@evident-ai/cli 3.5.2-dev.c2c9f26 → 3.5.2-dev.d192e6e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/index.js +2395 -590
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire as createRequire2 } from "module";
|
|
5
4
|
import { Command } from "commander";
|
|
6
5
|
|
|
7
6
|
// src/commands/login.ts
|
|
@@ -316,14 +315,14 @@ function blank() {
|
|
|
316
315
|
console.log();
|
|
317
316
|
}
|
|
318
317
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
319
|
-
return new Promise((
|
|
318
|
+
return new Promise((resolve5) => {
|
|
320
319
|
process.stdout.write(chalk.dim(prompt));
|
|
321
320
|
const handler = () => {
|
|
322
321
|
process.stdin.removeListener("data", handler);
|
|
323
322
|
process.stdin.setRawMode?.(false);
|
|
324
323
|
process.stdin.pause();
|
|
325
324
|
console.log();
|
|
326
|
-
|
|
325
|
+
resolve5();
|
|
327
326
|
};
|
|
328
327
|
if (process.stdin.isTTY) {
|
|
329
328
|
process.stdin.setRawMode?.(true);
|
|
@@ -333,7 +332,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
333
332
|
});
|
|
334
333
|
}
|
|
335
334
|
function sleep(ms) {
|
|
336
|
-
return new Promise((
|
|
335
|
+
return new Promise((resolve5) => setTimeout(resolve5, ms));
|
|
337
336
|
}
|
|
338
337
|
|
|
339
338
|
// src/commands/login.ts
|
|
@@ -411,19 +410,19 @@ async function tokenLogin() {
|
|
|
411
410
|
);
|
|
412
411
|
blank();
|
|
413
412
|
process.stdout.write("Paste token: ");
|
|
414
|
-
const token = await new Promise((
|
|
413
|
+
const token = await new Promise((resolve5) => {
|
|
415
414
|
let data = "";
|
|
416
415
|
process.stdin.setEncoding("utf8");
|
|
417
416
|
process.stdin.on("data", (chunk) => {
|
|
418
417
|
data += chunk;
|
|
419
418
|
});
|
|
420
419
|
process.stdin.on("end", () => {
|
|
421
|
-
|
|
420
|
+
resolve5(data.trim());
|
|
422
421
|
});
|
|
423
422
|
if (process.stdin.isTTY) {
|
|
424
423
|
process.stdin.once("data", (chunk) => {
|
|
425
424
|
process.stdin.pause();
|
|
426
|
-
|
|
425
|
+
resolve5(chunk.toString().trim());
|
|
427
426
|
});
|
|
428
427
|
process.stdin.resume();
|
|
429
428
|
}
|
|
@@ -704,6 +703,31 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
704
703
|
opencode_db_bytes: usage.opencodeDbBytes
|
|
705
704
|
});
|
|
706
705
|
}
|
|
706
|
+
async function getRunnerCredentialVault(agentId, authHeader) {
|
|
707
|
+
const empty = { prefix: null, vaultId: null, source: null };
|
|
708
|
+
try {
|
|
709
|
+
const apiUrl = getApiUrlConfig();
|
|
710
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/credential-vault`, {
|
|
711
|
+
headers: { Authorization: authHeader },
|
|
712
|
+
signal: AbortSignal.timeout(1e4)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) {
|
|
715
|
+
const serverMessage = await readErrorMessage(response);
|
|
716
|
+
return {
|
|
717
|
+
...empty,
|
|
718
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
return {
|
|
723
|
+
prefix: data.credential_vault_prefix,
|
|
724
|
+
vaultId: data.vault_id,
|
|
725
|
+
source: data.source
|
|
726
|
+
};
|
|
727
|
+
} catch (error2) {
|
|
728
|
+
return { ...empty, error: describeTimeoutError(error2, 1e4) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
707
731
|
async function getAgentInfo(agentId, authHeader) {
|
|
708
732
|
const apiUrl = getApiUrlConfig();
|
|
709
733
|
try {
|
|
@@ -878,6 +902,76 @@ async function status(options = {}) {
|
|
|
878
902
|
process.exit(result.exitCode);
|
|
879
903
|
}
|
|
880
904
|
|
|
905
|
+
// src/commands/credential-vault-env.ts
|
|
906
|
+
function shellQuote(value) {
|
|
907
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
908
|
+
}
|
|
909
|
+
function printAssignment(prefix) {
|
|
910
|
+
console.log(`CREDENTIAL_VAULT_PREFIX=${shellQuote(prefix ?? "")}`);
|
|
911
|
+
}
|
|
912
|
+
function safeReason(reason, credentials2, authHeader) {
|
|
913
|
+
let safe = reason;
|
|
914
|
+
for (const secret of [
|
|
915
|
+
authHeader,
|
|
916
|
+
credentials2?.token,
|
|
917
|
+
process.env.EVIDENT_RUNNER_KEY,
|
|
918
|
+
process.env.EVIDENT_AGENT_KEY,
|
|
919
|
+
process.env.EVIDENT_TOKEN
|
|
920
|
+
]) {
|
|
921
|
+
if (secret) safe = safe.replaceAll(secret, "[redacted]");
|
|
922
|
+
}
|
|
923
|
+
return safe.replace(/[\r\n]+/g, " ");
|
|
924
|
+
}
|
|
925
|
+
function printFallback(reason, credentials2, authHeader) {
|
|
926
|
+
printAssignment(null);
|
|
927
|
+
console.error(
|
|
928
|
+
`WARNING: ${safeReason(reason, credentials2, authHeader)}; using this runner's own credential prefix.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
async function credentialVaultEnv() {
|
|
932
|
+
let credentials2 = null;
|
|
933
|
+
let authHeader;
|
|
934
|
+
try {
|
|
935
|
+
credentials2 = await getAuthCredentials();
|
|
936
|
+
if (!credentials2) {
|
|
937
|
+
printFallback("no runner credentials are configured");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
authHeader = getAuthHeader(credentials2);
|
|
941
|
+
const runner = await resolveAgentIdFromKey(authHeader);
|
|
942
|
+
if (!runner.agent_id) {
|
|
943
|
+
printFallback(
|
|
944
|
+
runner.error ?? "the runner identity could not be resolved",
|
|
945
|
+
credentials2,
|
|
946
|
+
authHeader
|
|
947
|
+
);
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const lookup = await getRunnerCredentialVault(runner.agent_id, authHeader);
|
|
951
|
+
if (lookup.error) {
|
|
952
|
+
printFallback(`credential vault lookup failed: ${lookup.error}`, credentials2, authHeader);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (lookup.prefix === null) {
|
|
956
|
+
printFallback("this runner is not linked to a credential vault", credentials2, authHeader);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (/[\r\n]/.test(lookup.prefix)) {
|
|
960
|
+
printFallback("credential vault lookup returned an invalid prefix", credentials2, authHeader);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
printAssignment(lookup.prefix);
|
|
964
|
+
console.error(
|
|
965
|
+
`VAULT-MODE: credential stores use vault=${lookup.vaultId} source=${lookup.source} prefix=${lookup.prefix}.`
|
|
966
|
+
);
|
|
967
|
+
} catch (error2) {
|
|
968
|
+
const detail = credentials2 === null ? "credential resolution failed" : `credential vault lookup could not be completed: ${error2 instanceof Error ? error2.message : String(error2)}`;
|
|
969
|
+
printFallback(detail, credentials2, authHeader);
|
|
970
|
+
} finally {
|
|
971
|
+
process.exitCode = 0;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
881
975
|
// src/lib/claude-usage.ts
|
|
882
976
|
import { execFileSync } from "node:child_process";
|
|
883
977
|
import { readFileSync } from "node:fs";
|
|
@@ -1063,13 +1157,83 @@ async function claudeUsage() {
|
|
|
1063
1157
|
|
|
1064
1158
|
// src/commands/run.ts
|
|
1065
1159
|
import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync7, writeFileSync as writeFileSync6 } from "node:fs";
|
|
1066
|
-
import { homedir as
|
|
1067
|
-
import { isAbsolute as
|
|
1160
|
+
import { homedir as homedir7 } from "node:os";
|
|
1161
|
+
import { isAbsolute as isAbsolute4, join as join12, parse, resolve as resolvePath2 } from "node:path";
|
|
1068
1162
|
import chalk7 from "chalk";
|
|
1069
1163
|
|
|
1070
1164
|
// ../../packages/types/src/agents/index.ts
|
|
1071
1165
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1072
1166
|
|
|
1167
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1168
|
+
function roleOf(m) {
|
|
1169
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1170
|
+
if (typeof m.role === "string") return m.role;
|
|
1171
|
+
const infoRole = m.info?.role;
|
|
1172
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1173
|
+
}
|
|
1174
|
+
function idOf(m) {
|
|
1175
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1176
|
+
if (typeof m.id === "string") return m.id;
|
|
1177
|
+
const infoId = m.info?.id;
|
|
1178
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1179
|
+
}
|
|
1180
|
+
function parentIdOf(m) {
|
|
1181
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1182
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1183
|
+
const infoParent = m.info?.parentID;
|
|
1184
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1185
|
+
}
|
|
1186
|
+
function textPartsOf(m) {
|
|
1187
|
+
if (!m || typeof m !== "object") return [];
|
|
1188
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1189
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1190
|
+
}
|
|
1191
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1192
|
+
if (!messages || messages.length === 0) return [];
|
|
1193
|
+
const byParent = messages.filter(
|
|
1194
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1195
|
+
);
|
|
1196
|
+
if (byParent.length > 0) return byParent;
|
|
1197
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1198
|
+
if (userIndex === -1) return [];
|
|
1199
|
+
const run2 = [];
|
|
1200
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1201
|
+
const role = roleOf(messages[i]);
|
|
1202
|
+
if (role === "user") break;
|
|
1203
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1204
|
+
}
|
|
1205
|
+
return run2;
|
|
1206
|
+
}
|
|
1207
|
+
function lastText(assistantRun) {
|
|
1208
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1209
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1210
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1211
|
+
}
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1215
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1216
|
+
if (lastText(run2) !== null) return run2;
|
|
1217
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1218
|
+
if (userIndex === -1) return run2;
|
|
1219
|
+
let continuation;
|
|
1220
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1221
|
+
if (roleOf(messages[i]) === "user") {
|
|
1222
|
+
continuation = messages[i];
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
const continuationId = idOf(continuation);
|
|
1227
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1228
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1229
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1230
|
+
}
|
|
1231
|
+
return run2;
|
|
1232
|
+
}
|
|
1233
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1234
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1073
1237
|
// ../../packages/types/src/telemetry/index.ts
|
|
1074
1238
|
var TelemetryEventTypes = {
|
|
1075
1239
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1158,7 +1322,7 @@ import ora4 from "ora";
|
|
|
1158
1322
|
import { select as select4 } from "@inquirer/prompts";
|
|
1159
1323
|
|
|
1160
1324
|
// src/lib/telemetry.ts
|
|
1161
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1325
|
+
var CLI_VERSION = (true ? "3.5.2-dev.d192e6e" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1162
1326
|
function getCliVersion() {
|
|
1163
1327
|
return CLI_VERSION;
|
|
1164
1328
|
}
|
|
@@ -1172,6 +1336,23 @@ var authProvider = null;
|
|
|
1172
1336
|
function setTelemetryAuthProvider(provider) {
|
|
1173
1337
|
authProvider = provider;
|
|
1174
1338
|
}
|
|
1339
|
+
var diagnosticsReporter = null;
|
|
1340
|
+
function setTelemetryDiagnosticsReporter(reporter) {
|
|
1341
|
+
diagnosticsReporter = reporter;
|
|
1342
|
+
}
|
|
1343
|
+
function reportTelemetryDiagnostic(message) {
|
|
1344
|
+
if (!diagnosticsReporter) {
|
|
1345
|
+
console.error(message);
|
|
1346
|
+
return;
|
|
1347
|
+
}
|
|
1348
|
+
try {
|
|
1349
|
+
diagnosticsReporter(message);
|
|
1350
|
+
} catch (error2) {
|
|
1351
|
+
console.error(
|
|
1352
|
+
`Telemetry diagnostics reporter failed: ${error2 instanceof Error ? error2.message : String(error2)}; original diagnostic: ${message}`
|
|
1353
|
+
);
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1175
1356
|
var FLUSH_FAILURE_LOG_INTERVAL_MS = 6e4;
|
|
1176
1357
|
var lastFlushFailureLoggedAt = 0;
|
|
1177
1358
|
var suppressedFlushFailureCount = 0;
|
|
@@ -1237,7 +1418,9 @@ async function flushEvents() {
|
|
|
1237
1418
|
signal: controller.signal
|
|
1238
1419
|
});
|
|
1239
1420
|
if (!response.ok) {
|
|
1240
|
-
|
|
1421
|
+
reportTelemetryDiagnostic(
|
|
1422
|
+
`Telemetry could not send events (HTTP ${response.status}); runner and OpenCode are unaffected`
|
|
1423
|
+
);
|
|
1241
1424
|
}
|
|
1242
1425
|
} finally {
|
|
1243
1426
|
clearTimeout(timeout);
|
|
@@ -1247,7 +1430,9 @@ async function flushEvents() {
|
|
|
1247
1430
|
if (now - lastFlushFailureLoggedAt >= FLUSH_FAILURE_LOG_INTERVAL_MS) {
|
|
1248
1431
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1249
1432
|
const suffix = suppressedFlushFailureCount > 0 ? ` (${suppressedFlushFailureCount} more suppressed in the last ${FLUSH_FAILURE_LOG_INTERVAL_MS / 1e3}s)` : "";
|
|
1250
|
-
|
|
1433
|
+
reportTelemetryDiagnostic(
|
|
1434
|
+
`Telemetry flush failed: ${message}; runner and OpenCode are unaffected${suffix}`
|
|
1435
|
+
);
|
|
1251
1436
|
lastFlushFailureLoggedAt = now;
|
|
1252
1437
|
suppressedFlushFailureCount = 0;
|
|
1253
1438
|
} else {
|
|
@@ -1298,9 +1483,598 @@ var EventTypes = {
|
|
|
1298
1483
|
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
1299
1484
|
};
|
|
1300
1485
|
|
|
1486
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1487
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1488
|
+
import { dirname as dirname2 } from "node:path";
|
|
1489
|
+
|
|
1490
|
+
// src/lib/diagnostic-redaction.ts
|
|
1491
|
+
import { homedir as homedir2 } from "node:os";
|
|
1492
|
+
function redactHomeDirectory(message) {
|
|
1493
|
+
const home = homedir2();
|
|
1494
|
+
if (!home || home === "/") {
|
|
1495
|
+
return message;
|
|
1496
|
+
}
|
|
1497
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1498
|
+
let result = "";
|
|
1499
|
+
let searchFrom = 0;
|
|
1500
|
+
let homeIndex;
|
|
1501
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1502
|
+
const homeEnd = homeIndex + home.length;
|
|
1503
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1504
|
+
const nextCharacter = message[homeEnd];
|
|
1505
|
+
const atPathBoundary = (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1506
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1507
|
+
result += atPathBoundary ? "~" : home;
|
|
1508
|
+
searchFrom = homeEnd;
|
|
1509
|
+
}
|
|
1510
|
+
return result + message.slice(searchFrom);
|
|
1511
|
+
}
|
|
1512
|
+
function redactDiagnosticText(message) {
|
|
1513
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1514
|
+
return redactHomeDirectory(redacted);
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1518
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 100;
|
|
1519
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1520
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1521
|
+
var SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS = 5e3;
|
|
1522
|
+
function errorMessage(error2) {
|
|
1523
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1524
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1525
|
+
}
|
|
1526
|
+
function errorMessages(error2) {
|
|
1527
|
+
const messages = [];
|
|
1528
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1529
|
+
let current = error2;
|
|
1530
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1531
|
+
visited.add(current);
|
|
1532
|
+
if (typeof current === "string") {
|
|
1533
|
+
messages.push(current);
|
|
1534
|
+
break;
|
|
1535
|
+
}
|
|
1536
|
+
if (current instanceof Error) {
|
|
1537
|
+
messages.push(current.name, current.message);
|
|
1538
|
+
current = current.cause;
|
|
1539
|
+
continue;
|
|
1540
|
+
}
|
|
1541
|
+
if (typeof current === "object") {
|
|
1542
|
+
const record = current;
|
|
1543
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1544
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1545
|
+
current = record.cause;
|
|
1546
|
+
continue;
|
|
1547
|
+
}
|
|
1548
|
+
break;
|
|
1549
|
+
}
|
|
1550
|
+
return messages;
|
|
1551
|
+
}
|
|
1552
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1553
|
+
return errorMessages(error2).some(
|
|
1554
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1555
|
+
message
|
|
1556
|
+
)
|
|
1557
|
+
);
|
|
1558
|
+
}
|
|
1559
|
+
var SQLITE_CAUSE_TOKEN = /(?:^|cause:\s+)(SQLiteError:\s+[A-Za-z][A-Za-z0-9]*(?:\s+[A-Za-z][A-Za-z0-9]*){0,7})(?=\s*(?:[)\]"',;]|$))/i;
|
|
1560
|
+
var EFFECT_SQL_CAUSE_TOKEN = /(?:^|cause:\s+)(effect\/sql\/SqlError\/[A-Za-z][A-Za-z0-9_]*)/i;
|
|
1561
|
+
var PARAMS_FIELD = /\bparams\s*(?:=|:)/i;
|
|
1562
|
+
function isCauseBearingText(text) {
|
|
1563
|
+
return /^(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text.trim()) || /(?:^|\s)cause:\s+(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text);
|
|
1564
|
+
}
|
|
1565
|
+
function structuredLogFields(line) {
|
|
1566
|
+
const fields = [];
|
|
1567
|
+
for (let index = 0; index < line.length; index++) {
|
|
1568
|
+
if (line[index] === '"') {
|
|
1569
|
+
index++;
|
|
1570
|
+
while (index < line.length) {
|
|
1571
|
+
if (line[index] === "\\") {
|
|
1572
|
+
index++;
|
|
1573
|
+
} else if (line[index] === '"') {
|
|
1574
|
+
break;
|
|
1575
|
+
}
|
|
1576
|
+
index++;
|
|
1577
|
+
}
|
|
1578
|
+
continue;
|
|
1579
|
+
}
|
|
1580
|
+
const atFieldBoundary = index === 0 || /\s/.test(line[index - 1]);
|
|
1581
|
+
const key = line.startsWith("error=", index) ? "error=" : line.startsWith("cause=", index) ? "cause=" : null;
|
|
1582
|
+
if (!atFieldBoundary || key === null) continue;
|
|
1583
|
+
const valueStart = index + key.length;
|
|
1584
|
+
if (line[valueStart] !== '"') {
|
|
1585
|
+
const valueEnd2 = line.indexOf(" ", valueStart);
|
|
1586
|
+
fields.push(line.slice(valueStart, valueEnd2 === -1 ? line.length : valueEnd2));
|
|
1587
|
+
index = valueEnd2 === -1 ? line.length : valueEnd2;
|
|
1588
|
+
continue;
|
|
1589
|
+
}
|
|
1590
|
+
let valueEnd = valueStart + 1;
|
|
1591
|
+
for (; valueEnd < line.length; valueEnd++) {
|
|
1592
|
+
if (line[valueEnd] === "\\") {
|
|
1593
|
+
valueEnd++;
|
|
1594
|
+
} else if (line[valueEnd] === '"') {
|
|
1595
|
+
break;
|
|
1596
|
+
}
|
|
1597
|
+
}
|
|
1598
|
+
fields.push(line.slice(valueStart + 1, valueEnd));
|
|
1599
|
+
index = valueEnd;
|
|
1600
|
+
}
|
|
1601
|
+
return fields;
|
|
1602
|
+
}
|
|
1603
|
+
function structuredCauseMessages(text) {
|
|
1604
|
+
const fields = structuredLogFields(text);
|
|
1605
|
+
if (fields.length > 0) return fields;
|
|
1606
|
+
return isCauseBearingText(text) ? [text] : [];
|
|
1607
|
+
}
|
|
1608
|
+
function causeTokenFromText(text) {
|
|
1609
|
+
const paramsIndex = text.search(PARAMS_FIELD);
|
|
1610
|
+
for (const pattern of [SQLITE_CAUSE_TOKEN, EFFECT_SQL_CAUSE_TOKEN]) {
|
|
1611
|
+
const match = pattern.exec(text);
|
|
1612
|
+
if (match?.[1] && (paramsIndex === -1 || match.index < paramsIndex)) {
|
|
1613
|
+
return match[1];
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
return null;
|
|
1617
|
+
}
|
|
1618
|
+
function sessionStoreCauseToken(value) {
|
|
1619
|
+
for (const message of structuredCauseMessages(value)) {
|
|
1620
|
+
const token = causeTokenFromText(message);
|
|
1621
|
+
if (token !== null) return redactDiagnosticText(token).slice(0, 120);
|
|
1622
|
+
}
|
|
1623
|
+
return null;
|
|
1624
|
+
}
|
|
1625
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1626
|
+
try {
|
|
1627
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1628
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1629
|
+
if (availableBytes < requiredBytes) {
|
|
1630
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1631
|
+
}
|
|
1632
|
+
return null;
|
|
1633
|
+
} catch (err) {
|
|
1634
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1635
|
+
}
|
|
1636
|
+
}
|
|
1637
|
+
function readLogicalBytes(db) {
|
|
1638
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1639
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1640
|
+
return pageCount * pageSize;
|
|
1641
|
+
}
|
|
1642
|
+
function readFreelistCount(db) {
|
|
1643
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1644
|
+
}
|
|
1645
|
+
function readCheckpointResult(db) {
|
|
1646
|
+
db.exec("PRAGMA busy_timeout=0");
|
|
1647
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1648
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1649
|
+
}
|
|
1650
|
+
async function probeReclaimAvailability(input) {
|
|
1651
|
+
const { dbPath, requiredBytes } = input;
|
|
1652
|
+
let sqlite;
|
|
1653
|
+
try {
|
|
1654
|
+
sqlite = await import("node:sqlite");
|
|
1655
|
+
} catch (err) {
|
|
1656
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1657
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1658
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1659
|
+
}
|
|
1660
|
+
let autoVacuum = null;
|
|
1661
|
+
try {
|
|
1662
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1663
|
+
try {
|
|
1664
|
+
db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
|
|
1665
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1666
|
+
} finally {
|
|
1667
|
+
db.close();
|
|
1668
|
+
}
|
|
1669
|
+
} catch (err) {
|
|
1670
|
+
console.warn(
|
|
1671
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1672
|
+
);
|
|
1673
|
+
}
|
|
1674
|
+
if (autoVacuum !== 0) return null;
|
|
1675
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1676
|
+
}
|
|
1677
|
+
async function reclaimSessionDbSpace(input) {
|
|
1678
|
+
const {
|
|
1679
|
+
dbPath,
|
|
1680
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1681
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1682
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1683
|
+
now = () => performance.now(),
|
|
1684
|
+
hasLiveTurn = false,
|
|
1685
|
+
hasProvenanceAnomaly = false
|
|
1686
|
+
} = input;
|
|
1687
|
+
let sqlite;
|
|
1688
|
+
try {
|
|
1689
|
+
sqlite = await import("node:sqlite");
|
|
1690
|
+
} catch (err) {
|
|
1691
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1692
|
+
console.warn(
|
|
1693
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1694
|
+
);
|
|
1695
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1696
|
+
}
|
|
1697
|
+
const { DatabaseSync } = sqlite;
|
|
1698
|
+
let db;
|
|
1699
|
+
try {
|
|
1700
|
+
db = new DatabaseSync(dbPath);
|
|
1701
|
+
db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
|
|
1702
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1703
|
+
if (autoVacuum === 0) {
|
|
1704
|
+
if (hasLiveTurn || hasProvenanceAnomaly) {
|
|
1705
|
+
const detail = [
|
|
1706
|
+
...hasLiveTurn ? ["a session turn is live"] : [],
|
|
1707
|
+
...hasProvenanceAnomaly ? ["session-DB provenance is anomalous"] : []
|
|
1708
|
+
].join(" and ");
|
|
1709
|
+
console.warn(`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${detail}`);
|
|
1710
|
+
return {
|
|
1711
|
+
ok: false,
|
|
1712
|
+
skipped: "full-vacuum-blocked",
|
|
1713
|
+
detail
|
|
1714
|
+
};
|
|
1715
|
+
}
|
|
1716
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1717
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1718
|
+
if (skipReason !== null) {
|
|
1719
|
+
console.warn(
|
|
1720
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1721
|
+
);
|
|
1722
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1723
|
+
}
|
|
1724
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1725
|
+
const freelistBefore = readFreelistCount(db);
|
|
1726
|
+
const startedAt = now();
|
|
1727
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1728
|
+
db.exec("VACUUM");
|
|
1729
|
+
const afterBytes = readLogicalBytes(db);
|
|
1730
|
+
const freelistAfter = readFreelistCount(db);
|
|
1731
|
+
const checkpointStartedAt = now();
|
|
1732
|
+
const checkpoint = readCheckpointResult(db);
|
|
1733
|
+
const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
|
|
1734
|
+
return {
|
|
1735
|
+
ok: true,
|
|
1736
|
+
mode: "convert",
|
|
1737
|
+
beforeBytes,
|
|
1738
|
+
afterBytes,
|
|
1739
|
+
freelistBefore,
|
|
1740
|
+
freelistAfter,
|
|
1741
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1742
|
+
stoppedBy: "freelist-drained",
|
|
1743
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1744
|
+
checkpointElapsedMs,
|
|
1745
|
+
checkpoint
|
|
1746
|
+
};
|
|
1747
|
+
}
|
|
1748
|
+
if (autoVacuum === 2) {
|
|
1749
|
+
if (hasLiveTurn) {
|
|
1750
|
+
const detail = "a session turn is live";
|
|
1751
|
+
console.warn(
|
|
1752
|
+
`[reclaimSessionDbSpace] skipping incremental reclaim of ${dbPath}: ${detail}`
|
|
1753
|
+
);
|
|
1754
|
+
return { ok: false, skipped: "live-turn", detail };
|
|
1755
|
+
}
|
|
1756
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1757
|
+
const freelistBefore = readFreelistCount(db);
|
|
1758
|
+
const startedAt = now();
|
|
1759
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1760
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1761
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1762
|
+
let pagesAttempted = 0;
|
|
1763
|
+
let freelistAfter = freelistBefore;
|
|
1764
|
+
let stoppedBy = "freelist-drained";
|
|
1765
|
+
while (freelistAfter > 0) {
|
|
1766
|
+
if (pagesAttempted >= pageLimit) {
|
|
1767
|
+
stoppedBy = "page-ceiling";
|
|
1768
|
+
break;
|
|
1769
|
+
}
|
|
1770
|
+
if (now() - startedAt >= budget) {
|
|
1771
|
+
stoppedBy = "time-budget";
|
|
1772
|
+
break;
|
|
1773
|
+
}
|
|
1774
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1775
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1776
|
+
pagesAttempted += pages;
|
|
1777
|
+
freelistAfter = readFreelistCount(db);
|
|
1778
|
+
}
|
|
1779
|
+
const afterBytes = readLogicalBytes(db);
|
|
1780
|
+
const checkpointStartedAt = now();
|
|
1781
|
+
const checkpoint = readCheckpointResult(db);
|
|
1782
|
+
const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
|
|
1783
|
+
return {
|
|
1784
|
+
ok: true,
|
|
1785
|
+
mode: "incremental",
|
|
1786
|
+
beforeBytes,
|
|
1787
|
+
afterBytes,
|
|
1788
|
+
freelistBefore,
|
|
1789
|
+
freelistAfter,
|
|
1790
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1791
|
+
stoppedBy,
|
|
1792
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1793
|
+
checkpointElapsedMs,
|
|
1794
|
+
checkpoint
|
|
1795
|
+
};
|
|
1796
|
+
}
|
|
1797
|
+
console.warn(
|
|
1798
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1799
|
+
);
|
|
1800
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1801
|
+
} catch (err) {
|
|
1802
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1803
|
+
return {
|
|
1804
|
+
ok: false,
|
|
1805
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1806
|
+
detail: errorMessage(err)
|
|
1807
|
+
};
|
|
1808
|
+
} finally {
|
|
1809
|
+
db?.close();
|
|
1810
|
+
}
|
|
1811
|
+
}
|
|
1812
|
+
|
|
1813
|
+
// src/lib/opencode/dependency-install-diagnostic.ts
|
|
1814
|
+
import { lstatSync, realpathSync } from "node:fs";
|
|
1815
|
+
import { basename, dirname as dirname3, isAbsolute, join as join2, relative, resolve, sep } from "node:path";
|
|
1816
|
+
var DEPENDENCY_INSTALL_FAILURE_MESSAGE = "background dependency install failed";
|
|
1817
|
+
var MAX_DISTINCT_DIAGNOSTICS = 32;
|
|
1818
|
+
var EFFECT = "OpenCode could not install background dependencies for this scope; its plugins and tools may be unavailable";
|
|
1819
|
+
function readOpenCodeLogField(line, key) {
|
|
1820
|
+
const match = line.match(new RegExp(`(?:^|\\s)${key}=(?:"((?:\\\\.|[^"])*)"|(\\S+))`));
|
|
1821
|
+
if (!match) return void 0;
|
|
1822
|
+
if (match[1] === void 0) return match[2];
|
|
1823
|
+
return match[1].replace(/\\(["\\n])/g, (_match, escaped) => {
|
|
1824
|
+
return escaped === "n" ? "\n" : escaped;
|
|
1825
|
+
});
|
|
1826
|
+
}
|
|
1827
|
+
function parseTrigger(line) {
|
|
1828
|
+
if (readOpenCodeLogField(line, "message") !== DEPENDENCY_INSTALL_FAILURE_MESSAGE) {
|
|
1829
|
+
return null;
|
|
1830
|
+
}
|
|
1831
|
+
const error2 = readOpenCodeLogField(line, "error");
|
|
1832
|
+
if (!error2 || !/\bEACCES\b/.test(error2)) return null;
|
|
1833
|
+
const errnoMatch = error2.match(/\bEACCES:\s*permission denied,\s*([a-z_]+)\s+'([^']*)'/);
|
|
1834
|
+
const capturedPath = errnoMatch?.[2];
|
|
1835
|
+
return {
|
|
1836
|
+
dir: readOpenCodeLogField(line, "dir"),
|
|
1837
|
+
op: errnoMatch?.[1],
|
|
1838
|
+
path: capturedPath && isAbsolute(capturedPath) ? capturedPath : void 0
|
|
1839
|
+
};
|
|
1840
|
+
}
|
|
1841
|
+
function getErrorCode(error2) {
|
|
1842
|
+
if (typeof error2 === "object" && error2 !== null && "code" in error2) {
|
|
1843
|
+
const code = error2.code;
|
|
1844
|
+
if (typeof code === "string") return code;
|
|
1845
|
+
}
|
|
1846
|
+
return "UNKNOWN";
|
|
1847
|
+
}
|
|
1848
|
+
function isWithin(path, root) {
|
|
1849
|
+
const candidate = resolve(path);
|
|
1850
|
+
const resolvedRoot = resolve(root);
|
|
1851
|
+
return candidate === resolvedRoot || candidate.startsWith(`${resolvedRoot}${sep}`);
|
|
1852
|
+
}
|
|
1853
|
+
function rootSpecs(context) {
|
|
1854
|
+
return [
|
|
1855
|
+
{ token: "~", configured: resolve(context.homeDir) },
|
|
1856
|
+
{ token: "$WORKSPACE", configured: resolve(context.cwd) }
|
|
1857
|
+
];
|
|
1858
|
+
}
|
|
1859
|
+
function resolveRoots(context) {
|
|
1860
|
+
const resolved = [];
|
|
1861
|
+
const unresolved = [];
|
|
1862
|
+
for (const spec of rootSpecs(context)) {
|
|
1863
|
+
try {
|
|
1864
|
+
resolved.push({ ...spec, real: realpathSync(spec.configured) });
|
|
1865
|
+
} catch (_error) {
|
|
1866
|
+
unresolved.push(spec);
|
|
1867
|
+
}
|
|
1868
|
+
}
|
|
1869
|
+
return { resolved, unresolved };
|
|
1870
|
+
}
|
|
1871
|
+
function resolveNearestAncestor(failingPath, remaining, error2) {
|
|
1872
|
+
let ancestor = dirname3(failingPath);
|
|
1873
|
+
const suffix = [basename(failingPath), ...remaining];
|
|
1874
|
+
while (ancestor) {
|
|
1875
|
+
try {
|
|
1876
|
+
return {
|
|
1877
|
+
path: resolve(realpathSync(ancestor), ...suffix),
|
|
1878
|
+
failure: { path: resolve(failingPath), error: error2 }
|
|
1879
|
+
};
|
|
1880
|
+
} catch (ancestorError) {
|
|
1881
|
+
if (ancestor === dirname3(ancestor) || !getErrorCode(ancestorError).startsWith("E")) {
|
|
1882
|
+
return null;
|
|
1883
|
+
}
|
|
1884
|
+
suffix.unshift(basename(ancestor));
|
|
1885
|
+
ancestor = dirname3(ancestor);
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
return null;
|
|
1889
|
+
}
|
|
1890
|
+
function resolveExistingPath(absolutePath) {
|
|
1891
|
+
if (!isAbsolute(absolutePath)) return null;
|
|
1892
|
+
const candidate = resolve(absolutePath);
|
|
1893
|
+
const remaining = [];
|
|
1894
|
+
let current = candidate;
|
|
1895
|
+
while (current) {
|
|
1896
|
+
try {
|
|
1897
|
+
const stat2 = lstatSync(current);
|
|
1898
|
+
if (stat2.isSymbolicLink()) {
|
|
1899
|
+
if (remaining.length === 0) {
|
|
1900
|
+
return { path: join2(realpathSync(dirname3(current)), basename(current)) };
|
|
1901
|
+
}
|
|
1902
|
+
return { path: resolve(realpathSync(current), ...remaining) };
|
|
1903
|
+
}
|
|
1904
|
+
return { path: resolve(realpathSync(current), ...remaining) };
|
|
1905
|
+
} catch (error2) {
|
|
1906
|
+
const code = getErrorCode(error2);
|
|
1907
|
+
if (code !== "ENOENT" && code !== "ENOTDIR") {
|
|
1908
|
+
return resolveNearestAncestor(current, remaining, error2);
|
|
1909
|
+
}
|
|
1910
|
+
}
|
|
1911
|
+
if (current === dirname3(current)) return null;
|
|
1912
|
+
remaining.unshift(basename(current));
|
|
1913
|
+
current = dirname3(current);
|
|
1914
|
+
}
|
|
1915
|
+
return null;
|
|
1916
|
+
}
|
|
1917
|
+
function tokenise(absolutePath, roots) {
|
|
1918
|
+
const pathResolution = resolveExistingPath(absolutePath);
|
|
1919
|
+
if (!pathResolution) return null;
|
|
1920
|
+
const root = roots.reduce((selected, candidate) => {
|
|
1921
|
+
if (!isWithin(pathResolution.path, candidate.real)) return selected;
|
|
1922
|
+
if (!selected || candidate.real.length > selected.real.length || candidate.real.length === selected.real.length && candidate.token === "$WORKSPACE") {
|
|
1923
|
+
return candidate;
|
|
1924
|
+
}
|
|
1925
|
+
return selected;
|
|
1926
|
+
}, null);
|
|
1927
|
+
if (!root) return null;
|
|
1928
|
+
const suffix = relative(root.real, pathResolution.path);
|
|
1929
|
+
return {
|
|
1930
|
+
value: suffix ? `${root.token}/${suffix.split(sep).join("/")}` : root.token,
|
|
1931
|
+
root,
|
|
1932
|
+
resolutionFailure: pathResolution.failure
|
|
1933
|
+
};
|
|
1934
|
+
}
|
|
1935
|
+
function classifyScope(dir, roots) {
|
|
1936
|
+
if (!dir) return "other";
|
|
1937
|
+
const tokenisedDir = tokenise(dir, roots.resolved);
|
|
1938
|
+
if (tokenisedDir) return tokenisedDir.root.token === "$WORKSPACE" ? "workspace" : "global";
|
|
1939
|
+
for (const spec of roots.unresolved) {
|
|
1940
|
+
if (isWithin(dir, spec.configured)) {
|
|
1941
|
+
return spec.token === "$WORKSPACE" ? "workspace" : "global";
|
|
1942
|
+
}
|
|
1943
|
+
}
|
|
1944
|
+
return "other";
|
|
1945
|
+
}
|
|
1946
|
+
function unresolvedRootForPath(path, unresolved) {
|
|
1947
|
+
return unresolved.find((spec) => isWithin(path, spec.configured));
|
|
1948
|
+
}
|
|
1949
|
+
function statType(stat2) {
|
|
1950
|
+
if (stat2.isDirectory()) return "d";
|
|
1951
|
+
if (stat2.isFile()) return "f";
|
|
1952
|
+
if (stat2.isSymbolicLink()) return "l";
|
|
1953
|
+
return "o";
|
|
1954
|
+
}
|
|
1955
|
+
function renderStat(name, stat2) {
|
|
1956
|
+
const mode = (stat2.mode & 4095).toString(8).padStart(4, "0");
|
|
1957
|
+
return `${name}:${statType(stat2)}:${stat2.uid}:${stat2.gid}:${mode}`;
|
|
1958
|
+
}
|
|
1959
|
+
function renderStatError(name, error2) {
|
|
1960
|
+
const code = getErrorCode(error2);
|
|
1961
|
+
return `${name}:${renderErrorCode(code)}`;
|
|
1962
|
+
}
|
|
1963
|
+
function renderErrorCode(code) {
|
|
1964
|
+
if (code === "ENOENT") return "missing";
|
|
1965
|
+
return code.startsWith("E") ? code : `E${code}`;
|
|
1966
|
+
}
|
|
1967
|
+
function inspectChain(rootReal, rootToken, absolutePath, resolutionFailure) {
|
|
1968
|
+
const target = resolve(absolutePath);
|
|
1969
|
+
const relativePath = relative(rootReal, target);
|
|
1970
|
+
const components = relativePath ? relativePath.split(sep) : [];
|
|
1971
|
+
const entries = [];
|
|
1972
|
+
let current = rootReal;
|
|
1973
|
+
let status2 = "complete";
|
|
1974
|
+
let finalStat;
|
|
1975
|
+
for (const name of [rootToken, ...components]) {
|
|
1976
|
+
if (name !== rootToken) current = join2(current, name);
|
|
1977
|
+
if (resolutionFailure && resolve(current) === resolve(resolutionFailure.path)) {
|
|
1978
|
+
entries.push(renderStatError(name, resolutionFailure.error));
|
|
1979
|
+
status2 = "stat_failed";
|
|
1980
|
+
break;
|
|
1981
|
+
}
|
|
1982
|
+
try {
|
|
1983
|
+
const stat2 = lstatSync(current);
|
|
1984
|
+
entries.push(renderStat(name, stat2));
|
|
1985
|
+
finalStat = stat2;
|
|
1986
|
+
} catch (error2) {
|
|
1987
|
+
entries.push(renderStatError(name, error2));
|
|
1988
|
+
if (getErrorCode(error2) !== "ENOENT") status2 = "stat_failed";
|
|
1989
|
+
break;
|
|
1990
|
+
}
|
|
1991
|
+
}
|
|
1992
|
+
let symlinkTarget;
|
|
1993
|
+
let symlinkMeta;
|
|
1994
|
+
if (finalStat?.isSymbolicLink() && entries.length === components.length + 1) {
|
|
1995
|
+
try {
|
|
1996
|
+
symlinkTarget = realpathSync(current);
|
|
1997
|
+
try {
|
|
1998
|
+
symlinkMeta = renderStat("", lstatSync(symlinkTarget)).slice(1);
|
|
1999
|
+
} catch (error2) {
|
|
2000
|
+
symlinkMeta = renderErrorCode(getErrorCode(error2));
|
|
2001
|
+
status2 = "stat_failed";
|
|
2002
|
+
}
|
|
2003
|
+
} catch (error2) {
|
|
2004
|
+
symlinkTarget = "<unresolved>";
|
|
2005
|
+
symlinkMeta = renderErrorCode(getErrorCode(error2));
|
|
2006
|
+
status2 = "stat_failed";
|
|
2007
|
+
}
|
|
2008
|
+
}
|
|
2009
|
+
return { status: status2, chain: entries.join(","), symlinkTarget, symlinkMeta };
|
|
2010
|
+
}
|
|
2011
|
+
function buildDiagnostic(line, context) {
|
|
2012
|
+
const trigger = parseTrigger(line);
|
|
2013
|
+
if (!trigger) return null;
|
|
2014
|
+
const roots = resolveRoots(context);
|
|
2015
|
+
const scope = classifyScope(trigger.dir, roots);
|
|
2016
|
+
const pathSource = trigger.path ? "error" : trigger.dir ? "dir" : "none";
|
|
2017
|
+
const inspectedPath = trigger.path ?? trigger.dir;
|
|
2018
|
+
const microvmId = context.microvmId?.trim() || "unknown";
|
|
2019
|
+
const euid = context.euid === void 0 ? "unavailable" : String(context.euid);
|
|
2020
|
+
const egid = context.egid === void 0 ? "unavailable" : String(context.egid);
|
|
2021
|
+
const op = trigger.op ?? "unknown";
|
|
2022
|
+
if (!inspectedPath) {
|
|
2023
|
+
return {
|
|
2024
|
+
event: `OPENCODE-DEPENDENCY-INSTALL-EACCES: status=no_path path_source=${pathSource} scope=${scope} microvm_id=${microvmId} op=${op} path=unknown cli_euid=${euid} cli_egid=${egid} effect="${EFFECT}" chain=unavailable`,
|
|
2025
|
+
localDetail: null
|
|
2026
|
+
};
|
|
2027
|
+
}
|
|
2028
|
+
const tokenisedPath = tokenise(inspectedPath, roots.resolved);
|
|
2029
|
+
if (!tokenisedPath) {
|
|
2030
|
+
const unresolvedRoot = unresolvedRootForPath(inspectedPath, roots.unresolved);
|
|
2031
|
+
if (unresolvedRoot) {
|
|
2032
|
+
return {
|
|
2033
|
+
event: `OPENCODE-DEPENDENCY-INSTALL-EACCES: status=root_unresolved path_source=${pathSource} scope=${scope} microvm_id=${microvmId} op=${op} path=unknown cli_euid=${euid} cli_egid=${egid} effect="${EFFECT}" chain=unavailable`,
|
|
2034
|
+
localDetail: null
|
|
2035
|
+
};
|
|
2036
|
+
}
|
|
2037
|
+
return {
|
|
2038
|
+
event: `OPENCODE-DEPENDENCY-INSTALL-EACCES: status=outside_roots path_source=${pathSource} scope=${scope} microvm_id=${microvmId} op=${op} path=<outside-approved-roots> cli_euid=${euid} cli_egid=${egid} effect="${EFFECT}" chain=unavailable`,
|
|
2039
|
+
localDetail: `OPENCODE-DEPENDENCY-INSTALL-EACCES (local only): outside approved roots: ${inspectedPath}`
|
|
2040
|
+
};
|
|
2041
|
+
}
|
|
2042
|
+
const inspection = inspectChain(
|
|
2043
|
+
tokenisedPath.root.real,
|
|
2044
|
+
tokenisedPath.root.token,
|
|
2045
|
+
inspectedPath,
|
|
2046
|
+
tokenisedPath.resolutionFailure
|
|
2047
|
+
);
|
|
2048
|
+
const optionalSymlinkFields = inspection.symlinkTarget ? ` symlink_target=${tokenise(inspection.symlinkTarget, roots.resolved)?.value ?? "<outside-approved-roots>"}` + (inspection.symlinkMeta ? ` symlink_meta=${inspection.symlinkMeta}` : "") : "";
|
|
2049
|
+
return {
|
|
2050
|
+
event: `OPENCODE-DEPENDENCY-INSTALL-EACCES: status=${inspection.status} path_source=${pathSource} scope=${scope} microvm_id=${microvmId} op=${op} path=${tokenisedPath.value} cli_euid=${euid} cli_egid=${egid} effect="${EFFECT}" chain=${inspection.chain}${optionalSymlinkFields}`,
|
|
2051
|
+
localDetail: null
|
|
2052
|
+
};
|
|
2053
|
+
}
|
|
2054
|
+
function createDependencyInstallDiagnosticReporter({
|
|
2055
|
+
context,
|
|
2056
|
+
emitEvent: emitEvent2,
|
|
2057
|
+
emitLocal
|
|
2058
|
+
}) {
|
|
2059
|
+
const emittedEvents = /* @__PURE__ */ new Set();
|
|
2060
|
+
return (line) => {
|
|
2061
|
+
const diagnostic = buildDiagnostic(line, context);
|
|
2062
|
+
if (!diagnostic || emittedEvents.has(diagnostic.event)) return;
|
|
2063
|
+
if (emittedEvents.size >= MAX_DISTINCT_DIAGNOSTICS) return;
|
|
2064
|
+
emittedEvents.add(diagnostic.event);
|
|
2065
|
+
emitEvent2(diagnostic.event);
|
|
2066
|
+
if (diagnostic.localDetail) emitLocal(diagnostic.localDetail);
|
|
2067
|
+
};
|
|
2068
|
+
}
|
|
2069
|
+
|
|
1301
2070
|
// src/lib/runner-activity-telemetry.ts
|
|
1302
|
-
var
|
|
2071
|
+
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
2072
|
+
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
2073
|
+
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
2074
|
+
"cli.session-cleanup": /* @__PURE__ */ new Set(["info", "warn", "error"])
|
|
2075
|
+
};
|
|
1303
2076
|
var SEVERITY_BY_LEVEL = {
|
|
2077
|
+
info: "info",
|
|
1304
2078
|
warn: "warning",
|
|
1305
2079
|
error: "error"
|
|
1306
2080
|
};
|
|
@@ -1310,26 +2084,113 @@ function parseOpenCodeLogLine(line) {
|
|
|
1310
2084
|
if (!levelMatch) return null;
|
|
1311
2085
|
const level = levelMatch[1].toUpperCase();
|
|
1312
2086
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1313
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1314
|
-
|
|
2087
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
2088
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
2089
|
+
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
2090
|
+
const parsed = {
|
|
2091
|
+
level: level === "WARN" ? "warn" : "error",
|
|
2092
|
+
sessionID: sessionMatch?.[1],
|
|
2093
|
+
messageID: messageIDMatch?.[1]
|
|
2094
|
+
};
|
|
2095
|
+
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
2096
|
+
if (message !== void 0) parsed.message = message;
|
|
2097
|
+
return parsed;
|
|
1315
2098
|
}
|
|
1316
2099
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1317
|
-
|
|
2100
|
+
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
2101
|
+
function sanitiseDependencyInstallWarning(line) {
|
|
2102
|
+
const withoutDirectory = line.replace(
|
|
2103
|
+
/((?:^|\s)dir=)(?:"(?:\\.|[^"])*"|\/[^\s)"']+|\S+)/g,
|
|
2104
|
+
"$1<path>"
|
|
2105
|
+
);
|
|
2106
|
+
return withoutDirectory.replace(
|
|
2107
|
+
/((?:^|\s)error=)"((?:\\.|[^"])*)"/,
|
|
2108
|
+
(_match, prefix, error2) => `${prefix}"${error2.replace(/'\/[^']*'/g, "'<path>'")}"`
|
|
2109
|
+
);
|
|
2110
|
+
}
|
|
2111
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure, onDependencyInstallFailure) {
|
|
1318
2112
|
let buffer = Buffer.alloc(0);
|
|
2113
|
+
const repeatStates = /* @__PURE__ */ new Map();
|
|
2114
|
+
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
2115
|
+
const metadata = {
|
|
2116
|
+
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
2117
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
2118
|
+
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
2119
|
+
};
|
|
2120
|
+
forwardRunnerActivity(
|
|
2121
|
+
{
|
|
2122
|
+
level: parsed.level,
|
|
2123
|
+
error: parsed.message === DEPENDENCY_INSTALL_FAILURE_MESSAGE ? sanitiseDependencyInstallWarning(line) : line,
|
|
2124
|
+
metadata: Object.keys(metadata).length > 0 ? metadata : void 0,
|
|
2125
|
+
source: "opencode"
|
|
2126
|
+
},
|
|
2127
|
+
getContext()
|
|
2128
|
+
);
|
|
2129
|
+
};
|
|
2130
|
+
const expireRepeatStates = (now, currentKey) => {
|
|
2131
|
+
for (const [key, state] of repeatStates) {
|
|
2132
|
+
if (key === currentKey || now - state.lastForwardedAt < REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
2133
|
+
continue;
|
|
2134
|
+
}
|
|
2135
|
+
if (state.repeatsSuppressed > 0) {
|
|
2136
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
2137
|
+
}
|
|
2138
|
+
repeatStates.delete(key);
|
|
2139
|
+
}
|
|
2140
|
+
};
|
|
2141
|
+
const forwardWithRepeatSuppression = (line, parsed) => {
|
|
2142
|
+
const now = Date.now();
|
|
2143
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
2144
|
+
expireRepeatStates(now, key);
|
|
2145
|
+
if (key === void 0) {
|
|
2146
|
+
forwardLine(line, parsed);
|
|
2147
|
+
return;
|
|
2148
|
+
}
|
|
2149
|
+
const previous = repeatStates.get(key);
|
|
2150
|
+
if (!previous || now - previous.lastForwardedAt >= REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
2151
|
+
forwardLine(line, parsed, previous?.repeatsSuppressed ?? 0);
|
|
2152
|
+
repeatStates.set(key, {
|
|
2153
|
+
line,
|
|
2154
|
+
parsed,
|
|
2155
|
+
lastForwardedAt: now,
|
|
2156
|
+
repeatsSuppressed: 0
|
|
2157
|
+
});
|
|
2158
|
+
return;
|
|
2159
|
+
}
|
|
2160
|
+
previous.line = line;
|
|
2161
|
+
previous.parsed = parsed;
|
|
2162
|
+
previous.repeatsSuppressed++;
|
|
2163
|
+
};
|
|
1319
2164
|
const flushLine = (line) => {
|
|
1320
2165
|
const parsed = parseOpenCodeLogLine(line);
|
|
2166
|
+
const isDiskFailure = isSessionStoreWriteFailure(line);
|
|
2167
|
+
const cause = sessionStoreCauseToken(line);
|
|
2168
|
+
if (onSessionStoreWriteFailure && (isDiskFailure || cause !== null)) {
|
|
2169
|
+
try {
|
|
2170
|
+
onSessionStoreWriteFailure({
|
|
2171
|
+
isDiskFailure,
|
|
2172
|
+
sessionId: parsed?.sessionID ?? null,
|
|
2173
|
+
cause
|
|
2174
|
+
});
|
|
2175
|
+
} catch (err) {
|
|
2176
|
+
reportTelemetryDiagnostic(
|
|
2177
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
2178
|
+
);
|
|
2179
|
+
}
|
|
2180
|
+
}
|
|
1321
2181
|
if (!parsed) return;
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
2182
|
+
forwardWithRepeatSuppression(line, parsed);
|
|
2183
|
+
if (onDependencyInstallFailure && parsed.message === DEPENDENCY_INSTALL_FAILURE_MESSAGE) {
|
|
2184
|
+
try {
|
|
2185
|
+
onDependencyInstallFailure(line);
|
|
2186
|
+
} catch (err) {
|
|
2187
|
+
reportTelemetryDiagnostic(
|
|
2188
|
+
`Runner activity telemetry could not record dependency-install failure diagnostic: ${err instanceof Error ? err.message : String(err)}`
|
|
2189
|
+
);
|
|
2190
|
+
}
|
|
2191
|
+
}
|
|
1331
2192
|
};
|
|
1332
|
-
|
|
2193
|
+
const forward = ((chunk) => {
|
|
1333
2194
|
buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf-8")]);
|
|
1334
2195
|
let newlineIndex;
|
|
1335
2196
|
while ((newlineIndex = buffer.indexOf(10)) !== -1) {
|
|
@@ -1340,31 +2201,68 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1340
2201
|
flushLine(buffer.toString("utf-8"));
|
|
1341
2202
|
buffer = Buffer.alloc(0);
|
|
1342
2203
|
}
|
|
2204
|
+
});
|
|
2205
|
+
forward.flush = () => {
|
|
2206
|
+
for (const state of repeatStates.values()) {
|
|
2207
|
+
if (state.repeatsSuppressed > 0) {
|
|
2208
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
2209
|
+
}
|
|
2210
|
+
}
|
|
2211
|
+
repeatStates.clear();
|
|
1343
2212
|
};
|
|
2213
|
+
return forward;
|
|
1344
2214
|
}
|
|
1345
2215
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1346
2216
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1347
2217
|
var MAX_METADATA_ENTRIES = 20;
|
|
1348
|
-
var TRUNCATION_MARKER = "\u2026";
|
|
1349
|
-
function redact(message) {
|
|
1350
|
-
return message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1351
|
-
}
|
|
1352
2218
|
function truncate(message) {
|
|
1353
2219
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1354
|
-
|
|
2220
|
+
let dropped = message.length - MAX_MESSAGE_LENGTH;
|
|
2221
|
+
let marker = `\u2026${dropped} chars omitted\u2026`;
|
|
2222
|
+
let previousDropped = -1;
|
|
2223
|
+
while (dropped !== previousDropped) {
|
|
2224
|
+
previousDropped = dropped;
|
|
2225
|
+
const nextDropped = message.length - MAX_MESSAGE_LENGTH + marker.length;
|
|
2226
|
+
dropped = nextDropped;
|
|
2227
|
+
marker = `\u2026${dropped} chars omitted\u2026`;
|
|
2228
|
+
}
|
|
2229
|
+
const remaining = MAX_MESSAGE_LENGTH - marker.length;
|
|
2230
|
+
const headLength = Math.ceil(remaining * 0.6);
|
|
2231
|
+
const tailLength = remaining - headLength;
|
|
2232
|
+
const effectMatch = message.startsWith("OPENCODE-DEPENDENCY-INSTALL-EACCES:") ? message.match(/ effect="[^"]*"/) : null;
|
|
2233
|
+
if (effectMatch?.index !== void 0) {
|
|
2234
|
+
const effectEnd = effectMatch.index + effectMatch[0].length;
|
|
2235
|
+
if (effectEnd > headLength && effectEnd < message.length) {
|
|
2236
|
+
const effectLength = effectMatch[0].length;
|
|
2237
|
+
const nonEffectLength = remaining - effectLength;
|
|
2238
|
+
const preservedTailLength = Math.min(
|
|
2239
|
+
message.length - effectEnd,
|
|
2240
|
+
nonEffectLength - Math.min(effectMatch.index, Math.ceil(nonEffectLength * 0.6))
|
|
2241
|
+
);
|
|
2242
|
+
const preservedHeadLength = Math.min(
|
|
2243
|
+
effectMatch.index,
|
|
2244
|
+
nonEffectLength - preservedTailLength
|
|
2245
|
+
);
|
|
2246
|
+
return message.slice(0, preservedHeadLength) + marker + message.slice(effectMatch.index, effectEnd) + message.slice(message.length - preservedTailLength);
|
|
2247
|
+
}
|
|
2248
|
+
}
|
|
2249
|
+
return message.slice(0, headLength) + marker + message.slice(message.length - tailLength);
|
|
1355
2250
|
}
|
|
1356
2251
|
function sanitiseMetadata(metadata) {
|
|
1357
2252
|
if (!metadata) return {};
|
|
1358
2253
|
const entries = Object.entries(metadata).slice(0, MAX_METADATA_ENTRIES);
|
|
1359
2254
|
if (Object.keys(metadata).length > entries.length) {
|
|
1360
|
-
|
|
1361
|
-
`
|
|
2255
|
+
reportTelemetryDiagnostic(
|
|
2256
|
+
`Runner activity telemetry dropped ${Object.keys(metadata).length - entries.length} metadata entries; those entries were not sent`
|
|
1362
2257
|
);
|
|
1363
2258
|
}
|
|
1364
2259
|
const sanitised = [];
|
|
1365
2260
|
for (const [key, value] of entries) {
|
|
1366
2261
|
if (typeof value === "string") {
|
|
1367
|
-
sanitised.push([
|
|
2262
|
+
sanitised.push([
|
|
2263
|
+
key,
|
|
2264
|
+
truncate(redactDiagnosticText(value).slice(0, MAX_METADATA_VALUE_LENGTH))
|
|
2265
|
+
]);
|
|
1368
2266
|
} else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
|
|
1369
2267
|
sanitised.push([key, value]);
|
|
1370
2268
|
}
|
|
@@ -1382,8 +2280,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1382
2280
|
}
|
|
1383
2281
|
if (now - window.windowStartedAt >= RATE_LIMIT_WINDOW_MS) {
|
|
1384
2282
|
if (window.windowDroppedCount > 0) {
|
|
1385
|
-
|
|
1386
|
-
`
|
|
2283
|
+
reportTelemetryDiagnostic(
|
|
2284
|
+
`Runner activity telemetry rate cap reached: dropped ${window.windowDroppedCount} ${window.windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (limit ${RATE_LIMIT_MAX_EVENTS}/min); those entries were not sent for source "${source}"`
|
|
1387
2285
|
);
|
|
1388
2286
|
}
|
|
1389
2287
|
window.windowStartedAt = now;
|
|
@@ -1393,8 +2291,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1393
2291
|
if (window.windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
1394
2292
|
window.windowDroppedCount++;
|
|
1395
2293
|
if (window.windowDroppedCount === 1) {
|
|
1396
|
-
|
|
1397
|
-
`
|
|
2294
|
+
reportTelemetryDiagnostic(
|
|
2295
|
+
`Runner activity telemetry rate cap reached (${RATE_LIMIT_MAX_EVENTS} entries per ${RATE_LIMIT_WINDOW_MS / 1e3}s limit; further entries were not sent for source "${source}"`
|
|
1398
2296
|
);
|
|
1399
2297
|
}
|
|
1400
2298
|
return false;
|
|
@@ -1404,12 +2302,12 @@ function admitUnderRateLimit(source, now) {
|
|
|
1404
2302
|
}
|
|
1405
2303
|
function forwardRunnerActivity(entry, context) {
|
|
1406
2304
|
try {
|
|
1407
|
-
if (!FORWARDED_LEVELS.has(entry.level)) return;
|
|
1408
|
-
if (!context.agentId || !context.authHeader) return;
|
|
1409
2305
|
const source = entry.source ?? "cli.run";
|
|
2306
|
+
if (!FORWARDED_LEVELS_BY_SOURCE[source].has(entry.level)) return;
|
|
2307
|
+
if (!context.agentId || !context.authHeader) return;
|
|
1410
2308
|
if (!admitUnderRateLimit(source, Date.now())) return;
|
|
1411
2309
|
const rawMessage = entry.error ?? entry.message ?? "";
|
|
1412
|
-
const message = truncate(
|
|
2310
|
+
const message = truncate(redactDiagnosticText(rawMessage));
|
|
1413
2311
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1414
2312
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1415
2313
|
message,
|
|
@@ -1417,18 +2315,18 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1417
2315
|
agentId: context.agentId
|
|
1418
2316
|
});
|
|
1419
2317
|
} catch (err) {
|
|
1420
|
-
|
|
1421
|
-
`
|
|
2318
|
+
reportTelemetryDiagnostic(
|
|
2319
|
+
`Runner activity telemetry failed to forward runner activity: ${err instanceof Error ? err.message : String(err)}`
|
|
1422
2320
|
);
|
|
1423
2321
|
}
|
|
1424
2322
|
}
|
|
1425
2323
|
|
|
1426
2324
|
// src/lib/opencode/session-db-recovery-report.ts
|
|
1427
2325
|
import { readFileSync as readFileSync2, unlinkSync } from "node:fs";
|
|
1428
|
-
import { join as
|
|
2326
|
+
import { join as join3 } from "node:path";
|
|
1429
2327
|
function sessionDbRecoveryReportPath(homeDir, env) {
|
|
1430
2328
|
const override = env.EVIDENT_SESSION_DB_RECOVERY_REPORT?.trim();
|
|
1431
|
-
return override ||
|
|
2329
|
+
return override || join3(homeDir, ".local", "state", "evident", "session-db-recovery.jsonl");
|
|
1432
2330
|
}
|
|
1433
2331
|
function drainSessionDbRecoveryReport({
|
|
1434
2332
|
homeDir,
|
|
@@ -1558,12 +2456,23 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1558
2456
|
metadata: withoutContractFields(record),
|
|
1559
2457
|
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1560
2458
|
};
|
|
1561
|
-
case "session_db_boot_refused":
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
2459
|
+
case "session_db_boot_refused": {
|
|
2460
|
+
const message = (() => {
|
|
2461
|
+
switch (record.reason) {
|
|
2462
|
+
case "local_discard_failed":
|
|
2463
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
|
|
2464
|
+
case "replica_separation_unproven":
|
|
2465
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
|
|
2466
|
+
case "verification_timeout_before_separation":
|
|
2467
|
+
return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2468
|
+
case "verification_timeout_during_separation":
|
|
2469
|
+
return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2470
|
+
default:
|
|
2471
|
+
return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
|
|
2472
|
+
}
|
|
2473
|
+
})();
|
|
2474
|
+
return { level, metadata: withoutContractFields(record), message };
|
|
2475
|
+
}
|
|
1567
2476
|
case "schema_provenance_mismatch":
|
|
1568
2477
|
return {
|
|
1569
2478
|
level,
|
|
@@ -1690,7 +2599,7 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
1690
2599
|
if (health.healthy || health.authFailed) {
|
|
1691
2600
|
return health;
|
|
1692
2601
|
}
|
|
1693
|
-
await new Promise((
|
|
2602
|
+
await new Promise((resolve5) => setTimeout(resolve5, 1e3));
|
|
1694
2603
|
}
|
|
1695
2604
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
1696
2605
|
}
|
|
@@ -1701,15 +2610,53 @@ async function waitForOpenCode2Health(port, password, timeoutMs = 3e4) {
|
|
|
1701
2610
|
if (health.healthy || health.authFailed) {
|
|
1702
2611
|
return health;
|
|
1703
2612
|
}
|
|
1704
|
-
await new Promise((
|
|
2613
|
+
await new Promise((resolve5) => setTimeout(resolve5, 1e3));
|
|
1705
2614
|
}
|
|
1706
2615
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
1707
2616
|
}
|
|
1708
2617
|
|
|
1709
2618
|
// src/lib/http-timeout.ts
|
|
1710
2619
|
var REQUEST_TIMEOUT_MS = 6e4;
|
|
2620
|
+
var RequestTimeoutError = class extends Error {
|
|
2621
|
+
constructor(operation, timeoutMs, elapsedMs) {
|
|
2622
|
+
super(`${operation} timed out after ${elapsedMs}ms (limit ${timeoutMs}ms)`);
|
|
2623
|
+
this.operation = operation;
|
|
2624
|
+
this.timeoutMs = timeoutMs;
|
|
2625
|
+
this.elapsedMs = elapsedMs;
|
|
2626
|
+
this.name = "TimeoutError";
|
|
2627
|
+
}
|
|
2628
|
+
};
|
|
2629
|
+
function describeRequestOperation(input, init) {
|
|
2630
|
+
const method = (init?.method ?? input.method ?? "GET").toUpperCase();
|
|
2631
|
+
const rawUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
|
2632
|
+
let path;
|
|
2633
|
+
try {
|
|
2634
|
+
path = new URL(rawUrl).pathname;
|
|
2635
|
+
} catch (error2) {
|
|
2636
|
+
if (!(error2 instanceof TypeError)) throw error2;
|
|
2637
|
+
path = rawUrl.split(/[?#]/, 1)[0];
|
|
2638
|
+
}
|
|
2639
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
2640
|
+
const prefixedIdentifier = /^[a-z]+_[A-Za-z0-9_-]{10,}$/;
|
|
2641
|
+
const maskedPath = path.split("/").map((segment) => uuid.test(segment) || prefixedIdentifier.test(segment) ? ":id" : segment).join("/");
|
|
2642
|
+
return `${method} ${maskedPath}`;
|
|
2643
|
+
}
|
|
1711
2644
|
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
1712
|
-
return ((input, init) =>
|
|
2645
|
+
return (async (input, init) => {
|
|
2646
|
+
const startedAt = Date.now();
|
|
2647
|
+
try {
|
|
2648
|
+
return await fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) });
|
|
2649
|
+
} catch (error2) {
|
|
2650
|
+
if (error2?.name === "TimeoutError") {
|
|
2651
|
+
throw new RequestTimeoutError(
|
|
2652
|
+
describeRequestOperation(input, init),
|
|
2653
|
+
timeoutMs,
|
|
2654
|
+
Date.now() - startedAt
|
|
2655
|
+
);
|
|
2656
|
+
}
|
|
2657
|
+
throw error2;
|
|
2658
|
+
}
|
|
2659
|
+
});
|
|
1713
2660
|
}
|
|
1714
2661
|
|
|
1715
2662
|
// src/lib/opencode/client.ts
|
|
@@ -1775,7 +2722,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1775
2722
|
return null;
|
|
1776
2723
|
}
|
|
1777
2724
|
}
|
|
1778
|
-
function
|
|
2725
|
+
function roleOf2(m) {
|
|
1779
2726
|
if (!m || typeof m !== "object") return void 0;
|
|
1780
2727
|
if (typeof m.role === "string") return m.role;
|
|
1781
2728
|
const infoRole = m.info?.role;
|
|
@@ -1789,13 +2736,13 @@ function createdOf(m) {
|
|
|
1789
2736
|
if (!m || typeof m !== "object") return void 0;
|
|
1790
2737
|
return m.info?.time?.created ?? m.time?.created;
|
|
1791
2738
|
}
|
|
1792
|
-
function
|
|
2739
|
+
function idOf2(m) {
|
|
1793
2740
|
if (!m || typeof m !== "object") return void 0;
|
|
1794
2741
|
if (typeof m.id === "string") return m.id;
|
|
1795
2742
|
const infoId = m.info?.id;
|
|
1796
2743
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1797
2744
|
}
|
|
1798
|
-
function
|
|
2745
|
+
function parentIdOf2(m) {
|
|
1799
2746
|
if (!m || typeof m !== "object") return void 0;
|
|
1800
2747
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1801
2748
|
const infoParent = m.info?.parentID;
|
|
@@ -1811,8 +2758,12 @@ function errorOf(m) {
|
|
|
1811
2758
|
if (!m || typeof m !== "object") return void 0;
|
|
1812
2759
|
return m.info?.error ?? m.error;
|
|
1813
2760
|
}
|
|
2761
|
+
function isTerminalStopAnswer(m) {
|
|
2762
|
+
return completedOf(m) != null && errorOf(m) == null && finishOf(m) === "stop";
|
|
2763
|
+
}
|
|
1814
2764
|
function isAssistantInFlight(m) {
|
|
1815
2765
|
if (completedOf(m) == null) return true;
|
|
2766
|
+
if (errorOf(m) != null) return false;
|
|
1816
2767
|
return finishOf(m) === "tool-calls";
|
|
1817
2768
|
}
|
|
1818
2769
|
async function getSessionMessages(port, sessionId, client) {
|
|
@@ -1860,9 +2811,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
1860
2811
|
function isSessionActivelyGenerating(messages) {
|
|
1861
2812
|
if (!messages || messages.length === 0) return false;
|
|
1862
2813
|
const last = messages[messages.length - 1];
|
|
1863
|
-
if (
|
|
2814
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
1864
2815
|
return completedOf(last) == null;
|
|
1865
2816
|
}
|
|
2817
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
1866
2818
|
function sessionLastActivityMs(session) {
|
|
1867
2819
|
const candidates = [
|
|
1868
2820
|
session.time?.updated,
|
|
@@ -1877,30 +2829,71 @@ function sessionLastActivityMs(session) {
|
|
|
1877
2829
|
}
|
|
1878
2830
|
return null;
|
|
1879
2831
|
}
|
|
1880
|
-
|
|
1881
|
-
|
|
2832
|
+
function sessionListPath(options) {
|
|
2833
|
+
const params = new URLSearchParams();
|
|
2834
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2835
|
+
const query = params.toString();
|
|
2836
|
+
return query ? `/session?${query}` : "/session";
|
|
2837
|
+
}
|
|
2838
|
+
async function listSessionPage(port, client, options) {
|
|
2839
|
+
const path = sessionListPath(options);
|
|
1882
2840
|
try {
|
|
1883
|
-
const res = await requestWithClient(port, client,
|
|
2841
|
+
const res = await requestWithClient(port, client, path);
|
|
1884
2842
|
if (!res.ok) return null;
|
|
1885
2843
|
const body = await res.json();
|
|
1886
2844
|
return Array.isArray(body) ? body : null;
|
|
1887
2845
|
} catch (error2) {
|
|
1888
2846
|
console.error(
|
|
1889
|
-
`[
|
|
2847
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1890
2848
|
);
|
|
1891
2849
|
return null;
|
|
1892
2850
|
}
|
|
1893
2851
|
}
|
|
2852
|
+
async function listSessions(port, client, options) {
|
|
2853
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2854
|
+
return listSessionPage(port, client, options);
|
|
2855
|
+
}
|
|
2856
|
+
async function listAllSessions(port, client) {
|
|
2857
|
+
if (client?.version === "v2") {
|
|
2858
|
+
const sessions2 = await listV2Sessions(client);
|
|
2859
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2860
|
+
}
|
|
2861
|
+
const sessions = await listSessionPage(port, client, {
|
|
2862
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2863
|
+
});
|
|
2864
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2865
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2866
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2867
|
+
}
|
|
2868
|
+
return { sessions };
|
|
2869
|
+
}
|
|
1894
2870
|
async function deleteSession(port, id, client) {
|
|
1895
2871
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
1896
2872
|
try {
|
|
1897
|
-
const res = await requestWithClient(
|
|
1898
|
-
|
|
2873
|
+
const res = await requestWithClient(
|
|
2874
|
+
port,
|
|
2875
|
+
client,
|
|
2876
|
+
`/session/${id}`,
|
|
2877
|
+
{ method: "DELETE" },
|
|
2878
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2879
|
+
);
|
|
2880
|
+
if (res.status >= 200 && res.status < 300) {
|
|
2881
|
+
return { ok: true, status: res.status };
|
|
2882
|
+
}
|
|
2883
|
+
try {
|
|
2884
|
+
const body = await res.text();
|
|
2885
|
+
return body ? { ok: false, status: res.status, body } : { ok: false, status: res.status };
|
|
2886
|
+
} catch (error2) {
|
|
2887
|
+
console.error(
|
|
2888
|
+
`[deleteSession] DELETE /session/${id} response body read failed for HTTP ${res.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2889
|
+
);
|
|
2890
|
+
return { ok: false, status: res.status };
|
|
2891
|
+
}
|
|
1899
2892
|
} catch (error2) {
|
|
1900
2893
|
console.error(
|
|
1901
2894
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1902
2895
|
);
|
|
1903
|
-
return false;
|
|
2896
|
+
return { ok: false, status: "threw" };
|
|
1904
2897
|
}
|
|
1905
2898
|
}
|
|
1906
2899
|
async function sessionExists(port, id, client) {
|
|
@@ -2138,11 +3131,31 @@ function messageText(m) {
|
|
|
2138
3131
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2139
3132
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2140
3133
|
}
|
|
2141
|
-
|
|
3134
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
3135
|
+
if (!messages || messages.length === 0) return null;
|
|
3136
|
+
let best = null;
|
|
3137
|
+
for (const message of messages) {
|
|
3138
|
+
if (roleOf2(message) !== "user") continue;
|
|
3139
|
+
const id = idOf2(message);
|
|
3140
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
3141
|
+
if (messageText(message) !== content) continue;
|
|
3142
|
+
const created = createdOf(message) ?? 0;
|
|
3143
|
+
if (best === null || created > best.created) best = { id, created };
|
|
3144
|
+
}
|
|
3145
|
+
return best?.id ?? null;
|
|
3146
|
+
}
|
|
3147
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2142
3148
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2143
3149
|
const knownUserIds = new Set(
|
|
2144
|
-
(before ?? []).filter((m) =>
|
|
3150
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2145
3151
|
);
|
|
3152
|
+
if (callbacks) {
|
|
3153
|
+
try {
|
|
3154
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
3155
|
+
} catch (error2) {
|
|
3156
|
+
callbacks.onKnownUserIdsError(error2);
|
|
3157
|
+
}
|
|
3158
|
+
}
|
|
2146
3159
|
const parts = [{ type: "text", text: content }];
|
|
2147
3160
|
let pendingOutcomes = null;
|
|
2148
3161
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2181,25 +3194,13 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2181
3194
|
const READ_BACK_DELAY_MS = 150;
|
|
2182
3195
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2183
3196
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
const id = idOf(m);
|
|
2189
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2190
|
-
if (messageText(m) !== content) continue;
|
|
2191
|
-
const created = createdOf(m) ?? 0;
|
|
2192
|
-
if (best === null || created > best.created) {
|
|
2193
|
-
best = { id, created };
|
|
2194
|
-
}
|
|
2195
|
-
}
|
|
2196
|
-
if (best) {
|
|
2197
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2198
|
-
return best.id;
|
|
2199
|
-
}
|
|
3197
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
3198
|
+
if (dispatchedUserMessageId !== null) {
|
|
3199
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
3200
|
+
return dispatchedUserMessageId;
|
|
2200
3201
|
}
|
|
2201
3202
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2202
|
-
await new Promise((
|
|
3203
|
+
await new Promise((resolve5) => setTimeout(resolve5, READ_BACK_DELAY_MS));
|
|
2203
3204
|
}
|
|
2204
3205
|
}
|
|
2205
3206
|
return null;
|
|
@@ -2207,57 +3208,68 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2207
3208
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2208
3209
|
if (!messages || messages.length === 0) return null;
|
|
2209
3210
|
const byParent = messages.find(
|
|
2210
|
-
(m) =>
|
|
3211
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2211
3212
|
);
|
|
2212
3213
|
if (byParent) return byParent;
|
|
2213
|
-
const userIndex = messages.findIndex((m) =>
|
|
3214
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2214
3215
|
if (userIndex === -1) return null;
|
|
2215
3216
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2216
|
-
if (
|
|
3217
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2217
3218
|
}
|
|
2218
3219
|
return null;
|
|
2219
3220
|
}
|
|
2220
3221
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
2221
3222
|
if (!messages || messages.length === 0) return null;
|
|
2222
3223
|
let lastCorrelated = null;
|
|
2223
|
-
let lastNonErrored = null;
|
|
2224
3224
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2225
3225
|
const m = messages[i];
|
|
2226
|
-
if (
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
3226
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
3227
|
+
lastCorrelated = m;
|
|
3228
|
+
break;
|
|
3229
|
+
}
|
|
3230
|
+
if (lastCorrelated) {
|
|
3231
|
+
if (errorOf(lastCorrelated) == null) return lastCorrelated;
|
|
3232
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
3233
|
+
const m = messages[i];
|
|
3234
|
+
if (roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId && isTerminalStopAnswer(m)) {
|
|
3235
|
+
return m;
|
|
3236
|
+
}
|
|
2231
3237
|
}
|
|
3238
|
+
return lastCorrelated;
|
|
2232
3239
|
}
|
|
2233
|
-
|
|
2234
|
-
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
3240
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2235
3241
|
if (userIndex === -1) return null;
|
|
2236
3242
|
let last = null;
|
|
2237
|
-
let
|
|
3243
|
+
let lastAssistantIndex = -1;
|
|
2238
3244
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2239
|
-
const role =
|
|
3245
|
+
const role = roleOf2(messages[i]);
|
|
2240
3246
|
if (role === "user") break;
|
|
2241
3247
|
if (role === "assistant") {
|
|
2242
3248
|
last = messages[i];
|
|
2243
|
-
|
|
3249
|
+
lastAssistantIndex = i;
|
|
2244
3250
|
}
|
|
2245
3251
|
}
|
|
2246
|
-
|
|
3252
|
+
if (last === null || errorOf(last) == null) return last;
|
|
3253
|
+
for (let i = lastAssistantIndex; i > userIndex; i--) {
|
|
3254
|
+
const m = messages[i];
|
|
3255
|
+
if (roleOf2(m) === "user") break;
|
|
3256
|
+
if (roleOf2(m) === "assistant" && isTerminalStopAnswer(m)) return m;
|
|
3257
|
+
}
|
|
3258
|
+
return last;
|
|
2247
3259
|
}
|
|
2248
3260
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2249
3261
|
if (!messages || messages.length === 0) return [];
|
|
2250
3262
|
const byParent = messages.filter(
|
|
2251
|
-
(message) =>
|
|
3263
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2252
3264
|
);
|
|
2253
3265
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2254
3266
|
if (assistants.length === 0) {
|
|
2255
|
-
const userIndex = messages.findIndex((message) =>
|
|
3267
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2256
3268
|
if (userIndex === -1) return [];
|
|
2257
3269
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2258
3270
|
const message = messages[i];
|
|
2259
|
-
if (
|
|
2260
|
-
if (
|
|
3271
|
+
if (roleOf2(message) === "user") break;
|
|
3272
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2261
3273
|
}
|
|
2262
3274
|
}
|
|
2263
3275
|
const refs = [];
|
|
@@ -2297,7 +3309,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2297
3309
|
if (!messages || messages.length === 0) return [];
|
|
2298
3310
|
const calls = [];
|
|
2299
3311
|
for (const message of messages) {
|
|
2300
|
-
if (
|
|
3312
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2301
3313
|
for (const part of message.parts ?? []) {
|
|
2302
3314
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2303
3315
|
continue;
|
|
@@ -2327,7 +3339,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2327
3339
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2328
3340
|
const unattributed = [];
|
|
2329
3341
|
for (const message of messages ?? []) {
|
|
2330
|
-
if (
|
|
3342
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2331
3343
|
const created = finiteNumber(createdOf(message));
|
|
2332
3344
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2333
3345
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2349,8 +3361,9 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2349
3361
|
}
|
|
2350
3362
|
function sumAssistantUsage(messages) {
|
|
2351
3363
|
if (!messages || messages.length === 0) return null;
|
|
2352
|
-
const
|
|
2353
|
-
const
|
|
3364
|
+
const hasTrailingError = errorOf(messages[messages.length - 1]) != null;
|
|
3365
|
+
const hasTerminalStopAnswer = messages.some(isTerminalStopAnswer);
|
|
3366
|
+
const selected = hasTrailingError && hasTerminalStopAnswer ? messages.filter((message) => errorOf(message) == null) : messages;
|
|
2354
3367
|
let sawAnyUsage = false;
|
|
2355
3368
|
let inputSum = 0;
|
|
2356
3369
|
let outputSum = 0;
|
|
@@ -2359,9 +3372,11 @@ function sumAssistantUsage(messages) {
|
|
|
2359
3372
|
let cacheWriteSum = 0;
|
|
2360
3373
|
let costSum = 0;
|
|
2361
3374
|
let sawCost = false;
|
|
3375
|
+
let toolCallCount = 0;
|
|
2362
3376
|
let modelId = null;
|
|
2363
3377
|
let providerId = null;
|
|
2364
3378
|
for (const m of selected) {
|
|
3379
|
+
toolCallCount += m.parts?.filter((part) => part.type === "tool").length ?? 0;
|
|
2365
3380
|
const info = m.info;
|
|
2366
3381
|
if (!info) continue;
|
|
2367
3382
|
const tokens = info.tokens;
|
|
@@ -2398,19 +3413,18 @@ function sumAssistantUsage(messages) {
|
|
|
2398
3413
|
usage_tokens_cache_write: cacheWriteSum,
|
|
2399
3414
|
// NULL means OpenCode never reported a cost; it is distinct from a genuine
|
|
2400
3415
|
// zero-cost message, which sets `sawCost` with `costSum === 0`.
|
|
2401
|
-
usage_cost_usd: sawCost ? costSum : null
|
|
3416
|
+
usage_cost_usd: sawCost ? costSum : null,
|
|
3417
|
+
turn_tool_calls: toolCallCount
|
|
2402
3418
|
};
|
|
2403
3419
|
}
|
|
2404
3420
|
function messageUsage(messages, userMessageId) {
|
|
2405
3421
|
if (!messages || messages.length === 0) return null;
|
|
2406
3422
|
const byParentAll = messages.filter(
|
|
2407
|
-
(m) =>
|
|
3423
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2408
3424
|
);
|
|
2409
|
-
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2410
|
-
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
2411
3425
|
let correlated;
|
|
2412
|
-
if (
|
|
2413
|
-
correlated =
|
|
3426
|
+
if (byParentAll.length > 0) {
|
|
3427
|
+
correlated = byParentAll;
|
|
2414
3428
|
} else {
|
|
2415
3429
|
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
2416
3430
|
correlated = reply ? [reply] : [];
|
|
@@ -2420,7 +3434,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2420
3434
|
}
|
|
2421
3435
|
function messageRunState(messages, userMessageId) {
|
|
2422
3436
|
if (!messages || messages.length === 0) return "unknown";
|
|
2423
|
-
const hasUser = messages.some((m) =>
|
|
3437
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2424
3438
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2425
3439
|
if (!hasUser) {
|
|
2426
3440
|
if (!reply) return "unknown";
|
|
@@ -2456,14 +3470,38 @@ function messageError(messages, userMessageId) {
|
|
|
2456
3470
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2457
3471
|
const error2 = errorOf(reply);
|
|
2458
3472
|
if (error2 == null) return null;
|
|
2459
|
-
|
|
2460
|
-
if (typeof error2 === "
|
|
3473
|
+
let rendered;
|
|
3474
|
+
if (typeof error2 === "string") {
|
|
3475
|
+
rendered = error2;
|
|
3476
|
+
} else if (typeof error2 === "object") {
|
|
2461
3477
|
const e = error2;
|
|
2462
3478
|
const dataMessage = e.data?.message;
|
|
2463
|
-
if (typeof dataMessage === "string")
|
|
2464
|
-
|
|
3479
|
+
if (typeof dataMessage === "string") {
|
|
3480
|
+
rendered = dataMessage;
|
|
3481
|
+
} else if (typeof e.message === "string") {
|
|
3482
|
+
rendered = e.message;
|
|
3483
|
+
} else {
|
|
3484
|
+
rendered = "The agent run failed.";
|
|
3485
|
+
}
|
|
3486
|
+
} else {
|
|
3487
|
+
rendered = "The agent run failed.";
|
|
2465
3488
|
}
|
|
2466
|
-
|
|
3489
|
+
const toolPart = Array.isArray(reply?.parts) ? reply.parts.find(
|
|
3490
|
+
(part) => part && typeof part === "object" && part.type === "tool" && part.state?.status === "error"
|
|
3491
|
+
) : void 0;
|
|
3492
|
+
return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
|
|
3493
|
+
}
|
|
3494
|
+
function describeAbandonedReplyShape(messages, userMessageId) {
|
|
3495
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3496
|
+
if (!reply) return null;
|
|
3497
|
+
const parts = Array.isArray(reply.parts) ? reply.parts : [];
|
|
3498
|
+
const lastTool = [...parts].reverse().find((part) => part?.type === "tool");
|
|
3499
|
+
return {
|
|
3500
|
+
partTypes: parts.slice(-12).map((part) => typeof part?.type === "string" ? part.type : "(unknown)"),
|
|
3501
|
+
droppedPartCount: Math.max(0, parts.length - 12),
|
|
3502
|
+
lastToolName: typeof lastTool?.tool === "string" && lastTool.tool.length > 0 ? lastTool.tool : null,
|
|
3503
|
+
lastToolStatus: typeof lastTool?.state?.status === "string" && lastTool.state.status.length > 0 ? lastTool.state.status : null
|
|
3504
|
+
};
|
|
2467
3505
|
}
|
|
2468
3506
|
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2469
3507
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2512,7 +3550,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2512
3550
|
if (!messages || messages.length === 0) return null;
|
|
2513
3551
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2514
3552
|
const message = messages[i];
|
|
2515
|
-
if (
|
|
3553
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2516
3554
|
const created = createdOf(message);
|
|
2517
3555
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2518
3556
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2544,25 +3582,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2544
3582
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2545
3583
|
if (!messages || messages.length === 0) return false;
|
|
2546
3584
|
return messages.some(
|
|
2547
|
-
(m) =>
|
|
3585
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2548
3586
|
);
|
|
2549
3587
|
}
|
|
2550
3588
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2551
3589
|
if (!messages || messages.length === 0) return false;
|
|
2552
|
-
const userIndex = messages.findIndex((message) =>
|
|
3590
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2553
3591
|
if (userIndex === -1) return false;
|
|
2554
3592
|
let hasLaterUser = false;
|
|
2555
3593
|
let hasStartedLaterUser = false;
|
|
2556
3594
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2557
3595
|
const message = messages[i];
|
|
2558
|
-
if (
|
|
3596
|
+
if (roleOf2(message) !== "user") continue;
|
|
2559
3597
|
hasLaterUser = true;
|
|
2560
|
-
const laterUserMessageId =
|
|
3598
|
+
const laterUserMessageId = idOf2(message);
|
|
2561
3599
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2562
3600
|
return false;
|
|
2563
3601
|
}
|
|
2564
3602
|
if (messages.some(
|
|
2565
|
-
(candidate) =>
|
|
3603
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2566
3604
|
)) {
|
|
2567
3605
|
hasStartedLaterUser = true;
|
|
2568
3606
|
}
|
|
@@ -3033,13 +4071,28 @@ async function listV2Sessions(client) {
|
|
|
3033
4071
|
}
|
|
3034
4072
|
async function deleteV2Session(client, sessionId) {
|
|
3035
4073
|
try {
|
|
3036
|
-
await client.request(
|
|
3037
|
-
|
|
4074
|
+
const response = await client.request(
|
|
4075
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
4076
|
+
{ method: "DELETE" },
|
|
4077
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
4078
|
+
);
|
|
4079
|
+
if (response.status >= 200 && response.status < 300) {
|
|
4080
|
+
return { ok: true, status: response.status };
|
|
4081
|
+
}
|
|
4082
|
+
try {
|
|
4083
|
+
const body = await response.text();
|
|
4084
|
+
return body ? { ok: false, status: response.status, body } : { ok: false, status: response.status };
|
|
4085
|
+
} catch (error2) {
|
|
4086
|
+
console.error(
|
|
4087
|
+
`[deleteV2Session] DELETE /api/session/${sessionId} response body read failed for HTTP ${response.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4088
|
+
);
|
|
4089
|
+
return { ok: false, status: response.status };
|
|
4090
|
+
}
|
|
3038
4091
|
} catch (error2) {
|
|
3039
4092
|
console.error(
|
|
3040
4093
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3041
4094
|
);
|
|
3042
|
-
return false;
|
|
4095
|
+
return { ok: false, status: "threw" };
|
|
3043
4096
|
}
|
|
3044
4097
|
}
|
|
3045
4098
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3210,12 +4263,12 @@ async function readV2SessionErrorStream(client, options) {
|
|
|
3210
4263
|
|
|
3211
4264
|
// src/lib/opencode/credential-import-v2.ts
|
|
3212
4265
|
import { readFileSync as readFileSync4 } from "node:fs";
|
|
3213
|
-
import { join as
|
|
4266
|
+
import { join as join5 } from "node:path";
|
|
3214
4267
|
|
|
3215
4268
|
// src/lib/openai-usage.ts
|
|
3216
4269
|
import { readFileSync as readFileSync3 } from "node:fs";
|
|
3217
|
-
import { homedir as
|
|
3218
|
-
import { join as
|
|
4270
|
+
import { homedir as homedir3 } from "node:os";
|
|
4271
|
+
import { join as join4 } from "node:path";
|
|
3219
4272
|
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
3220
4273
|
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
3221
4274
|
var OpenAiUsageError = class extends Error {
|
|
@@ -3229,7 +4282,7 @@ function isLocalCredentialProblem2(err) {
|
|
|
3229
4282
|
}
|
|
3230
4283
|
function readOpenCodeChatGptCredentials() {
|
|
3231
4284
|
try {
|
|
3232
|
-
const raw = readFileSync3(
|
|
4285
|
+
const raw = readFileSync3(join4(homedir3(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
3233
4286
|
let parsed;
|
|
3234
4287
|
try {
|
|
3235
4288
|
parsed = JSON.parse(raw);
|
|
@@ -3370,16 +4423,17 @@ function hasPrimaryHeaders(headers) {
|
|
|
3370
4423
|
].some((name) => headers.has(name));
|
|
3371
4424
|
}
|
|
3372
4425
|
async function getOpenAiUsage(port, client) {
|
|
4426
|
+
const openCodeVersion = client?.version ?? "v1";
|
|
3373
4427
|
const credentials2 = readOpenCodeChatGptCredentials();
|
|
3374
4428
|
if (!credentials2) {
|
|
3375
4429
|
throw new OpenAiUsageError(
|
|
3376
|
-
"No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
4430
|
+
openCodeVersion === "v2" ? "OpenAI usage reporting is not available on OpenCode V2 because V2 keeps a connected ChatGPT login in a store Evident cannot read." : "No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3377
4431
|
"no_credentials"
|
|
3378
4432
|
);
|
|
3379
4433
|
}
|
|
3380
4434
|
if (credentials2.expiresAt < Date.now()) {
|
|
3381
4435
|
throw new OpenAiUsageError(
|
|
3382
|
-
"ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
4436
|
+
openCodeVersion === "v2" ? "The delivered ChatGPT login has expired. OpenCode V2 cannot be given a replacement by file; connect the account again from the runner's page in Evident." : "ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3383
4437
|
"credentials_expired"
|
|
3384
4438
|
);
|
|
3385
4439
|
}
|
|
@@ -3454,13 +4508,13 @@ var INTEGRATION_LIST_RETRY_DELAY_MS = 1e3;
|
|
|
3454
4508
|
function isRecord3(value) {
|
|
3455
4509
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3456
4510
|
}
|
|
3457
|
-
var sleep2 = (milliseconds) => new Promise((
|
|
4511
|
+
var sleep2 = (milliseconds) => new Promise((resolve5) => setTimeout(resolve5, milliseconds));
|
|
3458
4512
|
function emptyResult(unsupported = [], outcome = "completed") {
|
|
3459
4513
|
return { outcome, imported: [], alreadyConnected: [], unsupported, failed: [] };
|
|
3460
4514
|
}
|
|
3461
4515
|
function readOpenCodeCredentials(homeDir, log3) {
|
|
3462
4516
|
try {
|
|
3463
|
-
const raw = readFileSync4(
|
|
4517
|
+
const raw = readFileSync4(join5(homeDir, ...OPENCODE_AUTH_SEGMENTS), "utf8");
|
|
3464
4518
|
let parsed;
|
|
3465
4519
|
try {
|
|
3466
4520
|
parsed = JSON.parse(raw);
|
|
@@ -3660,9 +4714,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3660
4714
|
|
|
3661
4715
|
// src/lib/opencode/session-db-boot.ts
|
|
3662
4716
|
import { spawn as spawn2 } from "node:child_process";
|
|
3663
|
-
import { mkdirSync, statSync as
|
|
3664
|
-
import { homedir as
|
|
3665
|
-
import { dirname as
|
|
4717
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
4718
|
+
import { homedir as homedir4 } from "node:os";
|
|
4719
|
+
import { dirname as dirname4, resolve as resolvePath } from "node:path";
|
|
3666
4720
|
|
|
3667
4721
|
// src/lib/runner-synchroniser.ts
|
|
3668
4722
|
import { spawn } from "node:child_process";
|
|
@@ -3672,7 +4726,7 @@ function appendError(stderr, error2) {
|
|
|
3672
4726
|
${message}`;
|
|
3673
4727
|
}
|
|
3674
4728
|
function runSynchroniser(args, opts) {
|
|
3675
|
-
return new Promise((
|
|
4729
|
+
return new Promise((resolve5) => {
|
|
3676
4730
|
let child;
|
|
3677
4731
|
let stdout = "";
|
|
3678
4732
|
let stderr = "";
|
|
@@ -3686,7 +4740,7 @@ function runSynchroniser(args, opts) {
|
|
|
3686
4740
|
if (timer.handle) clearTimeout(timer.handle);
|
|
3687
4741
|
if (abortListener) opts.signal?.removeEventListener("abort", abortListener);
|
|
3688
4742
|
if (spawnListener) child.removeListener("spawn", spawnListener);
|
|
3689
|
-
|
|
4743
|
+
resolve5(result);
|
|
3690
4744
|
};
|
|
3691
4745
|
try {
|
|
3692
4746
|
child = spawn("runner-synchroniser", args, {
|
|
@@ -3742,9 +4796,12 @@ function runSynchroniser(args, opts) {
|
|
|
3742
4796
|
|
|
3743
4797
|
// src/lib/opencode/session-db-boot.ts
|
|
3744
4798
|
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
3745
|
-
var SESSION_DB_VERIFY_TIMEOUT_MS =
|
|
4799
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
|
|
3746
4800
|
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
3747
|
-
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS =
|
|
4801
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
|
|
4802
|
+
var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
|
|
4803
|
+
var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
|
|
4804
|
+
var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
|
|
3748
4805
|
function commandError(result) {
|
|
3749
4806
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
3750
4807
|
}
|
|
@@ -3787,7 +4844,7 @@ function clearMarker(options) {
|
|
|
3787
4844
|
function markNoReplicate(options, message) {
|
|
3788
4845
|
if (options.noReplicateMarker) {
|
|
3789
4846
|
try {
|
|
3790
|
-
mkdirSync(
|
|
4847
|
+
mkdirSync(dirname4(options.noReplicateMarker), { recursive: true });
|
|
3791
4848
|
writeFileSync(options.noReplicateMarker, "");
|
|
3792
4849
|
} catch (error2) {
|
|
3793
4850
|
options.log(
|
|
@@ -3846,7 +4903,7 @@ function parseSynchroniserEnv(stdout) {
|
|
|
3846
4903
|
return values;
|
|
3847
4904
|
}
|
|
3848
4905
|
function runCommand(command, args, options) {
|
|
3849
|
-
return new Promise((
|
|
4906
|
+
return new Promise((resolve5) => {
|
|
3850
4907
|
let child;
|
|
3851
4908
|
let stdout = "";
|
|
3852
4909
|
let stderr = "";
|
|
@@ -3855,7 +4912,7 @@ function runCommand(command, args, options) {
|
|
|
3855
4912
|
if (settled) return;
|
|
3856
4913
|
settled = true;
|
|
3857
4914
|
if (timer) clearTimeout(timer);
|
|
3858
|
-
|
|
4915
|
+
resolve5(result);
|
|
3859
4916
|
};
|
|
3860
4917
|
try {
|
|
3861
4918
|
child = spawn2(command, args, {
|
|
@@ -3863,7 +4920,7 @@ function runCommand(command, args, options) {
|
|
|
3863
4920
|
stdio: ["ignore", "pipe", "pipe"]
|
|
3864
4921
|
});
|
|
3865
4922
|
} catch (error2) {
|
|
3866
|
-
|
|
4923
|
+
resolve5({
|
|
3867
4924
|
code: null,
|
|
3868
4925
|
stdout,
|
|
3869
4926
|
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
@@ -3912,7 +4969,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3912
4969
|
return null;
|
|
3913
4970
|
}
|
|
3914
4971
|
try {
|
|
3915
|
-
if (
|
|
4972
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
3916
4973
|
} catch (error2) {
|
|
3917
4974
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
3918
4975
|
options.log(
|
|
@@ -3942,7 +4999,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3942
4999
|
return null;
|
|
3943
5000
|
}
|
|
3944
5001
|
try {
|
|
3945
|
-
mkdirSync(
|
|
5002
|
+
mkdirSync(dirname4(configPath), { recursive: true });
|
|
3946
5003
|
writeFileSync(configPath, rendered.stdout);
|
|
3947
5004
|
} catch (error2) {
|
|
3948
5005
|
options.log(
|
|
@@ -4017,14 +5074,14 @@ async function restoreSessionDb(options, configPath, env) {
|
|
|
4017
5074
|
case 0:
|
|
4018
5075
|
return;
|
|
4019
5076
|
case 31:
|
|
4020
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
5077
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4021
5078
|
options.log(
|
|
4022
5079
|
"SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot",
|
|
4023
5080
|
"warn"
|
|
4024
5081
|
);
|
|
4025
5082
|
return;
|
|
4026
5083
|
case 32:
|
|
4027
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
5084
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4028
5085
|
discardSessionDbDebris(options);
|
|
4029
5086
|
markNoReplicate(
|
|
4030
5087
|
options,
|
|
@@ -4055,20 +5112,37 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4055
5112
|
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
4056
5113
|
env: {
|
|
4057
5114
|
...env,
|
|
4058
|
-
// The synchroniser reads
|
|
4059
|
-
// 120_000, so the walkback gives up before the outer process bound.
|
|
5115
|
+
// The synchroniser reads all four values in SECONDS.
|
|
4060
5116
|
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
4061
5117
|
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
5118
|
+
),
|
|
5119
|
+
EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
|
|
5120
|
+
SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
|
|
5121
|
+
),
|
|
5122
|
+
EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
|
|
5123
|
+
SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
|
|
5124
|
+
),
|
|
5125
|
+
EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
|
|
5126
|
+
SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
|
|
4062
5127
|
)
|
|
4063
5128
|
}
|
|
4064
5129
|
});
|
|
4065
5130
|
logSynchroniserDiagnostics(result, options);
|
|
4066
5131
|
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
5132
|
+
if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
|
|
5133
|
+
const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
|
|
5134
|
+
options.log(
|
|
5135
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
|
|
5136
|
+
"error"
|
|
5137
|
+
);
|
|
5138
|
+
reportRecord("verify", "session_db_boot_refused", reason, null, options);
|
|
5139
|
+
return { verifyFatal: true, verifyFatalReason: reason };
|
|
5140
|
+
}
|
|
4067
5141
|
options.log(
|
|
4068
5142
|
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
4069
5143
|
"warn"
|
|
4070
5144
|
);
|
|
4071
|
-
return false;
|
|
5145
|
+
return { verifyFatal: false };
|
|
4072
5146
|
}
|
|
4073
5147
|
if (result.code === 34) {
|
|
4074
5148
|
reportRecord(
|
|
@@ -4078,14 +5152,14 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4078
5152
|
null,
|
|
4079
5153
|
options
|
|
4080
5154
|
);
|
|
4081
|
-
return true;
|
|
5155
|
+
return { verifyFatal: true };
|
|
4082
5156
|
}
|
|
4083
5157
|
if (result.code === 33) {
|
|
4084
5158
|
options.log(
|
|
4085
5159
|
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
4086
5160
|
"warn"
|
|
4087
5161
|
);
|
|
4088
|
-
return false;
|
|
5162
|
+
return { verifyFatal: false };
|
|
4089
5163
|
}
|
|
4090
5164
|
if (result.code !== 0) {
|
|
4091
5165
|
options.log(
|
|
@@ -4093,17 +5167,17 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4093
5167
|
"warn"
|
|
4094
5168
|
);
|
|
4095
5169
|
}
|
|
4096
|
-
return false;
|
|
5170
|
+
return { verifyFatal: false };
|
|
4097
5171
|
}
|
|
4098
5172
|
options.log(
|
|
4099
5173
|
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
4100
5174
|
"debug"
|
|
4101
5175
|
);
|
|
4102
|
-
return false;
|
|
5176
|
+
return { verifyFatal: false };
|
|
4103
5177
|
}
|
|
4104
5178
|
function fileExists(path) {
|
|
4105
5179
|
try {
|
|
4106
|
-
|
|
5180
|
+
statSync3(path);
|
|
4107
5181
|
return true;
|
|
4108
5182
|
} catch (error2) {
|
|
4109
5183
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4113,7 +5187,7 @@ function fileExists(path) {
|
|
|
4113
5187
|
async function restoreAndVerifySessionDb(options) {
|
|
4114
5188
|
const env = options.env ?? process.env;
|
|
4115
5189
|
clearMarker(options);
|
|
4116
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
5190
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4117
5191
|
const synchroniserEnv = await runSynchroniser(["env"], {
|
|
4118
5192
|
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
4119
5193
|
env
|
|
@@ -4186,13 +5260,13 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4186
5260
|
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
4187
5261
|
return { verifyFatal: false };
|
|
4188
5262
|
}
|
|
4189
|
-
return
|
|
5263
|
+
return verifySessionDb(options, configPath, env);
|
|
4190
5264
|
}
|
|
4191
5265
|
|
|
4192
5266
|
// src/lib/opencode/session-db-provenance.ts
|
|
4193
5267
|
import { createRequire } from "node:module";
|
|
4194
5268
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4195
|
-
import { dirname as
|
|
5269
|
+
import { dirname as dirname5, join as join6 } from "node:path";
|
|
4196
5270
|
var require2 = createRequire(import.meta.url);
|
|
4197
5271
|
function readSessionDbMigrationIds(dbPath) {
|
|
4198
5272
|
let db;
|
|
@@ -4229,7 +5303,7 @@ function readSessionDbMigrationIds(dbPath) {
|
|
|
4229
5303
|
}
|
|
4230
5304
|
function sessionDbProvenanceStatePath(homeDir, env) {
|
|
4231
5305
|
const override = env.EVIDENT_SESSION_DB_PROVENANCE_STATE?.trim();
|
|
4232
|
-
return override ||
|
|
5306
|
+
return override || join6(homeDir, ".local", "state", "evident", "session-db-provenance.json");
|
|
4233
5307
|
}
|
|
4234
5308
|
function loadSessionDbProvenanceState(path) {
|
|
4235
5309
|
let value;
|
|
@@ -4258,7 +5332,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4258
5332
|
}
|
|
4259
5333
|
function saveSessionDbProvenanceState(path, state) {
|
|
4260
5334
|
try {
|
|
4261
|
-
mkdirSync2(
|
|
5335
|
+
mkdirSync2(dirname5(path), { recursive: true });
|
|
4262
5336
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4263
5337
|
`, "utf8");
|
|
4264
5338
|
} catch (error2) {
|
|
@@ -4349,7 +5423,7 @@ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
|
4349
5423
|
if (child.exitCode !== null || child.signalCode !== null) {
|
|
4350
5424
|
return { outcome: "exited", code: child.exitCode, signal: child.signalCode };
|
|
4351
5425
|
}
|
|
4352
|
-
return new Promise((
|
|
5426
|
+
return new Promise((resolve5, reject) => {
|
|
4353
5427
|
let forced = false;
|
|
4354
5428
|
let settled = false;
|
|
4355
5429
|
const timer = setTimeout(() => {
|
|
@@ -4369,7 +5443,7 @@ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
|
4369
5443
|
settled = true;
|
|
4370
5444
|
clearTimeout(timer);
|
|
4371
5445
|
child.removeListener("exit", onExit);
|
|
4372
|
-
|
|
5446
|
+
resolve5(result);
|
|
4373
5447
|
};
|
|
4374
5448
|
const fail = (error2) => {
|
|
4375
5449
|
if (settled) return;
|
|
@@ -4839,10 +5913,13 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4839
5913
|
toDelete.push(s.id);
|
|
4840
5914
|
}
|
|
4841
5915
|
}
|
|
4842
|
-
|
|
5916
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5917
|
+
return toDelete.sort(
|
|
5918
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5919
|
+
);
|
|
4843
5920
|
}
|
|
4844
5921
|
var DEFAULT_INTERVAL = "1h";
|
|
4845
|
-
function
|
|
5922
|
+
function resolve2(flag, envValue, fallback) {
|
|
4846
5923
|
return flag ?? envValue ?? fallback;
|
|
4847
5924
|
}
|
|
4848
5925
|
function parseMaxCount(input) {
|
|
@@ -4858,9 +5935,9 @@ function parseMaxCount(input) {
|
|
|
4858
5935
|
}
|
|
4859
5936
|
function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
4860
5937
|
const warnings = [];
|
|
4861
|
-
const maxAgeRaw =
|
|
4862
|
-
const maxCountRaw =
|
|
4863
|
-
const intervalRaw =
|
|
5938
|
+
const maxAgeRaw = resolve2(flags.maxAge, env.EVIDENT_SESSION_CLEANUP_MAX_AGE);
|
|
5939
|
+
const maxCountRaw = resolve2(flags.maxCount, env.EVIDENT_SESSION_CLEANUP_MAX_COUNT);
|
|
5940
|
+
const intervalRaw = resolve2(
|
|
4864
5941
|
flags.interval,
|
|
4865
5942
|
env.EVIDENT_SESSION_CLEANUP_INTERVAL,
|
|
4866
5943
|
DEFAULT_INTERVAL
|
|
@@ -4899,13 +5976,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
4899
5976
|
}
|
|
4900
5977
|
|
|
4901
5978
|
// src/lib/opencode/session-db-size.ts
|
|
4902
|
-
import { statSync as
|
|
4903
|
-
import { join as
|
|
5979
|
+
import { statSync as statSync4 } from "node:fs";
|
|
5980
|
+
import { join as join7 } from "node:path";
|
|
4904
5981
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
4905
5982
|
function statSessionDbBytes(homeDir) {
|
|
4906
|
-
const dbPath =
|
|
5983
|
+
const dbPath = join7(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
4907
5984
|
try {
|
|
4908
|
-
return
|
|
5985
|
+
return statSync4(dbPath).size;
|
|
4909
5986
|
} catch (err) {
|
|
4910
5987
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
4911
5988
|
if (!isMissingFile) {
|
|
@@ -4931,14 +6008,14 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
4931
6008
|
}
|
|
4932
6009
|
|
|
4933
6010
|
// src/lib/opencode/log-tail.ts
|
|
4934
|
-
import { statSync as
|
|
4935
|
-
import { homedir as
|
|
4936
|
-
import { join as
|
|
6011
|
+
import { statSync as statSync5 } from "node:fs";
|
|
6012
|
+
import { homedir as homedir5 } from "node:os";
|
|
6013
|
+
import { join as join8 } from "node:path";
|
|
4937
6014
|
import { open as open2, stat } from "node:fs/promises";
|
|
4938
6015
|
var DEFAULT_POLL_INTERVAL_MS = 1e3;
|
|
4939
|
-
function resolveOpenCodeLogPath(homeDir =
|
|
4940
|
-
const dataDir = env.XDG_DATA_HOME ||
|
|
4941
|
-
return
|
|
6016
|
+
function resolveOpenCodeLogPath(homeDir = homedir5(), env = process.env) {
|
|
6017
|
+
const dataDir = env.XDG_DATA_HOME || join8(homeDir, ".local", "share");
|
|
6018
|
+
return join8(dataDir, "opencode", "log", "opencode.log");
|
|
4942
6019
|
}
|
|
4943
6020
|
function isEnoent(error2) {
|
|
4944
6021
|
return error2?.code === "ENOENT";
|
|
@@ -4954,7 +6031,7 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
4954
6031
|
let inode = null;
|
|
4955
6032
|
let baselineReady = true;
|
|
4956
6033
|
try {
|
|
4957
|
-
const initial =
|
|
6034
|
+
const initial = statSync5(logPath);
|
|
4958
6035
|
offset = initial.size;
|
|
4959
6036
|
inode = initial.ino;
|
|
4960
6037
|
} catch (error2) {
|
|
@@ -5000,136 +6077,17 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5000
6077
|
} catch (error2) {
|
|
5001
6078
|
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
5002
6079
|
} finally {
|
|
5003
|
-
polling = false;
|
|
5004
|
-
}
|
|
5005
|
-
};
|
|
5006
|
-
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
5007
|
-
void poll();
|
|
5008
|
-
return {
|
|
5009
|
-
stop: () => {
|
|
5010
|
-
stopped = true;
|
|
5011
|
-
clearInterval(interval);
|
|
5012
|
-
}
|
|
5013
|
-
};
|
|
5014
|
-
}
|
|
5015
|
-
|
|
5016
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5017
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5018
|
-
import { dirname as dirname4 } from "node:path";
|
|
5019
|
-
function errorMessage(error2) {
|
|
5020
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5021
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5022
|
-
}
|
|
5023
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5024
|
-
try {
|
|
5025
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5026
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5027
|
-
if (availableBytes < requiredBytes) {
|
|
5028
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5029
|
-
}
|
|
5030
|
-
return null;
|
|
5031
|
-
} catch (err) {
|
|
5032
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5033
|
-
}
|
|
5034
|
-
}
|
|
5035
|
-
function readLogicalBytes(db) {
|
|
5036
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5037
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5038
|
-
return pageCount * pageSize;
|
|
5039
|
-
}
|
|
5040
|
-
function readCheckpointResult(db) {
|
|
5041
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5042
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5043
|
-
}
|
|
5044
|
-
async function probeReclaimAvailability(input) {
|
|
5045
|
-
const { dbPath, requiredBytes } = input;
|
|
5046
|
-
let sqlite;
|
|
5047
|
-
try {
|
|
5048
|
-
sqlite = await import("node:sqlite");
|
|
5049
|
-
} catch (err) {
|
|
5050
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5051
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5052
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5053
|
-
}
|
|
5054
|
-
let autoVacuum = null;
|
|
5055
|
-
try {
|
|
5056
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5057
|
-
try {
|
|
5058
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5059
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5060
|
-
} finally {
|
|
5061
|
-
db.close();
|
|
5062
|
-
}
|
|
5063
|
-
} catch (err) {
|
|
5064
|
-
console.warn(
|
|
5065
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5066
|
-
);
|
|
5067
|
-
}
|
|
5068
|
-
if (autoVacuum !== 0) return null;
|
|
5069
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5070
|
-
}
|
|
5071
|
-
async function reclaimSessionDbSpace(input) {
|
|
5072
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5073
|
-
let sqlite;
|
|
5074
|
-
try {
|
|
5075
|
-
sqlite = await import("node:sqlite");
|
|
5076
|
-
} catch (err) {
|
|
5077
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5078
|
-
console.warn(
|
|
5079
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5080
|
-
);
|
|
5081
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5082
|
-
}
|
|
5083
|
-
const { DatabaseSync } = sqlite;
|
|
5084
|
-
let db;
|
|
5085
|
-
try {
|
|
5086
|
-
db = new DatabaseSync(dbPath);
|
|
5087
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5088
|
-
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5089
|
-
if (autoVacuum === 0) {
|
|
5090
|
-
if (!allowFullVacuum) {
|
|
5091
|
-
console.warn(
|
|
5092
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
5093
|
-
);
|
|
5094
|
-
return { ok: false, skipped: "full-vacuum-blocked" };
|
|
5095
|
-
}
|
|
5096
|
-
const fileBytesForGuard = statSync5(dbPath).size;
|
|
5097
|
-
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
5098
|
-
if (skipReason !== null) {
|
|
5099
|
-
console.warn(
|
|
5100
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
5101
|
-
);
|
|
5102
|
-
return { ok: false, skipped: "insufficient-disk-space" };
|
|
5103
|
-
}
|
|
5104
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5105
|
-
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
5106
|
-
db.exec("VACUUM");
|
|
5107
|
-
const afterBytes = readLogicalBytes(db);
|
|
5108
|
-
const checkpoint = readCheckpointResult(db);
|
|
5109
|
-
return { ok: true, mode: "convert", beforeBytes, afterBytes, checkpoint };
|
|
5110
|
-
}
|
|
5111
|
-
if (autoVacuum === 2) {
|
|
5112
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5113
|
-
const bound = Math.max(0, Math.trunc(maxPages));
|
|
5114
|
-
db.exec(`PRAGMA incremental_vacuum(${bound})`);
|
|
5115
|
-
const afterBytes = readLogicalBytes(db);
|
|
5116
|
-
const checkpoint = readCheckpointResult(db);
|
|
5117
|
-
return { ok: true, mode: "incremental", beforeBytes, afterBytes, checkpoint };
|
|
5118
|
-
}
|
|
5119
|
-
console.warn(
|
|
5120
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5121
|
-
);
|
|
5122
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5123
|
-
} catch (err) {
|
|
5124
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5125
|
-
return {
|
|
5126
|
-
ok: false,
|
|
5127
|
-
skipped: "reclaim-error",
|
|
5128
|
-
detail: errorMessage(err)
|
|
5129
|
-
};
|
|
5130
|
-
} finally {
|
|
5131
|
-
db?.close();
|
|
5132
|
-
}
|
|
6080
|
+
polling = false;
|
|
6081
|
+
}
|
|
6082
|
+
};
|
|
6083
|
+
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
6084
|
+
void poll();
|
|
6085
|
+
return {
|
|
6086
|
+
stop: () => {
|
|
6087
|
+
stopped = true;
|
|
6088
|
+
clearInterval(interval);
|
|
6089
|
+
}
|
|
6090
|
+
};
|
|
5133
6091
|
}
|
|
5134
6092
|
|
|
5135
6093
|
// src/lib/tunnel/connection.ts
|
|
@@ -5212,8 +6170,8 @@ var StreamForwarder = class {
|
|
|
5212
6170
|
pauseStream(sid, stream) {
|
|
5213
6171
|
if (!stream.paused) {
|
|
5214
6172
|
stream.paused = true;
|
|
5215
|
-
stream.pausePromise = new Promise((
|
|
5216
|
-
stream.resume =
|
|
6173
|
+
stream.pausePromise = new Promise((resolve5) => {
|
|
6174
|
+
stream.resume = resolve5;
|
|
5217
6175
|
});
|
|
5218
6176
|
}
|
|
5219
6177
|
if (stream.pauseTimer !== null) clearTimeout(stream.pauseTimer);
|
|
@@ -5280,12 +6238,12 @@ var StreamForwarder = class {
|
|
|
5280
6238
|
let endBody;
|
|
5281
6239
|
if (has_body) {
|
|
5282
6240
|
const chunks = [];
|
|
5283
|
-
bodyPromise = new Promise((
|
|
6241
|
+
bodyPromise = new Promise((resolve5) => {
|
|
5284
6242
|
pushBody = (buf) => {
|
|
5285
6243
|
chunks.push(buf);
|
|
5286
6244
|
};
|
|
5287
6245
|
endBody = () => {
|
|
5288
|
-
|
|
6246
|
+
resolve5(Buffer.concat(chunks));
|
|
5289
6247
|
};
|
|
5290
6248
|
});
|
|
5291
6249
|
}
|
|
@@ -5361,6 +6319,8 @@ var StreamForwarder = class {
|
|
|
5361
6319
|
if (done) break;
|
|
5362
6320
|
const chunk = Buffer.from(value);
|
|
5363
6321
|
for (let i = 0; i < chunk.length; i += MAX_FRAME_BYTES) {
|
|
6322
|
+
await stream.pausePromise;
|
|
6323
|
+
if (ac.signal.aborted) break;
|
|
5364
6324
|
const slice = chunk.subarray(i, i + MAX_FRAME_BYTES);
|
|
5365
6325
|
if (this.binaryFramesSupported) {
|
|
5366
6326
|
if (this.ws.readyState === WebSocket.OPEN) {
|
|
@@ -5403,6 +6363,7 @@ function classifyUpgradeRejection(headers) {
|
|
|
5403
6363
|
}
|
|
5404
6364
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
5405
6365
|
var BASE_RECONNECT_DELAY = 500;
|
|
6366
|
+
var RELAY_SILENCE_TIMEOUT_MS = 9e4;
|
|
5406
6367
|
function getReconnectDelay(attempt) {
|
|
5407
6368
|
const exponentialDelay = BASE_RECONNECT_DELAY * Math.pow(2, attempt);
|
|
5408
6369
|
const jitter = Math.random() * 1e3;
|
|
@@ -5453,7 +6414,7 @@ function connectTunnel(options) {
|
|
|
5453
6414
|
} = options;
|
|
5454
6415
|
const tunnelUrl = getTunnelUrlConfig();
|
|
5455
6416
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
5456
|
-
return new Promise((
|
|
6417
|
+
return new Promise((resolve5, reject) => {
|
|
5457
6418
|
const ws = new WebSocket2(url, {
|
|
5458
6419
|
headers: {
|
|
5459
6420
|
Authorization: authHeader,
|
|
@@ -5474,10 +6435,27 @@ function connectTunnel(options) {
|
|
|
5474
6435
|
ws.close();
|
|
5475
6436
|
reject(new Error("Connection timeout"));
|
|
5476
6437
|
}, 3e4);
|
|
6438
|
+
let relaySilenceTimeout;
|
|
6439
|
+
let relaySilenceTimedOut = false;
|
|
6440
|
+
const clearRelaySilenceTimeout = () => {
|
|
6441
|
+
if (relaySilenceTimeout === void 0) return;
|
|
6442
|
+
clearTimeout(relaySilenceTimeout);
|
|
6443
|
+
relaySilenceTimeout = void 0;
|
|
6444
|
+
};
|
|
6445
|
+
const armRelaySilenceTimeout = () => {
|
|
6446
|
+
clearRelaySilenceTimeout();
|
|
6447
|
+
relaySilenceTimeout = setTimeout(() => {
|
|
6448
|
+
relaySilenceTimeout = void 0;
|
|
6449
|
+
relaySilenceTimedOut = true;
|
|
6450
|
+
ws.terminate();
|
|
6451
|
+
}, RELAY_SILENCE_TIMEOUT_MS);
|
|
6452
|
+
relaySilenceTimeout.unref?.();
|
|
6453
|
+
};
|
|
5477
6454
|
let upgradeRejection = null;
|
|
5478
6455
|
let upgradeRejectionReason = null;
|
|
5479
6456
|
ws.on("unexpected-response", (_req, res) => {
|
|
5480
6457
|
clearTimeout(connectionTimeout);
|
|
6458
|
+
clearRelaySilenceTimeout();
|
|
5481
6459
|
const reason = classifyUpgradeRejection(res.headers);
|
|
5482
6460
|
upgradeRejectionReason = reason;
|
|
5483
6461
|
const chunks = [];
|
|
@@ -5507,6 +6485,7 @@ function connectTunnel(options) {
|
|
|
5507
6485
|
onInfo?.("WebSocket connection established");
|
|
5508
6486
|
});
|
|
5509
6487
|
ws.on("message", (data, isBinary) => {
|
|
6488
|
+
if (relaySilenceTimeout !== void 0) armRelaySilenceTimeout();
|
|
5510
6489
|
if (isBinary) {
|
|
5511
6490
|
try {
|
|
5512
6491
|
const frame = decodeBinaryBodyFrame(toUint8Array(data));
|
|
@@ -5540,10 +6519,11 @@ function connectTunnel(options) {
|
|
|
5540
6519
|
switch (message.type) {
|
|
5541
6520
|
case "connected": {
|
|
5542
6521
|
clearTimeout(connectionTimeout);
|
|
6522
|
+
armRelaySilenceTimeout();
|
|
5543
6523
|
forwarder.setBinaryFramesSupported(message.binary_frames === true);
|
|
5544
6524
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
5545
6525
|
onConnected?.(connectedAgentId);
|
|
5546
|
-
|
|
6526
|
+
resolve5({
|
|
5547
6527
|
ws,
|
|
5548
6528
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
5549
6529
|
});
|
|
@@ -5564,6 +6544,7 @@ function connectTunnel(options) {
|
|
|
5564
6544
|
});
|
|
5565
6545
|
ws.on("error", (error2) => {
|
|
5566
6546
|
clearTimeout(connectionTimeout);
|
|
6547
|
+
clearRelaySilenceTimeout();
|
|
5567
6548
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
5568
6549
|
if (upgradeRejectionReason === "do_code_updated") {
|
|
5569
6550
|
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
@@ -5575,7 +6556,8 @@ function connectTunnel(options) {
|
|
|
5575
6556
|
);
|
|
5576
6557
|
});
|
|
5577
6558
|
ws.on("close", (code, reason) => {
|
|
5578
|
-
|
|
6559
|
+
clearRelaySilenceTimeout();
|
|
6560
|
+
const reasonStr = (relaySilenceTimedOut ? "no relay heartbeat for 90s \u2014 reconnecting" : void 0) || reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
5579
6561
|
forwarder.abortAll();
|
|
5580
6562
|
onDisconnected?.(code, reasonStr);
|
|
5581
6563
|
});
|
|
@@ -5583,9 +6565,10 @@ function connectTunnel(options) {
|
|
|
5583
6565
|
}
|
|
5584
6566
|
|
|
5585
6567
|
// src/lib/tunnel/runner-connection.ts
|
|
5586
|
-
var RunnerConnection = class {
|
|
6568
|
+
var RunnerConnection = class _RunnerConnection {
|
|
5587
6569
|
opts;
|
|
5588
6570
|
sleep;
|
|
6571
|
+
now;
|
|
5589
6572
|
connection = null;
|
|
5590
6573
|
resolvedAgentId;
|
|
5591
6574
|
/** True while a (re)connect loop is in flight. */
|
|
@@ -5594,10 +6577,17 @@ var RunnerConnection = class {
|
|
|
5594
6577
|
reconnectPromise = null;
|
|
5595
6578
|
/** 1-based count of the current reconnect attempt streak. */
|
|
5596
6579
|
reconnectAttempt = 0;
|
|
6580
|
+
generation = 0;
|
|
6581
|
+
disconnectedAtMs = null;
|
|
6582
|
+
static RELAY_RECONNECT_CLOSE_REASONS = /* @__PURE__ */ new Set([
|
|
6583
|
+
"No pong received",
|
|
6584
|
+
"Orphaned socket"
|
|
6585
|
+
]);
|
|
5597
6586
|
constructor(opts) {
|
|
5598
6587
|
this.opts = opts;
|
|
5599
6588
|
this.resolvedAgentId = opts.agentId;
|
|
5600
6589
|
this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
6590
|
+
this.now = opts.now ?? Date.now;
|
|
5601
6591
|
}
|
|
5602
6592
|
get agentId() {
|
|
5603
6593
|
return this.resolvedAgentId;
|
|
@@ -5633,14 +6623,24 @@ var RunnerConnection = class {
|
|
|
5633
6623
|
port: this.opts.port,
|
|
5634
6624
|
openCodePassword: this.opts.openCodePassword,
|
|
5635
6625
|
onConnected: (agentId) => {
|
|
6626
|
+
const attempt = this.reconnectAttempt + 1;
|
|
5636
6627
|
this.reconnectAttempt = 0;
|
|
5637
6628
|
this.reconnecting = false;
|
|
5638
6629
|
this.resolvedAgentId = agentId;
|
|
5639
|
-
|
|
6630
|
+
this.generation++;
|
|
6631
|
+
const downtimeMs = this.disconnectedAtMs === null ? null : this.now() - this.disconnectedAtMs;
|
|
6632
|
+
this.disconnectedAtMs = null;
|
|
6633
|
+
events.onConnected(agentId, isReconnect, {
|
|
6634
|
+
generation: this.generation,
|
|
6635
|
+
attempt,
|
|
6636
|
+
downtimeMs
|
|
6637
|
+
});
|
|
5640
6638
|
},
|
|
5641
6639
|
onDisconnected: (code, reason) => {
|
|
5642
|
-
|
|
5643
|
-
|
|
6640
|
+
this.disconnectedAtMs = this.now();
|
|
6641
|
+
events.onDisconnected(code, reason, { generation: this.generation });
|
|
6642
|
+
const relayRequestedReconnect = code === 1e3 && _RunnerConnection.RELAY_RECONNECT_CLOSE_REASONS.has(reason);
|
|
6643
|
+
if (this.opts.isRunning() && (code !== 1e3 || relayRequestedReconnect) && !this.reconnecting) {
|
|
5644
6644
|
this.reconnectPromise = this.connectWithRetry(true).catch((err) => {
|
|
5645
6645
|
events.onError?.(`Reconnection failed: ${err.message}`);
|
|
5646
6646
|
});
|
|
@@ -5979,16 +6979,111 @@ function createResourceUsageCollector(homeDir) {
|
|
|
5979
6979
|
};
|
|
5980
6980
|
}
|
|
5981
6981
|
|
|
6982
|
+
// src/lib/resource-usage-low-disk.ts
|
|
6983
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
6984
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
6985
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
6986
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
6987
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
6988
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
6989
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
6990
|
+
var BAND_RANK = {
|
|
6991
|
+
low: 1,
|
|
6992
|
+
very_low: 2,
|
|
6993
|
+
critical: 3
|
|
6994
|
+
};
|
|
6995
|
+
function bandFor(freeBytes) {
|
|
6996
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
6997
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
6998
|
+
return "low";
|
|
6999
|
+
}
|
|
7000
|
+
function formatBytes(bytes) {
|
|
7001
|
+
if (bytes === null) return "unavailable";
|
|
7002
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
7003
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
7004
|
+
const value = bytes / 1024 ** unitIndex;
|
|
7005
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
7006
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
7007
|
+
}
|
|
7008
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
7009
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
7010
|
+
}
|
|
7011
|
+
function evaluateLowDiskWarning(input) {
|
|
7012
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
7013
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
7014
|
+
return { warning: null, state };
|
|
7015
|
+
}
|
|
7016
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
7017
|
+
if (recovered) {
|
|
7018
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
7019
|
+
}
|
|
7020
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
7021
|
+
if (!low) return { warning: null, state };
|
|
7022
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
7023
|
+
const currentRank = BAND_RANK[currentBand];
|
|
7024
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
7025
|
+
const worse = currentRank > previousRank;
|
|
7026
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
7027
|
+
const nextState = {
|
|
7028
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
7029
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
7030
|
+
};
|
|
7031
|
+
return {
|
|
7032
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
7033
|
+
state: nextState
|
|
7034
|
+
};
|
|
7035
|
+
}
|
|
7036
|
+
|
|
7037
|
+
// src/lib/opencode/session-store-health.ts
|
|
7038
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
7039
|
+
function createSessionStoreHealth(options = {}) {
|
|
7040
|
+
const now = options.now ?? (() => Date.now());
|
|
7041
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
7042
|
+
let lastUnwritableAt = null;
|
|
7043
|
+
let lastWritableAt = null;
|
|
7044
|
+
const causes = /* @__PURE__ */ new Map();
|
|
7045
|
+
return {
|
|
7046
|
+
observeUnwritable: () => {
|
|
7047
|
+
lastUnwritableAt = now();
|
|
7048
|
+
},
|
|
7049
|
+
observeWritable: () => {
|
|
7050
|
+
lastWritableAt = now();
|
|
7051
|
+
},
|
|
7052
|
+
isUnwritable: () => {
|
|
7053
|
+
if (lastUnwritableAt === null) return false;
|
|
7054
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
7055
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
7056
|
+
},
|
|
7057
|
+
observeCause: (sessionId, cause) => {
|
|
7058
|
+
if (sessionId === null) return;
|
|
7059
|
+
causes.delete(sessionId);
|
|
7060
|
+
causes.set(sessionId, { cause, observedAt: now() });
|
|
7061
|
+
while (causes.size > 256) {
|
|
7062
|
+
const oldest = causes.keys().next().value;
|
|
7063
|
+
if (typeof oldest !== "string") break;
|
|
7064
|
+
causes.delete(oldest);
|
|
7065
|
+
}
|
|
7066
|
+
},
|
|
7067
|
+
readCause: (sessionId) => {
|
|
7068
|
+
const observed = causes.get(sessionId);
|
|
7069
|
+
if (!observed) return null;
|
|
7070
|
+
causes.delete(sessionId);
|
|
7071
|
+
if (now() - observed.observedAt >= stalenessMs) return null;
|
|
7072
|
+
return observed.cause;
|
|
7073
|
+
}
|
|
7074
|
+
};
|
|
7075
|
+
}
|
|
7076
|
+
|
|
5982
7077
|
// src/lib/channels/driver.ts
|
|
5983
|
-
import { homedir as
|
|
7078
|
+
import { homedir as homedir6 } from "node:os";
|
|
5984
7079
|
|
|
5985
7080
|
// src/lib/runner-file-sync.ts
|
|
5986
|
-
import { join as
|
|
7081
|
+
import { join as join10 } from "node:path";
|
|
5987
7082
|
|
|
5988
7083
|
// src/lib/file-push.ts
|
|
5989
7084
|
import { randomUUID } from "node:crypto";
|
|
5990
7085
|
import { chmod, mkdir, open as open3, realpath, rename, unlink } from "node:fs/promises";
|
|
5991
|
-
import { basename, dirname as
|
|
7086
|
+
import { basename as basename2, dirname as dirname6, isAbsolute as isAbsolute2, join as join9, relative as relative2, resolve as resolve3, sep as sep2 } from "node:path";
|
|
5992
7087
|
var FILE_MODE = 384;
|
|
5993
7088
|
var DIRECTORY_MODE = 448;
|
|
5994
7089
|
async function writePushedFile(request2) {
|
|
@@ -6019,9 +7114,9 @@ async function writePushedFile(request2) {
|
|
|
6019
7114
|
}
|
|
6020
7115
|
try {
|
|
6021
7116
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
6022
|
-
|
|
7117
|
+
dirname6(candidate)
|
|
6023
7118
|
);
|
|
6024
|
-
const realTarget =
|
|
7119
|
+
const realTarget = join9(existingAncestor, ...missingSegments, basename2(candidate));
|
|
6025
7120
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
6026
7121
|
if (allowedDirectory === null) {
|
|
6027
7122
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
@@ -6031,8 +7126,8 @@ async function writePushedFile(request2) {
|
|
|
6031
7126
|
}
|
|
6032
7127
|
if (missingSegments.length > 0) {
|
|
6033
7128
|
await createMissingDirectories(existingAncestor, missingSegments);
|
|
6034
|
-
const realParent = await realpath(
|
|
6035
|
-
if (realParent !==
|
|
7129
|
+
const realParent = await realpath(dirname6(realTarget));
|
|
7130
|
+
if (realParent !== dirname6(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
6036
7131
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
6037
7132
|
path: realTarget,
|
|
6038
7133
|
bytes,
|
|
@@ -6057,15 +7152,15 @@ function expandAndValidate(requestedPath, homeDir) {
|
|
|
6057
7152
|
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
6058
7153
|
return null;
|
|
6059
7154
|
}
|
|
6060
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
7155
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join9(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
6061
7156
|
if (expanded.split(/[/\\]/).includes("..")) {
|
|
6062
7157
|
return null;
|
|
6063
7158
|
}
|
|
6064
|
-
if (!
|
|
7159
|
+
if (!isAbsolute2(expanded)) {
|
|
6065
7160
|
return null;
|
|
6066
7161
|
}
|
|
6067
|
-
const candidate =
|
|
6068
|
-
const name =
|
|
7162
|
+
const candidate = resolve3(expanded);
|
|
7163
|
+
const name = basename2(candidate);
|
|
6069
7164
|
return name === "" || name === "." || name === ".." ? null : candidate;
|
|
6070
7165
|
}
|
|
6071
7166
|
async function resolveNearestExistingAncestor(directory) {
|
|
@@ -6075,18 +7170,18 @@ async function resolveNearestExistingAncestor(directory) {
|
|
|
6075
7170
|
try {
|
|
6076
7171
|
return { existingAncestor: await realpath(current), missingSegments };
|
|
6077
7172
|
} catch (err) {
|
|
6078
|
-
const parent =
|
|
7173
|
+
const parent = dirname6(current);
|
|
6079
7174
|
if (err.code !== "ENOENT" || parent === current) {
|
|
6080
7175
|
throw err;
|
|
6081
7176
|
}
|
|
6082
|
-
missingSegments.unshift(
|
|
7177
|
+
missingSegments.unshift(basename2(current));
|
|
6083
7178
|
current = parent;
|
|
6084
7179
|
}
|
|
6085
7180
|
}
|
|
6086
7181
|
}
|
|
6087
7182
|
async function findContainingAllowedDirectory(allowedDirectories, realTarget) {
|
|
6088
7183
|
for (const directory of allowedDirectories) {
|
|
6089
|
-
if (!
|
|
7184
|
+
if (!isAbsolute2(directory)) {
|
|
6090
7185
|
log("warn", "file_push_allowed_directory_skipped", { directory, reason: "not_absolute" });
|
|
6091
7186
|
continue;
|
|
6092
7187
|
}
|
|
@@ -6124,19 +7219,19 @@ async function realpathCreatingIfMissing(directory) {
|
|
|
6124
7219
|
}
|
|
6125
7220
|
}
|
|
6126
7221
|
function contains(realDirectory, realTarget) {
|
|
6127
|
-
const rel =
|
|
6128
|
-
return rel !== "" && rel !== ".." && !rel.startsWith(`..${
|
|
7222
|
+
const rel = relative2(realDirectory, realTarget);
|
|
7223
|
+
return rel !== "" && rel !== ".." && !rel.startsWith(`..${sep2}`) && !isAbsolute2(rel);
|
|
6129
7224
|
}
|
|
6130
7225
|
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
6131
7226
|
let current = existingAncestor;
|
|
6132
7227
|
for (const segment of missingSegments) {
|
|
6133
|
-
current =
|
|
7228
|
+
current = join9(current, segment);
|
|
6134
7229
|
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
6135
7230
|
await chmod(current, DIRECTORY_MODE);
|
|
6136
7231
|
}
|
|
6137
7232
|
}
|
|
6138
7233
|
async function writeAtomically(realTarget, content) {
|
|
6139
|
-
const temporaryPath =
|
|
7234
|
+
const temporaryPath = join9(dirname6(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
6140
7235
|
let handle;
|
|
6141
7236
|
try {
|
|
6142
7237
|
handle = await open3(temporaryPath, "wx", FILE_MODE);
|
|
@@ -6266,12 +7361,12 @@ var NOT_APPLIED = {
|
|
|
6266
7361
|
opencodeAuthApplied: false
|
|
6267
7362
|
};
|
|
6268
7363
|
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
6269
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
6270
|
-
return expanded ===
|
|
7364
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join10(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
7365
|
+
return expanded === join10(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
6271
7366
|
}
|
|
6272
7367
|
function isOpenCodeAuthPath(requestedPath, homeDir) {
|
|
6273
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
6274
|
-
return expanded ===
|
|
7368
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join10(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
7369
|
+
return expanded === join10(homeDir, ...OPENCODE_AUTH_SEGMENTS);
|
|
6275
7370
|
}
|
|
6276
7371
|
async function applyOne(options, file) {
|
|
6277
7372
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
@@ -6415,6 +7510,11 @@ function messageIdOf(m) {
|
|
|
6415
7510
|
const infoId = m.info?.id;
|
|
6416
7511
|
return typeof infoId === "string" ? infoId : void 0;
|
|
6417
7512
|
}
|
|
7513
|
+
function composeAbandonmentNote(finish, replyShape, outcome) {
|
|
7514
|
+
const base = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") ` + (outcome === "without-answer" ? "without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs." : "after writing only part of an answer, and confirmed it is no longer working on this conversation. Evident delivered what was written and is reporting the run as finished.");
|
|
7515
|
+
if (replyShape === null) return base;
|
|
7516
|
+
return `${base} ${replyShape.lastToolName === null ? "No tool call was recorded" : `The last tool call was "${replyShape.lastToolName}" with status "${replyShape.lastToolStatus ?? "(unknown)"}"`}; the reply's final part types were [${replyShape.partTypes.join(", ")}]${replyShape.droppedPartCount > 0 ? ` (${replyShape.droppedPartCount} earlier part types omitted)` : ""}.`;
|
|
7517
|
+
}
|
|
6418
7518
|
function cleanImageMime(contentType) {
|
|
6419
7519
|
if (!contentType) return null;
|
|
6420
7520
|
const media = contentType.split(";")[0].trim().toLowerCase();
|
|
@@ -6435,8 +7535,13 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
|
|
|
6435
7535
|
var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
|
|
6436
7536
|
var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
6437
7537
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
7538
|
+
var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
|
|
7539
|
+
var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
7540
|
+
var SESSION_ERROR_RETAINED_DETAIL = "Runner-observed OpenCode session error: ";
|
|
7541
|
+
var SESSION_ERROR_OBSERVED_CAUSE = "Runner-observed OpenCode session-store cause: ";
|
|
6438
7542
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6439
7543
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
7544
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6440
7545
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6441
7546
|
var HEARTBEAT_MS = 6e4;
|
|
6442
7547
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6498,6 +7603,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6498
7603
|
maxActiveSessions;
|
|
6499
7604
|
watcherStallMs;
|
|
6500
7605
|
wedgeWarningIntervalMs;
|
|
7606
|
+
sessionStoreHealth;
|
|
6501
7607
|
openCodeClient;
|
|
6502
7608
|
/** Cache of conversationId → opencode sessionId. */
|
|
6503
7609
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6590,6 +7696,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6590
7696
|
* with a short TTL so an unmatched session cannot retain an event indefinitely.
|
|
6591
7697
|
*/
|
|
6592
7698
|
bufferedSessionErrors = /* @__PURE__ */ new Map();
|
|
7699
|
+
retainedSessionErrors = /* @__PURE__ */ new Map();
|
|
6593
7700
|
/**
|
|
6594
7701
|
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
6595
7702
|
* dispatched and are still in-flight. A message in this set is never
|
|
@@ -6622,10 +7729,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6622
7729
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
6623
7730
|
*/
|
|
6624
7731
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7732
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7733
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
6625
7734
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
6626
7735
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6627
7736
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6628
7737
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7738
|
+
/**
|
|
7739
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7740
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7741
|
+
* identical-text turn, so it never produces a correlation.
|
|
7742
|
+
*/
|
|
7743
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6629
7744
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6630
7745
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6631
7746
|
/**
|
|
@@ -6711,21 +7826,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6711
7826
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6712
7827
|
*/
|
|
6713
7828
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6714
|
-
/**
|
|
6715
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6716
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6717
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6718
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6719
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6720
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6721
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6722
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6723
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6724
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6725
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6726
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6727
|
-
*/
|
|
6728
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6729
7829
|
/**
|
|
6730
7830
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6731
7831
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6743,8 +7843,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6743
7843
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6744
7844
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6745
7845
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6746
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6747
|
-
*
|
|
7846
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7847
|
+
* the terminal-report fence and is never sent again).
|
|
6748
7848
|
*/
|
|
6749
7849
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6750
7850
|
/**
|
|
@@ -6863,10 +7963,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6863
7963
|
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
6864
7964
|
this.now = config.now ?? (() => Date.now());
|
|
6865
7965
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
6866
|
-
this.homeDir = config.homeDir ??
|
|
7966
|
+
this.homeDir = config.homeDir ?? homedir6();
|
|
6867
7967
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
6868
7968
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
6869
7969
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7970
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
6870
7971
|
}
|
|
6871
7972
|
get isV2() {
|
|
6872
7973
|
return this.openCodeClient.version === "v2";
|
|
@@ -6882,7 +7983,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6882
7983
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
6883
7984
|
}
|
|
6884
7985
|
async listSessions() {
|
|
6885
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
7986
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
7987
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
7988
|
+
});
|
|
6886
7989
|
}
|
|
6887
7990
|
async sessionExists(sessionId) {
|
|
6888
7991
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7059,6 +8162,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7059
8162
|
}
|
|
7060
8163
|
return ids;
|
|
7061
8164
|
}
|
|
8165
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
8166
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
8167
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
8168
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
8169
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
8170
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
8171
|
+
}
|
|
8172
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
8173
|
+
}
|
|
8174
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
8175
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
8176
|
+
this.awaitingCorrelation.delete(messageId);
|
|
8177
|
+
this.awaitingReadopt.delete(messageId);
|
|
8178
|
+
this.sessions.delete(conversationId);
|
|
8179
|
+
this.supersede(conversationId, sessionId);
|
|
8180
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
8181
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
8182
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
8183
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
8184
|
+
this.log({
|
|
8185
|
+
level: "error",
|
|
8186
|
+
message: errorMessage3,
|
|
8187
|
+
conversation_id: conversationId,
|
|
8188
|
+
message_id: messageId
|
|
8189
|
+
});
|
|
8190
|
+
const reportError = await this.reportUntrackableAck(
|
|
8191
|
+
conversationId,
|
|
8192
|
+
messageId,
|
|
8193
|
+
null,
|
|
8194
|
+
errorMessage3
|
|
8195
|
+
);
|
|
8196
|
+
if (reportError !== null) {
|
|
8197
|
+
this.log({
|
|
8198
|
+
level: "warn",
|
|
8199
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
8200
|
+
conversation_id: conversationId,
|
|
8201
|
+
message_id: messageId
|
|
8202
|
+
});
|
|
8203
|
+
}
|
|
8204
|
+
if (source === "readopt") {
|
|
8205
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
8206
|
+
}
|
|
8207
|
+
}
|
|
7062
8208
|
/**
|
|
7063
8209
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7064
8210
|
*
|
|
@@ -7238,6 +8384,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7238
8384
|
}
|
|
7239
8385
|
break;
|
|
7240
8386
|
}
|
|
8387
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
8388
|
+
if (correlation) {
|
|
8389
|
+
correlation.attempts += 1;
|
|
8390
|
+
let correlatedId = null;
|
|
8391
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
8392
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
8393
|
+
conv,
|
|
8394
|
+
message,
|
|
8395
|
+
correlation.sessionId
|
|
8396
|
+
);
|
|
8397
|
+
if (polled.ok)
|
|
8398
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
8399
|
+
} else if (correlation.knownUserIds === null) {
|
|
8400
|
+
this.log({
|
|
8401
|
+
level: "debug",
|
|
8402
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
8403
|
+
conversation_id: conv.id,
|
|
8404
|
+
message_id: message.id
|
|
8405
|
+
});
|
|
8406
|
+
}
|
|
8407
|
+
if (correlatedId !== null) {
|
|
8408
|
+
this.awaitingCorrelation.delete(message.id);
|
|
8409
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
8410
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
8411
|
+
this.dispatched.add(message.id);
|
|
8412
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
8413
|
+
dispatched += 1;
|
|
8414
|
+
this.log({
|
|
8415
|
+
level: "info",
|
|
8416
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
8417
|
+
conversation_id: conv.id,
|
|
8418
|
+
message_id: message.id
|
|
8419
|
+
});
|
|
8420
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
8421
|
+
opencode_message_id: correlatedId
|
|
8422
|
+
});
|
|
8423
|
+
continue;
|
|
8424
|
+
}
|
|
8425
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
8426
|
+
await this.handoffUntrackableCorrelation(
|
|
8427
|
+
conv.id,
|
|
8428
|
+
message.id,
|
|
8429
|
+
correlation.sessionId,
|
|
8430
|
+
correlation.attempts,
|
|
8431
|
+
"fresh"
|
|
8432
|
+
);
|
|
8433
|
+
break;
|
|
8434
|
+
}
|
|
8435
|
+
break;
|
|
8436
|
+
}
|
|
7241
8437
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7242
8438
|
if (effectiveOpencodeMessageId) {
|
|
7243
8439
|
const outcome = await this.resolveRedrive(
|
|
@@ -7259,6 +8455,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7259
8455
|
model: message.opencode_model ?? void 0
|
|
7260
8456
|
};
|
|
7261
8457
|
let opencodeMessageId;
|
|
8458
|
+
let preDispatchUserIds = null;
|
|
7262
8459
|
try {
|
|
7263
8460
|
this.log({
|
|
7264
8461
|
level: "info",
|
|
@@ -7288,7 +8485,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7288
8485
|
message.content,
|
|
7289
8486
|
options,
|
|
7290
8487
|
sendAttachments,
|
|
7291
|
-
this.openCodeClient
|
|
8488
|
+
this.openCodeClient,
|
|
8489
|
+
{
|
|
8490
|
+
onKnownUserIds: (ids) => {
|
|
8491
|
+
preDispatchUserIds = ids;
|
|
8492
|
+
},
|
|
8493
|
+
onKnownUserIdsError: (error2) => {
|
|
8494
|
+
this.log({
|
|
8495
|
+
level: "warn",
|
|
8496
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8497
|
+
conversation_id: conv.id,
|
|
8498
|
+
message_id: message.id
|
|
8499
|
+
});
|
|
8500
|
+
}
|
|
8501
|
+
}
|
|
7292
8502
|
)
|
|
7293
8503
|
);
|
|
7294
8504
|
} catch (err) {
|
|
@@ -7296,6 +8506,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7296
8506
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7297
8507
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7298
8508
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
8509
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7299
8510
|
this.log({
|
|
7300
8511
|
level: "error",
|
|
7301
8512
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7371,39 +8582,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7371
8582
|
if (this.isV2) {
|
|
7372
8583
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7373
8584
|
}
|
|
7374
|
-
|
|
7375
|
-
|
|
7376
|
-
|
|
7377
|
-
|
|
7378
|
-
|
|
7379
|
-
|
|
7380
|
-
message_id: message.id
|
|
7381
|
-
});
|
|
7382
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7383
|
-
continue;
|
|
7384
|
-
}
|
|
7385
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7386
|
-
this.sessions.delete(conv.id);
|
|
7387
|
-
this.supersede(conv.id, sessionId);
|
|
7388
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
8585
|
+
this.awaitingCorrelation.set(message.id, {
|
|
8586
|
+
sessionId,
|
|
8587
|
+
content: message.content,
|
|
8588
|
+
attempts: 0,
|
|
8589
|
+
knownUserIds: preDispatchUserIds
|
|
8590
|
+
});
|
|
7389
8591
|
this.log({
|
|
7390
|
-
level: "
|
|
7391
|
-
message:
|
|
8592
|
+
level: "warn",
|
|
8593
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7392
8594
|
conversation_id: conv.id,
|
|
7393
8595
|
message_id: message.id
|
|
7394
8596
|
});
|
|
7395
|
-
|
|
7396
|
-
this.log({
|
|
7397
|
-
level: "warn",
|
|
7398
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7399
|
-
conversation_id: conv.id,
|
|
7400
|
-
message_id: message.id
|
|
7401
|
-
});
|
|
7402
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7403
|
-
});
|
|
8597
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7404
8598
|
break;
|
|
7405
8599
|
}
|
|
7406
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7407
8600
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7408
8601
|
this.releasedOpencodeIds.delete(message.id);
|
|
7409
8602
|
this.dispatched.add(message.id);
|
|
@@ -8008,23 +9201,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8008
9201
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8009
9202
|
return 1;
|
|
8010
9203
|
}
|
|
8011
|
-
/**
|
|
8012
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8013
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8014
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8015
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8016
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8017
|
-
* count, since a new session is a genuinely different attempt.
|
|
8018
|
-
*/
|
|
8019
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8020
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8021
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8022
|
-
existing.count += 1;
|
|
8023
|
-
return existing.count;
|
|
8024
|
-
}
|
|
8025
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8026
|
-
return 1;
|
|
8027
|
-
}
|
|
8028
9204
|
/** Record one queued give-up toward the never-started bound. */
|
|
8029
9205
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8030
9206
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8336,8 +9512,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8336
9512
|
b2LastDescendantCheckMs: 0,
|
|
8337
9513
|
b2RootOngoingHeldLogged: false,
|
|
8338
9514
|
b2AbandonedSignalled: false,
|
|
9515
|
+
b2AbandonedWithoutAnswer: false,
|
|
8339
9516
|
ambiguousPinnedSinceMs: 0,
|
|
8340
9517
|
ambiguousResolved: false,
|
|
9518
|
+
sessionStoreFailurePending: false,
|
|
8341
9519
|
neverStartedResolved: false
|
|
8342
9520
|
});
|
|
8343
9521
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8359,6 +9537,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8359
9537
|
this.bufferedSessionErrors.delete(oldest);
|
|
8360
9538
|
}
|
|
8361
9539
|
}
|
|
9540
|
+
retainSessionError(event) {
|
|
9541
|
+
this.retainedSessionErrors.delete(event.sessionId);
|
|
9542
|
+
this.retainedSessionErrors.set(event.sessionId, {
|
|
9543
|
+
reason: event.reason,
|
|
9544
|
+
observedAt: this.now()
|
|
9545
|
+
});
|
|
9546
|
+
while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
|
|
9547
|
+
const oldest = this.retainedSessionErrors.keys().next().value;
|
|
9548
|
+
if (typeof oldest !== "string") break;
|
|
9549
|
+
this.retainedSessionErrors.delete(oldest);
|
|
9550
|
+
}
|
|
9551
|
+
}
|
|
9552
|
+
readRetainedSessionError(sessionId) {
|
|
9553
|
+
const retained = this.retainedSessionErrors.get(sessionId);
|
|
9554
|
+
if (!retained) return null;
|
|
9555
|
+
if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
|
|
9556
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9557
|
+
return null;
|
|
9558
|
+
}
|
|
9559
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9560
|
+
return retained.reason;
|
|
9561
|
+
}
|
|
8362
9562
|
/**
|
|
8363
9563
|
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
8364
9564
|
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
@@ -8454,8 +9654,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8454
9654
|
b2LastDescendantCheckMs: 0,
|
|
8455
9655
|
b2RootOngoingHeldLogged: false,
|
|
8456
9656
|
b2AbandonedSignalled: false,
|
|
9657
|
+
b2AbandonedWithoutAnswer: false,
|
|
8457
9658
|
ambiguousPinnedSinceMs: 0,
|
|
8458
9659
|
ambiguousResolved: false,
|
|
9660
|
+
sessionStoreFailurePending: false,
|
|
8459
9661
|
neverStartedResolved: false
|
|
8460
9662
|
});
|
|
8461
9663
|
}
|
|
@@ -8683,9 +9885,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8683
9885
|
return;
|
|
8684
9886
|
}
|
|
8685
9887
|
if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
|
|
9888
|
+
this.retainSessionError(event);
|
|
8686
9889
|
this.log({
|
|
8687
|
-
level: "
|
|
8688
|
-
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
|
|
9890
|
+
level: "warn",
|
|
9891
|
+
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
|
|
8689
9892
|
conversation_id: watcher.conv.id
|
|
8690
9893
|
});
|
|
8691
9894
|
return;
|
|
@@ -8715,9 +9918,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8715
9918
|
const messages = await this.getSessionMessages(event.sessionId);
|
|
8716
9919
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
8717
9920
|
if (state !== "queued") {
|
|
9921
|
+
this.retainSessionError(event);
|
|
8718
9922
|
this.log({
|
|
8719
|
-
level: "
|
|
8720
|
-
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state};
|
|
9923
|
+
level: "warn",
|
|
9924
|
+
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
|
|
8721
9925
|
conversation_id: watcher.conv.id,
|
|
8722
9926
|
message_id: inFlight.evidentMessageId
|
|
8723
9927
|
});
|
|
@@ -8869,6 +10073,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8869
10073
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8870
10074
|
this.neverStartedAttempts.delete(id);
|
|
8871
10075
|
}
|
|
10076
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
10077
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10078
|
+
return;
|
|
10079
|
+
}
|
|
8872
10080
|
if (inFlight.neverStartedResolved) {
|
|
8873
10081
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8874
10082
|
inFlight.neverStartedResolved = false;
|
|
@@ -8942,7 +10150,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8942
10150
|
if (state === "failed") {
|
|
8943
10151
|
this.anchorDeliveryDeadline(inFlight);
|
|
8944
10152
|
if (!inFlight.done) {
|
|
8945
|
-
const
|
|
10153
|
+
const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
|
|
10154
|
+
const retainedReason = this.readRetainedSessionError(sessionId);
|
|
10155
|
+
const retainedError = retainedReason === null || messageErrorText !== null && messageErrorText.includes(retainedReason) ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
|
|
10156
|
+
const observedCause = this.sessionStoreHealth?.readCause(sessionId);
|
|
10157
|
+
const causeError = observedCause === null || observedCause === void 0 ? null : `${SESSION_ERROR_OBSERVED_CAUSE}${observedCause}`;
|
|
10158
|
+
const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
|
|
10159
|
+
const runnerDetails = [retainedError, causeError, storeError].filter(
|
|
10160
|
+
(detail) => detail !== null
|
|
10161
|
+
);
|
|
10162
|
+
const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
|
|
8946
10163
|
this.log({
|
|
8947
10164
|
level: "error",
|
|
8948
10165
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -9028,10 +10245,33 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9028
10245
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9029
10246
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9030
10247
|
inFlight.b2AbandonedSignalled = false;
|
|
10248
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9031
10249
|
}
|
|
9032
10250
|
} else {
|
|
9033
10251
|
if (inFlight.b2AbandonedSignalled) {
|
|
9034
|
-
|
|
10252
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
10253
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
10254
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
10255
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
10256
|
+
sessionId,
|
|
10257
|
+
watcher,
|
|
10258
|
+
inFlight,
|
|
10259
|
+
messages,
|
|
10260
|
+
finish
|
|
10261
|
+
);
|
|
10262
|
+
} else {
|
|
10263
|
+
await this.settleMessageDone(
|
|
10264
|
+
sessionId,
|
|
10265
|
+
watcher,
|
|
10266
|
+
inFlight,
|
|
10267
|
+
messages,
|
|
10268
|
+
composeAbandonmentNote(
|
|
10269
|
+
finish,
|
|
10270
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10271
|
+
"with-answer"
|
|
10272
|
+
)
|
|
10273
|
+
);
|
|
10274
|
+
}
|
|
9035
10275
|
return;
|
|
9036
10276
|
}
|
|
9037
10277
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9048,22 +10288,62 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9048
10288
|
descendantOngoing,
|
|
9049
10289
|
rootOngoing
|
|
9050
10290
|
})) {
|
|
10291
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
10292
|
+
inFlight.sessionStoreFailurePending = true;
|
|
10293
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
10294
|
+
watched_for_ms: pinnedForMs,
|
|
10295
|
+
run_state: "failed",
|
|
10296
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10297
|
+
});
|
|
10298
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10299
|
+
return;
|
|
10300
|
+
}
|
|
9051
10301
|
inFlight.b2AbandonedSignalled = true;
|
|
10302
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
10303
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
10304
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
10305
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
10306
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
10307
|
+
watched_for_ms: pinnedForMs,
|
|
10308
|
+
run_state: "failed",
|
|
10309
|
+
finish,
|
|
10310
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
10311
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
10312
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10313
|
+
});
|
|
10314
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
10315
|
+
sessionId,
|
|
10316
|
+
watcher,
|
|
10317
|
+
inFlight,
|
|
10318
|
+
messages,
|
|
10319
|
+
finish
|
|
10320
|
+
);
|
|
10321
|
+
return;
|
|
10322
|
+
}
|
|
9052
10323
|
this.log({
|
|
9053
10324
|
level: "warn",
|
|
9054
10325
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9055
10326
|
conversation_id: conv.id,
|
|
9056
10327
|
message_id: id
|
|
9057
10328
|
});
|
|
9058
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9059
10329
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9060
10330
|
watched_for_ms: pinnedForMs,
|
|
9061
|
-
finish
|
|
10331
|
+
finish,
|
|
9062
10332
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9063
10333
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9064
10334
|
opencode_message_id: inFlight.opencodeMessageId
|
|
9065
10335
|
});
|
|
9066
|
-
await this.settleMessageDone(
|
|
10336
|
+
await this.settleMessageDone(
|
|
10337
|
+
sessionId,
|
|
10338
|
+
watcher,
|
|
10339
|
+
inFlight,
|
|
10340
|
+
messages,
|
|
10341
|
+
composeAbandonmentNote(
|
|
10342
|
+
finish,
|
|
10343
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10344
|
+
"with-answer"
|
|
10345
|
+
)
|
|
10346
|
+
);
|
|
9067
10347
|
return;
|
|
9068
10348
|
}
|
|
9069
10349
|
if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
|
|
@@ -9106,6 +10386,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9106
10386
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9107
10387
|
sessionOngoing: ongoing
|
|
9108
10388
|
})) {
|
|
10389
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
10390
|
+
inFlight.sessionStoreFailurePending = true;
|
|
10391
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
10392
|
+
watched_for_ms: pinnedForMs,
|
|
10393
|
+
run_state: "failed",
|
|
10394
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10395
|
+
});
|
|
10396
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10397
|
+
return;
|
|
10398
|
+
}
|
|
9109
10399
|
inFlight.ambiguousResolved = true;
|
|
9110
10400
|
this.log({
|
|
9111
10401
|
level: "warn",
|
|
@@ -9253,15 +10543,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9253
10543
|
}
|
|
9254
10544
|
}
|
|
9255
10545
|
/**
|
|
9256
|
-
* Settle a message whose run-state has resolved `'done'
|
|
9257
|
-
*
|
|
9258
|
-
*
|
|
9259
|
-
* b2-abandonment resolution) can reach the exact same completion behavior
|
|
9260
|
-
* (delivery-deadline anchoring, title resolution, usage extraction, and
|
|
9261
|
-
* `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
|
|
9262
|
-
* and risking the two copies silently drifting apart.
|
|
10546
|
+
* Settle a message whose run-state has resolved `'done'`. Normal completion and
|
|
10547
|
+
* confirmed b2 abandonment share the same delivery, usage, and terminal handling.
|
|
10548
|
+
* Normal completions omit the optional terminal diagnostic.
|
|
9263
10549
|
*/
|
|
9264
|
-
async settleMessageDone(sessionId, watcher, inFlight, messages) {
|
|
10550
|
+
async settleMessageDone(sessionId, watcher, inFlight, messages, note) {
|
|
9265
10551
|
const conv = watcher.conv;
|
|
9266
10552
|
this.anchorDeliveryDeadline(inFlight);
|
|
9267
10553
|
if (!inFlight.done) {
|
|
@@ -9288,7 +10574,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9288
10574
|
title,
|
|
9289
10575
|
usage,
|
|
9290
10576
|
usageAgentName,
|
|
9291
|
-
subagentInvocations
|
|
10577
|
+
subagentInvocations,
|
|
10578
|
+
note
|
|
9292
10579
|
);
|
|
9293
10580
|
} catch (err) {
|
|
9294
10581
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -9330,14 +10617,147 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9330
10617
|
}
|
|
9331
10618
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9332
10619
|
}
|
|
10620
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
10621
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
10622
|
+
const conv = watcher.conv;
|
|
10623
|
+
const error2 = composeAbandonmentNote(
|
|
10624
|
+
finish,
|
|
10625
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10626
|
+
"without-answer"
|
|
10627
|
+
);
|
|
10628
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10629
|
+
this.log({
|
|
10630
|
+
level: "warn",
|
|
10631
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
10632
|
+
conversation_id: conv.id,
|
|
10633
|
+
message_id: inFlight.evidentMessageId
|
|
10634
|
+
});
|
|
10635
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
10636
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
10637
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
10638
|
+
messages,
|
|
10639
|
+
inFlight.opencodeMessageId,
|
|
10640
|
+
inFlight.evidentMessageId
|
|
10641
|
+
);
|
|
10642
|
+
try {
|
|
10643
|
+
await this.markFailed(
|
|
10644
|
+
conv.id,
|
|
10645
|
+
inFlight.evidentMessageId,
|
|
10646
|
+
sessionId,
|
|
10647
|
+
error2,
|
|
10648
|
+
usage,
|
|
10649
|
+
void 0,
|
|
10650
|
+
usageAgentName,
|
|
10651
|
+
subagentInvocations
|
|
10652
|
+
);
|
|
10653
|
+
} catch (err) {
|
|
10654
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10655
|
+
if (err instanceof ChannelTerminalError) {
|
|
10656
|
+
this.log({
|
|
10657
|
+
level: "warn",
|
|
10658
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
10659
|
+
conversation_id: conv.id,
|
|
10660
|
+
message_id: inFlight.evidentMessageId
|
|
10661
|
+
});
|
|
10662
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10663
|
+
return;
|
|
10664
|
+
}
|
|
10665
|
+
if (this.now() >= inFlight.deadline) {
|
|
10666
|
+
this.log({
|
|
10667
|
+
level: "warn",
|
|
10668
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
10669
|
+
conversation_id: conv.id,
|
|
10670
|
+
message_id: inFlight.evidentMessageId
|
|
10671
|
+
});
|
|
10672
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10673
|
+
return;
|
|
10674
|
+
}
|
|
10675
|
+
this.log({
|
|
10676
|
+
level: "warn",
|
|
10677
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10678
|
+
conversation_id: conv.id,
|
|
10679
|
+
message_id: inFlight.evidentMessageId
|
|
10680
|
+
});
|
|
10681
|
+
return;
|
|
10682
|
+
}
|
|
10683
|
+
inFlight.done = true;
|
|
10684
|
+
await this.reportSubagentAuthFailures(
|
|
10685
|
+
watcher.conv.id,
|
|
10686
|
+
inFlight.opencodeMessageId,
|
|
10687
|
+
inFlight.evidentMessageId,
|
|
10688
|
+
messages
|
|
10689
|
+
);
|
|
10690
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10691
|
+
}
|
|
10692
|
+
/**
|
|
10693
|
+
* Report a queued message as failed after the never-started bound. The accepted
|
|
10694
|
+
* OpenCode copy is not cancelled, so the local fence remains on unsuccessful
|
|
10695
|
+
* terminal-report attempts to prevent a blind re-prompt.
|
|
10696
|
+
*/
|
|
10697
|
+
async settleMessageNeverStarted(sessionId, watcher, inFlight, attempts) {
|
|
10698
|
+
const conv = watcher.conv;
|
|
10699
|
+
const error2 = `OpenCode accepted this message but never started running it; after ${attempts} attempts it was still queued. Evident is reporting the run as failed. The copy OpenCode accepted has not been cancelled and may still run if OpenCode recovers.`;
|
|
10700
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10701
|
+
try {
|
|
10702
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10703
|
+
} catch (err) {
|
|
10704
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10705
|
+
if (err instanceof ChannelTerminalError) {
|
|
10706
|
+
this.log({
|
|
10707
|
+
level: "warn",
|
|
10708
|
+
message: `Failed to report never-started message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10709
|
+
conversation_id: conv.id,
|
|
10710
|
+
message_id: inFlight.evidentMessageId
|
|
10711
|
+
});
|
|
10712
|
+
this.recordReleasedOpencodeId(
|
|
10713
|
+
inFlight.evidentMessageId,
|
|
10714
|
+
sessionId,
|
|
10715
|
+
inFlight.opencodeMessageId
|
|
10716
|
+
);
|
|
10717
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10718
|
+
return;
|
|
10719
|
+
}
|
|
10720
|
+
if (this.now() >= inFlight.deadline) {
|
|
10721
|
+
this.log({
|
|
10722
|
+
level: "warn",
|
|
10723
|
+
message: `Failed to report never-started message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10724
|
+
conversation_id: conv.id,
|
|
10725
|
+
message_id: inFlight.evidentMessageId
|
|
10726
|
+
});
|
|
10727
|
+
this.recordReleasedOpencodeId(
|
|
10728
|
+
inFlight.evidentMessageId,
|
|
10729
|
+
sessionId,
|
|
10730
|
+
inFlight.opencodeMessageId
|
|
10731
|
+
);
|
|
10732
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10733
|
+
return;
|
|
10734
|
+
}
|
|
10735
|
+
this.log({
|
|
10736
|
+
level: "warn",
|
|
10737
|
+
message: `Failed to report never-started message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10738
|
+
conversation_id: conv.id,
|
|
10739
|
+
message_id: inFlight.evidentMessageId
|
|
10740
|
+
});
|
|
10741
|
+
return;
|
|
10742
|
+
}
|
|
10743
|
+
inFlight.done = true;
|
|
10744
|
+
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
10745
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10746
|
+
}
|
|
9333
10747
|
/**
|
|
9334
|
-
* Report a
|
|
9335
|
-
*
|
|
9336
|
-
*
|
|
10748
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10749
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10750
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
9337
10751
|
*/
|
|
9338
|
-
async
|
|
10752
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
9339
10753
|
const conv = watcher.conv;
|
|
9340
|
-
const error2 =
|
|
10754
|
+
const error2 = SESSION_STORE_UNWRITABLE_ERROR;
|
|
10755
|
+
this.log({
|
|
10756
|
+
level: "error",
|
|
10757
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10758
|
+
conversation_id: conv.id,
|
|
10759
|
+
message_id: inFlight.evidentMessageId
|
|
10760
|
+
});
|
|
9341
10761
|
this.anchorDeliveryDeadline(inFlight);
|
|
9342
10762
|
try {
|
|
9343
10763
|
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
@@ -9346,7 +10766,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9346
10766
|
if (err instanceof ChannelTerminalError) {
|
|
9347
10767
|
this.log({
|
|
9348
10768
|
level: "warn",
|
|
9349
|
-
message: `Failed to report
|
|
10769
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
9350
10770
|
conversation_id: conv.id,
|
|
9351
10771
|
message_id: inFlight.evidentMessageId
|
|
9352
10772
|
});
|
|
@@ -9361,7 +10781,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9361
10781
|
if (this.now() >= inFlight.deadline) {
|
|
9362
10782
|
this.log({
|
|
9363
10783
|
level: "warn",
|
|
9364
|
-
message: `Failed to report
|
|
10784
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
9365
10785
|
conversation_id: conv.id,
|
|
9366
10786
|
message_id: inFlight.evidentMessageId
|
|
9367
10787
|
});
|
|
@@ -9375,14 +10795,13 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9375
10795
|
}
|
|
9376
10796
|
this.log({
|
|
9377
10797
|
level: "warn",
|
|
9378
|
-
message: `Failed to report
|
|
10798
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
9379
10799
|
conversation_id: conv.id,
|
|
9380
10800
|
message_id: inFlight.evidentMessageId
|
|
9381
10801
|
});
|
|
9382
10802
|
return;
|
|
9383
10803
|
}
|
|
9384
10804
|
inFlight.done = true;
|
|
9385
|
-
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9386
10805
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9387
10806
|
}
|
|
9388
10807
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
@@ -9423,11 +10842,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9423
10842
|
}
|
|
9424
10843
|
}
|
|
9425
10844
|
}
|
|
9426
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10845
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9427
10846
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10847
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10848
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10849
|
+
this.awaitingCorrelation.delete(id);
|
|
10850
|
+
this.log({
|
|
10851
|
+
level: "debug",
|
|
10852
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10853
|
+
message_id: id
|
|
10854
|
+
});
|
|
10855
|
+
}
|
|
10856
|
+
}
|
|
9428
10857
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9429
10858
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9430
10859
|
this.untrackableAck.delete(id);
|
|
10860
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9431
10861
|
this.untrackableAckSignalled.delete(id);
|
|
9432
10862
|
this.untrackableAckWarned.delete(id);
|
|
9433
10863
|
this.log({
|
|
@@ -9467,7 +10897,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9467
10897
|
bySession.set(row.opencode_session_id, list);
|
|
9468
10898
|
}
|
|
9469
10899
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9470
|
-
let messages;
|
|
10900
|
+
let messages = null;
|
|
9471
10901
|
try {
|
|
9472
10902
|
if (this.isV2) {
|
|
9473
10903
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9476,9 +10906,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9476
10906
|
level: "warn",
|
|
9477
10907
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9478
10908
|
});
|
|
9479
|
-
|
|
10909
|
+
} else {
|
|
10910
|
+
messages = snapshot;
|
|
9480
10911
|
}
|
|
9481
|
-
messages = snapshot;
|
|
9482
10912
|
} else {
|
|
9483
10913
|
const polled = await pollSessionMessagesForRedrive(
|
|
9484
10914
|
this.port,
|
|
@@ -9491,15 +10921,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9491
10921
|
level: "warn",
|
|
9492
10922
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9493
10923
|
});
|
|
9494
|
-
|
|
10924
|
+
} else {
|
|
10925
|
+
messages = polled.messages;
|
|
9495
10926
|
}
|
|
9496
|
-
messages = polled.messages;
|
|
9497
10927
|
}
|
|
9498
10928
|
} catch (err) {
|
|
9499
10929
|
this.log({
|
|
9500
10930
|
level: "warn",
|
|
9501
10931
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9502
10932
|
});
|
|
10933
|
+
}
|
|
10934
|
+
if (messages === null) {
|
|
10935
|
+
for (const row of sessionRows) {
|
|
10936
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10937
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10938
|
+
}
|
|
10939
|
+
}
|
|
9503
10940
|
continue;
|
|
9504
10941
|
}
|
|
9505
10942
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9509,6 +10946,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9509
10946
|
}
|
|
9510
10947
|
}
|
|
9511
10948
|
}
|
|
10949
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10950
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10951
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10952
|
+
if (!correlation) return false;
|
|
10953
|
+
correlation.attempts += 1;
|
|
10954
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10955
|
+
if (correlatedId !== null) {
|
|
10956
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10957
|
+
const conv = this.convForRow(sessionId, row);
|
|
10958
|
+
const message = this.queuedMessageForRow(row);
|
|
10959
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10960
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10961
|
+
this.dispatched.add(row.id);
|
|
10962
|
+
this.readopted.add(row.id);
|
|
10963
|
+
this.ensureWatcherRunning(sessionId);
|
|
10964
|
+
this.log({
|
|
10965
|
+
level: "info",
|
|
10966
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10967
|
+
conversation_id: row.conversation_id,
|
|
10968
|
+
message_id: row.id
|
|
10969
|
+
});
|
|
10970
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10971
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10972
|
+
opencode_message_id: correlatedId
|
|
10973
|
+
});
|
|
10974
|
+
return true;
|
|
10975
|
+
}
|
|
10976
|
+
if (correlation.knownUserIds === null) {
|
|
10977
|
+
this.log({
|
|
10978
|
+
level: "debug",
|
|
10979
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10980
|
+
conversation_id: row.conversation_id,
|
|
10981
|
+
message_id: row.id
|
|
10982
|
+
});
|
|
10983
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10984
|
+
this.log({
|
|
10985
|
+
level: "warn",
|
|
10986
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10987
|
+
conversation_id: row.conversation_id,
|
|
10988
|
+
message_id: row.id
|
|
10989
|
+
});
|
|
10990
|
+
}
|
|
10991
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10992
|
+
await this.handoffUntrackableCorrelation(
|
|
10993
|
+
row.conversation_id,
|
|
10994
|
+
row.id,
|
|
10995
|
+
correlation.sessionId,
|
|
10996
|
+
correlation.attempts,
|
|
10997
|
+
"readopt"
|
|
10998
|
+
);
|
|
10999
|
+
}
|
|
11000
|
+
return true;
|
|
11001
|
+
}
|
|
9512
11002
|
/**
|
|
9513
11003
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9514
11004
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9516,8 +11006,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9516
11006
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
9517
11007
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
9518
11008
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
9519
|
-
* before the restart)
|
|
9520
|
-
*
|
|
11009
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
11010
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
11011
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
11012
|
+
* `forceReadoptRun`):
|
|
9521
11013
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
9522
11014
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
9523
11015
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -9526,7 +11018,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9526
11018
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
9527
11019
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
9528
11020
|
* tracking the stored id so the reply correlates by it;
|
|
9529
|
-
* - `unknown`/null id →
|
|
11021
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
11022
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
9530
11023
|
*
|
|
9531
11024
|
* Only `ChannelAuthError` propagates.
|
|
9532
11025
|
*/
|
|
@@ -9540,6 +11033,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9540
11033
|
});
|
|
9541
11034
|
return;
|
|
9542
11035
|
}
|
|
11036
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9543
11037
|
const ocId = row.opencode_message_id;
|
|
9544
11038
|
const state = messageRunState(messages, ocId ?? "");
|
|
9545
11039
|
if (state === "done") {
|
|
@@ -9606,12 +11100,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9606
11100
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
9607
11101
|
this.dontRedispatch.delete(row.id);
|
|
9608
11102
|
this.untrackableAck.delete(row.id);
|
|
11103
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9609
11104
|
this.untrackableAckSignalled.delete(row.id);
|
|
9610
11105
|
this.untrackableAckWarned.delete(row.id);
|
|
9611
11106
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
9612
11107
|
return;
|
|
9613
11108
|
}
|
|
9614
|
-
if (this.
|
|
11109
|
+
if (this.untrackableAck.has(row.id)) {
|
|
11110
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
11111
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
11112
|
+
this.untrackableAckWarned.add(row.id);
|
|
11113
|
+
this.log({
|
|
11114
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
11115
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
11116
|
+
conversation_id: row.conversation_id,
|
|
11117
|
+
message_id: row.id
|
|
11118
|
+
});
|
|
11119
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
11120
|
+
const reportError = await this.reportUntrackableAck(
|
|
11121
|
+
row.conversation_id,
|
|
11122
|
+
row.id,
|
|
11123
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
11124
|
+
errorMessage3
|
|
11125
|
+
);
|
|
11126
|
+
if (reportError !== null) {
|
|
11127
|
+
this.log({
|
|
11128
|
+
level: "warn",
|
|
11129
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
11130
|
+
conversation_id: row.conversation_id,
|
|
11131
|
+
message_id: row.id
|
|
11132
|
+
});
|
|
11133
|
+
}
|
|
11134
|
+
} else {
|
|
11135
|
+
this.log({
|
|
11136
|
+
level: "debug",
|
|
11137
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
11138
|
+
conversation_id: row.conversation_id,
|
|
11139
|
+
message_id: row.id
|
|
11140
|
+
});
|
|
11141
|
+
}
|
|
11142
|
+
return;
|
|
11143
|
+
}
|
|
11144
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
9615
11145
|
this.log({
|
|
9616
11146
|
level: "debug",
|
|
9617
11147
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -9789,6 +11319,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9789
11319
|
}
|
|
9790
11320
|
this.dontRedispatch.delete(row.id);
|
|
9791
11321
|
this.untrackableAck.delete(row.id);
|
|
11322
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9792
11323
|
this.untrackableAckSignalled.delete(row.id);
|
|
9793
11324
|
this.untrackableAckWarned.delete(row.id);
|
|
9794
11325
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -9808,8 +11339,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9808
11339
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
9809
11340
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
9810
11341
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
9811
|
-
* re-entry)
|
|
9812
|
-
*
|
|
11342
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
11343
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
11344
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
9813
11345
|
*
|
|
9814
11346
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
9815
11347
|
* `processed_at` (Invariant 1).
|
|
@@ -9872,6 +11404,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9872
11404
|
);
|
|
9873
11405
|
}
|
|
9874
11406
|
let ocId;
|
|
11407
|
+
let preDispatchUserIds = null;
|
|
9875
11408
|
try {
|
|
9876
11409
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
9877
11410
|
sessionId,
|
|
@@ -9881,7 +11414,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9881
11414
|
row.content,
|
|
9882
11415
|
options,
|
|
9883
11416
|
sendAttachments,
|
|
9884
|
-
this.openCodeClient
|
|
11417
|
+
this.openCodeClient,
|
|
11418
|
+
{
|
|
11419
|
+
onKnownUserIds: (ids) => {
|
|
11420
|
+
preDispatchUserIds = ids;
|
|
11421
|
+
},
|
|
11422
|
+
onKnownUserIdsError: (error2) => {
|
|
11423
|
+
this.log({
|
|
11424
|
+
level: "warn",
|
|
11425
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
11426
|
+
conversation_id: row.conversation_id,
|
|
11427
|
+
message_id: row.id
|
|
11428
|
+
});
|
|
11429
|
+
}
|
|
11430
|
+
}
|
|
9885
11431
|
)
|
|
9886
11432
|
);
|
|
9887
11433
|
} catch (err) {
|
|
@@ -9914,6 +11460,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9914
11460
|
}
|
|
9915
11461
|
} else {
|
|
9916
11462
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
11463
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
9917
11464
|
}
|
|
9918
11465
|
this.log({
|
|
9919
11466
|
level: "error",
|
|
@@ -9962,39 +11509,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9962
11509
|
}
|
|
9963
11510
|
if (ocId === null) {
|
|
9964
11511
|
this.awaitingReadopt.delete(row.id);
|
|
9965
|
-
|
|
9966
|
-
|
|
9967
|
-
|
|
9968
|
-
|
|
9969
|
-
|
|
9970
|
-
|
|
9971
|
-
this.log({
|
|
9972
|
-
level: "error",
|
|
9973
|
-
message: errorMessage3,
|
|
9974
|
-
conversation_id: row.conversation_id,
|
|
9975
|
-
message_id: row.id
|
|
9976
|
-
});
|
|
9977
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
9978
|
-
this.log({
|
|
9979
|
-
level: "warn",
|
|
9980
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
9981
|
-
conversation_id: row.conversation_id,
|
|
9982
|
-
message_id: row.id
|
|
9983
|
-
});
|
|
9984
|
-
});
|
|
9985
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
9986
|
-
return;
|
|
9987
|
-
}
|
|
11512
|
+
this.awaitingCorrelation.set(row.id, {
|
|
11513
|
+
sessionId,
|
|
11514
|
+
content: row.content,
|
|
11515
|
+
attempts: 0,
|
|
11516
|
+
knownUserIds: preDispatchUserIds
|
|
11517
|
+
});
|
|
9988
11518
|
this.log({
|
|
9989
11519
|
level: "warn",
|
|
9990
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
11520
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
9991
11521
|
conversation_id: row.conversation_id,
|
|
9992
11522
|
message_id: row.id
|
|
9993
11523
|
});
|
|
9994
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
9995
11524
|
return;
|
|
9996
11525
|
}
|
|
9997
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
9998
11526
|
const processedAtMs = this.processedAtMs(row);
|
|
9999
11527
|
this.registerReadopted(
|
|
10000
11528
|
readoptConv,
|
|
@@ -10333,14 +11861,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10333
11861
|
const fetchChildWithoutBlocking = async (sessionId) => {
|
|
10334
11862
|
const pending = fetchChildMessages(sessionId);
|
|
10335
11863
|
let timer;
|
|
10336
|
-
const timeout = new Promise((
|
|
11864
|
+
const timeout = new Promise((resolve5) => {
|
|
10337
11865
|
timer = setTimeout(() => {
|
|
10338
11866
|
this.log({
|
|
10339
11867
|
level: "warn",
|
|
10340
11868
|
message: `Best-effort subagent session fetch for message ${messageId.slice(0, 8)} and child session ${sessionId.slice(0, 8)} was slow \u2014 omitting invocation telemetry without delaying completion`,
|
|
10341
11869
|
message_id: messageId
|
|
10342
11870
|
});
|
|
10343
|
-
|
|
11871
|
+
resolve5(null);
|
|
10344
11872
|
}, SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS);
|
|
10345
11873
|
});
|
|
10346
11874
|
try {
|
|
@@ -10900,7 +12428,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10900
12428
|
* watcher retries next tick within the
|
|
10901
12429
|
* deadline, Finding 4).
|
|
10902
12430
|
*/
|
|
10903
|
-
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations) {
|
|
12431
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations, error2) {
|
|
10904
12432
|
const res = await this.fetchImpl(
|
|
10905
12433
|
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
10906
12434
|
{
|
|
@@ -10918,7 +12446,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10918
12446
|
...title ? { title } : {},
|
|
10919
12447
|
...usage ? usage : {},
|
|
10920
12448
|
...usageAgentName ? { usage_agent_name: usageAgentName } : {},
|
|
10921
|
-
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {}
|
|
12449
|
+
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {},
|
|
12450
|
+
...error2 ? { error: error2 } : {}
|
|
10922
12451
|
})
|
|
10923
12452
|
}
|
|
10924
12453
|
);
|
|
@@ -11581,7 +13110,7 @@ function commandError2(result) {
|
|
|
11581
13110
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
11582
13111
|
}
|
|
11583
13112
|
var runCommand2 = (command, args, opts) => {
|
|
11584
|
-
return new Promise((
|
|
13113
|
+
return new Promise((resolve5) => {
|
|
11585
13114
|
let child;
|
|
11586
13115
|
let stdout = "";
|
|
11587
13116
|
let stderr = "";
|
|
@@ -11591,7 +13120,7 @@ var runCommand2 = (command, args, opts) => {
|
|
|
11591
13120
|
if (settled) return;
|
|
11592
13121
|
settled = true;
|
|
11593
13122
|
if (timer.handle) clearTimeout(timer.handle);
|
|
11594
|
-
|
|
13123
|
+
resolve5(result);
|
|
11595
13124
|
};
|
|
11596
13125
|
try {
|
|
11597
13126
|
child = spawn5(command, args, { env: opts.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
@@ -11646,7 +13175,9 @@ async function installRunnerSecret({
|
|
|
11646
13175
|
}) {
|
|
11647
13176
|
const arn = env.RUNNER_SECRET_ARN?.trim();
|
|
11648
13177
|
if (!arn) {
|
|
11649
|
-
|
|
13178
|
+
if (!env.GH_TOKEN) {
|
|
13179
|
+
log3("runner secret is not configured; continuing without GitHub and MCP credentials");
|
|
13180
|
+
}
|
|
11650
13181
|
return false;
|
|
11651
13182
|
}
|
|
11652
13183
|
const result = await (commandRunner ?? runCommand2)(
|
|
@@ -11845,7 +13376,7 @@ async function configureGitHubAccess({ env, log: log3 }) {
|
|
|
11845
13376
|
// src/lib/opencode/config-overlay.ts
|
|
11846
13377
|
import { execFileSync as execFileSync2 } from "node:child_process";
|
|
11847
13378
|
import { copyFileSync, existsSync as existsSync2, statSync as statSync6 } from "node:fs";
|
|
11848
|
-
import { isAbsolute as
|
|
13379
|
+
import { isAbsolute as isAbsolute3, join as join11, resolve as resolve4 } from "node:path";
|
|
11849
13380
|
function isFile(filePath) {
|
|
11850
13381
|
return existsSync2(filePath) && statSync6(filePath).isFile();
|
|
11851
13382
|
}
|
|
@@ -11858,8 +13389,8 @@ function applyRunnerOpenCodeConfig({
|
|
|
11858
13389
|
log3("runner OpenCode config is not configured; using the baked project config", "debug");
|
|
11859
13390
|
return;
|
|
11860
13391
|
}
|
|
11861
|
-
const source =
|
|
11862
|
-
const target = isFile(
|
|
13392
|
+
const source = isAbsolute3(overlayPath) ? overlayPath : resolve4(cwd, overlayPath);
|
|
13393
|
+
const target = isFile(join11(cwd, "opencode.jsonc")) ? "opencode.jsonc" : "opencode.json";
|
|
11863
13394
|
if (!isFile(source)) {
|
|
11864
13395
|
log3(
|
|
11865
13396
|
`RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)`,
|
|
@@ -11867,7 +13398,7 @@ function applyRunnerOpenCodeConfig({
|
|
|
11867
13398
|
);
|
|
11868
13399
|
return;
|
|
11869
13400
|
}
|
|
11870
|
-
copyFileSync(source,
|
|
13401
|
+
copyFileSync(source, join11(cwd, target));
|
|
11871
13402
|
try {
|
|
11872
13403
|
execFileSync2("git", ["-C", cwd, "update-index", "--skip-worktree", target], {
|
|
11873
13404
|
stdio: "ignore"
|
|
@@ -11876,7 +13407,7 @@ function applyRunnerOpenCodeConfig({
|
|
|
11876
13407
|
const detail = error2 instanceof Error ? error2.message : String(error2);
|
|
11877
13408
|
log3(`could not mark ${target} skip-worktree: ${detail}; it may show as a local change`, "warn");
|
|
11878
13409
|
}
|
|
11879
|
-
log3(`Applied runner OpenCode config ${source} to ${
|
|
13410
|
+
log3(`Applied runner OpenCode config ${source} to ${join11(cwd, target)}`);
|
|
11880
13411
|
}
|
|
11881
13412
|
|
|
11882
13413
|
// src/lib/credential-sync.ts
|
|
@@ -11894,14 +13425,14 @@ function errorMessage2(error2) {
|
|
|
11894
13425
|
return error2 instanceof Error ? error2.message : String(error2);
|
|
11895
13426
|
}
|
|
11896
13427
|
function waitForSettlement(promise, timeoutMs) {
|
|
11897
|
-
return new Promise((
|
|
13428
|
+
return new Promise((resolve5) => {
|
|
11898
13429
|
let settled = false;
|
|
11899
13430
|
const timer = setTimeout(() => finish(false), Math.max(0, timeoutMs));
|
|
11900
13431
|
const finish = (value) => {
|
|
11901
13432
|
if (settled) return;
|
|
11902
13433
|
settled = true;
|
|
11903
13434
|
clearTimeout(timer);
|
|
11904
|
-
|
|
13435
|
+
resolve5(value);
|
|
11905
13436
|
};
|
|
11906
13437
|
promise.then(
|
|
11907
13438
|
() => finish(true),
|
|
@@ -12116,6 +13647,7 @@ function createCredentialSync({
|
|
|
12116
13647
|
|
|
12117
13648
|
// src/commands/run.ts
|
|
12118
13649
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
13650
|
+
var MAX_DISCONNECT_REASON_LENGTH = 200;
|
|
12119
13651
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
12120
13652
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
12121
13653
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
@@ -12151,8 +13683,8 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
12151
13683
|
if (trimmed === "") {
|
|
12152
13684
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
12153
13685
|
}
|
|
12154
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
12155
|
-
if (!
|
|
13686
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join12(homeDir, trimmed.slice(2)) : trimmed;
|
|
13687
|
+
if (!isAbsolute4(expanded)) {
|
|
12156
13688
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
12157
13689
|
}
|
|
12158
13690
|
const normalized = resolvePath2(expanded);
|
|
@@ -12252,8 +13784,7 @@ function resolveMaxActiveSessions(options, env = process.env) {
|
|
|
12252
13784
|
function meetsThreshold(state, level) {
|
|
12253
13785
|
return LOG_LEVELS[level] >= LOG_LEVELS[state.logLevel];
|
|
12254
13786
|
}
|
|
12255
|
-
function
|
|
12256
|
-
if (!meetsThreshold(state, level)) return;
|
|
13787
|
+
function writeConsoleLog(state, message, level) {
|
|
12257
13788
|
if (state.json) {
|
|
12258
13789
|
console.log(
|
|
12259
13790
|
JSON.stringify({
|
|
@@ -12262,16 +13793,30 @@ function log2(state, message, level = "info") {
|
|
|
12262
13793
|
message
|
|
12263
13794
|
})
|
|
12264
13795
|
);
|
|
12265
|
-
|
|
12266
|
-
const prefix = level === "error" ? chalk7.red("\u2717") : level === "warn" ? chalk7.yellow("!") : level === "debug" ? chalk7.dim("\xB7") : chalk7.green("\u2022");
|
|
12267
|
-
console.log(`${prefix} ${message}`);
|
|
13796
|
+
return;
|
|
12268
13797
|
}
|
|
13798
|
+
const prefix = level === "error" ? chalk7.red("\u2717") : level === "warn" ? chalk7.yellow("!") : level === "debug" ? chalk7.dim("\xB7") : chalk7.green("\u2022");
|
|
13799
|
+
console.log(`${prefix} ${message}`);
|
|
13800
|
+
}
|
|
13801
|
+
function log2(state, message, level = "info") {
|
|
13802
|
+
if (!meetsThreshold(state, level)) return;
|
|
13803
|
+
if (state.json || !state.interactive) writeConsoleLog(state, message, level);
|
|
13804
|
+
}
|
|
13805
|
+
function logLocalOnly(state, message, level = "info") {
|
|
13806
|
+
if (!meetsThreshold(state, level)) return;
|
|
13807
|
+
writeConsoleLog(state, message, level);
|
|
12269
13808
|
}
|
|
12270
13809
|
function logActivity(state, entry) {
|
|
12271
13810
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
12272
13811
|
if (!meetsThreshold(state, level)) return;
|
|
12273
13812
|
forwardRunnerActivity(
|
|
12274
|
-
{
|
|
13813
|
+
{
|
|
13814
|
+
level,
|
|
13815
|
+
message: entry.message,
|
|
13816
|
+
error: entry.error,
|
|
13817
|
+
metadata: entry.metadata,
|
|
13818
|
+
...entry.source ? { source: entry.source } : {}
|
|
13819
|
+
},
|
|
12275
13820
|
{ agentId: state.agentId, authHeader: state.authHeader }
|
|
12276
13821
|
);
|
|
12277
13822
|
const fullEntry = {
|
|
@@ -12293,7 +13838,7 @@ function logActivity(state, entry) {
|
|
|
12293
13838
|
}
|
|
12294
13839
|
function reportSessionDbRecovery(state) {
|
|
12295
13840
|
try {
|
|
12296
|
-
const report = drainSessionDbRecoveryReport({ homeDir:
|
|
13841
|
+
const report = drainSessionDbRecoveryReport({ homeDir: homedir7(), env: process.env });
|
|
12297
13842
|
for (const record of report.records) {
|
|
12298
13843
|
const activity = buildSessionDbRecoveryActivity(record);
|
|
12299
13844
|
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
@@ -12328,7 +13873,9 @@ function displayStatus(state) {
|
|
|
12328
13873
|
const opencode = state.opencodeConnected ? chalk7.green(`opencode: :${state.port}`) : chalk7.red(`opencode: :${state.port} (down)`);
|
|
12329
13874
|
const messages = state.messageCount > 0 ? chalk7.dim(` \xB7 ${state.messageCount} processed`) : "";
|
|
12330
13875
|
const last = state.activityLog[state.activityLog.length - 1];
|
|
12331
|
-
const detail = last ? chalk7.dim(
|
|
13876
|
+
const detail = last ? chalk7.dim(
|
|
13877
|
+
` \xB7 ${last.timestamp.toISOString()} ${last.type === "error" ? last.error ?? "" : last.message ?? ""}`
|
|
13878
|
+
) : "";
|
|
12332
13879
|
const agent = state.agentName ?? state.agentId;
|
|
12333
13880
|
console.log(
|
|
12334
13881
|
`${chalk7.bold("Evident")} ${chalk7.dim(agent)} ${tunnel} ${opencode}${messages}${detail}`
|
|
@@ -12347,7 +13894,7 @@ function reportOpenCodeV2CredentialImport(state, result) {
|
|
|
12347
13894
|
logActivity(state, {
|
|
12348
13895
|
type: "info",
|
|
12349
13896
|
level: "warn",
|
|
12350
|
-
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them.
|
|
13897
|
+
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them. Connect the provider again from the runner's page in Evident; no shell on this machine is needed.`
|
|
12351
13898
|
});
|
|
12352
13899
|
}
|
|
12353
13900
|
if (result.imported.length === 0 && result.unsupported.length === 0 && result.failed.length === 0) {
|
|
@@ -12372,7 +13919,7 @@ async function importOpenCodeV2CredentialsForState(state) {
|
|
|
12372
13919
|
}
|
|
12373
13920
|
const result = await importOpenCodeV2Credentials({
|
|
12374
13921
|
request: client.request,
|
|
12375
|
-
homeDir:
|
|
13922
|
+
homeDir: homedir7(),
|
|
12376
13923
|
directory,
|
|
12377
13924
|
guard: state.opencodeV2CredentialImportGuard,
|
|
12378
13925
|
log: (message, level = "info") => logActivity(state, { type: "info", level, message })
|
|
@@ -12458,6 +14005,8 @@ async function driveChannels(state, driver) {
|
|
|
12458
14005
|
let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
12459
14006
|
let lastSeenClaudeApplies = driver.fileSyncActivity().claudeCredentialApplies;
|
|
12460
14007
|
let lastSeenOpencodeAuthApplies = driver.fileSyncActivity().opencodeAuthApplyCount;
|
|
14008
|
+
const stayAwakeHolds = () => state.stayAwakeUntilMs !== null && Date.now() < state.stayAwakeUntilMs;
|
|
14009
|
+
let stayAwakeSuppressionLogged = false;
|
|
12461
14010
|
while (state.running) {
|
|
12462
14011
|
const cycleStartedAtMs = performance.now();
|
|
12463
14012
|
let idleThisCycle = false;
|
|
@@ -12559,7 +14108,7 @@ async function driveChannels(state, driver) {
|
|
|
12559
14108
|
}
|
|
12560
14109
|
}
|
|
12561
14110
|
}
|
|
12562
|
-
await new Promise((
|
|
14111
|
+
await new Promise((resolve5) => setTimeout(resolve5, CHANNEL_POLL_INTERVAL_MS));
|
|
12563
14112
|
const cycleMs = performance.now() - cycleStartedAtMs;
|
|
12564
14113
|
if (idleThisCycle) idleMs += cycleMs;
|
|
12565
14114
|
if (unreachableThisCycle) unreachableMs += cycleMs;
|
|
@@ -12572,17 +14121,29 @@ async function driveChannels(state, driver) {
|
|
|
12572
14121
|
if (state.interactive) displayStatus(state);
|
|
12573
14122
|
break;
|
|
12574
14123
|
}
|
|
12575
|
-
if (state.idleTimeout !== null && idlePolls >= 2 && idleMs > state.idleTimeout * 1e3) {
|
|
14124
|
+
if (state.idleTimeout !== null && idlePolls >= 2 && idleMs > state.idleTimeout * 1e3 && !stayAwakeHolds()) {
|
|
12576
14125
|
logActivity(state, { type: "info", message: "Idle timeout reached" });
|
|
12577
14126
|
if (state.interactive) displayStatus(state);
|
|
12578
14127
|
break;
|
|
12579
14128
|
}
|
|
14129
|
+
if (state.idleTimeout !== null && idlePolls >= 2 && idleMs > state.idleTimeout * 1e3 && stayAwakeHolds()) {
|
|
14130
|
+
if (!stayAwakeSuppressionLogged) {
|
|
14131
|
+
stayAwakeSuppressionLogged = true;
|
|
14132
|
+
logActivity(state, {
|
|
14133
|
+
type: "info",
|
|
14134
|
+
message: "Idle, but staying awake until the requested minimum passes"
|
|
14135
|
+
});
|
|
14136
|
+
}
|
|
14137
|
+
}
|
|
12580
14138
|
}
|
|
12581
14139
|
}
|
|
12582
14140
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12583
|
-
var
|
|
14141
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
14142
|
+
var SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES = 2;
|
|
14143
|
+
var SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES = 2;
|
|
14144
|
+
var MAX_SESSION_DELETE_BODY_LENGTH = 200;
|
|
12584
14145
|
function shouldWarnForReclaimSkip(reason) {
|
|
12585
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
14146
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12586
14147
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12587
14148
|
if (!version2) return false;
|
|
12588
14149
|
const major = Number(version2[1]);
|
|
@@ -12591,7 +14152,14 @@ function shouldWarnForReclaimSkip(reason) {
|
|
|
12591
14152
|
return major > 22 || major === 22 && (minor > 13 || minor === 13 && patch >= 0);
|
|
12592
14153
|
}
|
|
12593
14154
|
function sessionDbPath() {
|
|
12594
|
-
return
|
|
14155
|
+
return join12(homedir7(), ".local", "share", "opencode", "opencode.db");
|
|
14156
|
+
}
|
|
14157
|
+
function logSessionCleanupActivity(state, phase, entry) {
|
|
14158
|
+
logActivity(state, {
|
|
14159
|
+
...entry,
|
|
14160
|
+
source: "cli.session-cleanup",
|
|
14161
|
+
metadata: { ...entry.metadata, phase }
|
|
14162
|
+
});
|
|
12595
14163
|
}
|
|
12596
14164
|
function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBootMigrationCount) {
|
|
12597
14165
|
const record = {
|
|
@@ -12632,69 +14200,187 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12632
14200
|
async function runSweep(state, driver, config) {
|
|
12633
14201
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12634
14202
|
try {
|
|
12635
|
-
const
|
|
12636
|
-
|
|
14203
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
14204
|
+
const sessions = enumeration.sessions;
|
|
14205
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
14206
|
+
if (enumerationReason !== null) {
|
|
12637
14207
|
logActivity(state, {
|
|
12638
14208
|
type: "info",
|
|
12639
|
-
|
|
14209
|
+
level: "warn",
|
|
14210
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12640
14211
|
});
|
|
12641
|
-
return;
|
|
12642
14212
|
}
|
|
12643
|
-
const toDelete = selectSessionsToDelete(
|
|
12644
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12645
|
-
{
|
|
12646
|
-
maxAgeMs: config.maxAgeMs,
|
|
12647
|
-
maxCount: config.maxCount,
|
|
12648
|
-
nowMs: Date.now(),
|
|
12649
|
-
protectedIds: driver.protectedSessionIds()
|
|
12650
|
-
}
|
|
12651
|
-
);
|
|
12652
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12653
14213
|
let deleted = 0;
|
|
12654
14214
|
let failed = 0;
|
|
14215
|
+
let failed4xx = 0;
|
|
14216
|
+
let failed5xx = 0;
|
|
14217
|
+
let failedOtherStatus = 0;
|
|
14218
|
+
let failedThrew = 0;
|
|
12655
14219
|
let skippedNewlyActive = 0;
|
|
12656
|
-
|
|
12657
|
-
|
|
12658
|
-
|
|
12659
|
-
|
|
14220
|
+
let remaining = 0;
|
|
14221
|
+
let protectedNow = driver.protectedSessionIds();
|
|
14222
|
+
if (sessions !== null) {
|
|
14223
|
+
const toDelete = selectSessionsToDelete(
|
|
14224
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
14225
|
+
{
|
|
14226
|
+
maxAgeMs: config.maxAgeMs,
|
|
14227
|
+
maxCount: config.maxCount,
|
|
14228
|
+
nowMs: Date.now(),
|
|
14229
|
+
protectedIds: protectedNow
|
|
14230
|
+
}
|
|
14231
|
+
);
|
|
14232
|
+
const eligibleToDelete = toDelete.filter(
|
|
14233
|
+
(id) => (state.sessionDeleteFailures.get(id)?.isolatedFailures ?? 0) < SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES
|
|
14234
|
+
);
|
|
14235
|
+
const deleteCandidates = eligibleToDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
14236
|
+
remaining = eligibleToDelete.length - deleteCandidates.length;
|
|
14237
|
+
const answeredFailures = [];
|
|
14238
|
+
protectedNow = driver.protectedSessionIds();
|
|
14239
|
+
for (const id of deleteCandidates) {
|
|
14240
|
+
if (protectedNow.has(id)) {
|
|
14241
|
+
skippedNewlyActive++;
|
|
14242
|
+
logActivity(state, {
|
|
14243
|
+
type: "info",
|
|
14244
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
14245
|
+
});
|
|
14246
|
+
continue;
|
|
14247
|
+
}
|
|
14248
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
14249
|
+
if (outcome.ok) {
|
|
14250
|
+
deleted++;
|
|
14251
|
+
} else if (outcome.status === "threw") {
|
|
14252
|
+
failed++;
|
|
14253
|
+
failedThrew++;
|
|
14254
|
+
} else {
|
|
14255
|
+
failed++;
|
|
14256
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
14257
|
+
if (statusClass === 4) failed4xx++;
|
|
14258
|
+
else if (statusClass === 5) failed5xx++;
|
|
14259
|
+
else failedOtherStatus++;
|
|
14260
|
+
answeredFailures.push({
|
|
14261
|
+
id,
|
|
14262
|
+
status: outcome.status,
|
|
14263
|
+
...outcome.body !== void 0 ? { body: outcome.body } : {}
|
|
14264
|
+
});
|
|
14265
|
+
}
|
|
14266
|
+
}
|
|
14267
|
+
const givenUpFailures = [];
|
|
14268
|
+
if (failed > SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES) {
|
|
14269
|
+
for (const { id } of answeredFailures) state.sessionDeleteFailures.delete(id);
|
|
14270
|
+
} else {
|
|
14271
|
+
for (const failure of answeredFailures) {
|
|
14272
|
+
const previous = state.sessionDeleteFailures.get(failure.id);
|
|
14273
|
+
const record = previous?.status === failure.status ? { status: failure.status, isolatedFailures: previous.isolatedFailures + 1 } : { status: failure.status, isolatedFailures: 1 };
|
|
14274
|
+
state.sessionDeleteFailures.set(failure.id, record);
|
|
14275
|
+
if (record.isolatedFailures >= SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES) {
|
|
14276
|
+
givenUpFailures.push(failure);
|
|
14277
|
+
}
|
|
14278
|
+
}
|
|
14279
|
+
}
|
|
14280
|
+
for (const failure of givenUpFailures) {
|
|
14281
|
+
const excerpt = redactDiagnosticText(failure.body ?? "(no response body)").slice(
|
|
14282
|
+
0,
|
|
14283
|
+
MAX_SESSION_DELETE_BODY_LENGTH
|
|
14284
|
+
);
|
|
14285
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12660
14286
|
type: "info",
|
|
12661
|
-
|
|
14287
|
+
level: "warn",
|
|
14288
|
+
message: `Session cleanup: gave up on session ${failure.id} after ${SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES} isolated delete failures (status ${failure.status}): ${excerpt}`,
|
|
14289
|
+
metadata: {
|
|
14290
|
+
outcome: "session-delete-given-up",
|
|
14291
|
+
session_id: failure.id,
|
|
14292
|
+
status: failure.status,
|
|
14293
|
+
body_excerpt: excerpt,
|
|
14294
|
+
given_up_count: givenUpFailures.length
|
|
14295
|
+
}
|
|
12662
14296
|
});
|
|
12663
|
-
continue;
|
|
12664
14297
|
}
|
|
12665
|
-
|
|
12666
|
-
|
|
14298
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
14299
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
14300
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
14301
|
+
logActivity(state, {
|
|
14302
|
+
type: "info",
|
|
14303
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
14304
|
+
});
|
|
12667
14305
|
}
|
|
12668
|
-
const
|
|
12669
|
-
|
|
12670
|
-
|
|
12671
|
-
|
|
12672
|
-
|
|
12673
|
-
|
|
14306
|
+
const sessionMetadata = {
|
|
14307
|
+
sessions_inspected: sessions?.length ?? null,
|
|
14308
|
+
sessions_deleted: deleted,
|
|
14309
|
+
sessions_failed: failed,
|
|
14310
|
+
sessions_failed_4xx: failed4xx,
|
|
14311
|
+
sessions_failed_5xx: failed5xx,
|
|
14312
|
+
sessions_failed_other: failedOtherStatus,
|
|
14313
|
+
sessions_failed_threw: failedThrew,
|
|
14314
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
14315
|
+
sessions_remaining: remaining,
|
|
14316
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
14317
|
+
};
|
|
14318
|
+
const protectedForReclaim = driver.protectedSessionIds();
|
|
12674
14319
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12675
14320
|
dbPath: sessionDbPath(),
|
|
12676
|
-
|
|
12677
|
-
|
|
14321
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
14322
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
14323
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
14324
|
+
hasLiveTurn: protectedForReclaim.size > 0,
|
|
14325
|
+
hasProvenanceAnomaly: state.sessionDbProvenanceAnomaly
|
|
12678
14326
|
});
|
|
12679
14327
|
if (reclaimResult.ok) {
|
|
14328
|
+
state.sessionStoreHealth.observeWritable();
|
|
12680
14329
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12681
14330
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12682
14331
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
12683
|
-
|
|
14332
|
+
const loopElapsedMs = Math.max(
|
|
14333
|
+
0,
|
|
14334
|
+
reclaimResult.elapsedMs - reclaimResult.checkpointElapsedMs
|
|
14335
|
+
);
|
|
14336
|
+
const reclaimIsSlow = reclaimResult.elapsedMs >= 4 * SESSION_DB_RECLAIM_TIME_BUDGET_MS;
|
|
14337
|
+
const perPageNote = reclaimResult.pagesReclaimed > 0 ? `, per-page ${(loopElapsedMs / reclaimResult.pagesReclaimed).toFixed(2)} ms/page` : "";
|
|
14338
|
+
const slowReclaimNote = reclaimIsSlow ? ` (slow reclaim: total elapsed ${reclaimResult.elapsedMs} ms, checkpoint ${reclaimResult.checkpointElapsedMs} ms, loop ${loopElapsedMs} ms, budget ${SESSION_DB_RECLAIM_TIME_BUDGET_MS} ms${perPageNote})` : "";
|
|
14339
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12684
14340
|
type: "info",
|
|
12685
|
-
|
|
14341
|
+
level: reclaimIsSlow ? "warn" : "info",
|
|
14342
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote + slowReclaimNote,
|
|
14343
|
+
metadata: {
|
|
14344
|
+
outcome: "reclaimed",
|
|
14345
|
+
mode: reclaimResult.mode,
|
|
14346
|
+
before_bytes: reclaimResult.beforeBytes,
|
|
14347
|
+
after_bytes: reclaimResult.afterBytes,
|
|
14348
|
+
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
14349
|
+
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
14350
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
14351
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
14352
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
14353
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
14354
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
14355
|
+
checkpoint_elapsed_ms: reclaimResult.checkpointElapsedMs,
|
|
14356
|
+
...sessionMetadata
|
|
14357
|
+
}
|
|
12686
14358
|
});
|
|
12687
14359
|
} else {
|
|
12688
|
-
|
|
14360
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
14361
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
14362
|
+
}
|
|
14363
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12689
14364
|
type: "info",
|
|
12690
|
-
|
|
14365
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
14366
|
+
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
14367
|
+
metadata: {
|
|
14368
|
+
outcome: "reclaim-skipped",
|
|
14369
|
+
reason: reclaimResult.skipped,
|
|
14370
|
+
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
14371
|
+
...sessionMetadata
|
|
14372
|
+
}
|
|
12691
14373
|
});
|
|
12692
14374
|
}
|
|
12693
14375
|
} catch (error2) {
|
|
12694
14376
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
12695
|
-
|
|
14377
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12696
14378
|
type: "error",
|
|
12697
|
-
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}
|
|
14379
|
+
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`,
|
|
14380
|
+
metadata: {
|
|
14381
|
+
outcome: "sweep-failed",
|
|
14382
|
+
reason: "sweep-error"
|
|
14383
|
+
}
|
|
12698
14384
|
});
|
|
12699
14385
|
}
|
|
12700
14386
|
}
|
|
@@ -12710,14 +14396,19 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
12710
14396
|
for (const warning2 of config.warnings) {
|
|
12711
14397
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
12712
14398
|
}
|
|
12713
|
-
const dbBytes = statSessionDbBytes(
|
|
14399
|
+
const dbBytes = statSessionDbBytes(homedir7());
|
|
12714
14400
|
void (async () => {
|
|
12715
14401
|
const reclaimAvailability = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
12716
14402
|
if (reclaimAvailability !== null) {
|
|
12717
|
-
|
|
14403
|
+
logSessionCleanupActivity(state, "startup", {
|
|
12718
14404
|
type: "info",
|
|
12719
14405
|
level: shouldWarnForReclaimSkip(reclaimAvailability.reason) ? "warn" : "info",
|
|
12720
|
-
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : "")
|
|
14406
|
+
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : ""),
|
|
14407
|
+
metadata: {
|
|
14408
|
+
outcome: "preflight-skipped",
|
|
14409
|
+
reason: reclaimAvailability.reason,
|
|
14410
|
+
...reclaimAvailability.detail !== void 0 ? { detail: reclaimAvailability.detail } : {}
|
|
14411
|
+
}
|
|
12721
14412
|
});
|
|
12722
14413
|
}
|
|
12723
14414
|
const sizeWarning = buildSessionStoreSizeWarning({
|
|
@@ -12734,9 +14425,15 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
12734
14425
|
);
|
|
12735
14426
|
});
|
|
12736
14427
|
if (!config.enabled) return;
|
|
12737
|
-
|
|
14428
|
+
logSessionCleanupActivity(state, "startup", {
|
|
12738
14429
|
type: "info",
|
|
12739
|
-
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)
|
|
14430
|
+
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)`,
|
|
14431
|
+
metadata: {
|
|
14432
|
+
outcome: "armed",
|
|
14433
|
+
interval_ms: config.intervalMs,
|
|
14434
|
+
max_age_ms: config.maxAgeMs ?? null,
|
|
14435
|
+
max_count: config.maxCount ?? null
|
|
14436
|
+
}
|
|
12740
14437
|
});
|
|
12741
14438
|
const interval = setInterval(() => void runSweep(state, driver, config), config.intervalMs);
|
|
12742
14439
|
const firstSweep = setTimeout(
|
|
@@ -12915,7 +14612,7 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
12915
14612
|
});
|
|
12916
14613
|
return;
|
|
12917
14614
|
}
|
|
12918
|
-
const { collect, stop } = createResourceUsageCollector(
|
|
14615
|
+
const { collect, stop } = createResourceUsageCollector(homedir7());
|
|
12919
14616
|
state.stopResourceUsageSampling = stop;
|
|
12920
14617
|
let consecutiveFailures = 0;
|
|
12921
14618
|
const tick = async () => {
|
|
@@ -12928,6 +14625,19 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
12928
14625
|
message: `Resource usage collection: ${warning2}`
|
|
12929
14626
|
});
|
|
12930
14627
|
}
|
|
14628
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
14629
|
+
...usage,
|
|
14630
|
+
nowMs: Date.now(),
|
|
14631
|
+
state: state.lowDiskWarningState
|
|
14632
|
+
});
|
|
14633
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
14634
|
+
if (lowDiskDecision.warning !== null) {
|
|
14635
|
+
logActivity(state, {
|
|
14636
|
+
type: "info",
|
|
14637
|
+
level: "warn",
|
|
14638
|
+
message: lowDiskDecision.warning
|
|
14639
|
+
});
|
|
14640
|
+
}
|
|
12931
14641
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
12932
14642
|
if (result.ok) {
|
|
12933
14643
|
if (consecutiveFailures > 0) {
|
|
@@ -13122,8 +14832,8 @@ async function beginGracefulShutdown(state, trigger) {
|
|
|
13122
14832
|
return true;
|
|
13123
14833
|
}
|
|
13124
14834
|
);
|
|
13125
|
-
const timedOut = new Promise((
|
|
13126
|
-
timer = setTimeout(() =>
|
|
14835
|
+
const timedOut = new Promise((resolve5) => {
|
|
14836
|
+
timer = setTimeout(() => resolve5(false), telemetryBudgetMs);
|
|
13127
14837
|
});
|
|
13128
14838
|
if (!await Promise.race([flushed, timedOut])) {
|
|
13129
14839
|
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
@@ -13140,7 +14850,7 @@ async function run(options) {
|
|
|
13140
14850
|
let fileSyncDirectories;
|
|
13141
14851
|
try {
|
|
13142
14852
|
logLevel = resolveLogLevel(options);
|
|
13143
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
14853
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir7());
|
|
13144
14854
|
if (options.restoreSessionDb && !options.sessionDbNoReplicateMarker) {
|
|
13145
14855
|
throw new Error(
|
|
13146
14856
|
"--restore-session-db requires --session-db-no-replicate-marker <path>; restore failures must block Litestream replication"
|
|
@@ -13163,6 +14873,8 @@ async function run(options) {
|
|
|
13163
14873
|
port: options.port ?? 4096,
|
|
13164
14874
|
conversationFilter: options.conversation ?? null,
|
|
13165
14875
|
idleTimeout: options.idleTimeout ?? null,
|
|
14876
|
+
stayAwakeSeconds: null,
|
|
14877
|
+
stayAwakeUntilMs: null,
|
|
13166
14878
|
json: options.json ?? false,
|
|
13167
14879
|
interactive,
|
|
13168
14880
|
logLevel,
|
|
@@ -13174,6 +14886,7 @@ async function run(options) {
|
|
|
13174
14886
|
opencodeClient: null,
|
|
13175
14887
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13176
14888
|
sessionDbProvenanceAnomaly: false,
|
|
14889
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13177
14890
|
opencodeProcess: null,
|
|
13178
14891
|
stopOpenCodeLogTail: null,
|
|
13179
14892
|
litestreamProcess: null,
|
|
@@ -13185,16 +14898,19 @@ async function run(options) {
|
|
|
13185
14898
|
messageCount: 0,
|
|
13186
14899
|
lastProxiedActivityAt: null,
|
|
13187
14900
|
sessionCleanupTimers: [],
|
|
14901
|
+
sessionDeleteFailures: /* @__PURE__ */ new Map(),
|
|
13188
14902
|
claudeUsageTimer: null,
|
|
13189
14903
|
claudeUsageRearm: null,
|
|
13190
14904
|
openaiUsageTimer: null,
|
|
13191
14905
|
openaiUsageRearm: null,
|
|
13192
14906
|
resourceUsageTimer: null,
|
|
13193
14907
|
stopResourceUsageSampling: null,
|
|
14908
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13194
14909
|
credentialSync: null,
|
|
13195
14910
|
authHeader: ""
|
|
13196
14911
|
};
|
|
13197
14912
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
14913
|
+
setTelemetryDiagnosticsReporter((message) => log2(state, message, "warn"));
|
|
13198
14914
|
if (options.credentialSyncMarker) {
|
|
13199
14915
|
state.credentialSync = createCredentialSync({
|
|
13200
14916
|
markerPath: options.credentialSyncMarker,
|
|
@@ -13348,6 +15064,17 @@ async function run(options) {
|
|
|
13348
15064
|
}
|
|
13349
15065
|
spinner?.succeed(`Runner: ${validation.agent.name || state.agentId}`);
|
|
13350
15066
|
state.agentName = validation.agent.name;
|
|
15067
|
+
const requestedStayAwakeSeconds = validation.agent?.wake_stay_awake_seconds;
|
|
15068
|
+
if (typeof requestedStayAwakeSeconds === "number" && Number.isFinite(requestedStayAwakeSeconds) && requestedStayAwakeSeconds > 0) {
|
|
15069
|
+
state.stayAwakeSeconds = requestedStayAwakeSeconds;
|
|
15070
|
+
} else if (requestedStayAwakeSeconds !== void 0 && requestedStayAwakeSeconds !== null) {
|
|
15071
|
+
state.stayAwakeSeconds = null;
|
|
15072
|
+
log2(
|
|
15073
|
+
state,
|
|
15074
|
+
`Ignoring invalid wake_stay_awake_seconds (type: ${typeof requestedStayAwakeSeconds})`,
|
|
15075
|
+
"warn"
|
|
15076
|
+
);
|
|
15077
|
+
}
|
|
13351
15078
|
const microvmId = process.env.MICROVM_ID?.trim();
|
|
13352
15079
|
if (microvmId) {
|
|
13353
15080
|
const reported = await reportMicrovmId(state.agentId, state.authHeader, microvmId);
|
|
@@ -13363,47 +15090,104 @@ async function run(options) {
|
|
|
13363
15090
|
} else {
|
|
13364
15091
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
13365
15092
|
}
|
|
13366
|
-
if (options.restoreRunnerCredentials) {
|
|
13367
|
-
|
|
13368
|
-
|
|
13369
|
-
|
|
13370
|
-
|
|
13371
|
-
if (level === "error") {
|
|
13372
|
-
logActivity(state, { type: "error", error: message });
|
|
13373
|
-
} else {
|
|
13374
|
-
logActivity(state, { type: "info", level, message });
|
|
13375
|
-
}
|
|
15093
|
+
if (options.restoreRunnerCredentials || options.credentialSyncMarker) {
|
|
15094
|
+
const logCredentialVaultResolution = (message, level) => {
|
|
15095
|
+
log2(state, message, level);
|
|
15096
|
+
if (state.interactive && !state.json) {
|
|
15097
|
+
logActivity(state, { type: "info", level, message });
|
|
13376
15098
|
}
|
|
13377
15099
|
};
|
|
13378
|
-
const
|
|
15100
|
+
const vaultResolution = await getRunnerCredentialVault(state.agentId, state.authHeader);
|
|
15101
|
+
if (vaultResolution.error) {
|
|
15102
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
15103
|
+
logCredentialVaultResolution(
|
|
15104
|
+
`CREDENTIAL-VAULT-LOOKUP-FAILED: ${vaultResolution.error}; credentials will be read from and written to this runner's own prefix this boot`,
|
|
15105
|
+
"warn"
|
|
15106
|
+
);
|
|
15107
|
+
} else if (vaultResolution.prefix !== null && vaultResolution.vaultId !== null && vaultResolution.source !== null) {
|
|
15108
|
+
process.env.CREDENTIAL_VAULT_PREFIX = vaultResolution.prefix;
|
|
15109
|
+
logCredentialVaultResolution(
|
|
15110
|
+
`CREDENTIAL-VAULT: source=${vaultResolution.source} vault=${vaultResolution.vaultId} prefix=${vaultResolution.prefix}`,
|
|
15111
|
+
"info"
|
|
15112
|
+
);
|
|
15113
|
+
} else {
|
|
15114
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
15115
|
+
logCredentialVaultResolution(
|
|
15116
|
+
"CREDENTIAL-VAULT-NONE: this runner is not linked to a credential vault; using its own prefix",
|
|
15117
|
+
"debug"
|
|
15118
|
+
);
|
|
15119
|
+
}
|
|
15120
|
+
}
|
|
15121
|
+
const credentialContext = {
|
|
15122
|
+
env: process.env,
|
|
15123
|
+
log: (message, level = "info") => {
|
|
15124
|
+
if (level === "error") {
|
|
15125
|
+
logActivity(state, { type: "error", error: message });
|
|
15126
|
+
} else {
|
|
15127
|
+
logActivity(state, { type: "info", level, message });
|
|
15128
|
+
}
|
|
15129
|
+
}
|
|
15130
|
+
};
|
|
15131
|
+
const githubTokenPopulated = await installRunnerSecret(credentialContext);
|
|
15132
|
+
if (options.restoreRunnerCredentials) {
|
|
13379
15133
|
await restoreCredentialStores(credentialContext);
|
|
13380
|
-
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13381
15134
|
}
|
|
15135
|
+
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13382
15136
|
state.credentialSync?.arm();
|
|
13383
|
-
|
|
13384
|
-
|
|
13385
|
-
|
|
15137
|
+
const dependencyInstallDiagnosticReporter = createDependencyInstallDiagnosticReporter({
|
|
15138
|
+
context: {
|
|
15139
|
+
homeDir: homedir7(),
|
|
15140
|
+
cwd: process.cwd(),
|
|
15141
|
+
microvmId: process.env.MICROVM_ID?.trim(),
|
|
15142
|
+
euid: process.getuid?.(),
|
|
15143
|
+
egid: process.getgid?.()
|
|
15144
|
+
},
|
|
15145
|
+
emitEvent: (message) => logActivity(state, { type: "info", level: "warn", message }),
|
|
15146
|
+
emitLocal: (message) => logLocalOnly(state, message, "warn")
|
|
15147
|
+
});
|
|
15148
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
15149
|
+
() => ({
|
|
13386
15150
|
agentId: state.agentId,
|
|
13387
15151
|
authHeader: state.authHeader
|
|
13388
|
-
})
|
|
13389
|
-
|
|
15152
|
+
}),
|
|
15153
|
+
({ isDiskFailure, sessionId, cause }) => {
|
|
15154
|
+
if (isDiskFailure) state.sessionStoreHealth.observeUnwritable();
|
|
15155
|
+
if (cause !== null && sessionId !== null) {
|
|
15156
|
+
state.sessionStoreHealth.observeCause(sessionId, cause);
|
|
15157
|
+
}
|
|
15158
|
+
},
|
|
15159
|
+
dependencyInstallDiagnosticReporter
|
|
15160
|
+
);
|
|
15161
|
+
const openCodeLogTail = tailOpenCodeLogFile(
|
|
15162
|
+
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
15163
|
+
openCodeActivityForwarder
|
|
15164
|
+
);
|
|
15165
|
+
state.stopOpenCodeLogTail = () => {
|
|
15166
|
+
openCodeActivityForwarder.flush();
|
|
15167
|
+
openCodeLogTail.stop();
|
|
15168
|
+
};
|
|
13390
15169
|
let sessionDbVerifyFatal = false;
|
|
15170
|
+
let sessionDbVerifyFatalReason;
|
|
13391
15171
|
if (!options.restoreSessionDb) {
|
|
13392
|
-
|
|
15172
|
+
logActivity(state, {
|
|
15173
|
+
type: "info",
|
|
15174
|
+
level: "warn",
|
|
15175
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=flag-absent \u2014 Skipping session-DB restore: --restore-session-db was not passed"
|
|
15176
|
+
});
|
|
13393
15177
|
} else {
|
|
13394
15178
|
const health = await checkOpenCodeHealth(state.port);
|
|
13395
15179
|
if (health.healthy) {
|
|
13396
|
-
|
|
13397
|
-
|
|
13398
|
-
|
|
13399
|
-
"
|
|
13400
|
-
);
|
|
15180
|
+
logActivity(state, {
|
|
15181
|
+
type: "info",
|
|
15182
|
+
level: "warn",
|
|
15183
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-serving \u2014 Skipping session-DB restore: OpenCode is already serving this database"
|
|
15184
|
+
});
|
|
13401
15185
|
} else if (health.authFailed) {
|
|
13402
|
-
|
|
13403
|
-
|
|
13404
|
-
|
|
13405
|
-
"
|
|
13406
|
-
);
|
|
15186
|
+
logActivity(state, {
|
|
15187
|
+
type: "info",
|
|
15188
|
+
level: "warn",
|
|
15189
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-auth-challenge \u2014 Skipping session-DB restore: OpenCode answered with an authentication challenge; assuming this database is in use"
|
|
15190
|
+
});
|
|
13407
15191
|
} else {
|
|
13408
15192
|
const result = await restoreAndVerifySessionDb({
|
|
13409
15193
|
dbPath: sessionDbPath(),
|
|
@@ -13420,12 +15204,13 @@ async function run(options) {
|
|
|
13420
15204
|
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
13421
15205
|
});
|
|
13422
15206
|
sessionDbVerifyFatal = result.verifyFatal;
|
|
15207
|
+
sessionDbVerifyFatalReason = result.verifyFatalReason;
|
|
13423
15208
|
}
|
|
13424
15209
|
}
|
|
13425
15210
|
reportSessionDbRecovery(state);
|
|
13426
15211
|
if (sessionDbVerifyFatal) {
|
|
13427
15212
|
throw new Error(
|
|
13428
|
-
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
15213
|
+
sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
|
|
13429
15214
|
);
|
|
13430
15215
|
}
|
|
13431
15216
|
applyRunnerOpenCodeConfig({
|
|
@@ -13503,7 +15288,7 @@ async function run(options) {
|
|
|
13503
15288
|
const provenance = checkSessionDbProvenance({
|
|
13504
15289
|
dbPath: sessionDbPath(),
|
|
13505
15290
|
currentVersion: state.opencodeVersion,
|
|
13506
|
-
homeDir:
|
|
15291
|
+
homeDir: homedir7(),
|
|
13507
15292
|
env: process.env
|
|
13508
15293
|
});
|
|
13509
15294
|
if (provenance.anomaly) {
|
|
@@ -13670,10 +15455,11 @@ async function run(options) {
|
|
|
13670
15455
|
getAuthHeader: () => state.authHeader,
|
|
13671
15456
|
conversationFilter: state.conversationFilter,
|
|
13672
15457
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
15458
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13673
15459
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13674
15460
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13675
15461
|
fileSyncDirectories,
|
|
13676
|
-
homeDir:
|
|
15462
|
+
homeDir: homedir7(),
|
|
13677
15463
|
maxActiveSessions,
|
|
13678
15464
|
log: (entry) => (
|
|
13679
15465
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
@@ -13695,13 +15481,21 @@ async function run(options) {
|
|
|
13695
15481
|
openCodePassword: state.opencodePassword,
|
|
13696
15482
|
isRunning: () => state.running,
|
|
13697
15483
|
events: {
|
|
13698
|
-
onConnected: (agentId, isReconnect) => {
|
|
15484
|
+
onConnected: (agentId, isReconnect, transition) => {
|
|
13699
15485
|
state.connected = true;
|
|
13700
15486
|
state.agentId = agentId;
|
|
15487
|
+
const connectionMessage = isReconnect ? `Tunnel reconnected (runner: ${agentId}, generation: ${transition.generation}, attempt: ${transition.attempt}, offline for ${transition.downtimeMs}ms)` : `Tunnel connected (runner: ${agentId}, generation: ${transition.generation})`;
|
|
13701
15488
|
logActivity(state, {
|
|
13702
15489
|
type: "info",
|
|
13703
|
-
message:
|
|
15490
|
+
message: connectionMessage
|
|
13704
15491
|
});
|
|
15492
|
+
if (!isReconnect && state.stayAwakeSeconds !== null && state.stayAwakeUntilMs === null) {
|
|
15493
|
+
state.stayAwakeUntilMs = Date.now() + state.stayAwakeSeconds * 1e3;
|
|
15494
|
+
logActivity(state, {
|
|
15495
|
+
type: "info",
|
|
15496
|
+
message: `Staying awake for at least ${state.stayAwakeSeconds}s as requested by the wake`
|
|
15497
|
+
});
|
|
15498
|
+
}
|
|
13705
15499
|
if (options.tunnelReadyFile) {
|
|
13706
15500
|
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
13707
15501
|
if (marker.ok) {
|
|
@@ -13721,7 +15515,12 @@ async function run(options) {
|
|
|
13721
15515
|
version: state.opencodeVersion,
|
|
13722
15516
|
major: state.opencodeApiVersion,
|
|
13723
15517
|
connected: state.opencodeConnected
|
|
13724
|
-
})
|
|
15518
|
+
}),
|
|
15519
|
+
connection_generation: transition.generation,
|
|
15520
|
+
...isReconnect && transition.downtimeMs !== null ? {
|
|
15521
|
+
reconnect_attempt: transition.attempt,
|
|
15522
|
+
downtime_ms: transition.downtimeMs
|
|
15523
|
+
} : {}
|
|
13725
15524
|
});
|
|
13726
15525
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
13727
15526
|
if (state.interactive) displayStatus(state);
|
|
@@ -13743,13 +15542,18 @@ async function run(options) {
|
|
|
13743
15542
|
if (state.interactive) displayStatus(state);
|
|
13744
15543
|
});
|
|
13745
15544
|
},
|
|
13746
|
-
onDisconnected: (code, reason) => {
|
|
15545
|
+
onDisconnected: (code, reason, transition) => {
|
|
13747
15546
|
state.connected = false;
|
|
15547
|
+
const safeReason2 = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
13748
15548
|
logActivity(state, {
|
|
13749
15549
|
type: "info",
|
|
13750
|
-
message: `Tunnel disconnected (code: ${code}, reason: ${
|
|
15550
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason2})`
|
|
15551
|
+
});
|
|
15552
|
+
emitAgentDisconnected(state.agentId, {
|
|
15553
|
+
code,
|
|
15554
|
+
reason: safeReason2,
|
|
15555
|
+
connection_generation: transition.generation
|
|
13751
15556
|
});
|
|
13752
|
-
emitAgentDisconnected(state.agentId, { code, reason });
|
|
13753
15557
|
if (state.interactive) displayStatus(state);
|
|
13754
15558
|
},
|
|
13755
15559
|
onError: (error2) => {
|
|
@@ -13849,7 +15653,7 @@ async function run(options) {
|
|
|
13849
15653
|
},
|
|
13850
15654
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
13851
15655
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
13852
|
-
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
15656
|
+
forcedOnHint: state.opencodeApiVersion === "v2" ? "OpenCode V2 keeps its connected ChatGPT login in a store Evident cannot read" : "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
13853
15657
|
firstDelayMs: firstReportDelayMs,
|
|
13854
15658
|
nextDelayMs: usageReportDelayMs,
|
|
13855
15659
|
failureLogLevel: usageReportFailureLogLevel
|
|
@@ -13892,7 +15696,7 @@ async function run(options) {
|
|
|
13892
15696
|
}
|
|
13893
15697
|
|
|
13894
15698
|
// src/index.ts
|
|
13895
|
-
var
|
|
15699
|
+
var version = getCliVersion();
|
|
13896
15700
|
var program = new Command();
|
|
13897
15701
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
13898
15702
|
"--endpoint <url>",
|
|
@@ -13910,6 +15714,7 @@ program.command("login").description("Authenticate with Evident").option("--toke
|
|
|
13910
15714
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
13911
15715
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
13912
15716
|
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
15717
|
+
program.command("credential-vault-env").description("Print the shell assignment a runner image evals before starting the synchroniser").action(credentialVaultEnv);
|
|
13913
15718
|
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
13914
15719
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
13915
15720
|
"-a, --agent [id]",
|