@evident-ai/cli 3.5.2-dev.8d3aa06 → 3.5.2-dev.bd11262
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/index.js +2441 -600
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire as createRequire2 } from "module";
|
|
5
4
|
import { Command } from "commander";
|
|
6
5
|
|
|
7
6
|
// src/commands/login.ts
|
|
@@ -316,14 +315,14 @@ function blank() {
|
|
|
316
315
|
console.log();
|
|
317
316
|
}
|
|
318
317
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
319
|
-
return new Promise((
|
|
318
|
+
return new Promise((resolve5) => {
|
|
320
319
|
process.stdout.write(chalk.dim(prompt));
|
|
321
320
|
const handler = () => {
|
|
322
321
|
process.stdin.removeListener("data", handler);
|
|
323
322
|
process.stdin.setRawMode?.(false);
|
|
324
323
|
process.stdin.pause();
|
|
325
324
|
console.log();
|
|
326
|
-
|
|
325
|
+
resolve5();
|
|
327
326
|
};
|
|
328
327
|
if (process.stdin.isTTY) {
|
|
329
328
|
process.stdin.setRawMode?.(true);
|
|
@@ -333,7 +332,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
333
332
|
});
|
|
334
333
|
}
|
|
335
334
|
function sleep(ms) {
|
|
336
|
-
return new Promise((
|
|
335
|
+
return new Promise((resolve5) => setTimeout(resolve5, ms));
|
|
337
336
|
}
|
|
338
337
|
|
|
339
338
|
// src/commands/login.ts
|
|
@@ -411,19 +410,19 @@ async function tokenLogin() {
|
|
|
411
410
|
);
|
|
412
411
|
blank();
|
|
413
412
|
process.stdout.write("Paste token: ");
|
|
414
|
-
const token = await new Promise((
|
|
413
|
+
const token = await new Promise((resolve5) => {
|
|
415
414
|
let data = "";
|
|
416
415
|
process.stdin.setEncoding("utf8");
|
|
417
416
|
process.stdin.on("data", (chunk) => {
|
|
418
417
|
data += chunk;
|
|
419
418
|
});
|
|
420
419
|
process.stdin.on("end", () => {
|
|
421
|
-
|
|
420
|
+
resolve5(data.trim());
|
|
422
421
|
});
|
|
423
422
|
if (process.stdin.isTTY) {
|
|
424
423
|
process.stdin.once("data", (chunk) => {
|
|
425
424
|
process.stdin.pause();
|
|
426
|
-
|
|
425
|
+
resolve5(chunk.toString().trim());
|
|
427
426
|
});
|
|
428
427
|
process.stdin.resume();
|
|
429
428
|
}
|
|
@@ -704,6 +703,31 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
704
703
|
opencode_db_bytes: usage.opencodeDbBytes
|
|
705
704
|
});
|
|
706
705
|
}
|
|
706
|
+
async function getRunnerCredentialVault(agentId, authHeader) {
|
|
707
|
+
const empty = { prefix: null, vaultId: null, source: null };
|
|
708
|
+
try {
|
|
709
|
+
const apiUrl = getApiUrlConfig();
|
|
710
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/credential-vault`, {
|
|
711
|
+
headers: { Authorization: authHeader },
|
|
712
|
+
signal: AbortSignal.timeout(1e4)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) {
|
|
715
|
+
const serverMessage = await readErrorMessage(response);
|
|
716
|
+
return {
|
|
717
|
+
...empty,
|
|
718
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
return {
|
|
723
|
+
prefix: data.credential_vault_prefix,
|
|
724
|
+
vaultId: data.vault_id,
|
|
725
|
+
source: data.source
|
|
726
|
+
};
|
|
727
|
+
} catch (error2) {
|
|
728
|
+
return { ...empty, error: describeTimeoutError(error2, 1e4) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
707
731
|
async function getAgentInfo(agentId, authHeader) {
|
|
708
732
|
const apiUrl = getApiUrlConfig();
|
|
709
733
|
try {
|
|
@@ -878,6 +902,76 @@ async function status(options = {}) {
|
|
|
878
902
|
process.exit(result.exitCode);
|
|
879
903
|
}
|
|
880
904
|
|
|
905
|
+
// src/commands/credential-vault-env.ts
|
|
906
|
+
function shellQuote(value) {
|
|
907
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
908
|
+
}
|
|
909
|
+
function printAssignment(prefix) {
|
|
910
|
+
console.log(`CREDENTIAL_VAULT_PREFIX=${shellQuote(prefix ?? "")}`);
|
|
911
|
+
}
|
|
912
|
+
function safeReason(reason, credentials2, authHeader) {
|
|
913
|
+
let safe = reason;
|
|
914
|
+
for (const secret of [
|
|
915
|
+
authHeader,
|
|
916
|
+
credentials2?.token,
|
|
917
|
+
process.env.EVIDENT_RUNNER_KEY,
|
|
918
|
+
process.env.EVIDENT_AGENT_KEY,
|
|
919
|
+
process.env.EVIDENT_TOKEN
|
|
920
|
+
]) {
|
|
921
|
+
if (secret) safe = safe.replaceAll(secret, "[redacted]");
|
|
922
|
+
}
|
|
923
|
+
return safe.replace(/[\r\n]+/g, " ");
|
|
924
|
+
}
|
|
925
|
+
function printFallback(reason, credentials2, authHeader) {
|
|
926
|
+
printAssignment(null);
|
|
927
|
+
console.error(
|
|
928
|
+
`WARNING: ${safeReason(reason, credentials2, authHeader)}; using this runner's own credential prefix.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
async function credentialVaultEnv() {
|
|
932
|
+
let credentials2 = null;
|
|
933
|
+
let authHeader;
|
|
934
|
+
try {
|
|
935
|
+
credentials2 = await getAuthCredentials();
|
|
936
|
+
if (!credentials2) {
|
|
937
|
+
printFallback("no runner credentials are configured");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
authHeader = getAuthHeader(credentials2);
|
|
941
|
+
const runner = await resolveAgentIdFromKey(authHeader);
|
|
942
|
+
if (!runner.agent_id) {
|
|
943
|
+
printFallback(
|
|
944
|
+
runner.error ?? "the runner identity could not be resolved",
|
|
945
|
+
credentials2,
|
|
946
|
+
authHeader
|
|
947
|
+
);
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const lookup = await getRunnerCredentialVault(runner.agent_id, authHeader);
|
|
951
|
+
if (lookup.error) {
|
|
952
|
+
printFallback(`credential vault lookup failed: ${lookup.error}`, credentials2, authHeader);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (lookup.prefix === null) {
|
|
956
|
+
printFallback("this runner is not linked to a credential vault", credentials2, authHeader);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (/[\r\n]/.test(lookup.prefix)) {
|
|
960
|
+
printFallback("credential vault lookup returned an invalid prefix", credentials2, authHeader);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
printAssignment(lookup.prefix);
|
|
964
|
+
console.error(
|
|
965
|
+
`VAULT-MODE: credential stores use vault=${lookup.vaultId} source=${lookup.source} prefix=${lookup.prefix}.`
|
|
966
|
+
);
|
|
967
|
+
} catch (error2) {
|
|
968
|
+
const detail = credentials2 === null ? "credential resolution failed" : `credential vault lookup could not be completed: ${error2 instanceof Error ? error2.message : String(error2)}`;
|
|
969
|
+
printFallback(detail, credentials2, authHeader);
|
|
970
|
+
} finally {
|
|
971
|
+
process.exitCode = 0;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
881
975
|
// src/lib/claude-usage.ts
|
|
882
976
|
import { execFileSync } from "node:child_process";
|
|
883
977
|
import { readFileSync } from "node:fs";
|
|
@@ -1063,13 +1157,83 @@ async function claudeUsage() {
|
|
|
1063
1157
|
|
|
1064
1158
|
// src/commands/run.ts
|
|
1065
1159
|
import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync7, writeFileSync as writeFileSync6 } from "node:fs";
|
|
1066
|
-
import { homedir as
|
|
1067
|
-
import { isAbsolute as
|
|
1160
|
+
import { homedir as homedir7 } from "node:os";
|
|
1161
|
+
import { isAbsolute as isAbsolute4, join as join12, parse, resolve as resolvePath2 } from "node:path";
|
|
1068
1162
|
import chalk7 from "chalk";
|
|
1069
1163
|
|
|
1070
1164
|
// ../../packages/types/src/agents/index.ts
|
|
1071
1165
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1072
1166
|
|
|
1167
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1168
|
+
function roleOf(m) {
|
|
1169
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1170
|
+
if (typeof m.role === "string") return m.role;
|
|
1171
|
+
const infoRole = m.info?.role;
|
|
1172
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1173
|
+
}
|
|
1174
|
+
function idOf(m) {
|
|
1175
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1176
|
+
if (typeof m.id === "string") return m.id;
|
|
1177
|
+
const infoId = m.info?.id;
|
|
1178
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1179
|
+
}
|
|
1180
|
+
function parentIdOf(m) {
|
|
1181
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1182
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1183
|
+
const infoParent = m.info?.parentID;
|
|
1184
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1185
|
+
}
|
|
1186
|
+
function textPartsOf(m) {
|
|
1187
|
+
if (!m || typeof m !== "object") return [];
|
|
1188
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1189
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1190
|
+
}
|
|
1191
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1192
|
+
if (!messages || messages.length === 0) return [];
|
|
1193
|
+
const byParent = messages.filter(
|
|
1194
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1195
|
+
);
|
|
1196
|
+
if (byParent.length > 0) return byParent;
|
|
1197
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1198
|
+
if (userIndex === -1) return [];
|
|
1199
|
+
const run2 = [];
|
|
1200
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1201
|
+
const role = roleOf(messages[i]);
|
|
1202
|
+
if (role === "user") break;
|
|
1203
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1204
|
+
}
|
|
1205
|
+
return run2;
|
|
1206
|
+
}
|
|
1207
|
+
function lastText(assistantRun) {
|
|
1208
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1209
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1210
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1211
|
+
}
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1215
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1216
|
+
if (lastText(run2) !== null) return run2;
|
|
1217
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1218
|
+
if (userIndex === -1) return run2;
|
|
1219
|
+
let continuation;
|
|
1220
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1221
|
+
if (roleOf(messages[i]) === "user") {
|
|
1222
|
+
continuation = messages[i];
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
const continuationId = idOf(continuation);
|
|
1227
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1228
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1229
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1230
|
+
}
|
|
1231
|
+
return run2;
|
|
1232
|
+
}
|
|
1233
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1234
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1073
1237
|
// ../../packages/types/src/telemetry/index.ts
|
|
1074
1238
|
var TelemetryEventTypes = {
|
|
1075
1239
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1158,7 +1322,7 @@ import ora4 from "ora";
|
|
|
1158
1322
|
import { select as select4 } from "@inquirer/prompts";
|
|
1159
1323
|
|
|
1160
1324
|
// src/lib/telemetry.ts
|
|
1161
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1325
|
+
var CLI_VERSION = (true ? "3.5.2-dev.bd11262" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1162
1326
|
function getCliVersion() {
|
|
1163
1327
|
return CLI_VERSION;
|
|
1164
1328
|
}
|
|
@@ -1172,6 +1336,23 @@ var authProvider = null;
|
|
|
1172
1336
|
function setTelemetryAuthProvider(provider) {
|
|
1173
1337
|
authProvider = provider;
|
|
1174
1338
|
}
|
|
1339
|
+
var diagnosticsReporter = null;
|
|
1340
|
+
function setTelemetryDiagnosticsReporter(reporter) {
|
|
1341
|
+
diagnosticsReporter = reporter;
|
|
1342
|
+
}
|
|
1343
|
+
function reportTelemetryDiagnostic(message) {
|
|
1344
|
+
if (!diagnosticsReporter) {
|
|
1345
|
+
console.error(message);
|
|
1346
|
+
return;
|
|
1347
|
+
}
|
|
1348
|
+
try {
|
|
1349
|
+
diagnosticsReporter(message);
|
|
1350
|
+
} catch (error2) {
|
|
1351
|
+
console.error(
|
|
1352
|
+
`Telemetry diagnostics reporter failed: ${error2 instanceof Error ? error2.message : String(error2)}; original diagnostic: ${message}`
|
|
1353
|
+
);
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1175
1356
|
var FLUSH_FAILURE_LOG_INTERVAL_MS = 6e4;
|
|
1176
1357
|
var lastFlushFailureLoggedAt = 0;
|
|
1177
1358
|
var suppressedFlushFailureCount = 0;
|
|
@@ -1237,7 +1418,9 @@ async function flushEvents() {
|
|
|
1237
1418
|
signal: controller.signal
|
|
1238
1419
|
});
|
|
1239
1420
|
if (!response.ok) {
|
|
1240
|
-
|
|
1421
|
+
reportTelemetryDiagnostic(
|
|
1422
|
+
`Telemetry could not send events (HTTP ${response.status}); runner and OpenCode are unaffected`
|
|
1423
|
+
);
|
|
1241
1424
|
}
|
|
1242
1425
|
} finally {
|
|
1243
1426
|
clearTimeout(timeout);
|
|
@@ -1247,7 +1430,9 @@ async function flushEvents() {
|
|
|
1247
1430
|
if (now - lastFlushFailureLoggedAt >= FLUSH_FAILURE_LOG_INTERVAL_MS) {
|
|
1248
1431
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1249
1432
|
const suffix = suppressedFlushFailureCount > 0 ? ` (${suppressedFlushFailureCount} more suppressed in the last ${FLUSH_FAILURE_LOG_INTERVAL_MS / 1e3}s)` : "";
|
|
1250
|
-
|
|
1433
|
+
reportTelemetryDiagnostic(
|
|
1434
|
+
`Telemetry flush failed: ${message}; runner and OpenCode are unaffected${suffix}`
|
|
1435
|
+
);
|
|
1251
1436
|
lastFlushFailureLoggedAt = now;
|
|
1252
1437
|
suppressedFlushFailureCount = 0;
|
|
1253
1438
|
} else {
|
|
@@ -1298,9 +1483,598 @@ var EventTypes = {
|
|
|
1298
1483
|
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
1299
1484
|
};
|
|
1300
1485
|
|
|
1486
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1487
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1488
|
+
import { dirname as dirname2 } from "node:path";
|
|
1489
|
+
|
|
1490
|
+
// src/lib/diagnostic-redaction.ts
|
|
1491
|
+
import { homedir as homedir2 } from "node:os";
|
|
1492
|
+
function redactHomeDirectory(message) {
|
|
1493
|
+
const home = homedir2();
|
|
1494
|
+
if (!home || home === "/") {
|
|
1495
|
+
return message;
|
|
1496
|
+
}
|
|
1497
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1498
|
+
let result = "";
|
|
1499
|
+
let searchFrom = 0;
|
|
1500
|
+
let homeIndex;
|
|
1501
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1502
|
+
const homeEnd = homeIndex + home.length;
|
|
1503
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1504
|
+
const nextCharacter = message[homeEnd];
|
|
1505
|
+
const atPathBoundary = (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1506
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1507
|
+
result += atPathBoundary ? "~" : home;
|
|
1508
|
+
searchFrom = homeEnd;
|
|
1509
|
+
}
|
|
1510
|
+
return result + message.slice(searchFrom);
|
|
1511
|
+
}
|
|
1512
|
+
function redactDiagnosticText(message) {
|
|
1513
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1514
|
+
return redactHomeDirectory(redacted);
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1518
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 100;
|
|
1519
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1520
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1521
|
+
var SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS = 5e3;
|
|
1522
|
+
function errorMessage(error2) {
|
|
1523
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1524
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1525
|
+
}
|
|
1526
|
+
function errorMessages(error2) {
|
|
1527
|
+
const messages = [];
|
|
1528
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1529
|
+
let current = error2;
|
|
1530
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1531
|
+
visited.add(current);
|
|
1532
|
+
if (typeof current === "string") {
|
|
1533
|
+
messages.push(current);
|
|
1534
|
+
break;
|
|
1535
|
+
}
|
|
1536
|
+
if (current instanceof Error) {
|
|
1537
|
+
messages.push(current.name, current.message);
|
|
1538
|
+
current = current.cause;
|
|
1539
|
+
continue;
|
|
1540
|
+
}
|
|
1541
|
+
if (typeof current === "object") {
|
|
1542
|
+
const record = current;
|
|
1543
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1544
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1545
|
+
current = record.cause;
|
|
1546
|
+
continue;
|
|
1547
|
+
}
|
|
1548
|
+
break;
|
|
1549
|
+
}
|
|
1550
|
+
return messages;
|
|
1551
|
+
}
|
|
1552
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1553
|
+
return errorMessages(error2).some(
|
|
1554
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1555
|
+
message
|
|
1556
|
+
)
|
|
1557
|
+
);
|
|
1558
|
+
}
|
|
1559
|
+
var SQLITE_CAUSE_TOKEN = /(?:^|cause:\s+)(SQLiteError:\s+[A-Za-z][A-Za-z0-9]*(?:\s+[A-Za-z][A-Za-z0-9]*){0,7})(?=\s*(?:[)\]"',;]|$))/i;
|
|
1560
|
+
var EFFECT_SQL_CAUSE_TOKEN = /(?:^|cause:\s+)(effect\/sql\/SqlError\/[A-Za-z][A-Za-z0-9_]*)/i;
|
|
1561
|
+
var PARAMS_FIELD = /\bparams\s*(?:=|:)/i;
|
|
1562
|
+
function isCauseBearingText(text) {
|
|
1563
|
+
return /^(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text.trim()) || /(?:^|\s)cause:\s+(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text);
|
|
1564
|
+
}
|
|
1565
|
+
function structuredLogFields(line) {
|
|
1566
|
+
const fields = [];
|
|
1567
|
+
for (let index = 0; index < line.length; index++) {
|
|
1568
|
+
if (line[index] === '"') {
|
|
1569
|
+
index++;
|
|
1570
|
+
while (index < line.length) {
|
|
1571
|
+
if (line[index] === "\\") {
|
|
1572
|
+
index++;
|
|
1573
|
+
} else if (line[index] === '"') {
|
|
1574
|
+
break;
|
|
1575
|
+
}
|
|
1576
|
+
index++;
|
|
1577
|
+
}
|
|
1578
|
+
continue;
|
|
1579
|
+
}
|
|
1580
|
+
const atFieldBoundary = index === 0 || /\s/.test(line[index - 1]);
|
|
1581
|
+
const key = line.startsWith("error=", index) ? "error=" : line.startsWith("cause=", index) ? "cause=" : null;
|
|
1582
|
+
if (!atFieldBoundary || key === null) continue;
|
|
1583
|
+
const valueStart = index + key.length;
|
|
1584
|
+
if (line[valueStart] !== '"') {
|
|
1585
|
+
const valueEnd2 = line.indexOf(" ", valueStart);
|
|
1586
|
+
fields.push(line.slice(valueStart, valueEnd2 === -1 ? line.length : valueEnd2));
|
|
1587
|
+
index = valueEnd2 === -1 ? line.length : valueEnd2;
|
|
1588
|
+
continue;
|
|
1589
|
+
}
|
|
1590
|
+
let valueEnd = valueStart + 1;
|
|
1591
|
+
for (; valueEnd < line.length; valueEnd++) {
|
|
1592
|
+
if (line[valueEnd] === "\\") {
|
|
1593
|
+
valueEnd++;
|
|
1594
|
+
} else if (line[valueEnd] === '"') {
|
|
1595
|
+
break;
|
|
1596
|
+
}
|
|
1597
|
+
}
|
|
1598
|
+
fields.push(line.slice(valueStart + 1, valueEnd));
|
|
1599
|
+
index = valueEnd;
|
|
1600
|
+
}
|
|
1601
|
+
return fields;
|
|
1602
|
+
}
|
|
1603
|
+
function structuredCauseMessages(text) {
|
|
1604
|
+
const fields = structuredLogFields(text);
|
|
1605
|
+
if (fields.length > 0) return fields;
|
|
1606
|
+
return isCauseBearingText(text) ? [text] : [];
|
|
1607
|
+
}
|
|
1608
|
+
function causeTokenFromText(text) {
|
|
1609
|
+
const paramsIndex = text.search(PARAMS_FIELD);
|
|
1610
|
+
for (const pattern of [SQLITE_CAUSE_TOKEN, EFFECT_SQL_CAUSE_TOKEN]) {
|
|
1611
|
+
const match = pattern.exec(text);
|
|
1612
|
+
if (match?.[1] && (paramsIndex === -1 || match.index < paramsIndex)) {
|
|
1613
|
+
return match[1];
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
return null;
|
|
1617
|
+
}
|
|
1618
|
+
function sessionStoreCauseToken(value) {
|
|
1619
|
+
for (const message of structuredCauseMessages(value)) {
|
|
1620
|
+
const token = causeTokenFromText(message);
|
|
1621
|
+
if (token !== null) return redactDiagnosticText(token).slice(0, 120);
|
|
1622
|
+
}
|
|
1623
|
+
return null;
|
|
1624
|
+
}
|
|
1625
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1626
|
+
try {
|
|
1627
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1628
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1629
|
+
if (availableBytes < requiredBytes) {
|
|
1630
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1631
|
+
}
|
|
1632
|
+
return null;
|
|
1633
|
+
} catch (err) {
|
|
1634
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1635
|
+
}
|
|
1636
|
+
}
|
|
1637
|
+
function readLogicalBytes(db) {
|
|
1638
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1639
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1640
|
+
return pageCount * pageSize;
|
|
1641
|
+
}
|
|
1642
|
+
function readFreelistCount(db) {
|
|
1643
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1644
|
+
}
|
|
1645
|
+
function readCheckpointResult(db) {
|
|
1646
|
+
db.exec("PRAGMA busy_timeout=0");
|
|
1647
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1648
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1649
|
+
}
|
|
1650
|
+
async function probeReclaimAvailability(input) {
|
|
1651
|
+
const { dbPath, requiredBytes } = input;
|
|
1652
|
+
let sqlite;
|
|
1653
|
+
try {
|
|
1654
|
+
sqlite = await import("node:sqlite");
|
|
1655
|
+
} catch (err) {
|
|
1656
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1657
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1658
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1659
|
+
}
|
|
1660
|
+
let autoVacuum = null;
|
|
1661
|
+
try {
|
|
1662
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1663
|
+
try {
|
|
1664
|
+
db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
|
|
1665
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1666
|
+
} finally {
|
|
1667
|
+
db.close();
|
|
1668
|
+
}
|
|
1669
|
+
} catch (err) {
|
|
1670
|
+
console.warn(
|
|
1671
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1672
|
+
);
|
|
1673
|
+
}
|
|
1674
|
+
if (autoVacuum !== 0) return null;
|
|
1675
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1676
|
+
}
|
|
1677
|
+
async function reclaimSessionDbSpace(input) {
|
|
1678
|
+
const {
|
|
1679
|
+
dbPath,
|
|
1680
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1681
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1682
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1683
|
+
now = () => performance.now(),
|
|
1684
|
+
hasLiveTurn = false,
|
|
1685
|
+
hasProvenanceAnomaly = false
|
|
1686
|
+
} = input;
|
|
1687
|
+
let sqlite;
|
|
1688
|
+
try {
|
|
1689
|
+
sqlite = await import("node:sqlite");
|
|
1690
|
+
} catch (err) {
|
|
1691
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1692
|
+
console.warn(
|
|
1693
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1694
|
+
);
|
|
1695
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1696
|
+
}
|
|
1697
|
+
const { DatabaseSync } = sqlite;
|
|
1698
|
+
let db;
|
|
1699
|
+
try {
|
|
1700
|
+
db = new DatabaseSync(dbPath);
|
|
1701
|
+
db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
|
|
1702
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1703
|
+
if (autoVacuum === 0) {
|
|
1704
|
+
if (hasLiveTurn || hasProvenanceAnomaly) {
|
|
1705
|
+
const detail = [
|
|
1706
|
+
...hasLiveTurn ? ["a session turn is live"] : [],
|
|
1707
|
+
...hasProvenanceAnomaly ? ["session-DB provenance is anomalous"] : []
|
|
1708
|
+
].join(" and ");
|
|
1709
|
+
console.warn(`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${detail}`);
|
|
1710
|
+
return {
|
|
1711
|
+
ok: false,
|
|
1712
|
+
skipped: "full-vacuum-blocked",
|
|
1713
|
+
detail
|
|
1714
|
+
};
|
|
1715
|
+
}
|
|
1716
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1717
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1718
|
+
if (skipReason !== null) {
|
|
1719
|
+
console.warn(
|
|
1720
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1721
|
+
);
|
|
1722
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1723
|
+
}
|
|
1724
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1725
|
+
const freelistBefore = readFreelistCount(db);
|
|
1726
|
+
const startedAt = now();
|
|
1727
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1728
|
+
db.exec("VACUUM");
|
|
1729
|
+
const afterBytes = readLogicalBytes(db);
|
|
1730
|
+
const freelistAfter = readFreelistCount(db);
|
|
1731
|
+
const checkpointStartedAt = now();
|
|
1732
|
+
const checkpoint = readCheckpointResult(db);
|
|
1733
|
+
const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
|
|
1734
|
+
return {
|
|
1735
|
+
ok: true,
|
|
1736
|
+
mode: "convert",
|
|
1737
|
+
beforeBytes,
|
|
1738
|
+
afterBytes,
|
|
1739
|
+
freelistBefore,
|
|
1740
|
+
freelistAfter,
|
|
1741
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1742
|
+
stoppedBy: "freelist-drained",
|
|
1743
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1744
|
+
checkpointElapsedMs,
|
|
1745
|
+
checkpoint
|
|
1746
|
+
};
|
|
1747
|
+
}
|
|
1748
|
+
if (autoVacuum === 2) {
|
|
1749
|
+
if (hasLiveTurn) {
|
|
1750
|
+
const detail = "a session turn is live";
|
|
1751
|
+
console.warn(
|
|
1752
|
+
`[reclaimSessionDbSpace] skipping incremental reclaim of ${dbPath}: ${detail}`
|
|
1753
|
+
);
|
|
1754
|
+
return { ok: false, skipped: "live-turn", detail };
|
|
1755
|
+
}
|
|
1756
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1757
|
+
const freelistBefore = readFreelistCount(db);
|
|
1758
|
+
const startedAt = now();
|
|
1759
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1760
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1761
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1762
|
+
let pagesAttempted = 0;
|
|
1763
|
+
let freelistAfter = freelistBefore;
|
|
1764
|
+
let stoppedBy = "freelist-drained";
|
|
1765
|
+
while (freelistAfter > 0) {
|
|
1766
|
+
if (pagesAttempted >= pageLimit) {
|
|
1767
|
+
stoppedBy = "page-ceiling";
|
|
1768
|
+
break;
|
|
1769
|
+
}
|
|
1770
|
+
if (now() - startedAt >= budget) {
|
|
1771
|
+
stoppedBy = "time-budget";
|
|
1772
|
+
break;
|
|
1773
|
+
}
|
|
1774
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1775
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1776
|
+
pagesAttempted += pages;
|
|
1777
|
+
freelistAfter = readFreelistCount(db);
|
|
1778
|
+
}
|
|
1779
|
+
const afterBytes = readLogicalBytes(db);
|
|
1780
|
+
const checkpointStartedAt = now();
|
|
1781
|
+
const checkpoint = readCheckpointResult(db);
|
|
1782
|
+
const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
|
|
1783
|
+
return {
|
|
1784
|
+
ok: true,
|
|
1785
|
+
mode: "incremental",
|
|
1786
|
+
beforeBytes,
|
|
1787
|
+
afterBytes,
|
|
1788
|
+
freelistBefore,
|
|
1789
|
+
freelistAfter,
|
|
1790
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1791
|
+
stoppedBy,
|
|
1792
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1793
|
+
checkpointElapsedMs,
|
|
1794
|
+
checkpoint
|
|
1795
|
+
};
|
|
1796
|
+
}
|
|
1797
|
+
console.warn(
|
|
1798
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1799
|
+
);
|
|
1800
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1801
|
+
} catch (err) {
|
|
1802
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1803
|
+
return {
|
|
1804
|
+
ok: false,
|
|
1805
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1806
|
+
detail: errorMessage(err)
|
|
1807
|
+
};
|
|
1808
|
+
} finally {
|
|
1809
|
+
db?.close();
|
|
1810
|
+
}
|
|
1811
|
+
}
|
|
1812
|
+
|
|
1813
|
+
// src/lib/opencode/dependency-install-diagnostic.ts
|
|
1814
|
+
import { lstatSync, realpathSync } from "node:fs";
|
|
1815
|
+
import { basename, dirname as dirname3, isAbsolute, join as join2, relative, resolve, sep } from "node:path";
|
|
1816
|
+
var DEPENDENCY_INSTALL_FAILURE_MESSAGE = "background dependency install failed";
|
|
1817
|
+
var MAX_DISTINCT_DIAGNOSTICS = 32;
|
|
1818
|
+
var EFFECT = "OpenCode could not install background dependencies for this scope; its plugins and tools may be unavailable";
|
|
1819
|
+
function readOpenCodeLogField(line, key) {
|
|
1820
|
+
const match = line.match(new RegExp(`(?:^|\\s)${key}=(?:"((?:\\\\.|[^"])*)"|(\\S+))`));
|
|
1821
|
+
if (!match) return void 0;
|
|
1822
|
+
if (match[1] === void 0) return match[2];
|
|
1823
|
+
return match[1].replace(/\\(["\\n])/g, (_match, escaped) => {
|
|
1824
|
+
return escaped === "n" ? "\n" : escaped;
|
|
1825
|
+
});
|
|
1826
|
+
}
|
|
1827
|
+
function parseTrigger(line) {
|
|
1828
|
+
if (readOpenCodeLogField(line, "message") !== DEPENDENCY_INSTALL_FAILURE_MESSAGE) {
|
|
1829
|
+
return null;
|
|
1830
|
+
}
|
|
1831
|
+
const error2 = readOpenCodeLogField(line, "error");
|
|
1832
|
+
if (!error2 || !/\bEACCES\b/.test(error2)) return null;
|
|
1833
|
+
const errnoMatch = error2.match(/\bEACCES:\s*permission denied,\s*([a-z_]+)\s+'([^']*)'/);
|
|
1834
|
+
const capturedPath = errnoMatch?.[2];
|
|
1835
|
+
return {
|
|
1836
|
+
dir: readOpenCodeLogField(line, "dir"),
|
|
1837
|
+
op: errnoMatch?.[1],
|
|
1838
|
+
path: capturedPath && isAbsolute(capturedPath) ? capturedPath : void 0
|
|
1839
|
+
};
|
|
1840
|
+
}
|
|
1841
|
+
function getErrorCode(error2) {
|
|
1842
|
+
if (typeof error2 === "object" && error2 !== null && "code" in error2) {
|
|
1843
|
+
const code = error2.code;
|
|
1844
|
+
if (typeof code === "string") return code;
|
|
1845
|
+
}
|
|
1846
|
+
return "UNKNOWN";
|
|
1847
|
+
}
|
|
1848
|
+
function isWithin(path, root) {
|
|
1849
|
+
const candidate = resolve(path);
|
|
1850
|
+
const resolvedRoot = resolve(root);
|
|
1851
|
+
return candidate === resolvedRoot || candidate.startsWith(`${resolvedRoot}${sep}`);
|
|
1852
|
+
}
|
|
1853
|
+
function rootSpecs(context) {
|
|
1854
|
+
return [
|
|
1855
|
+
{ token: "~", configured: resolve(context.homeDir) },
|
|
1856
|
+
{ token: "$WORKSPACE", configured: resolve(context.cwd) }
|
|
1857
|
+
];
|
|
1858
|
+
}
|
|
1859
|
+
function resolveRoots(context) {
|
|
1860
|
+
const resolved = [];
|
|
1861
|
+
const unresolved = [];
|
|
1862
|
+
for (const spec of rootSpecs(context)) {
|
|
1863
|
+
try {
|
|
1864
|
+
resolved.push({ ...spec, real: realpathSync(spec.configured) });
|
|
1865
|
+
} catch (_error) {
|
|
1866
|
+
unresolved.push(spec);
|
|
1867
|
+
}
|
|
1868
|
+
}
|
|
1869
|
+
return { resolved, unresolved };
|
|
1870
|
+
}
|
|
1871
|
+
function resolveNearestAncestor(failingPath, remaining, error2) {
|
|
1872
|
+
let ancestor = dirname3(failingPath);
|
|
1873
|
+
const suffix = [basename(failingPath), ...remaining];
|
|
1874
|
+
while (ancestor) {
|
|
1875
|
+
try {
|
|
1876
|
+
return {
|
|
1877
|
+
path: resolve(realpathSync(ancestor), ...suffix),
|
|
1878
|
+
failure: { path: resolve(failingPath), error: error2 }
|
|
1879
|
+
};
|
|
1880
|
+
} catch (ancestorError) {
|
|
1881
|
+
if (ancestor === dirname3(ancestor) || !getErrorCode(ancestorError).startsWith("E")) {
|
|
1882
|
+
return null;
|
|
1883
|
+
}
|
|
1884
|
+
suffix.unshift(basename(ancestor));
|
|
1885
|
+
ancestor = dirname3(ancestor);
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
return null;
|
|
1889
|
+
}
|
|
1890
|
+
function resolveExistingPath(absolutePath) {
|
|
1891
|
+
if (!isAbsolute(absolutePath)) return null;
|
|
1892
|
+
const candidate = resolve(absolutePath);
|
|
1893
|
+
const remaining = [];
|
|
1894
|
+
let current = candidate;
|
|
1895
|
+
while (current) {
|
|
1896
|
+
try {
|
|
1897
|
+
const stat2 = lstatSync(current);
|
|
1898
|
+
if (stat2.isSymbolicLink()) {
|
|
1899
|
+
if (remaining.length === 0) {
|
|
1900
|
+
return { path: join2(realpathSync(dirname3(current)), basename(current)) };
|
|
1901
|
+
}
|
|
1902
|
+
return { path: resolve(realpathSync(current), ...remaining) };
|
|
1903
|
+
}
|
|
1904
|
+
return { path: resolve(realpathSync(current), ...remaining) };
|
|
1905
|
+
} catch (error2) {
|
|
1906
|
+
const code = getErrorCode(error2);
|
|
1907
|
+
if (code !== "ENOENT" && code !== "ENOTDIR") {
|
|
1908
|
+
return resolveNearestAncestor(current, remaining, error2);
|
|
1909
|
+
}
|
|
1910
|
+
}
|
|
1911
|
+
if (current === dirname3(current)) return null;
|
|
1912
|
+
remaining.unshift(basename(current));
|
|
1913
|
+
current = dirname3(current);
|
|
1914
|
+
}
|
|
1915
|
+
return null;
|
|
1916
|
+
}
|
|
1917
|
+
function tokenise(absolutePath, roots) {
|
|
1918
|
+
const pathResolution = resolveExistingPath(absolutePath);
|
|
1919
|
+
if (!pathResolution) return null;
|
|
1920
|
+
const root = roots.reduce((selected, candidate) => {
|
|
1921
|
+
if (!isWithin(pathResolution.path, candidate.real)) return selected;
|
|
1922
|
+
if (!selected || candidate.real.length > selected.real.length || candidate.real.length === selected.real.length && candidate.token === "$WORKSPACE") {
|
|
1923
|
+
return candidate;
|
|
1924
|
+
}
|
|
1925
|
+
return selected;
|
|
1926
|
+
}, null);
|
|
1927
|
+
if (!root) return null;
|
|
1928
|
+
const suffix = relative(root.real, pathResolution.path);
|
|
1929
|
+
return {
|
|
1930
|
+
value: suffix ? `${root.token}/${suffix.split(sep).join("/")}` : root.token,
|
|
1931
|
+
root,
|
|
1932
|
+
resolutionFailure: pathResolution.failure
|
|
1933
|
+
};
|
|
1934
|
+
}
|
|
1935
|
+
function classifyScope(dir, roots) {
|
|
1936
|
+
if (!dir) return "other";
|
|
1937
|
+
const tokenisedDir = tokenise(dir, roots.resolved);
|
|
1938
|
+
if (tokenisedDir) return tokenisedDir.root.token === "$WORKSPACE" ? "workspace" : "global";
|
|
1939
|
+
for (const spec of roots.unresolved) {
|
|
1940
|
+
if (isWithin(dir, spec.configured)) {
|
|
1941
|
+
return spec.token === "$WORKSPACE" ? "workspace" : "global";
|
|
1942
|
+
}
|
|
1943
|
+
}
|
|
1944
|
+
return "other";
|
|
1945
|
+
}
|
|
1946
|
+
function unresolvedRootForPath(path, unresolved) {
|
|
1947
|
+
return unresolved.find((spec) => isWithin(path, spec.configured));
|
|
1948
|
+
}
|
|
1949
|
+
function statType(stat2) {
|
|
1950
|
+
if (stat2.isDirectory()) return "d";
|
|
1951
|
+
if (stat2.isFile()) return "f";
|
|
1952
|
+
if (stat2.isSymbolicLink()) return "l";
|
|
1953
|
+
return "o";
|
|
1954
|
+
}
|
|
1955
|
+
function renderStat(name, stat2) {
|
|
1956
|
+
const mode = (stat2.mode & 4095).toString(8).padStart(4, "0");
|
|
1957
|
+
return `${name}:${statType(stat2)}:${stat2.uid}:${stat2.gid}:${mode}`;
|
|
1958
|
+
}
|
|
1959
|
+
function renderStatError(name, error2) {
|
|
1960
|
+
const code = getErrorCode(error2);
|
|
1961
|
+
return `${name}:${renderErrorCode(code)}`;
|
|
1962
|
+
}
|
|
1963
|
+
function renderErrorCode(code) {
|
|
1964
|
+
if (code === "ENOENT") return "missing";
|
|
1965
|
+
return code.startsWith("E") ? code : `E${code}`;
|
|
1966
|
+
}
|
|
1967
|
+
function inspectChain(rootReal, rootToken, absolutePath, resolutionFailure) {
|
|
1968
|
+
const target = resolve(absolutePath);
|
|
1969
|
+
const relativePath = relative(rootReal, target);
|
|
1970
|
+
const components = relativePath ? relativePath.split(sep) : [];
|
|
1971
|
+
const entries = [];
|
|
1972
|
+
let current = rootReal;
|
|
1973
|
+
let status2 = "complete";
|
|
1974
|
+
let finalStat;
|
|
1975
|
+
for (const name of [rootToken, ...components]) {
|
|
1976
|
+
if (name !== rootToken) current = join2(current, name);
|
|
1977
|
+
if (resolutionFailure && resolve(current) === resolve(resolutionFailure.path)) {
|
|
1978
|
+
entries.push(renderStatError(name, resolutionFailure.error));
|
|
1979
|
+
status2 = "stat_failed";
|
|
1980
|
+
break;
|
|
1981
|
+
}
|
|
1982
|
+
try {
|
|
1983
|
+
const stat2 = lstatSync(current);
|
|
1984
|
+
entries.push(renderStat(name, stat2));
|
|
1985
|
+
finalStat = stat2;
|
|
1986
|
+
} catch (error2) {
|
|
1987
|
+
entries.push(renderStatError(name, error2));
|
|
1988
|
+
if (getErrorCode(error2) !== "ENOENT") status2 = "stat_failed";
|
|
1989
|
+
break;
|
|
1990
|
+
}
|
|
1991
|
+
}
|
|
1992
|
+
let symlinkTarget;
|
|
1993
|
+
let symlinkMeta;
|
|
1994
|
+
if (finalStat?.isSymbolicLink() && entries.length === components.length + 1) {
|
|
1995
|
+
try {
|
|
1996
|
+
symlinkTarget = realpathSync(current);
|
|
1997
|
+
try {
|
|
1998
|
+
symlinkMeta = renderStat("", lstatSync(symlinkTarget)).slice(1);
|
|
1999
|
+
} catch (error2) {
|
|
2000
|
+
symlinkMeta = renderErrorCode(getErrorCode(error2));
|
|
2001
|
+
status2 = "stat_failed";
|
|
2002
|
+
}
|
|
2003
|
+
} catch (error2) {
|
|
2004
|
+
symlinkTarget = "<unresolved>";
|
|
2005
|
+
symlinkMeta = renderErrorCode(getErrorCode(error2));
|
|
2006
|
+
status2 = "stat_failed";
|
|
2007
|
+
}
|
|
2008
|
+
}
|
|
2009
|
+
return { status: status2, chain: entries.join(","), symlinkTarget, symlinkMeta };
|
|
2010
|
+
}
|
|
2011
|
+
function buildDiagnostic(line, context) {
|
|
2012
|
+
const trigger = parseTrigger(line);
|
|
2013
|
+
if (!trigger) return null;
|
|
2014
|
+
const roots = resolveRoots(context);
|
|
2015
|
+
const scope = classifyScope(trigger.dir, roots);
|
|
2016
|
+
const pathSource = trigger.path ? "error" : trigger.dir ? "dir" : "none";
|
|
2017
|
+
const inspectedPath = trigger.path ?? trigger.dir;
|
|
2018
|
+
const microvmId = context.microvmId?.trim() || "unknown";
|
|
2019
|
+
const euid = context.euid === void 0 ? "unavailable" : String(context.euid);
|
|
2020
|
+
const egid = context.egid === void 0 ? "unavailable" : String(context.egid);
|
|
2021
|
+
const op = trigger.op ?? "unknown";
|
|
2022
|
+
if (!inspectedPath) {
|
|
2023
|
+
return {
|
|
2024
|
+
event: `OPENCODE-DEPENDENCY-INSTALL-EACCES: status=no_path path_source=${pathSource} scope=${scope} microvm_id=${microvmId} op=${op} path=unknown cli_euid=${euid} cli_egid=${egid} effect="${EFFECT}" chain=unavailable`,
|
|
2025
|
+
localDetail: null
|
|
2026
|
+
};
|
|
2027
|
+
}
|
|
2028
|
+
const tokenisedPath = tokenise(inspectedPath, roots.resolved);
|
|
2029
|
+
if (!tokenisedPath) {
|
|
2030
|
+
const unresolvedRoot = unresolvedRootForPath(inspectedPath, roots.unresolved);
|
|
2031
|
+
if (unresolvedRoot) {
|
|
2032
|
+
return {
|
|
2033
|
+
event: `OPENCODE-DEPENDENCY-INSTALL-EACCES: status=root_unresolved path_source=${pathSource} scope=${scope} microvm_id=${microvmId} op=${op} path=unknown cli_euid=${euid} cli_egid=${egid} effect="${EFFECT}" chain=unavailable`,
|
|
2034
|
+
localDetail: null
|
|
2035
|
+
};
|
|
2036
|
+
}
|
|
2037
|
+
return {
|
|
2038
|
+
event: `OPENCODE-DEPENDENCY-INSTALL-EACCES: status=outside_roots path_source=${pathSource} scope=${scope} microvm_id=${microvmId} op=${op} path=<outside-approved-roots> cli_euid=${euid} cli_egid=${egid} effect="${EFFECT}" chain=unavailable`,
|
|
2039
|
+
localDetail: `OPENCODE-DEPENDENCY-INSTALL-EACCES (local only): outside approved roots: ${inspectedPath}`
|
|
2040
|
+
};
|
|
2041
|
+
}
|
|
2042
|
+
const inspection = inspectChain(
|
|
2043
|
+
tokenisedPath.root.real,
|
|
2044
|
+
tokenisedPath.root.token,
|
|
2045
|
+
inspectedPath,
|
|
2046
|
+
tokenisedPath.resolutionFailure
|
|
2047
|
+
);
|
|
2048
|
+
const optionalSymlinkFields = inspection.symlinkTarget ? ` symlink_target=${tokenise(inspection.symlinkTarget, roots.resolved)?.value ?? "<outside-approved-roots>"}` + (inspection.symlinkMeta ? ` symlink_meta=${inspection.symlinkMeta}` : "") : "";
|
|
2049
|
+
return {
|
|
2050
|
+
event: `OPENCODE-DEPENDENCY-INSTALL-EACCES: status=${inspection.status} path_source=${pathSource} scope=${scope} microvm_id=${microvmId} op=${op} path=${tokenisedPath.value} cli_euid=${euid} cli_egid=${egid} effect="${EFFECT}" chain=${inspection.chain}${optionalSymlinkFields}`,
|
|
2051
|
+
localDetail: null
|
|
2052
|
+
};
|
|
2053
|
+
}
|
|
2054
|
+
function createDependencyInstallDiagnosticReporter({
|
|
2055
|
+
context,
|
|
2056
|
+
emitEvent: emitEvent2,
|
|
2057
|
+
emitLocal
|
|
2058
|
+
}) {
|
|
2059
|
+
const emittedEvents = /* @__PURE__ */ new Set();
|
|
2060
|
+
return (line) => {
|
|
2061
|
+
const diagnostic = buildDiagnostic(line, context);
|
|
2062
|
+
if (!diagnostic || emittedEvents.has(diagnostic.event)) return;
|
|
2063
|
+
if (emittedEvents.size >= MAX_DISTINCT_DIAGNOSTICS) return;
|
|
2064
|
+
emittedEvents.add(diagnostic.event);
|
|
2065
|
+
emitEvent2(diagnostic.event);
|
|
2066
|
+
if (diagnostic.localDetail) emitLocal(diagnostic.localDetail);
|
|
2067
|
+
};
|
|
2068
|
+
}
|
|
2069
|
+
|
|
1301
2070
|
// src/lib/runner-activity-telemetry.ts
|
|
1302
|
-
var
|
|
2071
|
+
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
2072
|
+
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
2073
|
+
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
2074
|
+
"cli.session-cleanup": /* @__PURE__ */ new Set(["info", "warn", "error"])
|
|
2075
|
+
};
|
|
1303
2076
|
var SEVERITY_BY_LEVEL = {
|
|
2077
|
+
info: "info",
|
|
1304
2078
|
warn: "warning",
|
|
1305
2079
|
error: "error"
|
|
1306
2080
|
};
|
|
@@ -1310,26 +2084,113 @@ function parseOpenCodeLogLine(line) {
|
|
|
1310
2084
|
if (!levelMatch) return null;
|
|
1311
2085
|
const level = levelMatch[1].toUpperCase();
|
|
1312
2086
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1313
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1314
|
-
|
|
2087
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
2088
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
2089
|
+
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
2090
|
+
const parsed = {
|
|
2091
|
+
level: level === "WARN" ? "warn" : "error",
|
|
2092
|
+
sessionID: sessionMatch?.[1],
|
|
2093
|
+
messageID: messageIDMatch?.[1]
|
|
2094
|
+
};
|
|
2095
|
+
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
2096
|
+
if (message !== void 0) parsed.message = message;
|
|
2097
|
+
return parsed;
|
|
1315
2098
|
}
|
|
1316
2099
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1317
|
-
|
|
2100
|
+
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
2101
|
+
function sanitiseDependencyInstallWarning(line) {
|
|
2102
|
+
const withoutDirectory = line.replace(
|
|
2103
|
+
/((?:^|\s)dir=)(?:"(?:\\.|[^"])*"|\/[^\s)"']+|\S+)/g,
|
|
2104
|
+
"$1<path>"
|
|
2105
|
+
);
|
|
2106
|
+
return withoutDirectory.replace(
|
|
2107
|
+
/((?:^|\s)error=)"((?:\\.|[^"])*)"/,
|
|
2108
|
+
(_match, prefix, error2) => `${prefix}"${error2.replace(/'\/[^']*'/g, "'<path>'")}"`
|
|
2109
|
+
);
|
|
2110
|
+
}
|
|
2111
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure, onDependencyInstallFailure) {
|
|
1318
2112
|
let buffer = Buffer.alloc(0);
|
|
2113
|
+
const repeatStates = /* @__PURE__ */ new Map();
|
|
2114
|
+
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
2115
|
+
const metadata = {
|
|
2116
|
+
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
2117
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
2118
|
+
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
2119
|
+
};
|
|
2120
|
+
forwardRunnerActivity(
|
|
2121
|
+
{
|
|
2122
|
+
level: parsed.level,
|
|
2123
|
+
error: parsed.message === DEPENDENCY_INSTALL_FAILURE_MESSAGE ? sanitiseDependencyInstallWarning(line) : line,
|
|
2124
|
+
metadata: Object.keys(metadata).length > 0 ? metadata : void 0,
|
|
2125
|
+
source: "opencode"
|
|
2126
|
+
},
|
|
2127
|
+
getContext()
|
|
2128
|
+
);
|
|
2129
|
+
};
|
|
2130
|
+
const expireRepeatStates = (now, currentKey) => {
|
|
2131
|
+
for (const [key, state] of repeatStates) {
|
|
2132
|
+
if (key === currentKey || now - state.lastForwardedAt < REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
2133
|
+
continue;
|
|
2134
|
+
}
|
|
2135
|
+
if (state.repeatsSuppressed > 0) {
|
|
2136
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
2137
|
+
}
|
|
2138
|
+
repeatStates.delete(key);
|
|
2139
|
+
}
|
|
2140
|
+
};
|
|
2141
|
+
const forwardWithRepeatSuppression = (line, parsed) => {
|
|
2142
|
+
const now = Date.now();
|
|
2143
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
2144
|
+
expireRepeatStates(now, key);
|
|
2145
|
+
if (key === void 0) {
|
|
2146
|
+
forwardLine(line, parsed);
|
|
2147
|
+
return;
|
|
2148
|
+
}
|
|
2149
|
+
const previous = repeatStates.get(key);
|
|
2150
|
+
if (!previous || now - previous.lastForwardedAt >= REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
2151
|
+
forwardLine(line, parsed, previous?.repeatsSuppressed ?? 0);
|
|
2152
|
+
repeatStates.set(key, {
|
|
2153
|
+
line,
|
|
2154
|
+
parsed,
|
|
2155
|
+
lastForwardedAt: now,
|
|
2156
|
+
repeatsSuppressed: 0
|
|
2157
|
+
});
|
|
2158
|
+
return;
|
|
2159
|
+
}
|
|
2160
|
+
previous.line = line;
|
|
2161
|
+
previous.parsed = parsed;
|
|
2162
|
+
previous.repeatsSuppressed++;
|
|
2163
|
+
};
|
|
1319
2164
|
const flushLine = (line) => {
|
|
1320
2165
|
const parsed = parseOpenCodeLogLine(line);
|
|
2166
|
+
const isDiskFailure = isSessionStoreWriteFailure(line);
|
|
2167
|
+
const cause = sessionStoreCauseToken(line);
|
|
2168
|
+
if (onSessionStoreWriteFailure && (isDiskFailure || cause !== null)) {
|
|
2169
|
+
try {
|
|
2170
|
+
onSessionStoreWriteFailure({
|
|
2171
|
+
isDiskFailure,
|
|
2172
|
+
sessionId: parsed?.sessionID ?? null,
|
|
2173
|
+
cause
|
|
2174
|
+
});
|
|
2175
|
+
} catch (err) {
|
|
2176
|
+
reportTelemetryDiagnostic(
|
|
2177
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
2178
|
+
);
|
|
2179
|
+
}
|
|
2180
|
+
}
|
|
1321
2181
|
if (!parsed) return;
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
2182
|
+
forwardWithRepeatSuppression(line, parsed);
|
|
2183
|
+
if (onDependencyInstallFailure && parsed.message === DEPENDENCY_INSTALL_FAILURE_MESSAGE) {
|
|
2184
|
+
try {
|
|
2185
|
+
onDependencyInstallFailure(line);
|
|
2186
|
+
} catch (err) {
|
|
2187
|
+
reportTelemetryDiagnostic(
|
|
2188
|
+
`Runner activity telemetry could not record dependency-install failure diagnostic: ${err instanceof Error ? err.message : String(err)}`
|
|
2189
|
+
);
|
|
2190
|
+
}
|
|
2191
|
+
}
|
|
1331
2192
|
};
|
|
1332
|
-
|
|
2193
|
+
const forward = ((chunk) => {
|
|
1333
2194
|
buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf-8")]);
|
|
1334
2195
|
let newlineIndex;
|
|
1335
2196
|
while ((newlineIndex = buffer.indexOf(10)) !== -1) {
|
|
@@ -1340,31 +2201,68 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1340
2201
|
flushLine(buffer.toString("utf-8"));
|
|
1341
2202
|
buffer = Buffer.alloc(0);
|
|
1342
2203
|
}
|
|
2204
|
+
});
|
|
2205
|
+
forward.flush = () => {
|
|
2206
|
+
for (const state of repeatStates.values()) {
|
|
2207
|
+
if (state.repeatsSuppressed > 0) {
|
|
2208
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
2209
|
+
}
|
|
2210
|
+
}
|
|
2211
|
+
repeatStates.clear();
|
|
1343
2212
|
};
|
|
2213
|
+
return forward;
|
|
1344
2214
|
}
|
|
1345
2215
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1346
2216
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1347
2217
|
var MAX_METADATA_ENTRIES = 20;
|
|
1348
|
-
var TRUNCATION_MARKER = "\u2026";
|
|
1349
|
-
function redact(message) {
|
|
1350
|
-
return message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1351
|
-
}
|
|
1352
2218
|
function truncate(message) {
|
|
1353
2219
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1354
|
-
|
|
2220
|
+
let dropped = message.length - MAX_MESSAGE_LENGTH;
|
|
2221
|
+
let marker = `\u2026${dropped} chars omitted\u2026`;
|
|
2222
|
+
let previousDropped = -1;
|
|
2223
|
+
while (dropped !== previousDropped) {
|
|
2224
|
+
previousDropped = dropped;
|
|
2225
|
+
const nextDropped = message.length - MAX_MESSAGE_LENGTH + marker.length;
|
|
2226
|
+
dropped = nextDropped;
|
|
2227
|
+
marker = `\u2026${dropped} chars omitted\u2026`;
|
|
2228
|
+
}
|
|
2229
|
+
const remaining = MAX_MESSAGE_LENGTH - marker.length;
|
|
2230
|
+
const headLength = Math.ceil(remaining * 0.6);
|
|
2231
|
+
const tailLength = remaining - headLength;
|
|
2232
|
+
const effectMatch = message.startsWith("OPENCODE-DEPENDENCY-INSTALL-EACCES:") ? message.match(/ effect="[^"]*"/) : null;
|
|
2233
|
+
if (effectMatch?.index !== void 0) {
|
|
2234
|
+
const effectEnd = effectMatch.index + effectMatch[0].length;
|
|
2235
|
+
if (effectEnd > headLength && effectEnd < message.length) {
|
|
2236
|
+
const effectLength = effectMatch[0].length;
|
|
2237
|
+
const nonEffectLength = remaining - effectLength;
|
|
2238
|
+
const preservedTailLength = Math.min(
|
|
2239
|
+
message.length - effectEnd,
|
|
2240
|
+
nonEffectLength - Math.min(effectMatch.index, Math.ceil(nonEffectLength * 0.6))
|
|
2241
|
+
);
|
|
2242
|
+
const preservedHeadLength = Math.min(
|
|
2243
|
+
effectMatch.index,
|
|
2244
|
+
nonEffectLength - preservedTailLength
|
|
2245
|
+
);
|
|
2246
|
+
return message.slice(0, preservedHeadLength) + marker + message.slice(effectMatch.index, effectEnd) + message.slice(message.length - preservedTailLength);
|
|
2247
|
+
}
|
|
2248
|
+
}
|
|
2249
|
+
return message.slice(0, headLength) + marker + message.slice(message.length - tailLength);
|
|
1355
2250
|
}
|
|
1356
2251
|
function sanitiseMetadata(metadata) {
|
|
1357
2252
|
if (!metadata) return {};
|
|
1358
2253
|
const entries = Object.entries(metadata).slice(0, MAX_METADATA_ENTRIES);
|
|
1359
2254
|
if (Object.keys(metadata).length > entries.length) {
|
|
1360
|
-
|
|
1361
|
-
`
|
|
2255
|
+
reportTelemetryDiagnostic(
|
|
2256
|
+
`Runner activity telemetry dropped ${Object.keys(metadata).length - entries.length} metadata entries; those entries were not sent`
|
|
1362
2257
|
);
|
|
1363
2258
|
}
|
|
1364
2259
|
const sanitised = [];
|
|
1365
2260
|
for (const [key, value] of entries) {
|
|
1366
2261
|
if (typeof value === "string") {
|
|
1367
|
-
sanitised.push([
|
|
2262
|
+
sanitised.push([
|
|
2263
|
+
key,
|
|
2264
|
+
truncate(redactDiagnosticText(value).slice(0, MAX_METADATA_VALUE_LENGTH))
|
|
2265
|
+
]);
|
|
1368
2266
|
} else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
|
|
1369
2267
|
sanitised.push([key, value]);
|
|
1370
2268
|
}
|
|
@@ -1382,8 +2280,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1382
2280
|
}
|
|
1383
2281
|
if (now - window.windowStartedAt >= RATE_LIMIT_WINDOW_MS) {
|
|
1384
2282
|
if (window.windowDroppedCount > 0) {
|
|
1385
|
-
|
|
1386
|
-
`
|
|
2283
|
+
reportTelemetryDiagnostic(
|
|
2284
|
+
`Runner activity telemetry rate cap reached: dropped ${window.windowDroppedCount} ${window.windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (limit ${RATE_LIMIT_MAX_EVENTS}/min); those entries were not sent for source "${source}"`
|
|
1387
2285
|
);
|
|
1388
2286
|
}
|
|
1389
2287
|
window.windowStartedAt = now;
|
|
@@ -1393,8 +2291,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1393
2291
|
if (window.windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
1394
2292
|
window.windowDroppedCount++;
|
|
1395
2293
|
if (window.windowDroppedCount === 1) {
|
|
1396
|
-
|
|
1397
|
-
`
|
|
2294
|
+
reportTelemetryDiagnostic(
|
|
2295
|
+
`Runner activity telemetry rate cap reached (${RATE_LIMIT_MAX_EVENTS} entries per ${RATE_LIMIT_WINDOW_MS / 1e3}s limit; further entries were not sent for source "${source}"`
|
|
1398
2296
|
);
|
|
1399
2297
|
}
|
|
1400
2298
|
return false;
|
|
@@ -1404,12 +2302,12 @@ function admitUnderRateLimit(source, now) {
|
|
|
1404
2302
|
}
|
|
1405
2303
|
function forwardRunnerActivity(entry, context) {
|
|
1406
2304
|
try {
|
|
1407
|
-
if (!FORWARDED_LEVELS.has(entry.level)) return;
|
|
1408
|
-
if (!context.agentId || !context.authHeader) return;
|
|
1409
2305
|
const source = entry.source ?? "cli.run";
|
|
2306
|
+
if (!FORWARDED_LEVELS_BY_SOURCE[source].has(entry.level)) return;
|
|
2307
|
+
if (!context.agentId || !context.authHeader) return;
|
|
1410
2308
|
if (!admitUnderRateLimit(source, Date.now())) return;
|
|
1411
2309
|
const rawMessage = entry.error ?? entry.message ?? "";
|
|
1412
|
-
const message = truncate(
|
|
2310
|
+
const message = truncate(redactDiagnosticText(rawMessage));
|
|
1413
2311
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1414
2312
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1415
2313
|
message,
|
|
@@ -1417,18 +2315,18 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1417
2315
|
agentId: context.agentId
|
|
1418
2316
|
});
|
|
1419
2317
|
} catch (err) {
|
|
1420
|
-
|
|
1421
|
-
`
|
|
2318
|
+
reportTelemetryDiagnostic(
|
|
2319
|
+
`Runner activity telemetry failed to forward runner activity: ${err instanceof Error ? err.message : String(err)}`
|
|
1422
2320
|
);
|
|
1423
2321
|
}
|
|
1424
2322
|
}
|
|
1425
2323
|
|
|
1426
2324
|
// src/lib/opencode/session-db-recovery-report.ts
|
|
1427
2325
|
import { readFileSync as readFileSync2, unlinkSync } from "node:fs";
|
|
1428
|
-
import { join as
|
|
2326
|
+
import { join as join3 } from "node:path";
|
|
1429
2327
|
function sessionDbRecoveryReportPath(homeDir, env) {
|
|
1430
2328
|
const override = env.EVIDENT_SESSION_DB_RECOVERY_REPORT?.trim();
|
|
1431
|
-
return override ||
|
|
2329
|
+
return override || join3(homeDir, ".local", "state", "evident", "session-db-recovery.jsonl");
|
|
1432
2330
|
}
|
|
1433
2331
|
function drainSessionDbRecoveryReport({
|
|
1434
2332
|
homeDir,
|
|
@@ -1558,12 +2456,23 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1558
2456
|
metadata: withoutContractFields(record),
|
|
1559
2457
|
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1560
2458
|
};
|
|
1561
|
-
case "session_db_boot_refused":
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
2459
|
+
case "session_db_boot_refused": {
|
|
2460
|
+
const message = (() => {
|
|
2461
|
+
switch (record.reason) {
|
|
2462
|
+
case "local_discard_failed":
|
|
2463
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
|
|
2464
|
+
case "replica_separation_unproven":
|
|
2465
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
|
|
2466
|
+
case "verification_timeout_before_separation":
|
|
2467
|
+
return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2468
|
+
case "verification_timeout_during_separation":
|
|
2469
|
+
return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2470
|
+
default:
|
|
2471
|
+
return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
|
|
2472
|
+
}
|
|
2473
|
+
})();
|
|
2474
|
+
return { level, metadata: withoutContractFields(record), message };
|
|
2475
|
+
}
|
|
1567
2476
|
case "schema_provenance_mismatch":
|
|
1568
2477
|
return {
|
|
1569
2478
|
level,
|
|
@@ -1690,7 +2599,7 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
1690
2599
|
if (health.healthy || health.authFailed) {
|
|
1691
2600
|
return health;
|
|
1692
2601
|
}
|
|
1693
|
-
await new Promise((
|
|
2602
|
+
await new Promise((resolve5) => setTimeout(resolve5, 1e3));
|
|
1694
2603
|
}
|
|
1695
2604
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
1696
2605
|
}
|
|
@@ -1701,15 +2610,53 @@ async function waitForOpenCode2Health(port, password, timeoutMs = 3e4) {
|
|
|
1701
2610
|
if (health.healthy || health.authFailed) {
|
|
1702
2611
|
return health;
|
|
1703
2612
|
}
|
|
1704
|
-
await new Promise((
|
|
2613
|
+
await new Promise((resolve5) => setTimeout(resolve5, 1e3));
|
|
1705
2614
|
}
|
|
1706
2615
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
1707
2616
|
}
|
|
1708
2617
|
|
|
1709
2618
|
// src/lib/http-timeout.ts
|
|
1710
2619
|
var REQUEST_TIMEOUT_MS = 6e4;
|
|
2620
|
+
var RequestTimeoutError = class extends Error {
|
|
2621
|
+
constructor(operation, timeoutMs, elapsedMs) {
|
|
2622
|
+
super(`${operation} timed out after ${elapsedMs}ms (limit ${timeoutMs}ms)`);
|
|
2623
|
+
this.operation = operation;
|
|
2624
|
+
this.timeoutMs = timeoutMs;
|
|
2625
|
+
this.elapsedMs = elapsedMs;
|
|
2626
|
+
this.name = "TimeoutError";
|
|
2627
|
+
}
|
|
2628
|
+
};
|
|
2629
|
+
function describeRequestOperation(input, init) {
|
|
2630
|
+
const method = (init?.method ?? input.method ?? "GET").toUpperCase();
|
|
2631
|
+
const rawUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
|
2632
|
+
let path;
|
|
2633
|
+
try {
|
|
2634
|
+
path = new URL(rawUrl).pathname;
|
|
2635
|
+
} catch (error2) {
|
|
2636
|
+
if (!(error2 instanceof TypeError)) throw error2;
|
|
2637
|
+
path = rawUrl.split(/[?#]/, 1)[0];
|
|
2638
|
+
}
|
|
2639
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
2640
|
+
const prefixedIdentifier = /^[a-z]+_[A-Za-z0-9_-]{10,}$/;
|
|
2641
|
+
const maskedPath = path.split("/").map((segment) => uuid.test(segment) || prefixedIdentifier.test(segment) ? ":id" : segment).join("/");
|
|
2642
|
+
return `${method} ${maskedPath}`;
|
|
2643
|
+
}
|
|
1711
2644
|
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
1712
|
-
return ((input, init) =>
|
|
2645
|
+
return (async (input, init) => {
|
|
2646
|
+
const startedAt = Date.now();
|
|
2647
|
+
try {
|
|
2648
|
+
return await fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) });
|
|
2649
|
+
} catch (error2) {
|
|
2650
|
+
if (error2?.name === "TimeoutError") {
|
|
2651
|
+
throw new RequestTimeoutError(
|
|
2652
|
+
describeRequestOperation(input, init),
|
|
2653
|
+
timeoutMs,
|
|
2654
|
+
Date.now() - startedAt
|
|
2655
|
+
);
|
|
2656
|
+
}
|
|
2657
|
+
throw error2;
|
|
2658
|
+
}
|
|
2659
|
+
});
|
|
1713
2660
|
}
|
|
1714
2661
|
|
|
1715
2662
|
// src/lib/opencode/client.ts
|
|
@@ -1775,7 +2722,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1775
2722
|
return null;
|
|
1776
2723
|
}
|
|
1777
2724
|
}
|
|
1778
|
-
function
|
|
2725
|
+
function roleOf2(m) {
|
|
1779
2726
|
if (!m || typeof m !== "object") return void 0;
|
|
1780
2727
|
if (typeof m.role === "string") return m.role;
|
|
1781
2728
|
const infoRole = m.info?.role;
|
|
@@ -1789,13 +2736,13 @@ function createdOf(m) {
|
|
|
1789
2736
|
if (!m || typeof m !== "object") return void 0;
|
|
1790
2737
|
return m.info?.time?.created ?? m.time?.created;
|
|
1791
2738
|
}
|
|
1792
|
-
function
|
|
2739
|
+
function idOf2(m) {
|
|
1793
2740
|
if (!m || typeof m !== "object") return void 0;
|
|
1794
2741
|
if (typeof m.id === "string") return m.id;
|
|
1795
2742
|
const infoId = m.info?.id;
|
|
1796
2743
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1797
2744
|
}
|
|
1798
|
-
function
|
|
2745
|
+
function parentIdOf2(m) {
|
|
1799
2746
|
if (!m || typeof m !== "object") return void 0;
|
|
1800
2747
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1801
2748
|
const infoParent = m.info?.parentID;
|
|
@@ -1811,8 +2758,12 @@ function errorOf(m) {
|
|
|
1811
2758
|
if (!m || typeof m !== "object") return void 0;
|
|
1812
2759
|
return m.info?.error ?? m.error;
|
|
1813
2760
|
}
|
|
2761
|
+
function isTerminalStopAnswer(m) {
|
|
2762
|
+
return completedOf(m) != null && errorOf(m) == null && finishOf(m) === "stop";
|
|
2763
|
+
}
|
|
1814
2764
|
function isAssistantInFlight(m) {
|
|
1815
2765
|
if (completedOf(m) == null) return true;
|
|
2766
|
+
if (errorOf(m) != null) return false;
|
|
1816
2767
|
return finishOf(m) === "tool-calls";
|
|
1817
2768
|
}
|
|
1818
2769
|
async function getSessionMessages(port, sessionId, client) {
|
|
@@ -1860,9 +2811,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
1860
2811
|
function isSessionActivelyGenerating(messages) {
|
|
1861
2812
|
if (!messages || messages.length === 0) return false;
|
|
1862
2813
|
const last = messages[messages.length - 1];
|
|
1863
|
-
if (
|
|
2814
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
1864
2815
|
return completedOf(last) == null;
|
|
1865
2816
|
}
|
|
2817
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
1866
2818
|
function sessionLastActivityMs(session) {
|
|
1867
2819
|
const candidates = [
|
|
1868
2820
|
session.time?.updated,
|
|
@@ -1877,30 +2829,71 @@ function sessionLastActivityMs(session) {
|
|
|
1877
2829
|
}
|
|
1878
2830
|
return null;
|
|
1879
2831
|
}
|
|
1880
|
-
|
|
1881
|
-
|
|
2832
|
+
function sessionListPath(options) {
|
|
2833
|
+
const params = new URLSearchParams();
|
|
2834
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2835
|
+
const query = params.toString();
|
|
2836
|
+
return query ? `/session?${query}` : "/session";
|
|
2837
|
+
}
|
|
2838
|
+
async function listSessionPage(port, client, options) {
|
|
2839
|
+
const path = sessionListPath(options);
|
|
1882
2840
|
try {
|
|
1883
|
-
const res = await requestWithClient(port, client,
|
|
2841
|
+
const res = await requestWithClient(port, client, path);
|
|
1884
2842
|
if (!res.ok) return null;
|
|
1885
2843
|
const body = await res.json();
|
|
1886
2844
|
return Array.isArray(body) ? body : null;
|
|
1887
2845
|
} catch (error2) {
|
|
1888
2846
|
console.error(
|
|
1889
|
-
`[
|
|
2847
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1890
2848
|
);
|
|
1891
2849
|
return null;
|
|
1892
2850
|
}
|
|
1893
2851
|
}
|
|
2852
|
+
async function listSessions(port, client, options) {
|
|
2853
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2854
|
+
return listSessionPage(port, client, options);
|
|
2855
|
+
}
|
|
2856
|
+
async function listAllSessions(port, client) {
|
|
2857
|
+
if (client?.version === "v2") {
|
|
2858
|
+
const sessions2 = await listV2Sessions(client);
|
|
2859
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2860
|
+
}
|
|
2861
|
+
const sessions = await listSessionPage(port, client, {
|
|
2862
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2863
|
+
});
|
|
2864
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2865
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2866
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2867
|
+
}
|
|
2868
|
+
return { sessions };
|
|
2869
|
+
}
|
|
1894
2870
|
async function deleteSession(port, id, client) {
|
|
1895
2871
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
1896
2872
|
try {
|
|
1897
|
-
const res = await requestWithClient(
|
|
1898
|
-
|
|
2873
|
+
const res = await requestWithClient(
|
|
2874
|
+
port,
|
|
2875
|
+
client,
|
|
2876
|
+
`/session/${id}`,
|
|
2877
|
+
{ method: "DELETE" },
|
|
2878
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2879
|
+
);
|
|
2880
|
+
if (res.status >= 200 && res.status < 300) {
|
|
2881
|
+
return { ok: true, status: res.status };
|
|
2882
|
+
}
|
|
2883
|
+
try {
|
|
2884
|
+
const body = await res.text();
|
|
2885
|
+
return body ? { ok: false, status: res.status, body } : { ok: false, status: res.status };
|
|
2886
|
+
} catch (error2) {
|
|
2887
|
+
console.error(
|
|
2888
|
+
`[deleteSession] DELETE /session/${id} response body read failed for HTTP ${res.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2889
|
+
);
|
|
2890
|
+
return { ok: false, status: res.status };
|
|
2891
|
+
}
|
|
1899
2892
|
} catch (error2) {
|
|
1900
2893
|
console.error(
|
|
1901
2894
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1902
2895
|
);
|
|
1903
|
-
return false;
|
|
2896
|
+
return { ok: false, status: "threw" };
|
|
1904
2897
|
}
|
|
1905
2898
|
}
|
|
1906
2899
|
async function sessionExists(port, id, client) {
|
|
@@ -2138,11 +3131,31 @@ function messageText(m) {
|
|
|
2138
3131
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2139
3132
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2140
3133
|
}
|
|
2141
|
-
|
|
3134
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
3135
|
+
if (!messages || messages.length === 0) return null;
|
|
3136
|
+
let best = null;
|
|
3137
|
+
for (const message of messages) {
|
|
3138
|
+
if (roleOf2(message) !== "user") continue;
|
|
3139
|
+
const id = idOf2(message);
|
|
3140
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
3141
|
+
if (messageText(message) !== content) continue;
|
|
3142
|
+
const created = createdOf(message) ?? 0;
|
|
3143
|
+
if (best === null || created > best.created) best = { id, created };
|
|
3144
|
+
}
|
|
3145
|
+
return best?.id ?? null;
|
|
3146
|
+
}
|
|
3147
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2142
3148
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2143
3149
|
const knownUserIds = new Set(
|
|
2144
|
-
(before ?? []).filter((m) =>
|
|
3150
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2145
3151
|
);
|
|
3152
|
+
if (callbacks) {
|
|
3153
|
+
try {
|
|
3154
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
3155
|
+
} catch (error2) {
|
|
3156
|
+
callbacks.onKnownUserIdsError(error2);
|
|
3157
|
+
}
|
|
3158
|
+
}
|
|
2146
3159
|
const parts = [{ type: "text", text: content }];
|
|
2147
3160
|
let pendingOutcomes = null;
|
|
2148
3161
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2181,25 +3194,13 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2181
3194
|
const READ_BACK_DELAY_MS = 150;
|
|
2182
3195
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2183
3196
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
const id = idOf(m);
|
|
2189
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2190
|
-
if (messageText(m) !== content) continue;
|
|
2191
|
-
const created = createdOf(m) ?? 0;
|
|
2192
|
-
if (best === null || created > best.created) {
|
|
2193
|
-
best = { id, created };
|
|
2194
|
-
}
|
|
2195
|
-
}
|
|
2196
|
-
if (best) {
|
|
2197
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2198
|
-
return best.id;
|
|
2199
|
-
}
|
|
3197
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
3198
|
+
if (dispatchedUserMessageId !== null) {
|
|
3199
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
3200
|
+
return dispatchedUserMessageId;
|
|
2200
3201
|
}
|
|
2201
3202
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2202
|
-
await new Promise((
|
|
3203
|
+
await new Promise((resolve5) => setTimeout(resolve5, READ_BACK_DELAY_MS));
|
|
2203
3204
|
}
|
|
2204
3205
|
}
|
|
2205
3206
|
return null;
|
|
@@ -2207,57 +3208,68 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2207
3208
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2208
3209
|
if (!messages || messages.length === 0) return null;
|
|
2209
3210
|
const byParent = messages.find(
|
|
2210
|
-
(m) =>
|
|
3211
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2211
3212
|
);
|
|
2212
3213
|
if (byParent) return byParent;
|
|
2213
|
-
const userIndex = messages.findIndex((m) =>
|
|
3214
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2214
3215
|
if (userIndex === -1) return null;
|
|
2215
3216
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2216
|
-
if (
|
|
3217
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2217
3218
|
}
|
|
2218
3219
|
return null;
|
|
2219
3220
|
}
|
|
2220
3221
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
2221
3222
|
if (!messages || messages.length === 0) return null;
|
|
2222
3223
|
let lastCorrelated = null;
|
|
2223
|
-
let lastNonErrored = null;
|
|
2224
3224
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2225
3225
|
const m = messages[i];
|
|
2226
|
-
if (
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
3226
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
3227
|
+
lastCorrelated = m;
|
|
3228
|
+
break;
|
|
3229
|
+
}
|
|
3230
|
+
if (lastCorrelated) {
|
|
3231
|
+
if (errorOf(lastCorrelated) == null) return lastCorrelated;
|
|
3232
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
3233
|
+
const m = messages[i];
|
|
3234
|
+
if (roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId && isTerminalStopAnswer(m)) {
|
|
3235
|
+
return m;
|
|
3236
|
+
}
|
|
2231
3237
|
}
|
|
3238
|
+
return lastCorrelated;
|
|
2232
3239
|
}
|
|
2233
|
-
|
|
2234
|
-
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
3240
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2235
3241
|
if (userIndex === -1) return null;
|
|
2236
3242
|
let last = null;
|
|
2237
|
-
let
|
|
3243
|
+
let lastAssistantIndex = -1;
|
|
2238
3244
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2239
|
-
const role =
|
|
3245
|
+
const role = roleOf2(messages[i]);
|
|
2240
3246
|
if (role === "user") break;
|
|
2241
3247
|
if (role === "assistant") {
|
|
2242
3248
|
last = messages[i];
|
|
2243
|
-
|
|
3249
|
+
lastAssistantIndex = i;
|
|
2244
3250
|
}
|
|
2245
3251
|
}
|
|
2246
|
-
|
|
3252
|
+
if (last === null || errorOf(last) == null) return last;
|
|
3253
|
+
for (let i = lastAssistantIndex; i > userIndex; i--) {
|
|
3254
|
+
const m = messages[i];
|
|
3255
|
+
if (roleOf2(m) === "user") break;
|
|
3256
|
+
if (roleOf2(m) === "assistant" && isTerminalStopAnswer(m)) return m;
|
|
3257
|
+
}
|
|
3258
|
+
return last;
|
|
2247
3259
|
}
|
|
2248
3260
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2249
3261
|
if (!messages || messages.length === 0) return [];
|
|
2250
3262
|
const byParent = messages.filter(
|
|
2251
|
-
(message) =>
|
|
3263
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2252
3264
|
);
|
|
2253
3265
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2254
3266
|
if (assistants.length === 0) {
|
|
2255
|
-
const userIndex = messages.findIndex((message) =>
|
|
3267
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2256
3268
|
if (userIndex === -1) return [];
|
|
2257
3269
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2258
3270
|
const message = messages[i];
|
|
2259
|
-
if (
|
|
2260
|
-
if (
|
|
3271
|
+
if (roleOf2(message) === "user") break;
|
|
3272
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2261
3273
|
}
|
|
2262
3274
|
}
|
|
2263
3275
|
const refs = [];
|
|
@@ -2297,7 +3309,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2297
3309
|
if (!messages || messages.length === 0) return [];
|
|
2298
3310
|
const calls = [];
|
|
2299
3311
|
for (const message of messages) {
|
|
2300
|
-
if (
|
|
3312
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2301
3313
|
for (const part of message.parts ?? []) {
|
|
2302
3314
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2303
3315
|
continue;
|
|
@@ -2327,7 +3339,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2327
3339
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2328
3340
|
const unattributed = [];
|
|
2329
3341
|
for (const message of messages ?? []) {
|
|
2330
|
-
if (
|
|
3342
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2331
3343
|
const created = finiteNumber(createdOf(message));
|
|
2332
3344
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2333
3345
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2349,8 +3361,9 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2349
3361
|
}
|
|
2350
3362
|
function sumAssistantUsage(messages) {
|
|
2351
3363
|
if (!messages || messages.length === 0) return null;
|
|
2352
|
-
const
|
|
2353
|
-
const
|
|
3364
|
+
const hasTrailingError = errorOf(messages[messages.length - 1]) != null;
|
|
3365
|
+
const hasTerminalStopAnswer = messages.some(isTerminalStopAnswer);
|
|
3366
|
+
const selected = hasTrailingError && hasTerminalStopAnswer ? messages.filter((message) => errorOf(message) == null) : messages;
|
|
2354
3367
|
let sawAnyUsage = false;
|
|
2355
3368
|
let inputSum = 0;
|
|
2356
3369
|
let outputSum = 0;
|
|
@@ -2359,9 +3372,11 @@ function sumAssistantUsage(messages) {
|
|
|
2359
3372
|
let cacheWriteSum = 0;
|
|
2360
3373
|
let costSum = 0;
|
|
2361
3374
|
let sawCost = false;
|
|
3375
|
+
let toolCallCount = 0;
|
|
2362
3376
|
let modelId = null;
|
|
2363
3377
|
let providerId = null;
|
|
2364
3378
|
for (const m of selected) {
|
|
3379
|
+
toolCallCount += m.parts?.filter((part) => part.type === "tool").length ?? 0;
|
|
2365
3380
|
const info = m.info;
|
|
2366
3381
|
if (!info) continue;
|
|
2367
3382
|
const tokens = info.tokens;
|
|
@@ -2398,19 +3413,18 @@ function sumAssistantUsage(messages) {
|
|
|
2398
3413
|
usage_tokens_cache_write: cacheWriteSum,
|
|
2399
3414
|
// NULL means OpenCode never reported a cost; it is distinct from a genuine
|
|
2400
3415
|
// zero-cost message, which sets `sawCost` with `costSum === 0`.
|
|
2401
|
-
usage_cost_usd: sawCost ? costSum : null
|
|
3416
|
+
usage_cost_usd: sawCost ? costSum : null,
|
|
3417
|
+
turn_tool_calls: toolCallCount
|
|
2402
3418
|
};
|
|
2403
3419
|
}
|
|
2404
3420
|
function messageUsage(messages, userMessageId) {
|
|
2405
3421
|
if (!messages || messages.length === 0) return null;
|
|
2406
3422
|
const byParentAll = messages.filter(
|
|
2407
|
-
(m) =>
|
|
3423
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2408
3424
|
);
|
|
2409
|
-
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2410
|
-
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
2411
3425
|
let correlated;
|
|
2412
|
-
if (
|
|
2413
|
-
correlated =
|
|
3426
|
+
if (byParentAll.length > 0) {
|
|
3427
|
+
correlated = byParentAll;
|
|
2414
3428
|
} else {
|
|
2415
3429
|
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
2416
3430
|
correlated = reply ? [reply] : [];
|
|
@@ -2420,7 +3434,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2420
3434
|
}
|
|
2421
3435
|
function messageRunState(messages, userMessageId) {
|
|
2422
3436
|
if (!messages || messages.length === 0) return "unknown";
|
|
2423
|
-
const hasUser = messages.some((m) =>
|
|
3437
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2424
3438
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2425
3439
|
if (!hasUser) {
|
|
2426
3440
|
if (!reply) return "unknown";
|
|
@@ -2456,14 +3470,38 @@ function messageError(messages, userMessageId) {
|
|
|
2456
3470
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2457
3471
|
const error2 = errorOf(reply);
|
|
2458
3472
|
if (error2 == null) return null;
|
|
2459
|
-
|
|
2460
|
-
if (typeof error2 === "
|
|
3473
|
+
let rendered;
|
|
3474
|
+
if (typeof error2 === "string") {
|
|
3475
|
+
rendered = error2;
|
|
3476
|
+
} else if (typeof error2 === "object") {
|
|
2461
3477
|
const e = error2;
|
|
2462
3478
|
const dataMessage = e.data?.message;
|
|
2463
|
-
if (typeof dataMessage === "string")
|
|
2464
|
-
|
|
3479
|
+
if (typeof dataMessage === "string") {
|
|
3480
|
+
rendered = dataMessage;
|
|
3481
|
+
} else if (typeof e.message === "string") {
|
|
3482
|
+
rendered = e.message;
|
|
3483
|
+
} else {
|
|
3484
|
+
rendered = "The agent run failed.";
|
|
3485
|
+
}
|
|
3486
|
+
} else {
|
|
3487
|
+
rendered = "The agent run failed.";
|
|
2465
3488
|
}
|
|
2466
|
-
|
|
3489
|
+
const toolPart = Array.isArray(reply?.parts) ? reply.parts.find(
|
|
3490
|
+
(part) => part && typeof part === "object" && part.type === "tool" && part.state?.status === "error"
|
|
3491
|
+
) : void 0;
|
|
3492
|
+
return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
|
|
3493
|
+
}
|
|
3494
|
+
function describeAbandonedReplyShape(messages, userMessageId) {
|
|
3495
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3496
|
+
if (!reply) return null;
|
|
3497
|
+
const parts = Array.isArray(reply.parts) ? reply.parts : [];
|
|
3498
|
+
const lastTool = [...parts].reverse().find((part) => part?.type === "tool");
|
|
3499
|
+
return {
|
|
3500
|
+
partTypes: parts.slice(-12).map((part) => typeof part?.type === "string" ? part.type : "(unknown)"),
|
|
3501
|
+
droppedPartCount: Math.max(0, parts.length - 12),
|
|
3502
|
+
lastToolName: typeof lastTool?.tool === "string" && lastTool.tool.length > 0 ? lastTool.tool : null,
|
|
3503
|
+
lastToolStatus: typeof lastTool?.state?.status === "string" && lastTool.state.status.length > 0 ? lastTool.state.status : null
|
|
3504
|
+
};
|
|
2467
3505
|
}
|
|
2468
3506
|
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2469
3507
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2480,19 +3518,32 @@ function isAbortedTerminalReply(messages, userMessageId) {
|
|
|
2480
3518
|
}
|
|
2481
3519
|
return false;
|
|
2482
3520
|
}
|
|
3521
|
+
var CLAUDE_AUTH_EXPIRY_MESSAGES = Object.freeze([
|
|
3522
|
+
"Claude Code credentials are unavailable or expired. Run `claude` to refresh them."
|
|
3523
|
+
]);
|
|
3524
|
+
function extractReplyErrorShape(reply) {
|
|
3525
|
+
const error2 = errorOf(reply);
|
|
3526
|
+
if (error2 == null || typeof error2 !== "object") {
|
|
3527
|
+
return { name: null, statusCode: null };
|
|
3528
|
+
}
|
|
3529
|
+
const e = error2;
|
|
3530
|
+
const name = typeof e.name === "string" ? e.name.slice(0, 100) : null;
|
|
3531
|
+
const data = e.data;
|
|
3532
|
+
const statusCode = typeof data?.statusCode === "number" && Number.isFinite(data.statusCode) ? data.statusCode : null;
|
|
3533
|
+
return { name, statusCode };
|
|
3534
|
+
}
|
|
2483
3535
|
function classifyReplyAuthError(reply) {
|
|
2484
3536
|
const error2 = errorOf(reply);
|
|
2485
3537
|
if (error2 == null || typeof error2 !== "object") return null;
|
|
2486
3538
|
const e = error2;
|
|
2487
3539
|
const replyProviderId = reply?.info?.providerID ?? null;
|
|
2488
3540
|
const replyModelId = reply?.info?.modelID ?? null;
|
|
3541
|
+
const data = e.data;
|
|
2489
3542
|
if (e.name === "ProviderAuthError") {
|
|
2490
|
-
const data = e.data;
|
|
2491
3543
|
const providerId = typeof data?.providerID === "string" && data.providerID || replyProviderId;
|
|
2492
3544
|
return { kind: "model_auth", providerId, modelId: replyModelId, reason: "missing" };
|
|
2493
3545
|
}
|
|
2494
3546
|
if (e.name === "APIError") {
|
|
2495
|
-
const data = e.data;
|
|
2496
3547
|
const statusCode = data?.statusCode;
|
|
2497
3548
|
if (statusCode === 401 || statusCode === 403) {
|
|
2498
3549
|
return {
|
|
@@ -2503,6 +3554,15 @@ function classifyReplyAuthError(reply) {
|
|
|
2503
3554
|
};
|
|
2504
3555
|
}
|
|
2505
3556
|
}
|
|
3557
|
+
const errorMessage3 = typeof data?.message === "string" ? data.message : typeof e.message === "string" ? e.message : null;
|
|
3558
|
+
if (errorMessage3 !== null && CLAUDE_AUTH_EXPIRY_MESSAGES.some((knownMessage) => errorMessage3.includes(knownMessage))) {
|
|
3559
|
+
return {
|
|
3560
|
+
kind: "model_auth",
|
|
3561
|
+
providerId: typeof data?.providerID === "string" && data.providerID || replyProviderId,
|
|
3562
|
+
modelId: replyModelId,
|
|
3563
|
+
reason: "rejected"
|
|
3564
|
+
};
|
|
3565
|
+
}
|
|
2506
3566
|
return null;
|
|
2507
3567
|
}
|
|
2508
3568
|
function messageFailure(messages, userMessageId) {
|
|
@@ -2512,7 +3572,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2512
3572
|
if (!messages || messages.length === 0) return null;
|
|
2513
3573
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2514
3574
|
const message = messages[i];
|
|
2515
|
-
if (
|
|
3575
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2516
3576
|
const created = createdOf(message);
|
|
2517
3577
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2518
3578
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2544,25 +3604,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2544
3604
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2545
3605
|
if (!messages || messages.length === 0) return false;
|
|
2546
3606
|
return messages.some(
|
|
2547
|
-
(m) =>
|
|
3607
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2548
3608
|
);
|
|
2549
3609
|
}
|
|
2550
3610
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2551
3611
|
if (!messages || messages.length === 0) return false;
|
|
2552
|
-
const userIndex = messages.findIndex((message) =>
|
|
3612
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2553
3613
|
if (userIndex === -1) return false;
|
|
2554
3614
|
let hasLaterUser = false;
|
|
2555
3615
|
let hasStartedLaterUser = false;
|
|
2556
3616
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2557
3617
|
const message = messages[i];
|
|
2558
|
-
if (
|
|
3618
|
+
if (roleOf2(message) !== "user") continue;
|
|
2559
3619
|
hasLaterUser = true;
|
|
2560
|
-
const laterUserMessageId =
|
|
3620
|
+
const laterUserMessageId = idOf2(message);
|
|
2561
3621
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2562
3622
|
return false;
|
|
2563
3623
|
}
|
|
2564
3624
|
if (messages.some(
|
|
2565
|
-
(candidate) =>
|
|
3625
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2566
3626
|
)) {
|
|
2567
3627
|
hasStartedLaterUser = true;
|
|
2568
3628
|
}
|
|
@@ -3033,13 +4093,28 @@ async function listV2Sessions(client) {
|
|
|
3033
4093
|
}
|
|
3034
4094
|
async function deleteV2Session(client, sessionId) {
|
|
3035
4095
|
try {
|
|
3036
|
-
await client.request(
|
|
3037
|
-
|
|
4096
|
+
const response = await client.request(
|
|
4097
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
4098
|
+
{ method: "DELETE" },
|
|
4099
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
4100
|
+
);
|
|
4101
|
+
if (response.status >= 200 && response.status < 300) {
|
|
4102
|
+
return { ok: true, status: response.status };
|
|
4103
|
+
}
|
|
4104
|
+
try {
|
|
4105
|
+
const body = await response.text();
|
|
4106
|
+
return body ? { ok: false, status: response.status, body } : { ok: false, status: response.status };
|
|
4107
|
+
} catch (error2) {
|
|
4108
|
+
console.error(
|
|
4109
|
+
`[deleteV2Session] DELETE /api/session/${sessionId} response body read failed for HTTP ${response.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4110
|
+
);
|
|
4111
|
+
return { ok: false, status: response.status };
|
|
4112
|
+
}
|
|
3038
4113
|
} catch (error2) {
|
|
3039
4114
|
console.error(
|
|
3040
4115
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3041
4116
|
);
|
|
3042
|
-
return false;
|
|
4117
|
+
return { ok: false, status: "threw" };
|
|
3043
4118
|
}
|
|
3044
4119
|
}
|
|
3045
4120
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3210,12 +4285,12 @@ async function readV2SessionErrorStream(client, options) {
|
|
|
3210
4285
|
|
|
3211
4286
|
// src/lib/opencode/credential-import-v2.ts
|
|
3212
4287
|
import { readFileSync as readFileSync4 } from "node:fs";
|
|
3213
|
-
import { join as
|
|
4288
|
+
import { join as join5 } from "node:path";
|
|
3214
4289
|
|
|
3215
4290
|
// src/lib/openai-usage.ts
|
|
3216
4291
|
import { readFileSync as readFileSync3 } from "node:fs";
|
|
3217
|
-
import { homedir as
|
|
3218
|
-
import { join as
|
|
4292
|
+
import { homedir as homedir3 } from "node:os";
|
|
4293
|
+
import { join as join4 } from "node:path";
|
|
3219
4294
|
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
3220
4295
|
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
3221
4296
|
var OpenAiUsageError = class extends Error {
|
|
@@ -3229,7 +4304,7 @@ function isLocalCredentialProblem2(err) {
|
|
|
3229
4304
|
}
|
|
3230
4305
|
function readOpenCodeChatGptCredentials() {
|
|
3231
4306
|
try {
|
|
3232
|
-
const raw = readFileSync3(
|
|
4307
|
+
const raw = readFileSync3(join4(homedir3(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
3233
4308
|
let parsed;
|
|
3234
4309
|
try {
|
|
3235
4310
|
parsed = JSON.parse(raw);
|
|
@@ -3370,16 +4445,17 @@ function hasPrimaryHeaders(headers) {
|
|
|
3370
4445
|
].some((name) => headers.has(name));
|
|
3371
4446
|
}
|
|
3372
4447
|
async function getOpenAiUsage(port, client) {
|
|
4448
|
+
const openCodeVersion = client?.version ?? "v1";
|
|
3373
4449
|
const credentials2 = readOpenCodeChatGptCredentials();
|
|
3374
4450
|
if (!credentials2) {
|
|
3375
4451
|
throw new OpenAiUsageError(
|
|
3376
|
-
"No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
4452
|
+
openCodeVersion === "v2" ? "OpenAI usage reporting is not available on OpenCode V2 because V2 keeps a connected ChatGPT login in a store Evident cannot read." : "No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3377
4453
|
"no_credentials"
|
|
3378
4454
|
);
|
|
3379
4455
|
}
|
|
3380
4456
|
if (credentials2.expiresAt < Date.now()) {
|
|
3381
4457
|
throw new OpenAiUsageError(
|
|
3382
|
-
"ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
4458
|
+
openCodeVersion === "v2" ? "The delivered ChatGPT login has expired. OpenCode V2 cannot be given a replacement by file; connect the account again from the runner's page in Evident." : "ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3383
4459
|
"credentials_expired"
|
|
3384
4460
|
);
|
|
3385
4461
|
}
|
|
@@ -3454,13 +4530,13 @@ var INTEGRATION_LIST_RETRY_DELAY_MS = 1e3;
|
|
|
3454
4530
|
function isRecord3(value) {
|
|
3455
4531
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3456
4532
|
}
|
|
3457
|
-
var sleep2 = (milliseconds) => new Promise((
|
|
4533
|
+
var sleep2 = (milliseconds) => new Promise((resolve5) => setTimeout(resolve5, milliseconds));
|
|
3458
4534
|
function emptyResult(unsupported = [], outcome = "completed") {
|
|
3459
4535
|
return { outcome, imported: [], alreadyConnected: [], unsupported, failed: [] };
|
|
3460
4536
|
}
|
|
3461
4537
|
function readOpenCodeCredentials(homeDir, log3) {
|
|
3462
4538
|
try {
|
|
3463
|
-
const raw = readFileSync4(
|
|
4539
|
+
const raw = readFileSync4(join5(homeDir, ...OPENCODE_AUTH_SEGMENTS), "utf8");
|
|
3464
4540
|
let parsed;
|
|
3465
4541
|
try {
|
|
3466
4542
|
parsed = JSON.parse(raw);
|
|
@@ -3660,9 +4736,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3660
4736
|
|
|
3661
4737
|
// src/lib/opencode/session-db-boot.ts
|
|
3662
4738
|
import { spawn as spawn2 } from "node:child_process";
|
|
3663
|
-
import { mkdirSync, statSync as
|
|
3664
|
-
import { homedir as
|
|
3665
|
-
import { dirname as
|
|
4739
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
4740
|
+
import { homedir as homedir4 } from "node:os";
|
|
4741
|
+
import { dirname as dirname4, resolve as resolvePath } from "node:path";
|
|
3666
4742
|
|
|
3667
4743
|
// src/lib/runner-synchroniser.ts
|
|
3668
4744
|
import { spawn } from "node:child_process";
|
|
@@ -3672,7 +4748,7 @@ function appendError(stderr, error2) {
|
|
|
3672
4748
|
${message}`;
|
|
3673
4749
|
}
|
|
3674
4750
|
function runSynchroniser(args, opts) {
|
|
3675
|
-
return new Promise((
|
|
4751
|
+
return new Promise((resolve5) => {
|
|
3676
4752
|
let child;
|
|
3677
4753
|
let stdout = "";
|
|
3678
4754
|
let stderr = "";
|
|
@@ -3686,7 +4762,7 @@ function runSynchroniser(args, opts) {
|
|
|
3686
4762
|
if (timer.handle) clearTimeout(timer.handle);
|
|
3687
4763
|
if (abortListener) opts.signal?.removeEventListener("abort", abortListener);
|
|
3688
4764
|
if (spawnListener) child.removeListener("spawn", spawnListener);
|
|
3689
|
-
|
|
4765
|
+
resolve5(result);
|
|
3690
4766
|
};
|
|
3691
4767
|
try {
|
|
3692
4768
|
child = spawn("runner-synchroniser", args, {
|
|
@@ -3742,9 +4818,12 @@ function runSynchroniser(args, opts) {
|
|
|
3742
4818
|
|
|
3743
4819
|
// src/lib/opencode/session-db-boot.ts
|
|
3744
4820
|
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
3745
|
-
var SESSION_DB_VERIFY_TIMEOUT_MS =
|
|
4821
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
|
|
3746
4822
|
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
3747
|
-
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS =
|
|
4823
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
|
|
4824
|
+
var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
|
|
4825
|
+
var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
|
|
4826
|
+
var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
|
|
3748
4827
|
function commandError(result) {
|
|
3749
4828
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
3750
4829
|
}
|
|
@@ -3787,7 +4866,7 @@ function clearMarker(options) {
|
|
|
3787
4866
|
function markNoReplicate(options, message) {
|
|
3788
4867
|
if (options.noReplicateMarker) {
|
|
3789
4868
|
try {
|
|
3790
|
-
mkdirSync(
|
|
4869
|
+
mkdirSync(dirname4(options.noReplicateMarker), { recursive: true });
|
|
3791
4870
|
writeFileSync(options.noReplicateMarker, "");
|
|
3792
4871
|
} catch (error2) {
|
|
3793
4872
|
options.log(
|
|
@@ -3846,7 +4925,7 @@ function parseSynchroniserEnv(stdout) {
|
|
|
3846
4925
|
return values;
|
|
3847
4926
|
}
|
|
3848
4927
|
function runCommand(command, args, options) {
|
|
3849
|
-
return new Promise((
|
|
4928
|
+
return new Promise((resolve5) => {
|
|
3850
4929
|
let child;
|
|
3851
4930
|
let stdout = "";
|
|
3852
4931
|
let stderr = "";
|
|
@@ -3855,7 +4934,7 @@ function runCommand(command, args, options) {
|
|
|
3855
4934
|
if (settled) return;
|
|
3856
4935
|
settled = true;
|
|
3857
4936
|
if (timer) clearTimeout(timer);
|
|
3858
|
-
|
|
4937
|
+
resolve5(result);
|
|
3859
4938
|
};
|
|
3860
4939
|
try {
|
|
3861
4940
|
child = spawn2(command, args, {
|
|
@@ -3863,7 +4942,7 @@ function runCommand(command, args, options) {
|
|
|
3863
4942
|
stdio: ["ignore", "pipe", "pipe"]
|
|
3864
4943
|
});
|
|
3865
4944
|
} catch (error2) {
|
|
3866
|
-
|
|
4945
|
+
resolve5({
|
|
3867
4946
|
code: null,
|
|
3868
4947
|
stdout,
|
|
3869
4948
|
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
@@ -3912,7 +4991,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3912
4991
|
return null;
|
|
3913
4992
|
}
|
|
3914
4993
|
try {
|
|
3915
|
-
if (
|
|
4994
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
3916
4995
|
} catch (error2) {
|
|
3917
4996
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
3918
4997
|
options.log(
|
|
@@ -3942,7 +5021,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3942
5021
|
return null;
|
|
3943
5022
|
}
|
|
3944
5023
|
try {
|
|
3945
|
-
mkdirSync(
|
|
5024
|
+
mkdirSync(dirname4(configPath), { recursive: true });
|
|
3946
5025
|
writeFileSync(configPath, rendered.stdout);
|
|
3947
5026
|
} catch (error2) {
|
|
3948
5027
|
options.log(
|
|
@@ -4017,14 +5096,14 @@ async function restoreSessionDb(options, configPath, env) {
|
|
|
4017
5096
|
case 0:
|
|
4018
5097
|
return;
|
|
4019
5098
|
case 31:
|
|
4020
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
5099
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4021
5100
|
options.log(
|
|
4022
5101
|
"SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot",
|
|
4023
5102
|
"warn"
|
|
4024
5103
|
);
|
|
4025
5104
|
return;
|
|
4026
5105
|
case 32:
|
|
4027
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
5106
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4028
5107
|
discardSessionDbDebris(options);
|
|
4029
5108
|
markNoReplicate(
|
|
4030
5109
|
options,
|
|
@@ -4055,20 +5134,37 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4055
5134
|
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
4056
5135
|
env: {
|
|
4057
5136
|
...env,
|
|
4058
|
-
// The synchroniser reads
|
|
4059
|
-
// 120_000, so the walkback gives up before the outer process bound.
|
|
5137
|
+
// The synchroniser reads all four values in SECONDS.
|
|
4060
5138
|
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
4061
5139
|
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
5140
|
+
),
|
|
5141
|
+
EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
|
|
5142
|
+
SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
|
|
5143
|
+
),
|
|
5144
|
+
EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
|
|
5145
|
+
SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
|
|
5146
|
+
),
|
|
5147
|
+
EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
|
|
5148
|
+
SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
|
|
4062
5149
|
)
|
|
4063
5150
|
}
|
|
4064
5151
|
});
|
|
4065
5152
|
logSynchroniserDiagnostics(result, options);
|
|
4066
5153
|
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
5154
|
+
if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
|
|
5155
|
+
const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
|
|
5156
|
+
options.log(
|
|
5157
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
|
|
5158
|
+
"error"
|
|
5159
|
+
);
|
|
5160
|
+
reportRecord("verify", "session_db_boot_refused", reason, null, options);
|
|
5161
|
+
return { verifyFatal: true, verifyFatalReason: reason };
|
|
5162
|
+
}
|
|
4067
5163
|
options.log(
|
|
4068
5164
|
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
4069
5165
|
"warn"
|
|
4070
5166
|
);
|
|
4071
|
-
return false;
|
|
5167
|
+
return { verifyFatal: false };
|
|
4072
5168
|
}
|
|
4073
5169
|
if (result.code === 34) {
|
|
4074
5170
|
reportRecord(
|
|
@@ -4078,14 +5174,14 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4078
5174
|
null,
|
|
4079
5175
|
options
|
|
4080
5176
|
);
|
|
4081
|
-
return true;
|
|
5177
|
+
return { verifyFatal: true };
|
|
4082
5178
|
}
|
|
4083
5179
|
if (result.code === 33) {
|
|
4084
5180
|
options.log(
|
|
4085
5181
|
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
4086
5182
|
"warn"
|
|
4087
5183
|
);
|
|
4088
|
-
return false;
|
|
5184
|
+
return { verifyFatal: false };
|
|
4089
5185
|
}
|
|
4090
5186
|
if (result.code !== 0) {
|
|
4091
5187
|
options.log(
|
|
@@ -4093,17 +5189,17 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4093
5189
|
"warn"
|
|
4094
5190
|
);
|
|
4095
5191
|
}
|
|
4096
|
-
return false;
|
|
5192
|
+
return { verifyFatal: false };
|
|
4097
5193
|
}
|
|
4098
5194
|
options.log(
|
|
4099
5195
|
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
4100
5196
|
"debug"
|
|
4101
5197
|
);
|
|
4102
|
-
return false;
|
|
5198
|
+
return { verifyFatal: false };
|
|
4103
5199
|
}
|
|
4104
5200
|
function fileExists(path) {
|
|
4105
5201
|
try {
|
|
4106
|
-
|
|
5202
|
+
statSync3(path);
|
|
4107
5203
|
return true;
|
|
4108
5204
|
} catch (error2) {
|
|
4109
5205
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4113,7 +5209,7 @@ function fileExists(path) {
|
|
|
4113
5209
|
async function restoreAndVerifySessionDb(options) {
|
|
4114
5210
|
const env = options.env ?? process.env;
|
|
4115
5211
|
clearMarker(options);
|
|
4116
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
5212
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4117
5213
|
const synchroniserEnv = await runSynchroniser(["env"], {
|
|
4118
5214
|
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
4119
5215
|
env
|
|
@@ -4186,13 +5282,13 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4186
5282
|
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
4187
5283
|
return { verifyFatal: false };
|
|
4188
5284
|
}
|
|
4189
|
-
return
|
|
5285
|
+
return verifySessionDb(options, configPath, env);
|
|
4190
5286
|
}
|
|
4191
5287
|
|
|
4192
5288
|
// src/lib/opencode/session-db-provenance.ts
|
|
4193
5289
|
import { createRequire } from "node:module";
|
|
4194
5290
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4195
|
-
import { dirname as
|
|
5291
|
+
import { dirname as dirname5, join as join6 } from "node:path";
|
|
4196
5292
|
var require2 = createRequire(import.meta.url);
|
|
4197
5293
|
function readSessionDbMigrationIds(dbPath) {
|
|
4198
5294
|
let db;
|
|
@@ -4229,7 +5325,7 @@ function readSessionDbMigrationIds(dbPath) {
|
|
|
4229
5325
|
}
|
|
4230
5326
|
function sessionDbProvenanceStatePath(homeDir, env) {
|
|
4231
5327
|
const override = env.EVIDENT_SESSION_DB_PROVENANCE_STATE?.trim();
|
|
4232
|
-
return override ||
|
|
5328
|
+
return override || join6(homeDir, ".local", "state", "evident", "session-db-provenance.json");
|
|
4233
5329
|
}
|
|
4234
5330
|
function loadSessionDbProvenanceState(path) {
|
|
4235
5331
|
let value;
|
|
@@ -4258,7 +5354,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4258
5354
|
}
|
|
4259
5355
|
function saveSessionDbProvenanceState(path, state) {
|
|
4260
5356
|
try {
|
|
4261
|
-
mkdirSync2(
|
|
5357
|
+
mkdirSync2(dirname5(path), { recursive: true });
|
|
4262
5358
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4263
5359
|
`, "utf8");
|
|
4264
5360
|
} catch (error2) {
|
|
@@ -4349,7 +5445,7 @@ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
|
4349
5445
|
if (child.exitCode !== null || child.signalCode !== null) {
|
|
4350
5446
|
return { outcome: "exited", code: child.exitCode, signal: child.signalCode };
|
|
4351
5447
|
}
|
|
4352
|
-
return new Promise((
|
|
5448
|
+
return new Promise((resolve5, reject) => {
|
|
4353
5449
|
let forced = false;
|
|
4354
5450
|
let settled = false;
|
|
4355
5451
|
const timer = setTimeout(() => {
|
|
@@ -4369,7 +5465,7 @@ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
|
4369
5465
|
settled = true;
|
|
4370
5466
|
clearTimeout(timer);
|
|
4371
5467
|
child.removeListener("exit", onExit);
|
|
4372
|
-
|
|
5468
|
+
resolve5(result);
|
|
4373
5469
|
};
|
|
4374
5470
|
const fail = (error2) => {
|
|
4375
5471
|
if (settled) return;
|
|
@@ -4839,10 +5935,13 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4839
5935
|
toDelete.push(s.id);
|
|
4840
5936
|
}
|
|
4841
5937
|
}
|
|
4842
|
-
|
|
5938
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5939
|
+
return toDelete.sort(
|
|
5940
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5941
|
+
);
|
|
4843
5942
|
}
|
|
4844
5943
|
var DEFAULT_INTERVAL = "1h";
|
|
4845
|
-
function
|
|
5944
|
+
function resolve2(flag, envValue, fallback) {
|
|
4846
5945
|
return flag ?? envValue ?? fallback;
|
|
4847
5946
|
}
|
|
4848
5947
|
function parseMaxCount(input) {
|
|
@@ -4858,9 +5957,9 @@ function parseMaxCount(input) {
|
|
|
4858
5957
|
}
|
|
4859
5958
|
function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
4860
5959
|
const warnings = [];
|
|
4861
|
-
const maxAgeRaw =
|
|
4862
|
-
const maxCountRaw =
|
|
4863
|
-
const intervalRaw =
|
|
5960
|
+
const maxAgeRaw = resolve2(flags.maxAge, env.EVIDENT_SESSION_CLEANUP_MAX_AGE);
|
|
5961
|
+
const maxCountRaw = resolve2(flags.maxCount, env.EVIDENT_SESSION_CLEANUP_MAX_COUNT);
|
|
5962
|
+
const intervalRaw = resolve2(
|
|
4864
5963
|
flags.interval,
|
|
4865
5964
|
env.EVIDENT_SESSION_CLEANUP_INTERVAL,
|
|
4866
5965
|
DEFAULT_INTERVAL
|
|
@@ -4899,13 +5998,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
4899
5998
|
}
|
|
4900
5999
|
|
|
4901
6000
|
// src/lib/opencode/session-db-size.ts
|
|
4902
|
-
import { statSync as
|
|
4903
|
-
import { join as
|
|
6001
|
+
import { statSync as statSync4 } from "node:fs";
|
|
6002
|
+
import { join as join7 } from "node:path";
|
|
4904
6003
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
4905
6004
|
function statSessionDbBytes(homeDir) {
|
|
4906
|
-
const dbPath =
|
|
6005
|
+
const dbPath = join7(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
4907
6006
|
try {
|
|
4908
|
-
return
|
|
6007
|
+
return statSync4(dbPath).size;
|
|
4909
6008
|
} catch (err) {
|
|
4910
6009
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
4911
6010
|
if (!isMissingFile) {
|
|
@@ -4931,14 +6030,14 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
4931
6030
|
}
|
|
4932
6031
|
|
|
4933
6032
|
// src/lib/opencode/log-tail.ts
|
|
4934
|
-
import { statSync as
|
|
4935
|
-
import { homedir as
|
|
4936
|
-
import { join as
|
|
6033
|
+
import { statSync as statSync5 } from "node:fs";
|
|
6034
|
+
import { homedir as homedir5 } from "node:os";
|
|
6035
|
+
import { join as join8 } from "node:path";
|
|
4937
6036
|
import { open as open2, stat } from "node:fs/promises";
|
|
4938
6037
|
var DEFAULT_POLL_INTERVAL_MS = 1e3;
|
|
4939
|
-
function resolveOpenCodeLogPath(homeDir =
|
|
4940
|
-
const dataDir = env.XDG_DATA_HOME ||
|
|
4941
|
-
return
|
|
6038
|
+
function resolveOpenCodeLogPath(homeDir = homedir5(), env = process.env) {
|
|
6039
|
+
const dataDir = env.XDG_DATA_HOME || join8(homeDir, ".local", "share");
|
|
6040
|
+
return join8(dataDir, "opencode", "log", "opencode.log");
|
|
4942
6041
|
}
|
|
4943
6042
|
function isEnoent(error2) {
|
|
4944
6043
|
return error2?.code === "ENOENT";
|
|
@@ -4954,7 +6053,7 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
4954
6053
|
let inode = null;
|
|
4955
6054
|
let baselineReady = true;
|
|
4956
6055
|
try {
|
|
4957
|
-
const initial =
|
|
6056
|
+
const initial = statSync5(logPath);
|
|
4958
6057
|
offset = initial.size;
|
|
4959
6058
|
inode = initial.ino;
|
|
4960
6059
|
} catch (error2) {
|
|
@@ -5006,130 +6105,11 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5006
6105
|
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
5007
6106
|
void poll();
|
|
5008
6107
|
return {
|
|
5009
|
-
stop: () => {
|
|
5010
|
-
stopped = true;
|
|
5011
|
-
clearInterval(interval);
|
|
5012
|
-
}
|
|
5013
|
-
};
|
|
5014
|
-
}
|
|
5015
|
-
|
|
5016
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5017
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5018
|
-
import { dirname as dirname4 } from "node:path";
|
|
5019
|
-
function errorMessage(error2) {
|
|
5020
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5021
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5022
|
-
}
|
|
5023
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5024
|
-
try {
|
|
5025
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5026
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5027
|
-
if (availableBytes < requiredBytes) {
|
|
5028
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5029
|
-
}
|
|
5030
|
-
return null;
|
|
5031
|
-
} catch (err) {
|
|
5032
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5033
|
-
}
|
|
5034
|
-
}
|
|
5035
|
-
function readLogicalBytes(db) {
|
|
5036
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5037
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5038
|
-
return pageCount * pageSize;
|
|
5039
|
-
}
|
|
5040
|
-
function readCheckpointResult(db) {
|
|
5041
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5042
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5043
|
-
}
|
|
5044
|
-
async function probeReclaimAvailability(input) {
|
|
5045
|
-
const { dbPath, requiredBytes } = input;
|
|
5046
|
-
let sqlite;
|
|
5047
|
-
try {
|
|
5048
|
-
sqlite = await import("node:sqlite");
|
|
5049
|
-
} catch (err) {
|
|
5050
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5051
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5052
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5053
|
-
}
|
|
5054
|
-
let autoVacuum = null;
|
|
5055
|
-
try {
|
|
5056
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5057
|
-
try {
|
|
5058
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5059
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5060
|
-
} finally {
|
|
5061
|
-
db.close();
|
|
5062
|
-
}
|
|
5063
|
-
} catch (err) {
|
|
5064
|
-
console.warn(
|
|
5065
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5066
|
-
);
|
|
5067
|
-
}
|
|
5068
|
-
if (autoVacuum !== 0) return null;
|
|
5069
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5070
|
-
}
|
|
5071
|
-
async function reclaimSessionDbSpace(input) {
|
|
5072
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5073
|
-
let sqlite;
|
|
5074
|
-
try {
|
|
5075
|
-
sqlite = await import("node:sqlite");
|
|
5076
|
-
} catch (err) {
|
|
5077
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5078
|
-
console.warn(
|
|
5079
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5080
|
-
);
|
|
5081
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5082
|
-
}
|
|
5083
|
-
const { DatabaseSync } = sqlite;
|
|
5084
|
-
let db;
|
|
5085
|
-
try {
|
|
5086
|
-
db = new DatabaseSync(dbPath);
|
|
5087
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5088
|
-
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5089
|
-
if (autoVacuum === 0) {
|
|
5090
|
-
if (!allowFullVacuum) {
|
|
5091
|
-
console.warn(
|
|
5092
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
5093
|
-
);
|
|
5094
|
-
return { ok: false, skipped: "full-vacuum-blocked" };
|
|
5095
|
-
}
|
|
5096
|
-
const fileBytesForGuard = statSync5(dbPath).size;
|
|
5097
|
-
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
5098
|
-
if (skipReason !== null) {
|
|
5099
|
-
console.warn(
|
|
5100
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
5101
|
-
);
|
|
5102
|
-
return { ok: false, skipped: "insufficient-disk-space" };
|
|
5103
|
-
}
|
|
5104
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5105
|
-
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
5106
|
-
db.exec("VACUUM");
|
|
5107
|
-
const afterBytes = readLogicalBytes(db);
|
|
5108
|
-
const checkpoint = readCheckpointResult(db);
|
|
5109
|
-
return { ok: true, mode: "convert", beforeBytes, afterBytes, checkpoint };
|
|
5110
|
-
}
|
|
5111
|
-
if (autoVacuum === 2) {
|
|
5112
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5113
|
-
const bound = Math.max(0, Math.trunc(maxPages));
|
|
5114
|
-
db.exec(`PRAGMA incremental_vacuum(${bound})`);
|
|
5115
|
-
const afterBytes = readLogicalBytes(db);
|
|
5116
|
-
const checkpoint = readCheckpointResult(db);
|
|
5117
|
-
return { ok: true, mode: "incremental", beforeBytes, afterBytes, checkpoint };
|
|
5118
|
-
}
|
|
5119
|
-
console.warn(
|
|
5120
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5121
|
-
);
|
|
5122
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5123
|
-
} catch (err) {
|
|
5124
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5125
|
-
return {
|
|
5126
|
-
ok: false,
|
|
5127
|
-
skipped: "reclaim-error",
|
|
5128
|
-
detail: errorMessage(err)
|
|
5129
|
-
};
|
|
5130
|
-
} finally {
|
|
5131
|
-
db?.close();
|
|
5132
|
-
}
|
|
6108
|
+
stop: () => {
|
|
6109
|
+
stopped = true;
|
|
6110
|
+
clearInterval(interval);
|
|
6111
|
+
}
|
|
6112
|
+
};
|
|
5133
6113
|
}
|
|
5134
6114
|
|
|
5135
6115
|
// src/lib/tunnel/connection.ts
|
|
@@ -5212,8 +6192,8 @@ var StreamForwarder = class {
|
|
|
5212
6192
|
pauseStream(sid, stream) {
|
|
5213
6193
|
if (!stream.paused) {
|
|
5214
6194
|
stream.paused = true;
|
|
5215
|
-
stream.pausePromise = new Promise((
|
|
5216
|
-
stream.resume =
|
|
6195
|
+
stream.pausePromise = new Promise((resolve5) => {
|
|
6196
|
+
stream.resume = resolve5;
|
|
5217
6197
|
});
|
|
5218
6198
|
}
|
|
5219
6199
|
if (stream.pauseTimer !== null) clearTimeout(stream.pauseTimer);
|
|
@@ -5280,12 +6260,12 @@ var StreamForwarder = class {
|
|
|
5280
6260
|
let endBody;
|
|
5281
6261
|
if (has_body) {
|
|
5282
6262
|
const chunks = [];
|
|
5283
|
-
bodyPromise = new Promise((
|
|
6263
|
+
bodyPromise = new Promise((resolve5) => {
|
|
5284
6264
|
pushBody = (buf) => {
|
|
5285
6265
|
chunks.push(buf);
|
|
5286
6266
|
};
|
|
5287
6267
|
endBody = () => {
|
|
5288
|
-
|
|
6268
|
+
resolve5(Buffer.concat(chunks));
|
|
5289
6269
|
};
|
|
5290
6270
|
});
|
|
5291
6271
|
}
|
|
@@ -5361,6 +6341,8 @@ var StreamForwarder = class {
|
|
|
5361
6341
|
if (done) break;
|
|
5362
6342
|
const chunk = Buffer.from(value);
|
|
5363
6343
|
for (let i = 0; i < chunk.length; i += MAX_FRAME_BYTES) {
|
|
6344
|
+
await stream.pausePromise;
|
|
6345
|
+
if (ac.signal.aborted) break;
|
|
5364
6346
|
const slice = chunk.subarray(i, i + MAX_FRAME_BYTES);
|
|
5365
6347
|
if (this.binaryFramesSupported) {
|
|
5366
6348
|
if (this.ws.readyState === WebSocket.OPEN) {
|
|
@@ -5403,6 +6385,7 @@ function classifyUpgradeRejection(headers) {
|
|
|
5403
6385
|
}
|
|
5404
6386
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
5405
6387
|
var BASE_RECONNECT_DELAY = 500;
|
|
6388
|
+
var RELAY_SILENCE_TIMEOUT_MS = 9e4;
|
|
5406
6389
|
function getReconnectDelay(attempt) {
|
|
5407
6390
|
const exponentialDelay = BASE_RECONNECT_DELAY * Math.pow(2, attempt);
|
|
5408
6391
|
const jitter = Math.random() * 1e3;
|
|
@@ -5453,7 +6436,7 @@ function connectTunnel(options) {
|
|
|
5453
6436
|
} = options;
|
|
5454
6437
|
const tunnelUrl = getTunnelUrlConfig();
|
|
5455
6438
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
5456
|
-
return new Promise((
|
|
6439
|
+
return new Promise((resolve5, reject) => {
|
|
5457
6440
|
const ws = new WebSocket2(url, {
|
|
5458
6441
|
headers: {
|
|
5459
6442
|
Authorization: authHeader,
|
|
@@ -5474,10 +6457,27 @@ function connectTunnel(options) {
|
|
|
5474
6457
|
ws.close();
|
|
5475
6458
|
reject(new Error("Connection timeout"));
|
|
5476
6459
|
}, 3e4);
|
|
6460
|
+
let relaySilenceTimeout;
|
|
6461
|
+
let relaySilenceTimedOut = false;
|
|
6462
|
+
const clearRelaySilenceTimeout = () => {
|
|
6463
|
+
if (relaySilenceTimeout === void 0) return;
|
|
6464
|
+
clearTimeout(relaySilenceTimeout);
|
|
6465
|
+
relaySilenceTimeout = void 0;
|
|
6466
|
+
};
|
|
6467
|
+
const armRelaySilenceTimeout = () => {
|
|
6468
|
+
clearRelaySilenceTimeout();
|
|
6469
|
+
relaySilenceTimeout = setTimeout(() => {
|
|
6470
|
+
relaySilenceTimeout = void 0;
|
|
6471
|
+
relaySilenceTimedOut = true;
|
|
6472
|
+
ws.terminate();
|
|
6473
|
+
}, RELAY_SILENCE_TIMEOUT_MS);
|
|
6474
|
+
relaySilenceTimeout.unref?.();
|
|
6475
|
+
};
|
|
5477
6476
|
let upgradeRejection = null;
|
|
5478
6477
|
let upgradeRejectionReason = null;
|
|
5479
6478
|
ws.on("unexpected-response", (_req, res) => {
|
|
5480
6479
|
clearTimeout(connectionTimeout);
|
|
6480
|
+
clearRelaySilenceTimeout();
|
|
5481
6481
|
const reason = classifyUpgradeRejection(res.headers);
|
|
5482
6482
|
upgradeRejectionReason = reason;
|
|
5483
6483
|
const chunks = [];
|
|
@@ -5507,6 +6507,7 @@ function connectTunnel(options) {
|
|
|
5507
6507
|
onInfo?.("WebSocket connection established");
|
|
5508
6508
|
});
|
|
5509
6509
|
ws.on("message", (data, isBinary) => {
|
|
6510
|
+
if (relaySilenceTimeout !== void 0) armRelaySilenceTimeout();
|
|
5510
6511
|
if (isBinary) {
|
|
5511
6512
|
try {
|
|
5512
6513
|
const frame = decodeBinaryBodyFrame(toUint8Array(data));
|
|
@@ -5540,10 +6541,11 @@ function connectTunnel(options) {
|
|
|
5540
6541
|
switch (message.type) {
|
|
5541
6542
|
case "connected": {
|
|
5542
6543
|
clearTimeout(connectionTimeout);
|
|
6544
|
+
armRelaySilenceTimeout();
|
|
5543
6545
|
forwarder.setBinaryFramesSupported(message.binary_frames === true);
|
|
5544
6546
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
5545
6547
|
onConnected?.(connectedAgentId);
|
|
5546
|
-
|
|
6548
|
+
resolve5({
|
|
5547
6549
|
ws,
|
|
5548
6550
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
5549
6551
|
});
|
|
@@ -5564,6 +6566,7 @@ function connectTunnel(options) {
|
|
|
5564
6566
|
});
|
|
5565
6567
|
ws.on("error", (error2) => {
|
|
5566
6568
|
clearTimeout(connectionTimeout);
|
|
6569
|
+
clearRelaySilenceTimeout();
|
|
5567
6570
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
5568
6571
|
if (upgradeRejectionReason === "do_code_updated") {
|
|
5569
6572
|
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
@@ -5575,7 +6578,8 @@ function connectTunnel(options) {
|
|
|
5575
6578
|
);
|
|
5576
6579
|
});
|
|
5577
6580
|
ws.on("close", (code, reason) => {
|
|
5578
|
-
|
|
6581
|
+
clearRelaySilenceTimeout();
|
|
6582
|
+
const reasonStr = (relaySilenceTimedOut ? "no relay heartbeat for 90s \u2014 reconnecting" : void 0) || reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
5579
6583
|
forwarder.abortAll();
|
|
5580
6584
|
onDisconnected?.(code, reasonStr);
|
|
5581
6585
|
});
|
|
@@ -5583,9 +6587,10 @@ function connectTunnel(options) {
|
|
|
5583
6587
|
}
|
|
5584
6588
|
|
|
5585
6589
|
// src/lib/tunnel/runner-connection.ts
|
|
5586
|
-
var RunnerConnection = class {
|
|
6590
|
+
var RunnerConnection = class _RunnerConnection {
|
|
5587
6591
|
opts;
|
|
5588
6592
|
sleep;
|
|
6593
|
+
now;
|
|
5589
6594
|
connection = null;
|
|
5590
6595
|
resolvedAgentId;
|
|
5591
6596
|
/** True while a (re)connect loop is in flight. */
|
|
@@ -5594,10 +6599,17 @@ var RunnerConnection = class {
|
|
|
5594
6599
|
reconnectPromise = null;
|
|
5595
6600
|
/** 1-based count of the current reconnect attempt streak. */
|
|
5596
6601
|
reconnectAttempt = 0;
|
|
6602
|
+
generation = 0;
|
|
6603
|
+
disconnectedAtMs = null;
|
|
6604
|
+
static RELAY_RECONNECT_CLOSE_REASONS = /* @__PURE__ */ new Set([
|
|
6605
|
+
"No pong received",
|
|
6606
|
+
"Orphaned socket"
|
|
6607
|
+
]);
|
|
5597
6608
|
constructor(opts) {
|
|
5598
6609
|
this.opts = opts;
|
|
5599
6610
|
this.resolvedAgentId = opts.agentId;
|
|
5600
6611
|
this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
6612
|
+
this.now = opts.now ?? Date.now;
|
|
5601
6613
|
}
|
|
5602
6614
|
get agentId() {
|
|
5603
6615
|
return this.resolvedAgentId;
|
|
@@ -5633,14 +6645,24 @@ var RunnerConnection = class {
|
|
|
5633
6645
|
port: this.opts.port,
|
|
5634
6646
|
openCodePassword: this.opts.openCodePassword,
|
|
5635
6647
|
onConnected: (agentId) => {
|
|
6648
|
+
const attempt = this.reconnectAttempt + 1;
|
|
5636
6649
|
this.reconnectAttempt = 0;
|
|
5637
6650
|
this.reconnecting = false;
|
|
5638
6651
|
this.resolvedAgentId = agentId;
|
|
5639
|
-
|
|
6652
|
+
this.generation++;
|
|
6653
|
+
const downtimeMs = this.disconnectedAtMs === null ? null : this.now() - this.disconnectedAtMs;
|
|
6654
|
+
this.disconnectedAtMs = null;
|
|
6655
|
+
events.onConnected(agentId, isReconnect, {
|
|
6656
|
+
generation: this.generation,
|
|
6657
|
+
attempt,
|
|
6658
|
+
downtimeMs
|
|
6659
|
+
});
|
|
5640
6660
|
},
|
|
5641
6661
|
onDisconnected: (code, reason) => {
|
|
5642
|
-
|
|
5643
|
-
|
|
6662
|
+
this.disconnectedAtMs = this.now();
|
|
6663
|
+
events.onDisconnected(code, reason, { generation: this.generation });
|
|
6664
|
+
const relayRequestedReconnect = code === 1e3 && _RunnerConnection.RELAY_RECONNECT_CLOSE_REASONS.has(reason);
|
|
6665
|
+
if (this.opts.isRunning() && (code !== 1e3 || relayRequestedReconnect) && !this.reconnecting) {
|
|
5644
6666
|
this.reconnectPromise = this.connectWithRetry(true).catch((err) => {
|
|
5645
6667
|
events.onError?.(`Reconnection failed: ${err.message}`);
|
|
5646
6668
|
});
|
|
@@ -5979,16 +7001,111 @@ function createResourceUsageCollector(homeDir) {
|
|
|
5979
7001
|
};
|
|
5980
7002
|
}
|
|
5981
7003
|
|
|
7004
|
+
// src/lib/resource-usage-low-disk.ts
|
|
7005
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
7006
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
7007
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
7008
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
7009
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
7010
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
7011
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
7012
|
+
var BAND_RANK = {
|
|
7013
|
+
low: 1,
|
|
7014
|
+
very_low: 2,
|
|
7015
|
+
critical: 3
|
|
7016
|
+
};
|
|
7017
|
+
function bandFor(freeBytes) {
|
|
7018
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
7019
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
7020
|
+
return "low";
|
|
7021
|
+
}
|
|
7022
|
+
function formatBytes(bytes) {
|
|
7023
|
+
if (bytes === null) return "unavailable";
|
|
7024
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
7025
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
7026
|
+
const value = bytes / 1024 ** unitIndex;
|
|
7027
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
7028
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
7029
|
+
}
|
|
7030
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
7031
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
7032
|
+
}
|
|
7033
|
+
function evaluateLowDiskWarning(input) {
|
|
7034
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
7035
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
7036
|
+
return { warning: null, state };
|
|
7037
|
+
}
|
|
7038
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
7039
|
+
if (recovered) {
|
|
7040
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
7041
|
+
}
|
|
7042
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
7043
|
+
if (!low) return { warning: null, state };
|
|
7044
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
7045
|
+
const currentRank = BAND_RANK[currentBand];
|
|
7046
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
7047
|
+
const worse = currentRank > previousRank;
|
|
7048
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
7049
|
+
const nextState = {
|
|
7050
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
7051
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
7052
|
+
};
|
|
7053
|
+
return {
|
|
7054
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
7055
|
+
state: nextState
|
|
7056
|
+
};
|
|
7057
|
+
}
|
|
7058
|
+
|
|
7059
|
+
// src/lib/opencode/session-store-health.ts
|
|
7060
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
7061
|
+
function createSessionStoreHealth(options = {}) {
|
|
7062
|
+
const now = options.now ?? (() => Date.now());
|
|
7063
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
7064
|
+
let lastUnwritableAt = null;
|
|
7065
|
+
let lastWritableAt = null;
|
|
7066
|
+
const causes = /* @__PURE__ */ new Map();
|
|
7067
|
+
return {
|
|
7068
|
+
observeUnwritable: () => {
|
|
7069
|
+
lastUnwritableAt = now();
|
|
7070
|
+
},
|
|
7071
|
+
observeWritable: () => {
|
|
7072
|
+
lastWritableAt = now();
|
|
7073
|
+
},
|
|
7074
|
+
isUnwritable: () => {
|
|
7075
|
+
if (lastUnwritableAt === null) return false;
|
|
7076
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
7077
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
7078
|
+
},
|
|
7079
|
+
observeCause: (sessionId, cause) => {
|
|
7080
|
+
if (sessionId === null) return;
|
|
7081
|
+
causes.delete(sessionId);
|
|
7082
|
+
causes.set(sessionId, { cause, observedAt: now() });
|
|
7083
|
+
while (causes.size > 256) {
|
|
7084
|
+
const oldest = causes.keys().next().value;
|
|
7085
|
+
if (typeof oldest !== "string") break;
|
|
7086
|
+
causes.delete(oldest);
|
|
7087
|
+
}
|
|
7088
|
+
},
|
|
7089
|
+
readCause: (sessionId) => {
|
|
7090
|
+
const observed = causes.get(sessionId);
|
|
7091
|
+
if (!observed) return null;
|
|
7092
|
+
causes.delete(sessionId);
|
|
7093
|
+
if (now() - observed.observedAt >= stalenessMs) return null;
|
|
7094
|
+
return observed.cause;
|
|
7095
|
+
}
|
|
7096
|
+
};
|
|
7097
|
+
}
|
|
7098
|
+
|
|
5982
7099
|
// src/lib/channels/driver.ts
|
|
5983
|
-
import { homedir as
|
|
7100
|
+
import { homedir as homedir6 } from "node:os";
|
|
5984
7101
|
|
|
5985
7102
|
// src/lib/runner-file-sync.ts
|
|
5986
|
-
import { join as
|
|
7103
|
+
import { join as join10 } from "node:path";
|
|
5987
7104
|
|
|
5988
7105
|
// src/lib/file-push.ts
|
|
5989
7106
|
import { randomUUID } from "node:crypto";
|
|
5990
7107
|
import { chmod, mkdir, open as open3, realpath, rename, unlink } from "node:fs/promises";
|
|
5991
|
-
import { basename, dirname as
|
|
7108
|
+
import { basename as basename2, dirname as dirname6, isAbsolute as isAbsolute2, join as join9, relative as relative2, resolve as resolve3, sep as sep2 } from "node:path";
|
|
5992
7109
|
var FILE_MODE = 384;
|
|
5993
7110
|
var DIRECTORY_MODE = 448;
|
|
5994
7111
|
async function writePushedFile(request2) {
|
|
@@ -6019,9 +7136,9 @@ async function writePushedFile(request2) {
|
|
|
6019
7136
|
}
|
|
6020
7137
|
try {
|
|
6021
7138
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
6022
|
-
|
|
7139
|
+
dirname6(candidate)
|
|
6023
7140
|
);
|
|
6024
|
-
const realTarget =
|
|
7141
|
+
const realTarget = join9(existingAncestor, ...missingSegments, basename2(candidate));
|
|
6025
7142
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
6026
7143
|
if (allowedDirectory === null) {
|
|
6027
7144
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
@@ -6031,8 +7148,8 @@ async function writePushedFile(request2) {
|
|
|
6031
7148
|
}
|
|
6032
7149
|
if (missingSegments.length > 0) {
|
|
6033
7150
|
await createMissingDirectories(existingAncestor, missingSegments);
|
|
6034
|
-
const realParent = await realpath(
|
|
6035
|
-
if (realParent !==
|
|
7151
|
+
const realParent = await realpath(dirname6(realTarget));
|
|
7152
|
+
if (realParent !== dirname6(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
6036
7153
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
6037
7154
|
path: realTarget,
|
|
6038
7155
|
bytes,
|
|
@@ -6057,15 +7174,15 @@ function expandAndValidate(requestedPath, homeDir) {
|
|
|
6057
7174
|
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
6058
7175
|
return null;
|
|
6059
7176
|
}
|
|
6060
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
7177
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join9(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
6061
7178
|
if (expanded.split(/[/\\]/).includes("..")) {
|
|
6062
7179
|
return null;
|
|
6063
7180
|
}
|
|
6064
|
-
if (!
|
|
7181
|
+
if (!isAbsolute2(expanded)) {
|
|
6065
7182
|
return null;
|
|
6066
7183
|
}
|
|
6067
|
-
const candidate =
|
|
6068
|
-
const name =
|
|
7184
|
+
const candidate = resolve3(expanded);
|
|
7185
|
+
const name = basename2(candidate);
|
|
6069
7186
|
return name === "" || name === "." || name === ".." ? null : candidate;
|
|
6070
7187
|
}
|
|
6071
7188
|
async function resolveNearestExistingAncestor(directory) {
|
|
@@ -6075,18 +7192,18 @@ async function resolveNearestExistingAncestor(directory) {
|
|
|
6075
7192
|
try {
|
|
6076
7193
|
return { existingAncestor: await realpath(current), missingSegments };
|
|
6077
7194
|
} catch (err) {
|
|
6078
|
-
const parent =
|
|
7195
|
+
const parent = dirname6(current);
|
|
6079
7196
|
if (err.code !== "ENOENT" || parent === current) {
|
|
6080
7197
|
throw err;
|
|
6081
7198
|
}
|
|
6082
|
-
missingSegments.unshift(
|
|
7199
|
+
missingSegments.unshift(basename2(current));
|
|
6083
7200
|
current = parent;
|
|
6084
7201
|
}
|
|
6085
7202
|
}
|
|
6086
7203
|
}
|
|
6087
7204
|
async function findContainingAllowedDirectory(allowedDirectories, realTarget) {
|
|
6088
7205
|
for (const directory of allowedDirectories) {
|
|
6089
|
-
if (!
|
|
7206
|
+
if (!isAbsolute2(directory)) {
|
|
6090
7207
|
log("warn", "file_push_allowed_directory_skipped", { directory, reason: "not_absolute" });
|
|
6091
7208
|
continue;
|
|
6092
7209
|
}
|
|
@@ -6124,19 +7241,19 @@ async function realpathCreatingIfMissing(directory) {
|
|
|
6124
7241
|
}
|
|
6125
7242
|
}
|
|
6126
7243
|
function contains(realDirectory, realTarget) {
|
|
6127
|
-
const rel =
|
|
6128
|
-
return rel !== "" && rel !== ".." && !rel.startsWith(`..${
|
|
7244
|
+
const rel = relative2(realDirectory, realTarget);
|
|
7245
|
+
return rel !== "" && rel !== ".." && !rel.startsWith(`..${sep2}`) && !isAbsolute2(rel);
|
|
6129
7246
|
}
|
|
6130
7247
|
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
6131
7248
|
let current = existingAncestor;
|
|
6132
7249
|
for (const segment of missingSegments) {
|
|
6133
|
-
current =
|
|
7250
|
+
current = join9(current, segment);
|
|
6134
7251
|
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
6135
7252
|
await chmod(current, DIRECTORY_MODE);
|
|
6136
7253
|
}
|
|
6137
7254
|
}
|
|
6138
7255
|
async function writeAtomically(realTarget, content) {
|
|
6139
|
-
const temporaryPath =
|
|
7256
|
+
const temporaryPath = join9(dirname6(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
6140
7257
|
let handle;
|
|
6141
7258
|
try {
|
|
6142
7259
|
handle = await open3(temporaryPath, "wx", FILE_MODE);
|
|
@@ -6266,12 +7383,12 @@ var NOT_APPLIED = {
|
|
|
6266
7383
|
opencodeAuthApplied: false
|
|
6267
7384
|
};
|
|
6268
7385
|
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
6269
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
6270
|
-
return expanded ===
|
|
7386
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join10(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
7387
|
+
return expanded === join10(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
6271
7388
|
}
|
|
6272
7389
|
function isOpenCodeAuthPath(requestedPath, homeDir) {
|
|
6273
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
6274
|
-
return expanded ===
|
|
7390
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join10(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
7391
|
+
return expanded === join10(homeDir, ...OPENCODE_AUTH_SEGMENTS);
|
|
6275
7392
|
}
|
|
6276
7393
|
async function applyOne(options, file) {
|
|
6277
7394
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
@@ -6415,6 +7532,11 @@ function messageIdOf(m) {
|
|
|
6415
7532
|
const infoId = m.info?.id;
|
|
6416
7533
|
return typeof infoId === "string" ? infoId : void 0;
|
|
6417
7534
|
}
|
|
7535
|
+
function composeAbandonmentNote(finish, replyShape, outcome) {
|
|
7536
|
+
const base = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") ` + (outcome === "without-answer" ? "without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs." : "after writing only part of an answer, and confirmed it is no longer working on this conversation. Evident delivered what was written and is reporting the run as finished.");
|
|
7537
|
+
if (replyShape === null) return base;
|
|
7538
|
+
return `${base} ${replyShape.lastToolName === null ? "No tool call was recorded" : `The last tool call was "${replyShape.lastToolName}" with status "${replyShape.lastToolStatus ?? "(unknown)"}"`}; the reply's final part types were [${replyShape.partTypes.join(", ")}]${replyShape.droppedPartCount > 0 ? ` (${replyShape.droppedPartCount} earlier part types omitted)` : ""}.`;
|
|
7539
|
+
}
|
|
6418
7540
|
function cleanImageMime(contentType) {
|
|
6419
7541
|
if (!contentType) return null;
|
|
6420
7542
|
const media = contentType.split(";")[0].trim().toLowerCase();
|
|
@@ -6435,8 +7557,13 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
|
|
|
6435
7557
|
var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
|
|
6436
7558
|
var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
6437
7559
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
7560
|
+
var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
|
|
7561
|
+
var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
7562
|
+
var SESSION_ERROR_RETAINED_DETAIL = "Runner-observed OpenCode session error: ";
|
|
7563
|
+
var SESSION_ERROR_OBSERVED_CAUSE = "Runner-observed OpenCode session-store cause: ";
|
|
6438
7564
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6439
7565
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
7566
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6440
7567
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6441
7568
|
var HEARTBEAT_MS = 6e4;
|
|
6442
7569
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6498,6 +7625,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6498
7625
|
maxActiveSessions;
|
|
6499
7626
|
watcherStallMs;
|
|
6500
7627
|
wedgeWarningIntervalMs;
|
|
7628
|
+
sessionStoreHealth;
|
|
6501
7629
|
openCodeClient;
|
|
6502
7630
|
/** Cache of conversationId → opencode sessionId. */
|
|
6503
7631
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6590,6 +7718,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6590
7718
|
* with a short TTL so an unmatched session cannot retain an event indefinitely.
|
|
6591
7719
|
*/
|
|
6592
7720
|
bufferedSessionErrors = /* @__PURE__ */ new Map();
|
|
7721
|
+
retainedSessionErrors = /* @__PURE__ */ new Map();
|
|
6593
7722
|
/**
|
|
6594
7723
|
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
6595
7724
|
* dispatched and are still in-flight. A message in this set is never
|
|
@@ -6622,10 +7751,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6622
7751
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
6623
7752
|
*/
|
|
6624
7753
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7754
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7755
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
6625
7756
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
6626
7757
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6627
7758
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6628
7759
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7760
|
+
/**
|
|
7761
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7762
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7763
|
+
* identical-text turn, so it never produces a correlation.
|
|
7764
|
+
*/
|
|
7765
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6629
7766
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6630
7767
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6631
7768
|
/**
|
|
@@ -6711,21 +7848,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6711
7848
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6712
7849
|
*/
|
|
6713
7850
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6714
|
-
/**
|
|
6715
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6716
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6717
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6718
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6719
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6720
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6721
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6722
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6723
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6724
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6725
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6726
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6727
|
-
*/
|
|
6728
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6729
7851
|
/**
|
|
6730
7852
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6731
7853
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6743,8 +7865,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6743
7865
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6744
7866
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6745
7867
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6746
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6747
|
-
*
|
|
7868
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7869
|
+
* the terminal-report fence and is never sent again).
|
|
6748
7870
|
*/
|
|
6749
7871
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6750
7872
|
/**
|
|
@@ -6863,10 +7985,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6863
7985
|
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
6864
7986
|
this.now = config.now ?? (() => Date.now());
|
|
6865
7987
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
6866
|
-
this.homeDir = config.homeDir ??
|
|
7988
|
+
this.homeDir = config.homeDir ?? homedir6();
|
|
6867
7989
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
6868
7990
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
6869
7991
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7992
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
6870
7993
|
}
|
|
6871
7994
|
get isV2() {
|
|
6872
7995
|
return this.openCodeClient.version === "v2";
|
|
@@ -6882,7 +8005,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6882
8005
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
6883
8006
|
}
|
|
6884
8007
|
async listSessions() {
|
|
6885
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
8008
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
8009
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
8010
|
+
});
|
|
6886
8011
|
}
|
|
6887
8012
|
async sessionExists(sessionId) {
|
|
6888
8013
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7059,6 +8184,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7059
8184
|
}
|
|
7060
8185
|
return ids;
|
|
7061
8186
|
}
|
|
8187
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
8188
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
8189
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
8190
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
8191
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
8192
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
8193
|
+
}
|
|
8194
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
8195
|
+
}
|
|
8196
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
8197
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
8198
|
+
this.awaitingCorrelation.delete(messageId);
|
|
8199
|
+
this.awaitingReadopt.delete(messageId);
|
|
8200
|
+
this.sessions.delete(conversationId);
|
|
8201
|
+
this.supersede(conversationId, sessionId);
|
|
8202
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
8203
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
8204
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
8205
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
8206
|
+
this.log({
|
|
8207
|
+
level: "error",
|
|
8208
|
+
message: errorMessage3,
|
|
8209
|
+
conversation_id: conversationId,
|
|
8210
|
+
message_id: messageId
|
|
8211
|
+
});
|
|
8212
|
+
const reportError = await this.reportUntrackableAck(
|
|
8213
|
+
conversationId,
|
|
8214
|
+
messageId,
|
|
8215
|
+
null,
|
|
8216
|
+
errorMessage3
|
|
8217
|
+
);
|
|
8218
|
+
if (reportError !== null) {
|
|
8219
|
+
this.log({
|
|
8220
|
+
level: "warn",
|
|
8221
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
8222
|
+
conversation_id: conversationId,
|
|
8223
|
+
message_id: messageId
|
|
8224
|
+
});
|
|
8225
|
+
}
|
|
8226
|
+
if (source === "readopt") {
|
|
8227
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
8228
|
+
}
|
|
8229
|
+
}
|
|
7062
8230
|
/**
|
|
7063
8231
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7064
8232
|
*
|
|
@@ -7238,6 +8406,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7238
8406
|
}
|
|
7239
8407
|
break;
|
|
7240
8408
|
}
|
|
8409
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
8410
|
+
if (correlation) {
|
|
8411
|
+
correlation.attempts += 1;
|
|
8412
|
+
let correlatedId = null;
|
|
8413
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
8414
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
8415
|
+
conv,
|
|
8416
|
+
message,
|
|
8417
|
+
correlation.sessionId
|
|
8418
|
+
);
|
|
8419
|
+
if (polled.ok)
|
|
8420
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
8421
|
+
} else if (correlation.knownUserIds === null) {
|
|
8422
|
+
this.log({
|
|
8423
|
+
level: "debug",
|
|
8424
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
8425
|
+
conversation_id: conv.id,
|
|
8426
|
+
message_id: message.id
|
|
8427
|
+
});
|
|
8428
|
+
}
|
|
8429
|
+
if (correlatedId !== null) {
|
|
8430
|
+
this.awaitingCorrelation.delete(message.id);
|
|
8431
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
8432
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
8433
|
+
this.dispatched.add(message.id);
|
|
8434
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
8435
|
+
dispatched += 1;
|
|
8436
|
+
this.log({
|
|
8437
|
+
level: "info",
|
|
8438
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
8439
|
+
conversation_id: conv.id,
|
|
8440
|
+
message_id: message.id
|
|
8441
|
+
});
|
|
8442
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
8443
|
+
opencode_message_id: correlatedId
|
|
8444
|
+
});
|
|
8445
|
+
continue;
|
|
8446
|
+
}
|
|
8447
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
8448
|
+
await this.handoffUntrackableCorrelation(
|
|
8449
|
+
conv.id,
|
|
8450
|
+
message.id,
|
|
8451
|
+
correlation.sessionId,
|
|
8452
|
+
correlation.attempts,
|
|
8453
|
+
"fresh"
|
|
8454
|
+
);
|
|
8455
|
+
break;
|
|
8456
|
+
}
|
|
8457
|
+
break;
|
|
8458
|
+
}
|
|
7241
8459
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7242
8460
|
if (effectiveOpencodeMessageId) {
|
|
7243
8461
|
const outcome = await this.resolveRedrive(
|
|
@@ -7259,6 +8477,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7259
8477
|
model: message.opencode_model ?? void 0
|
|
7260
8478
|
};
|
|
7261
8479
|
let opencodeMessageId;
|
|
8480
|
+
let preDispatchUserIds = null;
|
|
7262
8481
|
try {
|
|
7263
8482
|
this.log({
|
|
7264
8483
|
level: "info",
|
|
@@ -7288,7 +8507,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7288
8507
|
message.content,
|
|
7289
8508
|
options,
|
|
7290
8509
|
sendAttachments,
|
|
7291
|
-
this.openCodeClient
|
|
8510
|
+
this.openCodeClient,
|
|
8511
|
+
{
|
|
8512
|
+
onKnownUserIds: (ids) => {
|
|
8513
|
+
preDispatchUserIds = ids;
|
|
8514
|
+
},
|
|
8515
|
+
onKnownUserIdsError: (error2) => {
|
|
8516
|
+
this.log({
|
|
8517
|
+
level: "warn",
|
|
8518
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8519
|
+
conversation_id: conv.id,
|
|
8520
|
+
message_id: message.id
|
|
8521
|
+
});
|
|
8522
|
+
}
|
|
8523
|
+
}
|
|
7292
8524
|
)
|
|
7293
8525
|
);
|
|
7294
8526
|
} catch (err) {
|
|
@@ -7296,6 +8528,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7296
8528
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7297
8529
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7298
8530
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
8531
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7299
8532
|
this.log({
|
|
7300
8533
|
level: "error",
|
|
7301
8534
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7371,39 +8604,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7371
8604
|
if (this.isV2) {
|
|
7372
8605
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7373
8606
|
}
|
|
7374
|
-
|
|
7375
|
-
|
|
7376
|
-
|
|
7377
|
-
|
|
7378
|
-
|
|
7379
|
-
|
|
7380
|
-
message_id: message.id
|
|
7381
|
-
});
|
|
7382
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7383
|
-
continue;
|
|
7384
|
-
}
|
|
7385
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7386
|
-
this.sessions.delete(conv.id);
|
|
7387
|
-
this.supersede(conv.id, sessionId);
|
|
7388
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
8607
|
+
this.awaitingCorrelation.set(message.id, {
|
|
8608
|
+
sessionId,
|
|
8609
|
+
content: message.content,
|
|
8610
|
+
attempts: 0,
|
|
8611
|
+
knownUserIds: preDispatchUserIds
|
|
8612
|
+
});
|
|
7389
8613
|
this.log({
|
|
7390
|
-
level: "
|
|
7391
|
-
message:
|
|
8614
|
+
level: "warn",
|
|
8615
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7392
8616
|
conversation_id: conv.id,
|
|
7393
8617
|
message_id: message.id
|
|
7394
8618
|
});
|
|
7395
|
-
|
|
7396
|
-
this.log({
|
|
7397
|
-
level: "warn",
|
|
7398
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7399
|
-
conversation_id: conv.id,
|
|
7400
|
-
message_id: message.id
|
|
7401
|
-
});
|
|
7402
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7403
|
-
});
|
|
8619
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7404
8620
|
break;
|
|
7405
8621
|
}
|
|
7406
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7407
8622
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7408
8623
|
this.releasedOpencodeIds.delete(message.id);
|
|
7409
8624
|
this.dispatched.add(message.id);
|
|
@@ -8008,23 +9223,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8008
9223
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8009
9224
|
return 1;
|
|
8010
9225
|
}
|
|
8011
|
-
/**
|
|
8012
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8013
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8014
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8015
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8016
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8017
|
-
* count, since a new session is a genuinely different attempt.
|
|
8018
|
-
*/
|
|
8019
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8020
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8021
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8022
|
-
existing.count += 1;
|
|
8023
|
-
return existing.count;
|
|
8024
|
-
}
|
|
8025
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8026
|
-
return 1;
|
|
8027
|
-
}
|
|
8028
9226
|
/** Record one queued give-up toward the never-started bound. */
|
|
8029
9227
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8030
9228
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8336,8 +9534,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8336
9534
|
b2LastDescendantCheckMs: 0,
|
|
8337
9535
|
b2RootOngoingHeldLogged: false,
|
|
8338
9536
|
b2AbandonedSignalled: false,
|
|
9537
|
+
b2AbandonedWithoutAnswer: false,
|
|
8339
9538
|
ambiguousPinnedSinceMs: 0,
|
|
8340
9539
|
ambiguousResolved: false,
|
|
9540
|
+
sessionStoreFailurePending: false,
|
|
8341
9541
|
neverStartedResolved: false
|
|
8342
9542
|
});
|
|
8343
9543
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8359,6 +9559,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8359
9559
|
this.bufferedSessionErrors.delete(oldest);
|
|
8360
9560
|
}
|
|
8361
9561
|
}
|
|
9562
|
+
retainSessionError(event) {
|
|
9563
|
+
this.retainedSessionErrors.delete(event.sessionId);
|
|
9564
|
+
this.retainedSessionErrors.set(event.sessionId, {
|
|
9565
|
+
reason: event.reason,
|
|
9566
|
+
observedAt: this.now()
|
|
9567
|
+
});
|
|
9568
|
+
while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
|
|
9569
|
+
const oldest = this.retainedSessionErrors.keys().next().value;
|
|
9570
|
+
if (typeof oldest !== "string") break;
|
|
9571
|
+
this.retainedSessionErrors.delete(oldest);
|
|
9572
|
+
}
|
|
9573
|
+
}
|
|
9574
|
+
readRetainedSessionError(sessionId) {
|
|
9575
|
+
const retained = this.retainedSessionErrors.get(sessionId);
|
|
9576
|
+
if (!retained) return null;
|
|
9577
|
+
if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
|
|
9578
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9579
|
+
return null;
|
|
9580
|
+
}
|
|
9581
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9582
|
+
return retained.reason;
|
|
9583
|
+
}
|
|
8362
9584
|
/**
|
|
8363
9585
|
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
8364
9586
|
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
@@ -8454,8 +9676,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8454
9676
|
b2LastDescendantCheckMs: 0,
|
|
8455
9677
|
b2RootOngoingHeldLogged: false,
|
|
8456
9678
|
b2AbandonedSignalled: false,
|
|
9679
|
+
b2AbandonedWithoutAnswer: false,
|
|
8457
9680
|
ambiguousPinnedSinceMs: 0,
|
|
8458
9681
|
ambiguousResolved: false,
|
|
9682
|
+
sessionStoreFailurePending: false,
|
|
8459
9683
|
neverStartedResolved: false
|
|
8460
9684
|
});
|
|
8461
9685
|
}
|
|
@@ -8683,9 +9907,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8683
9907
|
return;
|
|
8684
9908
|
}
|
|
8685
9909
|
if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
|
|
9910
|
+
this.retainSessionError(event);
|
|
8686
9911
|
this.log({
|
|
8687
|
-
level: "
|
|
8688
|
-
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
|
|
9912
|
+
level: "warn",
|
|
9913
|
+
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
|
|
8689
9914
|
conversation_id: watcher.conv.id
|
|
8690
9915
|
});
|
|
8691
9916
|
return;
|
|
@@ -8715,9 +9940,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8715
9940
|
const messages = await this.getSessionMessages(event.sessionId);
|
|
8716
9941
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
8717
9942
|
if (state !== "queued") {
|
|
9943
|
+
this.retainSessionError(event);
|
|
8718
9944
|
this.log({
|
|
8719
|
-
level: "
|
|
8720
|
-
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state};
|
|
9945
|
+
level: "warn",
|
|
9946
|
+
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
|
|
8721
9947
|
conversation_id: watcher.conv.id,
|
|
8722
9948
|
message_id: inFlight.evidentMessageId
|
|
8723
9949
|
});
|
|
@@ -8869,6 +10095,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8869
10095
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8870
10096
|
this.neverStartedAttempts.delete(id);
|
|
8871
10097
|
}
|
|
10098
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
10099
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10100
|
+
return;
|
|
10101
|
+
}
|
|
8872
10102
|
if (inFlight.neverStartedResolved) {
|
|
8873
10103
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8874
10104
|
inFlight.neverStartedResolved = false;
|
|
@@ -8942,7 +10172,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8942
10172
|
if (state === "failed") {
|
|
8943
10173
|
this.anchorDeliveryDeadline(inFlight);
|
|
8944
10174
|
if (!inFlight.done) {
|
|
8945
|
-
const
|
|
10175
|
+
const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
|
|
10176
|
+
const retainedReason = this.readRetainedSessionError(sessionId);
|
|
10177
|
+
const retainedError = retainedReason === null || messageErrorText !== null && messageErrorText.includes(retainedReason) ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
|
|
10178
|
+
const observedCause = this.sessionStoreHealth?.readCause(sessionId);
|
|
10179
|
+
const causeError = observedCause === null || observedCause === void 0 ? null : `${SESSION_ERROR_OBSERVED_CAUSE}${observedCause}`;
|
|
10180
|
+
const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
|
|
10181
|
+
const runnerDetails = [retainedError, causeError, storeError].filter(
|
|
10182
|
+
(detail) => detail !== null
|
|
10183
|
+
);
|
|
10184
|
+
const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
|
|
8946
10185
|
this.log({
|
|
8947
10186
|
level: "error",
|
|
8948
10187
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -9028,10 +10267,33 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9028
10267
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9029
10268
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9030
10269
|
inFlight.b2AbandonedSignalled = false;
|
|
10270
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9031
10271
|
}
|
|
9032
10272
|
} else {
|
|
9033
10273
|
if (inFlight.b2AbandonedSignalled) {
|
|
9034
|
-
|
|
10274
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
10275
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
10276
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
10277
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
10278
|
+
sessionId,
|
|
10279
|
+
watcher,
|
|
10280
|
+
inFlight,
|
|
10281
|
+
messages,
|
|
10282
|
+
finish
|
|
10283
|
+
);
|
|
10284
|
+
} else {
|
|
10285
|
+
await this.settleMessageDone(
|
|
10286
|
+
sessionId,
|
|
10287
|
+
watcher,
|
|
10288
|
+
inFlight,
|
|
10289
|
+
messages,
|
|
10290
|
+
composeAbandonmentNote(
|
|
10291
|
+
finish,
|
|
10292
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10293
|
+
"with-answer"
|
|
10294
|
+
)
|
|
10295
|
+
);
|
|
10296
|
+
}
|
|
9035
10297
|
return;
|
|
9036
10298
|
}
|
|
9037
10299
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9048,22 +10310,62 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9048
10310
|
descendantOngoing,
|
|
9049
10311
|
rootOngoing
|
|
9050
10312
|
})) {
|
|
10313
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
10314
|
+
inFlight.sessionStoreFailurePending = true;
|
|
10315
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
10316
|
+
watched_for_ms: pinnedForMs,
|
|
10317
|
+
run_state: "failed",
|
|
10318
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10319
|
+
});
|
|
10320
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10321
|
+
return;
|
|
10322
|
+
}
|
|
9051
10323
|
inFlight.b2AbandonedSignalled = true;
|
|
10324
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
10325
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
10326
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
10327
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
10328
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
10329
|
+
watched_for_ms: pinnedForMs,
|
|
10330
|
+
run_state: "failed",
|
|
10331
|
+
finish,
|
|
10332
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
10333
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
10334
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10335
|
+
});
|
|
10336
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
10337
|
+
sessionId,
|
|
10338
|
+
watcher,
|
|
10339
|
+
inFlight,
|
|
10340
|
+
messages,
|
|
10341
|
+
finish
|
|
10342
|
+
);
|
|
10343
|
+
return;
|
|
10344
|
+
}
|
|
9052
10345
|
this.log({
|
|
9053
10346
|
level: "warn",
|
|
9054
10347
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9055
10348
|
conversation_id: conv.id,
|
|
9056
10349
|
message_id: id
|
|
9057
10350
|
});
|
|
9058
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9059
10351
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9060
10352
|
watched_for_ms: pinnedForMs,
|
|
9061
|
-
finish
|
|
10353
|
+
finish,
|
|
9062
10354
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9063
10355
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9064
10356
|
opencode_message_id: inFlight.opencodeMessageId
|
|
9065
10357
|
});
|
|
9066
|
-
await this.settleMessageDone(
|
|
10358
|
+
await this.settleMessageDone(
|
|
10359
|
+
sessionId,
|
|
10360
|
+
watcher,
|
|
10361
|
+
inFlight,
|
|
10362
|
+
messages,
|
|
10363
|
+
composeAbandonmentNote(
|
|
10364
|
+
finish,
|
|
10365
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10366
|
+
"with-answer"
|
|
10367
|
+
)
|
|
10368
|
+
);
|
|
9067
10369
|
return;
|
|
9068
10370
|
}
|
|
9069
10371
|
if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
|
|
@@ -9106,6 +10408,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9106
10408
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9107
10409
|
sessionOngoing: ongoing
|
|
9108
10410
|
})) {
|
|
10411
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
10412
|
+
inFlight.sessionStoreFailurePending = true;
|
|
10413
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
10414
|
+
watched_for_ms: pinnedForMs,
|
|
10415
|
+
run_state: "failed",
|
|
10416
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10417
|
+
});
|
|
10418
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10419
|
+
return;
|
|
10420
|
+
}
|
|
9109
10421
|
inFlight.ambiguousResolved = true;
|
|
9110
10422
|
this.log({
|
|
9111
10423
|
level: "warn",
|
|
@@ -9253,15 +10565,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9253
10565
|
}
|
|
9254
10566
|
}
|
|
9255
10567
|
/**
|
|
9256
|
-
* Settle a message whose run-state has resolved `'done'
|
|
9257
|
-
*
|
|
9258
|
-
*
|
|
9259
|
-
* b2-abandonment resolution) can reach the exact same completion behavior
|
|
9260
|
-
* (delivery-deadline anchoring, title resolution, usage extraction, and
|
|
9261
|
-
* `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
|
|
9262
|
-
* and risking the two copies silently drifting apart.
|
|
10568
|
+
* Settle a message whose run-state has resolved `'done'`. Normal completion and
|
|
10569
|
+
* confirmed b2 abandonment share the same delivery, usage, and terminal handling.
|
|
10570
|
+
* Normal completions omit the optional terminal diagnostic.
|
|
9263
10571
|
*/
|
|
9264
|
-
async settleMessageDone(sessionId, watcher, inFlight, messages) {
|
|
10572
|
+
async settleMessageDone(sessionId, watcher, inFlight, messages, note) {
|
|
9265
10573
|
const conv = watcher.conv;
|
|
9266
10574
|
this.anchorDeliveryDeadline(inFlight);
|
|
9267
10575
|
if (!inFlight.done) {
|
|
@@ -9288,7 +10596,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9288
10596
|
title,
|
|
9289
10597
|
usage,
|
|
9290
10598
|
usageAgentName,
|
|
9291
|
-
subagentInvocations
|
|
10599
|
+
subagentInvocations,
|
|
10600
|
+
note
|
|
9292
10601
|
);
|
|
9293
10602
|
} catch (err) {
|
|
9294
10603
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -9330,14 +10639,147 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9330
10639
|
}
|
|
9331
10640
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9332
10641
|
}
|
|
10642
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
10643
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
10644
|
+
const conv = watcher.conv;
|
|
10645
|
+
const error2 = composeAbandonmentNote(
|
|
10646
|
+
finish,
|
|
10647
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10648
|
+
"without-answer"
|
|
10649
|
+
);
|
|
10650
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10651
|
+
this.log({
|
|
10652
|
+
level: "warn",
|
|
10653
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
10654
|
+
conversation_id: conv.id,
|
|
10655
|
+
message_id: inFlight.evidentMessageId
|
|
10656
|
+
});
|
|
10657
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
10658
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
10659
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
10660
|
+
messages,
|
|
10661
|
+
inFlight.opencodeMessageId,
|
|
10662
|
+
inFlight.evidentMessageId
|
|
10663
|
+
);
|
|
10664
|
+
try {
|
|
10665
|
+
await this.markFailed(
|
|
10666
|
+
conv.id,
|
|
10667
|
+
inFlight.evidentMessageId,
|
|
10668
|
+
sessionId,
|
|
10669
|
+
error2,
|
|
10670
|
+
usage,
|
|
10671
|
+
void 0,
|
|
10672
|
+
usageAgentName,
|
|
10673
|
+
subagentInvocations
|
|
10674
|
+
);
|
|
10675
|
+
} catch (err) {
|
|
10676
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10677
|
+
if (err instanceof ChannelTerminalError) {
|
|
10678
|
+
this.log({
|
|
10679
|
+
level: "warn",
|
|
10680
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
10681
|
+
conversation_id: conv.id,
|
|
10682
|
+
message_id: inFlight.evidentMessageId
|
|
10683
|
+
});
|
|
10684
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10685
|
+
return;
|
|
10686
|
+
}
|
|
10687
|
+
if (this.now() >= inFlight.deadline) {
|
|
10688
|
+
this.log({
|
|
10689
|
+
level: "warn",
|
|
10690
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
10691
|
+
conversation_id: conv.id,
|
|
10692
|
+
message_id: inFlight.evidentMessageId
|
|
10693
|
+
});
|
|
10694
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10695
|
+
return;
|
|
10696
|
+
}
|
|
10697
|
+
this.log({
|
|
10698
|
+
level: "warn",
|
|
10699
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10700
|
+
conversation_id: conv.id,
|
|
10701
|
+
message_id: inFlight.evidentMessageId
|
|
10702
|
+
});
|
|
10703
|
+
return;
|
|
10704
|
+
}
|
|
10705
|
+
inFlight.done = true;
|
|
10706
|
+
await this.reportSubagentAuthFailures(
|
|
10707
|
+
watcher.conv.id,
|
|
10708
|
+
inFlight.opencodeMessageId,
|
|
10709
|
+
inFlight.evidentMessageId,
|
|
10710
|
+
messages
|
|
10711
|
+
);
|
|
10712
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10713
|
+
}
|
|
10714
|
+
/**
|
|
10715
|
+
* Report a queued message as failed after the never-started bound. The accepted
|
|
10716
|
+
* OpenCode copy is not cancelled, so the local fence remains on unsuccessful
|
|
10717
|
+
* terminal-report attempts to prevent a blind re-prompt.
|
|
10718
|
+
*/
|
|
10719
|
+
async settleMessageNeverStarted(sessionId, watcher, inFlight, attempts) {
|
|
10720
|
+
const conv = watcher.conv;
|
|
10721
|
+
const error2 = `OpenCode accepted this message but never started running it; after ${attempts} attempts it was still queued. Evident is reporting the run as failed. The copy OpenCode accepted has not been cancelled and may still run if OpenCode recovers.`;
|
|
10722
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10723
|
+
try {
|
|
10724
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10725
|
+
} catch (err) {
|
|
10726
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10727
|
+
if (err instanceof ChannelTerminalError) {
|
|
10728
|
+
this.log({
|
|
10729
|
+
level: "warn",
|
|
10730
|
+
message: `Failed to report never-started message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10731
|
+
conversation_id: conv.id,
|
|
10732
|
+
message_id: inFlight.evidentMessageId
|
|
10733
|
+
});
|
|
10734
|
+
this.recordReleasedOpencodeId(
|
|
10735
|
+
inFlight.evidentMessageId,
|
|
10736
|
+
sessionId,
|
|
10737
|
+
inFlight.opencodeMessageId
|
|
10738
|
+
);
|
|
10739
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10740
|
+
return;
|
|
10741
|
+
}
|
|
10742
|
+
if (this.now() >= inFlight.deadline) {
|
|
10743
|
+
this.log({
|
|
10744
|
+
level: "warn",
|
|
10745
|
+
message: `Failed to report never-started message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10746
|
+
conversation_id: conv.id,
|
|
10747
|
+
message_id: inFlight.evidentMessageId
|
|
10748
|
+
});
|
|
10749
|
+
this.recordReleasedOpencodeId(
|
|
10750
|
+
inFlight.evidentMessageId,
|
|
10751
|
+
sessionId,
|
|
10752
|
+
inFlight.opencodeMessageId
|
|
10753
|
+
);
|
|
10754
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10755
|
+
return;
|
|
10756
|
+
}
|
|
10757
|
+
this.log({
|
|
10758
|
+
level: "warn",
|
|
10759
|
+
message: `Failed to report never-started message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10760
|
+
conversation_id: conv.id,
|
|
10761
|
+
message_id: inFlight.evidentMessageId
|
|
10762
|
+
});
|
|
10763
|
+
return;
|
|
10764
|
+
}
|
|
10765
|
+
inFlight.done = true;
|
|
10766
|
+
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
10767
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10768
|
+
}
|
|
9333
10769
|
/**
|
|
9334
|
-
* Report a
|
|
9335
|
-
*
|
|
9336
|
-
*
|
|
10770
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10771
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10772
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
9337
10773
|
*/
|
|
9338
|
-
async
|
|
10774
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
9339
10775
|
const conv = watcher.conv;
|
|
9340
|
-
const error2 =
|
|
10776
|
+
const error2 = SESSION_STORE_UNWRITABLE_ERROR;
|
|
10777
|
+
this.log({
|
|
10778
|
+
level: "error",
|
|
10779
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10780
|
+
conversation_id: conv.id,
|
|
10781
|
+
message_id: inFlight.evidentMessageId
|
|
10782
|
+
});
|
|
9341
10783
|
this.anchorDeliveryDeadline(inFlight);
|
|
9342
10784
|
try {
|
|
9343
10785
|
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
@@ -9346,7 +10788,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9346
10788
|
if (err instanceof ChannelTerminalError) {
|
|
9347
10789
|
this.log({
|
|
9348
10790
|
level: "warn",
|
|
9349
|
-
message: `Failed to report
|
|
10791
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
9350
10792
|
conversation_id: conv.id,
|
|
9351
10793
|
message_id: inFlight.evidentMessageId
|
|
9352
10794
|
});
|
|
@@ -9361,7 +10803,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9361
10803
|
if (this.now() >= inFlight.deadline) {
|
|
9362
10804
|
this.log({
|
|
9363
10805
|
level: "warn",
|
|
9364
|
-
message: `Failed to report
|
|
10806
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
9365
10807
|
conversation_id: conv.id,
|
|
9366
10808
|
message_id: inFlight.evidentMessageId
|
|
9367
10809
|
});
|
|
@@ -9375,14 +10817,13 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9375
10817
|
}
|
|
9376
10818
|
this.log({
|
|
9377
10819
|
level: "warn",
|
|
9378
|
-
message: `Failed to report
|
|
10820
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
9379
10821
|
conversation_id: conv.id,
|
|
9380
10822
|
message_id: inFlight.evidentMessageId
|
|
9381
10823
|
});
|
|
9382
10824
|
return;
|
|
9383
10825
|
}
|
|
9384
10826
|
inFlight.done = true;
|
|
9385
|
-
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9386
10827
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9387
10828
|
}
|
|
9388
10829
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
@@ -9423,11 +10864,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9423
10864
|
}
|
|
9424
10865
|
}
|
|
9425
10866
|
}
|
|
9426
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10867
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9427
10868
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10869
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10870
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10871
|
+
this.awaitingCorrelation.delete(id);
|
|
10872
|
+
this.log({
|
|
10873
|
+
level: "debug",
|
|
10874
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10875
|
+
message_id: id
|
|
10876
|
+
});
|
|
10877
|
+
}
|
|
10878
|
+
}
|
|
9428
10879
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9429
10880
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9430
10881
|
this.untrackableAck.delete(id);
|
|
10882
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9431
10883
|
this.untrackableAckSignalled.delete(id);
|
|
9432
10884
|
this.untrackableAckWarned.delete(id);
|
|
9433
10885
|
this.log({
|
|
@@ -9467,7 +10919,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9467
10919
|
bySession.set(row.opencode_session_id, list);
|
|
9468
10920
|
}
|
|
9469
10921
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9470
|
-
let messages;
|
|
10922
|
+
let messages = null;
|
|
9471
10923
|
try {
|
|
9472
10924
|
if (this.isV2) {
|
|
9473
10925
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9476,9 +10928,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9476
10928
|
level: "warn",
|
|
9477
10929
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9478
10930
|
});
|
|
9479
|
-
|
|
10931
|
+
} else {
|
|
10932
|
+
messages = snapshot;
|
|
9480
10933
|
}
|
|
9481
|
-
messages = snapshot;
|
|
9482
10934
|
} else {
|
|
9483
10935
|
const polled = await pollSessionMessagesForRedrive(
|
|
9484
10936
|
this.port,
|
|
@@ -9491,15 +10943,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9491
10943
|
level: "warn",
|
|
9492
10944
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9493
10945
|
});
|
|
9494
|
-
|
|
10946
|
+
} else {
|
|
10947
|
+
messages = polled.messages;
|
|
9495
10948
|
}
|
|
9496
|
-
messages = polled.messages;
|
|
9497
10949
|
}
|
|
9498
10950
|
} catch (err) {
|
|
9499
10951
|
this.log({
|
|
9500
10952
|
level: "warn",
|
|
9501
10953
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9502
10954
|
});
|
|
10955
|
+
}
|
|
10956
|
+
if (messages === null) {
|
|
10957
|
+
for (const row of sessionRows) {
|
|
10958
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10959
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10960
|
+
}
|
|
10961
|
+
}
|
|
9503
10962
|
continue;
|
|
9504
10963
|
}
|
|
9505
10964
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9509,6 +10968,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9509
10968
|
}
|
|
9510
10969
|
}
|
|
9511
10970
|
}
|
|
10971
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10972
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10973
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10974
|
+
if (!correlation) return false;
|
|
10975
|
+
correlation.attempts += 1;
|
|
10976
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10977
|
+
if (correlatedId !== null) {
|
|
10978
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10979
|
+
const conv = this.convForRow(sessionId, row);
|
|
10980
|
+
const message = this.queuedMessageForRow(row);
|
|
10981
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10982
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10983
|
+
this.dispatched.add(row.id);
|
|
10984
|
+
this.readopted.add(row.id);
|
|
10985
|
+
this.ensureWatcherRunning(sessionId);
|
|
10986
|
+
this.log({
|
|
10987
|
+
level: "info",
|
|
10988
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10989
|
+
conversation_id: row.conversation_id,
|
|
10990
|
+
message_id: row.id
|
|
10991
|
+
});
|
|
10992
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10993
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10994
|
+
opencode_message_id: correlatedId
|
|
10995
|
+
});
|
|
10996
|
+
return true;
|
|
10997
|
+
}
|
|
10998
|
+
if (correlation.knownUserIds === null) {
|
|
10999
|
+
this.log({
|
|
11000
|
+
level: "debug",
|
|
11001
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
11002
|
+
conversation_id: row.conversation_id,
|
|
11003
|
+
message_id: row.id
|
|
11004
|
+
});
|
|
11005
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
11006
|
+
this.log({
|
|
11007
|
+
level: "warn",
|
|
11008
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
11009
|
+
conversation_id: row.conversation_id,
|
|
11010
|
+
message_id: row.id
|
|
11011
|
+
});
|
|
11012
|
+
}
|
|
11013
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
11014
|
+
await this.handoffUntrackableCorrelation(
|
|
11015
|
+
row.conversation_id,
|
|
11016
|
+
row.id,
|
|
11017
|
+
correlation.sessionId,
|
|
11018
|
+
correlation.attempts,
|
|
11019
|
+
"readopt"
|
|
11020
|
+
);
|
|
11021
|
+
}
|
|
11022
|
+
return true;
|
|
11023
|
+
}
|
|
9512
11024
|
/**
|
|
9513
11025
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9514
11026
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9516,8 +11028,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9516
11028
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
9517
11029
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
9518
11030
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
9519
|
-
* before the restart)
|
|
9520
|
-
*
|
|
11031
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
11032
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
11033
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
11034
|
+
* `forceReadoptRun`):
|
|
9521
11035
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
9522
11036
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
9523
11037
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -9526,7 +11040,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9526
11040
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
9527
11041
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
9528
11042
|
* tracking the stored id so the reply correlates by it;
|
|
9529
|
-
* - `unknown`/null id →
|
|
11043
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
11044
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
9530
11045
|
*
|
|
9531
11046
|
* Only `ChannelAuthError` propagates.
|
|
9532
11047
|
*/
|
|
@@ -9540,6 +11055,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9540
11055
|
});
|
|
9541
11056
|
return;
|
|
9542
11057
|
}
|
|
11058
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9543
11059
|
const ocId = row.opencode_message_id;
|
|
9544
11060
|
const state = messageRunState(messages, ocId ?? "");
|
|
9545
11061
|
if (state === "done") {
|
|
@@ -9606,12 +11122,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9606
11122
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
9607
11123
|
this.dontRedispatch.delete(row.id);
|
|
9608
11124
|
this.untrackableAck.delete(row.id);
|
|
11125
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9609
11126
|
this.untrackableAckSignalled.delete(row.id);
|
|
9610
11127
|
this.untrackableAckWarned.delete(row.id);
|
|
9611
11128
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
9612
11129
|
return;
|
|
9613
11130
|
}
|
|
9614
|
-
if (this.
|
|
11131
|
+
if (this.untrackableAck.has(row.id)) {
|
|
11132
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
11133
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
11134
|
+
this.untrackableAckWarned.add(row.id);
|
|
11135
|
+
this.log({
|
|
11136
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
11137
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
11138
|
+
conversation_id: row.conversation_id,
|
|
11139
|
+
message_id: row.id
|
|
11140
|
+
});
|
|
11141
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
11142
|
+
const reportError = await this.reportUntrackableAck(
|
|
11143
|
+
row.conversation_id,
|
|
11144
|
+
row.id,
|
|
11145
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
11146
|
+
errorMessage3
|
|
11147
|
+
);
|
|
11148
|
+
if (reportError !== null) {
|
|
11149
|
+
this.log({
|
|
11150
|
+
level: "warn",
|
|
11151
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
11152
|
+
conversation_id: row.conversation_id,
|
|
11153
|
+
message_id: row.id
|
|
11154
|
+
});
|
|
11155
|
+
}
|
|
11156
|
+
} else {
|
|
11157
|
+
this.log({
|
|
11158
|
+
level: "debug",
|
|
11159
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
11160
|
+
conversation_id: row.conversation_id,
|
|
11161
|
+
message_id: row.id
|
|
11162
|
+
});
|
|
11163
|
+
}
|
|
11164
|
+
return;
|
|
11165
|
+
}
|
|
11166
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
9615
11167
|
this.log({
|
|
9616
11168
|
level: "debug",
|
|
9617
11169
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -9789,6 +11341,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9789
11341
|
}
|
|
9790
11342
|
this.dontRedispatch.delete(row.id);
|
|
9791
11343
|
this.untrackableAck.delete(row.id);
|
|
11344
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9792
11345
|
this.untrackableAckSignalled.delete(row.id);
|
|
9793
11346
|
this.untrackableAckWarned.delete(row.id);
|
|
9794
11347
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -9808,8 +11361,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9808
11361
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
9809
11362
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
9810
11363
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
9811
|
-
* re-entry)
|
|
9812
|
-
*
|
|
11364
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
11365
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
11366
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
9813
11367
|
*
|
|
9814
11368
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
9815
11369
|
* `processed_at` (Invariant 1).
|
|
@@ -9872,6 +11426,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9872
11426
|
);
|
|
9873
11427
|
}
|
|
9874
11428
|
let ocId;
|
|
11429
|
+
let preDispatchUserIds = null;
|
|
9875
11430
|
try {
|
|
9876
11431
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
9877
11432
|
sessionId,
|
|
@@ -9881,7 +11436,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9881
11436
|
row.content,
|
|
9882
11437
|
options,
|
|
9883
11438
|
sendAttachments,
|
|
9884
|
-
this.openCodeClient
|
|
11439
|
+
this.openCodeClient,
|
|
11440
|
+
{
|
|
11441
|
+
onKnownUserIds: (ids) => {
|
|
11442
|
+
preDispatchUserIds = ids;
|
|
11443
|
+
},
|
|
11444
|
+
onKnownUserIdsError: (error2) => {
|
|
11445
|
+
this.log({
|
|
11446
|
+
level: "warn",
|
|
11447
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
11448
|
+
conversation_id: row.conversation_id,
|
|
11449
|
+
message_id: row.id
|
|
11450
|
+
});
|
|
11451
|
+
}
|
|
11452
|
+
}
|
|
9885
11453
|
)
|
|
9886
11454
|
);
|
|
9887
11455
|
} catch (err) {
|
|
@@ -9914,6 +11482,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9914
11482
|
}
|
|
9915
11483
|
} else {
|
|
9916
11484
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
11485
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
9917
11486
|
}
|
|
9918
11487
|
this.log({
|
|
9919
11488
|
level: "error",
|
|
@@ -9962,39 +11531,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9962
11531
|
}
|
|
9963
11532
|
if (ocId === null) {
|
|
9964
11533
|
this.awaitingReadopt.delete(row.id);
|
|
9965
|
-
|
|
9966
|
-
|
|
9967
|
-
|
|
9968
|
-
|
|
9969
|
-
|
|
9970
|
-
|
|
9971
|
-
this.log({
|
|
9972
|
-
level: "error",
|
|
9973
|
-
message: errorMessage3,
|
|
9974
|
-
conversation_id: row.conversation_id,
|
|
9975
|
-
message_id: row.id
|
|
9976
|
-
});
|
|
9977
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
9978
|
-
this.log({
|
|
9979
|
-
level: "warn",
|
|
9980
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
9981
|
-
conversation_id: row.conversation_id,
|
|
9982
|
-
message_id: row.id
|
|
9983
|
-
});
|
|
9984
|
-
});
|
|
9985
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
9986
|
-
return;
|
|
9987
|
-
}
|
|
11534
|
+
this.awaitingCorrelation.set(row.id, {
|
|
11535
|
+
sessionId,
|
|
11536
|
+
content: row.content,
|
|
11537
|
+
attempts: 0,
|
|
11538
|
+
knownUserIds: preDispatchUserIds
|
|
11539
|
+
});
|
|
9988
11540
|
this.log({
|
|
9989
11541
|
level: "warn",
|
|
9990
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
11542
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
9991
11543
|
conversation_id: row.conversation_id,
|
|
9992
11544
|
message_id: row.id
|
|
9993
11545
|
});
|
|
9994
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
9995
11546
|
return;
|
|
9996
11547
|
}
|
|
9997
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
9998
11548
|
const processedAtMs = this.processedAtMs(row);
|
|
9999
11549
|
this.registerReadopted(
|
|
10000
11550
|
readoptConv,
|
|
@@ -10333,14 +11883,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10333
11883
|
const fetchChildWithoutBlocking = async (sessionId) => {
|
|
10334
11884
|
const pending = fetchChildMessages(sessionId);
|
|
10335
11885
|
let timer;
|
|
10336
|
-
const timeout = new Promise((
|
|
11886
|
+
const timeout = new Promise((resolve5) => {
|
|
10337
11887
|
timer = setTimeout(() => {
|
|
10338
11888
|
this.log({
|
|
10339
11889
|
level: "warn",
|
|
10340
11890
|
message: `Best-effort subagent session fetch for message ${messageId.slice(0, 8)} and child session ${sessionId.slice(0, 8)} was slow \u2014 omitting invocation telemetry without delaying completion`,
|
|
10341
11891
|
message_id: messageId
|
|
10342
11892
|
});
|
|
10343
|
-
|
|
11893
|
+
resolve5(null);
|
|
10344
11894
|
}, SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS);
|
|
10345
11895
|
});
|
|
10346
11896
|
try {
|
|
@@ -10900,7 +12450,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10900
12450
|
* watcher retries next tick within the
|
|
10901
12451
|
* deadline, Finding 4).
|
|
10902
12452
|
*/
|
|
10903
|
-
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations) {
|
|
12453
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations, error2) {
|
|
10904
12454
|
const res = await this.fetchImpl(
|
|
10905
12455
|
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
10906
12456
|
{
|
|
@@ -10918,7 +12468,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10918
12468
|
...title ? { title } : {},
|
|
10919
12469
|
...usage ? usage : {},
|
|
10920
12470
|
...usageAgentName ? { usage_agent_name: usageAgentName } : {},
|
|
10921
|
-
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {}
|
|
12471
|
+
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {},
|
|
12472
|
+
...error2 ? { error: error2 } : {}
|
|
10922
12473
|
})
|
|
10923
12474
|
}
|
|
10924
12475
|
);
|
|
@@ -10978,11 +12529,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10978
12529
|
if (subagentInvocations && subagentInvocations.length > 0) {
|
|
10979
12530
|
body.subagent_invocations = subagentInvocations;
|
|
10980
12531
|
}
|
|
10981
|
-
if (failure) {
|
|
10982
|
-
body.
|
|
10983
|
-
|
|
10984
|
-
|
|
10985
|
-
body.
|
|
12532
|
+
if (failure && failure.errorShape.name !== null) {
|
|
12533
|
+
body.failure_error_name = failure.errorShape.name;
|
|
12534
|
+
}
|
|
12535
|
+
if (failure && failure.errorShape.statusCode !== null) {
|
|
12536
|
+
body.failure_status_code = failure.errorShape.statusCode;
|
|
12537
|
+
}
|
|
12538
|
+
if (failure?.classification) {
|
|
12539
|
+
body.failure_kind = failure.classification.kind;
|
|
12540
|
+
body.failure_provider_id = failure.classification.providerId;
|
|
12541
|
+
body.failure_model_id = failure.classification.modelId;
|
|
12542
|
+
body.failure_reason = failure.classification.reason;
|
|
10986
12543
|
}
|
|
10987
12544
|
try {
|
|
10988
12545
|
await this.callWithRetry(
|
|
@@ -11003,27 +12560,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11003
12560
|
this.clearSubagentInvocationCaches(messageId);
|
|
11004
12561
|
}
|
|
11005
12562
|
/**
|
|
11006
|
-
* Classify an errored turn
|
|
12563
|
+
* Classify an errored turn and extract its bounded error shape for `markFailed`.
|
|
11007
12564
|
*
|
|
11008
12565
|
* `messageFailure` alone (structured OpenCode error → `model_auth`) covers
|
|
11009
12566
|
* most cases; when it returns `null` on this ALREADY-FAILED turn, fall back
|
|
11010
12567
|
* to the P1-2b zero-provider check — one extra loopback call to
|
|
11011
12568
|
* `hasAnyConfiguredProvider`, only reached when the structured classifier
|
|
11012
12569
|
* couldn't place it. Fails open (never throws): a fallback probe failure
|
|
11013
|
-
* (`null`/indeterminate) leaves the classification `null
|
|
11014
|
-
*
|
|
12570
|
+
* (`null`/indeterminate) leaves the classification `null`; the raw error shape
|
|
12571
|
+
* is still retained for diagnostics.
|
|
11015
12572
|
*/
|
|
11016
12573
|
async classifyModelAuthFailure(messages, userMessageId) {
|
|
11017
|
-
const classified = messageFailure(messages, userMessageId);
|
|
11018
|
-
if (classified != null) return classified;
|
|
11019
12574
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
11020
|
-
const
|
|
11021
|
-
|
|
12575
|
+
const errorShape = extractReplyErrorShape(reply);
|
|
12576
|
+
const classified = messageFailure(messages, userMessageId);
|
|
12577
|
+
if (classified != null) return { classification: classified, errorShape };
|
|
12578
|
+
const classification = applyZeroProviderFallback(
|
|
11022
12579
|
classified,
|
|
11023
|
-
|
|
12580
|
+
await this.hasAnyConfiguredProvider(),
|
|
11024
12581
|
reply?.info?.providerID ?? null,
|
|
11025
12582
|
reply?.info?.modelID ?? null
|
|
11026
12583
|
);
|
|
12584
|
+
return { classification, errorShape };
|
|
11027
12585
|
}
|
|
11028
12586
|
async recordSubagentModelAuthFailure(failure, conversationId, messageId) {
|
|
11029
12587
|
const providerId = failure.providerId ?? "(unknown)";
|
|
@@ -11581,7 +13139,7 @@ function commandError2(result) {
|
|
|
11581
13139
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
11582
13140
|
}
|
|
11583
13141
|
var runCommand2 = (command, args, opts) => {
|
|
11584
|
-
return new Promise((
|
|
13142
|
+
return new Promise((resolve5) => {
|
|
11585
13143
|
let child;
|
|
11586
13144
|
let stdout = "";
|
|
11587
13145
|
let stderr = "";
|
|
@@ -11591,7 +13149,7 @@ var runCommand2 = (command, args, opts) => {
|
|
|
11591
13149
|
if (settled) return;
|
|
11592
13150
|
settled = true;
|
|
11593
13151
|
if (timer.handle) clearTimeout(timer.handle);
|
|
11594
|
-
|
|
13152
|
+
resolve5(result);
|
|
11595
13153
|
};
|
|
11596
13154
|
try {
|
|
11597
13155
|
child = spawn5(command, args, { env: opts.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
@@ -11646,7 +13204,9 @@ async function installRunnerSecret({
|
|
|
11646
13204
|
}) {
|
|
11647
13205
|
const arn = env.RUNNER_SECRET_ARN?.trim();
|
|
11648
13206
|
if (!arn) {
|
|
11649
|
-
|
|
13207
|
+
if (!env.GH_TOKEN) {
|
|
13208
|
+
log3("runner secret is not configured; continuing without GitHub and MCP credentials");
|
|
13209
|
+
}
|
|
11650
13210
|
return false;
|
|
11651
13211
|
}
|
|
11652
13212
|
const result = await (commandRunner ?? runCommand2)(
|
|
@@ -11845,7 +13405,7 @@ async function configureGitHubAccess({ env, log: log3 }) {
|
|
|
11845
13405
|
// src/lib/opencode/config-overlay.ts
|
|
11846
13406
|
import { execFileSync as execFileSync2 } from "node:child_process";
|
|
11847
13407
|
import { copyFileSync, existsSync as existsSync2, statSync as statSync6 } from "node:fs";
|
|
11848
|
-
import { isAbsolute as
|
|
13408
|
+
import { isAbsolute as isAbsolute3, join as join11, resolve as resolve4 } from "node:path";
|
|
11849
13409
|
function isFile(filePath) {
|
|
11850
13410
|
return existsSync2(filePath) && statSync6(filePath).isFile();
|
|
11851
13411
|
}
|
|
@@ -11858,8 +13418,8 @@ function applyRunnerOpenCodeConfig({
|
|
|
11858
13418
|
log3("runner OpenCode config is not configured; using the baked project config", "debug");
|
|
11859
13419
|
return;
|
|
11860
13420
|
}
|
|
11861
|
-
const source =
|
|
11862
|
-
const target = isFile(
|
|
13421
|
+
const source = isAbsolute3(overlayPath) ? overlayPath : resolve4(cwd, overlayPath);
|
|
13422
|
+
const target = isFile(join11(cwd, "opencode.jsonc")) ? "opencode.jsonc" : "opencode.json";
|
|
11863
13423
|
if (!isFile(source)) {
|
|
11864
13424
|
log3(
|
|
11865
13425
|
`RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)`,
|
|
@@ -11867,7 +13427,7 @@ function applyRunnerOpenCodeConfig({
|
|
|
11867
13427
|
);
|
|
11868
13428
|
return;
|
|
11869
13429
|
}
|
|
11870
|
-
copyFileSync(source,
|
|
13430
|
+
copyFileSync(source, join11(cwd, target));
|
|
11871
13431
|
try {
|
|
11872
13432
|
execFileSync2("git", ["-C", cwd, "update-index", "--skip-worktree", target], {
|
|
11873
13433
|
stdio: "ignore"
|
|
@@ -11876,7 +13436,7 @@ function applyRunnerOpenCodeConfig({
|
|
|
11876
13436
|
const detail = error2 instanceof Error ? error2.message : String(error2);
|
|
11877
13437
|
log3(`could not mark ${target} skip-worktree: ${detail}; it may show as a local change`, "warn");
|
|
11878
13438
|
}
|
|
11879
|
-
log3(`Applied runner OpenCode config ${source} to ${
|
|
13439
|
+
log3(`Applied runner OpenCode config ${source} to ${join11(cwd, target)}`);
|
|
11880
13440
|
}
|
|
11881
13441
|
|
|
11882
13442
|
// src/lib/credential-sync.ts
|
|
@@ -11894,14 +13454,14 @@ function errorMessage2(error2) {
|
|
|
11894
13454
|
return error2 instanceof Error ? error2.message : String(error2);
|
|
11895
13455
|
}
|
|
11896
13456
|
function waitForSettlement(promise, timeoutMs) {
|
|
11897
|
-
return new Promise((
|
|
13457
|
+
return new Promise((resolve5) => {
|
|
11898
13458
|
let settled = false;
|
|
11899
13459
|
const timer = setTimeout(() => finish(false), Math.max(0, timeoutMs));
|
|
11900
13460
|
const finish = (value) => {
|
|
11901
13461
|
if (settled) return;
|
|
11902
13462
|
settled = true;
|
|
11903
13463
|
clearTimeout(timer);
|
|
11904
|
-
|
|
13464
|
+
resolve5(value);
|
|
11905
13465
|
};
|
|
11906
13466
|
promise.then(
|
|
11907
13467
|
() => finish(true),
|
|
@@ -12116,6 +13676,7 @@ function createCredentialSync({
|
|
|
12116
13676
|
|
|
12117
13677
|
// src/commands/run.ts
|
|
12118
13678
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
13679
|
+
var MAX_DISCONNECT_REASON_LENGTH = 200;
|
|
12119
13680
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
12120
13681
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
12121
13682
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
@@ -12151,8 +13712,8 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
12151
13712
|
if (trimmed === "") {
|
|
12152
13713
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
12153
13714
|
}
|
|
12154
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
12155
|
-
if (!
|
|
13715
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join12(homeDir, trimmed.slice(2)) : trimmed;
|
|
13716
|
+
if (!isAbsolute4(expanded)) {
|
|
12156
13717
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
12157
13718
|
}
|
|
12158
13719
|
const normalized = resolvePath2(expanded);
|
|
@@ -12252,8 +13813,7 @@ function resolveMaxActiveSessions(options, env = process.env) {
|
|
|
12252
13813
|
function meetsThreshold(state, level) {
|
|
12253
13814
|
return LOG_LEVELS[level] >= LOG_LEVELS[state.logLevel];
|
|
12254
13815
|
}
|
|
12255
|
-
function
|
|
12256
|
-
if (!meetsThreshold(state, level)) return;
|
|
13816
|
+
function writeConsoleLog(state, message, level) {
|
|
12257
13817
|
if (state.json) {
|
|
12258
13818
|
console.log(
|
|
12259
13819
|
JSON.stringify({
|
|
@@ -12262,16 +13822,30 @@ function log2(state, message, level = "info") {
|
|
|
12262
13822
|
message
|
|
12263
13823
|
})
|
|
12264
13824
|
);
|
|
12265
|
-
|
|
12266
|
-
const prefix = level === "error" ? chalk7.red("\u2717") : level === "warn" ? chalk7.yellow("!") : level === "debug" ? chalk7.dim("\xB7") : chalk7.green("\u2022");
|
|
12267
|
-
console.log(`${prefix} ${message}`);
|
|
13825
|
+
return;
|
|
12268
13826
|
}
|
|
13827
|
+
const prefix = level === "error" ? chalk7.red("\u2717") : level === "warn" ? chalk7.yellow("!") : level === "debug" ? chalk7.dim("\xB7") : chalk7.green("\u2022");
|
|
13828
|
+
console.log(`${prefix} ${message}`);
|
|
13829
|
+
}
|
|
13830
|
+
function log2(state, message, level = "info") {
|
|
13831
|
+
if (!meetsThreshold(state, level)) return;
|
|
13832
|
+
if (state.json || !state.interactive) writeConsoleLog(state, message, level);
|
|
13833
|
+
}
|
|
13834
|
+
function logLocalOnly(state, message, level = "info") {
|
|
13835
|
+
if (!meetsThreshold(state, level)) return;
|
|
13836
|
+
writeConsoleLog(state, message, level);
|
|
12269
13837
|
}
|
|
12270
13838
|
function logActivity(state, entry) {
|
|
12271
13839
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
12272
13840
|
if (!meetsThreshold(state, level)) return;
|
|
12273
13841
|
forwardRunnerActivity(
|
|
12274
|
-
{
|
|
13842
|
+
{
|
|
13843
|
+
level,
|
|
13844
|
+
message: entry.message,
|
|
13845
|
+
error: entry.error,
|
|
13846
|
+
metadata: entry.metadata,
|
|
13847
|
+
...entry.source ? { source: entry.source } : {}
|
|
13848
|
+
},
|
|
12275
13849
|
{ agentId: state.agentId, authHeader: state.authHeader }
|
|
12276
13850
|
);
|
|
12277
13851
|
const fullEntry = {
|
|
@@ -12293,7 +13867,7 @@ function logActivity(state, entry) {
|
|
|
12293
13867
|
}
|
|
12294
13868
|
function reportSessionDbRecovery(state) {
|
|
12295
13869
|
try {
|
|
12296
|
-
const report = drainSessionDbRecoveryReport({ homeDir:
|
|
13870
|
+
const report = drainSessionDbRecoveryReport({ homeDir: homedir7(), env: process.env });
|
|
12297
13871
|
for (const record of report.records) {
|
|
12298
13872
|
const activity = buildSessionDbRecoveryActivity(record);
|
|
12299
13873
|
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
@@ -12328,7 +13902,9 @@ function displayStatus(state) {
|
|
|
12328
13902
|
const opencode = state.opencodeConnected ? chalk7.green(`opencode: :${state.port}`) : chalk7.red(`opencode: :${state.port} (down)`);
|
|
12329
13903
|
const messages = state.messageCount > 0 ? chalk7.dim(` \xB7 ${state.messageCount} processed`) : "";
|
|
12330
13904
|
const last = state.activityLog[state.activityLog.length - 1];
|
|
12331
|
-
const detail = last ? chalk7.dim(
|
|
13905
|
+
const detail = last ? chalk7.dim(
|
|
13906
|
+
` \xB7 ${last.timestamp.toISOString()} ${last.type === "error" ? last.error ?? "" : last.message ?? ""}`
|
|
13907
|
+
) : "";
|
|
12332
13908
|
const agent = state.agentName ?? state.agentId;
|
|
12333
13909
|
console.log(
|
|
12334
13910
|
`${chalk7.bold("Evident")} ${chalk7.dim(agent)} ${tunnel} ${opencode}${messages}${detail}`
|
|
@@ -12347,7 +13923,7 @@ function reportOpenCodeV2CredentialImport(state, result) {
|
|
|
12347
13923
|
logActivity(state, {
|
|
12348
13924
|
type: "info",
|
|
12349
13925
|
level: "warn",
|
|
12350
|
-
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them.
|
|
13926
|
+
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them. Connect the provider again from the runner's page in Evident; no shell on this machine is needed.`
|
|
12351
13927
|
});
|
|
12352
13928
|
}
|
|
12353
13929
|
if (result.imported.length === 0 && result.unsupported.length === 0 && result.failed.length === 0) {
|
|
@@ -12372,7 +13948,7 @@ async function importOpenCodeV2CredentialsForState(state) {
|
|
|
12372
13948
|
}
|
|
12373
13949
|
const result = await importOpenCodeV2Credentials({
|
|
12374
13950
|
request: client.request,
|
|
12375
|
-
homeDir:
|
|
13951
|
+
homeDir: homedir7(),
|
|
12376
13952
|
directory,
|
|
12377
13953
|
guard: state.opencodeV2CredentialImportGuard,
|
|
12378
13954
|
log: (message, level = "info") => logActivity(state, { type: "info", level, message })
|
|
@@ -12458,6 +14034,8 @@ async function driveChannels(state, driver) {
|
|
|
12458
14034
|
let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
12459
14035
|
let lastSeenClaudeApplies = driver.fileSyncActivity().claudeCredentialApplies;
|
|
12460
14036
|
let lastSeenOpencodeAuthApplies = driver.fileSyncActivity().opencodeAuthApplyCount;
|
|
14037
|
+
const stayAwakeHolds = () => state.stayAwakeUntilMs !== null && Date.now() < state.stayAwakeUntilMs;
|
|
14038
|
+
let stayAwakeSuppressionLogged = false;
|
|
12461
14039
|
while (state.running) {
|
|
12462
14040
|
const cycleStartedAtMs = performance.now();
|
|
12463
14041
|
let idleThisCycle = false;
|
|
@@ -12559,7 +14137,7 @@ async function driveChannels(state, driver) {
|
|
|
12559
14137
|
}
|
|
12560
14138
|
}
|
|
12561
14139
|
}
|
|
12562
|
-
await new Promise((
|
|
14140
|
+
await new Promise((resolve5) => setTimeout(resolve5, CHANNEL_POLL_INTERVAL_MS));
|
|
12563
14141
|
const cycleMs = performance.now() - cycleStartedAtMs;
|
|
12564
14142
|
if (idleThisCycle) idleMs += cycleMs;
|
|
12565
14143
|
if (unreachableThisCycle) unreachableMs += cycleMs;
|
|
@@ -12572,17 +14150,29 @@ async function driveChannels(state, driver) {
|
|
|
12572
14150
|
if (state.interactive) displayStatus(state);
|
|
12573
14151
|
break;
|
|
12574
14152
|
}
|
|
12575
|
-
if (state.idleTimeout !== null && idlePolls >= 2 && idleMs > state.idleTimeout * 1e3) {
|
|
14153
|
+
if (state.idleTimeout !== null && idlePolls >= 2 && idleMs > state.idleTimeout * 1e3 && !stayAwakeHolds()) {
|
|
12576
14154
|
logActivity(state, { type: "info", message: "Idle timeout reached" });
|
|
12577
14155
|
if (state.interactive) displayStatus(state);
|
|
12578
14156
|
break;
|
|
12579
14157
|
}
|
|
14158
|
+
if (state.idleTimeout !== null && idlePolls >= 2 && idleMs > state.idleTimeout * 1e3 && stayAwakeHolds()) {
|
|
14159
|
+
if (!stayAwakeSuppressionLogged) {
|
|
14160
|
+
stayAwakeSuppressionLogged = true;
|
|
14161
|
+
logActivity(state, {
|
|
14162
|
+
type: "info",
|
|
14163
|
+
message: "Idle, but staying awake until the requested minimum passes"
|
|
14164
|
+
});
|
|
14165
|
+
}
|
|
14166
|
+
}
|
|
12580
14167
|
}
|
|
12581
14168
|
}
|
|
12582
14169
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12583
|
-
var
|
|
14170
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
14171
|
+
var SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES = 2;
|
|
14172
|
+
var SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES = 2;
|
|
14173
|
+
var MAX_SESSION_DELETE_BODY_LENGTH = 200;
|
|
12584
14174
|
function shouldWarnForReclaimSkip(reason) {
|
|
12585
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
14175
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12586
14176
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12587
14177
|
if (!version2) return false;
|
|
12588
14178
|
const major = Number(version2[1]);
|
|
@@ -12591,7 +14181,14 @@ function shouldWarnForReclaimSkip(reason) {
|
|
|
12591
14181
|
return major > 22 || major === 22 && (minor > 13 || minor === 13 && patch >= 0);
|
|
12592
14182
|
}
|
|
12593
14183
|
function sessionDbPath() {
|
|
12594
|
-
return
|
|
14184
|
+
return join12(homedir7(), ".local", "share", "opencode", "opencode.db");
|
|
14185
|
+
}
|
|
14186
|
+
function logSessionCleanupActivity(state, phase, entry) {
|
|
14187
|
+
logActivity(state, {
|
|
14188
|
+
...entry,
|
|
14189
|
+
source: "cli.session-cleanup",
|
|
14190
|
+
metadata: { ...entry.metadata, phase }
|
|
14191
|
+
});
|
|
12595
14192
|
}
|
|
12596
14193
|
function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBootMigrationCount) {
|
|
12597
14194
|
const record = {
|
|
@@ -12632,69 +14229,187 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12632
14229
|
async function runSweep(state, driver, config) {
|
|
12633
14230
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12634
14231
|
try {
|
|
12635
|
-
const
|
|
12636
|
-
|
|
14232
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
14233
|
+
const sessions = enumeration.sessions;
|
|
14234
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
14235
|
+
if (enumerationReason !== null) {
|
|
12637
14236
|
logActivity(state, {
|
|
12638
14237
|
type: "info",
|
|
12639
|
-
|
|
14238
|
+
level: "warn",
|
|
14239
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12640
14240
|
});
|
|
12641
|
-
return;
|
|
12642
14241
|
}
|
|
12643
|
-
const toDelete = selectSessionsToDelete(
|
|
12644
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12645
|
-
{
|
|
12646
|
-
maxAgeMs: config.maxAgeMs,
|
|
12647
|
-
maxCount: config.maxCount,
|
|
12648
|
-
nowMs: Date.now(),
|
|
12649
|
-
protectedIds: driver.protectedSessionIds()
|
|
12650
|
-
}
|
|
12651
|
-
);
|
|
12652
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12653
14242
|
let deleted = 0;
|
|
12654
14243
|
let failed = 0;
|
|
14244
|
+
let failed4xx = 0;
|
|
14245
|
+
let failed5xx = 0;
|
|
14246
|
+
let failedOtherStatus = 0;
|
|
14247
|
+
let failedThrew = 0;
|
|
12655
14248
|
let skippedNewlyActive = 0;
|
|
12656
|
-
|
|
12657
|
-
|
|
12658
|
-
|
|
12659
|
-
|
|
14249
|
+
let remaining = 0;
|
|
14250
|
+
let protectedNow = driver.protectedSessionIds();
|
|
14251
|
+
if (sessions !== null) {
|
|
14252
|
+
const toDelete = selectSessionsToDelete(
|
|
14253
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
14254
|
+
{
|
|
14255
|
+
maxAgeMs: config.maxAgeMs,
|
|
14256
|
+
maxCount: config.maxCount,
|
|
14257
|
+
nowMs: Date.now(),
|
|
14258
|
+
protectedIds: protectedNow
|
|
14259
|
+
}
|
|
14260
|
+
);
|
|
14261
|
+
const eligibleToDelete = toDelete.filter(
|
|
14262
|
+
(id) => (state.sessionDeleteFailures.get(id)?.isolatedFailures ?? 0) < SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES
|
|
14263
|
+
);
|
|
14264
|
+
const deleteCandidates = eligibleToDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
14265
|
+
remaining = eligibleToDelete.length - deleteCandidates.length;
|
|
14266
|
+
const answeredFailures = [];
|
|
14267
|
+
protectedNow = driver.protectedSessionIds();
|
|
14268
|
+
for (const id of deleteCandidates) {
|
|
14269
|
+
if (protectedNow.has(id)) {
|
|
14270
|
+
skippedNewlyActive++;
|
|
14271
|
+
logActivity(state, {
|
|
14272
|
+
type: "info",
|
|
14273
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
14274
|
+
});
|
|
14275
|
+
continue;
|
|
14276
|
+
}
|
|
14277
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
14278
|
+
if (outcome.ok) {
|
|
14279
|
+
deleted++;
|
|
14280
|
+
} else if (outcome.status === "threw") {
|
|
14281
|
+
failed++;
|
|
14282
|
+
failedThrew++;
|
|
14283
|
+
} else {
|
|
14284
|
+
failed++;
|
|
14285
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
14286
|
+
if (statusClass === 4) failed4xx++;
|
|
14287
|
+
else if (statusClass === 5) failed5xx++;
|
|
14288
|
+
else failedOtherStatus++;
|
|
14289
|
+
answeredFailures.push({
|
|
14290
|
+
id,
|
|
14291
|
+
status: outcome.status,
|
|
14292
|
+
...outcome.body !== void 0 ? { body: outcome.body } : {}
|
|
14293
|
+
});
|
|
14294
|
+
}
|
|
14295
|
+
}
|
|
14296
|
+
const givenUpFailures = [];
|
|
14297
|
+
if (failed > SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES) {
|
|
14298
|
+
for (const { id } of answeredFailures) state.sessionDeleteFailures.delete(id);
|
|
14299
|
+
} else {
|
|
14300
|
+
for (const failure of answeredFailures) {
|
|
14301
|
+
const previous = state.sessionDeleteFailures.get(failure.id);
|
|
14302
|
+
const record = previous?.status === failure.status ? { status: failure.status, isolatedFailures: previous.isolatedFailures + 1 } : { status: failure.status, isolatedFailures: 1 };
|
|
14303
|
+
state.sessionDeleteFailures.set(failure.id, record);
|
|
14304
|
+
if (record.isolatedFailures >= SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES) {
|
|
14305
|
+
givenUpFailures.push(failure);
|
|
14306
|
+
}
|
|
14307
|
+
}
|
|
14308
|
+
}
|
|
14309
|
+
for (const failure of givenUpFailures) {
|
|
14310
|
+
const excerpt = redactDiagnosticText(failure.body ?? "(no response body)").slice(
|
|
14311
|
+
0,
|
|
14312
|
+
MAX_SESSION_DELETE_BODY_LENGTH
|
|
14313
|
+
);
|
|
14314
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12660
14315
|
type: "info",
|
|
12661
|
-
|
|
14316
|
+
level: "warn",
|
|
14317
|
+
message: `Session cleanup: gave up on session ${failure.id} after ${SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES} isolated delete failures (status ${failure.status}): ${excerpt}`,
|
|
14318
|
+
metadata: {
|
|
14319
|
+
outcome: "session-delete-given-up",
|
|
14320
|
+
session_id: failure.id,
|
|
14321
|
+
status: failure.status,
|
|
14322
|
+
body_excerpt: excerpt,
|
|
14323
|
+
given_up_count: givenUpFailures.length
|
|
14324
|
+
}
|
|
12662
14325
|
});
|
|
12663
|
-
continue;
|
|
12664
14326
|
}
|
|
12665
|
-
|
|
12666
|
-
|
|
14327
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
14328
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
14329
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
14330
|
+
logActivity(state, {
|
|
14331
|
+
type: "info",
|
|
14332
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
14333
|
+
});
|
|
12667
14334
|
}
|
|
12668
|
-
const
|
|
12669
|
-
|
|
12670
|
-
|
|
12671
|
-
|
|
12672
|
-
|
|
12673
|
-
|
|
14335
|
+
const sessionMetadata = {
|
|
14336
|
+
sessions_inspected: sessions?.length ?? null,
|
|
14337
|
+
sessions_deleted: deleted,
|
|
14338
|
+
sessions_failed: failed,
|
|
14339
|
+
sessions_failed_4xx: failed4xx,
|
|
14340
|
+
sessions_failed_5xx: failed5xx,
|
|
14341
|
+
sessions_failed_other: failedOtherStatus,
|
|
14342
|
+
sessions_failed_threw: failedThrew,
|
|
14343
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
14344
|
+
sessions_remaining: remaining,
|
|
14345
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
14346
|
+
};
|
|
14347
|
+
const protectedForReclaim = driver.protectedSessionIds();
|
|
12674
14348
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12675
14349
|
dbPath: sessionDbPath(),
|
|
12676
|
-
|
|
12677
|
-
|
|
14350
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
14351
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
14352
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
14353
|
+
hasLiveTurn: protectedForReclaim.size > 0,
|
|
14354
|
+
hasProvenanceAnomaly: state.sessionDbProvenanceAnomaly
|
|
12678
14355
|
});
|
|
12679
14356
|
if (reclaimResult.ok) {
|
|
14357
|
+
state.sessionStoreHealth.observeWritable();
|
|
12680
14358
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12681
14359
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12682
14360
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
12683
|
-
|
|
14361
|
+
const loopElapsedMs = Math.max(
|
|
14362
|
+
0,
|
|
14363
|
+
reclaimResult.elapsedMs - reclaimResult.checkpointElapsedMs
|
|
14364
|
+
);
|
|
14365
|
+
const reclaimIsSlow = reclaimResult.elapsedMs >= 4 * SESSION_DB_RECLAIM_TIME_BUDGET_MS;
|
|
14366
|
+
const perPageNote = reclaimResult.pagesReclaimed > 0 ? `, per-page ${(loopElapsedMs / reclaimResult.pagesReclaimed).toFixed(2)} ms/page` : "";
|
|
14367
|
+
const slowReclaimNote = reclaimIsSlow ? ` (slow reclaim: total elapsed ${reclaimResult.elapsedMs} ms, checkpoint ${reclaimResult.checkpointElapsedMs} ms, loop ${loopElapsedMs} ms, budget ${SESSION_DB_RECLAIM_TIME_BUDGET_MS} ms${perPageNote})` : "";
|
|
14368
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12684
14369
|
type: "info",
|
|
12685
|
-
|
|
14370
|
+
level: reclaimIsSlow ? "warn" : "info",
|
|
14371
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote + slowReclaimNote,
|
|
14372
|
+
metadata: {
|
|
14373
|
+
outcome: "reclaimed",
|
|
14374
|
+
mode: reclaimResult.mode,
|
|
14375
|
+
before_bytes: reclaimResult.beforeBytes,
|
|
14376
|
+
after_bytes: reclaimResult.afterBytes,
|
|
14377
|
+
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
14378
|
+
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
14379
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
14380
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
14381
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
14382
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
14383
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
14384
|
+
checkpoint_elapsed_ms: reclaimResult.checkpointElapsedMs,
|
|
14385
|
+
...sessionMetadata
|
|
14386
|
+
}
|
|
12686
14387
|
});
|
|
12687
14388
|
} else {
|
|
12688
|
-
|
|
14389
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
14390
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
14391
|
+
}
|
|
14392
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12689
14393
|
type: "info",
|
|
12690
|
-
|
|
14394
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
14395
|
+
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
14396
|
+
metadata: {
|
|
14397
|
+
outcome: "reclaim-skipped",
|
|
14398
|
+
reason: reclaimResult.skipped,
|
|
14399
|
+
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
14400
|
+
...sessionMetadata
|
|
14401
|
+
}
|
|
12691
14402
|
});
|
|
12692
14403
|
}
|
|
12693
14404
|
} catch (error2) {
|
|
12694
14405
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
12695
|
-
|
|
14406
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12696
14407
|
type: "error",
|
|
12697
|
-
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}
|
|
14408
|
+
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`,
|
|
14409
|
+
metadata: {
|
|
14410
|
+
outcome: "sweep-failed",
|
|
14411
|
+
reason: "sweep-error"
|
|
14412
|
+
}
|
|
12698
14413
|
});
|
|
12699
14414
|
}
|
|
12700
14415
|
}
|
|
@@ -12710,14 +14425,19 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
12710
14425
|
for (const warning2 of config.warnings) {
|
|
12711
14426
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
12712
14427
|
}
|
|
12713
|
-
const dbBytes = statSessionDbBytes(
|
|
14428
|
+
const dbBytes = statSessionDbBytes(homedir7());
|
|
12714
14429
|
void (async () => {
|
|
12715
14430
|
const reclaimAvailability = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
12716
14431
|
if (reclaimAvailability !== null) {
|
|
12717
|
-
|
|
14432
|
+
logSessionCleanupActivity(state, "startup", {
|
|
12718
14433
|
type: "info",
|
|
12719
14434
|
level: shouldWarnForReclaimSkip(reclaimAvailability.reason) ? "warn" : "info",
|
|
12720
|
-
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : "")
|
|
14435
|
+
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : ""),
|
|
14436
|
+
metadata: {
|
|
14437
|
+
outcome: "preflight-skipped",
|
|
14438
|
+
reason: reclaimAvailability.reason,
|
|
14439
|
+
...reclaimAvailability.detail !== void 0 ? { detail: reclaimAvailability.detail } : {}
|
|
14440
|
+
}
|
|
12721
14441
|
});
|
|
12722
14442
|
}
|
|
12723
14443
|
const sizeWarning = buildSessionStoreSizeWarning({
|
|
@@ -12734,9 +14454,15 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
12734
14454
|
);
|
|
12735
14455
|
});
|
|
12736
14456
|
if (!config.enabled) return;
|
|
12737
|
-
|
|
14457
|
+
logSessionCleanupActivity(state, "startup", {
|
|
12738
14458
|
type: "info",
|
|
12739
|
-
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)
|
|
14459
|
+
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)`,
|
|
14460
|
+
metadata: {
|
|
14461
|
+
outcome: "armed",
|
|
14462
|
+
interval_ms: config.intervalMs,
|
|
14463
|
+
max_age_ms: config.maxAgeMs ?? null,
|
|
14464
|
+
max_count: config.maxCount ?? null
|
|
14465
|
+
}
|
|
12740
14466
|
});
|
|
12741
14467
|
const interval = setInterval(() => void runSweep(state, driver, config), config.intervalMs);
|
|
12742
14468
|
const firstSweep = setTimeout(
|
|
@@ -12766,10 +14492,12 @@ function scheduleUsageReporting(state, params) {
|
|
|
12766
14492
|
let phase = "dormant";
|
|
12767
14493
|
let rearmRequested = false;
|
|
12768
14494
|
const armProbe = () => {
|
|
14495
|
+
if (!state.running) return;
|
|
12769
14496
|
phase = "probe-pending";
|
|
12770
14497
|
params.setTimer(setTimeout(() => void tick(true), params.firstDelayMs()));
|
|
12771
14498
|
};
|
|
12772
14499
|
const scheduleNextTick = () => {
|
|
14500
|
+
if (!state.running) return;
|
|
12773
14501
|
if (rearmRequested) {
|
|
12774
14502
|
rearmRequested = false;
|
|
12775
14503
|
armProbe();
|
|
@@ -12803,6 +14531,7 @@ function scheduleUsageReporting(state, params) {
|
|
|
12803
14531
|
phase = "tick-in-flight";
|
|
12804
14532
|
try {
|
|
12805
14533
|
const usage = await params.fetchUsage();
|
|
14534
|
+
if (!state.running) return;
|
|
12806
14535
|
const result = await params.report(usage);
|
|
12807
14536
|
if (result.ok) {
|
|
12808
14537
|
if (consecutiveFailures > 0) {
|
|
@@ -12915,7 +14644,7 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
12915
14644
|
});
|
|
12916
14645
|
return;
|
|
12917
14646
|
}
|
|
12918
|
-
const { collect, stop } = createResourceUsageCollector(
|
|
14647
|
+
const { collect, stop } = createResourceUsageCollector(homedir7());
|
|
12919
14648
|
state.stopResourceUsageSampling = stop;
|
|
12920
14649
|
let consecutiveFailures = 0;
|
|
12921
14650
|
const tick = async () => {
|
|
@@ -12928,6 +14657,20 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
12928
14657
|
message: `Resource usage collection: ${warning2}`
|
|
12929
14658
|
});
|
|
12930
14659
|
}
|
|
14660
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
14661
|
+
...usage,
|
|
14662
|
+
nowMs: Date.now(),
|
|
14663
|
+
state: state.lowDiskWarningState
|
|
14664
|
+
});
|
|
14665
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
14666
|
+
if (lowDiskDecision.warning !== null) {
|
|
14667
|
+
logActivity(state, {
|
|
14668
|
+
type: "info",
|
|
14669
|
+
level: "warn",
|
|
14670
|
+
message: lowDiskDecision.warning
|
|
14671
|
+
});
|
|
14672
|
+
}
|
|
14673
|
+
if (!state.running) return;
|
|
12931
14674
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
12932
14675
|
if (result.ok) {
|
|
12933
14676
|
if (consecutiveFailures > 0) {
|
|
@@ -12960,7 +14703,9 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
12960
14703
|
message: `Resource usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
|
|
12961
14704
|
});
|
|
12962
14705
|
} finally {
|
|
12963
|
-
state.
|
|
14706
|
+
if (state.running) {
|
|
14707
|
+
state.resourceUsageTimer = setTimeout(() => void tick(), usageReportDelayMs());
|
|
14708
|
+
}
|
|
12964
14709
|
}
|
|
12965
14710
|
};
|
|
12966
14711
|
state.resourceUsageTimer = setTimeout(() => void tick(), firstReportDelayMs());
|
|
@@ -13122,8 +14867,8 @@ async function beginGracefulShutdown(state, trigger) {
|
|
|
13122
14867
|
return true;
|
|
13123
14868
|
}
|
|
13124
14869
|
);
|
|
13125
|
-
const timedOut = new Promise((
|
|
13126
|
-
timer = setTimeout(() =>
|
|
14870
|
+
const timedOut = new Promise((resolve5) => {
|
|
14871
|
+
timer = setTimeout(() => resolve5(false), telemetryBudgetMs);
|
|
13127
14872
|
});
|
|
13128
14873
|
if (!await Promise.race([flushed, timedOut])) {
|
|
13129
14874
|
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
@@ -13140,7 +14885,7 @@ async function run(options) {
|
|
|
13140
14885
|
let fileSyncDirectories;
|
|
13141
14886
|
try {
|
|
13142
14887
|
logLevel = resolveLogLevel(options);
|
|
13143
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
14888
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir7());
|
|
13144
14889
|
if (options.restoreSessionDb && !options.sessionDbNoReplicateMarker) {
|
|
13145
14890
|
throw new Error(
|
|
13146
14891
|
"--restore-session-db requires --session-db-no-replicate-marker <path>; restore failures must block Litestream replication"
|
|
@@ -13163,6 +14908,8 @@ async function run(options) {
|
|
|
13163
14908
|
port: options.port ?? 4096,
|
|
13164
14909
|
conversationFilter: options.conversation ?? null,
|
|
13165
14910
|
idleTimeout: options.idleTimeout ?? null,
|
|
14911
|
+
stayAwakeSeconds: null,
|
|
14912
|
+
stayAwakeUntilMs: null,
|
|
13166
14913
|
json: options.json ?? false,
|
|
13167
14914
|
interactive,
|
|
13168
14915
|
logLevel,
|
|
@@ -13174,6 +14921,7 @@ async function run(options) {
|
|
|
13174
14921
|
opencodeClient: null,
|
|
13175
14922
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13176
14923
|
sessionDbProvenanceAnomaly: false,
|
|
14924
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13177
14925
|
opencodeProcess: null,
|
|
13178
14926
|
stopOpenCodeLogTail: null,
|
|
13179
14927
|
litestreamProcess: null,
|
|
@@ -13185,16 +14933,19 @@ async function run(options) {
|
|
|
13185
14933
|
messageCount: 0,
|
|
13186
14934
|
lastProxiedActivityAt: null,
|
|
13187
14935
|
sessionCleanupTimers: [],
|
|
14936
|
+
sessionDeleteFailures: /* @__PURE__ */ new Map(),
|
|
13188
14937
|
claudeUsageTimer: null,
|
|
13189
14938
|
claudeUsageRearm: null,
|
|
13190
14939
|
openaiUsageTimer: null,
|
|
13191
14940
|
openaiUsageRearm: null,
|
|
13192
14941
|
resourceUsageTimer: null,
|
|
13193
14942
|
stopResourceUsageSampling: null,
|
|
14943
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13194
14944
|
credentialSync: null,
|
|
13195
14945
|
authHeader: ""
|
|
13196
14946
|
};
|
|
13197
14947
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
14948
|
+
setTelemetryDiagnosticsReporter((message) => log2(state, message, "warn"));
|
|
13198
14949
|
if (options.credentialSyncMarker) {
|
|
13199
14950
|
state.credentialSync = createCredentialSync({
|
|
13200
14951
|
markerPath: options.credentialSyncMarker,
|
|
@@ -13348,6 +15099,17 @@ async function run(options) {
|
|
|
13348
15099
|
}
|
|
13349
15100
|
spinner?.succeed(`Runner: ${validation.agent.name || state.agentId}`);
|
|
13350
15101
|
state.agentName = validation.agent.name;
|
|
15102
|
+
const requestedStayAwakeSeconds = validation.agent?.wake_stay_awake_seconds;
|
|
15103
|
+
if (typeof requestedStayAwakeSeconds === "number" && Number.isFinite(requestedStayAwakeSeconds) && requestedStayAwakeSeconds > 0) {
|
|
15104
|
+
state.stayAwakeSeconds = requestedStayAwakeSeconds;
|
|
15105
|
+
} else if (requestedStayAwakeSeconds !== void 0 && requestedStayAwakeSeconds !== null) {
|
|
15106
|
+
state.stayAwakeSeconds = null;
|
|
15107
|
+
log2(
|
|
15108
|
+
state,
|
|
15109
|
+
`Ignoring invalid wake_stay_awake_seconds (type: ${typeof requestedStayAwakeSeconds})`,
|
|
15110
|
+
"warn"
|
|
15111
|
+
);
|
|
15112
|
+
}
|
|
13351
15113
|
const microvmId = process.env.MICROVM_ID?.trim();
|
|
13352
15114
|
if (microvmId) {
|
|
13353
15115
|
const reported = await reportMicrovmId(state.agentId, state.authHeader, microvmId);
|
|
@@ -13363,47 +15125,104 @@ async function run(options) {
|
|
|
13363
15125
|
} else {
|
|
13364
15126
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
13365
15127
|
}
|
|
13366
|
-
if (options.restoreRunnerCredentials) {
|
|
13367
|
-
|
|
13368
|
-
|
|
13369
|
-
|
|
13370
|
-
|
|
13371
|
-
if (level === "error") {
|
|
13372
|
-
logActivity(state, { type: "error", error: message });
|
|
13373
|
-
} else {
|
|
13374
|
-
logActivity(state, { type: "info", level, message });
|
|
13375
|
-
}
|
|
15128
|
+
if (options.restoreRunnerCredentials || options.credentialSyncMarker) {
|
|
15129
|
+
const logCredentialVaultResolution = (message, level) => {
|
|
15130
|
+
log2(state, message, level);
|
|
15131
|
+
if (state.interactive && !state.json) {
|
|
15132
|
+
logActivity(state, { type: "info", level, message });
|
|
13376
15133
|
}
|
|
13377
15134
|
};
|
|
13378
|
-
const
|
|
15135
|
+
const vaultResolution = await getRunnerCredentialVault(state.agentId, state.authHeader);
|
|
15136
|
+
if (vaultResolution.error) {
|
|
15137
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
15138
|
+
logCredentialVaultResolution(
|
|
15139
|
+
`CREDENTIAL-VAULT-LOOKUP-FAILED: ${vaultResolution.error}; credentials will be read from and written to this runner's own prefix this boot`,
|
|
15140
|
+
"warn"
|
|
15141
|
+
);
|
|
15142
|
+
} else if (vaultResolution.prefix !== null && vaultResolution.vaultId !== null && vaultResolution.source !== null) {
|
|
15143
|
+
process.env.CREDENTIAL_VAULT_PREFIX = vaultResolution.prefix;
|
|
15144
|
+
logCredentialVaultResolution(
|
|
15145
|
+
`CREDENTIAL-VAULT: source=${vaultResolution.source} vault=${vaultResolution.vaultId} prefix=${vaultResolution.prefix}`,
|
|
15146
|
+
"info"
|
|
15147
|
+
);
|
|
15148
|
+
} else {
|
|
15149
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
15150
|
+
logCredentialVaultResolution(
|
|
15151
|
+
"CREDENTIAL-VAULT-NONE: this runner is not linked to a credential vault; using its own prefix",
|
|
15152
|
+
"debug"
|
|
15153
|
+
);
|
|
15154
|
+
}
|
|
15155
|
+
}
|
|
15156
|
+
const credentialContext = {
|
|
15157
|
+
env: process.env,
|
|
15158
|
+
log: (message, level = "info") => {
|
|
15159
|
+
if (level === "error") {
|
|
15160
|
+
logActivity(state, { type: "error", error: message });
|
|
15161
|
+
} else {
|
|
15162
|
+
logActivity(state, { type: "info", level, message });
|
|
15163
|
+
}
|
|
15164
|
+
}
|
|
15165
|
+
};
|
|
15166
|
+
const githubTokenPopulated = await installRunnerSecret(credentialContext);
|
|
15167
|
+
if (options.restoreRunnerCredentials) {
|
|
13379
15168
|
await restoreCredentialStores(credentialContext);
|
|
13380
|
-
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13381
15169
|
}
|
|
15170
|
+
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13382
15171
|
state.credentialSync?.arm();
|
|
13383
|
-
|
|
13384
|
-
|
|
13385
|
-
|
|
15172
|
+
const dependencyInstallDiagnosticReporter = createDependencyInstallDiagnosticReporter({
|
|
15173
|
+
context: {
|
|
15174
|
+
homeDir: homedir7(),
|
|
15175
|
+
cwd: process.cwd(),
|
|
15176
|
+
microvmId: process.env.MICROVM_ID?.trim(),
|
|
15177
|
+
euid: process.getuid?.(),
|
|
15178
|
+
egid: process.getgid?.()
|
|
15179
|
+
},
|
|
15180
|
+
emitEvent: (message) => logActivity(state, { type: "info", level: "warn", message }),
|
|
15181
|
+
emitLocal: (message) => logLocalOnly(state, message, "warn")
|
|
15182
|
+
});
|
|
15183
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
15184
|
+
() => ({
|
|
13386
15185
|
agentId: state.agentId,
|
|
13387
15186
|
authHeader: state.authHeader
|
|
13388
|
-
})
|
|
13389
|
-
|
|
15187
|
+
}),
|
|
15188
|
+
({ isDiskFailure, sessionId, cause }) => {
|
|
15189
|
+
if (isDiskFailure) state.sessionStoreHealth.observeUnwritable();
|
|
15190
|
+
if (cause !== null && sessionId !== null) {
|
|
15191
|
+
state.sessionStoreHealth.observeCause(sessionId, cause);
|
|
15192
|
+
}
|
|
15193
|
+
},
|
|
15194
|
+
dependencyInstallDiagnosticReporter
|
|
15195
|
+
);
|
|
15196
|
+
const openCodeLogTail = tailOpenCodeLogFile(
|
|
15197
|
+
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
15198
|
+
openCodeActivityForwarder
|
|
15199
|
+
);
|
|
15200
|
+
state.stopOpenCodeLogTail = () => {
|
|
15201
|
+
openCodeActivityForwarder.flush();
|
|
15202
|
+
openCodeLogTail.stop();
|
|
15203
|
+
};
|
|
13390
15204
|
let sessionDbVerifyFatal = false;
|
|
15205
|
+
let sessionDbVerifyFatalReason;
|
|
13391
15206
|
if (!options.restoreSessionDb) {
|
|
13392
|
-
|
|
15207
|
+
logActivity(state, {
|
|
15208
|
+
type: "info",
|
|
15209
|
+
level: "warn",
|
|
15210
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=flag-absent \u2014 Skipping session-DB restore: --restore-session-db was not passed"
|
|
15211
|
+
});
|
|
13393
15212
|
} else {
|
|
13394
15213
|
const health = await checkOpenCodeHealth(state.port);
|
|
13395
15214
|
if (health.healthy) {
|
|
13396
|
-
|
|
13397
|
-
|
|
13398
|
-
|
|
13399
|
-
"
|
|
13400
|
-
);
|
|
15215
|
+
logActivity(state, {
|
|
15216
|
+
type: "info",
|
|
15217
|
+
level: "warn",
|
|
15218
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-serving \u2014 Skipping session-DB restore: OpenCode is already serving this database"
|
|
15219
|
+
});
|
|
13401
15220
|
} else if (health.authFailed) {
|
|
13402
|
-
|
|
13403
|
-
|
|
13404
|
-
|
|
13405
|
-
"
|
|
13406
|
-
);
|
|
15221
|
+
logActivity(state, {
|
|
15222
|
+
type: "info",
|
|
15223
|
+
level: "warn",
|
|
15224
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-auth-challenge \u2014 Skipping session-DB restore: OpenCode answered with an authentication challenge; assuming this database is in use"
|
|
15225
|
+
});
|
|
13407
15226
|
} else {
|
|
13408
15227
|
const result = await restoreAndVerifySessionDb({
|
|
13409
15228
|
dbPath: sessionDbPath(),
|
|
@@ -13420,12 +15239,13 @@ async function run(options) {
|
|
|
13420
15239
|
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
13421
15240
|
});
|
|
13422
15241
|
sessionDbVerifyFatal = result.verifyFatal;
|
|
15242
|
+
sessionDbVerifyFatalReason = result.verifyFatalReason;
|
|
13423
15243
|
}
|
|
13424
15244
|
}
|
|
13425
15245
|
reportSessionDbRecovery(state);
|
|
13426
15246
|
if (sessionDbVerifyFatal) {
|
|
13427
15247
|
throw new Error(
|
|
13428
|
-
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
15248
|
+
sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
|
|
13429
15249
|
);
|
|
13430
15250
|
}
|
|
13431
15251
|
applyRunnerOpenCodeConfig({
|
|
@@ -13503,7 +15323,7 @@ async function run(options) {
|
|
|
13503
15323
|
const provenance = checkSessionDbProvenance({
|
|
13504
15324
|
dbPath: sessionDbPath(),
|
|
13505
15325
|
currentVersion: state.opencodeVersion,
|
|
13506
|
-
homeDir:
|
|
15326
|
+
homeDir: homedir7(),
|
|
13507
15327
|
env: process.env
|
|
13508
15328
|
});
|
|
13509
15329
|
if (provenance.anomaly) {
|
|
@@ -13670,10 +15490,11 @@ async function run(options) {
|
|
|
13670
15490
|
getAuthHeader: () => state.authHeader,
|
|
13671
15491
|
conversationFilter: state.conversationFilter,
|
|
13672
15492
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
15493
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13673
15494
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13674
15495
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13675
15496
|
fileSyncDirectories,
|
|
13676
|
-
homeDir:
|
|
15497
|
+
homeDir: homedir7(),
|
|
13677
15498
|
maxActiveSessions,
|
|
13678
15499
|
log: (entry) => (
|
|
13679
15500
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
@@ -13695,13 +15516,21 @@ async function run(options) {
|
|
|
13695
15516
|
openCodePassword: state.opencodePassword,
|
|
13696
15517
|
isRunning: () => state.running,
|
|
13697
15518
|
events: {
|
|
13698
|
-
onConnected: (agentId, isReconnect) => {
|
|
15519
|
+
onConnected: (agentId, isReconnect, transition) => {
|
|
13699
15520
|
state.connected = true;
|
|
13700
15521
|
state.agentId = agentId;
|
|
15522
|
+
const connectionMessage = isReconnect ? `Tunnel reconnected (runner: ${agentId}, generation: ${transition.generation}, attempt: ${transition.attempt}, offline for ${transition.downtimeMs}ms)` : `Tunnel connected (runner: ${agentId}, generation: ${transition.generation})`;
|
|
13701
15523
|
logActivity(state, {
|
|
13702
15524
|
type: "info",
|
|
13703
|
-
message:
|
|
15525
|
+
message: connectionMessage
|
|
13704
15526
|
});
|
|
15527
|
+
if (!isReconnect && state.stayAwakeSeconds !== null && state.stayAwakeUntilMs === null) {
|
|
15528
|
+
state.stayAwakeUntilMs = Date.now() + state.stayAwakeSeconds * 1e3;
|
|
15529
|
+
logActivity(state, {
|
|
15530
|
+
type: "info",
|
|
15531
|
+
message: `Staying awake for at least ${state.stayAwakeSeconds}s as requested by the wake`
|
|
15532
|
+
});
|
|
15533
|
+
}
|
|
13705
15534
|
if (options.tunnelReadyFile) {
|
|
13706
15535
|
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
13707
15536
|
if (marker.ok) {
|
|
@@ -13721,7 +15550,12 @@ async function run(options) {
|
|
|
13721
15550
|
version: state.opencodeVersion,
|
|
13722
15551
|
major: state.opencodeApiVersion,
|
|
13723
15552
|
connected: state.opencodeConnected
|
|
13724
|
-
})
|
|
15553
|
+
}),
|
|
15554
|
+
connection_generation: transition.generation,
|
|
15555
|
+
...isReconnect && transition.downtimeMs !== null ? {
|
|
15556
|
+
reconnect_attempt: transition.attempt,
|
|
15557
|
+
downtime_ms: transition.downtimeMs
|
|
15558
|
+
} : {}
|
|
13725
15559
|
});
|
|
13726
15560
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
13727
15561
|
if (state.interactive) displayStatus(state);
|
|
@@ -13743,13 +15577,18 @@ async function run(options) {
|
|
|
13743
15577
|
if (state.interactive) displayStatus(state);
|
|
13744
15578
|
});
|
|
13745
15579
|
},
|
|
13746
|
-
onDisconnected: (code, reason) => {
|
|
15580
|
+
onDisconnected: (code, reason, transition) => {
|
|
13747
15581
|
state.connected = false;
|
|
15582
|
+
const safeReason2 = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
13748
15583
|
logActivity(state, {
|
|
13749
15584
|
type: "info",
|
|
13750
|
-
message: `Tunnel disconnected (code: ${code}, reason: ${
|
|
15585
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason2})`
|
|
15586
|
+
});
|
|
15587
|
+
emitAgentDisconnected(state.agentId, {
|
|
15588
|
+
code,
|
|
15589
|
+
reason: safeReason2,
|
|
15590
|
+
connection_generation: transition.generation
|
|
13751
15591
|
});
|
|
13752
|
-
emitAgentDisconnected(state.agentId, { code, reason });
|
|
13753
15592
|
if (state.interactive) displayStatus(state);
|
|
13754
15593
|
},
|
|
13755
15594
|
onError: (error2) => {
|
|
@@ -13826,6 +15665,7 @@ async function run(options) {
|
|
|
13826
15665
|
if (error2.message === "Unauthorized") tunnelSpinner?.fail("Unauthorized");
|
|
13827
15666
|
throw error2;
|
|
13828
15667
|
}
|
|
15668
|
+
if (!state.running) return;
|
|
13829
15669
|
scheduleSessionCleanup(state, channelDriver, options);
|
|
13830
15670
|
state.claudeUsageRearm = scheduleClaudeUsageReporting(state, options);
|
|
13831
15671
|
state.openaiUsageRearm = scheduleUsageReporting(state, {
|
|
@@ -13849,7 +15689,7 @@ async function run(options) {
|
|
|
13849
15689
|
},
|
|
13850
15690
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
13851
15691
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
13852
|
-
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
15692
|
+
forcedOnHint: state.opencodeApiVersion === "v2" ? "OpenCode V2 keeps its connected ChatGPT login in a store Evident cannot read" : "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
13853
15693
|
firstDelayMs: firstReportDelayMs,
|
|
13854
15694
|
nextDelayMs: usageReportDelayMs,
|
|
13855
15695
|
failureLogLevel: usageReportFailureLogLevel
|
|
@@ -13892,7 +15732,7 @@ async function run(options) {
|
|
|
13892
15732
|
}
|
|
13893
15733
|
|
|
13894
15734
|
// src/index.ts
|
|
13895
|
-
var
|
|
15735
|
+
var version = getCliVersion();
|
|
13896
15736
|
var program = new Command();
|
|
13897
15737
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
13898
15738
|
"--endpoint <url>",
|
|
@@ -13910,6 +15750,7 @@ program.command("login").description("Authenticate with Evident").option("--toke
|
|
|
13910
15750
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
13911
15751
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
13912
15752
|
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
15753
|
+
program.command("credential-vault-env").description("Print the shell assignment a runner image evals before starting the synchroniser").action(credentialVaultEnv);
|
|
13913
15754
|
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
13914
15755
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
13915
15756
|
"-a, --agent [id]",
|