@evident-ai/cli 3.1.1-dev.341ff6c → 3.1.1-dev.35e0a15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +557 -88
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -11,8 +11,8 @@ import chalk2 from "chalk";
|
|
|
11
11
|
|
|
12
12
|
// src/lib/config.ts
|
|
13
13
|
import Conf from "conf";
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
14
|
+
import { chmodSync, existsSync, statSync } from "fs";
|
|
15
|
+
import { dirname } from "path";
|
|
16
16
|
var PRODUCTION_API_URL = "https://api.production.evident.run/v1";
|
|
17
17
|
var PRODUCTION_TUNNEL_URL = "wss://tunnel.production.evident.run";
|
|
18
18
|
var defaults = {
|
|
@@ -47,8 +47,35 @@ var credentials = new Conf({
|
|
|
47
47
|
projectName: "evident",
|
|
48
48
|
projectSuffix: "",
|
|
49
49
|
configName: "credentials",
|
|
50
|
-
defaults: {}
|
|
50
|
+
defaults: {},
|
|
51
|
+
configFileMode: 384
|
|
51
52
|
});
|
|
53
|
+
var CREDENTIALS_FILE_MODE = 384;
|
|
54
|
+
var CREDENTIALS_DIR_MODE = 448;
|
|
55
|
+
var permissionWarningEmitted = false;
|
|
56
|
+
function hardenCredentialsPermissions() {
|
|
57
|
+
if (process.platform === "win32") {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
const file = credentials.path;
|
|
61
|
+
for (const [path, mode] of [
|
|
62
|
+
[file, CREDENTIALS_FILE_MODE],
|
|
63
|
+
[dirname(file), CREDENTIALS_DIR_MODE]
|
|
64
|
+
]) {
|
|
65
|
+
try {
|
|
66
|
+
if (existsSync(path) && (statSync(path).mode & 511) !== mode) {
|
|
67
|
+
chmodSync(path, mode);
|
|
68
|
+
}
|
|
69
|
+
} catch (err) {
|
|
70
|
+
if (!permissionWarningEmitted) {
|
|
71
|
+
permissionWarningEmitted = true;
|
|
72
|
+
console.error(
|
|
73
|
+
`[config] could not restrict permissions on ${path}; the credentials file may be readable by other users on this machine: ${err instanceof Error ? err.message : String(err)}`
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
52
79
|
function getApiUrlConfig() {
|
|
53
80
|
return getApiUrl();
|
|
54
81
|
}
|
|
@@ -59,6 +86,7 @@ function credentialsKey() {
|
|
|
59
86
|
return getApiUrl();
|
|
60
87
|
}
|
|
61
88
|
function getCredentials() {
|
|
89
|
+
hardenCredentialsPermissions();
|
|
62
90
|
const byEndpoint = credentials.get("byEndpoint") ?? {};
|
|
63
91
|
return byEndpoint[credentialsKey()] ?? {};
|
|
64
92
|
}
|
|
@@ -70,14 +98,17 @@ function setCredentials(creds) {
|
|
|
70
98
|
expiresAt: creds.expiresAt
|
|
71
99
|
};
|
|
72
100
|
credentials.set("byEndpoint", byEndpoint);
|
|
101
|
+
hardenCredentialsPermissions();
|
|
73
102
|
}
|
|
74
103
|
function clearCredentials() {
|
|
75
104
|
const byEndpoint = credentials.get("byEndpoint") ?? {};
|
|
76
105
|
delete byEndpoint[credentialsKey()];
|
|
77
106
|
credentials.set("byEndpoint", byEndpoint);
|
|
107
|
+
hardenCredentialsPermissions();
|
|
78
108
|
}
|
|
79
109
|
function clearAllCredentials() {
|
|
80
110
|
credentials.clear();
|
|
111
|
+
hardenCredentialsPermissions();
|
|
81
112
|
}
|
|
82
113
|
function getCliName() {
|
|
83
114
|
const argv1 = process.argv[1] || "";
|
|
@@ -373,7 +404,8 @@ async function deviceFlowLogin(options) {
|
|
|
373
404
|
}
|
|
374
405
|
async function tokenLogin() {
|
|
375
406
|
console.log("Token login mode.");
|
|
376
|
-
console.log("
|
|
407
|
+
console.log("Run `evident login` on a machine with a browser to get a token.");
|
|
408
|
+
console.log("Manage or revoke existing tokens under Settings \u2192 CLI tokens.");
|
|
377
409
|
blank();
|
|
378
410
|
process.stdout.write("Paste token: ");
|
|
379
411
|
const token = await new Promise((resolve3) => {
|
|
@@ -467,8 +499,8 @@ async function whoami() {
|
|
|
467
499
|
}
|
|
468
500
|
|
|
469
501
|
// src/commands/run.ts
|
|
470
|
-
import { homedir as
|
|
471
|
-
import { isAbsolute as isAbsolute2, join as
|
|
502
|
+
import { homedir as homedir2 } from "os";
|
|
503
|
+
import { isAbsolute as isAbsolute2, join as join2, parse, resolve as resolvePath } from "path";
|
|
472
504
|
import chalk6 from "chalk";
|
|
473
505
|
|
|
474
506
|
// ../../packages/types/src/telemetry/index.ts
|
|
@@ -1500,6 +1532,9 @@ function isPreamblePinnedRunning(messages, userMessageId) {
|
|
|
1500
1532
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1501
1533
|
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
1502
1534
|
}
|
|
1535
|
+
function isB2AbandonmentConfirmed(params) {
|
|
1536
|
+
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false;
|
|
1537
|
+
}
|
|
1503
1538
|
function messageError(messages, userMessageId) {
|
|
1504
1539
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1505
1540
|
const error2 = errorOf(reply);
|
|
@@ -1513,6 +1548,42 @@ function messageError(messages, userMessageId) {
|
|
|
1513
1548
|
}
|
|
1514
1549
|
return "The agent run failed.";
|
|
1515
1550
|
}
|
|
1551
|
+
function messageFailure(messages, userMessageId) {
|
|
1552
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1553
|
+
const error2 = errorOf(reply);
|
|
1554
|
+
if (error2 == null || typeof error2 !== "object") return null;
|
|
1555
|
+
const e = error2;
|
|
1556
|
+
const replyProviderId = reply?.info?.providerID ?? null;
|
|
1557
|
+
const replyModelId = reply?.info?.modelID ?? null;
|
|
1558
|
+
if (e.name === "ProviderAuthError") {
|
|
1559
|
+
const data = e.data;
|
|
1560
|
+
const providerId = typeof data?.providerID === "string" && data.providerID || replyProviderId;
|
|
1561
|
+
return { kind: "model_auth", providerId, modelId: replyModelId, reason: "missing" };
|
|
1562
|
+
}
|
|
1563
|
+
if (e.name === "APIError") {
|
|
1564
|
+
const data = e.data;
|
|
1565
|
+
const statusCode = data?.statusCode;
|
|
1566
|
+
if (statusCode === 401 || statusCode === 403) {
|
|
1567
|
+
return {
|
|
1568
|
+
kind: "model_auth",
|
|
1569
|
+
providerId: replyProviderId,
|
|
1570
|
+
modelId: replyModelId,
|
|
1571
|
+
reason: "rejected"
|
|
1572
|
+
};
|
|
1573
|
+
}
|
|
1574
|
+
}
|
|
1575
|
+
return null;
|
|
1576
|
+
}
|
|
1577
|
+
function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProviderId, replyModelId = null) {
|
|
1578
|
+
if (classified != null) return classified;
|
|
1579
|
+
if (hasConfiguredProvider !== false) return null;
|
|
1580
|
+
return {
|
|
1581
|
+
kind: "model_auth",
|
|
1582
|
+
providerId: replyProviderId,
|
|
1583
|
+
modelId: replyModelId,
|
|
1584
|
+
reason: "missing"
|
|
1585
|
+
};
|
|
1586
|
+
}
|
|
1516
1587
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
1517
1588
|
if (!messages || messages.length === 0) return false;
|
|
1518
1589
|
return messages.some(
|
|
@@ -2041,13 +2112,25 @@ var RunnerConnection = class {
|
|
|
2041
2112
|
}
|
|
2042
2113
|
};
|
|
2043
2114
|
|
|
2115
|
+
// src/lib/tunnel/ready-marker.ts
|
|
2116
|
+
import { writeFileSync } from "fs";
|
|
2117
|
+
function writeTunnelReadyMarker(path, agentId) {
|
|
2118
|
+
try {
|
|
2119
|
+
writeFileSync(path, `${agentId}
|
|
2120
|
+
`);
|
|
2121
|
+
return { ok: true };
|
|
2122
|
+
} catch (error2) {
|
|
2123
|
+
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
2124
|
+
}
|
|
2125
|
+
}
|
|
2126
|
+
|
|
2044
2127
|
// src/lib/channels/driver.ts
|
|
2045
|
-
import { homedir
|
|
2128
|
+
import { homedir } from "os";
|
|
2046
2129
|
|
|
2047
2130
|
// src/lib/file-push.ts
|
|
2048
2131
|
import { randomUUID } from "crypto";
|
|
2049
2132
|
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
2050
|
-
import { basename, dirname, isAbsolute, join
|
|
2133
|
+
import { basename, dirname as dirname2, isAbsolute, join, relative, resolve as resolve2, sep } from "path";
|
|
2051
2134
|
var FILE_MODE = 384;
|
|
2052
2135
|
var DIRECTORY_MODE = 448;
|
|
2053
2136
|
async function writePushedFile(request) {
|
|
@@ -2078,9 +2161,9 @@ async function writePushedFile(request) {
|
|
|
2078
2161
|
}
|
|
2079
2162
|
try {
|
|
2080
2163
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
2081
|
-
|
|
2164
|
+
dirname2(candidate)
|
|
2082
2165
|
);
|
|
2083
|
-
const realTarget =
|
|
2166
|
+
const realTarget = join(existingAncestor, ...missingSegments, basename(candidate));
|
|
2084
2167
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
2085
2168
|
if (allowedDirectory === null) {
|
|
2086
2169
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
@@ -2090,8 +2173,8 @@ async function writePushedFile(request) {
|
|
|
2090
2173
|
}
|
|
2091
2174
|
if (missingSegments.length > 0) {
|
|
2092
2175
|
await createMissingDirectories(existingAncestor, missingSegments);
|
|
2093
|
-
const realParent = await realpath(
|
|
2094
|
-
if (realParent !==
|
|
2176
|
+
const realParent = await realpath(dirname2(realTarget));
|
|
2177
|
+
if (realParent !== dirname2(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
2095
2178
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
2096
2179
|
path: realTarget,
|
|
2097
2180
|
bytes,
|
|
@@ -2116,7 +2199,7 @@ function expandAndValidate(requestedPath, homeDir) {
|
|
|
2116
2199
|
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
2117
2200
|
return null;
|
|
2118
2201
|
}
|
|
2119
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
2202
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
2120
2203
|
if (expanded.split(/[/\\]/).includes("..")) {
|
|
2121
2204
|
return null;
|
|
2122
2205
|
}
|
|
@@ -2134,7 +2217,7 @@ async function resolveNearestExistingAncestor(directory) {
|
|
|
2134
2217
|
try {
|
|
2135
2218
|
return { existingAncestor: await realpath(current), missingSegments };
|
|
2136
2219
|
} catch (err) {
|
|
2137
|
-
const parent =
|
|
2220
|
+
const parent = dirname2(current);
|
|
2138
2221
|
if (err.code !== "ENOENT" || parent === current) {
|
|
2139
2222
|
throw err;
|
|
2140
2223
|
}
|
|
@@ -2189,13 +2272,13 @@ function contains(realDirectory, realTarget) {
|
|
|
2189
2272
|
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
2190
2273
|
let current = existingAncestor;
|
|
2191
2274
|
for (const segment of missingSegments) {
|
|
2192
|
-
current =
|
|
2275
|
+
current = join(current, segment);
|
|
2193
2276
|
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
2194
2277
|
await chmod(current, DIRECTORY_MODE);
|
|
2195
2278
|
}
|
|
2196
2279
|
}
|
|
2197
2280
|
async function writeAtomically(realTarget, content) {
|
|
2198
|
-
const temporaryPath =
|
|
2281
|
+
const temporaryPath = join(dirname2(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
2199
2282
|
let handle;
|
|
2200
2283
|
try {
|
|
2201
2284
|
handle = await open2(temporaryPath, "wx", FILE_MODE);
|
|
@@ -2467,6 +2550,8 @@ var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
|
2467
2550
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
2468
2551
|
var HEARTBEAT_MS = 6e4;
|
|
2469
2552
|
var ABSOLUTE_MAX_PROCESSING_MS = 6 * 60 * 60 * 1e3;
|
|
2553
|
+
var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
|
|
2554
|
+
var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
|
|
2470
2555
|
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
2471
2556
|
var MAX_SUPERSEDED_CONVERSATIONS = 256;
|
|
2472
2557
|
var ChannelAuthError = class extends Error {
|
|
@@ -2708,7 +2793,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2708
2793
|
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
2709
2794
|
this.now = config2.now ?? (() => Date.now());
|
|
2710
2795
|
this.fileSyncDirectories = config2.fileSyncDirectories ?? [];
|
|
2711
|
-
this.homeDir = config2.homeDir ??
|
|
2796
|
+
this.homeDir = config2.homeDir ?? homedir();
|
|
2712
2797
|
}
|
|
2713
2798
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
2714
2799
|
get opencodeBase() {
|
|
@@ -3285,12 +3370,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3285
3370
|
stuckReported: false,
|
|
3286
3371
|
lastAliveAt: 0,
|
|
3287
3372
|
aliveInFlight: false,
|
|
3373
|
+
titleSynced: false,
|
|
3374
|
+
titleSyncInFlight: false,
|
|
3288
3375
|
awaitingHumanLatched: false,
|
|
3289
3376
|
pausedOnQuestion: false,
|
|
3290
3377
|
pausedOnPermission: false,
|
|
3291
3378
|
pausedClearConfirmed: false,
|
|
3292
3379
|
pausedInFlight: false,
|
|
3293
|
-
deliveryDeadlineAnchored: false
|
|
3380
|
+
deliveryDeadlineAnchored: false,
|
|
3381
|
+
b2PinnedSinceMs: 0,
|
|
3382
|
+
b2LastDescendantCheckMs: 0,
|
|
3383
|
+
b2AbandonedSignalled: false
|
|
3294
3384
|
});
|
|
3295
3385
|
}
|
|
3296
3386
|
/**
|
|
@@ -3358,12 +3448,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3358
3448
|
// with no extra `re_adopted` signal needed (folds old WI-6).
|
|
3359
3449
|
lastAliveAt: 0,
|
|
3360
3450
|
aliveInFlight: false,
|
|
3451
|
+
titleSynced: false,
|
|
3452
|
+
titleSyncInFlight: false,
|
|
3361
3453
|
awaitingHumanLatched: false,
|
|
3362
3454
|
pausedOnQuestion: false,
|
|
3363
3455
|
pausedOnPermission: false,
|
|
3364
3456
|
pausedClearConfirmed: false,
|
|
3365
3457
|
pausedInFlight: false,
|
|
3366
|
-
deliveryDeadlineAnchored: false
|
|
3458
|
+
deliveryDeadlineAnchored: false,
|
|
3459
|
+
b2PinnedSinceMs: 0,
|
|
3460
|
+
b2LastDescendantCheckMs: 0,
|
|
3461
|
+
b2AbandonedSignalled: false
|
|
3367
3462
|
});
|
|
3368
3463
|
}
|
|
3369
3464
|
/**
|
|
@@ -3525,58 +3620,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3525
3620
|
}
|
|
3526
3621
|
}
|
|
3527
3622
|
if (state === "done") {
|
|
3528
|
-
this.
|
|
3529
|
-
if (!inFlight.done) {
|
|
3530
|
-
this.log({
|
|
3531
|
-
level: "info",
|
|
3532
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed \u2014 marking done`,
|
|
3533
|
-
conversation_id: conv.id,
|
|
3534
|
-
message_id: inFlight.evidentMessageId
|
|
3535
|
-
});
|
|
3536
|
-
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
3537
|
-
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
3538
|
-
try {
|
|
3539
|
-
await this.markDone(
|
|
3540
|
-
conv.id,
|
|
3541
|
-
inFlight.evidentMessageId,
|
|
3542
|
-
sessionId,
|
|
3543
|
-
inFlight.opencodeMessageId,
|
|
3544
|
-
title,
|
|
3545
|
-
usage
|
|
3546
|
-
);
|
|
3547
|
-
} catch (err) {
|
|
3548
|
-
if (err instanceof ChannelAuthError) throw err;
|
|
3549
|
-
if (err instanceof ChannelTerminalError) {
|
|
3550
|
-
this.log({
|
|
3551
|
-
level: "warn",
|
|
3552
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
3553
|
-
conversation_id: conv.id,
|
|
3554
|
-
message_id: inFlight.evidentMessageId
|
|
3555
|
-
});
|
|
3556
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3557
|
-
return;
|
|
3558
|
-
}
|
|
3559
|
-
if (this.now() >= inFlight.deadline) {
|
|
3560
|
-
this.log({
|
|
3561
|
-
level: "warn",
|
|
3562
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
3563
|
-
conversation_id: conv.id,
|
|
3564
|
-
message_id: inFlight.evidentMessageId
|
|
3565
|
-
});
|
|
3566
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3567
|
-
return;
|
|
3568
|
-
}
|
|
3569
|
-
this.log({
|
|
3570
|
-
level: "warn",
|
|
3571
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
3572
|
-
conversation_id: conv.id,
|
|
3573
|
-
message_id: inFlight.evidentMessageId
|
|
3574
|
-
});
|
|
3575
|
-
return;
|
|
3576
|
-
}
|
|
3577
|
-
inFlight.done = true;
|
|
3578
|
-
}
|
|
3579
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3623
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
3580
3624
|
return;
|
|
3581
3625
|
}
|
|
3582
3626
|
if (state === "failed") {
|
|
@@ -3590,8 +3634,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3590
3634
|
message_id: inFlight.evidentMessageId
|
|
3591
3635
|
});
|
|
3592
3636
|
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
3637
|
+
const failure = await this.classifyModelAuthFailure(messages, inFlight.opencodeMessageId);
|
|
3593
3638
|
try {
|
|
3594
|
-
await this.markFailed(
|
|
3639
|
+
await this.markFailed(
|
|
3640
|
+
conv.id,
|
|
3641
|
+
inFlight.evidentMessageId,
|
|
3642
|
+
sessionId,
|
|
3643
|
+
error2,
|
|
3644
|
+
usage,
|
|
3645
|
+
failure
|
|
3646
|
+
);
|
|
3595
3647
|
} catch (err) {
|
|
3596
3648
|
if (err instanceof ChannelAuthError) throw err;
|
|
3597
3649
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -3636,6 +3688,44 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3636
3688
|
});
|
|
3637
3689
|
}
|
|
3638
3690
|
const activelyRunning = state === "running" && !awaitingHuman;
|
|
3691
|
+
const pinnedNow = activelyRunning && isPreamblePinnedRunning(messages, inFlight.opencodeMessageId);
|
|
3692
|
+
const snapshotReadable = messages != null && messages.length > 0;
|
|
3693
|
+
if (!pinnedNow) {
|
|
3694
|
+
if (snapshotReadable) {
|
|
3695
|
+
inFlight.b2PinnedSinceMs = 0;
|
|
3696
|
+
inFlight.b2LastDescendantCheckMs = 0;
|
|
3697
|
+
inFlight.b2AbandonedSignalled = false;
|
|
3698
|
+
}
|
|
3699
|
+
} else {
|
|
3700
|
+
if (inFlight.b2AbandonedSignalled) {
|
|
3701
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
3702
|
+
return;
|
|
3703
|
+
}
|
|
3704
|
+
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
3705
|
+
const pinnedForMs = this.now() - inFlight.b2PinnedSinceMs;
|
|
3706
|
+
if (pinnedForMs >= B2_ABANDONMENT_MIN_PINNED_MS && this.now() - inFlight.b2LastDescendantCheckMs >= B2_ABANDONMENT_RECHECK_MS) {
|
|
3707
|
+
inFlight.b2LastDescendantCheckMs = this.now();
|
|
3708
|
+
const descendantOngoing = await this.isAnyDescendantSessionOngoing(sessionId);
|
|
3709
|
+
if (isB2AbandonmentConfirmed({
|
|
3710
|
+
pinnedForMs,
|
|
3711
|
+
minPinnedMs: B2_ABANDONMENT_MIN_PINNED_MS,
|
|
3712
|
+
descendantOngoing
|
|
3713
|
+
})) {
|
|
3714
|
+
inFlight.b2AbandonedSignalled = true;
|
|
3715
|
+
this.log({
|
|
3716
|
+
level: "warn",
|
|
3717
|
+
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
3718
|
+
conversation_id: conv.id,
|
|
3719
|
+
message_id: id
|
|
3720
|
+
});
|
|
3721
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
3722
|
+
watched_for_ms: pinnedForMs
|
|
3723
|
+
});
|
|
3724
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
3725
|
+
return;
|
|
3726
|
+
}
|
|
3727
|
+
}
|
|
3728
|
+
}
|
|
3639
3729
|
if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
|
|
3640
3730
|
this.log({
|
|
3641
3731
|
level: "warn",
|
|
@@ -3655,6 +3745,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3655
3745
|
inFlight.aliveInFlight = false;
|
|
3656
3746
|
if (ok) inFlight.lastAliveAt = this.now();
|
|
3657
3747
|
});
|
|
3748
|
+
if (!inFlight.titleSynced && !inFlight.titleSyncInFlight) {
|
|
3749
|
+
inFlight.titleSyncInFlight = true;
|
|
3750
|
+
void this.resolveSessionTitle(sessionId, conv.id).then(async (title) => {
|
|
3751
|
+
if (!title) {
|
|
3752
|
+
inFlight.titleSyncInFlight = false;
|
|
3753
|
+
return;
|
|
3754
|
+
}
|
|
3755
|
+
const ok = await this.patchConversationTitle(conv.id, title);
|
|
3756
|
+
inFlight.titleSyncInFlight = false;
|
|
3757
|
+
if (ok) inFlight.titleSynced = true;
|
|
3758
|
+
});
|
|
3759
|
+
}
|
|
3658
3760
|
}
|
|
3659
3761
|
if (awaitingHuman) {
|
|
3660
3762
|
if (!inFlight.awaitingHumanLatched) {
|
|
@@ -3692,6 +3794,70 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3692
3794
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3693
3795
|
}
|
|
3694
3796
|
}
|
|
3797
|
+
/**
|
|
3798
|
+
* Settle a message whose run-state has resolved `'done'` — extracted verbatim
|
|
3799
|
+
* (pure refactor, no behavior change) from `serviceInFlightMessage`'s former
|
|
3800
|
+
* inline `state === 'done'` branch body, so a SECOND caller (the #721
|
|
3801
|
+
* b2-abandonment resolution) can reach the exact same completion behavior
|
|
3802
|
+
* (delivery-deadline anchoring, title resolution, usage extraction, and
|
|
3803
|
+
* `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
|
|
3804
|
+
* and risking the two copies silently drifting apart.
|
|
3805
|
+
*/
|
|
3806
|
+
async settleMessageDone(sessionId, watcher, inFlight, messages) {
|
|
3807
|
+
const conv = watcher.conv;
|
|
3808
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
3809
|
+
if (!inFlight.done) {
|
|
3810
|
+
this.log({
|
|
3811
|
+
level: "info",
|
|
3812
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed \u2014 marking done`,
|
|
3813
|
+
conversation_id: conv.id,
|
|
3814
|
+
message_id: inFlight.evidentMessageId
|
|
3815
|
+
});
|
|
3816
|
+
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
3817
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
3818
|
+
try {
|
|
3819
|
+
await this.markDone(
|
|
3820
|
+
conv.id,
|
|
3821
|
+
inFlight.evidentMessageId,
|
|
3822
|
+
sessionId,
|
|
3823
|
+
inFlight.opencodeMessageId,
|
|
3824
|
+
title,
|
|
3825
|
+
usage
|
|
3826
|
+
);
|
|
3827
|
+
} catch (err) {
|
|
3828
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
3829
|
+
if (err instanceof ChannelTerminalError) {
|
|
3830
|
+
this.log({
|
|
3831
|
+
level: "warn",
|
|
3832
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
3833
|
+
conversation_id: conv.id,
|
|
3834
|
+
message_id: inFlight.evidentMessageId
|
|
3835
|
+
});
|
|
3836
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3837
|
+
return;
|
|
3838
|
+
}
|
|
3839
|
+
if (this.now() >= inFlight.deadline) {
|
|
3840
|
+
this.log({
|
|
3841
|
+
level: "warn",
|
|
3842
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
3843
|
+
conversation_id: conv.id,
|
|
3844
|
+
message_id: inFlight.evidentMessageId
|
|
3845
|
+
});
|
|
3846
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3847
|
+
return;
|
|
3848
|
+
}
|
|
3849
|
+
this.log({
|
|
3850
|
+
level: "warn",
|
|
3851
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
3852
|
+
conversation_id: conv.id,
|
|
3853
|
+
message_id: inFlight.evidentMessageId
|
|
3854
|
+
});
|
|
3855
|
+
return;
|
|
3856
|
+
}
|
|
3857
|
+
inFlight.done = true;
|
|
3858
|
+
}
|
|
3859
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3860
|
+
}
|
|
3695
3861
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
3696
3862
|
/**
|
|
3697
3863
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -3858,6 +4024,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3858
4024
|
if (state === "failed") {
|
|
3859
4025
|
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
3860
4026
|
const usage = messageUsage(messages, ocId ?? "");
|
|
4027
|
+
const failure = await this.classifyModelAuthFailure(messages, ocId ?? "");
|
|
3861
4028
|
this.log({
|
|
3862
4029
|
level: "error",
|
|
3863
4030
|
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -3865,7 +4032,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3865
4032
|
message_id: row.id
|
|
3866
4033
|
});
|
|
3867
4034
|
try {
|
|
3868
|
-
await this.markFailed(row.conversation_id, row.id, sessionId, error2, usage);
|
|
4035
|
+
await this.markFailed(row.conversation_id, row.id, sessionId, error2, usage, failure);
|
|
3869
4036
|
} catch (err) {
|
|
3870
4037
|
if (err instanceof ChannelAuthError) throw err;
|
|
3871
4038
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -4271,6 +4438,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4271
4438
|
}
|
|
4272
4439
|
return false;
|
|
4273
4440
|
}
|
|
4441
|
+
/**
|
|
4442
|
+
* Tri-state variant of the upward parentID membership walk (#721), used ONLY
|
|
4443
|
+
* by `isAnyDescendantSessionOngoing`. Walks the SAME cached
|
|
4444
|
+
* `resolveSessionParent` chain `sessionBelongsTo` uses above, but — unlike
|
|
4445
|
+
* `sessionBelongsTo`, which deliberately collapses "confirmed not a
|
|
4446
|
+
* descendant" and "the walk's fetch failed" into the same `false` (safe for
|
|
4447
|
+
* its OTHER callers: interaction attribution and the recovery-path
|
|
4448
|
+
* `isAnyDescendantSessionAlive`, both of which just retry next tick with no
|
|
4449
|
+
* safety consequence either way) — this variant keeps those two outcomes
|
|
4450
|
+
* SEPARATE, because `isAnyDescendantSessionOngoing`'s caller
|
|
4451
|
+
* (`isB2AbandonmentConfirmed`) must never treat "couldn't tell" as "confirmed
|
|
4452
|
+
* not ongoing".
|
|
4453
|
+
*
|
|
4454
|
+
* Return contract:
|
|
4455
|
+
* - `true` → the walk reached `rootSessionId` — `sessionId` IS a descendant.
|
|
4456
|
+
* - `false` → the walk reached a definitive, parent-less root session
|
|
4457
|
+
* WITHOUT ever matching `rootSessionId` — `sessionId` is
|
|
4458
|
+
* CONFIRMED NOT a descendant of it.
|
|
4459
|
+
* - `null` → INDETERMINATE: a `GET /session/:id` fetch failed partway
|
|
4460
|
+
* through the walk (`resolveSessionParent` returned `undefined`),
|
|
4461
|
+
* or the depth cap (32) was hit without a definitive answer (a
|
|
4462
|
+
* pathological/cyclic chain proves nothing either way). NEVER
|
|
4463
|
+
* treat this the same as `false` — see `sessionBelongsTo`'s own
|
|
4464
|
+
* doc comment above for why that collapse is safe THERE but not
|
|
4465
|
+
* here.
|
|
4466
|
+
*
|
|
4467
|
+
* `sessionBelongsTo` itself is UNCHANGED — this is an additive helper scoped
|
|
4468
|
+
* to the live-path descendant check, not a modification of shared code used
|
|
4469
|
+
* by interaction attribution or the recovery path.
|
|
4470
|
+
*/
|
|
4471
|
+
async resolveSessionMembership(sessionId, rootSessionId) {
|
|
4472
|
+
let current = sessionId;
|
|
4473
|
+
for (let depth = 0; current && depth < 32; depth++) {
|
|
4474
|
+
if (current === rootSessionId) return true;
|
|
4475
|
+
const parent = await this.resolveSessionParent(current);
|
|
4476
|
+
if (parent === void 0) return null;
|
|
4477
|
+
if (parent === null) return false;
|
|
4478
|
+
current = parent;
|
|
4479
|
+
}
|
|
4480
|
+
return null;
|
|
4481
|
+
}
|
|
4274
4482
|
/**
|
|
4275
4483
|
* Resolve (and cache) a session's `parentID` via `GET /session/:id`. Returns
|
|
4276
4484
|
* `null` for a root session (no parent) and `undefined` when opencode is
|
|
@@ -4355,6 +4563,54 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4355
4563
|
}
|
|
4356
4564
|
return null;
|
|
4357
4565
|
}
|
|
4566
|
+
/**
|
|
4567
|
+
* Best-effort mid-turn title sync (#711 follow-up): PATCH a resolved OpenCode
|
|
4568
|
+
* session title onto the conversation via the PLAIN conversation-update
|
|
4569
|
+
* endpoint (`PATCH /runners/:agentId/conversations/:conversationId`) — NOT the
|
|
4570
|
+
* message-status endpoint `markProcessing`/`markDone` use. Deliberately a
|
|
4571
|
+
* separate, lighter call: it carries no `status`, so it cannot re-trigger the
|
|
4572
|
+
* `processing`/`done` transition side effects (Slack notices, activity-log
|
|
4573
|
+
* rows, delivery jobs) those PATCHes gate on `transitioned` — this call only
|
|
4574
|
+
* ever touches `conversations.title`. That route (`routes/conversations.ts`)
|
|
4575
|
+
* skips a title write matching the stored value, so a redundant call with the
|
|
4576
|
+
* same title is a real no-op — it does not bump `updated_at`, which the
|
|
4577
|
+
* conversation list sorts and paginates on. (Note this is a DIFFERENT guard
|
|
4578
|
+
* from `threads.ts`'s "non-empty AND changed" one, which only covers the
|
|
4579
|
+
* message-status PATCH; the non-empty half is enforced here instead, by
|
|
4580
|
+
* `resolveSessionTitle` never returning an empty/placeholder title.)
|
|
4581
|
+
*
|
|
4582
|
+
* Telemetry-only / never blocks the caller, mirroring `postSignal`: a failure
|
|
4583
|
+
* is logged and the title is simply retried on the next heartbeat tick (the
|
|
4584
|
+
* caller only latches `titleSynced` on `true`).
|
|
4585
|
+
*/
|
|
4586
|
+
async patchConversationTitle(conversationId, title) {
|
|
4587
|
+
try {
|
|
4588
|
+
const res = await this.fetchImpl(
|
|
4589
|
+
`${this.apiUrl}/runners/${this.agentId}/conversations/${conversationId}`,
|
|
4590
|
+
{
|
|
4591
|
+
method: "PATCH",
|
|
4592
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
4593
|
+
body: JSON.stringify({ title })
|
|
4594
|
+
}
|
|
4595
|
+
);
|
|
4596
|
+
if (!res.ok) {
|
|
4597
|
+
this.log({
|
|
4598
|
+
level: "debug",
|
|
4599
|
+
message: `Mid-turn title sync PATCH for conversation ${conversationId.slice(0, 8)} returned HTTP ${res.status} (best-effort, will retry next heartbeat)`,
|
|
4600
|
+
conversation_id: conversationId
|
|
4601
|
+
});
|
|
4602
|
+
return false;
|
|
4603
|
+
}
|
|
4604
|
+
return true;
|
|
4605
|
+
} catch (err) {
|
|
4606
|
+
this.log({
|
|
4607
|
+
level: "debug",
|
|
4608
|
+
message: `Best-effort mid-turn title sync PATCH failed for conversation ${conversationId.slice(0, 8)} (will retry next heartbeat): ${err instanceof Error ? err.message : String(err)}`,
|
|
4609
|
+
conversation_id: conversationId
|
|
4610
|
+
});
|
|
4611
|
+
return false;
|
|
4612
|
+
}
|
|
4613
|
+
}
|
|
4358
4614
|
/**
|
|
4359
4615
|
* DEFENSIVE cross-check for the restart-recovery path (WI-2): is any descendant
|
|
4360
4616
|
* (`task` sub-agent) session under `rootSessionId` still genuinely doing work?
|
|
@@ -4413,6 +4669,84 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4413
4669
|
}
|
|
4414
4670
|
return false;
|
|
4415
4671
|
}
|
|
4672
|
+
/**
|
|
4673
|
+
* LIVE-PATH descendant-liveness check (#721): is any descendant (`task`
|
|
4674
|
+
* sub-agent) session under `rootSessionId` currently ONGOING per OpenCode's own
|
|
4675
|
+
* in-memory status map (`isSessionOngoing` — `busy`/`retry`)?
|
|
4676
|
+
*
|
|
4677
|
+
* Deliberately NOT `isAnyDescendantSessionAlive` (the RECOVERY-path
|
|
4678
|
+
* cross-check above): that method judges liveness from the child's OWN
|
|
4679
|
+
* TRANSCRIPT (`isSessionActivelyGenerating`), which is the right (only) option
|
|
4680
|
+
* on the recovery path because a restart WIPES `SessionStatus`. On the LIVE
|
|
4681
|
+
* path the local opencode server IS running, so its in-memory status map is
|
|
4682
|
+
* live and authoritative — and per ADR-0047 §4a ("the child has its own entry
|
|
4683
|
+
* [in the map]"), a `task` descendant's OWN busy/retry entry reflects its
|
|
4684
|
+
* ENTIRE turn (including any tool call it is itself executing), not a
|
|
4685
|
+
* per-message transcript snapshot. This sidesteps the "child's own tool is
|
|
4686
|
+
* executing, between its step's completion and the next generation step"
|
|
4687
|
+
* transcript gap that a transcript-based check would need a second,
|
|
4688
|
+
* sustained-window bound to guard against — it is simply not derived from
|
|
4689
|
+
* message timestamps at all.
|
|
4690
|
+
*
|
|
4691
|
+
* Why not just check `isSessionOngoing(port, rootSessionId)` (the ROOT's own
|
|
4692
|
+
* status, as the recovery path does per §4a)? Because on the LIVE path the
|
|
4693
|
+
* root session can be shared: a SECOND, unrelated user message can land on the
|
|
4694
|
+
* SAME session (issue #721's own root cause) and keep the root `busy` for a
|
|
4695
|
+
* reason that has nothing to do with THIS message's delegation. A `task`
|
|
4696
|
+
* descendant session is spawned for exactly one delegated turn and never
|
|
4697
|
+
* reused, so its OWN status-map entry is unambiguous evidence about that one
|
|
4698
|
+
* delegation — which the root's status is not.
|
|
4699
|
+
*
|
|
4700
|
+
* Why membership is checked via `resolveSessionMembership`, NOT
|
|
4701
|
+
* `sessionBelongsTo`: `sessionBelongsTo` collapses a transient
|
|
4702
|
+
* `GET /session/:id` fetch failure into "not a descendant", which would
|
|
4703
|
+
* silently drop a genuinely-live candidate from consideration on the one
|
|
4704
|
+
* unlucky tick its membership-walk fetch hiccups (#721).
|
|
4705
|
+
* `resolveSessionMembership` keeps that failure mode as a distinct `null`
|
|
4706
|
+
* (indeterminate) so it is folded into THIS method's own `indeterminate` flag
|
|
4707
|
+
* instead.
|
|
4708
|
+
*
|
|
4709
|
+
* Return contract (note the DIFFERENT judge vs. `isAnyDescendantSessionAlive`):
|
|
4710
|
+
* - `true` → some descendant session is `busy`/`retry` (genuinely ongoing).
|
|
4711
|
+
* - `false` → enumeration succeeded, EVERY candidate's MEMBERSHIP was
|
|
4712
|
+
* confirmed either way (`resolveSessionMembership` never
|
|
4713
|
+
* returned `null`), and every CONFIRMED descendant's status read
|
|
4714
|
+
* succeeded and is not ongoing (includes "no descendant session
|
|
4715
|
+
* exists at all" — e.g. a plain, non-`task` tool call).
|
|
4716
|
+
* - `null` → INDETERMINATE: `listSessions` failed, OR at least one
|
|
4717
|
+
* candidate's MEMBERSHIP could not be confirmed
|
|
4718
|
+
* (`resolveSessionMembership` returned `null` — a fetch failure
|
|
4719
|
+
* or pathological chain partway through the parent walk), OR at
|
|
4720
|
+
* least one CONFIRMED descendant's `isSessionOngoing` read
|
|
4721
|
+
* failed — and no OTHER candidate was already confirmed `true`.
|
|
4722
|
+
* The caller MUST NOT treat `null` the same as `false` here
|
|
4723
|
+
* (unlike the recovery cross-check's contract) — see
|
|
4724
|
+
* `isB2AbandonmentConfirmed`.
|
|
4725
|
+
*/
|
|
4726
|
+
async isAnyDescendantSessionOngoing(rootSessionId) {
|
|
4727
|
+
const sessions = await listSessions(this.port);
|
|
4728
|
+
if (!sessions) {
|
|
4729
|
+
this.log({
|
|
4730
|
+
level: "warn",
|
|
4731
|
+
message: `Could not enumerate sessions to check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating descendant liveness as indeterminate`
|
|
4732
|
+
});
|
|
4733
|
+
return null;
|
|
4734
|
+
}
|
|
4735
|
+
let indeterminate = false;
|
|
4736
|
+
for (const candidate of sessions) {
|
|
4737
|
+
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
4738
|
+
const membership = await this.resolveSessionMembership(candidate.id, rootSessionId);
|
|
4739
|
+
if (membership === null) {
|
|
4740
|
+
indeterminate = true;
|
|
4741
|
+
continue;
|
|
4742
|
+
}
|
|
4743
|
+
if (membership === false) continue;
|
|
4744
|
+
const ongoing = await isSessionOngoing(this.port, candidate.id);
|
|
4745
|
+
if (ongoing === true) return true;
|
|
4746
|
+
if (ongoing === null) indeterminate = true;
|
|
4747
|
+
}
|
|
4748
|
+
return indeterminate ? null : false;
|
|
4749
|
+
}
|
|
4416
4750
|
/**
|
|
4417
4751
|
* Cheap decision-telemetry label for a running row's LAST correlated reply
|
|
4418
4752
|
* (WI-2 Task 2.2): which running SHAPE it is, for the status-gated recovery log.
|
|
@@ -4676,7 +5010,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4676
5010
|
* exists but is wedged, so the next attempt must get a fresh one
|
|
4677
5011
|
* instead of reusing it — see WI-1's server-side null-clearing PATCH).
|
|
4678
5012
|
*/
|
|
4679
|
-
async markFailed(conversationId, messageId, sessionId, error2, usage) {
|
|
5013
|
+
async markFailed(conversationId, messageId, sessionId, error2, usage, failure) {
|
|
4680
5014
|
const body = { status: "failed" };
|
|
4681
5015
|
if (sessionId === null) {
|
|
4682
5016
|
body.opencode_session_id = null;
|
|
@@ -4685,6 +5019,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4685
5019
|
}
|
|
4686
5020
|
if (error2 !== void 0) body.error = error2;
|
|
4687
5021
|
if (usage) Object.assign(body, usage);
|
|
5022
|
+
if (failure) {
|
|
5023
|
+
body.failure_kind = failure.kind;
|
|
5024
|
+
body.failure_provider_id = failure.providerId;
|
|
5025
|
+
body.failure_model_id = failure.modelId;
|
|
5026
|
+
body.failure_reason = failure.reason;
|
|
5027
|
+
}
|
|
4688
5028
|
await this.callWithRetry(
|
|
4689
5029
|
"marking message as failed",
|
|
4690
5030
|
() => this.fetchImpl(
|
|
@@ -4697,6 +5037,29 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4697
5037
|
)
|
|
4698
5038
|
);
|
|
4699
5039
|
}
|
|
5040
|
+
/**
|
|
5041
|
+
* Classify an errored turn as a model-auth failure (#736 P1-4), for `markFailed`.
|
|
5042
|
+
*
|
|
5043
|
+
* `messageFailure` alone (structured OpenCode error → `model_auth`) covers
|
|
5044
|
+
* most cases; when it returns `null` on this ALREADY-FAILED turn, fall back
|
|
5045
|
+
* to the P1-2b zero-provider check — one extra loopback call to
|
|
5046
|
+
* `hasAnyConfiguredProvider`, only reached when the structured classifier
|
|
5047
|
+
* couldn't place it. Fails open (never throws): a fallback probe failure
|
|
5048
|
+
* (`null`/indeterminate) leaves the classification `null`, which produces
|
|
5049
|
+
* today's byte-identical PATCH body via `markFailed`'s `if (failure)` guard.
|
|
5050
|
+
*/
|
|
5051
|
+
async classifyModelAuthFailure(messages, userMessageId) {
|
|
5052
|
+
const classified = messageFailure(messages, userMessageId);
|
|
5053
|
+
if (classified != null) return classified;
|
|
5054
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
5055
|
+
const hasProvider = await hasAnyConfiguredProvider(this.port);
|
|
5056
|
+
return applyZeroProviderFallback(
|
|
5057
|
+
classified,
|
|
5058
|
+
hasProvider,
|
|
5059
|
+
reply?.info?.providerID ?? null,
|
|
5060
|
+
reply?.info?.modelID ?? null
|
|
5061
|
+
);
|
|
5062
|
+
}
|
|
4700
5063
|
/**
|
|
4701
5064
|
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
4702
5065
|
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
@@ -4873,7 +5236,7 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
4873
5236
|
console.log(chalk5.yellow("Tip: Run with the correct port:"));
|
|
4874
5237
|
console.log(
|
|
4875
5238
|
chalk5.dim(
|
|
4876
|
-
` ${getCliName()} run --
|
|
5239
|
+
` ${getCliName()} run --runner ${ctx.agentId} --port ${runningInstances[0].port}`
|
|
4877
5240
|
)
|
|
4878
5241
|
);
|
|
4879
5242
|
}
|
|
@@ -5003,19 +5366,21 @@ async function resolveAgentIdFromKey(authHeader) {
|
|
|
5003
5366
|
return { agent_id: data.agent_id };
|
|
5004
5367
|
}
|
|
5005
5368
|
return {
|
|
5006
|
-
error: "Cannot resolve runner ID: auth type is not agent_key. Please provide --
|
|
5369
|
+
error: "Cannot resolve runner ID: auth type is not agent_key. Please provide --runner explicitly."
|
|
5007
5370
|
};
|
|
5008
5371
|
} catch (error2) {
|
|
5009
5372
|
const message = error2 instanceof Error ? error2.message : "Unknown error";
|
|
5010
5373
|
return { error: `Failed to resolve runner from key: ${message}` };
|
|
5011
5374
|
}
|
|
5012
5375
|
}
|
|
5376
|
+
var BEST_EFFORT_NOTIFY_TIMEOUT_MS = 2e3;
|
|
5013
5377
|
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
5014
5378
|
const apiUrl = getApiUrlConfig();
|
|
5015
5379
|
try {
|
|
5016
5380
|
const response = await fetch(`${apiUrl}/runners/${agentId}/disconnect`, {
|
|
5017
5381
|
method: "POST",
|
|
5018
|
-
headers: { Authorization: authHeader }
|
|
5382
|
+
headers: { Authorization: authHeader },
|
|
5383
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
5019
5384
|
});
|
|
5020
5385
|
if (!response.ok) {
|
|
5021
5386
|
const serverMessage = await readErrorMessage(response);
|
|
@@ -5026,7 +5391,35 @@ async function notifyAgentDisconnected(agentId, authHeader) {
|
|
|
5026
5391
|
}
|
|
5027
5392
|
return { ok: true };
|
|
5028
5393
|
} catch (error2) {
|
|
5029
|
-
return { ok: false, error:
|
|
5394
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
5395
|
+
}
|
|
5396
|
+
}
|
|
5397
|
+
function describeBestEffortError(error2) {
|
|
5398
|
+
const name = error2?.name;
|
|
5399
|
+
if (name === "TimeoutError" || name === "AbortError") {
|
|
5400
|
+
return `timed out after ${BEST_EFFORT_NOTIFY_TIMEOUT_MS}ms`;
|
|
5401
|
+
}
|
|
5402
|
+
return error2 instanceof Error ? error2.message : String(error2);
|
|
5403
|
+
}
|
|
5404
|
+
async function reportMicrovmId(agentId, authHeader, microvmId) {
|
|
5405
|
+
try {
|
|
5406
|
+
const apiUrl = getApiUrlConfig();
|
|
5407
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/microvm`, {
|
|
5408
|
+
method: "POST",
|
|
5409
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
5410
|
+
body: JSON.stringify({ microvm_id: microvmId }),
|
|
5411
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
5412
|
+
});
|
|
5413
|
+
if (!response.ok) {
|
|
5414
|
+
const serverMessage = await readErrorMessage(response);
|
|
5415
|
+
return {
|
|
5416
|
+
ok: false,
|
|
5417
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
5418
|
+
};
|
|
5419
|
+
}
|
|
5420
|
+
return { ok: true };
|
|
5421
|
+
} catch (error2) {
|
|
5422
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
5030
5423
|
}
|
|
5031
5424
|
}
|
|
5032
5425
|
async function getAgentInfo(agentId, authHeader) {
|
|
@@ -5076,6 +5469,7 @@ var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
|
5076
5469
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
5077
5470
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
5078
5471
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
5472
|
+
var TELEMETRY_SHUTDOWN_TIMEOUT_MS = 5e3;
|
|
5079
5473
|
function resolveLogLevel(options) {
|
|
5080
5474
|
const accepted = Object.keys(LOG_LEVELS);
|
|
5081
5475
|
const validate = (value, source) => {
|
|
@@ -5106,7 +5500,7 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
5106
5500
|
if (trimmed === "") {
|
|
5107
5501
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
5108
5502
|
}
|
|
5109
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
5503
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join2(homeDir, trimmed.slice(2)) : trimmed;
|
|
5110
5504
|
if (!isAbsolute2(expanded)) {
|
|
5111
5505
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
5112
5506
|
}
|
|
@@ -5402,7 +5796,18 @@ async function notifyOffline(state) {
|
|
|
5402
5796
|
if (state.interactive) displayStatus(state);
|
|
5403
5797
|
}
|
|
5404
5798
|
}
|
|
5799
|
+
async function timeShutdownPhase(state, durations, name, run2) {
|
|
5800
|
+
const startedAt = Date.now();
|
|
5801
|
+
try {
|
|
5802
|
+
return await run2();
|
|
5803
|
+
} finally {
|
|
5804
|
+
const elapsedMs = Date.now() - startedAt;
|
|
5805
|
+
durations[name] = elapsedMs;
|
|
5806
|
+
log2(state, `Shutdown phase ${name}: ${elapsedMs}ms`);
|
|
5807
|
+
}
|
|
5808
|
+
}
|
|
5405
5809
|
async function cleanup(state, opts = {}) {
|
|
5810
|
+
const durations = {};
|
|
5406
5811
|
state.running = false;
|
|
5407
5812
|
for (const timer of state.sessionCleanupTimers) {
|
|
5408
5813
|
clearInterval(timer);
|
|
@@ -5416,7 +5821,13 @@ async function cleanup(state, opts = {}) {
|
|
|
5416
5821
|
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
5417
5822
|
displayStatus(state);
|
|
5418
5823
|
}
|
|
5419
|
-
const
|
|
5824
|
+
const driver = state.channelDriver;
|
|
5825
|
+
const settled = await timeShutdownPhase(
|
|
5826
|
+
state,
|
|
5827
|
+
durations,
|
|
5828
|
+
"drain",
|
|
5829
|
+
() => driver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS)
|
|
5830
|
+
);
|
|
5420
5831
|
if (!settled) {
|
|
5421
5832
|
logActivity(state, {
|
|
5422
5833
|
type: "info",
|
|
@@ -5425,13 +5836,15 @@ async function cleanup(state, opts = {}) {
|
|
|
5425
5836
|
if (state.interactive) displayStatus(state);
|
|
5426
5837
|
}
|
|
5427
5838
|
}
|
|
5428
|
-
await notifyOffline(state);
|
|
5839
|
+
await timeShutdownPhase(state, durations, "offline_notify", () => notifyOffline(state));
|
|
5429
5840
|
if (state.connection) {
|
|
5430
|
-
state.connection
|
|
5841
|
+
const connection = state.connection;
|
|
5842
|
+
await timeShutdownPhase(state, durations, "tunnel_close", () => connection.close());
|
|
5431
5843
|
state.connection = null;
|
|
5432
5844
|
}
|
|
5433
5845
|
if (state.opencodeProcess) {
|
|
5434
|
-
|
|
5846
|
+
const opencodeProcess = state.opencodeProcess;
|
|
5847
|
+
await timeShutdownPhase(state, durations, "opencode_stop", () => stopOpenCode(opencodeProcess));
|
|
5435
5848
|
if (state.interactive) {
|
|
5436
5849
|
logActivity(state, { type: "info", message: "Stopped OpenCode process" });
|
|
5437
5850
|
displayStatus(state);
|
|
@@ -5440,6 +5853,7 @@ async function cleanup(state, opts = {}) {
|
|
|
5440
5853
|
}
|
|
5441
5854
|
state.opencodeProcess = null;
|
|
5442
5855
|
}
|
|
5856
|
+
return durations;
|
|
5443
5857
|
}
|
|
5444
5858
|
async function run(options) {
|
|
5445
5859
|
const interactive = isInteractive(options.json);
|
|
@@ -5447,7 +5861,7 @@ async function run(options) {
|
|
|
5447
5861
|
let fileSyncDirectories;
|
|
5448
5862
|
try {
|
|
5449
5863
|
logLevel = resolveLogLevel(options);
|
|
5450
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
5864
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir2());
|
|
5451
5865
|
} catch (error2) {
|
|
5452
5866
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
5453
5867
|
if (options.json) {
|
|
@@ -5510,14 +5924,38 @@ async function run(options) {
|
|
|
5510
5924
|
const handleSignal = async () => {
|
|
5511
5925
|
if (state.shuttingDown) return;
|
|
5512
5926
|
state.shuttingDown = true;
|
|
5927
|
+
const shutdownStartedAt = Date.now();
|
|
5513
5928
|
if (state.interactive) {
|
|
5514
5929
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
5515
5930
|
displayStatus(state);
|
|
5516
5931
|
} else {
|
|
5517
5932
|
log2(state, "Shutting down...");
|
|
5518
5933
|
}
|
|
5519
|
-
await cleanup(state, { graceful: true });
|
|
5520
|
-
|
|
5934
|
+
const durations = await cleanup(state, { graceful: true });
|
|
5935
|
+
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
5936
|
+
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
5937
|
+
let timer;
|
|
5938
|
+
const flushed = shutdownTelemetry().then(
|
|
5939
|
+
() => true,
|
|
5940
|
+
(error2) => {
|
|
5941
|
+
log2(
|
|
5942
|
+
state,
|
|
5943
|
+
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
5944
|
+
"warn"
|
|
5945
|
+
);
|
|
5946
|
+
return true;
|
|
5947
|
+
}
|
|
5948
|
+
);
|
|
5949
|
+
const timedOut = new Promise((resolve3) => {
|
|
5950
|
+
timer = setTimeout(() => resolve3(false), telemetryBudgetMs);
|
|
5951
|
+
});
|
|
5952
|
+
if (!await Promise.race([flushed, timedOut])) {
|
|
5953
|
+
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
5954
|
+
}
|
|
5955
|
+
clearTimeout(timer);
|
|
5956
|
+
});
|
|
5957
|
+
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
5958
|
+
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
5521
5959
|
process.exit(0);
|
|
5522
5960
|
};
|
|
5523
5961
|
process.on("SIGINT", handleSignal);
|
|
@@ -5631,6 +6069,21 @@ async function run(options) {
|
|
|
5631
6069
|
}
|
|
5632
6070
|
spinner?.succeed(`Runner: ${validation.agent.name || state.agentId}`);
|
|
5633
6071
|
state.agentName = validation.agent.name;
|
|
6072
|
+
const microvmId = process.env.MICROVM_ID?.trim();
|
|
6073
|
+
if (microvmId) {
|
|
6074
|
+
const reported = await reportMicrovmId(state.agentId, state.authHeader, microvmId);
|
|
6075
|
+
if (reported.ok) {
|
|
6076
|
+
log2(state, "Reported MicroVM identity so this runner can be resumed rather than restarted");
|
|
6077
|
+
} else {
|
|
6078
|
+
const message = `Could not report MicroVM identity (future wakes will cold-start): ${reported.error}`;
|
|
6079
|
+
log2(state, message, "warn");
|
|
6080
|
+
if (state.interactive && !state.json) {
|
|
6081
|
+
logActivity(state, { type: "info", level: "warn", message });
|
|
6082
|
+
}
|
|
6083
|
+
}
|
|
6084
|
+
} else {
|
|
6085
|
+
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
6086
|
+
}
|
|
5634
6087
|
const ocSpinner = interactive && !state.json ? ora3("Checking OpenCode...").start() : null;
|
|
5635
6088
|
try {
|
|
5636
6089
|
const oc = await ensureOpenCodeRunning({
|
|
@@ -5682,7 +6135,7 @@ async function run(options) {
|
|
|
5682
6135
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
5683
6136
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
5684
6137
|
fileSyncDirectories,
|
|
5685
|
-
homeDir:
|
|
6138
|
+
homeDir: homedir2(),
|
|
5686
6139
|
log: (entry) => (
|
|
5687
6140
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
5688
6141
|
// survive the sink filter (they no longer collapse to info). `type`
|
|
@@ -5709,6 +6162,18 @@ async function run(options) {
|
|
|
5709
6162
|
type: "info",
|
|
5710
6163
|
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (runner: ${agentId})`
|
|
5711
6164
|
});
|
|
6165
|
+
if (options.tunnelReadyFile) {
|
|
6166
|
+
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
6167
|
+
if (marker.ok) {
|
|
6168
|
+
log2(state, `Wrote tunnel readiness marker to ${options.tunnelReadyFile}`, "debug");
|
|
6169
|
+
} else {
|
|
6170
|
+
log2(
|
|
6171
|
+
state,
|
|
6172
|
+
`Failed to write tunnel readiness marker to ${options.tunnelReadyFile}: ${marker.error}`,
|
|
6173
|
+
"error"
|
|
6174
|
+
);
|
|
6175
|
+
}
|
|
6176
|
+
}
|
|
5712
6177
|
emitAgentConnected(state.agentId, {
|
|
5713
6178
|
port: state.port,
|
|
5714
6179
|
cli_version: getCliVersion(),
|
|
@@ -5873,6 +6338,9 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
5873
6338
|
"Allow Evident to write files into this directory (repeatable). Omit to disable file sync entirely.",
|
|
5874
6339
|
(value, previous) => previous.concat([value]),
|
|
5875
6340
|
[]
|
|
6341
|
+
).option(
|
|
6342
|
+
"--tunnel-ready-file <path>",
|
|
6343
|
+
"Path to write once the tunnel is connected (set by the MicroVM hooks; unused on a developer machine)"
|
|
5876
6344
|
).action(
|
|
5877
6345
|
(options) => {
|
|
5878
6346
|
run({
|
|
@@ -5892,7 +6360,8 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
5892
6360
|
sessionCleanupInterval: options.sessionCleanupInterval,
|
|
5893
6361
|
// Raw values — expansion/validation is single-sourced in run.ts's
|
|
5894
6362
|
// resolveFileSyncDirectories.
|
|
5895
|
-
enableFileSyncTo: options.enableFileSyncTo
|
|
6363
|
+
enableFileSyncTo: options.enableFileSyncTo,
|
|
6364
|
+
tunnelReadyFile: options.tunnelReadyFile
|
|
5896
6365
|
});
|
|
5897
6366
|
}
|
|
5898
6367
|
);
|