@evident-ai/cli 3.1.1-dev.691c8d0 → 3.1.1-dev.698e558
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/index.js +1359 -138
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -11,8 +11,8 @@ import chalk2 from "chalk";
|
|
|
11
11
|
|
|
12
12
|
// src/lib/config.ts
|
|
13
13
|
import Conf from "conf";
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
14
|
+
import { chmodSync, existsSync, statSync } from "fs";
|
|
15
|
+
import { dirname } from "path";
|
|
16
16
|
var PRODUCTION_API_URL = "https://api.production.evident.run/v1";
|
|
17
17
|
var PRODUCTION_TUNNEL_URL = "wss://tunnel.production.evident.run";
|
|
18
18
|
var defaults = {
|
|
@@ -47,8 +47,35 @@ var credentials = new Conf({
|
|
|
47
47
|
projectName: "evident",
|
|
48
48
|
projectSuffix: "",
|
|
49
49
|
configName: "credentials",
|
|
50
|
-
defaults: {}
|
|
50
|
+
defaults: {},
|
|
51
|
+
configFileMode: 384
|
|
51
52
|
});
|
|
53
|
+
var CREDENTIALS_FILE_MODE = 384;
|
|
54
|
+
var CREDENTIALS_DIR_MODE = 448;
|
|
55
|
+
var permissionWarningEmitted = false;
|
|
56
|
+
function hardenCredentialsPermissions() {
|
|
57
|
+
if (process.platform === "win32") {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
const file = credentials.path;
|
|
61
|
+
for (const [path, mode] of [
|
|
62
|
+
[file, CREDENTIALS_FILE_MODE],
|
|
63
|
+
[dirname(file), CREDENTIALS_DIR_MODE]
|
|
64
|
+
]) {
|
|
65
|
+
try {
|
|
66
|
+
if (existsSync(path) && (statSync(path).mode & 511) !== mode) {
|
|
67
|
+
chmodSync(path, mode);
|
|
68
|
+
}
|
|
69
|
+
} catch (err) {
|
|
70
|
+
if (!permissionWarningEmitted) {
|
|
71
|
+
permissionWarningEmitted = true;
|
|
72
|
+
console.error(
|
|
73
|
+
`[config] could not restrict permissions on ${path}; the credentials file may be readable by other users on this machine: ${err instanceof Error ? err.message : String(err)}`
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
52
79
|
function getApiUrlConfig() {
|
|
53
80
|
return getApiUrl();
|
|
54
81
|
}
|
|
@@ -59,6 +86,7 @@ function credentialsKey() {
|
|
|
59
86
|
return getApiUrl();
|
|
60
87
|
}
|
|
61
88
|
function getCredentials() {
|
|
89
|
+
hardenCredentialsPermissions();
|
|
62
90
|
const byEndpoint = credentials.get("byEndpoint") ?? {};
|
|
63
91
|
return byEndpoint[credentialsKey()] ?? {};
|
|
64
92
|
}
|
|
@@ -70,14 +98,17 @@ function setCredentials(creds) {
|
|
|
70
98
|
expiresAt: creds.expiresAt
|
|
71
99
|
};
|
|
72
100
|
credentials.set("byEndpoint", byEndpoint);
|
|
101
|
+
hardenCredentialsPermissions();
|
|
73
102
|
}
|
|
74
103
|
function clearCredentials() {
|
|
75
104
|
const byEndpoint = credentials.get("byEndpoint") ?? {};
|
|
76
105
|
delete byEndpoint[credentialsKey()];
|
|
77
106
|
credentials.set("byEndpoint", byEndpoint);
|
|
107
|
+
hardenCredentialsPermissions();
|
|
78
108
|
}
|
|
79
109
|
function clearAllCredentials() {
|
|
80
110
|
credentials.clear();
|
|
111
|
+
hardenCredentialsPermissions();
|
|
81
112
|
}
|
|
82
113
|
function getCliName() {
|
|
83
114
|
const argv1 = process.argv[1] || "";
|
|
@@ -236,16 +267,28 @@ async function getToken() {
|
|
|
236
267
|
}
|
|
237
268
|
return null;
|
|
238
269
|
}
|
|
270
|
+
function toError(err) {
|
|
271
|
+
return err instanceof Error ? err : new Error(String(err));
|
|
272
|
+
}
|
|
239
273
|
async function deleteToken(options = {}) {
|
|
240
274
|
const keytar = await getKeytar();
|
|
275
|
+
const failures = [];
|
|
241
276
|
if (keytar) {
|
|
242
277
|
if (options.all) {
|
|
243
|
-
|
|
278
|
+
let accounts = [];
|
|
279
|
+
try {
|
|
280
|
+
accounts = await keytar.findCredentials(SERVICE_NAME);
|
|
281
|
+
} catch (err) {
|
|
282
|
+
failures.push({ type: "enumerate", error: toError(err) });
|
|
283
|
+
}
|
|
244
284
|
await Promise.all(
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
285
|
+
accounts.map(async (entry) => {
|
|
286
|
+
try {
|
|
287
|
+
await keytar.deletePassword(SERVICE_NAME, entry.account);
|
|
288
|
+
} catch (err) {
|
|
289
|
+
failures.push({ type: "delete", account: entry.account, error: toError(err) });
|
|
290
|
+
}
|
|
291
|
+
})
|
|
249
292
|
);
|
|
250
293
|
} else {
|
|
251
294
|
await keytar.deletePassword(SERVICE_NAME, keychainAccount());
|
|
@@ -256,6 +299,7 @@ async function deleteToken(options = {}) {
|
|
|
256
299
|
} else {
|
|
257
300
|
clearCredentials();
|
|
258
301
|
}
|
|
302
|
+
return { failures };
|
|
259
303
|
}
|
|
260
304
|
|
|
261
305
|
// src/utils/ui.ts
|
|
@@ -285,14 +329,14 @@ function blank() {
|
|
|
285
329
|
console.log();
|
|
286
330
|
}
|
|
287
331
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
288
|
-
return new Promise((
|
|
332
|
+
return new Promise((resolve3) => {
|
|
289
333
|
process.stdout.write(chalk.dim(prompt));
|
|
290
334
|
const handler = () => {
|
|
291
335
|
process.stdin.removeListener("data", handler);
|
|
292
336
|
process.stdin.setRawMode?.(false);
|
|
293
337
|
process.stdin.pause();
|
|
294
338
|
console.log();
|
|
295
|
-
|
|
339
|
+
resolve3();
|
|
296
340
|
};
|
|
297
341
|
if (process.stdin.isTTY) {
|
|
298
342
|
process.stdin.setRawMode?.(true);
|
|
@@ -302,7 +346,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
302
346
|
});
|
|
303
347
|
}
|
|
304
348
|
function sleep(ms) {
|
|
305
|
-
return new Promise((
|
|
349
|
+
return new Promise((resolve3) => setTimeout(resolve3, ms));
|
|
306
350
|
}
|
|
307
351
|
|
|
308
352
|
// src/commands/login.ts
|
|
@@ -373,22 +417,25 @@ async function deviceFlowLogin(options) {
|
|
|
373
417
|
}
|
|
374
418
|
async function tokenLogin() {
|
|
375
419
|
console.log("Token login mode.");
|
|
376
|
-
console.log("
|
|
420
|
+
console.log("Create a token under Settings \u2192 CLI tokens in the dashboard, then paste it below.");
|
|
421
|
+
console.log(
|
|
422
|
+
"(Alternatively, run `evident login` on a machine with a browser, or set EVIDENT_TOKEN for CI.)"
|
|
423
|
+
);
|
|
377
424
|
blank();
|
|
378
425
|
process.stdout.write("Paste token: ");
|
|
379
|
-
const token = await new Promise((
|
|
426
|
+
const token = await new Promise((resolve3) => {
|
|
380
427
|
let data = "";
|
|
381
428
|
process.stdin.setEncoding("utf8");
|
|
382
429
|
process.stdin.on("data", (chunk) => {
|
|
383
430
|
data += chunk;
|
|
384
431
|
});
|
|
385
432
|
process.stdin.on("end", () => {
|
|
386
|
-
|
|
433
|
+
resolve3(data.trim());
|
|
387
434
|
});
|
|
388
435
|
if (process.stdin.isTTY) {
|
|
389
436
|
process.stdin.once("data", (chunk) => {
|
|
390
437
|
process.stdin.pause();
|
|
391
|
-
|
|
438
|
+
resolve3(chunk.toString().trim());
|
|
392
439
|
});
|
|
393
440
|
process.stdin.resume();
|
|
394
441
|
}
|
|
@@ -397,13 +444,22 @@ async function tokenLogin() {
|
|
|
397
444
|
printError("No token provided.");
|
|
398
445
|
process.exit(1);
|
|
399
446
|
}
|
|
447
|
+
await validateAndStoreToken(token);
|
|
448
|
+
}
|
|
449
|
+
async function validateAndStoreToken(token) {
|
|
400
450
|
const spinner = ora("Validating token...").start();
|
|
401
451
|
try {
|
|
402
|
-
const result = await api.
|
|
452
|
+
const result = await api.get("/me", {
|
|
453
|
+
headers: { Authorization: `Bearer ${token}` }
|
|
454
|
+
});
|
|
455
|
+
if (!result.user) {
|
|
456
|
+
throw new Error(
|
|
457
|
+
"This token is not a user login (e.g. a runner key). Paste a CLI token instead."
|
|
458
|
+
);
|
|
459
|
+
}
|
|
403
460
|
await storeToken({
|
|
404
461
|
token,
|
|
405
|
-
user: result.user
|
|
406
|
-
expiresAt: result.expires_at
|
|
462
|
+
user: { email: result.user.email }
|
|
407
463
|
});
|
|
408
464
|
spinner.stop();
|
|
409
465
|
printSuccess(`Logged in as ${chalk2.bold(result.user.email)}`);
|
|
@@ -423,9 +479,22 @@ async function login(options) {
|
|
|
423
479
|
}
|
|
424
480
|
|
|
425
481
|
// src/commands/logout.ts
|
|
482
|
+
function describeFailure(failure) {
|
|
483
|
+
if (failure.type === "enumerate") {
|
|
484
|
+
return `could not list stored keychain entries (${failure.error.message})`;
|
|
485
|
+
}
|
|
486
|
+
return `${failure.account} (${failure.error.message})`;
|
|
487
|
+
}
|
|
426
488
|
async function logout(options = {}) {
|
|
427
489
|
if (options.all) {
|
|
428
|
-
await deleteToken({ all: true });
|
|
490
|
+
const result = await deleteToken({ all: true });
|
|
491
|
+
if (result.failures.length > 0) {
|
|
492
|
+
printError(
|
|
493
|
+
`Failed to fully clear your keychain: ${result.failures.map(describeFailure).join("; ")}. Your local credentials file was cleared, but stale keychain entries may remain \u2014 run \`evident logout --all\` again, or remove them manually from your OS keychain / credential manager.`
|
|
494
|
+
);
|
|
495
|
+
process.exitCode = 1;
|
|
496
|
+
return;
|
|
497
|
+
}
|
|
429
498
|
printSuccess("Logged out of all endpoints.");
|
|
430
499
|
return;
|
|
431
500
|
}
|
|
@@ -450,7 +519,9 @@ async function whoami() {
|
|
|
450
519
|
blank();
|
|
451
520
|
console.log(keyValue("Endpoint", apiUrl));
|
|
452
521
|
console.log(keyValue("User", chalk3.bold(credentials2.user.email)));
|
|
453
|
-
|
|
522
|
+
if (credentials2.user.id) {
|
|
523
|
+
console.log(keyValue("User ID", credentials2.user.id));
|
|
524
|
+
}
|
|
454
525
|
if (credentials2.expiresAt) {
|
|
455
526
|
const expiresAt = new Date(credentials2.expiresAt);
|
|
456
527
|
const now = /* @__PURE__ */ new Date();
|
|
@@ -467,9 +538,9 @@ async function whoami() {
|
|
|
467
538
|
}
|
|
468
539
|
|
|
469
540
|
// src/commands/run.ts
|
|
541
|
+
import { homedir as homedir2 } from "os";
|
|
542
|
+
import { isAbsolute as isAbsolute2, join as join2, parse, resolve as resolvePath } from "path";
|
|
470
543
|
import chalk6 from "chalk";
|
|
471
|
-
import ora3 from "ora";
|
|
472
|
-
import { select as select3 } from "@inquirer/prompts";
|
|
473
544
|
|
|
474
545
|
// ../../packages/types/src/telemetry/index.ts
|
|
475
546
|
var TelemetryEventTypes = {
|
|
@@ -478,13 +549,20 @@ var TelemetryEventTypes = {
|
|
|
478
549
|
AGENT_DISCONNECTED: "agent.disconnected",
|
|
479
550
|
AGENT_MESSAGE_PROCESSING: "agent.message_processing",
|
|
480
551
|
AGENT_MESSAGE_DONE: "agent.message_done",
|
|
481
|
-
AGENT_MESSAGE_FAILED: "agent.message_failed"
|
|
552
|
+
AGENT_MESSAGE_FAILED: "agent.message_failed",
|
|
553
|
+
// A `warn`/`error` runner-side log line forwarded server-side for
|
|
554
|
+
// observability (issue #916) — see `apps/cli/src/lib/runner-activity-telemetry.ts`.
|
|
555
|
+
RUNNER_ACTIVITY: "runner.activity"
|
|
482
556
|
};
|
|
483
557
|
|
|
484
558
|
// ../../packages/types/src/tunnel/index.ts
|
|
485
559
|
var MAX_FRAME_BYTES = 256 * 1024;
|
|
486
560
|
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
487
561
|
|
|
562
|
+
// ../../packages/types/src/runner-files.ts
|
|
563
|
+
var MAX_FILE_PUSH_BYTES = 64 * 1024;
|
|
564
|
+
var MAX_FILE_SYNC_DIRECTORIES = 16;
|
|
565
|
+
|
|
488
566
|
// ../../packages/types/src/logging/index.ts
|
|
489
567
|
var CORRELATION_ID_HEADER = "x-evident-correlation-id";
|
|
490
568
|
function log(level, event, fields) {
|
|
@@ -514,6 +592,10 @@ function stripQuery(url) {
|
|
|
514
592
|
}
|
|
515
593
|
}
|
|
516
594
|
|
|
595
|
+
// src/commands/run.ts
|
|
596
|
+
import ora3 from "ora";
|
|
597
|
+
import { select as select3 } from "@inquirer/prompts";
|
|
598
|
+
|
|
517
599
|
// src/lib/telemetry.ts
|
|
518
600
|
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
519
601
|
function getCliVersion() {
|
|
@@ -525,6 +607,13 @@ var isShuttingDown = false;
|
|
|
525
607
|
var FLUSH_INTERVAL_MS = 5e3;
|
|
526
608
|
var MAX_BUFFER_SIZE = 50;
|
|
527
609
|
var FLUSH_TIMEOUT_MS = 3e3;
|
|
610
|
+
var authProvider = null;
|
|
611
|
+
function setTelemetryAuthProvider(provider) {
|
|
612
|
+
authProvider = provider;
|
|
613
|
+
}
|
|
614
|
+
var FLUSH_FAILURE_LOG_INTERVAL_MS = 6e4;
|
|
615
|
+
var lastFlushFailureLoggedAt = 0;
|
|
616
|
+
var suppressedFlushFailureCount = 0;
|
|
528
617
|
function logEvent(eventType, options = {}) {
|
|
529
618
|
const event = {
|
|
530
619
|
event_type: eventType,
|
|
@@ -559,9 +648,16 @@ async function flushEvents() {
|
|
|
559
648
|
flushTimeout = null;
|
|
560
649
|
}
|
|
561
650
|
try {
|
|
562
|
-
const
|
|
563
|
-
|
|
564
|
-
|
|
651
|
+
const providerContext = authProvider?.();
|
|
652
|
+
let authHeader;
|
|
653
|
+
if (providerContext?.authHeader) {
|
|
654
|
+
authHeader = providerContext.authHeader;
|
|
655
|
+
} else {
|
|
656
|
+
const credentials2 = await getToken();
|
|
657
|
+
if (!credentials2) {
|
|
658
|
+
return;
|
|
659
|
+
}
|
|
660
|
+
authHeader = `Bearer ${credentials2.token}`;
|
|
565
661
|
}
|
|
566
662
|
const apiUrl = getApiUrlConfig();
|
|
567
663
|
const controller = new AbortController();
|
|
@@ -576,7 +672,7 @@ async function flushEvents() {
|
|
|
576
672
|
method: "POST",
|
|
577
673
|
headers: {
|
|
578
674
|
"Content-Type": "application/json",
|
|
579
|
-
Authorization:
|
|
675
|
+
Authorization: authHeader
|
|
580
676
|
},
|
|
581
677
|
body: JSON.stringify(request),
|
|
582
678
|
signal: controller.signal
|
|
@@ -588,8 +684,15 @@ async function flushEvents() {
|
|
|
588
684
|
clearTimeout(timeout);
|
|
589
685
|
}
|
|
590
686
|
} catch (error2) {
|
|
591
|
-
|
|
592
|
-
|
|
687
|
+
const now = Date.now();
|
|
688
|
+
if (now - lastFlushFailureLoggedAt >= FLUSH_FAILURE_LOG_INTERVAL_MS) {
|
|
689
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
690
|
+
const suffix = suppressedFlushFailureCount > 0 ? ` (${suppressedFlushFailureCount} more suppressed in the last ${FLUSH_FAILURE_LOG_INTERVAL_MS / 1e3}s)` : "";
|
|
691
|
+
console.error(`Telemetry flush error: ${message}${suffix}`);
|
|
692
|
+
lastFlushFailureLoggedAt = now;
|
|
693
|
+
suppressedFlushFailureCount = 0;
|
|
694
|
+
} else {
|
|
695
|
+
suppressedFlushFailureCount++;
|
|
593
696
|
}
|
|
594
697
|
}
|
|
595
698
|
}
|
|
@@ -658,6 +761,69 @@ var EventTypes = {
|
|
|
658
761
|
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
659
762
|
};
|
|
660
763
|
|
|
764
|
+
// src/lib/runner-activity-telemetry.ts
|
|
765
|
+
var FORWARDED_LEVELS = /* @__PURE__ */ new Set(["warn", "error"]);
|
|
766
|
+
var SEVERITY_BY_LEVEL = {
|
|
767
|
+
warn: "warning",
|
|
768
|
+
error: "error"
|
|
769
|
+
};
|
|
770
|
+
var MAX_MESSAGE_LENGTH = 500;
|
|
771
|
+
var TRUNCATION_MARKER = "\u2026";
|
|
772
|
+
function redact(message) {
|
|
773
|
+
return message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
774
|
+
}
|
|
775
|
+
function truncate(message) {
|
|
776
|
+
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
777
|
+
return message.slice(0, MAX_MESSAGE_LENGTH - TRUNCATION_MARKER.length) + TRUNCATION_MARKER;
|
|
778
|
+
}
|
|
779
|
+
var RATE_LIMIT_WINDOW_MS = 6e4;
|
|
780
|
+
var RATE_LIMIT_MAX_EVENTS = 30;
|
|
781
|
+
var windowStartedAt = 0;
|
|
782
|
+
var windowCount = 0;
|
|
783
|
+
var windowDroppedCount = 0;
|
|
784
|
+
function admitUnderRateLimit(now) {
|
|
785
|
+
if (now - windowStartedAt >= RATE_LIMIT_WINDOW_MS) {
|
|
786
|
+
if (windowDroppedCount > 0) {
|
|
787
|
+
console.error(
|
|
788
|
+
`[runner-activity-telemetry] rate cap reached: dropped ${windowDroppedCount} ${windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (cap ${RATE_LIMIT_MAX_EVENTS}/min)`
|
|
789
|
+
);
|
|
790
|
+
}
|
|
791
|
+
windowStartedAt = now;
|
|
792
|
+
windowCount = 0;
|
|
793
|
+
windowDroppedCount = 0;
|
|
794
|
+
}
|
|
795
|
+
if (windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
796
|
+
windowDroppedCount++;
|
|
797
|
+
if (windowDroppedCount === 1) {
|
|
798
|
+
console.error(
|
|
799
|
+
`[runner-activity-telemetry] rate cap reached (${RATE_LIMIT_MAX_EVENTS} per ${RATE_LIMIT_WINDOW_MS / 1e3}s) \u2014 dropping further entries this window`
|
|
800
|
+
);
|
|
801
|
+
}
|
|
802
|
+
return false;
|
|
803
|
+
}
|
|
804
|
+
windowCount++;
|
|
805
|
+
return true;
|
|
806
|
+
}
|
|
807
|
+
function forwardRunnerActivity(entry, context) {
|
|
808
|
+
try {
|
|
809
|
+
if (!FORWARDED_LEVELS.has(entry.level)) return;
|
|
810
|
+
if (!context.agentId || !context.authHeader) return;
|
|
811
|
+
if (!admitUnderRateLimit(Date.now())) return;
|
|
812
|
+
const rawMessage = entry.error ?? entry.message ?? "";
|
|
813
|
+
const message = truncate(redact(rawMessage));
|
|
814
|
+
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
815
|
+
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
816
|
+
message,
|
|
817
|
+
metadata: { source: "cli.run" },
|
|
818
|
+
agentId: context.agentId
|
|
819
|
+
});
|
|
820
|
+
} catch (err) {
|
|
821
|
+
console.error(
|
|
822
|
+
`[runner-activity-telemetry] failed to forward runner activity: ${err instanceof Error ? err.message : String(err)}`
|
|
823
|
+
);
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
|
|
661
827
|
// src/lib/auth.ts
|
|
662
828
|
async function getAuthCredentials() {
|
|
663
829
|
const runnerKey = process.env.EVIDENT_RUNNER_KEY;
|
|
@@ -725,7 +891,7 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
725
891
|
if (health.healthy) {
|
|
726
892
|
return health;
|
|
727
893
|
}
|
|
728
|
-
await new Promise((
|
|
894
|
+
await new Promise((resolve3) => setTimeout(resolve3, 1e3));
|
|
729
895
|
}
|
|
730
896
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
731
897
|
}
|
|
@@ -1364,7 +1530,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
1364
1530
|
}
|
|
1365
1531
|
}
|
|
1366
1532
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
1367
|
-
await new Promise((
|
|
1533
|
+
await new Promise((resolve3) => setTimeout(resolve3, READ_BACK_DELAY_MS));
|
|
1368
1534
|
}
|
|
1369
1535
|
}
|
|
1370
1536
|
return null;
|
|
@@ -1492,6 +1658,9 @@ function isPreamblePinnedRunning(messages, userMessageId) {
|
|
|
1492
1658
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1493
1659
|
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
1494
1660
|
}
|
|
1661
|
+
function isB2AbandonmentConfirmed(params) {
|
|
1662
|
+
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false;
|
|
1663
|
+
}
|
|
1495
1664
|
function messageError(messages, userMessageId) {
|
|
1496
1665
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1497
1666
|
const error2 = errorOf(reply);
|
|
@@ -1505,6 +1674,42 @@ function messageError(messages, userMessageId) {
|
|
|
1505
1674
|
}
|
|
1506
1675
|
return "The agent run failed.";
|
|
1507
1676
|
}
|
|
1677
|
+
function messageFailure(messages, userMessageId) {
|
|
1678
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1679
|
+
const error2 = errorOf(reply);
|
|
1680
|
+
if (error2 == null || typeof error2 !== "object") return null;
|
|
1681
|
+
const e = error2;
|
|
1682
|
+
const replyProviderId = reply?.info?.providerID ?? null;
|
|
1683
|
+
const replyModelId = reply?.info?.modelID ?? null;
|
|
1684
|
+
if (e.name === "ProviderAuthError") {
|
|
1685
|
+
const data = e.data;
|
|
1686
|
+
const providerId = typeof data?.providerID === "string" && data.providerID || replyProviderId;
|
|
1687
|
+
return { kind: "model_auth", providerId, modelId: replyModelId, reason: "missing" };
|
|
1688
|
+
}
|
|
1689
|
+
if (e.name === "APIError") {
|
|
1690
|
+
const data = e.data;
|
|
1691
|
+
const statusCode = data?.statusCode;
|
|
1692
|
+
if (statusCode === 401 || statusCode === 403) {
|
|
1693
|
+
return {
|
|
1694
|
+
kind: "model_auth",
|
|
1695
|
+
providerId: replyProviderId,
|
|
1696
|
+
modelId: replyModelId,
|
|
1697
|
+
reason: "rejected"
|
|
1698
|
+
};
|
|
1699
|
+
}
|
|
1700
|
+
}
|
|
1701
|
+
return null;
|
|
1702
|
+
}
|
|
1703
|
+
function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProviderId, replyModelId = null) {
|
|
1704
|
+
if (classified != null) return classified;
|
|
1705
|
+
if (hasConfiguredProvider !== false) return null;
|
|
1706
|
+
return {
|
|
1707
|
+
kind: "model_auth",
|
|
1708
|
+
providerId: replyProviderId,
|
|
1709
|
+
modelId: replyModelId,
|
|
1710
|
+
reason: "missing"
|
|
1711
|
+
};
|
|
1712
|
+
}
|
|
1508
1713
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
1509
1714
|
if (!messages || messages.length === 0) return false;
|
|
1510
1715
|
return messages.some(
|
|
@@ -1738,12 +1943,12 @@ var StreamForwarder = class {
|
|
|
1738
1943
|
let endBody;
|
|
1739
1944
|
if (has_body) {
|
|
1740
1945
|
const chunks = [];
|
|
1741
|
-
bodyPromise = new Promise((
|
|
1946
|
+
bodyPromise = new Promise((resolve3) => {
|
|
1742
1947
|
pushBody = (buf) => {
|
|
1743
1948
|
chunks.push(buf);
|
|
1744
1949
|
};
|
|
1745
1950
|
endBody = () => {
|
|
1746
|
-
|
|
1951
|
+
resolve3(Buffer.concat(chunks));
|
|
1747
1952
|
};
|
|
1748
1953
|
});
|
|
1749
1954
|
}
|
|
@@ -1860,7 +2065,7 @@ function connectTunnel(options) {
|
|
|
1860
2065
|
} = options;
|
|
1861
2066
|
const tunnelUrl = getTunnelUrlConfig();
|
|
1862
2067
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
1863
|
-
return new Promise((
|
|
2068
|
+
return new Promise((resolve3, reject) => {
|
|
1864
2069
|
const ws = new WebSocket2(url, {
|
|
1865
2070
|
headers: {
|
|
1866
2071
|
Authorization: authHeader
|
|
@@ -1915,7 +2120,7 @@ function connectTunnel(options) {
|
|
|
1915
2120
|
clearTimeout(connectionTimeout);
|
|
1916
2121
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
1917
2122
|
onConnected?.(connectedAgentId);
|
|
1918
|
-
|
|
2123
|
+
resolve3({
|
|
1919
2124
|
ws,
|
|
1920
2125
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
1921
2126
|
});
|
|
@@ -2033,6 +2238,416 @@ var RunnerConnection = class {
|
|
|
2033
2238
|
}
|
|
2034
2239
|
};
|
|
2035
2240
|
|
|
2241
|
+
// src/lib/tunnel/ready-marker.ts
|
|
2242
|
+
import { writeFileSync } from "fs";
|
|
2243
|
+
function writeTunnelReadyMarker(path, agentId) {
|
|
2244
|
+
try {
|
|
2245
|
+
writeFileSync(path, `${agentId}
|
|
2246
|
+
`);
|
|
2247
|
+
return { ok: true };
|
|
2248
|
+
} catch (error2) {
|
|
2249
|
+
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
2250
|
+
}
|
|
2251
|
+
}
|
|
2252
|
+
|
|
2253
|
+
// src/lib/channels/driver.ts
|
|
2254
|
+
import { homedir } from "os";
|
|
2255
|
+
|
|
2256
|
+
// src/lib/file-push.ts
|
|
2257
|
+
import { randomUUID } from "crypto";
|
|
2258
|
+
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
2259
|
+
import { basename, dirname as dirname2, isAbsolute, join, relative, resolve as resolve2, sep } from "path";
|
|
2260
|
+
var FILE_MODE = 384;
|
|
2261
|
+
var DIRECTORY_MODE = 448;
|
|
2262
|
+
async function writePushedFile(request) {
|
|
2263
|
+
const { requestedPath, content, allowedDirectories, homeDir } = request;
|
|
2264
|
+
const bytes = content.byteLength;
|
|
2265
|
+
if (allowedDirectories.length === 0) {
|
|
2266
|
+
return refuse("file_sync_disabled", "File sync is not enabled on this runner.", {
|
|
2267
|
+
path: requestedPath,
|
|
2268
|
+
bytes
|
|
2269
|
+
});
|
|
2270
|
+
}
|
|
2271
|
+
if (bytes > MAX_FILE_PUSH_BYTES) {
|
|
2272
|
+
return refuse(
|
|
2273
|
+
"file_too_large",
|
|
2274
|
+
`File is ${bytes} bytes; the limit is ${MAX_FILE_PUSH_BYTES}.`,
|
|
2275
|
+
{
|
|
2276
|
+
path: requestedPath,
|
|
2277
|
+
bytes
|
|
2278
|
+
}
|
|
2279
|
+
);
|
|
2280
|
+
}
|
|
2281
|
+
const candidate = expandAndValidate(requestedPath, homeDir);
|
|
2282
|
+
if (candidate === null) {
|
|
2283
|
+
return refuse("invalid_path", "The requested path is not a valid absolute file path.", {
|
|
2284
|
+
path: requestedPath,
|
|
2285
|
+
bytes
|
|
2286
|
+
});
|
|
2287
|
+
}
|
|
2288
|
+
try {
|
|
2289
|
+
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
2290
|
+
dirname2(candidate)
|
|
2291
|
+
);
|
|
2292
|
+
const realTarget = join(existingAncestor, ...missingSegments, basename(candidate));
|
|
2293
|
+
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
2294
|
+
if (allowedDirectory === null) {
|
|
2295
|
+
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
2296
|
+
path: realTarget,
|
|
2297
|
+
bytes
|
|
2298
|
+
});
|
|
2299
|
+
}
|
|
2300
|
+
if (missingSegments.length > 0) {
|
|
2301
|
+
await createMissingDirectories(existingAncestor, missingSegments);
|
|
2302
|
+
const realParent = await realpath(dirname2(realTarget));
|
|
2303
|
+
if (realParent !== dirname2(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
2304
|
+
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
2305
|
+
path: realTarget,
|
|
2306
|
+
bytes,
|
|
2307
|
+
reason: "parent_changed_after_create"
|
|
2308
|
+
});
|
|
2309
|
+
}
|
|
2310
|
+
}
|
|
2311
|
+
await writeAtomically(realTarget, content);
|
|
2312
|
+
log("info", "file_push_written", { path: realTarget, bytes });
|
|
2313
|
+
return { ok: true, path: realTarget };
|
|
2314
|
+
} catch (err) {
|
|
2315
|
+
const errno = err.code ?? "UNKNOWN";
|
|
2316
|
+
return refuse("write_failed", `The runner could not write the file (${errno}).`, {
|
|
2317
|
+
path: candidate,
|
|
2318
|
+
bytes,
|
|
2319
|
+
errno,
|
|
2320
|
+
...errorFields(err)
|
|
2321
|
+
});
|
|
2322
|
+
}
|
|
2323
|
+
}
|
|
2324
|
+
function expandAndValidate(requestedPath, homeDir) {
|
|
2325
|
+
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
2326
|
+
return null;
|
|
2327
|
+
}
|
|
2328
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
2329
|
+
if (expanded.split(/[/\\]/).includes("..")) {
|
|
2330
|
+
return null;
|
|
2331
|
+
}
|
|
2332
|
+
if (!isAbsolute(expanded)) {
|
|
2333
|
+
return null;
|
|
2334
|
+
}
|
|
2335
|
+
const candidate = resolve2(expanded);
|
|
2336
|
+
const name = basename(candidate);
|
|
2337
|
+
return name === "" || name === "." || name === ".." ? null : candidate;
|
|
2338
|
+
}
|
|
2339
|
+
async function resolveNearestExistingAncestor(directory) {
|
|
2340
|
+
const missingSegments = [];
|
|
2341
|
+
let current = directory;
|
|
2342
|
+
for (; ; ) {
|
|
2343
|
+
try {
|
|
2344
|
+
return { existingAncestor: await realpath(current), missingSegments };
|
|
2345
|
+
} catch (err) {
|
|
2346
|
+
const parent = dirname2(current);
|
|
2347
|
+
if (err.code !== "ENOENT" || parent === current) {
|
|
2348
|
+
throw err;
|
|
2349
|
+
}
|
|
2350
|
+
missingSegments.unshift(basename(current));
|
|
2351
|
+
current = parent;
|
|
2352
|
+
}
|
|
2353
|
+
}
|
|
2354
|
+
}
|
|
2355
|
+
async function findContainingAllowedDirectory(allowedDirectories, realTarget) {
|
|
2356
|
+
for (const directory of allowedDirectories) {
|
|
2357
|
+
if (!isAbsolute(directory)) {
|
|
2358
|
+
log("warn", "file_push_allowed_directory_skipped", { directory, reason: "not_absolute" });
|
|
2359
|
+
continue;
|
|
2360
|
+
}
|
|
2361
|
+
const realDirectory = await realpathCreatingIfMissing(directory);
|
|
2362
|
+
if (realDirectory !== null && contains(realDirectory, realTarget)) {
|
|
2363
|
+
return realDirectory;
|
|
2364
|
+
}
|
|
2365
|
+
}
|
|
2366
|
+
return null;
|
|
2367
|
+
}
|
|
2368
|
+
async function realpathCreatingIfMissing(directory) {
|
|
2369
|
+
try {
|
|
2370
|
+
return await realpath(directory);
|
|
2371
|
+
} catch (err) {
|
|
2372
|
+
if (err.code !== "ENOENT") {
|
|
2373
|
+
log("warn", "file_push_allowed_directory_skipped", {
|
|
2374
|
+
directory,
|
|
2375
|
+
reason: "unresolvable",
|
|
2376
|
+
...errorFields(err)
|
|
2377
|
+
});
|
|
2378
|
+
return null;
|
|
2379
|
+
}
|
|
2380
|
+
}
|
|
2381
|
+
try {
|
|
2382
|
+
await mkdir(directory, { recursive: true, mode: DIRECTORY_MODE });
|
|
2383
|
+
await chmod(directory, DIRECTORY_MODE);
|
|
2384
|
+
return await realpath(directory);
|
|
2385
|
+
} catch (err) {
|
|
2386
|
+
log("warn", "file_push_allowed_directory_skipped", {
|
|
2387
|
+
directory,
|
|
2388
|
+
reason: "create_failed",
|
|
2389
|
+
...errorFields(err)
|
|
2390
|
+
});
|
|
2391
|
+
return null;
|
|
2392
|
+
}
|
|
2393
|
+
}
|
|
2394
|
+
function contains(realDirectory, realTarget) {
|
|
2395
|
+
const rel = relative(realDirectory, realTarget);
|
|
2396
|
+
return rel !== "" && rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
|
|
2397
|
+
}
|
|
2398
|
+
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
2399
|
+
let current = existingAncestor;
|
|
2400
|
+
for (const segment of missingSegments) {
|
|
2401
|
+
current = join(current, segment);
|
|
2402
|
+
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
2403
|
+
await chmod(current, DIRECTORY_MODE);
|
|
2404
|
+
}
|
|
2405
|
+
}
|
|
2406
|
+
async function writeAtomically(realTarget, content) {
|
|
2407
|
+
const temporaryPath = join(dirname2(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
2408
|
+
let handle;
|
|
2409
|
+
try {
|
|
2410
|
+
handle = await open2(temporaryPath, "wx", FILE_MODE);
|
|
2411
|
+
await handle.writeFile(content);
|
|
2412
|
+
await handle.chmod(FILE_MODE);
|
|
2413
|
+
await handle.close();
|
|
2414
|
+
handle = void 0;
|
|
2415
|
+
await rename(temporaryPath, realTarget);
|
|
2416
|
+
} catch (err) {
|
|
2417
|
+
await discardTemporaryFile(temporaryPath, handle);
|
|
2418
|
+
throw err;
|
|
2419
|
+
}
|
|
2420
|
+
}
|
|
2421
|
+
async function discardTemporaryFile(temporaryPath, handle) {
|
|
2422
|
+
try {
|
|
2423
|
+
await handle?.close();
|
|
2424
|
+
} catch (err) {
|
|
2425
|
+
log("warn", "file_push_temp_close_failed", { path: temporaryPath, ...errorFields(err) });
|
|
2426
|
+
}
|
|
2427
|
+
try {
|
|
2428
|
+
await unlink(temporaryPath);
|
|
2429
|
+
} catch (err) {
|
|
2430
|
+
const errno = err.code;
|
|
2431
|
+
if (errno !== "ENOENT" && errno !== "ENOTDIR") {
|
|
2432
|
+
log("warn", "file_push_temp_cleanup_failed", { path: temporaryPath, ...errorFields(err) });
|
|
2433
|
+
}
|
|
2434
|
+
}
|
|
2435
|
+
}
|
|
2436
|
+
function refuse(code, message, fields) {
|
|
2437
|
+
log(code === "write_failed" ? "error" : "warn", "file_push_refused", { code, ...fields });
|
|
2438
|
+
return { ok: false, code, message };
|
|
2439
|
+
}
|
|
2440
|
+
|
|
2441
|
+
// src/lib/runner-file-sync.ts
|
|
2442
|
+
var MAX_ACK_ATTEMPTS = 5;
|
|
2443
|
+
async function syncPendingRunnerFiles(options) {
|
|
2444
|
+
const pending = await listPendingFiles(options);
|
|
2445
|
+
const pendingIds = new Set(pending.map((file) => file.id));
|
|
2446
|
+
for (const id of options.ackFailures.keys()) {
|
|
2447
|
+
if (!pendingIds.has(id)) options.ackFailures.delete(id);
|
|
2448
|
+
}
|
|
2449
|
+
if (pending.length === 0) return 0;
|
|
2450
|
+
options.log({
|
|
2451
|
+
level: "info",
|
|
2452
|
+
message: `Runner file sync: ${pending.length} file(s) queued for this runner`
|
|
2453
|
+
});
|
|
2454
|
+
let applied = 0;
|
|
2455
|
+
for (const file of pending) {
|
|
2456
|
+
if ((options.ackFailures.get(file.id) ?? 0) >= MAX_ACK_ATTEMPTS) continue;
|
|
2457
|
+
if (await applyOne(options, file)) applied += 1;
|
|
2458
|
+
}
|
|
2459
|
+
return applied;
|
|
2460
|
+
}
|
|
2461
|
+
async function listPendingFiles(options) {
|
|
2462
|
+
let res;
|
|
2463
|
+
try {
|
|
2464
|
+
res = await options.fetchImpl(`${options.apiUrl}/runners/${options.agentId}/files/pending`, {
|
|
2465
|
+
headers: { Authorization: options.getAuthHeader() }
|
|
2466
|
+
});
|
|
2467
|
+
} catch (err) {
|
|
2468
|
+
options.log({
|
|
2469
|
+
level: "warn",
|
|
2470
|
+
message: `Could not list pending runner files \u2014 retrying on the next drain: ${describe(err)}`
|
|
2471
|
+
});
|
|
2472
|
+
return [];
|
|
2473
|
+
}
|
|
2474
|
+
if (!res.ok) {
|
|
2475
|
+
options.log({
|
|
2476
|
+
level: res.status === 404 ? "debug" : "warn",
|
|
2477
|
+
message: `Listing pending runner files returned HTTP ${res.status} \u2014 retrying on the next drain`
|
|
2478
|
+
});
|
|
2479
|
+
return [];
|
|
2480
|
+
}
|
|
2481
|
+
let body;
|
|
2482
|
+
try {
|
|
2483
|
+
body = await res.json();
|
|
2484
|
+
} catch (err) {
|
|
2485
|
+
options.log({
|
|
2486
|
+
level: "warn",
|
|
2487
|
+
message: `Pending runner file list was not readable JSON \u2014 retrying on the next drain: ${describe(err)}`
|
|
2488
|
+
});
|
|
2489
|
+
return [];
|
|
2490
|
+
}
|
|
2491
|
+
if (!Array.isArray(body)) {
|
|
2492
|
+
options.log({
|
|
2493
|
+
level: "warn",
|
|
2494
|
+
message: "Pending runner file list was not an array \u2014 ignoring it for this drain"
|
|
2495
|
+
});
|
|
2496
|
+
return [];
|
|
2497
|
+
}
|
|
2498
|
+
const files = [];
|
|
2499
|
+
for (const entry of body) {
|
|
2500
|
+
const file = asPendingFile(entry);
|
|
2501
|
+
if (file === null) {
|
|
2502
|
+
options.log({
|
|
2503
|
+
level: "warn",
|
|
2504
|
+
message: "Ignoring a malformed pending runner file entry (expected id, path and size)"
|
|
2505
|
+
});
|
|
2506
|
+
continue;
|
|
2507
|
+
}
|
|
2508
|
+
files.push(file);
|
|
2509
|
+
}
|
|
2510
|
+
return files;
|
|
2511
|
+
}
|
|
2512
|
+
function asPendingFile(entry) {
|
|
2513
|
+
if (entry === null || typeof entry !== "object") return null;
|
|
2514
|
+
const { id, path, size } = entry;
|
|
2515
|
+
if (typeof id !== "string" || id === "") return null;
|
|
2516
|
+
if (typeof path !== "string" || path === "") return null;
|
|
2517
|
+
if (typeof size !== "number" || !Number.isFinite(size) || size < 0) return null;
|
|
2518
|
+
return { id, path, size };
|
|
2519
|
+
}
|
|
2520
|
+
async function applyOne(options, file) {
|
|
2521
|
+
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
2522
|
+
if (options.allowedDirectories.length === 0) {
|
|
2523
|
+
options.log({
|
|
2524
|
+
level: "warn",
|
|
2525
|
+
message: `Runner file ${label} rejected: file sync is not enabled on this runner (start it with --enable-file-sync-to)`
|
|
2526
|
+
});
|
|
2527
|
+
await ack(options, file, "rejected", "file_sync_disabled");
|
|
2528
|
+
return false;
|
|
2529
|
+
}
|
|
2530
|
+
if (file.size > MAX_FILE_PUSH_BYTES) {
|
|
2531
|
+
options.log({
|
|
2532
|
+
level: "warn",
|
|
2533
|
+
message: `Runner file ${label} rejected: declared ${file.size} bytes, the limit is ${MAX_FILE_PUSH_BYTES}`
|
|
2534
|
+
});
|
|
2535
|
+
await ack(options, file, "rejected", "file_too_large");
|
|
2536
|
+
return false;
|
|
2537
|
+
}
|
|
2538
|
+
const download = await downloadContent(options, file, label);
|
|
2539
|
+
if (!download.ok) {
|
|
2540
|
+
if (download.terminal) await ack(options, file, "rejected", download.code);
|
|
2541
|
+
return false;
|
|
2542
|
+
}
|
|
2543
|
+
let outcome;
|
|
2544
|
+
try {
|
|
2545
|
+
outcome = await writePushedFile({
|
|
2546
|
+
requestedPath: file.path,
|
|
2547
|
+
content: download.content,
|
|
2548
|
+
allowedDirectories: options.allowedDirectories,
|
|
2549
|
+
homeDir: options.homeDir
|
|
2550
|
+
});
|
|
2551
|
+
} catch (err) {
|
|
2552
|
+
options.log({
|
|
2553
|
+
level: "error",
|
|
2554
|
+
message: `Runner file ${label} could not be written: ${describe(err)}`
|
|
2555
|
+
});
|
|
2556
|
+
await ack(options, file, "rejected", "write_failed");
|
|
2557
|
+
return false;
|
|
2558
|
+
}
|
|
2559
|
+
if (!outcome.ok) {
|
|
2560
|
+
options.log({
|
|
2561
|
+
level: "warn",
|
|
2562
|
+
message: `Runner file ${label} rejected (${outcome.code}): ${outcome.message}`
|
|
2563
|
+
});
|
|
2564
|
+
await ack(options, file, "rejected", outcome.code);
|
|
2565
|
+
return false;
|
|
2566
|
+
}
|
|
2567
|
+
options.log({
|
|
2568
|
+
level: "info",
|
|
2569
|
+
message: `Runner file ${label} applied (${download.content.byteLength} bytes)`
|
|
2570
|
+
});
|
|
2571
|
+
await ack(options, file, "applied");
|
|
2572
|
+
return true;
|
|
2573
|
+
}
|
|
2574
|
+
function durableDownloadCode(status) {
|
|
2575
|
+
return status === 413 ? "file_too_large" : "write_failed";
|
|
2576
|
+
}
|
|
2577
|
+
async function downloadContent(options, file, label) {
|
|
2578
|
+
try {
|
|
2579
|
+
const res = await options.fetchImpl(
|
|
2580
|
+
`${options.apiUrl}/runners/${options.agentId}/files/${file.id}/content`,
|
|
2581
|
+
{ headers: { Authorization: options.getAuthHeader() } }
|
|
2582
|
+
);
|
|
2583
|
+
if (!res.ok) {
|
|
2584
|
+
const terminal = res.status >= 400 && res.status < 500 && res.status !== 401 && res.status !== 403 && res.status !== 408 && res.status !== 429;
|
|
2585
|
+
if (!terminal) {
|
|
2586
|
+
options.log({
|
|
2587
|
+
level: "warn",
|
|
2588
|
+
message: `Downloading runner file ${label} returned HTTP ${res.status} \u2014 retrying on the next drain`
|
|
2589
|
+
});
|
|
2590
|
+
return { ok: false, terminal: false };
|
|
2591
|
+
}
|
|
2592
|
+
const code = durableDownloadCode(res.status);
|
|
2593
|
+
options.log({
|
|
2594
|
+
level: "error",
|
|
2595
|
+
message: `Downloading runner file ${label} returned HTTP ${res.status} \u2014 rejecting it as ${code} (the bytes never reached the writer)`
|
|
2596
|
+
});
|
|
2597
|
+
return { ok: false, terminal: true, code };
|
|
2598
|
+
}
|
|
2599
|
+
return { ok: true, content: Buffer.from(await res.arrayBuffer()) };
|
|
2600
|
+
} catch (err) {
|
|
2601
|
+
options.log({
|
|
2602
|
+
level: "warn",
|
|
2603
|
+
message: `Downloading runner file ${label} failed \u2014 retrying on the next drain: ${describe(err)}`
|
|
2604
|
+
});
|
|
2605
|
+
return { ok: false, terminal: false };
|
|
2606
|
+
}
|
|
2607
|
+
}
|
|
2608
|
+
async function ack(options, file, status, reason) {
|
|
2609
|
+
const outcome = `${status}${reason ? ` (${reason})` : ""}`;
|
|
2610
|
+
try {
|
|
2611
|
+
const res = await options.fetchImpl(
|
|
2612
|
+
`${options.apiUrl}/runners/${options.agentId}/files/${file.id}/ack`,
|
|
2613
|
+
{
|
|
2614
|
+
method: "POST",
|
|
2615
|
+
headers: {
|
|
2616
|
+
Authorization: options.getAuthHeader(),
|
|
2617
|
+
"Content-Type": "application/json"
|
|
2618
|
+
},
|
|
2619
|
+
body: JSON.stringify(reason ? { status, reason } : { status })
|
|
2620
|
+
}
|
|
2621
|
+
);
|
|
2622
|
+
if (!res.ok) {
|
|
2623
|
+
recordAckFailure(
|
|
2624
|
+
options,
|
|
2625
|
+
file,
|
|
2626
|
+
`Acking runner file ${file.id.slice(0, 8)} as ${outcome} returned HTTP ${res.status}`
|
|
2627
|
+
);
|
|
2628
|
+
return;
|
|
2629
|
+
}
|
|
2630
|
+
options.ackFailures.delete(file.id);
|
|
2631
|
+
} catch (err) {
|
|
2632
|
+
recordAckFailure(
|
|
2633
|
+
options,
|
|
2634
|
+
file,
|
|
2635
|
+
`Acking runner file ${file.id.slice(0, 8)} as ${outcome} failed: ${describe(err)}`
|
|
2636
|
+
);
|
|
2637
|
+
}
|
|
2638
|
+
}
|
|
2639
|
+
function recordAckFailure(options, file, what) {
|
|
2640
|
+
const attempts = (options.ackFailures.get(file.id) ?? 0) + 1;
|
|
2641
|
+
options.ackFailures.set(file.id, attempts);
|
|
2642
|
+
options.log({
|
|
2643
|
+
level: "error",
|
|
2644
|
+
message: attempts >= MAX_ACK_ATTEMPTS ? `${what} \u2014 giving up after ${attempts} attempts. It stays pending until the server expires it; restart the runner to retry.` : `${what} \u2014 it stays pending until a later drain re-acks it (attempt ${attempts} of ${MAX_ACK_ATTEMPTS})`
|
|
2645
|
+
});
|
|
2646
|
+
}
|
|
2647
|
+
function describe(err) {
|
|
2648
|
+
return err instanceof Error ? err.message : String(err);
|
|
2649
|
+
}
|
|
2650
|
+
|
|
2036
2651
|
// src/lib/channels/driver.ts
|
|
2037
2652
|
function messageIdOf(m) {
|
|
2038
2653
|
if (!m || typeof m !== "object") return void 0;
|
|
@@ -2061,6 +2676,8 @@ var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
|
2061
2676
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
2062
2677
|
var HEARTBEAT_MS = 6e4;
|
|
2063
2678
|
var ABSOLUTE_MAX_PROCESSING_MS = 6 * 60 * 60 * 1e3;
|
|
2679
|
+
var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
|
|
2680
|
+
var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
|
|
2064
2681
|
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
2065
2682
|
var MAX_SUPERSEDED_CONVERSATIONS = 256;
|
|
2066
2683
|
var ChannelAuthError = class extends Error {
|
|
@@ -2099,6 +2716,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2099
2716
|
pausedMaxWaitMs;
|
|
2100
2717
|
stuckQueuedMs;
|
|
2101
2718
|
now;
|
|
2719
|
+
fileSyncDirectories;
|
|
2720
|
+
homeDir;
|
|
2102
2721
|
/** Cache of conversationId → opencode sessionId. */
|
|
2103
2722
|
sessions = /* @__PURE__ */ new Map();
|
|
2104
2723
|
/**
|
|
@@ -2251,6 +2870,24 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2251
2870
|
sessionTitles = /* @__PURE__ */ new Map();
|
|
2252
2871
|
/** Serialises drains so a reconnect during a drain doesn't double-process. */
|
|
2253
2872
|
draining = false;
|
|
2873
|
+
/**
|
|
2874
|
+
* Serialises runner-file syncs (#559) so the ~2s poll tick and a concurrent
|
|
2875
|
+
* drain ping don't download, write and ack the same file twice.
|
|
2876
|
+
*/
|
|
2877
|
+
syncingFiles = false;
|
|
2878
|
+
/**
|
|
2879
|
+
* Consecutive failed acks per pending file (#559). Lives on the driver so it
|
|
2880
|
+
* survives across drains — without it, a file whose ack keeps failing is
|
|
2881
|
+
* re-downloaded and re-written every ~2s until the server expires it.
|
|
2882
|
+
*/
|
|
2883
|
+
fileAckFailures = /* @__PURE__ */ new Map();
|
|
2884
|
+
/**
|
|
2885
|
+
* Monotonic count of files this runner has pulled and written (#559). Only
|
|
2886
|
+
* ever increases, so `run.ts` detects work by comparing it against the value
|
|
2887
|
+
* it saw on the previous cycle — including work that landed mid-sleep, the
|
|
2888
|
+
* same trick `lastProxiedActivityAt` uses.
|
|
2889
|
+
*/
|
|
2890
|
+
appliedFileCount = 0;
|
|
2254
2891
|
/**
|
|
2255
2892
|
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
2256
2893
|
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
@@ -2281,6 +2918,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2281
2918
|
this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
2282
2919
|
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
2283
2920
|
this.now = config2.now ?? (() => Date.now());
|
|
2921
|
+
this.fileSyncDirectories = config2.fileSyncDirectories ?? [];
|
|
2922
|
+
this.homeDir = config2.homeDir ?? homedir();
|
|
2284
2923
|
}
|
|
2285
2924
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
2286
2925
|
get opencodeBase() {
|
|
@@ -2308,6 +2947,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2308
2947
|
);
|
|
2309
2948
|
return run2;
|
|
2310
2949
|
}
|
|
2950
|
+
/**
|
|
2951
|
+
* Pull-and-apply any files Evident has queued for this runner (#559), riding
|
|
2952
|
+
* the EXISTING drain cycle — `run.ts` calls it from the same ~2s channel poll
|
|
2953
|
+
* and drain ping that call `drainPending()`. There is deliberately no channel,
|
|
2954
|
+
* control frame or poll loop of its own: worst-case latency is one poll tick.
|
|
2955
|
+
*
|
|
2956
|
+
* NEVER throws and never surfaces a `ChannelAuthError`: a file failure must not
|
|
2957
|
+
* cost a conversation turn. Failures are logged and either acked as a terminal
|
|
2958
|
+
* outcome or left pending for the next drain (see `runner-file-sync.ts`).
|
|
2959
|
+
*
|
|
2960
|
+
* Re-entrant calls are skipped (the poll tick and a drain ping can overlap).
|
|
2961
|
+
*
|
|
2962
|
+
* @returns the number of files written to disk.
|
|
2963
|
+
*/
|
|
2964
|
+
async syncPendingFiles() {
|
|
2965
|
+
if (this.stopped) return 0;
|
|
2966
|
+
if (this.syncingFiles) return 0;
|
|
2967
|
+
this.syncingFiles = true;
|
|
2968
|
+
try {
|
|
2969
|
+
const applied = await syncPendingRunnerFiles({
|
|
2970
|
+
agentId: this.agentId,
|
|
2971
|
+
apiUrl: this.apiUrl,
|
|
2972
|
+
getAuthHeader: this.getAuthHeader,
|
|
2973
|
+
fetchImpl: this.fetchImpl,
|
|
2974
|
+
allowedDirectories: this.fileSyncDirectories,
|
|
2975
|
+
homeDir: this.homeDir,
|
|
2976
|
+
ackFailures: this.fileAckFailures,
|
|
2977
|
+
log: this.log
|
|
2978
|
+
});
|
|
2979
|
+
this.appliedFileCount += applied;
|
|
2980
|
+
return applied;
|
|
2981
|
+
} catch (err) {
|
|
2982
|
+
this.log({
|
|
2983
|
+
level: "error",
|
|
2984
|
+
message: `Runner file sync failed unexpectedly (message processing is unaffected): ${err instanceof Error ? err.message : String(err)}`
|
|
2985
|
+
});
|
|
2986
|
+
return 0;
|
|
2987
|
+
} finally {
|
|
2988
|
+
this.syncingFiles = false;
|
|
2989
|
+
}
|
|
2990
|
+
}
|
|
2311
2991
|
async runDrain() {
|
|
2312
2992
|
let dispatched = 0;
|
|
2313
2993
|
try {
|
|
@@ -2341,6 +3021,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2341
3021
|
}
|
|
2342
3022
|
return false;
|
|
2343
3023
|
}
|
|
3024
|
+
/**
|
|
3025
|
+
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
3026
|
+
*
|
|
3027
|
+
* Pulling a file is real work that `drainPending()` knows nothing about, so
|
|
3028
|
+
* without this a near-idle runner counts a credential pull as an empty tick
|
|
3029
|
+
* and `--idle-timeout` can `process.exit` mid-pull — leaving a
|
|
3030
|
+
* `.evident-push-*.tmp` behind — or immediately after the write, before the
|
|
3031
|
+
* browser has run the authorize/callback that activates it (the user then sees
|
|
3032
|
+
* `saved_not_activated` for a runner that was fine).
|
|
3033
|
+
*
|
|
3034
|
+
* Two signals because one cannot cover both cases: `inFlight` is the pull
|
|
3035
|
+
* happening RIGHT NOW (it may outlive the tick that started it), and
|
|
3036
|
+
* `appliedFiles` is monotonic so a pull that started AND finished between two
|
|
3037
|
+
* idle checks still shows up as an advance.
|
|
3038
|
+
*
|
|
3039
|
+
* CALLER CONTRACT: sample `inFlight` BEFORE calling `syncPendingFiles()` for
|
|
3040
|
+
* the cycle. `syncPendingFiles` sets the flag synchronously, so a caller that
|
|
3041
|
+
* samples afterwards reads `true` every single cycle and can never idle out.
|
|
3042
|
+
*/
|
|
3043
|
+
fileSyncActivity() {
|
|
3044
|
+
return { appliedFiles: this.appliedFileCount, inFlight: this.syncingFiles };
|
|
3045
|
+
}
|
|
2344
3046
|
/**
|
|
2345
3047
|
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
2346
3048
|
* exactly those with a live (dispatched-but-not-done / paused) turn, i.e. a
|
|
@@ -2402,7 +3104,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2402
3104
|
await this.sleep(step);
|
|
2403
3105
|
}
|
|
2404
3106
|
}
|
|
2405
|
-
while (this.hasInFlightWatchers()) {
|
|
3107
|
+
while (this.hasInFlightWatchers() || this.syncingFiles) {
|
|
2406
3108
|
if (this.now() >= deadline) return false;
|
|
2407
3109
|
await this.sleep(step);
|
|
2408
3110
|
}
|
|
@@ -2602,7 +3304,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2602
3304
|
const directory = await this.resolveOpenCodeDirectory();
|
|
2603
3305
|
const sessionId = await createOpenCodeSession(this.port, directory);
|
|
2604
3306
|
this.sessions.set(conversationId, sessionId);
|
|
2605
|
-
await this.persistSession(conversationId, sessionId).catch(() => {
|
|
3307
|
+
await this.persistSession(conversationId, sessionId).catch((err) => {
|
|
3308
|
+
this.log({
|
|
3309
|
+
level: "warn",
|
|
3310
|
+
message: `Persisting the OpenCode session binding ${sessionId.slice(0, 8)} for conversation ${conversationId.slice(0, 8)} failed (best-effort, not retried) \u2014 the completion PATCH also carries opencode_session_id, so the binding is repaired when the turn finishes: ${err instanceof Error ? err.message : String(err)}`,
|
|
3311
|
+
conversation_id: conversationId
|
|
3312
|
+
});
|
|
2606
3313
|
});
|
|
2607
3314
|
return sessionId;
|
|
2608
3315
|
}
|
|
@@ -2794,12 +3501,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2794
3501
|
stuckReported: false,
|
|
2795
3502
|
lastAliveAt: 0,
|
|
2796
3503
|
aliveInFlight: false,
|
|
3504
|
+
titleSynced: false,
|
|
3505
|
+
titleSyncInFlight: false,
|
|
2797
3506
|
awaitingHumanLatched: false,
|
|
2798
3507
|
pausedOnQuestion: false,
|
|
2799
3508
|
pausedOnPermission: false,
|
|
2800
3509
|
pausedClearConfirmed: false,
|
|
2801
3510
|
pausedInFlight: false,
|
|
2802
|
-
deliveryDeadlineAnchored: false
|
|
3511
|
+
deliveryDeadlineAnchored: false,
|
|
3512
|
+
b2PinnedSinceMs: 0,
|
|
3513
|
+
b2LastDescendantCheckMs: 0,
|
|
3514
|
+
b2AbandonedSignalled: false
|
|
2803
3515
|
});
|
|
2804
3516
|
}
|
|
2805
3517
|
/**
|
|
@@ -2867,12 +3579,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2867
3579
|
// with no extra `re_adopted` signal needed (folds old WI-6).
|
|
2868
3580
|
lastAliveAt: 0,
|
|
2869
3581
|
aliveInFlight: false,
|
|
3582
|
+
titleSynced: false,
|
|
3583
|
+
titleSyncInFlight: false,
|
|
2870
3584
|
awaitingHumanLatched: false,
|
|
2871
3585
|
pausedOnQuestion: false,
|
|
2872
3586
|
pausedOnPermission: false,
|
|
2873
3587
|
pausedClearConfirmed: false,
|
|
2874
3588
|
pausedInFlight: false,
|
|
2875
|
-
deliveryDeadlineAnchored: false
|
|
3589
|
+
deliveryDeadlineAnchored: false,
|
|
3590
|
+
b2PinnedSinceMs: 0,
|
|
3591
|
+
b2LastDescendantCheckMs: 0,
|
|
3592
|
+
b2AbandonedSignalled: false
|
|
2876
3593
|
});
|
|
2877
3594
|
}
|
|
2878
3595
|
/**
|
|
@@ -3034,58 +3751,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3034
3751
|
}
|
|
3035
3752
|
}
|
|
3036
3753
|
if (state === "done") {
|
|
3037
|
-
this.
|
|
3038
|
-
if (!inFlight.done) {
|
|
3039
|
-
this.log({
|
|
3040
|
-
level: "info",
|
|
3041
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed \u2014 marking done`,
|
|
3042
|
-
conversation_id: conv.id,
|
|
3043
|
-
message_id: inFlight.evidentMessageId
|
|
3044
|
-
});
|
|
3045
|
-
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
3046
|
-
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
3047
|
-
try {
|
|
3048
|
-
await this.markDone(
|
|
3049
|
-
conv.id,
|
|
3050
|
-
inFlight.evidentMessageId,
|
|
3051
|
-
sessionId,
|
|
3052
|
-
inFlight.opencodeMessageId,
|
|
3053
|
-
title,
|
|
3054
|
-
usage
|
|
3055
|
-
);
|
|
3056
|
-
} catch (err) {
|
|
3057
|
-
if (err instanceof ChannelAuthError) throw err;
|
|
3058
|
-
if (err instanceof ChannelTerminalError) {
|
|
3059
|
-
this.log({
|
|
3060
|
-
level: "warn",
|
|
3061
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
3062
|
-
conversation_id: conv.id,
|
|
3063
|
-
message_id: inFlight.evidentMessageId
|
|
3064
|
-
});
|
|
3065
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3066
|
-
return;
|
|
3067
|
-
}
|
|
3068
|
-
if (this.now() >= inFlight.deadline) {
|
|
3069
|
-
this.log({
|
|
3070
|
-
level: "warn",
|
|
3071
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
3072
|
-
conversation_id: conv.id,
|
|
3073
|
-
message_id: inFlight.evidentMessageId
|
|
3074
|
-
});
|
|
3075
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3076
|
-
return;
|
|
3077
|
-
}
|
|
3078
|
-
this.log({
|
|
3079
|
-
level: "warn",
|
|
3080
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
3081
|
-
conversation_id: conv.id,
|
|
3082
|
-
message_id: inFlight.evidentMessageId
|
|
3083
|
-
});
|
|
3084
|
-
return;
|
|
3085
|
-
}
|
|
3086
|
-
inFlight.done = true;
|
|
3087
|
-
}
|
|
3088
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3754
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
3089
3755
|
return;
|
|
3090
3756
|
}
|
|
3091
3757
|
if (state === "failed") {
|
|
@@ -3099,8 +3765,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3099
3765
|
message_id: inFlight.evidentMessageId
|
|
3100
3766
|
});
|
|
3101
3767
|
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
3768
|
+
const failure = await this.classifyModelAuthFailure(messages, inFlight.opencodeMessageId);
|
|
3102
3769
|
try {
|
|
3103
|
-
await this.markFailed(
|
|
3770
|
+
await this.markFailed(
|
|
3771
|
+
conv.id,
|
|
3772
|
+
inFlight.evidentMessageId,
|
|
3773
|
+
sessionId,
|
|
3774
|
+
error2,
|
|
3775
|
+
usage,
|
|
3776
|
+
failure
|
|
3777
|
+
);
|
|
3104
3778
|
} catch (err) {
|
|
3105
3779
|
if (err instanceof ChannelAuthError) throw err;
|
|
3106
3780
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -3145,6 +3819,44 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3145
3819
|
});
|
|
3146
3820
|
}
|
|
3147
3821
|
const activelyRunning = state === "running" && !awaitingHuman;
|
|
3822
|
+
const pinnedNow = activelyRunning && isPreamblePinnedRunning(messages, inFlight.opencodeMessageId);
|
|
3823
|
+
const snapshotReadable = messages != null && messages.length > 0;
|
|
3824
|
+
if (!pinnedNow) {
|
|
3825
|
+
if (snapshotReadable) {
|
|
3826
|
+
inFlight.b2PinnedSinceMs = 0;
|
|
3827
|
+
inFlight.b2LastDescendantCheckMs = 0;
|
|
3828
|
+
inFlight.b2AbandonedSignalled = false;
|
|
3829
|
+
}
|
|
3830
|
+
} else {
|
|
3831
|
+
if (inFlight.b2AbandonedSignalled) {
|
|
3832
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
3833
|
+
return;
|
|
3834
|
+
}
|
|
3835
|
+
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
3836
|
+
const pinnedForMs = this.now() - inFlight.b2PinnedSinceMs;
|
|
3837
|
+
if (pinnedForMs >= B2_ABANDONMENT_MIN_PINNED_MS && this.now() - inFlight.b2LastDescendantCheckMs >= B2_ABANDONMENT_RECHECK_MS) {
|
|
3838
|
+
inFlight.b2LastDescendantCheckMs = this.now();
|
|
3839
|
+
const descendantOngoing = await this.isAnyDescendantSessionOngoing(sessionId);
|
|
3840
|
+
if (isB2AbandonmentConfirmed({
|
|
3841
|
+
pinnedForMs,
|
|
3842
|
+
minPinnedMs: B2_ABANDONMENT_MIN_PINNED_MS,
|
|
3843
|
+
descendantOngoing
|
|
3844
|
+
})) {
|
|
3845
|
+
inFlight.b2AbandonedSignalled = true;
|
|
3846
|
+
this.log({
|
|
3847
|
+
level: "warn",
|
|
3848
|
+
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
3849
|
+
conversation_id: conv.id,
|
|
3850
|
+
message_id: id
|
|
3851
|
+
});
|
|
3852
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
3853
|
+
watched_for_ms: pinnedForMs
|
|
3854
|
+
});
|
|
3855
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
3856
|
+
return;
|
|
3857
|
+
}
|
|
3858
|
+
}
|
|
3859
|
+
}
|
|
3148
3860
|
if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
|
|
3149
3861
|
this.log({
|
|
3150
3862
|
level: "warn",
|
|
@@ -3164,6 +3876,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3164
3876
|
inFlight.aliveInFlight = false;
|
|
3165
3877
|
if (ok) inFlight.lastAliveAt = this.now();
|
|
3166
3878
|
});
|
|
3879
|
+
if (!inFlight.titleSynced && !inFlight.titleSyncInFlight) {
|
|
3880
|
+
inFlight.titleSyncInFlight = true;
|
|
3881
|
+
void this.resolveSessionTitle(sessionId, conv.id).then(async (title) => {
|
|
3882
|
+
if (!title) {
|
|
3883
|
+
inFlight.titleSyncInFlight = false;
|
|
3884
|
+
return;
|
|
3885
|
+
}
|
|
3886
|
+
const ok = await this.patchConversationTitle(conv.id, title);
|
|
3887
|
+
inFlight.titleSyncInFlight = false;
|
|
3888
|
+
if (ok) inFlight.titleSynced = true;
|
|
3889
|
+
});
|
|
3890
|
+
}
|
|
3167
3891
|
}
|
|
3168
3892
|
if (awaitingHuman) {
|
|
3169
3893
|
if (!inFlight.awaitingHumanLatched) {
|
|
@@ -3201,6 +3925,70 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3201
3925
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3202
3926
|
}
|
|
3203
3927
|
}
|
|
3928
|
+
/**
|
|
3929
|
+
* Settle a message whose run-state has resolved `'done'` — extracted verbatim
|
|
3930
|
+
* (pure refactor, no behavior change) from `serviceInFlightMessage`'s former
|
|
3931
|
+
* inline `state === 'done'` branch body, so a SECOND caller (the #721
|
|
3932
|
+
* b2-abandonment resolution) can reach the exact same completion behavior
|
|
3933
|
+
* (delivery-deadline anchoring, title resolution, usage extraction, and
|
|
3934
|
+
* `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
|
|
3935
|
+
* and risking the two copies silently drifting apart.
|
|
3936
|
+
*/
|
|
3937
|
+
async settleMessageDone(sessionId, watcher, inFlight, messages) {
|
|
3938
|
+
const conv = watcher.conv;
|
|
3939
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
3940
|
+
if (!inFlight.done) {
|
|
3941
|
+
this.log({
|
|
3942
|
+
level: "info",
|
|
3943
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed \u2014 marking done`,
|
|
3944
|
+
conversation_id: conv.id,
|
|
3945
|
+
message_id: inFlight.evidentMessageId
|
|
3946
|
+
});
|
|
3947
|
+
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
3948
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
3949
|
+
try {
|
|
3950
|
+
await this.markDone(
|
|
3951
|
+
conv.id,
|
|
3952
|
+
inFlight.evidentMessageId,
|
|
3953
|
+
sessionId,
|
|
3954
|
+
inFlight.opencodeMessageId,
|
|
3955
|
+
title,
|
|
3956
|
+
usage
|
|
3957
|
+
);
|
|
3958
|
+
} catch (err) {
|
|
3959
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
3960
|
+
if (err instanceof ChannelTerminalError) {
|
|
3961
|
+
this.log({
|
|
3962
|
+
level: "warn",
|
|
3963
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
3964
|
+
conversation_id: conv.id,
|
|
3965
|
+
message_id: inFlight.evidentMessageId
|
|
3966
|
+
});
|
|
3967
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3968
|
+
return;
|
|
3969
|
+
}
|
|
3970
|
+
if (this.now() >= inFlight.deadline) {
|
|
3971
|
+
this.log({
|
|
3972
|
+
level: "warn",
|
|
3973
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
3974
|
+
conversation_id: conv.id,
|
|
3975
|
+
message_id: inFlight.evidentMessageId
|
|
3976
|
+
});
|
|
3977
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3978
|
+
return;
|
|
3979
|
+
}
|
|
3980
|
+
this.log({
|
|
3981
|
+
level: "warn",
|
|
3982
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
3983
|
+
conversation_id: conv.id,
|
|
3984
|
+
message_id: inFlight.evidentMessageId
|
|
3985
|
+
});
|
|
3986
|
+
return;
|
|
3987
|
+
}
|
|
3988
|
+
inFlight.done = true;
|
|
3989
|
+
}
|
|
3990
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3991
|
+
}
|
|
3204
3992
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
3205
3993
|
/**
|
|
3206
3994
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -3367,6 +4155,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3367
4155
|
if (state === "failed") {
|
|
3368
4156
|
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
3369
4157
|
const usage = messageUsage(messages, ocId ?? "");
|
|
4158
|
+
const failure = await this.classifyModelAuthFailure(messages, ocId ?? "");
|
|
3370
4159
|
this.log({
|
|
3371
4160
|
level: "error",
|
|
3372
4161
|
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -3374,7 +4163,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3374
4163
|
message_id: row.id
|
|
3375
4164
|
});
|
|
3376
4165
|
try {
|
|
3377
|
-
await this.markFailed(row.conversation_id, row.id, sessionId, error2, usage);
|
|
4166
|
+
await this.markFailed(row.conversation_id, row.id, sessionId, error2, usage, failure);
|
|
3378
4167
|
} catch (err) {
|
|
3379
4168
|
if (err instanceof ChannelAuthError) throw err;
|
|
3380
4169
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -3780,6 +4569,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3780
4569
|
}
|
|
3781
4570
|
return false;
|
|
3782
4571
|
}
|
|
4572
|
+
/**
|
|
4573
|
+
* Tri-state variant of the upward parentID membership walk (#721), used ONLY
|
|
4574
|
+
* by `isAnyDescendantSessionOngoing`. Walks the SAME cached
|
|
4575
|
+
* `resolveSessionParent` chain `sessionBelongsTo` uses above, but — unlike
|
|
4576
|
+
* `sessionBelongsTo`, which deliberately collapses "confirmed not a
|
|
4577
|
+
* descendant" and "the walk's fetch failed" into the same `false` (safe for
|
|
4578
|
+
* its OTHER callers: interaction attribution and the recovery-path
|
|
4579
|
+
* `isAnyDescendantSessionAlive`, both of which just retry next tick with no
|
|
4580
|
+
* safety consequence either way) — this variant keeps those two outcomes
|
|
4581
|
+
* SEPARATE, because `isAnyDescendantSessionOngoing`'s caller
|
|
4582
|
+
* (`isB2AbandonmentConfirmed`) must never treat "couldn't tell" as "confirmed
|
|
4583
|
+
* not ongoing".
|
|
4584
|
+
*
|
|
4585
|
+
* Return contract:
|
|
4586
|
+
* - `true` → the walk reached `rootSessionId` — `sessionId` IS a descendant.
|
|
4587
|
+
* - `false` → the walk reached a definitive, parent-less root session
|
|
4588
|
+
* WITHOUT ever matching `rootSessionId` — `sessionId` is
|
|
4589
|
+
* CONFIRMED NOT a descendant of it.
|
|
4590
|
+
* - `null` → INDETERMINATE: a `GET /session/:id` fetch failed partway
|
|
4591
|
+
* through the walk (`resolveSessionParent` returned `undefined`),
|
|
4592
|
+
* or the depth cap (32) was hit without a definitive answer (a
|
|
4593
|
+
* pathological/cyclic chain proves nothing either way). NEVER
|
|
4594
|
+
* treat this the same as `false` — see `sessionBelongsTo`'s own
|
|
4595
|
+
* doc comment above for why that collapse is safe THERE but not
|
|
4596
|
+
* here.
|
|
4597
|
+
*
|
|
4598
|
+
* `sessionBelongsTo` itself is UNCHANGED — this is an additive helper scoped
|
|
4599
|
+
* to the live-path descendant check, not a modification of shared code used
|
|
4600
|
+
* by interaction attribution or the recovery path.
|
|
4601
|
+
*/
|
|
4602
|
+
async resolveSessionMembership(sessionId, rootSessionId) {
|
|
4603
|
+
let current = sessionId;
|
|
4604
|
+
for (let depth = 0; current && depth < 32; depth++) {
|
|
4605
|
+
if (current === rootSessionId) return true;
|
|
4606
|
+
const parent = await this.resolveSessionParent(current);
|
|
4607
|
+
if (parent === void 0) return null;
|
|
4608
|
+
if (parent === null) return false;
|
|
4609
|
+
current = parent;
|
|
4610
|
+
}
|
|
4611
|
+
return null;
|
|
4612
|
+
}
|
|
3783
4613
|
/**
|
|
3784
4614
|
* Resolve (and cache) a session's `parentID` via `GET /session/:id`. Returns
|
|
3785
4615
|
* `null` for a root session (no parent) and `undefined` when opencode is
|
|
@@ -3864,6 +4694,54 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3864
4694
|
}
|
|
3865
4695
|
return null;
|
|
3866
4696
|
}
|
|
4697
|
+
/**
|
|
4698
|
+
* Best-effort mid-turn title sync (#711 follow-up): PATCH a resolved OpenCode
|
|
4699
|
+
* session title onto the conversation via the PLAIN conversation-update
|
|
4700
|
+
* endpoint (`PATCH /runners/:agentId/conversations/:conversationId`) — NOT the
|
|
4701
|
+
* message-status endpoint `markProcessing`/`markDone` use. Deliberately a
|
|
4702
|
+
* separate, lighter call: it carries no `status`, so it cannot re-trigger the
|
|
4703
|
+
* `processing`/`done` transition side effects (Slack notices, activity-log
|
|
4704
|
+
* rows, delivery jobs) those PATCHes gate on `transitioned` — this call only
|
|
4705
|
+
* ever touches `conversations.title`. That route (`routes/conversations.ts`)
|
|
4706
|
+
* skips a title write matching the stored value, so a redundant call with the
|
|
4707
|
+
* same title is a real no-op — it does not bump `updated_at`, which the
|
|
4708
|
+
* conversation list sorts and paginates on. (Note this is a DIFFERENT guard
|
|
4709
|
+
* from `threads.ts`'s "non-empty AND changed" one, which only covers the
|
|
4710
|
+
* message-status PATCH; the non-empty half is enforced here instead, by
|
|
4711
|
+
* `resolveSessionTitle` never returning an empty/placeholder title.)
|
|
4712
|
+
*
|
|
4713
|
+
* Telemetry-only / never blocks the caller, mirroring `postSignal`: a failure
|
|
4714
|
+
* is logged and the title is simply retried on the next heartbeat tick (the
|
|
4715
|
+
* caller only latches `titleSynced` on `true`).
|
|
4716
|
+
*/
|
|
4717
|
+
async patchConversationTitle(conversationId, title) {
|
|
4718
|
+
try {
|
|
4719
|
+
const res = await this.fetchImpl(
|
|
4720
|
+
`${this.apiUrl}/runners/${this.agentId}/conversations/${conversationId}`,
|
|
4721
|
+
{
|
|
4722
|
+
method: "PATCH",
|
|
4723
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
4724
|
+
body: JSON.stringify({ title })
|
|
4725
|
+
}
|
|
4726
|
+
);
|
|
4727
|
+
if (!res.ok) {
|
|
4728
|
+
this.log({
|
|
4729
|
+
level: "debug",
|
|
4730
|
+
message: `Mid-turn title sync PATCH for conversation ${conversationId.slice(0, 8)} returned HTTP ${res.status} (best-effort, will retry next heartbeat)`,
|
|
4731
|
+
conversation_id: conversationId
|
|
4732
|
+
});
|
|
4733
|
+
return false;
|
|
4734
|
+
}
|
|
4735
|
+
return true;
|
|
4736
|
+
} catch (err) {
|
|
4737
|
+
this.log({
|
|
4738
|
+
level: "debug",
|
|
4739
|
+
message: `Best-effort mid-turn title sync PATCH failed for conversation ${conversationId.slice(0, 8)} (will retry next heartbeat): ${err instanceof Error ? err.message : String(err)}`,
|
|
4740
|
+
conversation_id: conversationId
|
|
4741
|
+
});
|
|
4742
|
+
return false;
|
|
4743
|
+
}
|
|
4744
|
+
}
|
|
3867
4745
|
/**
|
|
3868
4746
|
* DEFENSIVE cross-check for the restart-recovery path (WI-2): is any descendant
|
|
3869
4747
|
* (`task` sub-agent) session under `rootSessionId` still genuinely doing work?
|
|
@@ -3922,6 +4800,84 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3922
4800
|
}
|
|
3923
4801
|
return false;
|
|
3924
4802
|
}
|
|
4803
|
+
/**
|
|
4804
|
+
* LIVE-PATH descendant-liveness check (#721): is any descendant (`task`
|
|
4805
|
+
* sub-agent) session under `rootSessionId` currently ONGOING per OpenCode's own
|
|
4806
|
+
* in-memory status map (`isSessionOngoing` — `busy`/`retry`)?
|
|
4807
|
+
*
|
|
4808
|
+
* Deliberately NOT `isAnyDescendantSessionAlive` (the RECOVERY-path
|
|
4809
|
+
* cross-check above): that method judges liveness from the child's OWN
|
|
4810
|
+
* TRANSCRIPT (`isSessionActivelyGenerating`), which is the right (only) option
|
|
4811
|
+
* on the recovery path because a restart WIPES `SessionStatus`. On the LIVE
|
|
4812
|
+
* path the local opencode server IS running, so its in-memory status map is
|
|
4813
|
+
* live and authoritative — and per ADR-0047 §4a ("the child has its own entry
|
|
4814
|
+
* [in the map]"), a `task` descendant's OWN busy/retry entry reflects its
|
|
4815
|
+
* ENTIRE turn (including any tool call it is itself executing), not a
|
|
4816
|
+
* per-message transcript snapshot. This sidesteps the "child's own tool is
|
|
4817
|
+
* executing, between its step's completion and the next generation step"
|
|
4818
|
+
* transcript gap that a transcript-based check would need a second,
|
|
4819
|
+
* sustained-window bound to guard against — it is simply not derived from
|
|
4820
|
+
* message timestamps at all.
|
|
4821
|
+
*
|
|
4822
|
+
* Why not just check `isSessionOngoing(port, rootSessionId)` (the ROOT's own
|
|
4823
|
+
* status, as the recovery path does per §4a)? Because on the LIVE path the
|
|
4824
|
+
* root session can be shared: a SECOND, unrelated user message can land on the
|
|
4825
|
+
* SAME session (issue #721's own root cause) and keep the root `busy` for a
|
|
4826
|
+
* reason that has nothing to do with THIS message's delegation. A `task`
|
|
4827
|
+
* descendant session is spawned for exactly one delegated turn and never
|
|
4828
|
+
* reused, so its OWN status-map entry is unambiguous evidence about that one
|
|
4829
|
+
* delegation — which the root's status is not.
|
|
4830
|
+
*
|
|
4831
|
+
* Why membership is checked via `resolveSessionMembership`, NOT
|
|
4832
|
+
* `sessionBelongsTo`: `sessionBelongsTo` collapses a transient
|
|
4833
|
+
* `GET /session/:id` fetch failure into "not a descendant", which would
|
|
4834
|
+
* silently drop a genuinely-live candidate from consideration on the one
|
|
4835
|
+
* unlucky tick its membership-walk fetch hiccups (#721).
|
|
4836
|
+
* `resolveSessionMembership` keeps that failure mode as a distinct `null`
|
|
4837
|
+
* (indeterminate) so it is folded into THIS method's own `indeterminate` flag
|
|
4838
|
+
* instead.
|
|
4839
|
+
*
|
|
4840
|
+
* Return contract (note the DIFFERENT judge vs. `isAnyDescendantSessionAlive`):
|
|
4841
|
+
* - `true` → some descendant session is `busy`/`retry` (genuinely ongoing).
|
|
4842
|
+
* - `false` → enumeration succeeded, EVERY candidate's MEMBERSHIP was
|
|
4843
|
+
* confirmed either way (`resolveSessionMembership` never
|
|
4844
|
+
* returned `null`), and every CONFIRMED descendant's status read
|
|
4845
|
+
* succeeded and is not ongoing (includes "no descendant session
|
|
4846
|
+
* exists at all" — e.g. a plain, non-`task` tool call).
|
|
4847
|
+
* - `null` → INDETERMINATE: `listSessions` failed, OR at least one
|
|
4848
|
+
* candidate's MEMBERSHIP could not be confirmed
|
|
4849
|
+
* (`resolveSessionMembership` returned `null` — a fetch failure
|
|
4850
|
+
* or pathological chain partway through the parent walk), OR at
|
|
4851
|
+
* least one CONFIRMED descendant's `isSessionOngoing` read
|
|
4852
|
+
* failed — and no OTHER candidate was already confirmed `true`.
|
|
4853
|
+
* The caller MUST NOT treat `null` the same as `false` here
|
|
4854
|
+
* (unlike the recovery cross-check's contract) — see
|
|
4855
|
+
* `isB2AbandonmentConfirmed`.
|
|
4856
|
+
*/
|
|
4857
|
+
async isAnyDescendantSessionOngoing(rootSessionId) {
|
|
4858
|
+
const sessions = await listSessions(this.port);
|
|
4859
|
+
if (!sessions) {
|
|
4860
|
+
this.log({
|
|
4861
|
+
level: "warn",
|
|
4862
|
+
message: `Could not enumerate sessions to check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating descendant liveness as indeterminate`
|
|
4863
|
+
});
|
|
4864
|
+
return null;
|
|
4865
|
+
}
|
|
4866
|
+
let indeterminate = false;
|
|
4867
|
+
for (const candidate of sessions) {
|
|
4868
|
+
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
4869
|
+
const membership = await this.resolveSessionMembership(candidate.id, rootSessionId);
|
|
4870
|
+
if (membership === null) {
|
|
4871
|
+
indeterminate = true;
|
|
4872
|
+
continue;
|
|
4873
|
+
}
|
|
4874
|
+
if (membership === false) continue;
|
|
4875
|
+
const ongoing = await isSessionOngoing(this.port, candidate.id);
|
|
4876
|
+
if (ongoing === true) return true;
|
|
4877
|
+
if (ongoing === null) indeterminate = true;
|
|
4878
|
+
}
|
|
4879
|
+
return indeterminate ? null : false;
|
|
4880
|
+
}
|
|
3925
4881
|
/**
|
|
3926
4882
|
* Cheap decision-telemetry label for a running row's LAST correlated reply
|
|
3927
4883
|
* (WI-2 Task 2.2): which running SHAPE it is, for the status-gated recovery log.
|
|
@@ -4185,7 +5141,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4185
5141
|
* exists but is wedged, so the next attempt must get a fresh one
|
|
4186
5142
|
* instead of reusing it — see WI-1's server-side null-clearing PATCH).
|
|
4187
5143
|
*/
|
|
4188
|
-
async markFailed(conversationId, messageId, sessionId, error2, usage) {
|
|
5144
|
+
async markFailed(conversationId, messageId, sessionId, error2, usage, failure) {
|
|
4189
5145
|
const body = { status: "failed" };
|
|
4190
5146
|
if (sessionId === null) {
|
|
4191
5147
|
body.opencode_session_id = null;
|
|
@@ -4194,6 +5150,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4194
5150
|
}
|
|
4195
5151
|
if (error2 !== void 0) body.error = error2;
|
|
4196
5152
|
if (usage) Object.assign(body, usage);
|
|
5153
|
+
if (failure) {
|
|
5154
|
+
body.failure_kind = failure.kind;
|
|
5155
|
+
body.failure_provider_id = failure.providerId;
|
|
5156
|
+
body.failure_model_id = failure.modelId;
|
|
5157
|
+
body.failure_reason = failure.reason;
|
|
5158
|
+
}
|
|
4197
5159
|
await this.callWithRetry(
|
|
4198
5160
|
"marking message as failed",
|
|
4199
5161
|
() => this.fetchImpl(
|
|
@@ -4206,6 +5168,29 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4206
5168
|
)
|
|
4207
5169
|
);
|
|
4208
5170
|
}
|
|
5171
|
+
/**
|
|
5172
|
+
* Classify an errored turn as a model-auth failure (#736 P1-4), for `markFailed`.
|
|
5173
|
+
*
|
|
5174
|
+
* `messageFailure` alone (structured OpenCode error → `model_auth`) covers
|
|
5175
|
+
* most cases; when it returns `null` on this ALREADY-FAILED turn, fall back
|
|
5176
|
+
* to the P1-2b zero-provider check — one extra loopback call to
|
|
5177
|
+
* `hasAnyConfiguredProvider`, only reached when the structured classifier
|
|
5178
|
+
* couldn't place it. Fails open (never throws): a fallback probe failure
|
|
5179
|
+
* (`null`/indeterminate) leaves the classification `null`, which produces
|
|
5180
|
+
* today's byte-identical PATCH body via `markFailed`'s `if (failure)` guard.
|
|
5181
|
+
*/
|
|
5182
|
+
async classifyModelAuthFailure(messages, userMessageId) {
|
|
5183
|
+
const classified = messageFailure(messages, userMessageId);
|
|
5184
|
+
if (classified != null) return classified;
|
|
5185
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
5186
|
+
const hasProvider = await hasAnyConfiguredProvider(this.port);
|
|
5187
|
+
return applyZeroProviderFallback(
|
|
5188
|
+
classified,
|
|
5189
|
+
hasProvider,
|
|
5190
|
+
reply?.info?.providerID ?? null,
|
|
5191
|
+
reply?.info?.modelID ?? null
|
|
5192
|
+
);
|
|
5193
|
+
}
|
|
4209
5194
|
/**
|
|
4210
5195
|
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
4211
5196
|
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
@@ -4358,10 +5343,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4358
5343
|
import chalk5 from "chalk";
|
|
4359
5344
|
import ora2 from "ora";
|
|
4360
5345
|
import { select as select2 } from "@inquirer/prompts";
|
|
5346
|
+
var INTERACTIVE_START_TIMEOUT_MS = 3e4;
|
|
4361
5347
|
async function ensureOpenCodeRunning(ctx) {
|
|
4362
5348
|
const healthCheck = await checkOpenCodeHealth(ctx.port);
|
|
4363
5349
|
if (healthCheck.healthy) {
|
|
4364
|
-
return {
|
|
5350
|
+
return {
|
|
5351
|
+
port: ctx.port,
|
|
5352
|
+
process: null,
|
|
5353
|
+
version: healthCheck.version ?? null,
|
|
5354
|
+
notReadyReason: null
|
|
5355
|
+
};
|
|
4365
5356
|
}
|
|
4366
5357
|
const runningInstances = await findHealthyOpenCodeInstances();
|
|
4367
5358
|
if (runningInstances.length > 0) {
|
|
@@ -4382,7 +5373,7 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
4382
5373
|
console.log(chalk5.yellow("Tip: Run with the correct port:"));
|
|
4383
5374
|
console.log(
|
|
4384
5375
|
chalk5.dim(
|
|
4385
|
-
` ${getCliName()} run --
|
|
5376
|
+
` ${getCliName()} run --runner ${ctx.agentId} --port ${runningInstances[0].port}`
|
|
4386
5377
|
)
|
|
4387
5378
|
);
|
|
4388
5379
|
}
|
|
@@ -4402,14 +5393,22 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
4402
5393
|
if (!ctx.interactive) {
|
|
4403
5394
|
ctx.log(`OpenCode is not running on port ${ctx.port}. Starting it automatically...`);
|
|
4404
5395
|
const proc = await startOpenCode(ctx.port);
|
|
4405
|
-
const health = await waitForOpenCodeHealth(ctx.port,
|
|
5396
|
+
const health = await waitForOpenCodeHealth(ctx.port, ctx.startTimeoutMs);
|
|
4406
5397
|
if (!health.healthy) {
|
|
4407
|
-
|
|
4408
|
-
|
|
4409
|
-
|
|
5398
|
+
return {
|
|
5399
|
+
port: ctx.port,
|
|
5400
|
+
process: proc,
|
|
5401
|
+
version: null,
|
|
5402
|
+
notReadyReason: `it did not become healthy within ${Math.round(ctx.startTimeoutMs / 1e3)}s`
|
|
5403
|
+
};
|
|
4410
5404
|
}
|
|
4411
5405
|
ctx.log(`OpenCode started on port ${ctx.port}${health.version ? ` (v${health.version})` : ""}`);
|
|
4412
|
-
return {
|
|
5406
|
+
return {
|
|
5407
|
+
port: ctx.port,
|
|
5408
|
+
process: proc,
|
|
5409
|
+
version: health.version ?? null,
|
|
5410
|
+
notReadyReason: null
|
|
5411
|
+
};
|
|
4413
5412
|
}
|
|
4414
5413
|
let port = ctx.port;
|
|
4415
5414
|
if (isPortInUse(port)) {
|
|
@@ -4462,15 +5461,15 @@ Port ${port} is already in use.`));
|
|
|
4462
5461
|
if (action === "start") {
|
|
4463
5462
|
const spinner = ora2("Starting OpenCode...").start();
|
|
4464
5463
|
const proc = await startOpenCode(port);
|
|
4465
|
-
const health = await waitForOpenCodeHealth(port,
|
|
5464
|
+
const health = await waitForOpenCodeHealth(port, INTERACTIVE_START_TIMEOUT_MS);
|
|
4466
5465
|
if (!health.healthy) {
|
|
4467
5466
|
spinner.fail("Failed to start OpenCode");
|
|
4468
5467
|
throw new Error("OpenCode failed to start");
|
|
4469
5468
|
}
|
|
4470
5469
|
spinner.stop();
|
|
4471
|
-
return { port, process: proc, version: health.version ?? null };
|
|
5470
|
+
return { port, process: proc, version: health.version ?? null, notReadyReason: null };
|
|
4472
5471
|
}
|
|
4473
|
-
return { port, process: null, version: null };
|
|
5472
|
+
return { port, process: null, version: null, notReadyReason: "you chose to continue without it" };
|
|
4474
5473
|
}
|
|
4475
5474
|
|
|
4476
5475
|
// src/commands/agent-lookup.ts
|
|
@@ -4512,19 +5511,21 @@ async function resolveAgentIdFromKey(authHeader) {
|
|
|
4512
5511
|
return { agent_id: data.agent_id };
|
|
4513
5512
|
}
|
|
4514
5513
|
return {
|
|
4515
|
-
error: "Cannot resolve runner ID: auth type is not agent_key. Please provide --
|
|
5514
|
+
error: "Cannot resolve runner ID: auth type is not agent_key. Please provide --runner explicitly."
|
|
4516
5515
|
};
|
|
4517
5516
|
} catch (error2) {
|
|
4518
5517
|
const message = error2 instanceof Error ? error2.message : "Unknown error";
|
|
4519
5518
|
return { error: `Failed to resolve runner from key: ${message}` };
|
|
4520
5519
|
}
|
|
4521
5520
|
}
|
|
5521
|
+
var BEST_EFFORT_NOTIFY_TIMEOUT_MS = 2e3;
|
|
4522
5522
|
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
4523
5523
|
const apiUrl = getApiUrlConfig();
|
|
4524
5524
|
try {
|
|
4525
5525
|
const response = await fetch(`${apiUrl}/runners/${agentId}/disconnect`, {
|
|
4526
5526
|
method: "POST",
|
|
4527
|
-
headers: { Authorization: authHeader }
|
|
5527
|
+
headers: { Authorization: authHeader },
|
|
5528
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
4528
5529
|
});
|
|
4529
5530
|
if (!response.ok) {
|
|
4530
5531
|
const serverMessage = await readErrorMessage(response);
|
|
@@ -4535,7 +5536,35 @@ async function notifyAgentDisconnected(agentId, authHeader) {
|
|
|
4535
5536
|
}
|
|
4536
5537
|
return { ok: true };
|
|
4537
5538
|
} catch (error2) {
|
|
4538
|
-
return { ok: false, error:
|
|
5539
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
5540
|
+
}
|
|
5541
|
+
}
|
|
5542
|
+
function describeBestEffortError(error2) {
|
|
5543
|
+
const name = error2?.name;
|
|
5544
|
+
if (name === "TimeoutError" || name === "AbortError") {
|
|
5545
|
+
return `timed out after ${BEST_EFFORT_NOTIFY_TIMEOUT_MS}ms`;
|
|
5546
|
+
}
|
|
5547
|
+
return error2 instanceof Error ? error2.message : String(error2);
|
|
5548
|
+
}
|
|
5549
|
+
async function reportMicrovmId(agentId, authHeader, microvmId) {
|
|
5550
|
+
try {
|
|
5551
|
+
const apiUrl = getApiUrlConfig();
|
|
5552
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/microvm`, {
|
|
5553
|
+
method: "POST",
|
|
5554
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
5555
|
+
body: JSON.stringify({ microvm_id: microvmId }),
|
|
5556
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
5557
|
+
});
|
|
5558
|
+
if (!response.ok) {
|
|
5559
|
+
const serverMessage = await readErrorMessage(response);
|
|
5560
|
+
return {
|
|
5561
|
+
ok: false,
|
|
5562
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
5563
|
+
};
|
|
5564
|
+
}
|
|
5565
|
+
return { ok: true };
|
|
5566
|
+
} catch (error2) {
|
|
5567
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
4539
5568
|
}
|
|
4540
5569
|
}
|
|
4541
5570
|
async function getAgentInfo(agentId, authHeader) {
|
|
@@ -4585,6 +5614,7 @@ var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
|
4585
5614
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
4586
5615
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
4587
5616
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
5617
|
+
var TELEMETRY_SHUTDOWN_TIMEOUT_MS = 5e3;
|
|
4588
5618
|
function resolveLogLevel(options) {
|
|
4589
5619
|
const accepted = Object.keys(LOG_LEVELS);
|
|
4590
5620
|
const validate = (value, source) => {
|
|
@@ -4608,6 +5638,63 @@ function resolveLogLevel(options) {
|
|
|
4608
5638
|
}
|
|
4609
5639
|
return "info";
|
|
4610
5640
|
}
|
|
5641
|
+
function resolveFileSyncDirectories(raw, homeDir) {
|
|
5642
|
+
const directories = [];
|
|
5643
|
+
for (const entry of raw ?? []) {
|
|
5644
|
+
const trimmed = entry.trim();
|
|
5645
|
+
if (trimmed === "") {
|
|
5646
|
+
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
5647
|
+
}
|
|
5648
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join2(homeDir, trimmed.slice(2)) : trimmed;
|
|
5649
|
+
if (!isAbsolute2(expanded)) {
|
|
5650
|
+
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
5651
|
+
}
|
|
5652
|
+
const normalized = resolvePath(expanded);
|
|
5653
|
+
if (parse(normalized).root === normalized) {
|
|
5654
|
+
throw new Error(
|
|
5655
|
+
`--enable-file-sync-to will not allow-list the filesystem root ("${entry}"); name the specific directory the credentials belong in (for example ~/.claude)`
|
|
5656
|
+
);
|
|
5657
|
+
}
|
|
5658
|
+
if (!directories.includes(normalized)) {
|
|
5659
|
+
directories.push(normalized);
|
|
5660
|
+
}
|
|
5661
|
+
}
|
|
5662
|
+
if (directories.length > MAX_FILE_SYNC_DIRECTORIES) {
|
|
5663
|
+
throw new Error(
|
|
5664
|
+
`--enable-file-sync-to accepts at most ${MAX_FILE_SYNC_DIRECTORIES} directories; got ${directories.length}`
|
|
5665
|
+
);
|
|
5666
|
+
}
|
|
5667
|
+
return directories;
|
|
5668
|
+
}
|
|
5669
|
+
var DEFAULT_OPENCODE_START_TIMEOUT_SECONDS = 180;
|
|
5670
|
+
var MAX_OPENCODE_START_TIMEOUT_SECONDS = 3600;
|
|
5671
|
+
var OPENCODE_START_TIMEOUT_ENV = "EVIDENT_OPENCODE_START_TIMEOUT";
|
|
5672
|
+
function resolveOpenCodeStartTimeoutMs(options, env = process.env) {
|
|
5673
|
+
const defaultMs = DEFAULT_OPENCODE_START_TIMEOUT_SECONDS * 1e3;
|
|
5674
|
+
let raw;
|
|
5675
|
+
let source;
|
|
5676
|
+
if (options.opencodeStartTimeout !== void 0) {
|
|
5677
|
+
raw = options.opencodeStartTimeout;
|
|
5678
|
+
source = "--opencode-start-timeout";
|
|
5679
|
+
} else if (env[OPENCODE_START_TIMEOUT_ENV] !== void 0 && env[OPENCODE_START_TIMEOUT_ENV] !== "") {
|
|
5680
|
+
raw = env[OPENCODE_START_TIMEOUT_ENV];
|
|
5681
|
+
source = OPENCODE_START_TIMEOUT_ENV;
|
|
5682
|
+
} else {
|
|
5683
|
+
return { timeoutMs: defaultMs, warnings: [] };
|
|
5684
|
+
}
|
|
5685
|
+
const trimmed = raw.trim();
|
|
5686
|
+
const seconds = Number(trimmed);
|
|
5687
|
+
const isPositiveInteger = /^\d+$/.test(trimmed) && Number.isInteger(seconds) && seconds > 0;
|
|
5688
|
+
if (!isPositiveInteger || seconds > MAX_OPENCODE_START_TIMEOUT_SECONDS) {
|
|
5689
|
+
return {
|
|
5690
|
+
timeoutMs: defaultMs,
|
|
5691
|
+
warnings: [
|
|
5692
|
+
`Ignoring invalid ${source} "${raw}": expected a positive integer number of seconds (at most ${MAX_OPENCODE_START_TIMEOUT_SECONDS}); using the default ${DEFAULT_OPENCODE_START_TIMEOUT_SECONDS}s`
|
|
5693
|
+
]
|
|
5694
|
+
};
|
|
5695
|
+
}
|
|
5696
|
+
return { timeoutMs: seconds * 1e3, warnings: [] };
|
|
5697
|
+
}
|
|
4611
5698
|
function meetsThreshold(state, level) {
|
|
4612
5699
|
return LOG_LEVELS[level] >= LOG_LEVELS[state.logLevel];
|
|
4613
5700
|
}
|
|
@@ -4629,6 +5716,10 @@ function log2(state, message, level = "info") {
|
|
|
4629
5716
|
function logActivity(state, entry) {
|
|
4630
5717
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
4631
5718
|
if (!meetsThreshold(state, level)) return;
|
|
5719
|
+
forwardRunnerActivity(
|
|
5720
|
+
{ level, message: entry.message, error: entry.error },
|
|
5721
|
+
{ agentId: state.agentId, authHeader: state.authHeader }
|
|
5722
|
+
);
|
|
4632
5723
|
const fullEntry = {
|
|
4633
5724
|
...entry,
|
|
4634
5725
|
level,
|
|
@@ -4729,18 +5820,29 @@ async function handleAuthError(state, error2) {
|
|
|
4729
5820
|
async function driveChannels(state, driver) {
|
|
4730
5821
|
let idlePolls = 0;
|
|
4731
5822
|
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
5823
|
+
let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
4732
5824
|
while (state.running) {
|
|
4733
5825
|
if (state.connection?.reconnecting && state.connection.reconnectPromise) {
|
|
4734
5826
|
logActivity(state, { type: "info", message: "Waiting for tunnel reconnection..." });
|
|
4735
5827
|
if (state.interactive) displayStatus(state);
|
|
4736
5828
|
await state.connection.reconnectPromise;
|
|
4737
5829
|
}
|
|
5830
|
+
const carriedOverFileSync = driver.fileSyncActivity().inFlight;
|
|
5831
|
+
void driver.syncPendingFiles().catch(
|
|
5832
|
+
(error2) => logActivity(state, {
|
|
5833
|
+
type: "error",
|
|
5834
|
+
error: `Runner file sync failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
5835
|
+
})
|
|
5836
|
+
);
|
|
4738
5837
|
try {
|
|
4739
5838
|
const processed = await driver.drainPending();
|
|
4740
5839
|
state.messageCount += processed;
|
|
4741
5840
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
4742
5841
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
4743
|
-
|
|
5842
|
+
const appliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
5843
|
+
const fileActivity = carriedOverFileSync || appliedFiles !== lastSeenAppliedFiles;
|
|
5844
|
+
lastSeenAppliedFiles = appliedFiles;
|
|
5845
|
+
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
|
|
4744
5846
|
idlePolls = 0;
|
|
4745
5847
|
if (processed > 0 && state.interactive) displayStatus(state);
|
|
4746
5848
|
} else if (state.idleTimeout !== null) {
|
|
@@ -4769,7 +5871,7 @@ async function driveChannels(state, driver) {
|
|
|
4769
5871
|
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage}` });
|
|
4770
5872
|
if (state.interactive) displayStatus(state);
|
|
4771
5873
|
}
|
|
4772
|
-
await new Promise((
|
|
5874
|
+
await new Promise((resolve3) => setTimeout(resolve3, CHANNEL_POLL_INTERVAL_MS));
|
|
4773
5875
|
if (state.idleTimeout !== null && idlePolls >= 2) {
|
|
4774
5876
|
const idleMs = idlePolls * CHANNEL_POLL_INTERVAL_MS;
|
|
4775
5877
|
if (idleMs > state.idleTimeout * 1e3) {
|
|
@@ -4872,7 +5974,18 @@ async function notifyOffline(state) {
|
|
|
4872
5974
|
if (state.interactive) displayStatus(state);
|
|
4873
5975
|
}
|
|
4874
5976
|
}
|
|
5977
|
+
async function timeShutdownPhase(state, durations, name, run2) {
|
|
5978
|
+
const startedAt = Date.now();
|
|
5979
|
+
try {
|
|
5980
|
+
return await run2();
|
|
5981
|
+
} finally {
|
|
5982
|
+
const elapsedMs = Date.now() - startedAt;
|
|
5983
|
+
durations[name] = elapsedMs;
|
|
5984
|
+
log2(state, `Shutdown phase ${name}: ${elapsedMs}ms`);
|
|
5985
|
+
}
|
|
5986
|
+
}
|
|
4875
5987
|
async function cleanup(state, opts = {}) {
|
|
5988
|
+
const durations = {};
|
|
4876
5989
|
state.running = false;
|
|
4877
5990
|
for (const timer of state.sessionCleanupTimers) {
|
|
4878
5991
|
clearInterval(timer);
|
|
@@ -4886,7 +5999,13 @@ async function cleanup(state, opts = {}) {
|
|
|
4886
5999
|
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
4887
6000
|
displayStatus(state);
|
|
4888
6001
|
}
|
|
4889
|
-
const
|
|
6002
|
+
const driver = state.channelDriver;
|
|
6003
|
+
const settled = await timeShutdownPhase(
|
|
6004
|
+
state,
|
|
6005
|
+
durations,
|
|
6006
|
+
"drain",
|
|
6007
|
+
() => driver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS)
|
|
6008
|
+
);
|
|
4890
6009
|
if (!settled) {
|
|
4891
6010
|
logActivity(state, {
|
|
4892
6011
|
type: "info",
|
|
@@ -4895,13 +6014,15 @@ async function cleanup(state, opts = {}) {
|
|
|
4895
6014
|
if (state.interactive) displayStatus(state);
|
|
4896
6015
|
}
|
|
4897
6016
|
}
|
|
4898
|
-
await notifyOffline(state);
|
|
6017
|
+
await timeShutdownPhase(state, durations, "offline_notify", () => notifyOffline(state));
|
|
4899
6018
|
if (state.connection) {
|
|
4900
|
-
state.connection
|
|
6019
|
+
const connection = state.connection;
|
|
6020
|
+
await timeShutdownPhase(state, durations, "tunnel_close", () => connection.close());
|
|
4901
6021
|
state.connection = null;
|
|
4902
6022
|
}
|
|
4903
6023
|
if (state.opencodeProcess) {
|
|
4904
|
-
|
|
6024
|
+
const opencodeProcess = state.opencodeProcess;
|
|
6025
|
+
await timeShutdownPhase(state, durations, "opencode_stop", () => stopOpenCode(opencodeProcess));
|
|
4905
6026
|
if (state.interactive) {
|
|
4906
6027
|
logActivity(state, { type: "info", message: "Stopped OpenCode process" });
|
|
4907
6028
|
displayStatus(state);
|
|
@@ -4910,12 +6031,15 @@ async function cleanup(state, opts = {}) {
|
|
|
4910
6031
|
}
|
|
4911
6032
|
state.opencodeProcess = null;
|
|
4912
6033
|
}
|
|
6034
|
+
return durations;
|
|
4913
6035
|
}
|
|
4914
6036
|
async function run(options) {
|
|
4915
6037
|
const interactive = isInteractive(options.json);
|
|
4916
6038
|
let logLevel;
|
|
6039
|
+
let fileSyncDirectories;
|
|
4917
6040
|
try {
|
|
4918
6041
|
logLevel = resolveLogLevel(options);
|
|
6042
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir2());
|
|
4919
6043
|
} catch (error2) {
|
|
4920
6044
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
4921
6045
|
if (options.json) {
|
|
@@ -4950,6 +6074,12 @@ async function run(options) {
|
|
|
4950
6074
|
sessionCleanupTimers: [],
|
|
4951
6075
|
authHeader: ""
|
|
4952
6076
|
};
|
|
6077
|
+
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
6078
|
+
if (fileSyncDirectories.length > 0) {
|
|
6079
|
+
log2(state, `File sync enabled for: ${fileSyncDirectories.join(", ")}`);
|
|
6080
|
+
} else {
|
|
6081
|
+
log2(state, "File sync is disabled (no --enable-file-sync-to given)", "debug");
|
|
6082
|
+
}
|
|
4953
6083
|
if (!options.runner && options.agent) {
|
|
4954
6084
|
telemetry.info(
|
|
4955
6085
|
EventTypes.DEPRECATED_AGENT_FLAG_USED,
|
|
@@ -4973,14 +6103,38 @@ async function run(options) {
|
|
|
4973
6103
|
const handleSignal = async () => {
|
|
4974
6104
|
if (state.shuttingDown) return;
|
|
4975
6105
|
state.shuttingDown = true;
|
|
6106
|
+
const shutdownStartedAt = Date.now();
|
|
4976
6107
|
if (state.interactive) {
|
|
4977
6108
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
4978
6109
|
displayStatus(state);
|
|
4979
6110
|
} else {
|
|
4980
6111
|
log2(state, "Shutting down...");
|
|
4981
6112
|
}
|
|
4982
|
-
await cleanup(state, { graceful: true });
|
|
4983
|
-
|
|
6113
|
+
const durations = await cleanup(state, { graceful: true });
|
|
6114
|
+
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
6115
|
+
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
6116
|
+
let timer;
|
|
6117
|
+
const flushed = shutdownTelemetry().then(
|
|
6118
|
+
() => true,
|
|
6119
|
+
(error2) => {
|
|
6120
|
+
log2(
|
|
6121
|
+
state,
|
|
6122
|
+
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
6123
|
+
"warn"
|
|
6124
|
+
);
|
|
6125
|
+
return true;
|
|
6126
|
+
}
|
|
6127
|
+
);
|
|
6128
|
+
const timedOut = new Promise((resolve3) => {
|
|
6129
|
+
timer = setTimeout(() => resolve3(false), telemetryBudgetMs);
|
|
6130
|
+
});
|
|
6131
|
+
if (!await Promise.race([flushed, timedOut])) {
|
|
6132
|
+
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
6133
|
+
}
|
|
6134
|
+
clearTimeout(timer);
|
|
6135
|
+
});
|
|
6136
|
+
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
6137
|
+
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
4984
6138
|
process.exit(0);
|
|
4985
6139
|
};
|
|
4986
6140
|
process.on("SIGINT", handleSignal);
|
|
@@ -5094,40 +6248,67 @@ async function run(options) {
|
|
|
5094
6248
|
}
|
|
5095
6249
|
spinner?.succeed(`Runner: ${validation.agent.name || state.agentId}`);
|
|
5096
6250
|
state.agentName = validation.agent.name;
|
|
6251
|
+
const microvmId = process.env.MICROVM_ID?.trim();
|
|
6252
|
+
if (microvmId) {
|
|
6253
|
+
const reported = await reportMicrovmId(state.agentId, state.authHeader, microvmId);
|
|
6254
|
+
if (reported.ok) {
|
|
6255
|
+
log2(state, "Reported MicroVM identity so this runner can be resumed rather than restarted");
|
|
6256
|
+
} else {
|
|
6257
|
+
const message = `Could not report MicroVM identity (future wakes will cold-start): ${reported.error}`;
|
|
6258
|
+
log2(state, message, "warn");
|
|
6259
|
+
if (state.interactive && !state.json) {
|
|
6260
|
+
logActivity(state, { type: "info", level: "warn", message });
|
|
6261
|
+
}
|
|
6262
|
+
}
|
|
6263
|
+
} else {
|
|
6264
|
+
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
6265
|
+
}
|
|
6266
|
+
const { timeoutMs: opencodeStartTimeoutMs, warnings: opencodeStartTimeoutWarnings } = resolveOpenCodeStartTimeoutMs(options, process.env);
|
|
6267
|
+
for (const warning2 of opencodeStartTimeoutWarnings) {
|
|
6268
|
+
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
6269
|
+
}
|
|
5097
6270
|
const ocSpinner = interactive && !state.json ? ora3("Checking OpenCode...").start() : null;
|
|
5098
6271
|
try {
|
|
5099
6272
|
const oc = await ensureOpenCodeRunning({
|
|
5100
6273
|
port: state.port,
|
|
5101
6274
|
interactive: state.interactive,
|
|
5102
6275
|
agentId: state.agentId,
|
|
5103
|
-
log: (message) => log2(state, message)
|
|
6276
|
+
log: (message) => log2(state, message),
|
|
6277
|
+
startTimeoutMs: opencodeStartTimeoutMs
|
|
5104
6278
|
});
|
|
5105
6279
|
state.port = oc.port;
|
|
5106
6280
|
state.opencodeProcess = oc.process;
|
|
5107
6281
|
state.opencodeVersion = oc.version;
|
|
5108
|
-
state.opencodeConnected = oc.
|
|
6282
|
+
state.opencodeConnected = oc.notReadyReason === null;
|
|
5109
6283
|
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
5110
6284
|
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
5111
|
-
|
|
5112
|
-
|
|
5113
|
-
|
|
5114
|
-
|
|
5115
|
-
|
|
6285
|
+
if (!state.interactive && oc.notReadyReason !== null) {
|
|
6286
|
+
const message = `OpenCode is not ready on port ${state.port}: ${oc.notReadyReason}. The runner will still come online, but messages will fail until opencode answers \u2014 raise the wait with --opencode-start-timeout <seconds> (env ${OPENCODE_START_TIMEOUT_ENV}).`;
|
|
6287
|
+
logActivity(state, { type: "info", level: "warn", message });
|
|
6288
|
+
} else {
|
|
6289
|
+
const versionWarning = buildOpenCodeVersionWarning(state.opencodeVersion);
|
|
6290
|
+
if (versionWarning) {
|
|
6291
|
+
log2(state, versionWarning, "warn");
|
|
6292
|
+
if (state.interactive && !state.json) {
|
|
6293
|
+
logActivity(state, { type: "info", level: "warn", message: versionWarning });
|
|
6294
|
+
}
|
|
5116
6295
|
}
|
|
5117
|
-
|
|
5118
|
-
|
|
5119
|
-
|
|
5120
|
-
|
|
5121
|
-
|
|
5122
|
-
|
|
5123
|
-
|
|
5124
|
-
|
|
5125
|
-
|
|
5126
|
-
|
|
5127
|
-
|
|
5128
|
-
|
|
5129
|
-
|
|
5130
|
-
|
|
6296
|
+
const noProviderWarning = buildNoProviderWarning(
|
|
6297
|
+
await hasAnyConfiguredProvider(state.port)
|
|
6298
|
+
);
|
|
6299
|
+
if (noProviderWarning) {
|
|
6300
|
+
log2(state, noProviderWarning, "warn");
|
|
6301
|
+
if (state.interactive && !state.json) {
|
|
6302
|
+
logActivity(state, { type: "info", level: "warn", message: noProviderWarning });
|
|
6303
|
+
blank();
|
|
6304
|
+
console.log(chalk6.yellow("\u26A0 No OpenCode model provider is configured."));
|
|
6305
|
+
console.log(
|
|
6306
|
+
chalk6.dim(
|
|
6307
|
+
`Run ${chalk6.cyan("opencode auth login")} to set one up \u2014 messages will fail until then.`
|
|
6308
|
+
)
|
|
6309
|
+
);
|
|
6310
|
+
blank();
|
|
6311
|
+
}
|
|
5131
6312
|
}
|
|
5132
6313
|
}
|
|
5133
6314
|
} catch (error2) {
|
|
@@ -5142,6 +6323,10 @@ async function run(options) {
|
|
|
5142
6323
|
getAuthHeader: () => state.authHeader,
|
|
5143
6324
|
conversationFilter: state.conversationFilter,
|
|
5144
6325
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
6326
|
+
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
6327
|
+
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
6328
|
+
fileSyncDirectories,
|
|
6329
|
+
homeDir: homedir2(),
|
|
5145
6330
|
log: (entry) => (
|
|
5146
6331
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
5147
6332
|
// survive the sink filter (they no longer collapse to info). `type`
|
|
@@ -5168,6 +6353,18 @@ async function run(options) {
|
|
|
5168
6353
|
type: "info",
|
|
5169
6354
|
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (runner: ${agentId})`
|
|
5170
6355
|
});
|
|
6356
|
+
if (options.tunnelReadyFile) {
|
|
6357
|
+
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
6358
|
+
if (marker.ok) {
|
|
6359
|
+
log2(state, `Wrote tunnel readiness marker to ${options.tunnelReadyFile}`, "debug");
|
|
6360
|
+
} else {
|
|
6361
|
+
log2(
|
|
6362
|
+
state,
|
|
6363
|
+
`Failed to write tunnel readiness marker to ${options.tunnelReadyFile}: ${marker.error}`,
|
|
6364
|
+
"error"
|
|
6365
|
+
);
|
|
6366
|
+
}
|
|
6367
|
+
}
|
|
5171
6368
|
emitAgentConnected(state.agentId, {
|
|
5172
6369
|
port: state.port,
|
|
5173
6370
|
cli_version: getCliVersion(),
|
|
@@ -5223,6 +6420,12 @@ async function run(options) {
|
|
|
5223
6420
|
onDrainPing: () => {
|
|
5224
6421
|
if (!state.running) return;
|
|
5225
6422
|
logActivity(state, { type: "info", message: "Drain ping received \u2014 draining" });
|
|
6423
|
+
void channelDriver.syncPendingFiles().catch(
|
|
6424
|
+
(error2) => logActivity(state, {
|
|
6425
|
+
type: "error",
|
|
6426
|
+
error: `Runner file sync failed on ping: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
6427
|
+
})
|
|
6428
|
+
);
|
|
5226
6429
|
channelDriver.drainPending().then((processed) => {
|
|
5227
6430
|
if (processed > 0) {
|
|
5228
6431
|
state.messageCount += processed;
|
|
@@ -5312,7 +6515,10 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
5312
6515
|
).option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option(
|
|
5313
6516
|
"--log-level <level>",
|
|
5314
6517
|
"Log verbosity: debug | info | warn | error (default: info). Env: EVIDENT_LOG_LEVEL"
|
|
5315
|
-
).option("-v, --verbose", "Alias for --log-level debug (ignored if --log-level is set)").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option(
|
|
6518
|
+
).option("-v, --verbose", "Alias for --log-level debug (ignored if --log-level is set)").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option(
|
|
6519
|
+
"--opencode-start-timeout <seconds>",
|
|
6520
|
+
"Seconds to wait for OpenCode to become healthy when the runner starts it (default: 180). Env: EVIDENT_OPENCODE_START_TIMEOUT"
|
|
6521
|
+
).option("--json", "Output in JSON format").option(
|
|
5316
6522
|
"--session-cleanup-max-age <duration>",
|
|
5317
6523
|
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
5318
6524
|
).option(
|
|
@@ -5321,6 +6527,14 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
5321
6527
|
).option(
|
|
5322
6528
|
"--session-cleanup-interval <duration>",
|
|
5323
6529
|
"How often the cleanup sweep runs (default: 1h). Env: EVIDENT_SESSION_CLEANUP_INTERVAL"
|
|
6530
|
+
).option(
|
|
6531
|
+
"--enable-file-sync-to <dir>",
|
|
6532
|
+
"Allow Evident to write files into this directory (repeatable). Omit to disable file sync entirely.",
|
|
6533
|
+
(value, previous) => previous.concat([value]),
|
|
6534
|
+
[]
|
|
6535
|
+
).option(
|
|
6536
|
+
"--tunnel-ready-file <path>",
|
|
6537
|
+
"Path to write once the tunnel is connected (set by the MicroVM hooks; unused on a developer machine)"
|
|
5324
6538
|
).action(
|
|
5325
6539
|
(options) => {
|
|
5326
6540
|
run({
|
|
@@ -5333,11 +6547,18 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
5333
6547
|
verbose: options.verbose,
|
|
5334
6548
|
conversation: options.conversation,
|
|
5335
6549
|
idleTimeout: options.idleTimeout ? parseInt(options.idleTimeout, 10) : void 0,
|
|
6550
|
+
// Raw string — validation/precedence is single-sourced in run.ts's
|
|
6551
|
+
// resolveOpenCodeStartTimeoutMs (flag > EVIDENT_OPENCODE_START_TIMEOUT > 180s default).
|
|
6552
|
+
opencodeStartTimeout: options.opencodeStartTimeout,
|
|
5336
6553
|
json: options.json,
|
|
5337
6554
|
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
5338
6555
|
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
5339
6556
|
sessionCleanupMaxCount: options.sessionCleanupMaxCount,
|
|
5340
|
-
sessionCleanupInterval: options.sessionCleanupInterval
|
|
6557
|
+
sessionCleanupInterval: options.sessionCleanupInterval,
|
|
6558
|
+
// Raw values — expansion/validation is single-sourced in run.ts's
|
|
6559
|
+
// resolveFileSyncDirectories.
|
|
6560
|
+
enableFileSyncTo: options.enableFileSyncTo,
|
|
6561
|
+
tunnelReadyFile: options.tunnelReadyFile
|
|
5341
6562
|
});
|
|
5342
6563
|
}
|
|
5343
6564
|
);
|