@evident-ai/cli 3.4.1-dev.cb56b3f → 3.4.1-dev.d2c12e9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +393 -83
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -763,6 +763,13 @@ function toReportedOpenAiWindow(window) {
|
|
|
763
763
|
resets_at: window.resetsAt
|
|
764
764
|
};
|
|
765
765
|
}
|
|
766
|
+
function toReportedOpenAiSubscription(snapshot) {
|
|
767
|
+
if (!snapshot.subscription) return null;
|
|
768
|
+
return {
|
|
769
|
+
owner_email: snapshot.subscription.ownerEmail,
|
|
770
|
+
plan_type: snapshot.subscription.planType
|
|
771
|
+
};
|
|
772
|
+
}
|
|
766
773
|
async function reportOpenAiUsage(agentId, authHeader, snapshot) {
|
|
767
774
|
try {
|
|
768
775
|
const apiUrl = getApiUrlConfig();
|
|
@@ -773,7 +780,8 @@ async function reportOpenAiUsage(agentId, authHeader, snapshot) {
|
|
|
773
780
|
primary: toReportedOpenAiWindow(snapshot.primary),
|
|
774
781
|
secondary: toReportedOpenAiWindow(snapshot.secondary),
|
|
775
782
|
has_credits: snapshot.hasCredits,
|
|
776
|
-
credits_unlimited: snapshot.creditsUnlimited
|
|
783
|
+
credits_unlimited: snapshot.creditsUnlimited,
|
|
784
|
+
subscription: toReportedOpenAiSubscription(snapshot)
|
|
777
785
|
}),
|
|
778
786
|
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
779
787
|
});
|
|
@@ -797,6 +805,7 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
797
805
|
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
798
806
|
body: JSON.stringify({
|
|
799
807
|
cpu_percent: usage.cpuPercent,
|
|
808
|
+
cpu_peak_percent: usage.cpuPeakPercent,
|
|
800
809
|
cpu_count: usage.cpuCount,
|
|
801
810
|
memory_total_bytes: usage.memoryTotalBytes,
|
|
802
811
|
memory_available_bytes: usage.memoryAvailableBytes,
|
|
@@ -1207,6 +1216,7 @@ var TelemetryEventTypes = {
|
|
|
1207
1216
|
// ../../packages/types/src/tunnel/index.ts
|
|
1208
1217
|
var MAX_FRAME_BYTES = 256 * 1024;
|
|
1209
1218
|
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
1219
|
+
var TUNNEL_USAGE_REARM_PING_PATH = "/__evident/usage-rearm";
|
|
1210
1220
|
|
|
1211
1221
|
// ../../packages/types/src/runner-files.ts
|
|
1212
1222
|
var MAX_FILE_PUSH_BYTES = 64 * 1024;
|
|
@@ -3142,6 +3152,44 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
3142
3152
|
}
|
|
3143
3153
|
return lastOk ?? last;
|
|
3144
3154
|
}
|
|
3155
|
+
function collectSubagentSessions(messages, userMessageId) {
|
|
3156
|
+
if (!messages || messages.length === 0) return [];
|
|
3157
|
+
const byParent = messages.filter(
|
|
3158
|
+
(message) => roleOf(message) === "assistant" && parentIdOf(message) === userMessageId
|
|
3159
|
+
);
|
|
3160
|
+
const assistants = byParent.length > 0 ? byParent : [];
|
|
3161
|
+
if (assistants.length === 0) {
|
|
3162
|
+
const userIndex = messages.findIndex((message) => idOf(message) === userMessageId);
|
|
3163
|
+
if (userIndex === -1) return [];
|
|
3164
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
3165
|
+
const message = messages[i];
|
|
3166
|
+
if (roleOf(message) === "user") break;
|
|
3167
|
+
if (roleOf(message) === "assistant") assistants.push(message);
|
|
3168
|
+
}
|
|
3169
|
+
}
|
|
3170
|
+
const refs = [];
|
|
3171
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3172
|
+
for (const message of assistants) {
|
|
3173
|
+
const parts = Array.isArray(message.parts) ? message.parts : [];
|
|
3174
|
+
for (const part of parts) {
|
|
3175
|
+
if (!part || typeof part !== "object" || part.type !== "tool" || part.tool !== "task")
|
|
3176
|
+
continue;
|
|
3177
|
+
const state = part.state;
|
|
3178
|
+
if (!state || typeof state !== "object") continue;
|
|
3179
|
+
const metadata = state.metadata;
|
|
3180
|
+
if (!metadata || typeof metadata !== "object") continue;
|
|
3181
|
+
const sessionId = metadata.sessionId;
|
|
3182
|
+
if (typeof sessionId !== "string" || sessionId.length === 0 || seen.has(sessionId)) continue;
|
|
3183
|
+
seen.add(sessionId);
|
|
3184
|
+
const start = state.time?.start;
|
|
3185
|
+
refs.push({
|
|
3186
|
+
sessionId,
|
|
3187
|
+
startedAtMs: typeof start === "number" && Number.isFinite(start) ? start : null
|
|
3188
|
+
});
|
|
3189
|
+
}
|
|
3190
|
+
}
|
|
3191
|
+
return refs;
|
|
3192
|
+
}
|
|
3145
3193
|
function messageUsage(messages, userMessageId) {
|
|
3146
3194
|
if (!messages || messages.length === 0) return null;
|
|
3147
3195
|
const byParentAll = messages.filter(
|
|
@@ -3270,8 +3318,7 @@ function isAbortedTerminalReply(messages, userMessageId) {
|
|
|
3270
3318
|
}
|
|
3271
3319
|
return false;
|
|
3272
3320
|
}
|
|
3273
|
-
function
|
|
3274
|
-
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3321
|
+
function classifyReplyAuthError(reply) {
|
|
3275
3322
|
const error2 = errorOf(reply);
|
|
3276
3323
|
if (error2 == null || typeof error2 !== "object") return null;
|
|
3277
3324
|
const e = error2;
|
|
@@ -3296,6 +3343,32 @@ function messageFailure(messages, userMessageId) {
|
|
|
3296
3343
|
}
|
|
3297
3344
|
return null;
|
|
3298
3345
|
}
|
|
3346
|
+
function messageFailure(messages, userMessageId) {
|
|
3347
|
+
return classifyReplyAuthError(findLastAssistantReplyFor(messages, userMessageId));
|
|
3348
|
+
}
|
|
3349
|
+
function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
3350
|
+
if (!messages || messages.length === 0) return null;
|
|
3351
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
3352
|
+
const message = messages[i];
|
|
3353
|
+
if (roleOf(message) !== "assistant") continue;
|
|
3354
|
+
const created = createdOf(message);
|
|
3355
|
+
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
3356
|
+
const failure = classifyReplyAuthError(message);
|
|
3357
|
+
if (failure) {
|
|
3358
|
+
if (!failure.providerId) return null;
|
|
3359
|
+
return { providerId: failure.providerId, outcome: "failed", failure };
|
|
3360
|
+
}
|
|
3361
|
+
const providerId = message.info?.providerID;
|
|
3362
|
+
if (errorOf(message) == null && typeof providerId === "string" && providerId.length > 0) {
|
|
3363
|
+
return { providerId, outcome: "succeeded" };
|
|
3364
|
+
}
|
|
3365
|
+
return null;
|
|
3366
|
+
}
|
|
3367
|
+
return null;
|
|
3368
|
+
}
|
|
3369
|
+
function findSubagentAuthOutcome(messages, sinceMs) {
|
|
3370
|
+
return findLatestSubagentAuthOutcome(messages, sinceMs);
|
|
3371
|
+
}
|
|
3299
3372
|
function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProviderId, replyModelId = null) {
|
|
3300
3373
|
if (classified != null) return classified;
|
|
3301
3374
|
if (hasConfiguredProvider !== false) return null;
|
|
@@ -3626,7 +3699,6 @@ var StreamForwarder = class {
|
|
|
3626
3699
|
handleFrame(frame) {
|
|
3627
3700
|
switch (frame.type) {
|
|
3628
3701
|
case "open":
|
|
3629
|
-
this.callbacks.onOpen?.(frame.sid, frame.method, frame.path);
|
|
3630
3702
|
void this.handleOpen(frame);
|
|
3631
3703
|
break;
|
|
3632
3704
|
case "req_data":
|
|
@@ -3662,12 +3734,21 @@ var StreamForwarder = class {
|
|
|
3662
3734
|
const { sid, method, path, headers, has_body } = frame;
|
|
3663
3735
|
const correlationId = headers?.[CORRELATION_ID_HEADER];
|
|
3664
3736
|
const startedAt = Date.now();
|
|
3737
|
+
if (path !== TUNNEL_DRAIN_PING_PATH && path !== TUNNEL_USAGE_REARM_PING_PATH) {
|
|
3738
|
+
this.callbacks.onOpen?.(sid, method, path);
|
|
3739
|
+
}
|
|
3665
3740
|
if (path === TUNNEL_DRAIN_PING_PATH) {
|
|
3666
3741
|
this.callbacks.onDrainPing?.();
|
|
3667
3742
|
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
3668
3743
|
this.send({ type: "res_end", sid });
|
|
3669
3744
|
return;
|
|
3670
3745
|
}
|
|
3746
|
+
if (path === TUNNEL_USAGE_REARM_PING_PATH) {
|
|
3747
|
+
this.callbacks.onUsageRearmPing?.();
|
|
3748
|
+
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
3749
|
+
this.send({ type: "res_end", sid });
|
|
3750
|
+
return;
|
|
3751
|
+
}
|
|
3671
3752
|
if (process.env.DEBUG) {
|
|
3672
3753
|
log("debug", "agent_request", {
|
|
3673
3754
|
correlation_id: correlationId,
|
|
@@ -3812,7 +3893,8 @@ function connectTunnel(options) {
|
|
|
3812
3893
|
onResponse,
|
|
3813
3894
|
onInfo,
|
|
3814
3895
|
onWarning,
|
|
3815
|
-
onDrainPing
|
|
3896
|
+
onDrainPing,
|
|
3897
|
+
onUsageRearmPing
|
|
3816
3898
|
} = options;
|
|
3817
3899
|
const tunnelUrl = getTunnelUrlConfig();
|
|
3818
3900
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
@@ -3824,7 +3906,8 @@ function connectTunnel(options) {
|
|
|
3824
3906
|
});
|
|
3825
3907
|
const forwarder = new StreamForwarder(ws, port, {
|
|
3826
3908
|
onHead: () => onResponse?.(),
|
|
3827
|
-
onDrainPing: () => onDrainPing?.()
|
|
3909
|
+
onDrainPing: () => onDrainPing?.(),
|
|
3910
|
+
onUsageRearmPing: () => onUsageRearmPing?.()
|
|
3828
3911
|
});
|
|
3829
3912
|
const connectionTimeout = setTimeout(() => {
|
|
3830
3913
|
ws.close();
|
|
@@ -3985,6 +4068,7 @@ var RunnerConnection = class {
|
|
|
3985
4068
|
onError: (error2) => events.onError?.(error2),
|
|
3986
4069
|
onResponse: () => events.onResponse?.(),
|
|
3987
4070
|
onDrainPing: () => events.onDrainPing?.(),
|
|
4071
|
+
onUsageRearmPing: () => events.onUsageRearmPing?.(),
|
|
3988
4072
|
onInfo: (message) => events.onInfo?.(message),
|
|
3989
4073
|
onWarning: (message) => events.onWarning?.(message)
|
|
3990
4074
|
});
|
|
@@ -4103,6 +4187,23 @@ function readOpenCodeChatGptCredentials() {
|
|
|
4103
4187
|
return null;
|
|
4104
4188
|
}
|
|
4105
4189
|
}
|
|
4190
|
+
function parseChatGptIdentity(accessToken) {
|
|
4191
|
+
const segments = accessToken.split(".");
|
|
4192
|
+
if (segments.length !== 3) return null;
|
|
4193
|
+
let payload;
|
|
4194
|
+
try {
|
|
4195
|
+
const parsed = JSON.parse(Buffer.from(segments[1], "base64url").toString("utf8"));
|
|
4196
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null;
|
|
4197
|
+
payload = parsed;
|
|
4198
|
+
} catch {
|
|
4199
|
+
return null;
|
|
4200
|
+
}
|
|
4201
|
+
const profile = payload["https://api.openai.com/profile"];
|
|
4202
|
+
const auth = payload["https://api.openai.com/auth"];
|
|
4203
|
+
const ownerEmail = typeof profile?.email === "string" ? profile.email.trim() || null : null;
|
|
4204
|
+
const planType = typeof auth?.chatgpt_plan_type === "string" ? auth.chatgpt_plan_type.trim() || null : null;
|
|
4205
|
+
return ownerEmail === null && planType === null ? null : { ownerEmail, planType };
|
|
4206
|
+
}
|
|
4106
4207
|
function toWindow2(headers, name) {
|
|
4107
4208
|
const utilizationHeader = headers.get(`x-codex-${name}-used-percent`);
|
|
4108
4209
|
const windowMinutesHeader = headers.get(`x-codex-${name}-window-minutes`);
|
|
@@ -4178,6 +4279,7 @@ async function getOpenAiUsage(port) {
|
|
|
4178
4279
|
"credentials_expired"
|
|
4179
4280
|
);
|
|
4180
4281
|
}
|
|
4282
|
+
const subscription = parseChatGptIdentity(credentials2.accessToken);
|
|
4181
4283
|
const models = await resolveProbeModels(port);
|
|
4182
4284
|
if (models.length === 0) {
|
|
4183
4285
|
throw new OpenAiUsageError("No supported OpenAI probe model is available.", "no_probe_model");
|
|
@@ -4210,7 +4312,7 @@ async function getOpenAiUsage(port) {
|
|
|
4210
4312
|
"no_usable_window"
|
|
4211
4313
|
);
|
|
4212
4314
|
}
|
|
4213
|
-
return usage;
|
|
4315
|
+
return { ...usage, subscription };
|
|
4214
4316
|
}
|
|
4215
4317
|
if (res.status === 401) {
|
|
4216
4318
|
throw new OpenAiUsageError(
|
|
@@ -4411,45 +4513,84 @@ function readDisk(homeDir) {
|
|
|
4411
4513
|
};
|
|
4412
4514
|
}
|
|
4413
4515
|
}
|
|
4414
|
-
|
|
4415
|
-
|
|
4416
|
-
|
|
4516
|
+
var CPU_PEAK_WINDOW_MS = 6e4;
|
|
4517
|
+
var CPU_PEAK_WINDOW_SAMPLE_COUNT = 4;
|
|
4518
|
+
var CPU_PEAK_SAMPLE_INTERVAL_MS = CPU_PEAK_WINDOW_MS / CPU_PEAK_WINDOW_SAMPLE_COUNT;
|
|
4519
|
+
function createCpuPeakSampler() {
|
|
4520
|
+
const sampleHistory = new Array(CPU_PEAK_WINDOW_SAMPLE_COUNT);
|
|
4521
|
+
sampleHistory[0] = readCpuSample();
|
|
4522
|
+
let nextSampleIndex = 1;
|
|
4523
|
+
let sampleCount = 1;
|
|
4524
|
+
let peak = null;
|
|
4525
|
+
const timer = setInterval(() => {
|
|
4417
4526
|
const current = readCpuSample();
|
|
4418
|
-
const
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
const warnings = [];
|
|
4425
|
-
if (disk.warning) warnings.push(disk.warning);
|
|
4426
|
-
if (ecsWarning) warnings.push(ecsWarning);
|
|
4427
|
-
let cpuPercent = hostCpuPercent;
|
|
4428
|
-
let cpuCount = hostCpuCount;
|
|
4429
|
-
let memoryTotalBytes = totalmem();
|
|
4430
|
-
let memoryAvailableBytes = freemem();
|
|
4431
|
-
if (limits !== null) {
|
|
4432
|
-
cpuCount = limits.cpuCount;
|
|
4433
|
-
memoryTotalBytes = limits.memoryTotalBytes;
|
|
4434
|
-
memoryAvailableBytes = clamp(
|
|
4435
|
-
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
4436
|
-
0,
|
|
4437
|
-
limits.memoryTotalBytes
|
|
4438
|
-
);
|
|
4439
|
-
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4527
|
+
const sampleFromWindowAgo = sampleCount === CPU_PEAK_WINDOW_SAMPLE_COUNT ? sampleHistory[nextSampleIndex] : void 0;
|
|
4528
|
+
if (sampleFromWindowAgo !== void 0) {
|
|
4529
|
+
const percentage = cpuPercentBetween(sampleFromWindowAgo, current);
|
|
4530
|
+
if (percentage !== null) {
|
|
4531
|
+
peak = peak === null ? percentage : Math.max(peak, percentage);
|
|
4532
|
+
}
|
|
4440
4533
|
}
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4446
|
-
|
|
4447
|
-
|
|
4448
|
-
|
|
4449
|
-
|
|
4450
|
-
|
|
4451
|
-
|
|
4452
|
-
|
|
4534
|
+
sampleHistory[nextSampleIndex] = current;
|
|
4535
|
+
nextSampleIndex = (nextSampleIndex + 1) % CPU_PEAK_WINDOW_SAMPLE_COUNT;
|
|
4536
|
+
sampleCount = Math.min(sampleCount + 1, CPU_PEAK_WINDOW_SAMPLE_COUNT);
|
|
4537
|
+
}, CPU_PEAK_SAMPLE_INTERVAL_MS);
|
|
4538
|
+
return {
|
|
4539
|
+
takeAndReset: () => {
|
|
4540
|
+
const currentPeak = peak;
|
|
4541
|
+
peak = null;
|
|
4542
|
+
return currentPeak;
|
|
4543
|
+
},
|
|
4544
|
+
stop: () => clearInterval(timer)
|
|
4545
|
+
};
|
|
4546
|
+
}
|
|
4547
|
+
function createResourceUsageCollector(homeDir) {
|
|
4548
|
+
let previous = readCpuSample();
|
|
4549
|
+
const cpuPeakSampler = createCpuPeakSampler();
|
|
4550
|
+
return {
|
|
4551
|
+
collect: async () => {
|
|
4552
|
+
const current = readCpuSample();
|
|
4553
|
+
const hostCpuPercent = cpuPercentBetween(previous, current);
|
|
4554
|
+
const hostCpuPeakPercent = cpuPeakSampler.takeAndReset();
|
|
4555
|
+
const hostCpuCount = cpus().length;
|
|
4556
|
+
previous = current;
|
|
4557
|
+
const disk = readDisk(homeDir);
|
|
4558
|
+
const opencodeDbBytes = statSessionDbBytes(homeDir);
|
|
4559
|
+
const { limits, warning: ecsWarning } = await readEcsTaskLimits(process.env);
|
|
4560
|
+
const warnings = [];
|
|
4561
|
+
if (disk.warning) warnings.push(disk.warning);
|
|
4562
|
+
if (ecsWarning) warnings.push(ecsWarning);
|
|
4563
|
+
let cpuPercent = hostCpuPercent;
|
|
4564
|
+
let cpuPeakPercent = hostCpuPeakPercent;
|
|
4565
|
+
let cpuCount = hostCpuCount;
|
|
4566
|
+
let memoryTotalBytes = totalmem();
|
|
4567
|
+
let memoryAvailableBytes = freemem();
|
|
4568
|
+
if (limits !== null) {
|
|
4569
|
+
cpuCount = limits.cpuCount;
|
|
4570
|
+
memoryTotalBytes = limits.memoryTotalBytes;
|
|
4571
|
+
memoryAvailableBytes = clamp(
|
|
4572
|
+
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
4573
|
+
0,
|
|
4574
|
+
limits.memoryTotalBytes
|
|
4575
|
+
);
|
|
4576
|
+
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4577
|
+
cpuPeakPercent = hostCpuPeakPercent === null ? null : clamp(round2(hostCpuPeakPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4578
|
+
}
|
|
4579
|
+
return {
|
|
4580
|
+
usage: {
|
|
4581
|
+
cpuPercent,
|
|
4582
|
+
cpuPeakPercent,
|
|
4583
|
+
cpuCount,
|
|
4584
|
+
memoryTotalBytes,
|
|
4585
|
+
memoryAvailableBytes,
|
|
4586
|
+
diskTotalBytes: disk.totalBytes,
|
|
4587
|
+
diskFreeBytes: disk.freeBytes,
|
|
4588
|
+
opencodeDbBytes
|
|
4589
|
+
},
|
|
4590
|
+
warnings
|
|
4591
|
+
};
|
|
4592
|
+
},
|
|
4593
|
+
stop: cpuPeakSampler.stop
|
|
4453
4594
|
};
|
|
4454
4595
|
}
|
|
4455
4596
|
|
|
@@ -5272,6 +5413,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5272
5413
|
* and stops opencode.
|
|
5273
5414
|
*/
|
|
5274
5415
|
stopped = false;
|
|
5416
|
+
recycleRequestedFlag = false;
|
|
5275
5417
|
constructor(config) {
|
|
5276
5418
|
this.agentId = config.agentId;
|
|
5277
5419
|
this.port = config.port;
|
|
@@ -5377,6 +5519,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5377
5519
|
let dispatched = 0;
|
|
5378
5520
|
try {
|
|
5379
5521
|
const conversations = await this.getPendingConversations();
|
|
5522
|
+
if (this.recycleRequestedFlag) {
|
|
5523
|
+
this.stop();
|
|
5524
|
+
}
|
|
5380
5525
|
if (conversations.length > 0) {
|
|
5381
5526
|
const total = conversations.reduce((sum, c) => sum + (c.pending_message_count ?? 0), 0);
|
|
5382
5527
|
this.log({
|
|
@@ -5505,6 +5650,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5505
5650
|
stop() {
|
|
5506
5651
|
this.stopped = true;
|
|
5507
5652
|
}
|
|
5653
|
+
/**
|
|
5654
|
+
* The server clears this request when a new MicroVM identity is recorded, so a
|
|
5655
|
+
* same-VM tunnel reconnect does not consume it. This is a plain read rather
|
|
5656
|
+
* than a consume; `run.ts` guards the action once-only.
|
|
5657
|
+
*/
|
|
5658
|
+
get recycleRequested() {
|
|
5659
|
+
return this.recycleRequestedFlag;
|
|
5660
|
+
}
|
|
5508
5661
|
/**
|
|
5509
5662
|
* Wait (up to `timeoutMs`) for in-flight watcher work to settle during a
|
|
5510
5663
|
* graceful shutdown, so a turn whose reply is ready — or completes within the
|
|
@@ -6024,6 +6177,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6024
6177
|
});
|
|
6025
6178
|
await this.markFailed(conv.id, message.id, sessionId, error2, usage, failure);
|
|
6026
6179
|
}
|
|
6180
|
+
if (ocId !== null) {
|
|
6181
|
+
await this.reportSubagentAuthFailures(conv.id, ocId, message.id, messages);
|
|
6182
|
+
}
|
|
6027
6183
|
} catch (err) {
|
|
6028
6184
|
if (err instanceof ChannelAuthError) throw err;
|
|
6029
6185
|
this.log({
|
|
@@ -6988,6 +7144,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6988
7144
|
return;
|
|
6989
7145
|
}
|
|
6990
7146
|
inFlight.done = true;
|
|
7147
|
+
await this.reportSubagentAuthFailures(
|
|
7148
|
+
watcher.conv.id,
|
|
7149
|
+
inFlight.opencodeMessageId,
|
|
7150
|
+
inFlight.evidentMessageId,
|
|
7151
|
+
messages
|
|
7152
|
+
);
|
|
6991
7153
|
}
|
|
6992
7154
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
6993
7155
|
return;
|
|
@@ -7230,6 +7392,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7230
7392
|
return;
|
|
7231
7393
|
}
|
|
7232
7394
|
inFlight.done = true;
|
|
7395
|
+
await this.reportSubagentAuthFailures(
|
|
7396
|
+
watcher.conv.id,
|
|
7397
|
+
inFlight.opencodeMessageId,
|
|
7398
|
+
inFlight.evidentMessageId,
|
|
7399
|
+
messages
|
|
7400
|
+
);
|
|
7233
7401
|
}
|
|
7234
7402
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
7235
7403
|
}
|
|
@@ -7400,6 +7568,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7400
7568
|
});
|
|
7401
7569
|
return;
|
|
7402
7570
|
}
|
|
7571
|
+
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
7403
7572
|
this.dontRedispatch.delete(row.id);
|
|
7404
7573
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
7405
7574
|
return;
|
|
@@ -7561,6 +7730,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7561
7730
|
});
|
|
7562
7731
|
return;
|
|
7563
7732
|
}
|
|
7733
|
+
if (ocId !== null) {
|
|
7734
|
+
await this.reportSubagentAuthFailures(row.conversation_id, ocId, row.id, messages);
|
|
7735
|
+
}
|
|
7564
7736
|
this.dontRedispatch.delete(row.id);
|
|
7565
7737
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
7566
7738
|
}
|
|
@@ -8276,6 +8448,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8276
8448
|
throw new Error(`Failed to get pending conversations: HTTP ${res.status}`);
|
|
8277
8449
|
}
|
|
8278
8450
|
const data = await res.json();
|
|
8451
|
+
this.recycleRequestedFlag = data.recycle_requested === true;
|
|
8279
8452
|
let conversations = data.conversations;
|
|
8280
8453
|
if (this.conversationFilter) {
|
|
8281
8454
|
conversations = conversations.filter((c) => c.id === this.conversationFilter);
|
|
@@ -8511,6 +8684,111 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8511
8684
|
reply?.info?.modelID ?? null
|
|
8512
8685
|
);
|
|
8513
8686
|
}
|
|
8687
|
+
async recordSubagentModelAuthFailure(failure, conversationId, messageId) {
|
|
8688
|
+
const providerId = failure.providerId ?? "(unknown)";
|
|
8689
|
+
try {
|
|
8690
|
+
const res = await this.fetchImpl(
|
|
8691
|
+
`${this.apiUrl}/runners/${this.agentId}/model-auth-failures`,
|
|
8692
|
+
{
|
|
8693
|
+
method: "POST",
|
|
8694
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
8695
|
+
body: JSON.stringify({
|
|
8696
|
+
provider_id: failure.providerId,
|
|
8697
|
+
model_id: failure.modelId,
|
|
8698
|
+
reason: failure.reason
|
|
8699
|
+
})
|
|
8700
|
+
}
|
|
8701
|
+
);
|
|
8702
|
+
if (!res.ok) {
|
|
8703
|
+
this.log({
|
|
8704
|
+
level: "warn",
|
|
8705
|
+
message: `Sub-agent model-auth report for provider ${providerId} (${failure.reason}) returned HTTP ${res.status} (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)})`,
|
|
8706
|
+
conversation_id: conversationId,
|
|
8707
|
+
message_id: messageId
|
|
8708
|
+
});
|
|
8709
|
+
}
|
|
8710
|
+
} catch (err) {
|
|
8711
|
+
this.log({
|
|
8712
|
+
level: "warn",
|
|
8713
|
+
message: `Sub-agent model-auth report for provider ${providerId} (${failure.reason}) failed with status network-error (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8714
|
+
conversation_id: conversationId,
|
|
8715
|
+
message_id: messageId
|
|
8716
|
+
});
|
|
8717
|
+
}
|
|
8718
|
+
}
|
|
8719
|
+
async clearSubagentModelAuthFailure(providerId, conversationId, messageId) {
|
|
8720
|
+
try {
|
|
8721
|
+
const res = await this.fetchImpl(
|
|
8722
|
+
`${this.apiUrl}/runners/${this.agentId}/model-auth-failures/${encodeURIComponent(providerId)}`,
|
|
8723
|
+
{
|
|
8724
|
+
method: "DELETE",
|
|
8725
|
+
headers: { Authorization: this.getAuthHeader() }
|
|
8726
|
+
}
|
|
8727
|
+
);
|
|
8728
|
+
if (!res.ok) {
|
|
8729
|
+
this.log({
|
|
8730
|
+
level: "warn",
|
|
8731
|
+
message: `Sub-agent model-auth clear for provider ${providerId} returned HTTP ${res.status} (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)})`,
|
|
8732
|
+
conversation_id: conversationId,
|
|
8733
|
+
message_id: messageId
|
|
8734
|
+
});
|
|
8735
|
+
}
|
|
8736
|
+
} catch (err) {
|
|
8737
|
+
this.log({
|
|
8738
|
+
level: "warn",
|
|
8739
|
+
message: `Sub-agent model-auth clear for provider ${providerId} failed with status network-error (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8740
|
+
conversation_id: conversationId,
|
|
8741
|
+
message_id: messageId
|
|
8742
|
+
});
|
|
8743
|
+
}
|
|
8744
|
+
}
|
|
8745
|
+
async reportSubagentAuthFailures(conversationId, opencodeMessageId, evidentMessageId, messages) {
|
|
8746
|
+
const refs = collectSubagentSessions(messages, opencodeMessageId);
|
|
8747
|
+
if (refs.length === 0) return;
|
|
8748
|
+
const failedProviders = /* @__PURE__ */ new Map();
|
|
8749
|
+
const succeededProviders = /* @__PURE__ */ new Set();
|
|
8750
|
+
for (const ref of refs) {
|
|
8751
|
+
try {
|
|
8752
|
+
const childMessages = await getSessionMessages(this.port, ref.sessionId);
|
|
8753
|
+
if (childMessages === null) {
|
|
8754
|
+
this.log({
|
|
8755
|
+
level: "debug",
|
|
8756
|
+
message: `Could not read sub-agent session ${ref.sessionId} while checking credential failures \u2014 skipping it`,
|
|
8757
|
+
conversation_id: conversationId,
|
|
8758
|
+
message_id: evidentMessageId
|
|
8759
|
+
});
|
|
8760
|
+
continue;
|
|
8761
|
+
}
|
|
8762
|
+
const outcome = findSubagentAuthOutcome(childMessages, ref.startedAtMs);
|
|
8763
|
+
if (!outcome) continue;
|
|
8764
|
+
if (outcome.outcome === "failed") {
|
|
8765
|
+
failedProviders.set(outcome.providerId, outcome.failure);
|
|
8766
|
+
} else {
|
|
8767
|
+
succeededProviders.add(outcome.providerId);
|
|
8768
|
+
}
|
|
8769
|
+
} catch (err) {
|
|
8770
|
+
this.log({
|
|
8771
|
+
level: "warn",
|
|
8772
|
+
message: `Failed to inspect sub-agent session ${ref.sessionId} for credential failures (conversation ${conversationId.slice(0, 8)}, message ${evidentMessageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8773
|
+
conversation_id: conversationId,
|
|
8774
|
+
message_id: evidentMessageId
|
|
8775
|
+
});
|
|
8776
|
+
}
|
|
8777
|
+
}
|
|
8778
|
+
for (const [providerId, failure] of failedProviders) {
|
|
8779
|
+
this.log({
|
|
8780
|
+
level: "warn",
|
|
8781
|
+
message: `Sub-agent turn failed on provider ${providerId} (${failure.reason}) \u2014 recording credential evidence`,
|
|
8782
|
+
conversation_id: conversationId,
|
|
8783
|
+
message_id: evidentMessageId
|
|
8784
|
+
});
|
|
8785
|
+
await this.recordSubagentModelAuthFailure(failure, conversationId, evidentMessageId);
|
|
8786
|
+
}
|
|
8787
|
+
for (const providerId of succeededProviders) {
|
|
8788
|
+
if (failedProviders.has(providerId)) continue;
|
|
8789
|
+
await this.clearSubagentModelAuthFailure(providerId, conversationId, evidentMessageId);
|
|
8790
|
+
}
|
|
8791
|
+
}
|
|
8514
8792
|
/**
|
|
8515
8793
|
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
8516
8794
|
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
@@ -9634,6 +9912,10 @@ async function driveChannels(state, driver) {
|
|
|
9634
9912
|
consecutiveDrainFailures = 0;
|
|
9635
9913
|
unreachableMs = 0;
|
|
9636
9914
|
state.messageCount += processed;
|
|
9915
|
+
if (driver.recycleRequested) {
|
|
9916
|
+
await beginGracefulShutdown(state, "recycle");
|
|
9917
|
+
return;
|
|
9918
|
+
}
|
|
9637
9919
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
9638
9920
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
9639
9921
|
const fileActivitySnapshot = driver.fileSyncActivity();
|
|
@@ -10037,7 +10319,8 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
10037
10319
|
});
|
|
10038
10320
|
return;
|
|
10039
10321
|
}
|
|
10040
|
-
const collect = createResourceUsageCollector(homedir5());
|
|
10322
|
+
const { collect, stop } = createResourceUsageCollector(homedir5());
|
|
10323
|
+
state.stopResourceUsageSampling = stop;
|
|
10041
10324
|
let consecutiveFailures = 0;
|
|
10042
10325
|
const tick = async () => {
|
|
10043
10326
|
try {
|
|
@@ -10147,6 +10430,8 @@ async function cleanup(state, opts = {}) {
|
|
|
10147
10430
|
clearTimeout(state.resourceUsageTimer);
|
|
10148
10431
|
state.resourceUsageTimer = null;
|
|
10149
10432
|
}
|
|
10433
|
+
state.stopResourceUsageSampling?.();
|
|
10434
|
+
state.stopResourceUsageSampling = null;
|
|
10150
10435
|
const credentialSync = state.credentialSync;
|
|
10151
10436
|
const flushCredentials = credentialSync ? async (phase, publish) => {
|
|
10152
10437
|
await timeShutdownPhase(state, durations, phase, async () => {
|
|
@@ -10225,6 +10510,44 @@ async function cleanup(state, opts = {}) {
|
|
|
10225
10510
|
}
|
|
10226
10511
|
return durations;
|
|
10227
10512
|
}
|
|
10513
|
+
async function beginGracefulShutdown(state, trigger) {
|
|
10514
|
+
if (state.shuttingDown) return;
|
|
10515
|
+
state.shuttingDown = true;
|
|
10516
|
+
const shutdownStartedAt = Date.now();
|
|
10517
|
+
const shutdownMessage = trigger === "recycle" ? "Recycle requested \u2014 finishing in-flight work, then exiting" : "Shutting down...";
|
|
10518
|
+
if (state.interactive) {
|
|
10519
|
+
logActivity(state, { type: "info", message: shutdownMessage });
|
|
10520
|
+
displayStatus(state);
|
|
10521
|
+
} else {
|
|
10522
|
+
log2(state, shutdownMessage);
|
|
10523
|
+
}
|
|
10524
|
+
const durations = await cleanup(state, { graceful: true });
|
|
10525
|
+
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
10526
|
+
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
10527
|
+
let timer;
|
|
10528
|
+
const flushed = shutdownTelemetry().then(
|
|
10529
|
+
() => true,
|
|
10530
|
+
(error2) => {
|
|
10531
|
+
log2(
|
|
10532
|
+
state,
|
|
10533
|
+
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
10534
|
+
"warn"
|
|
10535
|
+
);
|
|
10536
|
+
return true;
|
|
10537
|
+
}
|
|
10538
|
+
);
|
|
10539
|
+
const timedOut = new Promise((resolve4) => {
|
|
10540
|
+
timer = setTimeout(() => resolve4(false), telemetryBudgetMs);
|
|
10541
|
+
});
|
|
10542
|
+
if (!await Promise.race([flushed, timedOut])) {
|
|
10543
|
+
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
10544
|
+
}
|
|
10545
|
+
clearTimeout(timer);
|
|
10546
|
+
});
|
|
10547
|
+
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
10548
|
+
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
10549
|
+
process.exit(0);
|
|
10550
|
+
}
|
|
10228
10551
|
async function run(options) {
|
|
10229
10552
|
const interactive = isInteractive(options.json);
|
|
10230
10553
|
let logLevel;
|
|
@@ -10276,6 +10599,7 @@ async function run(options) {
|
|
|
10276
10599
|
openaiUsageTimer: null,
|
|
10277
10600
|
openaiUsageRearm: null,
|
|
10278
10601
|
resourceUsageTimer: null,
|
|
10602
|
+
stopResourceUsageSampling: null,
|
|
10279
10603
|
credentialSync: null,
|
|
10280
10604
|
authHeader: ""
|
|
10281
10605
|
};
|
|
@@ -10318,43 +10642,7 @@ async function run(options) {
|
|
|
10318
10642
|
"warn"
|
|
10319
10643
|
);
|
|
10320
10644
|
}
|
|
10321
|
-
const handleSignal =
|
|
10322
|
-
if (state.shuttingDown) return;
|
|
10323
|
-
state.shuttingDown = true;
|
|
10324
|
-
const shutdownStartedAt = Date.now();
|
|
10325
|
-
if (state.interactive) {
|
|
10326
|
-
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
10327
|
-
displayStatus(state);
|
|
10328
|
-
} else {
|
|
10329
|
-
log2(state, "Shutting down...");
|
|
10330
|
-
}
|
|
10331
|
-
const durations = await cleanup(state, { graceful: true });
|
|
10332
|
-
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
10333
|
-
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
10334
|
-
let timer;
|
|
10335
|
-
const flushed = shutdownTelemetry().then(
|
|
10336
|
-
() => true,
|
|
10337
|
-
(error2) => {
|
|
10338
|
-
log2(
|
|
10339
|
-
state,
|
|
10340
|
-
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
10341
|
-
"warn"
|
|
10342
|
-
);
|
|
10343
|
-
return true;
|
|
10344
|
-
}
|
|
10345
|
-
);
|
|
10346
|
-
const timedOut = new Promise((resolve4) => {
|
|
10347
|
-
timer = setTimeout(() => resolve4(false), telemetryBudgetMs);
|
|
10348
|
-
});
|
|
10349
|
-
if (!await Promise.race([flushed, timedOut])) {
|
|
10350
|
-
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
10351
|
-
}
|
|
10352
|
-
clearTimeout(timer);
|
|
10353
|
-
});
|
|
10354
|
-
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
10355
|
-
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
10356
|
-
process.exit(0);
|
|
10357
|
-
};
|
|
10645
|
+
const handleSignal = () => beginGracefulShutdown(state, "signal");
|
|
10358
10646
|
process.on("SIGINT", handleSignal);
|
|
10359
10647
|
process.on("SIGTERM", handleSignal);
|
|
10360
10648
|
try {
|
|
@@ -10872,6 +11160,18 @@ async function run(options) {
|
|
|
10872
11160
|
if (state.interactive) displayStatus(state);
|
|
10873
11161
|
});
|
|
10874
11162
|
},
|
|
11163
|
+
// Both loops are rearmed because `rearm()` is idempotent for the
|
|
11164
|
+
// provider that did not just connect, and is a no-op when reporting is off.
|
|
11165
|
+
onUsageRearmPing: () => {
|
|
11166
|
+
if (!state.running) return;
|
|
11167
|
+
logActivity(state, {
|
|
11168
|
+
type: "info",
|
|
11169
|
+
level: "debug",
|
|
11170
|
+
message: "Usage rearm ping received"
|
|
11171
|
+
});
|
|
11172
|
+
state.claudeUsageRearm?.();
|
|
11173
|
+
state.openaiUsageRearm?.();
|
|
11174
|
+
},
|
|
10875
11175
|
onInfo: (message) => logActivity(state, { type: "info", message })
|
|
10876
11176
|
}
|
|
10877
11177
|
});
|
|
@@ -10892,7 +11192,17 @@ async function run(options) {
|
|
|
10892
11192
|
setTimer: (timer) => {
|
|
10893
11193
|
state.openaiUsageTimer = timer;
|
|
10894
11194
|
},
|
|
10895
|
-
fetchUsage: () =>
|
|
11195
|
+
fetchUsage: async () => {
|
|
11196
|
+
const usage = await getOpenAiUsage(state.port);
|
|
11197
|
+
if (usage.subscription === null) {
|
|
11198
|
+
logActivity(state, {
|
|
11199
|
+
type: "info",
|
|
11200
|
+
level: "debug",
|
|
11201
|
+
message: "OpenAI usage subscription could not be identified from the local credential"
|
|
11202
|
+
});
|
|
11203
|
+
}
|
|
11204
|
+
return usage;
|
|
11205
|
+
},
|
|
10896
11206
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
10897
11207
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
10898
11208
|
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|