@vtxmacro/cli 2026.8.44 → 2026.8.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/vtx.js +624 -185
- package/package.json +1 -1
package/bin/vtx.js
CHANGED
|
@@ -38,7 +38,7 @@ var init_agent_cli_release = __esm({
|
|
|
38
38
|
"agent-cli-release.json"() {
|
|
39
39
|
agent_cli_release_default = {
|
|
40
40
|
package_name: "@vtxmacro/cli",
|
|
41
|
-
package_version: "2026.8.
|
|
41
|
+
package_version: "2026.8.46",
|
|
42
42
|
codex_package_name: "@openai/codex",
|
|
43
43
|
codex_version: "0.147.0",
|
|
44
44
|
copilot_sdk_package_name: "@github/copilot-sdk",
|
|
@@ -14379,7 +14379,7 @@ var init_zod = __esm({
|
|
|
14379
14379
|
});
|
|
14380
14380
|
|
|
14381
14381
|
// lib/external-inference-contract.ts
|
|
14382
|
-
var EXTERNAL_INFERENCE_CONTRACT_VERSION, EXTERNAL_INFERENCE_CONTROLLER_SCHEMA_VERSION, EXTERNAL_INFERENCE_ADVERTISEMENT_SCHEMA_VERSION, EXTERNAL_INFERENCE_ENVELOPE_SCHEMA_VERSION, EXTERNAL_INFERENCE_JOB_INPUT_SCHEMA_VERSION, LEGACY_EXTERNAL_INFERENCE_JOB_INPUT_SCHEMA_VERSION, EXTERNAL_INFERENCE_USAGE_SCHEMA_VERSION, EXTERNAL_INFERENCE_INPUT_HASH_DOMAIN, MAX_EXTERNAL_INFERENCE_STATUS_HOSTS, MAX_GENERATION, MAX_SAFE_INTEGER, MAX_PROMPT_BYTES, MAX_IMMUTABLE_JOB_INPUT_BYTES, MAX_CANDIDATE_PLAINTEXT_BYTES, MAX_CIPHERTEXT_CHARS, SHA256_INITIAL_STATE, SHA256_ROUND_CONSTANTS, rotateRight, sha256BytesHex, sha256Utf8Hex, computeImmutableJobInputSha256, identifierSchema, operationIdentifierSchema, sha256HexSchema, generationSchema, zeroBasedGenerationSchema, nonNegativeSafeIntegerSchema, positiveSafeIntegerSchema, sqlIntegerTokenCountSchema, timestampSchema, safeCodeSchema, externalInferenceAdapterIdSchema, reasoningEffortSchema, modelNameSchema, displayNameSchema, authenticatedAccountEmailSchema, protocolVersionSchema, base64UrlAlphabet, isCanonicalBase64Url, base64Url32BytesSchema, base64UrlNonceSchema, base64UrlCiphertextSchema, claimHandleSchema, boundedJsonTextSchema, boundedPromptTextSchema, laneSchema, botModeSchema, executionModeSchema, responseModeSchema, dispatchOutcomeSchema, responseOutcomeSchema, nativeModelControllerSchema, externalAgentModelControllerSchema, externalAgentUnboundControllerSchema, orderedFallbackHostIdsSchema, externalInferenceFallbackHostIdsSchema, liveModelControllerSchema, snapshotModelControllerSchema, modelIdentityMappingSchema, advertisedModelSchema, hostAdvertisementSchema, envelopeAadSchema, validateEnvelopeAad, commonEnvelopeShape, sealedKeyWrapEnvelopeSchema, sealedInputEnvelopeSchema, sealedCandidateEnvelopeSchema, sealedEnvelopeSchema, screenerCandidateRuntimeBindingSchema, screenerClientLeaseBindingSchema, screenerRuntimeBindingSchema, serverRuntimeBindingSchema, clientRuntimeBindingSchema, runtimeBindingSchema, immutableJobInputSchema, canonicalizeJsonValue, serializeImmutableJobInput, reportedTokenUsageSchema, unavailableTokenUsageSchema, tokenUsageSchema, agentConnectRequestSchema, agentNextRequestSchema, agentHeartbeatRequestSchema, emptyAgentNextResultSchema, claimedAgentNextResultSchema, agentNextResultSchema, agentCompleteRequestSchema, agentFailRequestSchema, requestedModelSelectionSchema, completedModelProvenanceSchema, attemptModelProvenanceSchema, providerWeeklyQuotaReadSchema, hostHeartbeatRequestSchema, jobClaimRequestSchema, emptyJobClaimSchema, claimedJobSchema, jobClaimResultSchema, attemptStartRequestSchema, jobHeartbeatRequestSchema, continueHeartbeatDirectiveSchema, cancelHeartbeatDirectiveSchema, jobHeartbeatResultSchema, adapterOutcomeEvidenceSchema, completionBaseShape, validateCompletion, providerResponseCompletionSchema, decisionCandidateCompletionSchema, jobCompleteRequestSchema, jobFailRequestSchema, jobCancelRequestSchema, jobCancellationReceiptSchema, jobDeliveryRequestSchema, deliveryBaseShape, validateDeliveryCandidate, providerResponseDeliverySchema, decisionCandidateDeliverySchema, jobDeliveryResultSchema, jobConsumptionRequestSchema, jobConsumptionReceiptSchema, jobStateSchema, jobStatusReadSchema, hostStatusReadSchema, controllerBindingStatusReadSchema, queueStatusReadSchema, outcomeStatusReadSchema, externalInferenceVolumeWindowReadSchema, externalInferenceFailureCountReadSchema, externalInferenceHostDiagnosticsReadSchema, externalInferenceProfileActivityReadSchema, externalInferenceStatusReadSchema, nativeResearchEvidenceSourceSchema, externalAgentResearchEvidenceSourceSchema, researchEvidenceSourceSchema, reportedResearchTokenUsageSchema, unavailableResearchTokenUsageSchema, researchTokenUsageSchema, externalInferenceProductLineageBaseShape, traderExternalInferenceProductLineageSchema, decisionChatExternalInferenceProductLineageSchema, screenerExternalInferenceProductLineageSchema, externalInferenceProductLineageSchema, normalProductLinkageSchema, evidenceMaterializationMarkerSchema, deliveredExternalAgentSourceIdentitySchema, deliveredRequestEvidenceSchema, researchEvidenceBaseShape, researchEvidenceBaseSchema, validateExactUtf8, validateResearchEvidence, providerResponseResearchEvidenceSchema, decisionCandidateResearchEvidenceSchema, researchEvidenceSchema;
|
|
14382
|
+
var EXTERNAL_INFERENCE_CONTRACT_VERSION, EXTERNAL_INFERENCE_CONTROLLER_SCHEMA_VERSION, EXTERNAL_INFERENCE_ADVERTISEMENT_SCHEMA_VERSION, EXTERNAL_INFERENCE_ENVELOPE_SCHEMA_VERSION, EXTERNAL_INFERENCE_JOB_INPUT_SCHEMA_VERSION, LEGACY_EXTERNAL_INFERENCE_JOB_INPUT_SCHEMA_VERSION, EXTERNAL_INFERENCE_USAGE_SCHEMA_VERSION, EXTERNAL_INFERENCE_INPUT_HASH_DOMAIN, MAX_EXTERNAL_INFERENCE_STATUS_HOSTS, MAX_GENERATION, MAX_SAFE_INTEGER, MAX_PROMPT_BYTES, MAX_IMMUTABLE_JOB_INPUT_BYTES, MAX_CANDIDATE_PLAINTEXT_BYTES, MAX_CIPHERTEXT_CHARS, SHA256_INITIAL_STATE, SHA256_ROUND_CONSTANTS, rotateRight, sha256BytesHex, sha256Utf8Hex, computeImmutableJobInputSha256, identifierSchema, operationIdentifierSchema, sha256HexSchema, generationSchema, zeroBasedGenerationSchema, nonNegativeSafeIntegerSchema, positiveSafeIntegerSchema, sqlIntegerTokenCountSchema, timestampSchema, safeCodeSchema, processExitCodeSchema, externalInferenceAdapterIdSchema, reasoningEffortSchema, modelNameSchema, displayNameSchema, authenticatedAccountEmailSchema, protocolVersionSchema, base64UrlAlphabet, isCanonicalBase64Url, base64Url32BytesSchema, base64UrlNonceSchema, base64UrlCiphertextSchema, claimHandleSchema, boundedJsonTextSchema, boundedPromptTextSchema, laneSchema, botModeSchema, executionModeSchema, responseModeSchema, dispatchOutcomeSchema, responseOutcomeSchema, nativeModelControllerSchema, externalAgentModelControllerSchema, externalAgentUnboundControllerSchema, orderedFallbackHostIdsSchema, externalInferenceFallbackHostIdsSchema, liveModelControllerSchema, snapshotModelControllerSchema, modelIdentityMappingSchema, advertisedModelSchema, hostAdvertisementSchema, envelopeAadSchema, validateEnvelopeAad, commonEnvelopeShape, sealedKeyWrapEnvelopeSchema, sealedInputEnvelopeSchema, sealedCandidateEnvelopeSchema, sealedEnvelopeSchema, screenerCandidateRuntimeBindingSchema, screenerClientLeaseBindingSchema, screenerRuntimeBindingSchema, serverRuntimeBindingSchema, clientRuntimeBindingSchema, runtimeBindingSchema, immutableJobInputSchema, canonicalizeJsonValue, serializeImmutableJobInput, reportedTokenUsageSchema, unavailableTokenUsageSchema, tokenUsageSchema, agentConnectRequestSchema, agentNextRequestSchema, agentHeartbeatRequestSchema, emptyAgentNextResultSchema, claimedAgentNextResultSchema, agentNextResultSchema, agentCompleteRequestSchema, agentFailRequestSchema, requestedModelSelectionSchema, completedModelProvenanceSchema, attemptModelProvenanceSchema, providerWeeklyQuotaReadSchema, hostHeartbeatRequestSchema, jobClaimRequestSchema, emptyJobClaimSchema, claimedJobSchema, jobClaimResultSchema, attemptStartRequestSchema, jobHeartbeatRequestSchema, continueHeartbeatDirectiveSchema, cancelHeartbeatDirectiveSchema, jobHeartbeatResultSchema, adapterOutcomeEvidenceSchema, completionBaseShape, validateCompletion, providerResponseCompletionSchema, decisionCandidateCompletionSchema, jobCompleteRequestSchema, codexProcessExitDiagnosticsSchema, jobFailRequestSchema, jobCancelRequestSchema, jobCancellationReceiptSchema, jobDeliveryRequestSchema, deliveryBaseShape, validateDeliveryCandidate, providerResponseDeliverySchema, decisionCandidateDeliverySchema, jobDeliveryResultSchema, jobConsumptionRequestSchema, jobConsumptionReceiptSchema, jobStateSchema, jobStatusReadSchema, hostStatusReadSchema, controllerBindingStatusReadSchema, queueStatusReadSchema, outcomeStatusReadSchema, externalInferenceVolumeWindowReadSchema, externalInferenceFailureCountReadSchema, externalInferenceHostDiagnosticsReadSchema, externalInferenceProfileActivityReadSchema, externalInferenceStatusReadSchema, nativeResearchEvidenceSourceSchema, externalAgentResearchEvidenceSourceSchema, researchEvidenceSourceSchema, reportedResearchTokenUsageSchema, unavailableResearchTokenUsageSchema, researchTokenUsageSchema, externalInferenceProductLineageBaseShape, traderExternalInferenceProductLineageSchema, decisionChatExternalInferenceProductLineageSchema, screenerExternalInferenceProductLineageSchema, externalInferenceProductLineageSchema, normalProductLinkageSchema, evidenceMaterializationMarkerSchema, deliveredExternalAgentSourceIdentitySchema, deliveredRequestEvidenceSchema, researchEvidenceBaseShape, researchEvidenceBaseSchema, validateExactUtf8, validateResearchEvidence, providerResponseResearchEvidenceSchema, decisionCandidateResearchEvidenceSchema, researchEvidenceSchema;
|
|
14383
14383
|
var init_external_inference_contract = __esm({
|
|
14384
14384
|
"lib/external-inference-contract.ts"() {
|
|
14385
14385
|
"use strict";
|
|
@@ -14582,6 +14582,7 @@ var init_external_inference_contract = __esm({
|
|
|
14582
14582
|
sqlIntegerTokenCountSchema = external_exports.number().int().min(0).max(2147483647);
|
|
14583
14583
|
timestampSchema = external_exports.iso.datetime({ offset: true });
|
|
14584
14584
|
safeCodeSchema = external_exports.string().min(1).max(96).regex(/^[a-z0-9][a-z0-9._-]{0,95}$/);
|
|
14585
|
+
processExitCodeSchema = external_exports.number().int().min(-(2 ** 31)).max(2 ** 31 - 1);
|
|
14585
14586
|
externalInferenceAdapterIdSchema = external_exports.string().min(1).max(64).regex(/^[a-z0-9][a-z0-9._-]{0,63}$/);
|
|
14586
14587
|
reasoningEffortSchema = external_exports.string().min(1).max(32).regex(/^[a-z][a-z0-9_-]{0,31}$/);
|
|
14587
14588
|
modelNameSchema = external_exports.string().min(1).max(256);
|
|
@@ -15435,6 +15436,55 @@ var init_external_inference_contract = __esm({
|
|
|
15435
15436
|
providerResponseCompletionSchema,
|
|
15436
15437
|
decisionCandidateCompletionSchema
|
|
15437
15438
|
]);
|
|
15439
|
+
codexProcessExitDiagnosticsSchema = external_exports.strictObject({
|
|
15440
|
+
schema_version: external_exports.literal("external_inference_process_exit_v1"),
|
|
15441
|
+
runtime: external_exports.literal("codex_app_server"),
|
|
15442
|
+
source: external_exports.enum(["child_receipt", "direct_process", "guardian_process"]),
|
|
15443
|
+
session_generation: generationSchema,
|
|
15444
|
+
expected_shutdown: external_exports.boolean(),
|
|
15445
|
+
exit_class: external_exports.enum(["zero", "nonzero", "signal", "unknown"]),
|
|
15446
|
+
exit_code: processExitCodeSchema.nullable(),
|
|
15447
|
+
signal: safeCodeSchema.nullable(),
|
|
15448
|
+
uptime_ms: nonNegativeSafeIntegerSchema,
|
|
15449
|
+
pending_request_count: nonNegativeSafeIntegerSchema.nullable(),
|
|
15450
|
+
active_thread_count: nonNegativeSafeIntegerSchema.nullable(),
|
|
15451
|
+
stderr_nonempty: external_exports.boolean().nullable(),
|
|
15452
|
+
stderr_bytes_observed: external_exports.number().int().min(0).max(65536).nullable(),
|
|
15453
|
+
stderr_truncated: external_exports.boolean().nullable(),
|
|
15454
|
+
stderr_class: external_exports.enum([
|
|
15455
|
+
"unknown",
|
|
15456
|
+
"none",
|
|
15457
|
+
"out_of_memory",
|
|
15458
|
+
"runtime_fatal",
|
|
15459
|
+
"memory_fault",
|
|
15460
|
+
"pipe_or_connection",
|
|
15461
|
+
"unclassified"
|
|
15462
|
+
])
|
|
15463
|
+
}).superRefine((value, context) => {
|
|
15464
|
+
const expectedExitClass = value.signal !== null ? "signal" : value.exit_code === null ? "unknown" : value.exit_code === 0 ? "zero" : "nonzero";
|
|
15465
|
+
if (value.exit_class !== expectedExitClass) {
|
|
15466
|
+
context.addIssue({ code: "custom", message: "process exit class does not match code and signal", path: ["exit_class"] });
|
|
15467
|
+
}
|
|
15468
|
+
if (value.signal !== null && value.exit_code !== null) {
|
|
15469
|
+
context.addIssue({ code: "custom", message: "signal exits cannot also report an exit code", path: ["exit_code"] });
|
|
15470
|
+
}
|
|
15471
|
+
if (value.stderr_class === "unknown") {
|
|
15472
|
+
if (value.stderr_nonempty !== null || value.stderr_bytes_observed !== null || value.stderr_truncated !== null) {
|
|
15473
|
+
context.addIssue({ code: "custom", message: "unknown stderr diagnostics must not infer observations", path: ["stderr_class"] });
|
|
15474
|
+
}
|
|
15475
|
+
} else if (value.stderr_nonempty === null || value.stderr_bytes_observed === null || value.stderr_truncated === null) {
|
|
15476
|
+
context.addIssue({ code: "custom", message: "observed stderr diagnostics must be complete", path: ["stderr_class"] });
|
|
15477
|
+
} else if (!value.stderr_nonempty) {
|
|
15478
|
+
if (value.stderr_bytes_observed !== 0 || value.stderr_truncated || value.stderr_class !== "none") {
|
|
15479
|
+
context.addIssue({ code: "custom", message: "empty stderr diagnostics are inconsistent", path: ["stderr_nonempty"] });
|
|
15480
|
+
}
|
|
15481
|
+
} else if (value.stderr_bytes_observed === 0 || value.stderr_class === "none") {
|
|
15482
|
+
context.addIssue({ code: "custom", message: "non-empty stderr diagnostics are inconsistent", path: ["stderr_nonempty"] });
|
|
15483
|
+
}
|
|
15484
|
+
if (value.stderr_truncated && value.stderr_bytes_observed !== 65536) {
|
|
15485
|
+
context.addIssue({ code: "custom", message: "truncated stderr diagnostics must report the observation cap", path: ["stderr_truncated"] });
|
|
15486
|
+
}
|
|
15487
|
+
});
|
|
15438
15488
|
jobFailRequestSchema = external_exports.strictObject({
|
|
15439
15489
|
schema_version: external_exports.literal("external_inference_job_fail_v1"),
|
|
15440
15490
|
contract_version: external_exports.literal(EXTERNAL_INFERENCE_CONTRACT_VERSION),
|
|
@@ -15457,6 +15507,7 @@ var init_external_inference_contract = __esm({
|
|
|
15457
15507
|
failure_category: safeCodeSchema,
|
|
15458
15508
|
failure_code: safeCodeSchema,
|
|
15459
15509
|
retryable: external_exports.boolean(),
|
|
15510
|
+
process_exit: codexProcessExitDiagnosticsSchema.nullable().optional(),
|
|
15460
15511
|
membership_disposition: external_exports.enum([
|
|
15461
15512
|
"retry_same_host",
|
|
15462
15513
|
"cascade_cooldown",
|
|
@@ -15473,6 +15524,13 @@ var init_external_inference_contract = __esm({
|
|
|
15473
15524
|
if (value.attempt_status !== expectedStatus) {
|
|
15474
15525
|
context.addIssue({ code: "custom", message: "attempt_status must match dispatch outcome", path: ["attempt_status"] });
|
|
15475
15526
|
}
|
|
15527
|
+
if (value.process_exit != null && (value.failure_category !== "transport" || value.failure_code !== "codex_process_closed" || value.process_exit.expected_shutdown)) {
|
|
15528
|
+
context.addIssue({
|
|
15529
|
+
code: "custom",
|
|
15530
|
+
message: "process exit diagnostics require an unexpected Codex process-close failure",
|
|
15531
|
+
path: ["process_exit"]
|
|
15532
|
+
});
|
|
15533
|
+
}
|
|
15476
15534
|
if (value.outcome.dispatch_outcome === "outcome_unknown" && value.membership_disposition !== void 0 && value.membership_disposition !== "quarantine_ambiguous") {
|
|
15477
15535
|
context.addIssue({ code: "custom", message: "outcome-unknown failures must be quarantined", path: ["membership_disposition"] });
|
|
15478
15536
|
}
|
|
@@ -19593,6 +19651,7 @@ var init_codex_app_server = __esm({
|
|
|
19593
19651
|
this.retryAtMs = options.retryAtMs ?? null;
|
|
19594
19652
|
this.usage = options.usage ?? null;
|
|
19595
19653
|
this.rateLimits = options.rateLimits ?? null;
|
|
19654
|
+
this.processExit = options.processExit ?? null;
|
|
19596
19655
|
}
|
|
19597
19656
|
};
|
|
19598
19657
|
objectOrNull = (value) => value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
@@ -19897,7 +19956,7 @@ var init_codex_app_server = __esm({
|
|
|
19897
19956
|
};
|
|
19898
19957
|
GUARDIAN_SCRIPT = String.raw`
|
|
19899
19958
|
import { spawn } from 'node:child_process';
|
|
19900
|
-
import { chmod, open, rename, rm, writeFile } from 'node:fs/promises';
|
|
19959
|
+
import { chmod, open, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
19901
19960
|
import { dirname } from 'node:path';
|
|
19902
19961
|
const input = JSON.parse(Buffer.from(process.argv[1], 'base64url').toString('utf8'));
|
|
19903
19962
|
// The guardian owns the process boundary for every attempt. Apply a private
|
|
@@ -19927,7 +19986,7 @@ const replaceReceipt = async (temporary) => {
|
|
|
19927
19986
|
};
|
|
19928
19987
|
let receiptWriteSequence = 0;
|
|
19929
19988
|
let receiptWriteChain = Promise.resolve();
|
|
19930
|
-
const writeReceipt = (state, childPid) => {
|
|
19989
|
+
const writeReceipt = (state, childPid, termination = null) => {
|
|
19931
19990
|
const write = async () => {
|
|
19932
19991
|
const body = JSON.stringify({
|
|
19933
19992
|
schema_version: 'vtx_codex_guardian_v1',
|
|
@@ -19936,6 +19995,7 @@ const writeReceipt = (state, childPid) => {
|
|
|
19936
19995
|
guardian_pid: process.pid,
|
|
19937
19996
|
child_pid: childPid ?? null,
|
|
19938
19997
|
boot_identity: input.bootIdentity,
|
|
19998
|
+
termination,
|
|
19939
19999
|
updated_at: new Date().toISOString(),
|
|
19940
20000
|
}) + '\n';
|
|
19941
20001
|
receiptWriteSequence += 1;
|
|
@@ -19960,8 +20020,11 @@ const writeReceipt = (state, childPid) => {
|
|
|
19960
20020
|
};
|
|
19961
20021
|
let child = null;
|
|
19962
20022
|
let shuttingDown = false;
|
|
20023
|
+
let shutdownRequested = false;
|
|
20024
|
+
let childStartedAt = null;
|
|
19963
20025
|
let forceTimer = null;
|
|
19964
|
-
const
|
|
20026
|
+
const stopChild = (expected) => {
|
|
20027
|
+
if (expected) shutdownRequested = true;
|
|
19965
20028
|
if (shuttingDown) return;
|
|
19966
20029
|
shuttingDown = true;
|
|
19967
20030
|
if (!child) return;
|
|
@@ -19971,6 +20034,19 @@ const shutdown = () => {
|
|
|
19971
20034
|
}, input.shutdownGraceMs);
|
|
19972
20035
|
forceTimer.unref();
|
|
19973
20036
|
};
|
|
20037
|
+
let shutdownStarted = false;
|
|
20038
|
+
const shutdown = async () => {
|
|
20039
|
+
if (shutdownStarted) return;
|
|
20040
|
+
shutdownStarted = true;
|
|
20041
|
+
let expected = false;
|
|
20042
|
+
try {
|
|
20043
|
+
expected = (await readFile(input.shutdownIntentPath, 'utf8')) === input.processToken;
|
|
20044
|
+
await rm(input.shutdownIntentPath, { force: true });
|
|
20045
|
+
} catch {
|
|
20046
|
+
expected = false;
|
|
20047
|
+
}
|
|
20048
|
+
stopChild(expected);
|
|
20049
|
+
};
|
|
19974
20050
|
process.stdin.once('end', shutdown);
|
|
19975
20051
|
process.stdin.once('close', shutdown);
|
|
19976
20052
|
process.stdout.on('error', () => undefined);
|
|
@@ -19986,6 +20062,7 @@ child = spawn(input.binary, input.args, {
|
|
|
19986
20062
|
windowsHide: true,
|
|
19987
20063
|
detached: false,
|
|
19988
20064
|
});
|
|
20065
|
+
childStartedAt = Date.now();
|
|
19989
20066
|
child.stdin.on('error', () => undefined);
|
|
19990
20067
|
process.stdin.pipe(child.stdin, { end: false });
|
|
19991
20068
|
child.stdout.pipe(process.stdout);
|
|
@@ -19994,18 +20071,33 @@ child.once('spawn', async () => {
|
|
|
19994
20071
|
try {
|
|
19995
20072
|
await writeReceipt('running', child.pid ?? null);
|
|
19996
20073
|
} catch {
|
|
19997
|
-
|
|
20074
|
+
stopChild(false);
|
|
19998
20075
|
}
|
|
19999
20076
|
});
|
|
20000
|
-
child.once('error',
|
|
20001
|
-
child.once('close', async () => {
|
|
20077
|
+
child.once('error', () => stopChild(false));
|
|
20078
|
+
child.once('close', async (code, signal) => {
|
|
20002
20079
|
if (forceTimer) clearTimeout(forceTimer);
|
|
20003
20080
|
// A pending promise alone does not keep Node's event loop alive after the
|
|
20004
20081
|
// child and parent stdin have both closed. Hold one local handle until the
|
|
20005
20082
|
// serialized terminal tombstone is durably replaced.
|
|
20006
20083
|
const receiptKeepAlive = setInterval(() => undefined, 1_000);
|
|
20007
20084
|
try {
|
|
20008
|
-
|
|
20085
|
+
const normalizedSignal = typeof signal === 'string' && /^SIG[A-Z0-9]{1,28}$/.test(signal)
|
|
20086
|
+
? signal.toLowerCase()
|
|
20087
|
+
: null;
|
|
20088
|
+
const normalizedCode = normalizedSignal === null && Number.isInteger(code)
|
|
20089
|
+
&& code >= -(2 ** 31) && code <= (2 ** 31) - 1
|
|
20090
|
+
? code
|
|
20091
|
+
: null;
|
|
20092
|
+
await writeReceipt('terminated', child.pid ?? null, {
|
|
20093
|
+
exit_code: normalizedCode,
|
|
20094
|
+
signal: normalizedSignal,
|
|
20095
|
+
expected: shutdownRequested,
|
|
20096
|
+
uptime_ms: Math.min(
|
|
20097
|
+
Math.max(0, Date.now() - (childStartedAt ?? Date.now())),
|
|
20098
|
+
Number.MAX_SAFE_INTEGER,
|
|
20099
|
+
),
|
|
20100
|
+
});
|
|
20009
20101
|
} finally {
|
|
20010
20102
|
clearInterval(receiptKeepAlive);
|
|
20011
20103
|
process.exit(0);
|
|
@@ -20057,6 +20149,7 @@ child.once('close', async () => {
|
|
|
20057
20149
|
guardian_pid: null,
|
|
20058
20150
|
child_pid: null,
|
|
20059
20151
|
boot_identity: guardian.bootIdentity ?? null,
|
|
20152
|
+
termination: null,
|
|
20060
20153
|
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
20061
20154
|
})}
|
|
20062
20155
|
`;
|
|
@@ -20081,12 +20174,15 @@ child.once('close', async () => {
|
|
|
20081
20174
|
};
|
|
20082
20175
|
parseGuardianReceipt = (raw) => {
|
|
20083
20176
|
const value = JSON.parse(raw);
|
|
20084
|
-
|
|
20177
|
+
const rawTermination = value.termination;
|
|
20178
|
+
const termination = rawTermination === null || rawTermination === void 0 ? null : objectOrNull(rawTermination);
|
|
20179
|
+
if (value.schema_version !== "vtx_codex_guardian_v1" || typeof value.process_token !== "string" || !/^[0-9a-f]{64}$/u.test(value.process_token) || !["spawn_intent", "starting", "running", "terminated"].includes(String(value.state)) || value.guardian_pid !== null && !Number.isSafeInteger(value.guardian_pid) || value.state === "spawn_intent" && value.guardian_pid !== null || value.state !== "spawn_intent" && !Number.isSafeInteger(value.guardian_pid) || value.child_pid !== null && !Number.isSafeInteger(value.child_pid) || value.boot_identity !== null && value.boot_identity !== void 0 && typeof value.boot_identity !== "string" || typeof value.updated_at !== "string" || !Number.isFinite(Date.parse(value.updated_at)) || rawTermination !== null && rawTermination !== void 0 && termination === null || termination !== null && (value.state !== "terminated" || termination.exit_code !== null && (!Number.isInteger(termination.exit_code) || Number(termination.exit_code) < -(2 ** 31) || Number(termination.exit_code) > 2 ** 31 - 1) || termination.signal !== null && (typeof termination.signal !== "string" || !/^sig[a-z0-9]{1,28}$/u.test(termination.signal)) || termination.signal !== null && termination.exit_code !== null || typeof termination.expected !== "boolean" || !Number.isSafeInteger(termination.uptime_ms) || Number(termination.uptime_ms) < 0)) {
|
|
20085
20180
|
throw new Error("Codex guardian receipt is invalid.");
|
|
20086
20181
|
}
|
|
20087
20182
|
return {
|
|
20088
20183
|
...value,
|
|
20089
|
-
boot_identity: typeof value.boot_identity === "string" ? value.boot_identity : null
|
|
20184
|
+
boot_identity: typeof value.boot_identity === "string" ? value.boot_identity : null,
|
|
20185
|
+
termination
|
|
20090
20186
|
};
|
|
20091
20187
|
};
|
|
20092
20188
|
readCodexGuardianReceipt = async (path) => {
|
|
@@ -20130,6 +20226,7 @@ child.once('close', async () => {
|
|
|
20130
20226
|
args,
|
|
20131
20227
|
processToken: options.guardian.processToken,
|
|
20132
20228
|
receiptPath: options.guardian.receiptPath,
|
|
20229
|
+
shutdownIntentPath: `${options.guardian.receiptPath}.shutdown-intent`,
|
|
20133
20230
|
shutdownGraceMs: Math.max(25, options.guardian.shutdownGraceMs ?? 2e3),
|
|
20134
20231
|
bootIdentity: options.guardian.bootIdentity
|
|
20135
20232
|
}), "utf8").toString("base64url");
|
|
@@ -20151,13 +20248,33 @@ child.once('close', async () => {
|
|
|
20151
20248
|
this.nextRequestId = 1;
|
|
20152
20249
|
this.fatalError = null;
|
|
20153
20250
|
this.closed = false;
|
|
20251
|
+
this.processCloseObserved = false;
|
|
20252
|
+
this.shutdownRequested = false;
|
|
20253
|
+
this.stderrBytesObserved = 0;
|
|
20254
|
+
this.stderrNonempty = false;
|
|
20255
|
+
this.stderrTruncated = false;
|
|
20256
|
+
this.stderrTail = "";
|
|
20257
|
+
this.stderrClass = "none";
|
|
20154
20258
|
this.process = options.process;
|
|
20155
20259
|
this.codexHome = options.codexHome;
|
|
20156
|
-
this.
|
|
20260
|
+
this.sessionGeneration = options.sessionGeneration;
|
|
20261
|
+
this.startedAtMs = Date.now();
|
|
20262
|
+
this.guardian = options.guardian;
|
|
20263
|
+
this.onProcessExit = options.onProcessExit;
|
|
20264
|
+
this.process.stderr.on("data", (chunk) => this.observeStderr(chunk));
|
|
20157
20265
|
const lines = createInterface({ input: this.process.stdout, crlfDelay: Infinity });
|
|
20158
20266
|
lines.on("line", (line) => this.handleLine(line));
|
|
20159
|
-
this.process.once("error", () =>
|
|
20160
|
-
|
|
20267
|
+
this.process.once("error", () => {
|
|
20268
|
+
});
|
|
20269
|
+
this.process.once("close", (code, signal) => {
|
|
20270
|
+
void this.handleProcessClose(code, signal);
|
|
20271
|
+
});
|
|
20272
|
+
}
|
|
20273
|
+
static {
|
|
20274
|
+
this.STDERR_OBSERVATION_CAP_BYTES = 65536;
|
|
20275
|
+
}
|
|
20276
|
+
static {
|
|
20277
|
+
this.STDERR_CLASSIFICATION_TAIL_CHARS = 1024;
|
|
20161
20278
|
}
|
|
20162
20279
|
static async start(options) {
|
|
20163
20280
|
const spawnProcess = options.spawnProcess ?? defaultSpawn;
|
|
@@ -20218,7 +20335,13 @@ child.once('close', async () => {
|
|
|
20218
20335
|
});
|
|
20219
20336
|
}
|
|
20220
20337
|
}
|
|
20221
|
-
const session = new _CodexAppServerSession({
|
|
20338
|
+
const session = new _CodexAppServerSession({
|
|
20339
|
+
process: child,
|
|
20340
|
+
codexHome: options.codexHome,
|
|
20341
|
+
sessionGeneration: options.sessionGeneration ?? 1,
|
|
20342
|
+
guardian: guardian ?? null,
|
|
20343
|
+
onProcessExit: options.onProcessExit ?? null
|
|
20344
|
+
});
|
|
20222
20345
|
try {
|
|
20223
20346
|
const initialized = await session.request("initialize", {
|
|
20224
20347
|
clientInfo: {
|
|
@@ -20333,13 +20456,14 @@ child.once('close', async () => {
|
|
|
20333
20456
|
}
|
|
20334
20457
|
for (const listener of this.notificationListeners) listener({ method, params });
|
|
20335
20458
|
}
|
|
20336
|
-
failTransport(code) {
|
|
20459
|
+
failTransport(code, processExit = null) {
|
|
20337
20460
|
if (this.fatalError) return;
|
|
20338
20461
|
this.fatalError = new CodexAppServerError({
|
|
20339
20462
|
message: "Codex app-server transport became unavailable.",
|
|
20340
20463
|
category: "transport",
|
|
20341
20464
|
code,
|
|
20342
|
-
retryable: true
|
|
20465
|
+
retryable: true,
|
|
20466
|
+
processExit
|
|
20343
20467
|
});
|
|
20344
20468
|
for (const pending of this.pending.values()) {
|
|
20345
20469
|
if (pending.timer) clearTimeout(pending.timer);
|
|
@@ -20440,6 +20564,77 @@ child.once('close', async () => {
|
|
|
20440
20564
|
this.fatalListeners.add(listener);
|
|
20441
20565
|
return () => this.fatalListeners.delete(listener);
|
|
20442
20566
|
}
|
|
20567
|
+
observeStderr(chunk) {
|
|
20568
|
+
const bytes3 = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf8");
|
|
20569
|
+
if (bytes3.length === 0) return;
|
|
20570
|
+
this.stderrNonempty = true;
|
|
20571
|
+
const remaining = Math.max(
|
|
20572
|
+
0,
|
|
20573
|
+
_CodexAppServerSession.STDERR_OBSERVATION_CAP_BYTES - this.stderrBytesObserved
|
|
20574
|
+
);
|
|
20575
|
+
if (remaining === 0) {
|
|
20576
|
+
this.stderrTruncated = true;
|
|
20577
|
+
return;
|
|
20578
|
+
}
|
|
20579
|
+
const observed = bytes3.subarray(0, remaining);
|
|
20580
|
+
this.stderrBytesObserved += observed.length;
|
|
20581
|
+
if (bytes3.length > remaining) this.stderrTruncated = true;
|
|
20582
|
+
this.stderrTail = `${this.stderrTail}${observed.toString("utf8")}`.slice(-_CodexAppServerSession.STDERR_CLASSIFICATION_TAIL_CHARS).toLowerCase();
|
|
20583
|
+
const classified = /heap out of memory|allocation failed|out of memory/u.test(this.stderrTail) ? "out_of_memory" : /fatal runtime error|fatal error|\bpanic(?:ked)?\b/u.test(this.stderrTail) ? "runtime_fatal" : /access violation|segmentation fault|\bsegfault\b/u.test(this.stderrTail) ? "memory_fault" : /broken pipe|econnreset|connection reset/u.test(this.stderrTail) ? "pipe_or_connection" : null;
|
|
20584
|
+
if (classified) this.stderrClass = classified;
|
|
20585
|
+
else if (this.stderrClass === "none") this.stderrClass = "unclassified";
|
|
20586
|
+
}
|
|
20587
|
+
async handleProcessClose(rawCode, rawSignal) {
|
|
20588
|
+
this.processCloseObserved = true;
|
|
20589
|
+
let termination = null;
|
|
20590
|
+
if (this.guardian) {
|
|
20591
|
+
try {
|
|
20592
|
+
const receipt = await readCodexGuardianReceipt(this.guardian.receiptPath);
|
|
20593
|
+
if (receipt?.process_token === this.guardian.processToken && receipt.state === "terminated") {
|
|
20594
|
+
termination = receipt.termination;
|
|
20595
|
+
}
|
|
20596
|
+
} catch {
|
|
20597
|
+
}
|
|
20598
|
+
}
|
|
20599
|
+
const diagnostics = this.processExitDiagnostics(rawCode, rawSignal, termination);
|
|
20600
|
+
if (this.guardian) {
|
|
20601
|
+
await rm2(`${this.guardian.receiptPath}.shutdown-intent`, { force: true }).catch(() => void 0);
|
|
20602
|
+
}
|
|
20603
|
+
try {
|
|
20604
|
+
this.onProcessExit?.(diagnostics);
|
|
20605
|
+
} catch {
|
|
20606
|
+
}
|
|
20607
|
+
this.failTransport("codex_process_closed", diagnostics);
|
|
20608
|
+
}
|
|
20609
|
+
processExitDiagnostics(rawCode, rawSignal, termination) {
|
|
20610
|
+
const source = termination ? "child_receipt" : this.guardian ? "guardian_process" : "direct_process";
|
|
20611
|
+
const reportedCode = termination?.exit_code ?? rawCode;
|
|
20612
|
+
const reportedSignal = termination?.signal ?? rawSignal;
|
|
20613
|
+
const signal = typeof reportedSignal === "string" && /^(?:SIG[A-Z0-9]{1,28}|sig[a-z0-9]{1,28})$/u.test(reportedSignal) ? reportedSignal.toLowerCase() : null;
|
|
20614
|
+
const exitCode = signal === null && typeof reportedCode === "number" && Number.isInteger(reportedCode) && reportedCode >= -(2 ** 31) && reportedCode <= 2 ** 31 - 1 ? reportedCode : null;
|
|
20615
|
+
const diagnostics = {
|
|
20616
|
+
schema_version: "external_inference_process_exit_v1",
|
|
20617
|
+
runtime: "codex_app_server",
|
|
20618
|
+
source,
|
|
20619
|
+
session_generation: this.sessionGeneration,
|
|
20620
|
+
expected_shutdown: termination?.expected ?? this.shutdownRequested,
|
|
20621
|
+
exit_class: signal !== null ? "signal" : exitCode === null ? "unknown" : exitCode === 0 ? "zero" : "nonzero",
|
|
20622
|
+
exit_code: exitCode,
|
|
20623
|
+
signal,
|
|
20624
|
+
uptime_ms: termination?.uptime_ms ?? Math.min(
|
|
20625
|
+
Math.max(0, Date.now() - this.startedAtMs),
|
|
20626
|
+
Number.MAX_SAFE_INTEGER
|
|
20627
|
+
),
|
|
20628
|
+
pending_request_count: this.pending.size,
|
|
20629
|
+
active_thread_count: this.threads.size,
|
|
20630
|
+
stderr_nonempty: this.stderrNonempty,
|
|
20631
|
+
stderr_bytes_observed: this.stderrBytesObserved,
|
|
20632
|
+
stderr_truncated: this.stderrTruncated,
|
|
20633
|
+
stderr_class: this.stderrNonempty ? this.stderrClass : "none"
|
|
20634
|
+
};
|
|
20635
|
+
this.stderrTail = "";
|
|
20636
|
+
return diagnostics;
|
|
20637
|
+
}
|
|
20443
20638
|
isUsable() {
|
|
20444
20639
|
return !this.closed && this.fatalError === null && this.process.exitCode === null;
|
|
20445
20640
|
}
|
|
@@ -20864,6 +21059,7 @@ child.once('close', async () => {
|
|
|
20864
21059
|
code: error48.code,
|
|
20865
21060
|
retryable: error48.retryable,
|
|
20866
21061
|
dispatchOutcome: requestWritten ? "outcome_unknown" : "not_dispatched",
|
|
21062
|
+
processExit: error48.processExit,
|
|
20867
21063
|
cause: error48
|
|
20868
21064
|
}));
|
|
20869
21065
|
});
|
|
@@ -21067,6 +21263,7 @@ child.once('close', async () => {
|
|
|
21067
21263
|
code: error48.code,
|
|
21068
21264
|
retryable: error48.retryable,
|
|
21069
21265
|
dispatchOutcome: "outcome_unknown",
|
|
21266
|
+
processExit: error48.processExit,
|
|
21070
21267
|
cause: error48
|
|
21071
21268
|
});
|
|
21072
21269
|
}
|
|
@@ -21319,6 +21516,20 @@ child.once('close', async () => {
|
|
|
21319
21516
|
async close() {
|
|
21320
21517
|
if (this.closed) return;
|
|
21321
21518
|
this.closed = true;
|
|
21519
|
+
if (this.processCloseObserved || this.process.exitCode !== null) {
|
|
21520
|
+
if (this.guardian) {
|
|
21521
|
+
await rm2(`${this.guardian.receiptPath}.shutdown-intent`, { force: true }).catch(() => void 0);
|
|
21522
|
+
}
|
|
21523
|
+
return;
|
|
21524
|
+
}
|
|
21525
|
+
this.shutdownRequested = true;
|
|
21526
|
+
if (this.guardian) {
|
|
21527
|
+
await writeFile(
|
|
21528
|
+
`${this.guardian.receiptPath}.shutdown-intent`,
|
|
21529
|
+
this.guardian.processToken,
|
|
21530
|
+
{ mode: 384 }
|
|
21531
|
+
).catch(() => void 0);
|
|
21532
|
+
}
|
|
21322
21533
|
this.process.stdin.end();
|
|
21323
21534
|
const exited = await new Promise((resolve6) => {
|
|
21324
21535
|
if (this.process.exitCode !== null) {
|
|
@@ -21527,11 +21738,12 @@ import {
|
|
|
21527
21738
|
import { createHash as createHash4, randomBytes as randomBytes3 } from "node:crypto";
|
|
21528
21739
|
import { tmpdir as tmpdir2 } from "node:os";
|
|
21529
21740
|
import { isAbsolute as isAbsolute3, join as join4, relative as relative2, resolve as resolve3, sep as sep2 } from "node:path";
|
|
21530
|
-
var MAX_PROMPT_BYTES2, CODEX_MODEL_NAME_PATTERN, CODEX_REASONING_EFFORT_PATTERN, codexRecoveryCheckpointProcessFingerprint, proveCodexSameBootQuiescence, tokenUsageReceiptSchema, turnResultReceiptSchema, terminalReceiptSchema, recoveryCheckpointSchema, recoveryFileSchema, FileCodexAttemptRecoveryStore, utf8Bytes, assertAttemptActive, tomlString, permissionConfig, ensureDedicatedCodexHome, createIsolatedCodexAttemptResources, createIsolatedCodexHostResources, validateAttemptInput, isStrictDescendant, assertRecoveryResourceScope, removeRecoveredThread, confirmGuardianTerminatedForRecovery, reconcileCodexAttemptRecovery, CodexSubscriptionAdapter;
|
|
21741
|
+
var MAX_PROMPT_BYTES2, CODEX_MODEL_NAME_PATTERN, CODEX_REASONING_EFFORT_PATTERN, codexRecoveryCheckpointProcessFingerprint, proveCodexSameBootQuiescence, tokenUsageReceiptSchema, turnResultReceiptSchema, terminalReceiptSchema, recoveryCheckpointSchema, recoveryFileSchema, FileCodexAttemptRecoveryStore, utf8Bytes, assertAttemptActive, tomlString, permissionConfig, ensureDedicatedCodexHome, createIsolatedCodexAttemptResources, createIsolatedCodexHostResources, validateAttemptInput, isStrictDescendant, assertRecoveryResourceScope, removeRecoveredThread, confirmGuardianTerminatedForRecovery, recoveredCodexProcessExit, reconcileCodexAttemptRecovery, CodexSubscriptionAdapter;
|
|
21531
21742
|
var init_codex_adapter = __esm({
|
|
21532
21743
|
"lib/inference-host/codex-adapter.ts"() {
|
|
21533
21744
|
"use strict";
|
|
21534
21745
|
init_zod();
|
|
21746
|
+
init_external_inference_contract();
|
|
21535
21747
|
init_codex_app_server();
|
|
21536
21748
|
init_codex_binary();
|
|
21537
21749
|
init_config();
|
|
@@ -21606,7 +21818,8 @@ var init_codex_adapter = __esm({
|
|
|
21606
21818
|
dispatchOutcome: external_exports.enum(["not_dispatched", "confirmed_dispatched", "outcome_unknown"]),
|
|
21607
21819
|
httpStatusCode: external_exports.number().int().min(100).max(599).nullable().optional(),
|
|
21608
21820
|
retryAtMs: external_exports.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).nullable().optional(),
|
|
21609
|
-
usage: tokenUsageReceiptSchema.nullable().optional()
|
|
21821
|
+
usage: tokenUsageReceiptSchema.nullable().optional(),
|
|
21822
|
+
processExit: codexProcessExitDiagnosticsSchema.nullable().optional()
|
|
21610
21823
|
})
|
|
21611
21824
|
})
|
|
21612
21825
|
]);
|
|
@@ -21622,6 +21835,7 @@ var init_codex_adapter = __esm({
|
|
|
21622
21835
|
processToken: external_exports.string().regex(/^[0-9a-f]{64}$/u).nullable(),
|
|
21623
21836
|
processReceiptPath: external_exports.string().min(1).max(4096).nullable(),
|
|
21624
21837
|
processState: external_exports.enum(["unmanaged", "spawn_intent", "running", "terminated"]),
|
|
21838
|
+
sessionGeneration: external_exports.number().int().positive().max(2147483647).nullable().optional(),
|
|
21625
21839
|
bootIdentity: external_exports.string().min(1).max(4096).nullable().optional(),
|
|
21626
21840
|
cleanupConfirmed: external_exports.boolean(),
|
|
21627
21841
|
adapterRequestId: external_exports.string().min(1).max(512).nullable(),
|
|
@@ -22016,6 +22230,30 @@ var init_codex_adapter = __esm({
|
|
|
22016
22230
|
throw error48;
|
|
22017
22231
|
}
|
|
22018
22232
|
};
|
|
22233
|
+
recoveredCodexProcessExit = (checkpoint, receipt) => {
|
|
22234
|
+
const termination = receipt?.termination;
|
|
22235
|
+
const sessionGeneration = checkpoint.sessionGeneration ?? null;
|
|
22236
|
+
if (!termination || termination.expected || sessionGeneration === null) return null;
|
|
22237
|
+
const signal = termination.signal;
|
|
22238
|
+
const exitCode = signal === null ? termination.exit_code : null;
|
|
22239
|
+
return {
|
|
22240
|
+
schema_version: "external_inference_process_exit_v1",
|
|
22241
|
+
runtime: "codex_app_server",
|
|
22242
|
+
source: "child_receipt",
|
|
22243
|
+
session_generation: sessionGeneration,
|
|
22244
|
+
expected_shutdown: false,
|
|
22245
|
+
exit_class: signal !== null ? "signal" : exitCode === null ? "unknown" : exitCode === 0 ? "zero" : "nonzero",
|
|
22246
|
+
exit_code: exitCode,
|
|
22247
|
+
signal,
|
|
22248
|
+
uptime_ms: termination.uptime_ms,
|
|
22249
|
+
pending_request_count: null,
|
|
22250
|
+
active_thread_count: null,
|
|
22251
|
+
stderr_nonempty: null,
|
|
22252
|
+
stderr_bytes_observed: null,
|
|
22253
|
+
stderr_truncated: null,
|
|
22254
|
+
stderr_class: "unknown"
|
|
22255
|
+
};
|
|
22256
|
+
};
|
|
22019
22257
|
reconcileCodexAttemptRecovery = async (options) => {
|
|
22020
22258
|
const attempts = await options.recoveryHooks.loadAll?.() ?? {};
|
|
22021
22259
|
const currentBootIdentity = options.bootIdentity ?? await readInferenceSystemBootIdentity();
|
|
@@ -22056,11 +22294,27 @@ var init_codex_adapter = __esm({
|
|
|
22056
22294
|
assertRecoveryResourceScope(checkpoint);
|
|
22057
22295
|
await removeRecoveredThread(checkpoint, options.codexHome);
|
|
22058
22296
|
await rm3(checkpoint.resourceRoot, { recursive: true, force: true });
|
|
22297
|
+
const recoveredProcessExit = recoveredCodexProcessExit(
|
|
22298
|
+
checkpoint,
|
|
22299
|
+
await readCodexGuardianReceipt(guardian.receiptPath)
|
|
22300
|
+
);
|
|
22301
|
+
const dispatchOutcome = crossedBoot ? "outcome_unknown" : checkpoint.dispatchOutcome;
|
|
22059
22302
|
await options.recoveryHooks.save({
|
|
22060
22303
|
...checkpoint,
|
|
22061
22304
|
processState: "terminated",
|
|
22062
22305
|
cleanupConfirmed: true,
|
|
22063
|
-
dispatchOutcome
|
|
22306
|
+
dispatchOutcome,
|
|
22307
|
+
terminal: checkpoint.terminal ?? (recoveredProcessExit ? {
|
|
22308
|
+
kind: "failed",
|
|
22309
|
+
error: {
|
|
22310
|
+
message: "Codex app-server transport became unavailable.",
|
|
22311
|
+
category: "transport",
|
|
22312
|
+
code: "codex_process_closed",
|
|
22313
|
+
retryable: true,
|
|
22314
|
+
dispatchOutcome,
|
|
22315
|
+
processExit: recoveredProcessExit
|
|
22316
|
+
}
|
|
22317
|
+
} : null)
|
|
22064
22318
|
});
|
|
22065
22319
|
reconciled += 1;
|
|
22066
22320
|
}
|
|
@@ -22074,8 +22328,15 @@ var init_codex_adapter = __esm({
|
|
|
22074
22328
|
this.durableSessionStart = null;
|
|
22075
22329
|
this.closing = false;
|
|
22076
22330
|
this.adapterClosePromise = null;
|
|
22331
|
+
this.sessionGeneration = 0;
|
|
22077
22332
|
this.dependencies = dependencies;
|
|
22078
22333
|
}
|
|
22334
|
+
emitDiagnostic(event) {
|
|
22335
|
+
try {
|
|
22336
|
+
this.dependencies.emitDiagnosticEvent?.(event);
|
|
22337
|
+
} catch {
|
|
22338
|
+
}
|
|
22339
|
+
}
|
|
22079
22340
|
async readAccountRateLimits(deadlineAtMs, signal) {
|
|
22080
22341
|
const codexHome = this.dependencies.codexHome;
|
|
22081
22342
|
if (!codexHome) {
|
|
@@ -22094,6 +22355,9 @@ var init_codex_adapter = __esm({
|
|
|
22094
22355
|
return await durableSession.session.readAccountRateLimits(deadlineAtMs, signal);
|
|
22095
22356
|
}
|
|
22096
22357
|
async startDurableSession(codexHome, deadlineAtMs) {
|
|
22358
|
+
this.sessionGeneration = Math.min(this.sessionGeneration + 1, 2147483647);
|
|
22359
|
+
const sessionGeneration = this.sessionGeneration;
|
|
22360
|
+
const activeAttemptIds = /* @__PURE__ */ new Set();
|
|
22097
22361
|
const binary = await (this.dependencies.resolveBinary ?? resolvePinnedCodexBinary)();
|
|
22098
22362
|
const guardianManaged = Boolean(
|
|
22099
22363
|
this.dependencies.recoveryHooks && !this.dependencies.spawnProcess
|
|
@@ -22108,10 +22372,26 @@ var init_codex_adapter = __esm({
|
|
|
22108
22372
|
codexHome,
|
|
22109
22373
|
deadlineAtMs,
|
|
22110
22374
|
spawnProcess: this.dependencies.spawnProcess,
|
|
22111
|
-
guardian: processToken && processReceiptPath ? { processToken, receiptPath: processReceiptPath, bootIdentity: bootIdentity ?? void 0 } : void 0
|
|
22375
|
+
guardian: processToken && processReceiptPath ? { processToken, receiptPath: processReceiptPath, bootIdentity: bootIdentity ?? void 0 } : void 0,
|
|
22376
|
+
sessionGeneration,
|
|
22377
|
+
onProcessExit: (diagnostics) => {
|
|
22378
|
+
this.emitDiagnostic({
|
|
22379
|
+
event: "codex_app_server_closed",
|
|
22380
|
+
process_exit: diagnostics,
|
|
22381
|
+
active_attempt_ids: [...activeAttemptIds].sort().slice(0, 64),
|
|
22382
|
+
active_attempt_ids_truncated: activeAttemptIds.size > 64
|
|
22383
|
+
});
|
|
22384
|
+
}
|
|
22385
|
+
});
|
|
22386
|
+
this.emitDiagnostic({
|
|
22387
|
+
event: "codex_app_server_session_started",
|
|
22388
|
+
session_generation: sessionGeneration,
|
|
22389
|
+
replacement: sessionGeneration > 1
|
|
22112
22390
|
});
|
|
22113
22391
|
return {
|
|
22114
22392
|
session,
|
|
22393
|
+
sessionGeneration,
|
|
22394
|
+
activeAttemptIds,
|
|
22115
22395
|
codexHome,
|
|
22116
22396
|
capabilityPreflight: null,
|
|
22117
22397
|
processToken,
|
|
@@ -22351,6 +22631,7 @@ var init_codex_adapter = __esm({
|
|
|
22351
22631
|
const receipt = await readCodexGuardianReceipt(checkpoint.processReceiptPath);
|
|
22352
22632
|
if (checkpoint.processState === "terminated" || receipt === null || receipt.state === "terminated") {
|
|
22353
22633
|
await rm3(checkpoint.processReceiptPath, { force: true });
|
|
22634
|
+
await rm3(`${checkpoint.processReceiptPath}.shutdown-intent`, { force: true });
|
|
22354
22635
|
}
|
|
22355
22636
|
}
|
|
22356
22637
|
}
|
|
@@ -22398,6 +22679,7 @@ var init_codex_adapter = __esm({
|
|
|
22398
22679
|
for (const session of sessions) {
|
|
22399
22680
|
if (session.processReceiptPath && unconfirmedReceiptPaths !== null && !unconfirmedReceiptPaths.has(resolve3(session.processReceiptPath))) {
|
|
22400
22681
|
await rm3(session.processReceiptPath, { force: true });
|
|
22682
|
+
await rm3(`${session.processReceiptPath}.shutdown-intent`, { force: true });
|
|
22401
22683
|
}
|
|
22402
22684
|
}
|
|
22403
22685
|
})();
|
|
@@ -22442,7 +22724,8 @@ var init_codex_adapter = __esm({
|
|
|
22442
22724
|
dispatchOutcome: recovered.dispatchOutcome,
|
|
22443
22725
|
httpStatusCode: recovered.httpStatusCode,
|
|
22444
22726
|
retryAtMs: recovered.retryAtMs,
|
|
22445
|
-
usage: recovered.usage
|
|
22727
|
+
usage: recovered.usage,
|
|
22728
|
+
processExit: recovered.processExit
|
|
22446
22729
|
});
|
|
22447
22730
|
}
|
|
22448
22731
|
throw new CodexAppServerError({
|
|
@@ -22475,6 +22758,7 @@ var init_codex_adapter = __esm({
|
|
|
22475
22758
|
input.deadlineAtMs,
|
|
22476
22759
|
input.signal
|
|
22477
22760
|
);
|
|
22761
|
+
sessionLease.activeAttemptIds.add(input.attemptId);
|
|
22478
22762
|
session = sessionLease.session;
|
|
22479
22763
|
baseCheckpoint = {
|
|
22480
22764
|
schemaVersion: "vtx_codex_attempt_recovery_v1",
|
|
@@ -22488,6 +22772,7 @@ var init_codex_adapter = __esm({
|
|
|
22488
22772
|
processToken: sessionLease.processToken,
|
|
22489
22773
|
processReceiptPath: sessionLease.processReceiptPath,
|
|
22490
22774
|
processState: sessionLease.processToken ? "running" : "unmanaged",
|
|
22775
|
+
sessionGeneration: sessionLease.sessionGeneration,
|
|
22491
22776
|
bootIdentity: sessionLease.bootIdentity,
|
|
22492
22777
|
cleanupConfirmed: false,
|
|
22493
22778
|
adapterRequestId: null,
|
|
@@ -22586,7 +22871,8 @@ var init_codex_adapter = __esm({
|
|
|
22586
22871
|
dispatchOutcome,
|
|
22587
22872
|
httpStatusCode: error48.httpStatusCode,
|
|
22588
22873
|
retryAtMs: error48.retryAtMs,
|
|
22589
|
-
usage: error48.usage
|
|
22874
|
+
usage: error48.usage,
|
|
22875
|
+
processExit: error48.processExit
|
|
22590
22876
|
}
|
|
22591
22877
|
}
|
|
22592
22878
|
};
|
|
@@ -22602,6 +22888,7 @@ var init_codex_adapter = __esm({
|
|
|
22602
22888
|
httpStatusCode: error48.httpStatusCode,
|
|
22603
22889
|
retryAtMs: error48.retryAtMs,
|
|
22604
22890
|
usage: error48.usage,
|
|
22891
|
+
processExit: error48.processExit,
|
|
22605
22892
|
cause: error48
|
|
22606
22893
|
});
|
|
22607
22894
|
}
|
|
@@ -22632,6 +22919,7 @@ var init_codex_adapter = __esm({
|
|
|
22632
22919
|
}
|
|
22633
22920
|
throw classified;
|
|
22634
22921
|
} finally {
|
|
22922
|
+
sessionLease?.activeAttemptIds.delete(input.attemptId);
|
|
22635
22923
|
const cleanup = async () => {
|
|
22636
22924
|
let cleanupError = sessionInvalidationError;
|
|
22637
22925
|
if (session && threadId) {
|
|
@@ -29762,7 +30050,7 @@ function createDefaultInferenceHostRunnerDependencies(options) {
|
|
|
29762
30050
|
envelopePublicKey: options.envelopePublicKey
|
|
29763
30051
|
};
|
|
29764
30052
|
}
|
|
29765
|
-
var import_ajv, RUNTIME_RECEIPT_SCHEMA_VERSION, ATTEMPT_RECEIPT_SCHEMA_VERSION, DEFAULT_ADVERTISEMENT_TTL_MS, DEFAULT_ADVERTISEMENT_REFRESH_LEAD_MS, DEFAULT_HOST_HEARTBEAT_MS, DEFAULT_PROVIDER_RATE_LIMIT_REFRESH_MS, DEFAULT_ATTEMPT_HEARTBEAT_MS, DEFAULT_DRAIN_TIMEOUT_MS, DEFAULT_REMOTE_RETRY_LIMIT, MIN_SLEEP_MS, DEFAULT_CODEX_ACCOUNT_COOLDOWN_MS, MIN_CLAIM_START_WINDOW_MS, MAX_ATTEMPT_START_RETRY_DELAY_MS, UNBOUNDED_AVAILABLE_SLOTS, buildInferenceAdvertisedModels, summarizeInferenceHostRuntimeRecovery, FileInferenceHostRuntimeReceiptStore, InferenceHostRecoveryRequiredError, InferenceHostRunnerError, defaultSleep, abortError, defaultRegisterSignalHandlers, safeInteger, exactObjectKeys, validIso, validateAttemptReceipt, validateRuntimeReceipt, sha256, stableOperationId, buildAttemptStartRequest, isoAt, providerWeeklyQuotaFromRateLimits, finitePositiveOption, validateRunnerOptions, assertLocalCredentialIdentity, retryableRemoteError, remoteRetryAfterMs, controlPlaneFatal, defaultValidateOutput, objectRecord, positiveUtf8ByteLimit, assertCompilableOutputSchema, parseOutputContract, membershipFailureDisposition, safeFailureCode, attemptStartRetryFallbackMs, exactJson2, runnerIdentityMismatch, assertAdvertisementResult, assertHostHeartbeatResult, assertClaimResult, assertStartResult, assertJobHeartbeatResult, assertTerminalResult, classifyFailure, ambiguousHeartbeatTransport, reportedTokenUsage, reportedUsage, InferenceHostRunner;
|
|
30053
|
+
var import_ajv, RUNTIME_RECEIPT_SCHEMA_VERSION, ATTEMPT_RECEIPT_SCHEMA_VERSION, DEFAULT_ADVERTISEMENT_TTL_MS, DEFAULT_ADVERTISEMENT_REFRESH_LEAD_MS, DEFAULT_HOST_HEARTBEAT_MS, DEFAULT_PROVIDER_RATE_LIMIT_REFRESH_MS, DEFAULT_ATTEMPT_HEARTBEAT_MS, DEFAULT_DRAIN_TIMEOUT_MS, DEFAULT_REMOTE_RETRY_LIMIT, MIN_SLEEP_MS, MIN_HOST_HEARTBEAT_GAP_DIAGNOSTIC_MS, HOST_HEARTBEAT_GAP_DIAGNOSTIC_FACTOR, DEFAULT_CODEX_ACCOUNT_COOLDOWN_MS, MIN_CLAIM_START_WINDOW_MS, MAX_ATTEMPT_START_RETRY_DELAY_MS, UNBOUNDED_AVAILABLE_SLOTS, buildInferenceAdvertisedModels, summarizeInferenceHostRuntimeRecovery, FileInferenceHostRuntimeReceiptStore, InferenceHostRecoveryRequiredError, InferenceHostRunnerError, defaultSleep, abortError, defaultRegisterSignalHandlers, safeInteger, exactObjectKeys, validIso, validateAttemptReceipt, validateRuntimeReceipt, sha256, stableOperationId, buildAttemptStartRequest, isoAt, providerWeeklyQuotaFromRateLimits, finitePositiveOption, validateRunnerOptions, assertLocalCredentialIdentity, retryableRemoteError, remoteRetryAfterMs, controlPlaneFatal, defaultValidateOutput, objectRecord, positiveUtf8ByteLimit, assertCompilableOutputSchema, parseOutputContract, membershipFailureDisposition, safeFailureCode, attemptStartRetryFallbackMs, exactJson2, runnerIdentityMismatch, assertAdvertisementResult, assertHostHeartbeatResult, assertClaimResult, assertStartResult, assertJobHeartbeatResult, assertTerminalResult, classifyFailure, ambiguousHeartbeatTransport, reportedTokenUsage, reportedUsage, InferenceHostRunner;
|
|
29766
30054
|
var init_runner = __esm({
|
|
29767
30055
|
"lib/inference-host/runner.ts"() {
|
|
29768
30056
|
"use strict";
|
|
@@ -29784,6 +30072,8 @@ var init_runner = __esm({
|
|
|
29784
30072
|
DEFAULT_DRAIN_TIMEOUT_MS = 3e4;
|
|
29785
30073
|
DEFAULT_REMOTE_RETRY_LIMIT = 3;
|
|
29786
30074
|
MIN_SLEEP_MS = 10;
|
|
30075
|
+
MIN_HOST_HEARTBEAT_GAP_DIAGNOSTIC_MS = 1e4;
|
|
30076
|
+
HOST_HEARTBEAT_GAP_DIAGNOSTIC_FACTOR = 3;
|
|
29787
30077
|
DEFAULT_CODEX_ACCOUNT_COOLDOWN_MS = 5 * 60 * 1e3;
|
|
29788
30078
|
MIN_CLAIM_START_WINDOW_MS = 5e3;
|
|
29789
30079
|
MAX_ATTEMPT_START_RETRY_DELAY_MS = 15e3;
|
|
@@ -30454,6 +30744,9 @@ var init_runner = __esm({
|
|
|
30454
30744
|
let completed = 0;
|
|
30455
30745
|
let failed = 0;
|
|
30456
30746
|
let acceptAttemptUpdates = true;
|
|
30747
|
+
let stopHostHeartbeatLoop = () => {
|
|
30748
|
+
};
|
|
30749
|
+
let hostHeartbeatLoop = Promise.resolve();
|
|
30457
30750
|
let wakeDrain;
|
|
30458
30751
|
const drainSignal = new Promise((resolve6) => {
|
|
30459
30752
|
wakeDrain = resolve6;
|
|
@@ -30469,6 +30762,7 @@ var init_runner = __esm({
|
|
|
30469
30762
|
if (drainRequested && priority(reason) <= priority(drainReason)) return;
|
|
30470
30763
|
drainRequested = true;
|
|
30471
30764
|
drainReason = reason;
|
|
30765
|
+
stopHostHeartbeatLoop();
|
|
30472
30766
|
wakeDrain();
|
|
30473
30767
|
};
|
|
30474
30768
|
try {
|
|
@@ -30600,6 +30894,18 @@ var init_runner = __esm({
|
|
|
30600
30894
|
updated_at: isoAt(now())
|
|
30601
30895
|
};
|
|
30602
30896
|
await this.persistReceipt(receipt, now);
|
|
30897
|
+
let receiptMutationChain = Promise.resolve();
|
|
30898
|
+
const mutateReceipt = async (mutation) => {
|
|
30899
|
+
let result2;
|
|
30900
|
+
const operation = receiptMutationChain.then(async () => {
|
|
30901
|
+
result2 = mutation(receipt);
|
|
30902
|
+
receipt = result2;
|
|
30903
|
+
await this.persistReceipt(receipt, now);
|
|
30904
|
+
});
|
|
30905
|
+
receiptMutationChain = operation;
|
|
30906
|
+
await operation;
|
|
30907
|
+
return result2;
|
|
30908
|
+
};
|
|
30603
30909
|
const envelopePublicKey = await this.dependencies.envelopePublicKey(credential);
|
|
30604
30910
|
const retryExact = async (operation, boundary = {}) => {
|
|
30605
30911
|
let lastError;
|
|
@@ -30642,40 +30948,48 @@ var init_runner = __esm({
|
|
|
30642
30948
|
const publishAdvertisement = async (health, forceAdvance, retryRemote = true, allowDefinitiveReseed = true) => {
|
|
30643
30949
|
let advertisement = receipt.pending_advertisement;
|
|
30644
30950
|
if (!advertisement) {
|
|
30645
|
-
const
|
|
30646
|
-
|
|
30647
|
-
|
|
30648
|
-
|
|
30649
|
-
|
|
30650
|
-
|
|
30651
|
-
|
|
30652
|
-
|
|
30653
|
-
|
|
30654
|
-
|
|
30655
|
-
|
|
30656
|
-
|
|
30657
|
-
|
|
30658
|
-
|
|
30659
|
-
|
|
30660
|
-
|
|
30661
|
-
|
|
30662
|
-
|
|
30663
|
-
|
|
30664
|
-
|
|
30665
|
-
|
|
30666
|
-
|
|
30667
|
-
|
|
30668
|
-
|
|
30669
|
-
|
|
30670
|
-
|
|
30671
|
-
|
|
30951
|
+
const updated = await mutateReceipt((current) => {
|
|
30952
|
+
if (current.pending_advertisement) return current;
|
|
30953
|
+
const generation = current.registered && forceAdvance ? current.advertisement_generation + 1 : current.advertisement_generation;
|
|
30954
|
+
const advertisedAt = now();
|
|
30955
|
+
const pendingAdvertisement = hostAdvertisementSchema.parse({
|
|
30956
|
+
schema_version: EXTERNAL_INFERENCE_ADVERTISEMENT_SCHEMA_VERSION,
|
|
30957
|
+
contract_version: EXTERNAL_INFERENCE_CONTRACT_VERSION,
|
|
30958
|
+
operation_id: stableOperationId("advertise", [
|
|
30959
|
+
localState.host_id,
|
|
30960
|
+
localState.host_generation,
|
|
30961
|
+
generation,
|
|
30962
|
+
localState.key_generation,
|
|
30963
|
+
advertisedAt
|
|
30964
|
+
]),
|
|
30965
|
+
host_id: localState.host_id,
|
|
30966
|
+
host_generation: localState.host_generation,
|
|
30967
|
+
advertisement_generation: generation,
|
|
30968
|
+
key_generation: localState.key_generation,
|
|
30969
|
+
adapter: settings.adapterId,
|
|
30970
|
+
display_name: this.options.displayName.trim(),
|
|
30971
|
+
...this.options.authenticatedAccountIdentity ? { authenticated_account_identity: this.options.authenticatedAccountIdentity } : {},
|
|
30972
|
+
authenticated_account_email: this.options.authenticatedAccountEmail ?? null,
|
|
30973
|
+
authenticated_account_plan: this.options.authenticatedAccountPlan ?? null,
|
|
30974
|
+
protocol_version: this.options.protocolVersion,
|
|
30975
|
+
envelope_public_key: envelopePublicKey,
|
|
30976
|
+
health,
|
|
30977
|
+
advertised_at: isoAt(advertisedAt),
|
|
30978
|
+
expires_at: isoAt(advertisedAt + settings.advertisementTtlMs),
|
|
30979
|
+
models: settings.advertisedModels
|
|
30980
|
+
});
|
|
30981
|
+
return {
|
|
30982
|
+
...current,
|
|
30983
|
+
pending_advertisement: pendingAdvertisement,
|
|
30984
|
+
updated_at: isoAt(now())
|
|
30985
|
+
};
|
|
30672
30986
|
});
|
|
30673
|
-
|
|
30674
|
-
|
|
30675
|
-
|
|
30676
|
-
|
|
30677
|
-
|
|
30678
|
-
|
|
30987
|
+
advertisement = updated.pending_advertisement;
|
|
30988
|
+
}
|
|
30989
|
+
if (!advertisement) {
|
|
30990
|
+
throw new InferenceHostRecoveryRequiredError(
|
|
30991
|
+
"The durable host advertisement receipt is unavailable."
|
|
30992
|
+
);
|
|
30679
30993
|
}
|
|
30680
30994
|
const tool = receipt.registered ? "inference.host.advertise" : "inference.host.register";
|
|
30681
30995
|
let result2;
|
|
@@ -30690,26 +31004,24 @@ var init_runner = __esm({
|
|
|
30690
31004
|
const definitivelyNotApplied = error48 instanceof ExternalInferenceMcpError ? error48.definitivelyNotApplied : Boolean(error48 && typeof error48 === "object" && error48.definitivelyNotApplied === true);
|
|
30691
31005
|
const expectedGeneration = receipt.registered ? receipt.advertisement_generation + 1 : receipt.advertisement_generation;
|
|
30692
31006
|
if (definitivelyNotApplied && advertisement.advertisement_generation === expectedGeneration && allowDefinitiveReseed) {
|
|
30693
|
-
|
|
30694
|
-
...
|
|
30695
|
-
pending_advertisement: null,
|
|
31007
|
+
await mutateReceipt((current) => ({
|
|
31008
|
+
...current,
|
|
31009
|
+
pending_advertisement: current.pending_advertisement?.operation_id === advertisement.operation_id ? null : current.pending_advertisement,
|
|
30696
31010
|
updated_at: isoAt(now())
|
|
30697
|
-
};
|
|
30698
|
-
await this.persistReceipt(receipt, now);
|
|
31011
|
+
}));
|
|
30699
31012
|
return await publishAdvertisement(health, forceAdvance, retryRemote, false);
|
|
30700
31013
|
}
|
|
30701
31014
|
throw error48;
|
|
30702
31015
|
}
|
|
30703
31016
|
assertAdvertisementResult(advertisement, result2);
|
|
30704
|
-
|
|
30705
|
-
...
|
|
31017
|
+
await mutateReceipt((current) => ({
|
|
31018
|
+
...current,
|
|
30706
31019
|
registered: true,
|
|
30707
31020
|
advertisement_generation: result2.host.advertisement_generation,
|
|
30708
31021
|
advertisement_expires_at: advertisement.expires_at,
|
|
30709
|
-
pending_advertisement: null,
|
|
31022
|
+
pending_advertisement: current.pending_advertisement?.operation_id === advertisement.operation_id ? null : current.pending_advertisement,
|
|
30710
31023
|
updated_at: isoAt(now())
|
|
30711
|
-
};
|
|
30712
|
-
await this.persistReceipt(receipt, now);
|
|
31024
|
+
}));
|
|
30713
31025
|
};
|
|
30714
31026
|
const wasRegistered = receipt.registered;
|
|
30715
31027
|
await publishAdvertisement("healthy", wasRegistered);
|
|
@@ -30722,32 +31034,34 @@ var init_runner = __esm({
|
|
|
30722
31034
|
let providerRateLimitsObservedAtMs = now();
|
|
30723
31035
|
const hostHeartbeat = async (status, requestOptions = {
|
|
30724
31036
|
signal: this.options.signal
|
|
30725
|
-
}, retryRemote = true, allowOfflineProjection = false, definitiveReseedAttempt = 0) => {
|
|
30726
|
-
|
|
30727
|
-
|
|
30728
|
-
|
|
30729
|
-
|
|
30730
|
-
|
|
30731
|
-
|
|
30732
|
-
|
|
30733
|
-
|
|
30734
|
-
|
|
30735
|
-
|
|
30736
|
-
|
|
30737
|
-
|
|
30738
|
-
|
|
30739
|
-
|
|
30740
|
-
|
|
30741
|
-
|
|
30742
|
-
|
|
30743
|
-
|
|
30744
|
-
|
|
30745
|
-
|
|
30746
|
-
|
|
30747
|
-
|
|
30748
|
-
|
|
30749
|
-
|
|
30750
|
-
|
|
31037
|
+
}, retryRemote = true, allowOfflineProjection = false, definitiveReseedAttempt = 0, advertisementGeneration = receipt.advertisement_generation) => {
|
|
31038
|
+
let request;
|
|
31039
|
+
await mutateReceipt((current) => {
|
|
31040
|
+
const sequence = current.host_heartbeat_sequence + 1;
|
|
31041
|
+
request = {
|
|
31042
|
+
schema_version: "external_inference_host_heartbeat_v1",
|
|
31043
|
+
contract_version: EXTERNAL_INFERENCE_CONTRACT_VERSION,
|
|
31044
|
+
host_id: localState.host_id,
|
|
31045
|
+
host_generation: localState.host_generation,
|
|
31046
|
+
advertisement_generation: advertisementGeneration,
|
|
31047
|
+
key_generation: localState.key_generation,
|
|
31048
|
+
sequence,
|
|
31049
|
+
status,
|
|
31050
|
+
active_attempts: active.size,
|
|
31051
|
+
available_slots: settings.maxConcurrency === null ? UNBOUNDED_AVAILABLE_SLOTS - active.size : Math.max(0, settings.maxConcurrency - active.size),
|
|
31052
|
+
provider_weekly_quota: providerWeeklyQuotaFromRateLimits(
|
|
31053
|
+
providerRateLimits,
|
|
31054
|
+
providerRateLimitsObservedAtMs,
|
|
31055
|
+
now()
|
|
31056
|
+
),
|
|
31057
|
+
observed_at: isoAt(now())
|
|
31058
|
+
};
|
|
31059
|
+
return {
|
|
31060
|
+
...current,
|
|
31061
|
+
host_heartbeat_sequence: sequence,
|
|
31062
|
+
updated_at: isoAt(now())
|
|
31063
|
+
};
|
|
31064
|
+
});
|
|
30751
31065
|
const heartbeatOperation = () => mcp.callTool(
|
|
30752
31066
|
"inference.host.heartbeat",
|
|
30753
31067
|
request,
|
|
@@ -30764,7 +31078,8 @@ var init_runner = __esm({
|
|
|
30764
31078
|
requestOptions,
|
|
30765
31079
|
retryRemote,
|
|
30766
31080
|
allowOfflineProjection,
|
|
30767
|
-
definitiveReseedAttempt + 1
|
|
31081
|
+
definitiveReseedAttempt + 1,
|
|
31082
|
+
advertisementGeneration
|
|
30768
31083
|
);
|
|
30769
31084
|
}
|
|
30770
31085
|
throw error48;
|
|
@@ -30779,12 +31094,129 @@ var init_runner = __esm({
|
|
|
30779
31094
|
}
|
|
30780
31095
|
};
|
|
30781
31096
|
await hostHeartbeat("healthy");
|
|
30782
|
-
let
|
|
31097
|
+
let lastHostHeartbeatSucceededAt = now();
|
|
31098
|
+
let hostHeartbeatGapOpen = false;
|
|
31099
|
+
const hostHeartbeatGapThresholdMs = Math.max(
|
|
31100
|
+
MIN_HOST_HEARTBEAT_GAP_DIAGNOSTIC_MS,
|
|
31101
|
+
settings.hostHeartbeatMs * HOST_HEARTBEAT_GAP_DIAGNOSTIC_FACTOR
|
|
31102
|
+
);
|
|
31103
|
+
const observeHostHeartbeatGap = (observedAt) => {
|
|
31104
|
+
const gapMs = Math.max(0, observedAt - lastHostHeartbeatSucceededAt);
|
|
31105
|
+
if (gapMs >= hostHeartbeatGapThresholdMs && !hostHeartbeatGapOpen) {
|
|
31106
|
+
hostHeartbeatGapOpen = true;
|
|
31107
|
+
emitDiagnostic("host_heartbeat_gap_detected", {
|
|
31108
|
+
gap_ms: gapMs,
|
|
31109
|
+
configured_interval_ms: settings.hostHeartbeatMs,
|
|
31110
|
+
diagnostic_threshold_ms: hostHeartbeatGapThresholdMs,
|
|
31111
|
+
active_attempts: active.size,
|
|
31112
|
+
host_heartbeat_sequence: receipt.host_heartbeat_sequence
|
|
31113
|
+
});
|
|
31114
|
+
}
|
|
31115
|
+
return gapMs;
|
|
31116
|
+
};
|
|
30783
31117
|
let nextAdvertisementAttemptAt = 0;
|
|
30784
31118
|
let nextProviderRateLimitRefreshAt = now() + settings.providerRateLimitRefreshMs;
|
|
30785
31119
|
let nextClaimAt = Math.max(now(), providerRetryAtMs ?? 0);
|
|
30786
31120
|
let pendingClaimPromotions = 0;
|
|
30787
31121
|
let onceClaimed = false;
|
|
31122
|
+
const hostHeartbeatAbort = new AbortController();
|
|
31123
|
+
stopHostHeartbeatLoop = () => hostHeartbeatAbort.abort();
|
|
31124
|
+
let nextHostHeartbeatAt = now() + settings.hostHeartbeatMs;
|
|
31125
|
+
hostHeartbeatLoop = (async () => {
|
|
31126
|
+
while (!drainRequested && !hostHeartbeatAbort.signal.aborted) {
|
|
31127
|
+
try {
|
|
31128
|
+
await sleep4(
|
|
31129
|
+
Math.max(MIN_SLEEP_MS, nextHostHeartbeatAt - now()),
|
|
31130
|
+
hostHeartbeatAbort.signal
|
|
31131
|
+
);
|
|
31132
|
+
} catch (error48) {
|
|
31133
|
+
if (hostHeartbeatAbort.signal.aborted || drainRequested) return;
|
|
31134
|
+
throw error48;
|
|
31135
|
+
}
|
|
31136
|
+
if (drainRequested || hostHeartbeatAbort.signal.aborted) return;
|
|
31137
|
+
const heartbeatStartedAt = now();
|
|
31138
|
+
observeHostHeartbeatGap(heartbeatStartedAt);
|
|
31139
|
+
const attemptedAdvertisementGeneration = receipt.advertisement_generation;
|
|
31140
|
+
try {
|
|
31141
|
+
const expiresAt = receipt.advertisement_expires_at ? Date.parse(receipt.advertisement_expires_at) : 0;
|
|
31142
|
+
const refreshPendingAcrossExpiry = receipt.pending_advertisement !== null && expiresAt <= heartbeatStartedAt;
|
|
31143
|
+
await hostHeartbeat(
|
|
31144
|
+
"healthy",
|
|
31145
|
+
{ signal: hostHeartbeatAbort.signal },
|
|
31146
|
+
false,
|
|
31147
|
+
refreshPendingAcrossExpiry,
|
|
31148
|
+
0,
|
|
31149
|
+
attemptedAdvertisementGeneration
|
|
31150
|
+
);
|
|
31151
|
+
} catch (error48) {
|
|
31152
|
+
if (hostHeartbeatAbort.signal.aborted || drainRequested) return;
|
|
31153
|
+
const errorCode = error48 && typeof error48 === "object" && "code" in error48 ? String(error48.code) : "";
|
|
31154
|
+
const pendingAdvertisement = receipt.pending_advertisement;
|
|
31155
|
+
const recoveryAdvertisementGeneration = receipt.advertisement_generation > attemptedAdvertisementGeneration ? receipt.advertisement_generation : errorCode === "generation_stale" && pendingAdvertisement && pendingAdvertisement.advertisement_generation === attemptedAdvertisementGeneration + 1 ? pendingAdvertisement.advertisement_generation : null;
|
|
31156
|
+
if (errorCode === "generation_stale" && recoveryAdvertisementGeneration !== null) {
|
|
31157
|
+
try {
|
|
31158
|
+
await hostHeartbeat(
|
|
31159
|
+
"healthy",
|
|
31160
|
+
{ signal: hostHeartbeatAbort.signal },
|
|
31161
|
+
false,
|
|
31162
|
+
false,
|
|
31163
|
+
0,
|
|
31164
|
+
recoveryAdvertisementGeneration
|
|
31165
|
+
);
|
|
31166
|
+
if (receipt.advertisement_generation < recoveryAdvertisementGeneration) {
|
|
31167
|
+
await mutateReceipt((current) => current.advertisement_generation >= recoveryAdvertisementGeneration ? current : {
|
|
31168
|
+
...current,
|
|
31169
|
+
advertisement_generation: recoveryAdvertisementGeneration,
|
|
31170
|
+
advertisement_expires_at: pendingAdvertisement?.expires_at ?? current.advertisement_expires_at,
|
|
31171
|
+
updated_at: isoAt(now())
|
|
31172
|
+
});
|
|
31173
|
+
}
|
|
31174
|
+
} catch (recoveryError) {
|
|
31175
|
+
if (hostHeartbeatAbort.signal.aborted || drainRequested) return;
|
|
31176
|
+
if (controlPlaneFatal(recoveryError)) {
|
|
31177
|
+
requestDrain("authority_lost");
|
|
31178
|
+
return;
|
|
31179
|
+
}
|
|
31180
|
+
if (!retryableRemoteError(recoveryError)) {
|
|
31181
|
+
requestDrain("host_heartbeat_failed");
|
|
31182
|
+
return;
|
|
31183
|
+
}
|
|
31184
|
+
nextHostHeartbeatAt = now() + remoteRetryAfterMs(recoveryError, settings.hostHeartbeatMs);
|
|
31185
|
+
continue;
|
|
31186
|
+
}
|
|
31187
|
+
} else if (controlPlaneFatal(error48)) {
|
|
31188
|
+
requestDrain("authority_lost");
|
|
31189
|
+
return;
|
|
31190
|
+
} else if (!retryableRemoteError(error48)) {
|
|
31191
|
+
requestDrain("host_heartbeat_failed");
|
|
31192
|
+
return;
|
|
31193
|
+
} else {
|
|
31194
|
+
nextHostHeartbeatAt = now() + remoteRetryAfterMs(error48, settings.hostHeartbeatMs);
|
|
31195
|
+
continue;
|
|
31196
|
+
}
|
|
31197
|
+
}
|
|
31198
|
+
const heartbeatSucceededAt = now();
|
|
31199
|
+
const gapMs = observeHostHeartbeatGap(heartbeatSucceededAt);
|
|
31200
|
+
if (hostHeartbeatGapOpen) {
|
|
31201
|
+
emitDiagnostic("host_heartbeat_gap_recovered", {
|
|
31202
|
+
gap_ms: gapMs,
|
|
31203
|
+
configured_interval_ms: settings.hostHeartbeatMs,
|
|
31204
|
+
diagnostic_threshold_ms: hostHeartbeatGapThresholdMs,
|
|
31205
|
+
active_attempts: active.size,
|
|
31206
|
+
host_heartbeat_sequence: receipt.host_heartbeat_sequence
|
|
31207
|
+
});
|
|
31208
|
+
hostHeartbeatGapOpen = false;
|
|
31209
|
+
}
|
|
31210
|
+
lastHostHeartbeatSucceededAt = heartbeatSucceededAt;
|
|
31211
|
+
nextHostHeartbeatAt = Math.max(
|
|
31212
|
+
nextHostHeartbeatAt + settings.hostHeartbeatMs,
|
|
31213
|
+
heartbeatSucceededAt + MIN_SLEEP_MS
|
|
31214
|
+
);
|
|
31215
|
+
}
|
|
31216
|
+
})().catch((error48) => {
|
|
31217
|
+
if (hostHeartbeatAbort.signal.aborted || drainRequested) return;
|
|
31218
|
+
requestDrain(controlPlaneFatal(error48) ? "authority_lost" : "host_heartbeat_failed");
|
|
31219
|
+
});
|
|
30788
31220
|
const recoveryQueue = [];
|
|
30789
31221
|
const attemptStartRetryCounts = /* @__PURE__ */ new Map();
|
|
30790
31222
|
const launchClaim = (claim, recovery, claimRequest) => {
|
|
@@ -30820,15 +31252,14 @@ var init_runner = __esm({
|
|
|
30820
31252
|
credential,
|
|
30821
31253
|
mcp,
|
|
30822
31254
|
receipt: () => receipt,
|
|
30823
|
-
updateReceipt: async (
|
|
31255
|
+
updateReceipt: async (mutation) => {
|
|
30824
31256
|
if (!acceptAttemptUpdates) {
|
|
30825
31257
|
throw new InferenceHostRunnerError(
|
|
30826
31258
|
"runtime_closed",
|
|
30827
31259
|
"The drained inference runtime no longer accepts attempt updates."
|
|
30828
31260
|
);
|
|
30829
31261
|
}
|
|
30830
|
-
|
|
30831
|
-
await this.persistReceipt(receipt, now);
|
|
31262
|
+
await mutateReceipt(mutation);
|
|
30832
31263
|
},
|
|
30833
31264
|
retryExact,
|
|
30834
31265
|
signal: controller.signal,
|
|
@@ -30926,10 +31357,11 @@ var init_runner = __esm({
|
|
|
30926
31357
|
return attemptId;
|
|
30927
31358
|
};
|
|
30928
31359
|
const removeRecoveredAttempt = async (attemptId) => {
|
|
30929
|
-
|
|
30930
|
-
|
|
30931
|
-
|
|
30932
|
-
|
|
31360
|
+
await mutateReceipt((current) => {
|
|
31361
|
+
const attempts = { ...current.attempts };
|
|
31362
|
+
delete attempts[attemptId];
|
|
31363
|
+
return { ...current, attempts, updated_at: isoAt(now()) };
|
|
31364
|
+
});
|
|
30933
31365
|
};
|
|
30934
31366
|
for (const recovery of Object.values({ ...receipt.attempts })) {
|
|
30935
31367
|
if (recovery.phase === "terminal_pending") {
|
|
@@ -31044,59 +31476,37 @@ var init_runner = __esm({
|
|
|
31044
31476
|
nextClaimAt = Math.max(nextClaimAt, nextAdvertisementAttemptAt);
|
|
31045
31477
|
}
|
|
31046
31478
|
}
|
|
31047
|
-
if (currentTime >= nextHostHeartbeatAt) {
|
|
31048
|
-
try {
|
|
31049
|
-
const refreshPendingAcrossExpiry = receipt.pending_advertisement !== null && expiresAt <= currentTime && nextAdvertisementAttemptAt > currentTime;
|
|
31050
|
-
await hostHeartbeat(
|
|
31051
|
-
"healthy",
|
|
31052
|
-
{ signal: this.options.signal },
|
|
31053
|
-
false,
|
|
31054
|
-
refreshPendingAcrossExpiry
|
|
31055
|
-
);
|
|
31056
|
-
} catch (error48) {
|
|
31057
|
-
if (controlPlaneFatal(error48)) {
|
|
31058
|
-
requestDrain("authority_lost");
|
|
31059
|
-
break;
|
|
31060
|
-
}
|
|
31061
|
-
if (!retryableRemoteError(error48)) {
|
|
31062
|
-
requestDrain("host_heartbeat_failed");
|
|
31063
|
-
break;
|
|
31064
|
-
}
|
|
31065
|
-
nextHostHeartbeatAt = now() + remoteRetryAfterMs(error48, settings.hostHeartbeatMs);
|
|
31066
|
-
}
|
|
31067
|
-
if (!drainRequested && currentTime >= nextHostHeartbeatAt) {
|
|
31068
|
-
nextHostHeartbeatAt = now() + settings.hostHeartbeatMs;
|
|
31069
|
-
}
|
|
31070
|
-
}
|
|
31071
31479
|
while (recoveryQueue.length > 0 && (settings.maxConcurrency === null || active.size < settings.maxConcurrency)) {
|
|
31072
31480
|
const recovery = recoveryQueue.shift();
|
|
31073
31481
|
launchClaim(recovery.claim, recovery);
|
|
31074
31482
|
}
|
|
31075
31483
|
while (!drainRequested && recoveryQueue.length === 0 && (!this.options.once || !onceClaimed) && (settings.maxConcurrency === null || active.size < settings.maxConcurrency) && pendingClaimPromotions === 0 && now() >= nextClaimAt) {
|
|
31076
31484
|
if (!receipt.pending_claim_request) {
|
|
31077
|
-
|
|
31078
|
-
|
|
31079
|
-
|
|
31080
|
-
|
|
31081
|
-
|
|
31082
|
-
|
|
31083
|
-
|
|
31084
|
-
|
|
31085
|
-
|
|
31086
|
-
localState.
|
|
31087
|
-
|
|
31088
|
-
|
|
31089
|
-
|
|
31090
|
-
|
|
31091
|
-
|
|
31092
|
-
|
|
31093
|
-
|
|
31094
|
-
|
|
31095
|
-
|
|
31096
|
-
|
|
31097
|
-
|
|
31098
|
-
|
|
31099
|
-
|
|
31485
|
+
await mutateReceipt((current) => {
|
|
31486
|
+
if (current.pending_claim_request) return current;
|
|
31487
|
+
const claimSequence = current.claim_sequence + 1;
|
|
31488
|
+
const pendingClaimRequest = {
|
|
31489
|
+
schema_version: "external_inference_job_claim_v1",
|
|
31490
|
+
contract_version: EXTERNAL_INFERENCE_CONTRACT_VERSION,
|
|
31491
|
+
host_id: localState.host_id,
|
|
31492
|
+
host_generation: localState.host_generation,
|
|
31493
|
+
advertisement_generation: current.advertisement_generation,
|
|
31494
|
+
key_generation: localState.key_generation,
|
|
31495
|
+
claim_request_id: stableOperationId("claim", [
|
|
31496
|
+
localState.host_id,
|
|
31497
|
+
localState.host_generation,
|
|
31498
|
+
current.advertisement_generation,
|
|
31499
|
+
claimSequence
|
|
31500
|
+
]),
|
|
31501
|
+
requested_at: isoAt(now())
|
|
31502
|
+
};
|
|
31503
|
+
return {
|
|
31504
|
+
...current,
|
|
31505
|
+
claim_sequence: claimSequence,
|
|
31506
|
+
pending_claim_request: pendingClaimRequest,
|
|
31507
|
+
updated_at: isoAt(now())
|
|
31508
|
+
};
|
|
31509
|
+
});
|
|
31100
31510
|
}
|
|
31101
31511
|
const request = receipt.pending_claim_request;
|
|
31102
31512
|
let claim;
|
|
@@ -31112,12 +31522,11 @@ var init_runner = __esm({
|
|
|
31112
31522
|
requestDrain("authority_lost");
|
|
31113
31523
|
} else if (!this.options.signal?.aborted && error48 instanceof ExternalInferenceMcpError && error48.definitivelyNotApplied) {
|
|
31114
31524
|
if (request.advertisement_generation < receipt.advertisement_generation) {
|
|
31115
|
-
|
|
31116
|
-
...
|
|
31117
|
-
pending_claim_request: null,
|
|
31525
|
+
await mutateReceipt((current) => ({
|
|
31526
|
+
...current,
|
|
31527
|
+
pending_claim_request: current.pending_claim_request?.claim_request_id === request.claim_request_id ? null : current.pending_claim_request,
|
|
31118
31528
|
updated_at: isoAt(now())
|
|
31119
|
-
};
|
|
31120
|
-
await this.persistReceipt(receipt, now);
|
|
31529
|
+
}));
|
|
31121
31530
|
nextClaimAt = now();
|
|
31122
31531
|
} else {
|
|
31123
31532
|
requestDrain("claim_failed");
|
|
@@ -31130,23 +31539,21 @@ var init_runner = __esm({
|
|
|
31130
31539
|
break;
|
|
31131
31540
|
}
|
|
31132
31541
|
if (claim.claim_state === "empty") {
|
|
31133
|
-
|
|
31134
|
-
...
|
|
31135
|
-
pending_claim_request: null,
|
|
31542
|
+
await mutateReceipt((current) => ({
|
|
31543
|
+
...current,
|
|
31544
|
+
pending_claim_request: current.pending_claim_request?.claim_request_id === request.claim_request_id ? null : current.pending_claim_request,
|
|
31136
31545
|
updated_at: isoAt(now())
|
|
31137
|
-
};
|
|
31138
|
-
await this.persistReceipt(receipt, now);
|
|
31546
|
+
}));
|
|
31139
31547
|
nextClaimAt = now() + Math.max(MIN_SLEEP_MS, claim.retry_after_ms);
|
|
31140
31548
|
if (this.options.once) requestDrain("once_empty");
|
|
31141
31549
|
break;
|
|
31142
31550
|
}
|
|
31143
31551
|
if (Date.parse(claim.lease_expires_at) <= now() + MIN_CLAIM_START_WINDOW_MS || Date.parse(claim.deadline_at) <= now() + MIN_CLAIM_START_WINDOW_MS) {
|
|
31144
|
-
|
|
31145
|
-
...
|
|
31146
|
-
pending_claim_request: null,
|
|
31552
|
+
await mutateReceipt((current) => ({
|
|
31553
|
+
...current,
|
|
31554
|
+
pending_claim_request: current.pending_claim_request?.claim_request_id === request.claim_request_id ? null : current.pending_claim_request,
|
|
31147
31555
|
updated_at: isoAt(now())
|
|
31148
|
-
};
|
|
31149
|
-
await this.persistReceipt(receipt, now);
|
|
31556
|
+
}));
|
|
31150
31557
|
nextClaimAt = now();
|
|
31151
31558
|
break;
|
|
31152
31559
|
}
|
|
@@ -31157,7 +31564,8 @@ var init_runner = __esm({
|
|
|
31157
31564
|
const advertisementDueAt = nextAdvertisementAttemptAt > now() ? nextAdvertisementAttemptAt : receipt.advertisement_expires_at ? Date.parse(receipt.advertisement_expires_at) - settings.advertisementRefreshLeadMs : now() + MIN_SLEEP_MS;
|
|
31158
31565
|
const wakeAt = Math.min(
|
|
31159
31566
|
nextClaimAt,
|
|
31160
|
-
|
|
31567
|
+
nextProviderRateLimitRefreshAt,
|
|
31568
|
+
now() + settings.hostHeartbeatMs,
|
|
31161
31569
|
Math.max(now() + MIN_SLEEP_MS, advertisementDueAt)
|
|
31162
31570
|
);
|
|
31163
31571
|
await Promise.race([
|
|
@@ -31166,6 +31574,8 @@ var init_runner = __esm({
|
|
|
31166
31574
|
...active.values()
|
|
31167
31575
|
]);
|
|
31168
31576
|
}
|
|
31577
|
+
stopHostHeartbeatLoop();
|
|
31578
|
+
await hostHeartbeatLoop;
|
|
31169
31579
|
try {
|
|
31170
31580
|
await hostHeartbeat("draining", {
|
|
31171
31581
|
deadlineAtMs: now() + settings.drainTimeoutMs,
|
|
@@ -31200,6 +31610,8 @@ var init_runner = __esm({
|
|
|
31200
31610
|
});
|
|
31201
31611
|
return { claimed, completed, failed, drain_reason: drainReason };
|
|
31202
31612
|
} finally {
|
|
31613
|
+
stopHostHeartbeatLoop();
|
|
31614
|
+
await hostHeartbeatLoop;
|
|
31203
31615
|
removeAbortListener();
|
|
31204
31616
|
unregisterSignals();
|
|
31205
31617
|
try {
|
|
@@ -31271,35 +31683,39 @@ var init_runner = __esm({
|
|
|
31271
31683
|
);
|
|
31272
31684
|
}
|
|
31273
31685
|
if (!options.resumeReceipt) {
|
|
31274
|
-
|
|
31275
|
-
|
|
31276
|
-
|
|
31277
|
-
|
|
31278
|
-
|
|
31279
|
-
|
|
31280
|
-
|
|
31686
|
+
await options.updateReceipt((current) => {
|
|
31687
|
+
const pendingClaimRequest = options.claimRequest && current.pending_claim_request?.claim_request_id === options.claimRequest.claim_request_id ? null : current.pending_claim_request ?? null;
|
|
31688
|
+
return {
|
|
31689
|
+
...current,
|
|
31690
|
+
pending_claim_request: pendingClaimRequest,
|
|
31691
|
+
attempts: { ...current.attempts, [attemptId]: attemptReceipt },
|
|
31692
|
+
updated_at: isoAt(now())
|
|
31693
|
+
};
|
|
31281
31694
|
});
|
|
31282
31695
|
options.onClaimPersisted?.();
|
|
31283
31696
|
}
|
|
31284
31697
|
const updateAttempt = async (changes) => {
|
|
31285
|
-
|
|
31698
|
+
const nextAttemptReceipt = validateAttemptReceipt({
|
|
31286
31699
|
...attemptReceipt,
|
|
31287
31700
|
...changes,
|
|
31288
31701
|
updated_at: isoAt(now())
|
|
31289
31702
|
});
|
|
31290
|
-
|
|
31291
|
-
|
|
31292
|
-
|
|
31703
|
+
attemptReceipt = nextAttemptReceipt;
|
|
31704
|
+
await options.updateReceipt((current) => ({
|
|
31705
|
+
...current,
|
|
31706
|
+
attempts: { ...current.attempts, [attemptId]: nextAttemptReceipt },
|
|
31293
31707
|
updated_at: isoAt(now())
|
|
31294
|
-
});
|
|
31708
|
+
}));
|
|
31295
31709
|
};
|
|
31296
31710
|
const removeAttempt = async () => {
|
|
31297
|
-
|
|
31298
|
-
|
|
31299
|
-
|
|
31300
|
-
|
|
31301
|
-
|
|
31302
|
-
|
|
31711
|
+
await options.updateReceipt((current) => {
|
|
31712
|
+
const attempts = { ...current.attempts };
|
|
31713
|
+
delete attempts[attemptId];
|
|
31714
|
+
return {
|
|
31715
|
+
...current,
|
|
31716
|
+
attempts,
|
|
31717
|
+
updated_at: isoAt(now())
|
|
31718
|
+
};
|
|
31303
31719
|
});
|
|
31304
31720
|
};
|
|
31305
31721
|
if (attemptReceipt.phase === "claimed") {
|
|
@@ -31612,6 +32028,7 @@ var init_runner = __esm({
|
|
|
31612
32028
|
});
|
|
31613
32029
|
}
|
|
31614
32030
|
const classified = classifyFailure(heartbeatFailure ?? error48, adapterResult);
|
|
32031
|
+
const processExit = heartbeatFailure === void 0 && error48 instanceof CodexAppServerError && error48.code === "codex_process_closed" && error48.processExit?.expected_shutdown === false ? error48.processExit : null;
|
|
31615
32032
|
const heartbeatAmbiguous = ambiguousHeartbeatTransport(heartbeatFailure);
|
|
31616
32033
|
const failure = heartbeatAmbiguous ? {
|
|
31617
32034
|
...classified,
|
|
@@ -31665,6 +32082,7 @@ var init_runner = __esm({
|
|
|
31665
32082
|
failure_category: failure.category,
|
|
31666
32083
|
failure_code: failure.code,
|
|
31667
32084
|
retryable: failure.retryable,
|
|
32085
|
+
...processExit ? { process_exit: processExit } : {},
|
|
31668
32086
|
membership_disposition: membershipFailureDisposition({
|
|
31669
32087
|
...failure,
|
|
31670
32088
|
dispatchOutcome
|
|
@@ -31699,6 +32117,7 @@ var init_runner = __esm({
|
|
|
31699
32117
|
failure_category: failureRequest.failure_category,
|
|
31700
32118
|
failure_code: failureRequest.failure_code,
|
|
31701
32119
|
retryable: failureRequest.retryable,
|
|
32120
|
+
process_exit: failureRequest.process_exit ?? null,
|
|
31702
32121
|
dispatch_outcome: failureRequest.outcome.dispatch_outcome,
|
|
31703
32122
|
response_outcome: failureRequest.outcome.response_outcome,
|
|
31704
32123
|
latency_ms: adapterResult?.latencyMs ?? null,
|
|
@@ -33692,6 +34111,17 @@ Durable service:
|
|
|
33692
34111
|
}
|
|
33693
34112
|
};
|
|
33694
34113
|
defaultRunForeground = async (options) => {
|
|
34114
|
+
const localState = await readInferenceHostLocalState(options.config.statePath);
|
|
34115
|
+
const emitCodexDiagnostic = (event) => {
|
|
34116
|
+
options.emitDiagnosticEvent?.({
|
|
34117
|
+
...event,
|
|
34118
|
+
schema_version: "vtx_inference_host_diagnostic_v1",
|
|
34119
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
34120
|
+
instance_name: options.config.instanceName,
|
|
34121
|
+
host_id: localState?.host_id ?? null,
|
|
34122
|
+
display_name: options.displayName
|
|
34123
|
+
});
|
|
34124
|
+
};
|
|
33695
34125
|
const adapter = new CodexSubscriptionAdapter({
|
|
33696
34126
|
resolveBinary: async () => options.verifiedBinary,
|
|
33697
34127
|
createAttemptResources: options.hostResources.createAttemptResources,
|
|
@@ -33699,7 +34129,8 @@ Durable service:
|
|
|
33699
34129
|
guardianReceiptRoot: codexGuardianReceiptRoot(options.config),
|
|
33700
34130
|
recoveryHooks: new FileCodexAttemptRecoveryStore(
|
|
33701
34131
|
codexRecoveryPath(options.config)
|
|
33702
|
-
)
|
|
34132
|
+
),
|
|
34133
|
+
emitDiagnosticEvent: emitCodexDiagnostic
|
|
33703
34134
|
});
|
|
33704
34135
|
const dependencies = createDefaultInferenceHostRunnerDependencies({
|
|
33705
34136
|
apiUrl: options.config.apiUrl,
|
|
@@ -34376,7 +34807,15 @@ Waiting for approval...
|
|
|
34376
34807
|
const emitDiagnosticEvent = parsed.json ? (event) => (dependencies.emitStdout ?? ((text) => {
|
|
34377
34808
|
process.stdout.write(text);
|
|
34378
34809
|
}))(`${JSON.stringify(event)}
|
|
34379
|
-
`) :
|
|
34810
|
+
`) : adapterId === "codex" ? (event) => {
|
|
34811
|
+
if (!["codex_app_server_closed", "codex_app_server_session_started"].includes(
|
|
34812
|
+
String(event.event ?? "")
|
|
34813
|
+
)) return;
|
|
34814
|
+
(dependencies.emitStderr ?? ((text) => {
|
|
34815
|
+
process.stderr.write(text);
|
|
34816
|
+
}))(`[VTX inference-host] ${JSON.stringify(event)}
|
|
34817
|
+
`);
|
|
34818
|
+
} : void 0;
|
|
34380
34819
|
if (adapterId !== "codex") {
|
|
34381
34820
|
const prepared = await (dependencies.prepareDurableAdapter ?? preparePortableDurableAdapter)(
|
|
34382
34821
|
adapterId,
|