@vtxmacro/cli 2026.9.8 → 2026.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/vtx-service-bootstrap.js +1 -1
- package/bin/vtx.js +374 -124
- package/package.json +1 -1
|
@@ -16,7 +16,7 @@ import { fileURLToPath } from "node:url";
|
|
|
16
16
|
// agent-cli-release.json
|
|
17
17
|
var agent_cli_release_default = {
|
|
18
18
|
package_name: "@vtxmacro/cli",
|
|
19
|
-
package_version: "2026.9.
|
|
19
|
+
package_version: "2026.9.9",
|
|
20
20
|
codex_package_name: "@openai/codex",
|
|
21
21
|
codex_version: "0.147.0",
|
|
22
22
|
copilot_sdk_package_name: "@github/copilot-sdk",
|
package/bin/vtx.js
CHANGED
|
@@ -47,7 +47,7 @@ var init_agent_cli_release = __esm({
|
|
|
47
47
|
"agent-cli-release.json"() {
|
|
48
48
|
agent_cli_release_default = {
|
|
49
49
|
package_name: "@vtxmacro/cli",
|
|
50
|
-
package_version: "2026.9.
|
|
50
|
+
package_version: "2026.9.9",
|
|
51
51
|
codex_package_name: "@openai/codex",
|
|
52
52
|
codex_version: "0.147.0",
|
|
53
53
|
copilot_sdk_package_name: "@github/copilot-sdk",
|
|
@@ -23919,6 +23919,7 @@ var init_codex_adapter = __esm({
|
|
|
23919
23919
|
}
|
|
23920
23920
|
if (error48 instanceof CodexAppServerError) {
|
|
23921
23921
|
const dispatchOutcome = error48.code === "server_account_context_expired_before_provider_dispatch" ? "not_dispatched" : error48.dispatchOutcome === "not_dispatched" ? currentDispatchOutcome : error48.dispatchOutcome;
|
|
23922
|
+
input.onTerminalDispatchOutcome?.(dispatchOutcome);
|
|
23922
23923
|
if (baseCheckpoint && !completedResult) {
|
|
23923
23924
|
latestCheckpoint = {
|
|
23924
23925
|
...baseCheckpoint,
|
|
@@ -23962,6 +23963,7 @@ var init_codex_adapter = __esm({
|
|
|
23962
23963
|
dispatchOutcome: currentDispatchOutcome,
|
|
23963
23964
|
cause: error48
|
|
23964
23965
|
});
|
|
23966
|
+
input.onTerminalDispatchOutcome?.(classified.dispatchOutcome);
|
|
23965
23967
|
if (baseCheckpoint && !completedResult) {
|
|
23966
23968
|
latestCheckpoint = {
|
|
23967
23969
|
...baseCheckpoint,
|
|
@@ -24716,9 +24718,14 @@ ${input.outputSchemaJson}`
|
|
|
24716
24718
|
terminalStatus: "completed"
|
|
24717
24719
|
};
|
|
24718
24720
|
} catch (error48) {
|
|
24719
|
-
if (error48 instanceof CodexAppServerError)
|
|
24721
|
+
if (error48 instanceof CodexAppServerError) {
|
|
24722
|
+
input.onTerminalDispatchOutcome?.(error48.dispatchOutcome);
|
|
24723
|
+
throw error48;
|
|
24724
|
+
}
|
|
24720
24725
|
const cancelled = input.signal?.aborted === true;
|
|
24721
24726
|
const deadline = Date.now() >= input.deadlineAtMs;
|
|
24727
|
+
const dispatchOutcome = providerReceiptObserved ? "confirmed_dispatched" : dispatchEntered ? "outcome_unknown" : "not_dispatched";
|
|
24728
|
+
input.onTerminalDispatchOutcome?.(dispatchOutcome);
|
|
24722
24729
|
if (dispatchEntered && session) {
|
|
24723
24730
|
await session.abort().catch(() => void 0);
|
|
24724
24731
|
}
|
|
@@ -24727,7 +24734,7 @@ ${input.outputSchemaJson}`
|
|
|
24727
24734
|
category: cancelled ? "cancelled" : deadline ? "timeout" : "adapter",
|
|
24728
24735
|
code: cancelled ? "cancelled" : deadline ? "deadline_exceeded" : "copilot_adapter_failure",
|
|
24729
24736
|
retryable: false,
|
|
24730
|
-
dispatchOutcome
|
|
24737
|
+
dispatchOutcome,
|
|
24731
24738
|
cause: error48
|
|
24732
24739
|
});
|
|
24733
24740
|
} finally {
|
|
@@ -25500,12 +25507,14 @@ ${input.outputSchemaJson}`,
|
|
|
25500
25507
|
} catch (error48) {
|
|
25501
25508
|
const cancelled = input.signal?.aborted === true;
|
|
25502
25509
|
const deadline = !cancelled && boundary.signal.aborted;
|
|
25503
|
-
|
|
25510
|
+
const mapped = mapDeepSeekFailure({
|
|
25504
25511
|
error: error48,
|
|
25505
25512
|
cancelled,
|
|
25506
25513
|
deadline,
|
|
25507
25514
|
dispatchOutcome: proxyDispatchOutcome(proxy)
|
|
25508
25515
|
});
|
|
25516
|
+
input.onTerminalDispatchOutcome?.(mapped.dispatchOutcome);
|
|
25517
|
+
throw mapped;
|
|
25509
25518
|
} finally {
|
|
25510
25519
|
boundary.cleanup();
|
|
25511
25520
|
await proxy.close();
|
|
@@ -33336,6 +33345,7 @@ var init_runner = __esm({
|
|
|
33336
33345
|
let stopHostHeartbeatLoop = () => {
|
|
33337
33346
|
};
|
|
33338
33347
|
let hostHeartbeatLoop = Promise.resolve();
|
|
33348
|
+
let providerDispatchWatchdogDrainRequested = false;
|
|
33339
33349
|
const agentAbort = new AbortController();
|
|
33340
33350
|
let agentLoop = Promise.resolve();
|
|
33341
33351
|
let wakeDrain;
|
|
@@ -34135,6 +34145,24 @@ var init_runner = __esm({
|
|
|
34135
34145
|
active_attempts: active.size
|
|
34136
34146
|
});
|
|
34137
34147
|
},
|
|
34148
|
+
onAttemptStage: (stage) => {
|
|
34149
|
+
emitDiagnostic("attempt_stage_changed", {
|
|
34150
|
+
stage,
|
|
34151
|
+
job_id: claim.job_id,
|
|
34152
|
+
attempt_id: attemptId,
|
|
34153
|
+
active_attempts: active.size
|
|
34154
|
+
});
|
|
34155
|
+
},
|
|
34156
|
+
onProviderDispatchWatchdog: (observation) => {
|
|
34157
|
+
providerDispatchWatchdogDrainRequested = true;
|
|
34158
|
+
emitDiagnostic("provider_dispatch_watchdog_expired", {
|
|
34159
|
+
...observation,
|
|
34160
|
+
job_id: claim.job_id,
|
|
34161
|
+
attempt_id: attemptId,
|
|
34162
|
+
active_attempts: active.size
|
|
34163
|
+
});
|
|
34164
|
+
requestDrain("provider_dispatch_watchdog_expired");
|
|
34165
|
+
},
|
|
34138
34166
|
startRetryCount: nextStartRetryCount,
|
|
34139
34167
|
onAttemptStartRetry: (observation) => {
|
|
34140
34168
|
attemptStartRetryCounts.set(attemptId, observation.retry_count);
|
|
@@ -34501,7 +34529,8 @@ var init_runner = __esm({
|
|
|
34501
34529
|
});
|
|
34502
34530
|
});
|
|
34503
34531
|
const shouldRollover = !this.options.once && (settings.maxConcurrency === null || active.size + prepared.length < settings.maxConcurrency);
|
|
34504
|
-
|
|
34532
|
+
pendingClaimPromotions += 1;
|
|
34533
|
+
const batchReceiptDurability = mutateReceipt((current) => {
|
|
34505
34534
|
if (current.pending_claim_request?.claim_request_id !== request.claim_request_id) runnerIdentityMismatch("Batched claim receipt");
|
|
34506
34535
|
const attempts = { ...current.attempts };
|
|
34507
34536
|
for (const attempt of prepared) {
|
|
@@ -34517,6 +34546,8 @@ var init_runner = __esm({
|
|
|
34517
34546
|
updated_at: isoAt(now())
|
|
34518
34547
|
};
|
|
34519
34548
|
return shouldRollover ? preparePendingClaim(promoted) : promoted;
|
|
34549
|
+
}).then(() => void 0).finally(() => {
|
|
34550
|
+
pendingClaimPromotions = Math.max(0, pendingClaimPromotions - 1);
|
|
34520
34551
|
});
|
|
34521
34552
|
for (let index = 0; index < prepared.length; index += 1) {
|
|
34522
34553
|
const item = claim.claims[index];
|
|
@@ -34529,7 +34560,8 @@ var init_runner = __esm({
|
|
|
34529
34560
|
provider_dispatch_freshness_remaining_ms: item.provider_dispatch_freshness_remaining_ms
|
|
34530
34561
|
},
|
|
34531
34562
|
roundTripMs: claimTransportResidualMs,
|
|
34532
|
-
claimResponseReceivedAtMs
|
|
34563
|
+
claimResponseReceivedAtMs,
|
|
34564
|
+
receiptDurability: batchReceiptDurability
|
|
34533
34565
|
});
|
|
34534
34566
|
}
|
|
34535
34567
|
if (this.options.once) requestDrain("once_complete");
|
|
@@ -34573,13 +34605,17 @@ var init_runner = __esm({
|
|
|
34573
34605
|
]);
|
|
34574
34606
|
}
|
|
34575
34607
|
stopHostHeartbeatLoop();
|
|
34576
|
-
|
|
34577
|
-
|
|
34578
|
-
|
|
34579
|
-
|
|
34580
|
-
|
|
34581
|
-
|
|
34582
|
-
|
|
34608
|
+
if (providerDispatchWatchdogDrainRequested) {
|
|
34609
|
+
await settlesWithin(hostHeartbeatLoop, MIN_SLEEP_MS);
|
|
34610
|
+
} else {
|
|
34611
|
+
await hostHeartbeatLoop;
|
|
34612
|
+
await settlesWithin(
|
|
34613
|
+
hostHeartbeat("draining", {
|
|
34614
|
+
deadlineAtMs: now() + settings.drainTimeoutMs,
|
|
34615
|
+
timeoutMs: settings.drainTimeoutMs
|
|
34616
|
+
}),
|
|
34617
|
+
settings.drainTimeoutMs
|
|
34618
|
+
);
|
|
34583
34619
|
}
|
|
34584
34620
|
if (active.size > 0) {
|
|
34585
34621
|
let timedOut = false;
|
|
@@ -34617,7 +34653,11 @@ var init_runner = __esm({
|
|
|
34617
34653
|
]);
|
|
34618
34654
|
await settlesWithin(agentCleanup, settings.drainTimeoutMs);
|
|
34619
34655
|
stopHostHeartbeatLoop();
|
|
34620
|
-
|
|
34656
|
+
if (providerDispatchWatchdogDrainRequested) {
|
|
34657
|
+
await settlesWithin(hostHeartbeatLoop, MIN_SLEEP_MS);
|
|
34658
|
+
} else {
|
|
34659
|
+
await hostHeartbeatLoop;
|
|
34660
|
+
}
|
|
34621
34661
|
removeAbortListener();
|
|
34622
34662
|
unregisterSignals();
|
|
34623
34663
|
try {
|
|
@@ -34695,6 +34735,53 @@ var init_runner = __esm({
|
|
|
34695
34735
|
"The persisted same-attempt recovery scope is invalid."
|
|
34696
34736
|
);
|
|
34697
34737
|
}
|
|
34738
|
+
let lastPreProviderStage = "claimed";
|
|
34739
|
+
const reportPreProviderStage = (stage) => {
|
|
34740
|
+
if (lastPreProviderStage === stage && stage !== "claimed") return;
|
|
34741
|
+
lastPreProviderStage = stage;
|
|
34742
|
+
options.onAttemptStage?.(stage);
|
|
34743
|
+
};
|
|
34744
|
+
reportPreProviderStage("claimed");
|
|
34745
|
+
const attemptAbort = new AbortController();
|
|
34746
|
+
const relayAbort = () => attemptAbort.abort();
|
|
34747
|
+
signal.addEventListener("abort", relayAbort, { once: true });
|
|
34748
|
+
if (signal.aborted) attemptAbort.abort();
|
|
34749
|
+
let watchdogArmedAtMs = now();
|
|
34750
|
+
let providerDispatchWatchdog = null;
|
|
34751
|
+
let providerDispatchWatchdogExpired = false;
|
|
34752
|
+
let resolveProviderDispatchWatchdog;
|
|
34753
|
+
const providerDispatchWatchdogSignal = new Promise((resolve6) => {
|
|
34754
|
+
resolveProviderDispatchWatchdog = resolve6;
|
|
34755
|
+
});
|
|
34756
|
+
const clearProviderDispatchWatchdog = () => {
|
|
34757
|
+
if (providerDispatchWatchdog !== null) clearTimeout(providerDispatchWatchdog);
|
|
34758
|
+
providerDispatchWatchdog = null;
|
|
34759
|
+
};
|
|
34760
|
+
const armProviderDispatchWatchdog = (deadlineAtMs) => {
|
|
34761
|
+
clearProviderDispatchWatchdog();
|
|
34762
|
+
if (deadlineAtMs === void 0 || attemptReceipt.phase === "dispatched" || attemptAbort.signal.aborted) return;
|
|
34763
|
+
watchdogArmedAtMs = now();
|
|
34764
|
+
providerDispatchWatchdog = setTimeout(() => {
|
|
34765
|
+
providerDispatchWatchdog = null;
|
|
34766
|
+
if (providerDispatchedAt !== null || attemptAbort.signal.aborted) return;
|
|
34767
|
+
providerDispatchWatchdogExpired = true;
|
|
34768
|
+
options.onProviderDispatchWatchdog?.({
|
|
34769
|
+
last_stage: lastPreProviderStage,
|
|
34770
|
+
elapsed_ms: Math.max(0, now() - watchdogArmedAtMs),
|
|
34771
|
+
remaining_ms: Math.max(0, deadlineAtMs - now())
|
|
34772
|
+
});
|
|
34773
|
+
resolveProviderDispatchWatchdog();
|
|
34774
|
+
attemptAbort.abort();
|
|
34775
|
+
}, Math.max(MIN_SLEEP_MS, deadlineAtMs - now()));
|
|
34776
|
+
};
|
|
34777
|
+
const awaitPreProviderDurability = async (operation) => Promise.race([
|
|
34778
|
+
operation,
|
|
34779
|
+
providerDispatchWatchdogSignal.then(() => {
|
|
34780
|
+
throw new InferenceHostRecoveryRequiredError(
|
|
34781
|
+
"Provider dispatch stopped because its durable receipt missed the freshness deadline."
|
|
34782
|
+
);
|
|
34783
|
+
})
|
|
34784
|
+
]);
|
|
34698
34785
|
const persistAttempt = async (changes, clearPendingClaim = false) => {
|
|
34699
34786
|
const nextAttemptReceipt = validateAttemptReceipt({
|
|
34700
34787
|
...attemptReceipt,
|
|
@@ -34761,6 +34848,84 @@ var init_runner = __esm({
|
|
|
34761
34848
|
});
|
|
34762
34849
|
options.onClaimPersisted?.();
|
|
34763
34850
|
};
|
|
34851
|
+
const terminalizeProviderDispatchWatchdog = async () => {
|
|
34852
|
+
const failureCode = "server_account_context_expired_before_provider_dispatch";
|
|
34853
|
+
const failure = {
|
|
34854
|
+
category: "timeout",
|
|
34855
|
+
code: failureCode,
|
|
34856
|
+
retryable: false,
|
|
34857
|
+
dispatchOutcome: "not_dispatched",
|
|
34858
|
+
effectiveModel: null,
|
|
34859
|
+
effectiveEffort: null
|
|
34860
|
+
};
|
|
34861
|
+
const failureId = stableOperationId("failure", [
|
|
34862
|
+
claim.job_id,
|
|
34863
|
+
attemptId,
|
|
34864
|
+
failure.dispatchOutcome,
|
|
34865
|
+
failure.category,
|
|
34866
|
+
failure.code
|
|
34867
|
+
]);
|
|
34868
|
+
const failureRequest = jobFailRequestSchema.parse({
|
|
34869
|
+
schema_version: "external_inference_job_fail_v1",
|
|
34870
|
+
contract_version: EXTERNAL_INFERENCE_CONTRACT_VERSION,
|
|
34871
|
+
job_id: claim.job_id,
|
|
34872
|
+
host_id: claim.host_id,
|
|
34873
|
+
host_generation: claim.host_generation,
|
|
34874
|
+
advertisement_generation: claim.advertisement_generation,
|
|
34875
|
+
key_generation: claim.key_generation,
|
|
34876
|
+
claim_generation: claim.claim_generation,
|
|
34877
|
+
controller_generation: claim.controller_generation,
|
|
34878
|
+
claim_handle: claim.claim_handle,
|
|
34879
|
+
attempt_id: attemptId,
|
|
34880
|
+
failure_id: failureId,
|
|
34881
|
+
response_mode: claim.response_mode,
|
|
34882
|
+
output_schema_version: claim.output_schema_version,
|
|
34883
|
+
attempt_status: "failed_no_response",
|
|
34884
|
+
outcome: {
|
|
34885
|
+
dispatch_outcome: failure.dispatchOutcome,
|
|
34886
|
+
response_outcome: "none"
|
|
34887
|
+
},
|
|
34888
|
+
model: {
|
|
34889
|
+
requested_model: claim.requested_model,
|
|
34890
|
+
requested_reasoning_effort: claim.requested_reasoning_effort,
|
|
34891
|
+
effective_model: null,
|
|
34892
|
+
effective_reasoning_effort: null,
|
|
34893
|
+
adapter_request_id: null,
|
|
34894
|
+
adapter_response_id: null
|
|
34895
|
+
},
|
|
34896
|
+
usage: null,
|
|
34897
|
+
failure_category: failure.category,
|
|
34898
|
+
failure_code: failure.code,
|
|
34899
|
+
retryable: failure.retryable,
|
|
34900
|
+
provider_dispatch_freshness_remaining_ms: 0,
|
|
34901
|
+
membership_disposition: membershipFailureDisposition(failure),
|
|
34902
|
+
failed_at: isoAt(now())
|
|
34903
|
+
});
|
|
34904
|
+
void persistAttempt({
|
|
34905
|
+
phase: "terminal_pending",
|
|
34906
|
+
dispatch_outcome: "not_dispatched",
|
|
34907
|
+
terminal_operation_id: failureId,
|
|
34908
|
+
terminal_request: failureRequest
|
|
34909
|
+
}).catch(() => void 0);
|
|
34910
|
+
const evidenceDeadlineAtMs = Date.parse(jobInput.evidence_expires_at);
|
|
34911
|
+
const failureResult = await retryExact(() => mcp.callTool(
|
|
34912
|
+
"inference.job.fail",
|
|
34913
|
+
failureRequest,
|
|
34914
|
+
{ signal, deadlineAtMs: evidenceDeadlineAtMs }
|
|
34915
|
+
), { signal, deadlineAtMs: evidenceDeadlineAtMs });
|
|
34916
|
+
assertTerminalResult(failureRequest, failureResult);
|
|
34917
|
+
options.onAttemptOutcome?.({
|
|
34918
|
+
outcome: "failed",
|
|
34919
|
+
failure_category: failureRequest.failure_category,
|
|
34920
|
+
failure_code: failureRequest.failure_code,
|
|
34921
|
+
retryable: failureRequest.retryable,
|
|
34922
|
+
dispatch_outcome: failureRequest.outcome.dispatch_outcome,
|
|
34923
|
+
response_outcome: failureRequest.outcome.response_outcome,
|
|
34924
|
+
latency_ms: null,
|
|
34925
|
+
usage: null
|
|
34926
|
+
});
|
|
34927
|
+
return "failed";
|
|
34928
|
+
};
|
|
34764
34929
|
let outputContract = null;
|
|
34765
34930
|
let inputValidationError;
|
|
34766
34931
|
try {
|
|
@@ -34769,121 +34934,150 @@ var init_runner = __esm({
|
|
|
34769
34934
|
} catch (error48) {
|
|
34770
34935
|
inputValidationError = error48;
|
|
34771
34936
|
}
|
|
34772
|
-
|
|
34773
|
-
|
|
34774
|
-
|
|
34775
|
-
|
|
34776
|
-
|
|
34777
|
-
|
|
34778
|
-
|
|
34779
|
-
|
|
34780
|
-
|
|
34781
|
-
|
|
34782
|
-
|
|
34783
|
-
|
|
34784
|
-
|
|
34785
|
-
|
|
34786
|
-
|
|
34787
|
-
|
|
34788
|
-
|
|
34789
|
-
|
|
34790
|
-
|
|
34791
|
-
|
|
34792
|
-
|
|
34793
|
-
|
|
34794
|
-
|
|
34795
|
-
startCallCompletedAtMs = now();
|
|
34796
|
-
return result2;
|
|
34797
|
-
});
|
|
34798
|
-
const [startOutcome, claimPromotionOutcome] = await Promise.allSettled([
|
|
34799
|
-
startCallPromise,
|
|
34800
|
-
claimPromotionPromise
|
|
34801
|
-
]);
|
|
34802
|
-
if (claimPromotionOutcome.status === "rejected") {
|
|
34803
|
-
throw claimPromotionOutcome.reason;
|
|
34804
|
-
}
|
|
34805
|
-
if (startOutcome.status === "rejected") {
|
|
34806
|
-
const error48 = startOutcome.reason;
|
|
34807
|
-
if (error48 instanceof ExternalInferenceMcpError && error48.serverFailureCode === "server_account_context_expired_before_provider_dispatch") {
|
|
34808
|
-
await removeAttempt();
|
|
34809
|
-
options.onAttemptOutcome?.({
|
|
34810
|
-
outcome: "freshness_rejected_before_provider_dispatch",
|
|
34811
|
-
failure_category: "freshness",
|
|
34812
|
-
failure_code: error48.serverFailureCode,
|
|
34813
|
-
dispatch_outcome: "not_dispatched"
|
|
34814
|
-
});
|
|
34815
|
-
return "failed";
|
|
34816
|
-
}
|
|
34817
|
-
if (attemptDeadlineAtMs <= now() + MIN_CLAIM_START_WINDOW_MS) {
|
|
34818
|
-
await removeAttempt();
|
|
34819
|
-
return "failed";
|
|
34937
|
+
try {
|
|
34938
|
+
if (attemptReceipt.phase === "claimed") {
|
|
34939
|
+
const attemptDeadlineAtMs = Math.min(
|
|
34940
|
+
Date.parse(claim.deadline_at),
|
|
34941
|
+
Date.parse(claim.lease_expires_at)
|
|
34942
|
+
);
|
|
34943
|
+
if (attemptDeadlineAtMs <= now() + MIN_CLAIM_START_WINDOW_MS) {
|
|
34944
|
+
await removeAttempt();
|
|
34945
|
+
clearProviderDispatchWatchdog();
|
|
34946
|
+
signal.removeEventListener("abort", relayAbort);
|
|
34947
|
+
return "failed";
|
|
34948
|
+
}
|
|
34949
|
+
const startCallStartedAtMs = now();
|
|
34950
|
+
let startCallCompletedAtMs = startCallStartedAtMs;
|
|
34951
|
+
const acceptStartResult = (result2, freshnessAnchorAtMs, roundTripMs) => {
|
|
34952
|
+
assertStartResult(startRequest, result2);
|
|
34953
|
+
if (serverProviderDispatchDeadlineMs === void 0) return;
|
|
34954
|
+
const serverRemaining = result2.provider_dispatch_freshness_remaining_ms;
|
|
34955
|
+
if (!Number.isSafeInteger(serverRemaining) || Number(serverRemaining) < 0) {
|
|
34956
|
+
throw new InferenceHostRunnerError(
|
|
34957
|
+
"provider_dispatch_freshness_receipt_invalid",
|
|
34958
|
+
"The Server provider-dispatch freshness receipt is unavailable."
|
|
34959
|
+
);
|
|
34820
34960
|
}
|
|
34821
|
-
|
|
34822
|
-
|
|
34823
|
-
const
|
|
34824
|
-
|
|
34825
|
-
|
|
34961
|
+
startRoundTripMs = Math.max(0, roundTripMs);
|
|
34962
|
+
serverReportedFreshnessRemainingMs = Number(serverRemaining);
|
|
34963
|
+
const conservativeRemainingMs = Math.max(
|
|
34964
|
+
0,
|
|
34965
|
+
serverReportedFreshnessRemainingMs - startRoundTripMs
|
|
34826
34966
|
);
|
|
34827
|
-
|
|
34828
|
-
|
|
34829
|
-
|
|
34967
|
+
providerDispatchNotAfterMs = freshnessAnchorAtMs + conservativeRemainingMs;
|
|
34968
|
+
armProviderDispatchWatchdog(providerDispatchNotAfterMs);
|
|
34969
|
+
};
|
|
34970
|
+
if (options.prestarted) {
|
|
34971
|
+
startCallCompletedAtMs = now();
|
|
34972
|
+
acceptStartResult(
|
|
34973
|
+
options.prestarted.result,
|
|
34974
|
+
options.prestarted.claimResponseReceivedAtMs,
|
|
34975
|
+
options.prestarted.roundTripMs
|
|
34830
34976
|
);
|
|
34831
|
-
|
|
34832
|
-
|
|
34833
|
-
|
|
34977
|
+
if (options.prestarted.receiptDurability) {
|
|
34978
|
+
reportPreProviderStage("receipt_persistence");
|
|
34979
|
+
await awaitPreProviderDurability(options.prestarted.receiptDurability);
|
|
34980
|
+
}
|
|
34981
|
+
} else {
|
|
34982
|
+
reportPreProviderStage("receipt_persistence");
|
|
34983
|
+
const claimPromotionPromise = options.resumeReceipt ? Promise.resolve() : awaitPreProviderDurability(
|
|
34984
|
+
persistAttempt({ phase: "claimed" }, true)
|
|
34834
34985
|
);
|
|
34835
|
-
const
|
|
34836
|
-
|
|
34837
|
-
|
|
34838
|
-
|
|
34839
|
-
|
|
34840
|
-
|
|
34841
|
-
|
|
34842
|
-
|
|
34843
|
-
|
|
34844
|
-
|
|
34845
|
-
|
|
34846
|
-
|
|
34986
|
+
const startCallPromise = retryExact(() => mcp.callTool(
|
|
34987
|
+
"inference.job.start",
|
|
34988
|
+
startRequest,
|
|
34989
|
+
{ signal, deadlineAtMs: attemptDeadlineAtMs }
|
|
34990
|
+
), { signal, deadlineAtMs: attemptDeadlineAtMs }).then((result2) => {
|
|
34991
|
+
startCallCompletedAtMs = now();
|
|
34992
|
+
acceptStartResult(
|
|
34993
|
+
result2,
|
|
34994
|
+
startCallCompletedAtMs,
|
|
34995
|
+
startCallCompletedAtMs - startCallStartedAtMs
|
|
34996
|
+
);
|
|
34997
|
+
return result2;
|
|
34847
34998
|
});
|
|
34848
|
-
await
|
|
34849
|
-
|
|
34999
|
+
const [startOutcome, claimPromotionOutcome] = await Promise.allSettled([
|
|
35000
|
+
startCallPromise,
|
|
35001
|
+
claimPromotionPromise
|
|
35002
|
+
]);
|
|
35003
|
+
if (claimPromotionOutcome.status === "rejected") {
|
|
35004
|
+
throw claimPromotionOutcome.reason;
|
|
35005
|
+
}
|
|
35006
|
+
if (startOutcome.status === "rejected") {
|
|
35007
|
+
const error48 = startOutcome.reason;
|
|
35008
|
+
if (error48 instanceof ExternalInferenceMcpError && error48.serverFailureCode === "server_account_context_expired_before_provider_dispatch") {
|
|
35009
|
+
await removeAttempt();
|
|
35010
|
+
options.onAttemptOutcome?.({
|
|
35011
|
+
outcome: "freshness_rejected_before_provider_dispatch",
|
|
35012
|
+
failure_category: "freshness",
|
|
35013
|
+
failure_code: error48.serverFailureCode,
|
|
35014
|
+
dispatch_outcome: "not_dispatched"
|
|
35015
|
+
});
|
|
35016
|
+
clearProviderDispatchWatchdog();
|
|
35017
|
+
signal.removeEventListener("abort", relayAbort);
|
|
35018
|
+
return "failed";
|
|
35019
|
+
}
|
|
35020
|
+
if (attemptDeadlineAtMs <= now() + MIN_CLAIM_START_WINDOW_MS) {
|
|
35021
|
+
await removeAttempt();
|
|
35022
|
+
clearProviderDispatchWatchdog();
|
|
35023
|
+
signal.removeEventListener("abort", relayAbort);
|
|
35024
|
+
return "failed";
|
|
35025
|
+
}
|
|
35026
|
+
if (signal.aborted) throw error48;
|
|
35027
|
+
if (controlPlaneFatal(error48)) throw error48;
|
|
35028
|
+
const remainingMs = Math.max(
|
|
35029
|
+
MIN_SLEEP_MS,
|
|
35030
|
+
attemptDeadlineAtMs - now() - MIN_CLAIM_START_WINDOW_MS
|
|
35031
|
+
);
|
|
35032
|
+
const retryCount = Math.min(
|
|
35033
|
+
Math.max(1, options.startRetryCount),
|
|
35034
|
+
Number.MAX_SAFE_INTEGER
|
|
35035
|
+
);
|
|
35036
|
+
const retryDelayMs = Math.min(
|
|
35037
|
+
remoteRetryAfterMs(error48, attemptStartRetryFallbackMs(retryCount)),
|
|
35038
|
+
remainingMs
|
|
35039
|
+
);
|
|
35040
|
+
const rawErrorCode = error48 && typeof error48 === "object" && "code" in error48 ? String(error48.code) : "start_acknowledgement_failed";
|
|
35041
|
+
options.onAttemptStartRetry?.({
|
|
35042
|
+
failure_stage: "job_start_acknowledgement",
|
|
35043
|
+
error_code: safeFailureCode(rawErrorCode, "start_acknowledgement_failed"),
|
|
35044
|
+
http_status_code: error48 instanceof ExternalInferenceMcpError ? error48.httpStatusCode : null,
|
|
35045
|
+
retryable: retryableRemoteError(error48),
|
|
35046
|
+
definitively_not_applied: Boolean(
|
|
35047
|
+
error48 && typeof error48 === "object" && "definitivelyNotApplied" in error48 && error48.definitivelyNotApplied === true
|
|
35048
|
+
),
|
|
35049
|
+
retry_after_ms: error48 instanceof ExternalInferenceMcpError ? error48.retryAfterMs : null,
|
|
35050
|
+
retry_count: retryCount,
|
|
35051
|
+
next_delay_ms: retryDelayMs
|
|
35052
|
+
});
|
|
35053
|
+
await options.sleep(retryDelayMs, signal);
|
|
35054
|
+
clearProviderDispatchWatchdog();
|
|
35055
|
+
signal.removeEventListener("abort", relayAbort);
|
|
35056
|
+
return "retry_claimed";
|
|
35057
|
+
}
|
|
34850
35058
|
}
|
|
34851
|
-
startResult = startOutcome.value;
|
|
34852
35059
|
}
|
|
34853
|
-
|
|
34854
|
-
|
|
34855
|
-
|
|
34856
|
-
|
|
34857
|
-
|
|
34858
|
-
"
|
|
34859
|
-
|
|
34860
|
-
)
|
|
34861
|
-
}
|
|
34862
|
-
if (!options.prestarted) {
|
|
34863
|
-
startRoundTripMs = Math.max(0, startCallCompletedAtMs - startCallStartedAtMs);
|
|
34864
|
-
}
|
|
34865
|
-
serverReportedFreshnessRemainingMs = Number(serverRemaining);
|
|
34866
|
-
const conservativeRemainingMs = Math.max(
|
|
34867
|
-
0,
|
|
34868
|
-
serverReportedFreshnessRemainingMs - startRoundTripMs
|
|
35060
|
+
if (attemptReceipt.phase !== "dispatched") {
|
|
35061
|
+
const providerDispatchable = inputValidationError === void 0 && outputContract !== null;
|
|
35062
|
+
reportPreProviderStage("receipt_persistence");
|
|
35063
|
+
await awaitPreProviderDurability(
|
|
35064
|
+
persistAttempt({
|
|
35065
|
+
phase: providerDispatchable ? "dispatched" : "started",
|
|
35066
|
+
dispatch_outcome: providerDispatchable ? "outcome_unknown" : "not_dispatched"
|
|
35067
|
+
})
|
|
34869
35068
|
);
|
|
34870
|
-
const freshnessAnchorAtMs = options.prestarted ? options.prestarted.claimResponseReceivedAtMs : startCallCompletedAtMs;
|
|
34871
|
-
providerDispatchNotAfterMs = freshnessAnchorAtMs + conservativeRemainingMs;
|
|
34872
35069
|
}
|
|
34873
|
-
|
|
34874
|
-
|
|
34875
|
-
|
|
34876
|
-
|
|
34877
|
-
|
|
34878
|
-
|
|
34879
|
-
}
|
|
35070
|
+
reportPreProviderStage("thread_setup");
|
|
35071
|
+
} catch (error48) {
|
|
35072
|
+
clearProviderDispatchWatchdog();
|
|
35073
|
+
signal.removeEventListener("abort", relayAbort);
|
|
35074
|
+
if (providerDispatchWatchdogExpired) {
|
|
35075
|
+
return terminalizeProviderDispatchWatchdog();
|
|
35076
|
+
}
|
|
35077
|
+
throw error48;
|
|
34880
35078
|
}
|
|
34881
35079
|
let adapterResult;
|
|
34882
35080
|
let heartbeatFailure;
|
|
34883
|
-
const attemptAbort = new AbortController();
|
|
34884
|
-
const relayAbort = () => attemptAbort.abort();
|
|
34885
|
-
signal.addEventListener("abort", relayAbort, { once: true });
|
|
34886
|
-
if (signal.aborted) attemptAbort.abort();
|
|
34887
35081
|
let heartbeatStopped = false;
|
|
34888
35082
|
const heartbeatLoop = (async () => {
|
|
34889
35083
|
while (!heartbeatStopped && !attemptAbort.signal.aborted) {
|
|
@@ -34959,7 +35153,12 @@ var init_runner = __esm({
|
|
|
34959
35153
|
dispatchOutcome: "outcome_unknown"
|
|
34960
35154
|
});
|
|
34961
35155
|
}
|
|
34962
|
-
|
|
35156
|
+
let resolveTerminalDispatchOutcome;
|
|
35157
|
+
const terminalDispatchOutcomeSignal = new Promise((resolve6) => {
|
|
35158
|
+
resolveTerminalDispatchOutcome = resolve6;
|
|
35159
|
+
});
|
|
35160
|
+
let terminalDispatchOutcomeObserved = false;
|
|
35161
|
+
const adapterAttempt = this.dependencies.codexAdapter.runAttempt({
|
|
34963
35162
|
attemptId,
|
|
34964
35163
|
inputSha256: jobInput.input_sha256,
|
|
34965
35164
|
systemPrompt: jobInput.system_prompt,
|
|
@@ -34972,6 +35171,8 @@ var init_runner = __esm({
|
|
|
34972
35171
|
onProviderDispatch: () => {
|
|
34973
35172
|
const dispatchedAtMs = now();
|
|
34974
35173
|
providerDispatchedAt ??= isoAt(dispatchedAtMs);
|
|
35174
|
+
clearProviderDispatchWatchdog();
|
|
35175
|
+
options.onAttemptStage?.("provider_dispatched");
|
|
34975
35176
|
if (providerDispatchNotAfterMs === void 0) return;
|
|
34976
35177
|
providerDispatchFreshnessRemainingMs = Math.max(
|
|
34977
35178
|
0,
|
|
@@ -34984,8 +35185,49 @@ var init_runner = __esm({
|
|
|
34984
35185
|
start_round_trip_ms: startRoundTripMs
|
|
34985
35186
|
});
|
|
34986
35187
|
},
|
|
35188
|
+
onTerminalDispatchOutcome: (outcome) => {
|
|
35189
|
+
if (terminalDispatchOutcomeObserved) return;
|
|
35190
|
+
terminalDispatchOutcomeObserved = true;
|
|
35191
|
+
resolveTerminalDispatchOutcome(outcome);
|
|
35192
|
+
},
|
|
34987
35193
|
signal: attemptAbort.signal
|
|
34988
35194
|
});
|
|
35195
|
+
adapterResult = await Promise.race([
|
|
35196
|
+
adapterAttempt,
|
|
35197
|
+
providerDispatchWatchdogSignal.then(async () => {
|
|
35198
|
+
const classified = await Promise.race([
|
|
35199
|
+
adapterAttempt.then(
|
|
35200
|
+
(result2) => ({ kind: "result", result: result2 }),
|
|
35201
|
+
(error48) => ({ kind: "error", error: error48 })
|
|
35202
|
+
),
|
|
35203
|
+
terminalDispatchOutcomeSignal.then((outcome) => ({
|
|
35204
|
+
kind: "dispatch_outcome",
|
|
35205
|
+
outcome
|
|
35206
|
+
})),
|
|
35207
|
+
options.sleep(options.attemptHeartbeatMs).then(() => ({
|
|
35208
|
+
kind: "unsettled"
|
|
35209
|
+
}))
|
|
35210
|
+
]);
|
|
35211
|
+
if (classified.kind === "result") return classified.result;
|
|
35212
|
+
if (classified.kind === "error") throw classified.error;
|
|
35213
|
+
if (classified.kind === "dispatch_outcome" && classified.outcome === "not_dispatched") {
|
|
35214
|
+
throw new CodexAppServerError({
|
|
35215
|
+
message: "The Server account context expired before provider dispatch.",
|
|
35216
|
+
category: "timeout",
|
|
35217
|
+
code: "server_account_context_expired_before_provider_dispatch",
|
|
35218
|
+
retryable: false,
|
|
35219
|
+
dispatchOutcome: "not_dispatched"
|
|
35220
|
+
});
|
|
35221
|
+
}
|
|
35222
|
+
throw new CodexAppServerError({
|
|
35223
|
+
message: "Provider dispatch could not be excluded after the freshness watchdog expired.",
|
|
35224
|
+
category: "timeout",
|
|
35225
|
+
code: "provider_dispatch_outcome_unknown",
|
|
35226
|
+
retryable: false,
|
|
35227
|
+
dispatchOutcome: classified.kind === "dispatch_outcome" ? classified.outcome : "outcome_unknown"
|
|
35228
|
+
});
|
|
35229
|
+
})
|
|
35230
|
+
]);
|
|
34989
35231
|
adapterResultReadyAt = isoAt(now());
|
|
34990
35232
|
await persistAttempt({
|
|
34991
35233
|
phase: "dispatched",
|
|
@@ -35170,8 +35412,15 @@ var init_runner = __esm({
|
|
|
35170
35412
|
rateLimits: error48.rateLimits
|
|
35171
35413
|
});
|
|
35172
35414
|
}
|
|
35173
|
-
const
|
|
35174
|
-
|
|
35415
|
+
const effectiveError = providerDispatchWatchdogExpired && error48 instanceof CodexAppServerError && error48.dispatchOutcome === "not_dispatched" ? new CodexAppServerError({
|
|
35416
|
+
message: "The Server account context expired before provider dispatch.",
|
|
35417
|
+
category: "timeout",
|
|
35418
|
+
code: "server_account_context_expired_before_provider_dispatch",
|
|
35419
|
+
retryable: false,
|
|
35420
|
+
dispatchOutcome: "not_dispatched"
|
|
35421
|
+
}) : error48;
|
|
35422
|
+
const classified = classifyFailure(heartbeatFailure ?? effectiveError, adapterResult);
|
|
35423
|
+
const processExit = heartbeatFailure === void 0 && effectiveError instanceof CodexAppServerError && effectiveError.code === "codex_process_closed" && effectiveError.processExit?.expected_shutdown === false ? effectiveError.processExit : null;
|
|
35175
35424
|
const heartbeatAmbiguous = ambiguousHeartbeatTransport(heartbeatFailure);
|
|
35176
35425
|
const failure = heartbeatAmbiguous ? {
|
|
35177
35426
|
...classified,
|
|
@@ -35180,7 +35429,7 @@ var init_runner = __esm({
|
|
|
35180
35429
|
effectiveModel: adapterResult?.effectiveModel ?? classified.effectiveModel,
|
|
35181
35430
|
effectiveEffort: adapterResult?.effectiveReasoningEffort ?? classified.effectiveEffort
|
|
35182
35431
|
} : classified;
|
|
35183
|
-
const dispatchOutcome = heartbeatAmbiguous ? "outcome_unknown" :
|
|
35432
|
+
const dispatchOutcome = heartbeatAmbiguous ? "outcome_unknown" : effectiveError instanceof CodexAppServerError ? effectiveError.dispatchOutcome : failure.dispatchOutcome;
|
|
35184
35433
|
await persistAttempt({
|
|
35185
35434
|
phase: dispatchOutcome === "not_dispatched" ? "started" : "dispatched",
|
|
35186
35435
|
dispatch_outcome: dispatchOutcome
|
|
@@ -35221,7 +35470,7 @@ var init_runner = __esm({
|
|
|
35221
35470
|
adapter_request_id: canReportEffective ? adapterResult?.adapterRequestId ?? null : null,
|
|
35222
35471
|
adapter_response_id: canReportEffective ? adapterResult?.adapterResponseId ?? null : null
|
|
35223
35472
|
},
|
|
35224
|
-
usage: adapterResult ? reportedUsage(adapterResult, usageSource) :
|
|
35473
|
+
usage: adapterResult ? reportedUsage(adapterResult, usageSource) : effectiveError instanceof CodexAppServerError && effectiveError.usage ? reportedTokenUsage(effectiveError.usage, usageSource) : null,
|
|
35225
35474
|
failure_category: failure.category,
|
|
35226
35475
|
failure_code: failure.code,
|
|
35227
35476
|
retryable: failure.retryable,
|
|
@@ -35271,6 +35520,7 @@ var init_runner = __esm({
|
|
|
35271
35520
|
});
|
|
35272
35521
|
return "failed";
|
|
35273
35522
|
} finally {
|
|
35523
|
+
clearProviderDispatchWatchdog();
|
|
35274
35524
|
signal.removeEventListener("abort", relayAbort);
|
|
35275
35525
|
await stopHeartbeat();
|
|
35276
35526
|
}
|
|
@@ -43405,7 +43655,7 @@ var init_hyperliquid_account_state_adapter = __esm({
|
|
|
43405
43655
|
}
|
|
43406
43656
|
};
|
|
43407
43657
|
}
|
|
43408
|
-
if (cached2 && now - cached2.fetchedAtMs <= staleIf429MaxAgeMs && hasActiveInfoRateLimitCooldown(apiUrl, ["userFills"], now)) {
|
|
43658
|
+
if (!input.forceRefresh && cached2 && now - cached2.fetchedAtMs <= staleIf429MaxAgeMs && hasActiveInfoRateLimitCooldown(apiUrl, ["userFills"], now)) {
|
|
43409
43659
|
const scopedFills = filterFillsBySymbols(cached2.rawFills, input.symbols);
|
|
43410
43660
|
const tradeHistory2 = scopedFills.map(normalizeBrowserFillToTradeHistory).filter((fill) => fill !== null);
|
|
43411
43661
|
return {
|
|
@@ -43465,7 +43715,7 @@ var init_hyperliquid_account_state_adapter = __esm({
|
|
|
43465
43715
|
} catch (error48) {
|
|
43466
43716
|
recordInfoRateLimitCooldownFromError(apiUrl, error48, staleIf429MaxAgeMs);
|
|
43467
43717
|
const staleCache = readBrowserUserFillsCache(cacheKey);
|
|
43468
|
-
if (isRateLimitError(error48)) {
|
|
43718
|
+
if (isRateLimitError(error48) && !input.forceRefresh) {
|
|
43469
43719
|
evidenceSource = "stale_if_429";
|
|
43470
43720
|
rawWalletFills = staleCache && Date.now() - staleCache.fetchedAtMs <= staleIf429MaxAgeMs ? staleCache.rawFills : [];
|
|
43471
43721
|
providerRowCount = staleCache?.providerRowCount ?? 0;
|