@vtxmacro/cli 2026.8.45 → 2026.8.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/vtx.js +269 -176
- package/package.json +1 -1
package/bin/vtx.js
CHANGED
|
@@ -38,7 +38,7 @@ var init_agent_cli_release = __esm({
|
|
|
38
38
|
"agent-cli-release.json"() {
|
|
39
39
|
agent_cli_release_default = {
|
|
40
40
|
package_name: "@vtxmacro/cli",
|
|
41
|
-
package_version: "2026.8.
|
|
41
|
+
package_version: "2026.8.47",
|
|
42
42
|
codex_package_name: "@openai/codex",
|
|
43
43
|
codex_version: "0.147.0",
|
|
44
44
|
copilot_sdk_package_name: "@github/copilot-sdk",
|
|
@@ -18431,11 +18431,12 @@ var init_mcp_client = __esm({
|
|
|
18431
18431
|
const retryableClaimRejection = name === "inference.job.claim" && !definitivelyNotApplied && (retryableInfrastructureRejection || errorText.includes("External inference host is not live enough to claim work") || errorText.includes("External inference host request generation is stale"));
|
|
18432
18432
|
const retryableHeartbeatClockSkew = name === "inference.host.heartbeat" && definitivelyNotApplied;
|
|
18433
18433
|
const retryableAdvertisementClockSkew = (name === "inference.host.register" || name === "inference.host.advertise") && definitivelyNotApplied;
|
|
18434
|
+
const staleHostHeartbeatGeneration = name === "inference.host.heartbeat" && errorText.includes("Heartbeat generations do not match the current host authority.");
|
|
18434
18435
|
throw new ExternalInferenceMcpError(
|
|
18435
|
-
"tool_rejected",
|
|
18436
|
+
staleHostHeartbeatGeneration ? "generation_stale" : "tool_rejected",
|
|
18436
18437
|
`Insights MCP rejected ${name}.`,
|
|
18437
18438
|
{
|
|
18438
|
-
definitivelyNotApplied,
|
|
18439
|
+
definitivelyNotApplied: definitivelyNotApplied || staleHostHeartbeatGeneration,
|
|
18439
18440
|
// Claim polling is an idempotent control-plane operation. During an
|
|
18440
18441
|
// API rotation the MCP server can return a tool error after the
|
|
18441
18442
|
// request reached the application boundary, so retain and replay
|
|
@@ -30744,6 +30745,9 @@ var init_runner = __esm({
|
|
|
30744
30745
|
let completed = 0;
|
|
30745
30746
|
let failed = 0;
|
|
30746
30747
|
let acceptAttemptUpdates = true;
|
|
30748
|
+
let stopHostHeartbeatLoop = () => {
|
|
30749
|
+
};
|
|
30750
|
+
let hostHeartbeatLoop = Promise.resolve();
|
|
30747
30751
|
let wakeDrain;
|
|
30748
30752
|
const drainSignal = new Promise((resolve6) => {
|
|
30749
30753
|
wakeDrain = resolve6;
|
|
@@ -30759,6 +30763,7 @@ var init_runner = __esm({
|
|
|
30759
30763
|
if (drainRequested && priority(reason) <= priority(drainReason)) return;
|
|
30760
30764
|
drainRequested = true;
|
|
30761
30765
|
drainReason = reason;
|
|
30766
|
+
stopHostHeartbeatLoop();
|
|
30762
30767
|
wakeDrain();
|
|
30763
30768
|
};
|
|
30764
30769
|
try {
|
|
@@ -30890,6 +30895,18 @@ var init_runner = __esm({
|
|
|
30890
30895
|
updated_at: isoAt(now())
|
|
30891
30896
|
};
|
|
30892
30897
|
await this.persistReceipt(receipt, now);
|
|
30898
|
+
let receiptMutationChain = Promise.resolve();
|
|
30899
|
+
const mutateReceipt = async (mutation) => {
|
|
30900
|
+
let result2;
|
|
30901
|
+
const operation = receiptMutationChain.then(async () => {
|
|
30902
|
+
result2 = mutation(receipt);
|
|
30903
|
+
receipt = result2;
|
|
30904
|
+
await this.persistReceipt(receipt, now);
|
|
30905
|
+
});
|
|
30906
|
+
receiptMutationChain = operation;
|
|
30907
|
+
await operation;
|
|
30908
|
+
return result2;
|
|
30909
|
+
};
|
|
30893
30910
|
const envelopePublicKey = await this.dependencies.envelopePublicKey(credential);
|
|
30894
30911
|
const retryExact = async (operation, boundary = {}) => {
|
|
30895
30912
|
let lastError;
|
|
@@ -30932,40 +30949,48 @@ var init_runner = __esm({
|
|
|
30932
30949
|
const publishAdvertisement = async (health, forceAdvance, retryRemote = true, allowDefinitiveReseed = true) => {
|
|
30933
30950
|
let advertisement = receipt.pending_advertisement;
|
|
30934
30951
|
if (!advertisement) {
|
|
30935
|
-
const
|
|
30936
|
-
|
|
30937
|
-
|
|
30938
|
-
|
|
30939
|
-
|
|
30940
|
-
|
|
30941
|
-
|
|
30942
|
-
|
|
30943
|
-
|
|
30944
|
-
|
|
30945
|
-
|
|
30946
|
-
|
|
30947
|
-
|
|
30948
|
-
|
|
30949
|
-
|
|
30950
|
-
|
|
30951
|
-
|
|
30952
|
-
|
|
30953
|
-
|
|
30954
|
-
|
|
30955
|
-
|
|
30956
|
-
|
|
30957
|
-
|
|
30958
|
-
|
|
30959
|
-
|
|
30960
|
-
|
|
30961
|
-
|
|
30952
|
+
const updated = await mutateReceipt((current) => {
|
|
30953
|
+
if (current.pending_advertisement) return current;
|
|
30954
|
+
const generation = current.registered && forceAdvance ? current.advertisement_generation + 1 : current.advertisement_generation;
|
|
30955
|
+
const advertisedAt = now();
|
|
30956
|
+
const pendingAdvertisement = hostAdvertisementSchema.parse({
|
|
30957
|
+
schema_version: EXTERNAL_INFERENCE_ADVERTISEMENT_SCHEMA_VERSION,
|
|
30958
|
+
contract_version: EXTERNAL_INFERENCE_CONTRACT_VERSION,
|
|
30959
|
+
operation_id: stableOperationId("advertise", [
|
|
30960
|
+
localState.host_id,
|
|
30961
|
+
localState.host_generation,
|
|
30962
|
+
generation,
|
|
30963
|
+
localState.key_generation,
|
|
30964
|
+
advertisedAt
|
|
30965
|
+
]),
|
|
30966
|
+
host_id: localState.host_id,
|
|
30967
|
+
host_generation: localState.host_generation,
|
|
30968
|
+
advertisement_generation: generation,
|
|
30969
|
+
key_generation: localState.key_generation,
|
|
30970
|
+
adapter: settings.adapterId,
|
|
30971
|
+
display_name: this.options.displayName.trim(),
|
|
30972
|
+
...this.options.authenticatedAccountIdentity ? { authenticated_account_identity: this.options.authenticatedAccountIdentity } : {},
|
|
30973
|
+
authenticated_account_email: this.options.authenticatedAccountEmail ?? null,
|
|
30974
|
+
authenticated_account_plan: this.options.authenticatedAccountPlan ?? null,
|
|
30975
|
+
protocol_version: this.options.protocolVersion,
|
|
30976
|
+
envelope_public_key: envelopePublicKey,
|
|
30977
|
+
health,
|
|
30978
|
+
advertised_at: isoAt(advertisedAt),
|
|
30979
|
+
expires_at: isoAt(advertisedAt + settings.advertisementTtlMs),
|
|
30980
|
+
models: settings.advertisedModels
|
|
30981
|
+
});
|
|
30982
|
+
return {
|
|
30983
|
+
...current,
|
|
30984
|
+
pending_advertisement: pendingAdvertisement,
|
|
30985
|
+
updated_at: isoAt(now())
|
|
30986
|
+
};
|
|
30962
30987
|
});
|
|
30963
|
-
|
|
30964
|
-
|
|
30965
|
-
|
|
30966
|
-
|
|
30967
|
-
|
|
30968
|
-
|
|
30988
|
+
advertisement = updated.pending_advertisement;
|
|
30989
|
+
}
|
|
30990
|
+
if (!advertisement) {
|
|
30991
|
+
throw new InferenceHostRecoveryRequiredError(
|
|
30992
|
+
"The durable host advertisement receipt is unavailable."
|
|
30993
|
+
);
|
|
30969
30994
|
}
|
|
30970
30995
|
const tool = receipt.registered ? "inference.host.advertise" : "inference.host.register";
|
|
30971
30996
|
let result2;
|
|
@@ -30980,26 +31005,24 @@ var init_runner = __esm({
|
|
|
30980
31005
|
const definitivelyNotApplied = error48 instanceof ExternalInferenceMcpError ? error48.definitivelyNotApplied : Boolean(error48 && typeof error48 === "object" && error48.definitivelyNotApplied === true);
|
|
30981
31006
|
const expectedGeneration = receipt.registered ? receipt.advertisement_generation + 1 : receipt.advertisement_generation;
|
|
30982
31007
|
if (definitivelyNotApplied && advertisement.advertisement_generation === expectedGeneration && allowDefinitiveReseed) {
|
|
30983
|
-
|
|
30984
|
-
...
|
|
30985
|
-
pending_advertisement: null,
|
|
31008
|
+
await mutateReceipt((current) => ({
|
|
31009
|
+
...current,
|
|
31010
|
+
pending_advertisement: current.pending_advertisement?.operation_id === advertisement.operation_id ? null : current.pending_advertisement,
|
|
30986
31011
|
updated_at: isoAt(now())
|
|
30987
|
-
};
|
|
30988
|
-
await this.persistReceipt(receipt, now);
|
|
31012
|
+
}));
|
|
30989
31013
|
return await publishAdvertisement(health, forceAdvance, retryRemote, false);
|
|
30990
31014
|
}
|
|
30991
31015
|
throw error48;
|
|
30992
31016
|
}
|
|
30993
31017
|
assertAdvertisementResult(advertisement, result2);
|
|
30994
|
-
|
|
30995
|
-
...
|
|
31018
|
+
await mutateReceipt((current) => ({
|
|
31019
|
+
...current,
|
|
30996
31020
|
registered: true,
|
|
30997
31021
|
advertisement_generation: result2.host.advertisement_generation,
|
|
30998
31022
|
advertisement_expires_at: advertisement.expires_at,
|
|
30999
|
-
pending_advertisement: null,
|
|
31023
|
+
pending_advertisement: current.pending_advertisement?.operation_id === advertisement.operation_id ? null : current.pending_advertisement,
|
|
31000
31024
|
updated_at: isoAt(now())
|
|
31001
|
-
};
|
|
31002
|
-
await this.persistReceipt(receipt, now);
|
|
31025
|
+
}));
|
|
31003
31026
|
};
|
|
31004
31027
|
const wasRegistered = receipt.registered;
|
|
31005
31028
|
await publishAdvertisement("healthy", wasRegistered);
|
|
@@ -31012,32 +31035,34 @@ var init_runner = __esm({
|
|
|
31012
31035
|
let providerRateLimitsObservedAtMs = now();
|
|
31013
31036
|
const hostHeartbeat = async (status, requestOptions = {
|
|
31014
31037
|
signal: this.options.signal
|
|
31015
|
-
}, retryRemote = true, allowOfflineProjection = false, definitiveReseedAttempt = 0) => {
|
|
31016
|
-
|
|
31017
|
-
|
|
31018
|
-
|
|
31019
|
-
|
|
31020
|
-
|
|
31021
|
-
|
|
31022
|
-
|
|
31023
|
-
|
|
31024
|
-
|
|
31025
|
-
|
|
31026
|
-
|
|
31027
|
-
|
|
31028
|
-
|
|
31029
|
-
|
|
31030
|
-
|
|
31031
|
-
|
|
31032
|
-
|
|
31033
|
-
|
|
31034
|
-
|
|
31035
|
-
|
|
31036
|
-
|
|
31037
|
-
|
|
31038
|
-
|
|
31039
|
-
|
|
31040
|
-
|
|
31038
|
+
}, retryRemote = true, allowOfflineProjection = false, definitiveReseedAttempt = 0, advertisementGeneration = receipt.advertisement_generation) => {
|
|
31039
|
+
let request;
|
|
31040
|
+
await mutateReceipt((current) => {
|
|
31041
|
+
const sequence = current.host_heartbeat_sequence + 1;
|
|
31042
|
+
request = {
|
|
31043
|
+
schema_version: "external_inference_host_heartbeat_v1",
|
|
31044
|
+
contract_version: EXTERNAL_INFERENCE_CONTRACT_VERSION,
|
|
31045
|
+
host_id: localState.host_id,
|
|
31046
|
+
host_generation: localState.host_generation,
|
|
31047
|
+
advertisement_generation: advertisementGeneration,
|
|
31048
|
+
key_generation: localState.key_generation,
|
|
31049
|
+
sequence,
|
|
31050
|
+
status,
|
|
31051
|
+
active_attempts: active.size,
|
|
31052
|
+
available_slots: settings.maxConcurrency === null ? UNBOUNDED_AVAILABLE_SLOTS - active.size : Math.max(0, settings.maxConcurrency - active.size),
|
|
31053
|
+
provider_weekly_quota: providerWeeklyQuotaFromRateLimits(
|
|
31054
|
+
providerRateLimits,
|
|
31055
|
+
providerRateLimitsObservedAtMs,
|
|
31056
|
+
now()
|
|
31057
|
+
),
|
|
31058
|
+
observed_at: isoAt(now())
|
|
31059
|
+
};
|
|
31060
|
+
return {
|
|
31061
|
+
...current,
|
|
31062
|
+
host_heartbeat_sequence: sequence,
|
|
31063
|
+
updated_at: isoAt(now())
|
|
31064
|
+
};
|
|
31065
|
+
});
|
|
31041
31066
|
const heartbeatOperation = () => mcp.callTool(
|
|
31042
31067
|
"inference.host.heartbeat",
|
|
31043
31068
|
request,
|
|
@@ -31054,7 +31079,8 @@ var init_runner = __esm({
|
|
|
31054
31079
|
requestOptions,
|
|
31055
31080
|
retryRemote,
|
|
31056
31081
|
allowOfflineProjection,
|
|
31057
|
-
definitiveReseedAttempt + 1
|
|
31082
|
+
definitiveReseedAttempt + 1,
|
|
31083
|
+
advertisementGeneration
|
|
31058
31084
|
);
|
|
31059
31085
|
}
|
|
31060
31086
|
throw error48;
|
|
@@ -31089,12 +31115,109 @@ var init_runner = __esm({
|
|
|
31089
31115
|
}
|
|
31090
31116
|
return gapMs;
|
|
31091
31117
|
};
|
|
31092
|
-
let nextHostHeartbeatAt = now() + settings.hostHeartbeatMs;
|
|
31093
31118
|
let nextAdvertisementAttemptAt = 0;
|
|
31094
31119
|
let nextProviderRateLimitRefreshAt = now() + settings.providerRateLimitRefreshMs;
|
|
31095
31120
|
let nextClaimAt = Math.max(now(), providerRetryAtMs ?? 0);
|
|
31096
31121
|
let pendingClaimPromotions = 0;
|
|
31097
31122
|
let onceClaimed = false;
|
|
31123
|
+
const hostHeartbeatAbort = new AbortController();
|
|
31124
|
+
stopHostHeartbeatLoop = () => hostHeartbeatAbort.abort();
|
|
31125
|
+
let nextHostHeartbeatAt = now() + settings.hostHeartbeatMs;
|
|
31126
|
+
hostHeartbeatLoop = (async () => {
|
|
31127
|
+
while (!drainRequested && !hostHeartbeatAbort.signal.aborted) {
|
|
31128
|
+
try {
|
|
31129
|
+
await sleep4(
|
|
31130
|
+
Math.max(MIN_SLEEP_MS, nextHostHeartbeatAt - now()),
|
|
31131
|
+
hostHeartbeatAbort.signal
|
|
31132
|
+
);
|
|
31133
|
+
} catch (error48) {
|
|
31134
|
+
if (hostHeartbeatAbort.signal.aborted || drainRequested) return;
|
|
31135
|
+
throw error48;
|
|
31136
|
+
}
|
|
31137
|
+
if (drainRequested || hostHeartbeatAbort.signal.aborted) return;
|
|
31138
|
+
const heartbeatStartedAt = now();
|
|
31139
|
+
observeHostHeartbeatGap(heartbeatStartedAt);
|
|
31140
|
+
const attemptedAdvertisementGeneration = receipt.advertisement_generation;
|
|
31141
|
+
try {
|
|
31142
|
+
const expiresAt = receipt.advertisement_expires_at ? Date.parse(receipt.advertisement_expires_at) : 0;
|
|
31143
|
+
const refreshPendingAcrossExpiry = receipt.pending_advertisement !== null && expiresAt <= heartbeatStartedAt;
|
|
31144
|
+
await hostHeartbeat(
|
|
31145
|
+
"healthy",
|
|
31146
|
+
{ signal: hostHeartbeatAbort.signal },
|
|
31147
|
+
false,
|
|
31148
|
+
refreshPendingAcrossExpiry,
|
|
31149
|
+
0,
|
|
31150
|
+
attemptedAdvertisementGeneration
|
|
31151
|
+
);
|
|
31152
|
+
} catch (error48) {
|
|
31153
|
+
if (hostHeartbeatAbort.signal.aborted || drainRequested) return;
|
|
31154
|
+
const errorCode = error48 && typeof error48 === "object" && "code" in error48 ? String(error48.code) : "";
|
|
31155
|
+
const pendingAdvertisement = receipt.pending_advertisement;
|
|
31156
|
+
const recoveryAdvertisementGeneration = receipt.advertisement_generation > attemptedAdvertisementGeneration ? receipt.advertisement_generation : errorCode === "generation_stale" && pendingAdvertisement && pendingAdvertisement.advertisement_generation === attemptedAdvertisementGeneration + 1 ? pendingAdvertisement.advertisement_generation : null;
|
|
31157
|
+
if (errorCode === "generation_stale" && recoveryAdvertisementGeneration !== null) {
|
|
31158
|
+
try {
|
|
31159
|
+
await hostHeartbeat(
|
|
31160
|
+
"healthy",
|
|
31161
|
+
{ signal: hostHeartbeatAbort.signal },
|
|
31162
|
+
false,
|
|
31163
|
+
false,
|
|
31164
|
+
0,
|
|
31165
|
+
recoveryAdvertisementGeneration
|
|
31166
|
+
);
|
|
31167
|
+
if (receipt.advertisement_generation < recoveryAdvertisementGeneration) {
|
|
31168
|
+
await mutateReceipt((current) => current.advertisement_generation >= recoveryAdvertisementGeneration ? current : {
|
|
31169
|
+
...current,
|
|
31170
|
+
advertisement_generation: recoveryAdvertisementGeneration,
|
|
31171
|
+
advertisement_expires_at: pendingAdvertisement?.expires_at ?? current.advertisement_expires_at,
|
|
31172
|
+
updated_at: isoAt(now())
|
|
31173
|
+
});
|
|
31174
|
+
}
|
|
31175
|
+
} catch (recoveryError) {
|
|
31176
|
+
if (hostHeartbeatAbort.signal.aborted || drainRequested) return;
|
|
31177
|
+
if (controlPlaneFatal(recoveryError)) {
|
|
31178
|
+
requestDrain("authority_lost");
|
|
31179
|
+
return;
|
|
31180
|
+
}
|
|
31181
|
+
if (!retryableRemoteError(recoveryError)) {
|
|
31182
|
+
requestDrain("host_heartbeat_failed");
|
|
31183
|
+
return;
|
|
31184
|
+
}
|
|
31185
|
+
nextHostHeartbeatAt = now() + remoteRetryAfterMs(recoveryError, settings.hostHeartbeatMs);
|
|
31186
|
+
continue;
|
|
31187
|
+
}
|
|
31188
|
+
} else if (controlPlaneFatal(error48)) {
|
|
31189
|
+
requestDrain("authority_lost");
|
|
31190
|
+
return;
|
|
31191
|
+
} else if (!retryableRemoteError(error48)) {
|
|
31192
|
+
requestDrain("host_heartbeat_failed");
|
|
31193
|
+
return;
|
|
31194
|
+
} else {
|
|
31195
|
+
nextHostHeartbeatAt = now() + remoteRetryAfterMs(error48, settings.hostHeartbeatMs);
|
|
31196
|
+
continue;
|
|
31197
|
+
}
|
|
31198
|
+
}
|
|
31199
|
+
const heartbeatSucceededAt = now();
|
|
31200
|
+
const gapMs = observeHostHeartbeatGap(heartbeatSucceededAt);
|
|
31201
|
+
if (hostHeartbeatGapOpen) {
|
|
31202
|
+
emitDiagnostic("host_heartbeat_gap_recovered", {
|
|
31203
|
+
gap_ms: gapMs,
|
|
31204
|
+
configured_interval_ms: settings.hostHeartbeatMs,
|
|
31205
|
+
diagnostic_threshold_ms: hostHeartbeatGapThresholdMs,
|
|
31206
|
+
active_attempts: active.size,
|
|
31207
|
+
host_heartbeat_sequence: receipt.host_heartbeat_sequence
|
|
31208
|
+
});
|
|
31209
|
+
hostHeartbeatGapOpen = false;
|
|
31210
|
+
}
|
|
31211
|
+
lastHostHeartbeatSucceededAt = heartbeatSucceededAt;
|
|
31212
|
+
nextHostHeartbeatAt = Math.max(
|
|
31213
|
+
nextHostHeartbeatAt + settings.hostHeartbeatMs,
|
|
31214
|
+
heartbeatSucceededAt + MIN_SLEEP_MS
|
|
31215
|
+
);
|
|
31216
|
+
}
|
|
31217
|
+
})().catch((error48) => {
|
|
31218
|
+
if (hostHeartbeatAbort.signal.aborted || drainRequested) return;
|
|
31219
|
+
requestDrain(controlPlaneFatal(error48) ? "authority_lost" : "host_heartbeat_failed");
|
|
31220
|
+
});
|
|
31098
31221
|
const recoveryQueue = [];
|
|
31099
31222
|
const attemptStartRetryCounts = /* @__PURE__ */ new Map();
|
|
31100
31223
|
const launchClaim = (claim, recovery, claimRequest) => {
|
|
@@ -31130,15 +31253,14 @@ var init_runner = __esm({
|
|
|
31130
31253
|
credential,
|
|
31131
31254
|
mcp,
|
|
31132
31255
|
receipt: () => receipt,
|
|
31133
|
-
updateReceipt: async (
|
|
31256
|
+
updateReceipt: async (mutation) => {
|
|
31134
31257
|
if (!acceptAttemptUpdates) {
|
|
31135
31258
|
throw new InferenceHostRunnerError(
|
|
31136
31259
|
"runtime_closed",
|
|
31137
31260
|
"The drained inference runtime no longer accepts attempt updates."
|
|
31138
31261
|
);
|
|
31139
31262
|
}
|
|
31140
|
-
|
|
31141
|
-
await this.persistReceipt(receipt, now);
|
|
31263
|
+
await mutateReceipt(mutation);
|
|
31142
31264
|
},
|
|
31143
31265
|
retryExact,
|
|
31144
31266
|
signal: controller.signal,
|
|
@@ -31236,10 +31358,11 @@ var init_runner = __esm({
|
|
|
31236
31358
|
return attemptId;
|
|
31237
31359
|
};
|
|
31238
31360
|
const removeRecoveredAttempt = async (attemptId) => {
|
|
31239
|
-
|
|
31240
|
-
|
|
31241
|
-
|
|
31242
|
-
|
|
31361
|
+
await mutateReceipt((current) => {
|
|
31362
|
+
const attempts = { ...current.attempts };
|
|
31363
|
+
delete attempts[attemptId];
|
|
31364
|
+
return { ...current, attempts, updated_at: isoAt(now()) };
|
|
31365
|
+
});
|
|
31243
31366
|
};
|
|
31244
31367
|
for (const recovery of Object.values({ ...receipt.attempts })) {
|
|
31245
31368
|
if (recovery.phase === "terminal_pending") {
|
|
@@ -31354,73 +31477,37 @@ var init_runner = __esm({
|
|
|
31354
31477
|
nextClaimAt = Math.max(nextClaimAt, nextAdvertisementAttemptAt);
|
|
31355
31478
|
}
|
|
31356
31479
|
}
|
|
31357
|
-
if (currentTime >= nextHostHeartbeatAt) {
|
|
31358
|
-
observeHostHeartbeatGap(currentTime);
|
|
31359
|
-
try {
|
|
31360
|
-
const refreshPendingAcrossExpiry = receipt.pending_advertisement !== null && expiresAt <= currentTime && nextAdvertisementAttemptAt > currentTime;
|
|
31361
|
-
await hostHeartbeat(
|
|
31362
|
-
"healthy",
|
|
31363
|
-
{ signal: this.options.signal },
|
|
31364
|
-
false,
|
|
31365
|
-
refreshPendingAcrossExpiry
|
|
31366
|
-
);
|
|
31367
|
-
const heartbeatSucceededAt = now();
|
|
31368
|
-
const gapMs = observeHostHeartbeatGap(heartbeatSucceededAt);
|
|
31369
|
-
if (hostHeartbeatGapOpen) {
|
|
31370
|
-
emitDiagnostic("host_heartbeat_gap_recovered", {
|
|
31371
|
-
gap_ms: gapMs,
|
|
31372
|
-
configured_interval_ms: settings.hostHeartbeatMs,
|
|
31373
|
-
diagnostic_threshold_ms: hostHeartbeatGapThresholdMs,
|
|
31374
|
-
active_attempts: active.size,
|
|
31375
|
-
host_heartbeat_sequence: receipt.host_heartbeat_sequence
|
|
31376
|
-
});
|
|
31377
|
-
hostHeartbeatGapOpen = false;
|
|
31378
|
-
}
|
|
31379
|
-
lastHostHeartbeatSucceededAt = heartbeatSucceededAt;
|
|
31380
|
-
} catch (error48) {
|
|
31381
|
-
if (controlPlaneFatal(error48)) {
|
|
31382
|
-
requestDrain("authority_lost");
|
|
31383
|
-
break;
|
|
31384
|
-
}
|
|
31385
|
-
if (!retryableRemoteError(error48)) {
|
|
31386
|
-
requestDrain("host_heartbeat_failed");
|
|
31387
|
-
break;
|
|
31388
|
-
}
|
|
31389
|
-
nextHostHeartbeatAt = now() + remoteRetryAfterMs(error48, settings.hostHeartbeatMs);
|
|
31390
|
-
}
|
|
31391
|
-
if (!drainRequested && currentTime >= nextHostHeartbeatAt) {
|
|
31392
|
-
nextHostHeartbeatAt = now() + settings.hostHeartbeatMs;
|
|
31393
|
-
}
|
|
31394
|
-
}
|
|
31395
31480
|
while (recoveryQueue.length > 0 && (settings.maxConcurrency === null || active.size < settings.maxConcurrency)) {
|
|
31396
31481
|
const recovery = recoveryQueue.shift();
|
|
31397
31482
|
launchClaim(recovery.claim, recovery);
|
|
31398
31483
|
}
|
|
31399
31484
|
while (!drainRequested && recoveryQueue.length === 0 && (!this.options.once || !onceClaimed) && (settings.maxConcurrency === null || active.size < settings.maxConcurrency) && pendingClaimPromotions === 0 && now() >= nextClaimAt) {
|
|
31400
31485
|
if (!receipt.pending_claim_request) {
|
|
31401
|
-
|
|
31402
|
-
|
|
31403
|
-
|
|
31404
|
-
|
|
31405
|
-
|
|
31406
|
-
|
|
31407
|
-
|
|
31408
|
-
|
|
31409
|
-
|
|
31410
|
-
localState.
|
|
31411
|
-
|
|
31412
|
-
|
|
31413
|
-
|
|
31414
|
-
|
|
31415
|
-
|
|
31416
|
-
|
|
31417
|
-
|
|
31418
|
-
|
|
31419
|
-
|
|
31420
|
-
|
|
31421
|
-
|
|
31422
|
-
|
|
31423
|
-
|
|
31486
|
+
await mutateReceipt((current) => {
|
|
31487
|
+
if (current.pending_claim_request) return current;
|
|
31488
|
+
const claimSequence = current.claim_sequence + 1;
|
|
31489
|
+
const pendingClaimRequest = {
|
|
31490
|
+
schema_version: "external_inference_job_claim_v1",
|
|
31491
|
+
contract_version: EXTERNAL_INFERENCE_CONTRACT_VERSION,
|
|
31492
|
+
host_id: localState.host_id,
|
|
31493
|
+
host_generation: localState.host_generation,
|
|
31494
|
+
advertisement_generation: current.advertisement_generation,
|
|
31495
|
+
key_generation: localState.key_generation,
|
|
31496
|
+
claim_request_id: stableOperationId("claim", [
|
|
31497
|
+
localState.host_id,
|
|
31498
|
+
localState.host_generation,
|
|
31499
|
+
current.advertisement_generation,
|
|
31500
|
+
claimSequence
|
|
31501
|
+
]),
|
|
31502
|
+
requested_at: isoAt(now())
|
|
31503
|
+
};
|
|
31504
|
+
return {
|
|
31505
|
+
...current,
|
|
31506
|
+
claim_sequence: claimSequence,
|
|
31507
|
+
pending_claim_request: pendingClaimRequest,
|
|
31508
|
+
updated_at: isoAt(now())
|
|
31509
|
+
};
|
|
31510
|
+
});
|
|
31424
31511
|
}
|
|
31425
31512
|
const request = receipt.pending_claim_request;
|
|
31426
31513
|
let claim;
|
|
@@ -31436,12 +31523,11 @@ var init_runner = __esm({
|
|
|
31436
31523
|
requestDrain("authority_lost");
|
|
31437
31524
|
} else if (!this.options.signal?.aborted && error48 instanceof ExternalInferenceMcpError && error48.definitivelyNotApplied) {
|
|
31438
31525
|
if (request.advertisement_generation < receipt.advertisement_generation) {
|
|
31439
|
-
|
|
31440
|
-
...
|
|
31441
|
-
pending_claim_request: null,
|
|
31526
|
+
await mutateReceipt((current) => ({
|
|
31527
|
+
...current,
|
|
31528
|
+
pending_claim_request: current.pending_claim_request?.claim_request_id === request.claim_request_id ? null : current.pending_claim_request,
|
|
31442
31529
|
updated_at: isoAt(now())
|
|
31443
|
-
};
|
|
31444
|
-
await this.persistReceipt(receipt, now);
|
|
31530
|
+
}));
|
|
31445
31531
|
nextClaimAt = now();
|
|
31446
31532
|
} else {
|
|
31447
31533
|
requestDrain("claim_failed");
|
|
@@ -31454,23 +31540,21 @@ var init_runner = __esm({
|
|
|
31454
31540
|
break;
|
|
31455
31541
|
}
|
|
31456
31542
|
if (claim.claim_state === "empty") {
|
|
31457
|
-
|
|
31458
|
-
...
|
|
31459
|
-
pending_claim_request: null,
|
|
31543
|
+
await mutateReceipt((current) => ({
|
|
31544
|
+
...current,
|
|
31545
|
+
pending_claim_request: current.pending_claim_request?.claim_request_id === request.claim_request_id ? null : current.pending_claim_request,
|
|
31460
31546
|
updated_at: isoAt(now())
|
|
31461
|
-
};
|
|
31462
|
-
await this.persistReceipt(receipt, now);
|
|
31547
|
+
}));
|
|
31463
31548
|
nextClaimAt = now() + Math.max(MIN_SLEEP_MS, claim.retry_after_ms);
|
|
31464
31549
|
if (this.options.once) requestDrain("once_empty");
|
|
31465
31550
|
break;
|
|
31466
31551
|
}
|
|
31467
31552
|
if (Date.parse(claim.lease_expires_at) <= now() + MIN_CLAIM_START_WINDOW_MS || Date.parse(claim.deadline_at) <= now() + MIN_CLAIM_START_WINDOW_MS) {
|
|
31468
|
-
|
|
31469
|
-
...
|
|
31470
|
-
pending_claim_request: null,
|
|
31553
|
+
await mutateReceipt((current) => ({
|
|
31554
|
+
...current,
|
|
31555
|
+
pending_claim_request: current.pending_claim_request?.claim_request_id === request.claim_request_id ? null : current.pending_claim_request,
|
|
31471
31556
|
updated_at: isoAt(now())
|
|
31472
|
-
};
|
|
31473
|
-
await this.persistReceipt(receipt, now);
|
|
31557
|
+
}));
|
|
31474
31558
|
nextClaimAt = now();
|
|
31475
31559
|
break;
|
|
31476
31560
|
}
|
|
@@ -31481,7 +31565,8 @@ var init_runner = __esm({
|
|
|
31481
31565
|
const advertisementDueAt = nextAdvertisementAttemptAt > now() ? nextAdvertisementAttemptAt : receipt.advertisement_expires_at ? Date.parse(receipt.advertisement_expires_at) - settings.advertisementRefreshLeadMs : now() + MIN_SLEEP_MS;
|
|
31482
31566
|
const wakeAt = Math.min(
|
|
31483
31567
|
nextClaimAt,
|
|
31484
|
-
|
|
31568
|
+
nextProviderRateLimitRefreshAt,
|
|
31569
|
+
now() + settings.hostHeartbeatMs,
|
|
31485
31570
|
Math.max(now() + MIN_SLEEP_MS, advertisementDueAt)
|
|
31486
31571
|
);
|
|
31487
31572
|
await Promise.race([
|
|
@@ -31490,6 +31575,8 @@ var init_runner = __esm({
|
|
|
31490
31575
|
...active.values()
|
|
31491
31576
|
]);
|
|
31492
31577
|
}
|
|
31578
|
+
stopHostHeartbeatLoop();
|
|
31579
|
+
await hostHeartbeatLoop;
|
|
31493
31580
|
try {
|
|
31494
31581
|
await hostHeartbeat("draining", {
|
|
31495
31582
|
deadlineAtMs: now() + settings.drainTimeoutMs,
|
|
@@ -31524,6 +31611,8 @@ var init_runner = __esm({
|
|
|
31524
31611
|
});
|
|
31525
31612
|
return { claimed, completed, failed, drain_reason: drainReason };
|
|
31526
31613
|
} finally {
|
|
31614
|
+
stopHostHeartbeatLoop();
|
|
31615
|
+
await hostHeartbeatLoop;
|
|
31527
31616
|
removeAbortListener();
|
|
31528
31617
|
unregisterSignals();
|
|
31529
31618
|
try {
|
|
@@ -31595,35 +31684,39 @@ var init_runner = __esm({
|
|
|
31595
31684
|
);
|
|
31596
31685
|
}
|
|
31597
31686
|
if (!options.resumeReceipt) {
|
|
31598
|
-
|
|
31599
|
-
|
|
31600
|
-
|
|
31601
|
-
|
|
31602
|
-
|
|
31603
|
-
|
|
31604
|
-
|
|
31687
|
+
await options.updateReceipt((current) => {
|
|
31688
|
+
const pendingClaimRequest = options.claimRequest && current.pending_claim_request?.claim_request_id === options.claimRequest.claim_request_id ? null : current.pending_claim_request ?? null;
|
|
31689
|
+
return {
|
|
31690
|
+
...current,
|
|
31691
|
+
pending_claim_request: pendingClaimRequest,
|
|
31692
|
+
attempts: { ...current.attempts, [attemptId]: attemptReceipt },
|
|
31693
|
+
updated_at: isoAt(now())
|
|
31694
|
+
};
|
|
31605
31695
|
});
|
|
31606
31696
|
options.onClaimPersisted?.();
|
|
31607
31697
|
}
|
|
31608
31698
|
const updateAttempt = async (changes) => {
|
|
31609
|
-
|
|
31699
|
+
const nextAttemptReceipt = validateAttemptReceipt({
|
|
31610
31700
|
...attemptReceipt,
|
|
31611
31701
|
...changes,
|
|
31612
31702
|
updated_at: isoAt(now())
|
|
31613
31703
|
});
|
|
31614
|
-
|
|
31615
|
-
|
|
31616
|
-
|
|
31704
|
+
attemptReceipt = nextAttemptReceipt;
|
|
31705
|
+
await options.updateReceipt((current) => ({
|
|
31706
|
+
...current,
|
|
31707
|
+
attempts: { ...current.attempts, [attemptId]: nextAttemptReceipt },
|
|
31617
31708
|
updated_at: isoAt(now())
|
|
31618
|
-
});
|
|
31709
|
+
}));
|
|
31619
31710
|
};
|
|
31620
31711
|
const removeAttempt = async () => {
|
|
31621
|
-
|
|
31622
|
-
|
|
31623
|
-
|
|
31624
|
-
|
|
31625
|
-
|
|
31626
|
-
|
|
31712
|
+
await options.updateReceipt((current) => {
|
|
31713
|
+
const attempts = { ...current.attempts };
|
|
31714
|
+
delete attempts[attemptId];
|
|
31715
|
+
return {
|
|
31716
|
+
...current,
|
|
31717
|
+
attempts,
|
|
31718
|
+
updated_at: isoAt(now())
|
|
31719
|
+
};
|
|
31627
31720
|
});
|
|
31628
31721
|
};
|
|
31629
31722
|
if (attemptReceipt.phase === "claimed") {
|