agent-embassy 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +50 -5
- package/README.zh-CN.md +32 -5
- package/SECURITY.md +26 -2
- package/assets/live-dashboard/app.css +14 -5
- package/dist/src/gateway/claude-helper-protocol.d.ts +3 -0
- package/dist/src/gateway/claude-helper-protocol.js +15 -1
- package/dist/src/gateway/claude-helper-protocol.js.map +1 -1
- package/dist/src/gateway/claude-helper-supervisor.d.ts +2 -0
- package/dist/src/gateway/claude-helper-supervisor.js +25 -1
- package/dist/src/gateway/claude-helper-supervisor.js.map +1 -1
- package/dist/src/gateway/cli-copy.en.d.ts +1 -0
- package/dist/src/gateway/cli-copy.en.js +1 -0
- package/dist/src/gateway/cli-copy.en.js.map +1 -1
- package/dist/src/gateway/cli-copy.zh-CN.d.ts +1 -0
- package/dist/src/gateway/cli-copy.zh-CN.js +1 -0
- package/dist/src/gateway/cli-copy.zh-CN.js.map +1 -1
- package/dist/src/gateway/cli.d.ts +1 -1
- package/dist/src/gateway/cli.js +20 -7
- package/dist/src/gateway/cli.js.map +1 -1
- package/dist/src/gateway/codex-app-server.js +6 -2
- package/dist/src/gateway/codex-app-server.js.map +1 -1
- package/dist/src/gateway/codex-local-transport.d.ts +13 -0
- package/dist/src/gateway/codex-local-transport.js +21 -2
- package/dist/src/gateway/codex-local-transport.js.map +1 -1
- package/dist/src/gateway/config.js +5 -3
- package/dist/src/gateway/config.js.map +1 -1
- package/dist/src/gateway/control.d.ts +11 -1
- package/dist/src/gateway/control.js +48 -14
- package/dist/src/gateway/control.js.map +1 -1
- package/dist/src/gateway/dashboard-copy.d.ts +1 -1
- package/dist/src/gateway/dashboard-copy.en.d.ts +6 -0
- package/dist/src/gateway/dashboard-copy.en.js +11 -5
- package/dist/src/gateway/dashboard-copy.en.js.map +1 -1
- package/dist/src/gateway/dashboard-copy.js +6 -0
- package/dist/src/gateway/dashboard-copy.js.map +1 -1
- package/dist/src/gateway/dashboard-copy.zh-CN.d.ts +6 -0
- package/dist/src/gateway/dashboard-copy.zh-CN.js +11 -5
- package/dist/src/gateway/dashboard-copy.zh-CN.js.map +1 -1
- package/dist/src/gateway/dashboard-model.d.ts +4 -2
- package/dist/src/gateway/dashboard-model.js +6 -0
- package/dist/src/gateway/dashboard-model.js.map +1 -1
- package/dist/src/gateway/dashboard.js +5 -3
- package/dist/src/gateway/dashboard.js.map +1 -1
- package/dist/src/gateway/live-dashboard-app/app.js +48 -10
- package/dist/src/gateway/live-dashboard-command.js +10 -0
- package/dist/src/gateway/live-dashboard-command.js.map +1 -1
- package/dist/src/gateway/live-dashboard-http.d.ts +3 -0
- package/dist/src/gateway/live-dashboard-http.js +13 -0
- package/dist/src/gateway/live-dashboard-http.js.map +1 -1
- package/dist/src/gateway/live-dashboard-stream.js +26 -25
- package/dist/src/gateway/live-dashboard-stream.js.map +1 -1
- package/dist/src/gateway/provenance-envelope.d.ts +16 -0
- package/dist/src/gateway/provenance-envelope.js +80 -0
- package/dist/src/gateway/provenance-envelope.js.map +1 -0
- package/dist/src/gateway/providers.d.ts +43 -16
- package/dist/src/gateway/providers.js +688 -61
- package/dist/src/gateway/providers.js.map +1 -1
- package/dist/src/gateway/server.d.ts +2 -0
- package/dist/src/gateway/server.js +9 -3
- package/dist/src/gateway/server.js.map +1 -1
- package/dist/src/gateway/service.d.ts +102 -10
- package/dist/src/gateway/service.js +1309 -54
- package/dist/src/gateway/service.js.map +1 -1
- package/dist/src/gateway/store.d.ts +39 -3
- package/dist/src/gateway/store.js +528 -26
- package/dist/src/gateway/store.js.map +1 -1
- package/dist/src/gateway/types.d.ts +90 -5
- package/dist/src/gateway/types.js +6 -0
- package/dist/src/gateway/types.js.map +1 -1
- package/docs/GATEWAY-ARCHITECTURE.md +98 -11
- package/package.json +3 -2
- package/skills/embassy-peer/SKILL.md +15 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createHash, randomBytes } from "node:crypto";
|
|
2
2
|
import { BridgeError } from "../errors.js";
|
|
3
3
|
import { KeyedMutex } from "../mutex.js";
|
|
4
|
-
import { certifiedCompatibilityVersions, compatibilitySurfaces, attachCompatibilityCertification, evaluateCompatibilityAttestation, isCompatibilityCertification, isCompatibilityCertificationReport, isCompatibilityCheckReport, } from "./compatibility.js";
|
|
4
|
+
import { certifiedCompatibilityVersions, compatibilitySurfaces, attachCompatibilityCertification, evaluateCompatibilityAttestation, isCompatibilityAttestation, isCompatibilityCertification, isCompatibilityCertificationReport, isCompatibilityCheckReport, } from "./compatibility.js";
|
|
5
5
|
import { createGatewayConversationId, isGatewaySnapshot, startGatewayControlServer, } from "./control.js";
|
|
6
6
|
import { publishGatewayDashboard } from "./dashboard.js";
|
|
7
7
|
import { createDeliveryMachine, isTerminalDeliveryMachine, projectDelivery, projectDeliveryWakeups, transitionDelivery, } from "./delivery-machine.js";
|
|
@@ -22,6 +22,12 @@ const NATIVE_HELD_THRESHOLD_MS = 1_000;
|
|
|
22
22
|
const RUNTIME_ACTIVITY_CAPACITY = 256;
|
|
23
23
|
const DELIVERY_SETTLEMENT_RETRY_MS = 250;
|
|
24
24
|
const DELIVERY_DASHBOARD_REFRESH_MS = 15_000;
|
|
25
|
+
const CLAUDE_CLEAN_PREWRITE_ROUTE_CODES = new Set([
|
|
26
|
+
"CLAUDE_PEER_TARGET_UNKNOWN",
|
|
27
|
+
"CLAUDE_PEER_TARGET_STALE",
|
|
28
|
+
"CLAUDE_PEER_TARGET_CHANGED",
|
|
29
|
+
"CLAUDE_PEER_WORKSPACE_UNATTESTED",
|
|
30
|
+
]);
|
|
25
31
|
function progressWatchActivity(text, conversationId, actorAlias, explicitIdleMinutes) {
|
|
26
32
|
const prefixedOpen = text.startsWith("TRACK:");
|
|
27
33
|
const completionSignal = text.startsWith("DONE:");
|
|
@@ -49,6 +55,44 @@ function bindingKey(binding) {
|
|
|
49
55
|
binding.ownerLease,
|
|
50
56
|
].join("\0");
|
|
51
57
|
}
|
|
58
|
+
function endpointRefreshCallbackKey(event) {
|
|
59
|
+
const attestation = {
|
|
60
|
+
schemaVersion: event.attestation.schemaVersion,
|
|
61
|
+
surface: event.attestation.surface,
|
|
62
|
+
version: event.attestation.version,
|
|
63
|
+
tier: event.attestation.tier,
|
|
64
|
+
checkedAt: event.attestation.checkedAt,
|
|
65
|
+
probes: event.attestation.probes.map((probe) => ({
|
|
66
|
+
name: probe.name,
|
|
67
|
+
outcome: probe.outcome,
|
|
68
|
+
...(probe.safeErrorCode === undefined
|
|
69
|
+
? {}
|
|
70
|
+
: { safeErrorCode: probe.safeErrorCode }),
|
|
71
|
+
})),
|
|
72
|
+
...(event.attestation.certification === undefined
|
|
73
|
+
? {}
|
|
74
|
+
: { certification: { ...event.attestation.certification } }),
|
|
75
|
+
...(event.attestation.safeErrorCode === undefined
|
|
76
|
+
? {}
|
|
77
|
+
: { safeErrorCode: event.attestation.safeErrorCode }),
|
|
78
|
+
};
|
|
79
|
+
return event.outcome === "compatible"
|
|
80
|
+
? JSON.stringify({
|
|
81
|
+
outcome: event.outcome,
|
|
82
|
+
previous: event.previous,
|
|
83
|
+
current: event.current,
|
|
84
|
+
attestation,
|
|
85
|
+
routes: [...event.routes]
|
|
86
|
+
.map((route) => ({ ...route }))
|
|
87
|
+
.sort((left, right) => left.routeHandle.localeCompare(right.routeHandle)),
|
|
88
|
+
})
|
|
89
|
+
: JSON.stringify({
|
|
90
|
+
outcome: event.outcome,
|
|
91
|
+
previous: event.previous,
|
|
92
|
+
candidate: event.candidate,
|
|
93
|
+
attestation,
|
|
94
|
+
});
|
|
95
|
+
}
|
|
52
96
|
function stableLease(provider, value) {
|
|
53
97
|
return `lease_${createHash("sha256").update(provider).update("\0").update(value).digest("base64url")}`;
|
|
54
98
|
}
|
|
@@ -101,16 +145,42 @@ export class GatewayService {
|
|
|
101
145
|
activeDispatchByTarget = new Map();
|
|
102
146
|
scheduledDispatchTargets = new Set();
|
|
103
147
|
dispatchRunnerTargets = new Set();
|
|
148
|
+
/** Claude sends that proved no write wait for exact provider-idle evidence. */
|
|
149
|
+
claudePrewriteIdleWaitTargets = new Set();
|
|
104
150
|
heldRedispatchTimers = new Map();
|
|
105
151
|
pendingClaudeReplies = new Map();
|
|
106
152
|
deliveryTrackers = new Map();
|
|
107
153
|
deliveryTokens = new Map();
|
|
108
154
|
runtimeAlerts = [];
|
|
109
155
|
activityEvents = [];
|
|
156
|
+
activeCodexIncompatibility;
|
|
110
157
|
activitySequence = 0;
|
|
111
158
|
detachedReceiptWrites = new Set();
|
|
112
159
|
nativeIngressByConversation = new Map();
|
|
113
160
|
callbackQueue = [];
|
|
161
|
+
/**
|
|
162
|
+
* Endpoint replacement is authority-bearing lifecycle evidence, not an
|
|
163
|
+
* ordinary refreshable observation. Keep one coalesced slot outside the
|
|
164
|
+
* bounded callback queue so delivery evidence can never displace it (or be
|
|
165
|
+
* displaced by it).
|
|
166
|
+
*/
|
|
167
|
+
endpointRefreshCallback;
|
|
168
|
+
activeEndpointRefreshCallback;
|
|
169
|
+
endpointRefreshRetryRequested;
|
|
170
|
+
endpointRefreshConflictBoundary;
|
|
171
|
+
activeEndpointRefreshCallbackKey;
|
|
172
|
+
endpointRefreshCallbackPoisoned = false;
|
|
173
|
+
endpointRefreshConflictPending = false;
|
|
174
|
+
/** Last successfully activated transition per allowed Codex host. */
|
|
175
|
+
activatedEndpointRefreshes = new Map();
|
|
176
|
+
/** Durable reanchors awaiting their exact native-listener activation. */
|
|
177
|
+
durablyAppliedEndpointRefreshes = new Map();
|
|
178
|
+
/** Selector transitions staged until their exact registration owns a helper. */
|
|
179
|
+
pendingSelectorEndpointActivations = new Map();
|
|
180
|
+
/** Last incompatible transition already projected per allowed Codex host. */
|
|
181
|
+
handledIncompatibleEndpointRefreshes = new Map();
|
|
182
|
+
/** Process-local dedupe for automatic public activity rows across retries. */
|
|
183
|
+
endpointRefreshActivityKeys = new Set();
|
|
114
184
|
callbackWorker;
|
|
115
185
|
callbackScheduled = false;
|
|
116
186
|
callbackCapacity;
|
|
@@ -136,6 +206,12 @@ export class GatewayService {
|
|
|
136
206
|
/** Route-scoped fences; unrelated Codex registrations remain dispatchable. */
|
|
137
207
|
codexSuccessionDispatchFrozen = new Set();
|
|
138
208
|
codexSuccessionPoisoned = new Set();
|
|
209
|
+
/**
|
|
210
|
+
* Exact stale authority whose process-owned native advertisement was already
|
|
211
|
+
* removed, but whose durable orphan deletion did not commit. Keep dispatch
|
|
212
|
+
* frozen and permit only the same exact cleanup transaction to retry.
|
|
213
|
+
*/
|
|
214
|
+
codexOrphanCleanupPending = new Map();
|
|
139
215
|
codexSuccessionState;
|
|
140
216
|
pendingCodexSuccessionRecovery;
|
|
141
217
|
/**
|
|
@@ -171,6 +247,7 @@ export class GatewayService {
|
|
|
171
247
|
async start(signal) {
|
|
172
248
|
if (this.running || this.closing)
|
|
173
249
|
return;
|
|
250
|
+
let bootReactivatedIdleAliases = [];
|
|
174
251
|
const assertStartActive = () => {
|
|
175
252
|
if (signal?.aborted === true || this.closing) {
|
|
176
253
|
throw new BridgeError("GATEWAY_START_CANCELLED", "Gateway startup was cancelled before it became ready.", true);
|
|
@@ -191,7 +268,7 @@ export class GatewayService {
|
|
|
191
268
|
throw new BridgeError("GATEWAY_ADAPTER_NOT_ALLOWED", "A provider adapter is duplicated or outside the host allowlist.");
|
|
192
269
|
}
|
|
193
270
|
seen.add(key);
|
|
194
|
-
const observation = await adapter.initialize(this.callbacksFor(adapter
|
|
271
|
+
const observation = await adapter.initialize(this.callbacksFor(adapter));
|
|
195
272
|
assertStartActive();
|
|
196
273
|
await this.store.observeConnector({
|
|
197
274
|
identity: adapter.identity,
|
|
@@ -215,6 +292,8 @@ export class GatewayService {
|
|
|
215
292
|
}
|
|
216
293
|
}
|
|
217
294
|
await this.recoverCodexSuccessionAfterRestartLocked();
|
|
295
|
+
bootReactivatedIdleAliases =
|
|
296
|
+
await this.reactivateRetainedCodexRoutesAfterRestartLocked();
|
|
218
297
|
assertStartActive();
|
|
219
298
|
const control = await startGatewayControlServer({
|
|
220
299
|
stateDir: this.store.rootDir,
|
|
@@ -235,6 +314,9 @@ export class GatewayService {
|
|
|
235
314
|
this.nextDashboardRefreshAt = undefined;
|
|
236
315
|
await this.refreshNextProgressWatchWakeLocked();
|
|
237
316
|
this.scheduleLifecycleWakeLocked();
|
|
317
|
+
for (const alias of bootReactivatedIdleAliases) {
|
|
318
|
+
this.scheduleDispatch(alias);
|
|
319
|
+
}
|
|
238
320
|
await this.publish();
|
|
239
321
|
assertStartActive();
|
|
240
322
|
}
|
|
@@ -284,7 +366,10 @@ export class GatewayService {
|
|
|
284
366
|
for (const messageId of [...this.providerTurnContinuations.keys()]) {
|
|
285
367
|
await this.finishProviderTurnContinuationLocked(messageId, "cancelled");
|
|
286
368
|
}
|
|
369
|
+
const retainedQueuedMessageIds = new Set(await this.store.inspectQueuedMessageIds());
|
|
287
370
|
for (const messageId of [...this.messageContexts.keys()]) {
|
|
371
|
+
if (retainedQueuedMessageIds.has(messageId))
|
|
372
|
+
continue;
|
|
288
373
|
const queued = await this.store.settleQueuedMessage({
|
|
289
374
|
messageId,
|
|
290
375
|
state: "cancelled",
|
|
@@ -300,6 +385,8 @@ export class GatewayService {
|
|
|
300
385
|
});
|
|
301
386
|
}
|
|
302
387
|
for (const tracker of this.deliveryTrackers.values()) {
|
|
388
|
+
if (retainedQueuedMessageIds.has(tracker.messageId))
|
|
389
|
+
continue;
|
|
303
390
|
if (!isTerminalDeliveryMachine(tracker.machine)) {
|
|
304
391
|
await this.advanceDeliveryLocked(tracker.messageId, {
|
|
305
392
|
type: "shutdown",
|
|
@@ -340,6 +427,7 @@ export class GatewayService {
|
|
|
340
427
|
this.activeDispatchByTarget.clear();
|
|
341
428
|
this.scheduledDispatchTargets.clear();
|
|
342
429
|
this.dispatchRunnerTargets.clear();
|
|
430
|
+
this.claudePrewriteIdleWaitTargets.clear();
|
|
343
431
|
for (const timer of this.heldRedispatchTimers.values()) {
|
|
344
432
|
this.timers.clearTimeout(timer);
|
|
345
433
|
}
|
|
@@ -351,6 +439,19 @@ export class GatewayService {
|
|
|
351
439
|
this.detachedReceiptWrites.clear();
|
|
352
440
|
this.nativeIngressByConversation.clear();
|
|
353
441
|
this.callbackQueue.length = 0;
|
|
442
|
+
this.endpointRefreshCallback = undefined;
|
|
443
|
+
this.activeEndpointRefreshCallback = undefined;
|
|
444
|
+
this.endpointRefreshRetryRequested = undefined;
|
|
445
|
+
this.endpointRefreshConflictBoundary = undefined;
|
|
446
|
+
this.activeEndpointRefreshCallbackKey = undefined;
|
|
447
|
+
this.endpointRefreshConflictPending = false;
|
|
448
|
+
this.endpointRefreshCallbackPoisoned = true;
|
|
449
|
+
this.activatedEndpointRefreshes.clear();
|
|
450
|
+
this.durablyAppliedEndpointRefreshes.clear();
|
|
451
|
+
this.pendingSelectorEndpointActivations.clear();
|
|
452
|
+
this.handledIncompatibleEndpointRefreshes.clear();
|
|
453
|
+
this.endpointRefreshActivityKeys.clear();
|
|
454
|
+
this.codexOrphanCleanupPending.clear();
|
|
354
455
|
this.candidates.clear();
|
|
355
456
|
this.routeBindings.clear();
|
|
356
457
|
this.routeStates.clear();
|
|
@@ -390,6 +491,11 @@ export class GatewayService {
|
|
|
390
491
|
action: "codex_unregistered",
|
|
391
492
|
aliases: [params.alias],
|
|
392
493
|
}),
|
|
494
|
+
removeStaleCodexRegistration: async (params) => await this.exclusiveDecision(async () => this.removeStaleCodexRegistration(params.alias), {
|
|
495
|
+
kind: "recovery",
|
|
496
|
+
action: "codex_orphan_removed",
|
|
497
|
+
aliases: [params.alias],
|
|
498
|
+
}),
|
|
393
499
|
selectClaude: async (params) => await this.exclusiveDecision(async () => this.selectClaude(params), {
|
|
394
500
|
kind: "selection",
|
|
395
501
|
action: "claude_selected",
|
|
@@ -663,9 +769,10 @@ export class GatewayService {
|
|
|
663
769
|
}
|
|
664
770
|
return createHash("sha256").update(encoded, "utf8").digest("base64url");
|
|
665
771
|
}
|
|
666
|
-
callbacksFor(
|
|
772
|
+
callbacksFor(adapter) {
|
|
667
773
|
return {
|
|
668
774
|
onDelivery: (event) => {
|
|
775
|
+
const source = adapter.identity;
|
|
669
776
|
if (!this.running ||
|
|
670
777
|
!MESSAGE_ID.test(event.messageId) ||
|
|
671
778
|
this.deliveryCorrelation(event.messageId) === undefined ||
|
|
@@ -694,6 +801,7 @@ export class GatewayService {
|
|
|
694
801
|
});
|
|
695
802
|
},
|
|
696
803
|
onRouteState: (event) => {
|
|
804
|
+
const source = adapter.identity;
|
|
697
805
|
if (!this.running ||
|
|
698
806
|
event.endpoint.provider !== source.provider ||
|
|
699
807
|
event.endpoint.hostId !== source.hostId ||
|
|
@@ -716,6 +824,7 @@ export class GatewayService {
|
|
|
716
824
|
});
|
|
717
825
|
},
|
|
718
826
|
onClaudeReply: (event) => {
|
|
827
|
+
const source = adapter.identity;
|
|
719
828
|
if (!this.running ||
|
|
720
829
|
event.endpoint.provider !== source.provider ||
|
|
721
830
|
event.endpoint.hostId !== source.hostId ||
|
|
@@ -742,6 +851,7 @@ export class GatewayService {
|
|
|
742
851
|
});
|
|
743
852
|
},
|
|
744
853
|
onClaudeMessage: (event) => {
|
|
854
|
+
const source = adapter.identity;
|
|
745
855
|
const ingressFailureCode = this.closing
|
|
746
856
|
? "GATEWAY_SHUTDOWN"
|
|
747
857
|
: "NATIVE_INGRESS_INVALID";
|
|
@@ -779,6 +889,7 @@ export class GatewayService {
|
|
|
779
889
|
}
|
|
780
890
|
},
|
|
781
891
|
onProtocolNotice: (event) => {
|
|
892
|
+
const source = adapter.identity;
|
|
782
893
|
if (!this.running || !SAFE_CODE.test(event.code))
|
|
783
894
|
return;
|
|
784
895
|
this.enqueueCallback({
|
|
@@ -787,11 +898,61 @@ export class GatewayService {
|
|
|
787
898
|
value: { code: event.code },
|
|
788
899
|
});
|
|
789
900
|
},
|
|
901
|
+
onEndpointRefresh: (event) => {
|
|
902
|
+
if (!this.running ||
|
|
903
|
+
adapter.identity.provider !== "codex" ||
|
|
904
|
+
event.previous.provider !== "codex" ||
|
|
905
|
+
event.previous.hostId !== adapter.identity.hostId ||
|
|
906
|
+
!isCompatibilityAttestation(event.attestation) ||
|
|
907
|
+
event.attestation.surface !== "codex") {
|
|
908
|
+
return;
|
|
909
|
+
}
|
|
910
|
+
if (event.outcome === "compatible") {
|
|
911
|
+
const routeHandles = event.routes.map((route) => route.routeHandle);
|
|
912
|
+
if (event.current.provider !== "codex" ||
|
|
913
|
+
event.current.hostId !== event.previous.hostId ||
|
|
914
|
+
event.current.endpointGeneration !==
|
|
915
|
+
adapter.identity.endpointGeneration ||
|
|
916
|
+
event.attestation.version !== adapter.protocolVersion ||
|
|
917
|
+
event.attestation.tier === "incompatible" ||
|
|
918
|
+
routeHandles.length > this.config.limits.maxRoutes ||
|
|
919
|
+
new Set(routeHandles).size !== routeHandles.length ||
|
|
920
|
+
routeHandles.some((routeHandle) => !PRIVATE_HANDLE.test(routeHandle))) {
|
|
921
|
+
return;
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
else if (event.candidate.provider !== "codex" ||
|
|
925
|
+
event.candidate.hostId !== event.previous.hostId ||
|
|
926
|
+
event.previous.endpointGeneration !==
|
|
927
|
+
adapter.identity.endpointGeneration ||
|
|
928
|
+
event.attestation.tier !== "incompatible") {
|
|
929
|
+
return;
|
|
930
|
+
}
|
|
931
|
+
this.enqueueCallback({
|
|
932
|
+
type: "endpoint_refresh",
|
|
933
|
+
adapter,
|
|
934
|
+
value: event.outcome === "compatible"
|
|
935
|
+
? {
|
|
936
|
+
...event,
|
|
937
|
+
previous: { ...event.previous },
|
|
938
|
+
current: { ...event.current },
|
|
939
|
+
routes: event.routes.map((route) => ({ ...route })),
|
|
940
|
+
}
|
|
941
|
+
: {
|
|
942
|
+
...event,
|
|
943
|
+
previous: { ...event.previous },
|
|
944
|
+
candidate: { ...event.candidate },
|
|
945
|
+
},
|
|
946
|
+
});
|
|
947
|
+
},
|
|
790
948
|
};
|
|
791
949
|
}
|
|
792
950
|
enqueueCallback(event) {
|
|
793
951
|
if (!this.acceptingCallbacks)
|
|
794
952
|
return false;
|
|
953
|
+
if (event.type === "endpoint_refresh") {
|
|
954
|
+
return this.enqueueEndpointRefreshCallback(event);
|
|
955
|
+
}
|
|
795
956
|
if (event.type === "delivery") {
|
|
796
957
|
const duplicate = this.callbackQueue.findIndex((candidate) => candidate.type === "delivery" &&
|
|
797
958
|
candidate.value.messageId === event.value.messageId &&
|
|
@@ -834,7 +995,7 @@ export class GatewayService {
|
|
|
834
995
|
// Delivery write/approval observations are authoritative evidence for
|
|
835
996
|
// no-replay and deadline arbitration. Never evict them. A stale route
|
|
836
997
|
// observation is safe to replace because discovery re-observes it.
|
|
837
|
-
const replaceable = this.callbackQueue.findIndex((candidate) => candidate.type === "route");
|
|
998
|
+
const replaceable = this.callbackQueue.findIndex((candidate) => candidate.type === "route" || candidate.type === "protocol_notice");
|
|
838
999
|
if (replaceable < 0) {
|
|
839
1000
|
this.dashboardHealthy = false;
|
|
840
1001
|
return false;
|
|
@@ -842,29 +1003,49 @@ export class GatewayService {
|
|
|
842
1003
|
this.callbackQueue.splice(replaceable, 1);
|
|
843
1004
|
}
|
|
844
1005
|
this.callbackQueue.push(event);
|
|
845
|
-
|
|
846
|
-
return true;
|
|
847
|
-
this.callbackScheduled = true;
|
|
848
|
-
setImmediate(() => {
|
|
849
|
-
this.callbackScheduled = false;
|
|
850
|
-
const worker = this.mutex.run("service", async () => {
|
|
851
|
-
await this.drainCallbackQueueLocked();
|
|
852
|
-
});
|
|
853
|
-
this.callbackWorker = worker;
|
|
854
|
-
worker
|
|
855
|
-
.catch(() => {
|
|
856
|
-
this.dashboardHealthy = false;
|
|
857
|
-
})
|
|
858
|
-
.finally(() => {
|
|
859
|
-
if (this.callbackWorker === worker)
|
|
860
|
-
this.callbackWorker = undefined;
|
|
861
|
-
if (this.callbackQueue.length > 0 && !this.closing) {
|
|
862
|
-
this.enqueueCallbackWorkerOnly();
|
|
863
|
-
}
|
|
864
|
-
});
|
|
865
|
-
});
|
|
1006
|
+
this.enqueueCallbackWorkerOnly();
|
|
866
1007
|
return true;
|
|
867
1008
|
}
|
|
1009
|
+
enqueueEndpointRefreshCallback(event) {
|
|
1010
|
+
if (this.endpointRefreshCallbackPoisoned)
|
|
1011
|
+
return false;
|
|
1012
|
+
const key = endpointRefreshCallbackKey(event.value);
|
|
1013
|
+
if (this.activeEndpointRefreshCallbackKey !== undefined) {
|
|
1014
|
+
if (this.activeEndpointRefreshCallbackKey === key) {
|
|
1015
|
+
this.endpointRefreshRetryRequested = event;
|
|
1016
|
+
return true;
|
|
1017
|
+
}
|
|
1018
|
+
this.poisonEndpointRefreshCallbacks(event);
|
|
1019
|
+
return false;
|
|
1020
|
+
}
|
|
1021
|
+
if (this.endpointRefreshCallback !== undefined) {
|
|
1022
|
+
if (endpointRefreshCallbackKey(this.endpointRefreshCallback.value) === key) {
|
|
1023
|
+
return true;
|
|
1024
|
+
}
|
|
1025
|
+
this.poisonEndpointRefreshCallbacks(event);
|
|
1026
|
+
return false;
|
|
1027
|
+
}
|
|
1028
|
+
this.endpointRefreshCallback = event;
|
|
1029
|
+
this.enqueueCallbackWorkerOnly();
|
|
1030
|
+
return true;
|
|
1031
|
+
}
|
|
1032
|
+
poisonEndpointRefreshCallbacks(conflicting) {
|
|
1033
|
+
this.endpointRefreshConflictBoundary =
|
|
1034
|
+
this.activeEndpointRefreshCallback ??
|
|
1035
|
+
this.endpointRefreshCallback ??
|
|
1036
|
+
conflicting;
|
|
1037
|
+
this.endpointRefreshCallback = undefined;
|
|
1038
|
+
this.endpointRefreshRetryRequested = undefined;
|
|
1039
|
+
this.endpointRefreshCallbackPoisoned = true;
|
|
1040
|
+
this.endpointRefreshConflictPending = true;
|
|
1041
|
+
this.dashboardHealthy = false;
|
|
1042
|
+
this.enqueueCallbackWorkerOnly();
|
|
1043
|
+
}
|
|
1044
|
+
hasPendingCallbackWork() {
|
|
1045
|
+
return (this.callbackQueue.length > 0 ||
|
|
1046
|
+
this.endpointRefreshCallback !== undefined ||
|
|
1047
|
+
this.endpointRefreshConflictPending);
|
|
1048
|
+
}
|
|
868
1049
|
enqueueCallbackWorkerOnly() {
|
|
869
1050
|
if (this.callbackScheduled)
|
|
870
1051
|
return;
|
|
@@ -882,7 +1063,7 @@ export class GatewayService {
|
|
|
882
1063
|
.finally(() => {
|
|
883
1064
|
if (this.callbackWorker === worker)
|
|
884
1065
|
this.callbackWorker = undefined;
|
|
885
|
-
if (this.
|
|
1066
|
+
if (this.hasPendingCallbackWork() && !this.closing) {
|
|
886
1067
|
this.enqueueCallbackWorkerOnly();
|
|
887
1068
|
}
|
|
888
1069
|
});
|
|
@@ -895,10 +1076,95 @@ export class GatewayService {
|
|
|
895
1076
|
await this.publish();
|
|
896
1077
|
}
|
|
897
1078
|
this.pruneTransient();
|
|
898
|
-
while (this.
|
|
899
|
-
|
|
900
|
-
|
|
1079
|
+
while (this.hasPendingCallbackWork()) {
|
|
1080
|
+
// Preserve the complete ordinary queue order. In particular, every
|
|
1081
|
+
// delivery observation already retained before endpoint replacement is
|
|
1082
|
+
// reduced before the lifecycle transition can settle its generation.
|
|
1083
|
+
const ordinaryBatch = this.callbackQueue.splice(0, this.callbackQueue.length);
|
|
1084
|
+
for (const event of ordinaryBatch) {
|
|
1085
|
+
try {
|
|
1086
|
+
if (event.type === "delivery") {
|
|
1087
|
+
await this.onDelivery(event.source, event.value, event.receivedAt);
|
|
1088
|
+
}
|
|
1089
|
+
else if (event.type === "route") {
|
|
1090
|
+
await this.onRouteState(event.source, event.value);
|
|
1091
|
+
}
|
|
1092
|
+
else if (event.type === "claude_reply") {
|
|
1093
|
+
await this.onClaudeReply(event.value);
|
|
1094
|
+
}
|
|
1095
|
+
else if (event.type === "claude_message") {
|
|
1096
|
+
await this.onClaudeMessage(event.value);
|
|
1097
|
+
}
|
|
1098
|
+
else if (event.type === "protocol_notice") {
|
|
1099
|
+
this.addRuntimeAlert(event.value.code, "warning", {
|
|
1100
|
+
provider: event.source.provider,
|
|
1101
|
+
host: event.source.hostId,
|
|
1102
|
+
});
|
|
1103
|
+
await this.changed();
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
catch {
|
|
1107
|
+
this.dashboardHealthy = false;
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
await this.drainPreDeadlineDeliveryCallbacksLocked();
|
|
1111
|
+
if (this.endpointRefreshConflictPending) {
|
|
1112
|
+
this.endpointRefreshConflictPending = false;
|
|
1113
|
+
const boundary = this.endpointRefreshConflictBoundary;
|
|
1114
|
+
if (boundary !== undefined) {
|
|
1115
|
+
await this.failClosedEndpointRefreshConflict(boundary);
|
|
1116
|
+
}
|
|
1117
|
+
else {
|
|
1118
|
+
this.addRuntimeAlert("CODEX_ENDPOINT_REFRESH_CONFLICT", "error", { provider: "codex" });
|
|
1119
|
+
await this.changed();
|
|
1120
|
+
}
|
|
901
1121
|
continue;
|
|
1122
|
+
}
|
|
1123
|
+
const endpointRefresh = this.endpointRefreshCallback;
|
|
1124
|
+
if (endpointRefresh === undefined)
|
|
1125
|
+
continue;
|
|
1126
|
+
const key = endpointRefreshCallbackKey(endpointRefresh.value);
|
|
1127
|
+
this.endpointRefreshCallback = undefined;
|
|
1128
|
+
this.activeEndpointRefreshCallback = endpointRefresh;
|
|
1129
|
+
this.activeEndpointRefreshCallbackKey = key;
|
|
1130
|
+
let succeeded = false;
|
|
1131
|
+
try {
|
|
1132
|
+
await this.onCodexEndpointRefresh(endpointRefresh.adapter, endpointRefresh.value);
|
|
1133
|
+
succeeded = true;
|
|
1134
|
+
}
|
|
1135
|
+
catch {
|
|
1136
|
+
this.dashboardHealthy = false;
|
|
1137
|
+
}
|
|
1138
|
+
finally {
|
|
1139
|
+
this.activeEndpointRefreshCallback = undefined;
|
|
1140
|
+
if (this.activeEndpointRefreshCallbackKey === key) {
|
|
1141
|
+
this.activeEndpointRefreshCallbackKey = undefined;
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
const requestedRetry = this.endpointRefreshRetryRequested;
|
|
1145
|
+
this.endpointRefreshRetryRequested = undefined;
|
|
1146
|
+
if (!succeeded &&
|
|
1147
|
+
!this.endpointRefreshCallbackPoisoned &&
|
|
1148
|
+
requestedRetry !== undefined) {
|
|
1149
|
+
this.endpointRefreshCallback = requestedRetry;
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
/**
|
|
1154
|
+
* A selector can synchronously admit callbacks while its promise is still
|
|
1155
|
+
* running under the service mutex. Drain exactly the ordinary work that was
|
|
1156
|
+
* retained before the selector returned before applying a returned endpoint
|
|
1157
|
+
* transition. Delivery evidence wins within that fixed snapshot; callbacks
|
|
1158
|
+
* admitted while it is reduced remain queued for the ordinary worker, and
|
|
1159
|
+
* the dedicated endpoint-transition slot is never consumed recursively.
|
|
1160
|
+
*/
|
|
1161
|
+
async drainSelectorCallbackSnapshotLocked() {
|
|
1162
|
+
const snapshot = this.callbackQueue.splice(0, this.callbackQueue.length);
|
|
1163
|
+
const ordered = [
|
|
1164
|
+
...snapshot.filter((event) => event.type === "delivery"),
|
|
1165
|
+
...snapshot.filter((event) => event.type !== "delivery"),
|
|
1166
|
+
];
|
|
1167
|
+
for (const event of ordered) {
|
|
902
1168
|
try {
|
|
903
1169
|
if (event.type === "delivery") {
|
|
904
1170
|
await this.onDelivery(event.source, event.value, event.receivedAt);
|
|
@@ -912,7 +1178,7 @@ export class GatewayService {
|
|
|
912
1178
|
else if (event.type === "claude_message") {
|
|
913
1179
|
await this.onClaudeMessage(event.value);
|
|
914
1180
|
}
|
|
915
|
-
else {
|
|
1181
|
+
else if (event.type === "protocol_notice") {
|
|
916
1182
|
this.addRuntimeAlert(event.value.code, "warning", {
|
|
917
1183
|
provider: event.source.provider,
|
|
918
1184
|
host: event.source.hostId,
|
|
@@ -924,6 +1190,59 @@ export class GatewayService {
|
|
|
924
1190
|
this.dashboardHealthy = false;
|
|
925
1191
|
}
|
|
926
1192
|
}
|
|
1193
|
+
// Synchronous dispatch may have admitted authoritative delivery evidence,
|
|
1194
|
+
// and rejected native ingress may have started a detached terminal write.
|
|
1195
|
+
// Join both before the endpoint boundary can move to its next generation.
|
|
1196
|
+
await this.drainPreDeadlineDeliveryCallbacksLocked();
|
|
1197
|
+
await this.drainDetachedReceiptWritesLocked();
|
|
1198
|
+
}
|
|
1199
|
+
async failClosedEndpointRefreshConflict(boundary) {
|
|
1200
|
+
const identities = [
|
|
1201
|
+
boundary.value.previous,
|
|
1202
|
+
...(boundary.value.outcome === "compatible"
|
|
1203
|
+
? [boundary.value.current]
|
|
1204
|
+
: []),
|
|
1205
|
+
];
|
|
1206
|
+
for (const identity of identities) {
|
|
1207
|
+
const routes = (await this.store.inspectPrivateCodexRoutes()).filter((route) => route.binding.hostId === identity.hostId &&
|
|
1208
|
+
route.binding.endpointGeneration === identity.endpointGeneration);
|
|
1209
|
+
const aliases = routes.map((route) => route.alias);
|
|
1210
|
+
const inFlightSettlements = aliases.length === 0
|
|
1211
|
+
? []
|
|
1212
|
+
: await this.planRoutesInFlightSettlementsLocked(aliases, {
|
|
1213
|
+
unwrittenOutcome: "failed",
|
|
1214
|
+
safeErrorCode: "CODEX_ENDPOINT_REFRESH_CONFLICT",
|
|
1215
|
+
});
|
|
1216
|
+
try {
|
|
1217
|
+
const settlements = await this.store.markConnectorOffline(identity, "CODEX_ENDPOINT_REFRESH_CONFLICT", inFlightSettlements, { preserveQueued: true });
|
|
1218
|
+
for (const settlement of settlements) {
|
|
1219
|
+
await this.applyTerminalSettlementLocked(settlement);
|
|
1220
|
+
}
|
|
1221
|
+
for (const route of routes) {
|
|
1222
|
+
this.routeStates.set(route.alias, "stale");
|
|
1223
|
+
}
|
|
1224
|
+
break;
|
|
1225
|
+
}
|
|
1226
|
+
catch (error) {
|
|
1227
|
+
if (error instanceof BridgeError &&
|
|
1228
|
+
error.code === "CONNECTOR_NOT_FOUND") {
|
|
1229
|
+
continue;
|
|
1230
|
+
}
|
|
1231
|
+
throw error;
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
this.addRuntimeAlert("CODEX_ENDPOINT_REFRESH_CONFLICT", "error", { provider: "codex", host: boundary.value.previous.hostId });
|
|
1235
|
+
await this.changed();
|
|
1236
|
+
}
|
|
1237
|
+
async requireEndpointRefreshUnconflicted() {
|
|
1238
|
+
if (!this.endpointRefreshCallbackPoisoned)
|
|
1239
|
+
return;
|
|
1240
|
+
this.endpointRefreshConflictPending = false;
|
|
1241
|
+
const boundary = this.endpointRefreshConflictBoundary;
|
|
1242
|
+
if (boundary !== undefined) {
|
|
1243
|
+
await this.failClosedEndpointRefreshConflict(boundary);
|
|
1244
|
+
}
|
|
1245
|
+
throw new BridgeError("CODEX_ENDPOINT_REFRESH_CONFLICT", "Conflicting Codex endpoint transitions require a clean broker restart.");
|
|
927
1246
|
}
|
|
928
1247
|
isTerminalAdapterDelivery(event) {
|
|
929
1248
|
return (event.state !== "transport_uncertain" &&
|
|
@@ -1030,6 +1349,144 @@ export class GatewayService {
|
|
|
1030
1349
|
generation: authority.journal.new.generation,
|
|
1031
1350
|
}));
|
|
1032
1351
|
}
|
|
1352
|
+
/**
|
|
1353
|
+
* Rebuild process-owned route and listener authority for retained Codex
|
|
1354
|
+
* registrations during staged startup. The provider's selectRoute performs
|
|
1355
|
+
* the exact loaded/list + history-free resume proof; durable state advances
|
|
1356
|
+
* only after the new native advertisement and status are live.
|
|
1357
|
+
*/
|
|
1358
|
+
async reactivateRetainedCodexRoutesAfterRestartLocked() {
|
|
1359
|
+
const idleAliases = [];
|
|
1360
|
+
const retained = await this.store.inspectPrivateCodexRoutes();
|
|
1361
|
+
for (const route of retained) {
|
|
1362
|
+
if (!route.enabled ||
|
|
1363
|
+
route.state !== "stale" ||
|
|
1364
|
+
route.compatibility !== "expired" ||
|
|
1365
|
+
this.pendingCodexSuccessionRecovery?.journal.new.alias === route.alias ||
|
|
1366
|
+
this.codexSuccessionPoisoned.has(route.alias)) {
|
|
1367
|
+
continue;
|
|
1368
|
+
}
|
|
1369
|
+
const codexAdapter = this.adapters.find((adapter) => adapter.identity.provider === "codex" &&
|
|
1370
|
+
adapter.identity.hostId === route.binding.hostId);
|
|
1371
|
+
const claudeAdapter = this.adapters.find((adapter) => adapter.identity.provider === "claude" &&
|
|
1372
|
+
adapter.identity.hostId === route.binding.hostId);
|
|
1373
|
+
if (codexAdapter === undefined || claudeAdapter === undefined) {
|
|
1374
|
+
this.addRuntimeAlert("CODEX_BOOT_REACTIVATION_SKIPPED", "warning", { provider: "codex", host: route.binding.hostId, alias: route.alias });
|
|
1375
|
+
continue;
|
|
1376
|
+
}
|
|
1377
|
+
let routeSelected = false;
|
|
1378
|
+
let advertiseAttempted = false;
|
|
1379
|
+
let storeReactivated = false;
|
|
1380
|
+
let deferredEndpointRefresh = false;
|
|
1381
|
+
try {
|
|
1382
|
+
const selected = await codexAdapter.selectRoute({
|
|
1383
|
+
alias: route.alias,
|
|
1384
|
+
routeHandle: route.binding.routeHandle,
|
|
1385
|
+
});
|
|
1386
|
+
routeSelected = true;
|
|
1387
|
+
await this.drainSelectorCallbackSnapshotLocked();
|
|
1388
|
+
if (selected.routeHandle !== route.binding.routeHandle) {
|
|
1389
|
+
throw new BridgeError("CODEX_BOOT_REACTIVATION_MISMATCH", "The retained Codex task changed during broker startup reactivation.");
|
|
1390
|
+
}
|
|
1391
|
+
if (selected.endpointRefresh !== undefined) {
|
|
1392
|
+
deferredEndpointRefresh = await this.onCodexEndpointRefresh(codexAdapter, selected.endpointRefresh, {
|
|
1393
|
+
deferActivation: true,
|
|
1394
|
+
deferredRoute: {
|
|
1395
|
+
alias: route.alias,
|
|
1396
|
+
threadId: route.binding.routeHandle,
|
|
1397
|
+
state: selected.state,
|
|
1398
|
+
},
|
|
1399
|
+
});
|
|
1400
|
+
if (!deferredEndpointRefresh) {
|
|
1401
|
+
throw new BridgeError("CODEX_BOOT_REACTIVATION_MISMATCH", "The retained Codex task could not stage a compatible endpoint refresh during startup.");
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
const binding = {
|
|
1405
|
+
...codexAdapter.identity,
|
|
1406
|
+
routeHandle: route.binding.routeHandle,
|
|
1407
|
+
ownerLease: route.binding.ownerLease,
|
|
1408
|
+
};
|
|
1409
|
+
if (deferredEndpointRefresh) {
|
|
1410
|
+
const afterStaging = await this.store.inspectPrivateRoute(route.alias);
|
|
1411
|
+
storeReactivated =
|
|
1412
|
+
afterStaging !== undefined &&
|
|
1413
|
+
afterStaging.enabled &&
|
|
1414
|
+
afterStaging.compatibility === "compatible" &&
|
|
1415
|
+
bindingKey(afterStaging.binding) === bindingKey(binding);
|
|
1416
|
+
}
|
|
1417
|
+
advertiseAttempted = true;
|
|
1418
|
+
const advertise = claudeAdapter.advertiseNativeCodexPeer;
|
|
1419
|
+
if (advertise === undefined) {
|
|
1420
|
+
throw new BridgeError("CODEX_BOOT_REACTIVATION_UNAVAILABLE", "The retained Codex task cannot rebuild its native advertisement.");
|
|
1421
|
+
}
|
|
1422
|
+
await advertise.call(claudeAdapter, {
|
|
1423
|
+
alias: route.alias,
|
|
1424
|
+
cwd: this.nativePeerCwd,
|
|
1425
|
+
});
|
|
1426
|
+
await claudeAdapter.updateNativeCodexPeerStatus?.(route.alias, selected.state === "idle"
|
|
1427
|
+
? "idle"
|
|
1428
|
+
: selected.state === "awaiting_approval"
|
|
1429
|
+
? "waiting"
|
|
1430
|
+
: "busy");
|
|
1431
|
+
const listenerGeneration = this.requireCurrentNativeCodexGeneration(claudeAdapter, route.alias);
|
|
1432
|
+
if (!storeReactivated) {
|
|
1433
|
+
await this.store.rebindStaleRoute({
|
|
1434
|
+
alias: route.alias,
|
|
1435
|
+
currentOwnerLease: route.binding.ownerLease,
|
|
1436
|
+
newBinding: binding,
|
|
1437
|
+
reason: "endpoint_reobserved",
|
|
1438
|
+
journalReason: "boot_reactivation",
|
|
1439
|
+
state: selected.state,
|
|
1440
|
+
});
|
|
1441
|
+
storeReactivated = true;
|
|
1442
|
+
}
|
|
1443
|
+
this.rememberBinding(route.alias, binding, selected.state);
|
|
1444
|
+
this.lockCodexRegistration({
|
|
1445
|
+
alias: route.alias,
|
|
1446
|
+
threadId: route.binding.routeHandle,
|
|
1447
|
+
hostId: route.binding.hostId,
|
|
1448
|
+
busyPolicy: "queue",
|
|
1449
|
+
}, listenerGeneration);
|
|
1450
|
+
if (deferredEndpointRefresh &&
|
|
1451
|
+
selected.endpointRefresh !== undefined) {
|
|
1452
|
+
await this.onCodexEndpointRefresh(codexAdapter, selected.endpointRefresh, {
|
|
1453
|
+
requiredRoute: {
|
|
1454
|
+
alias: route.alias,
|
|
1455
|
+
threadId: route.binding.routeHandle,
|
|
1456
|
+
state: selected.state,
|
|
1457
|
+
},
|
|
1458
|
+
});
|
|
1459
|
+
}
|
|
1460
|
+
if (this.requireCurrentNativeCodexGeneration(claudeAdapter, route.alias) !== listenerGeneration) {
|
|
1461
|
+
throw new BridgeError("CODEX_BOOT_REACTIVATION_NATIVE_LISTENER_MISSING", "The retained Codex task's native listener changed during boot reactivation.");
|
|
1462
|
+
}
|
|
1463
|
+
if (selected.state === "idle")
|
|
1464
|
+
idleAliases.push(route.alias);
|
|
1465
|
+
}
|
|
1466
|
+
catch {
|
|
1467
|
+
let cleanupFailed = false;
|
|
1468
|
+
if (advertiseAttempted) {
|
|
1469
|
+
await claudeAdapter
|
|
1470
|
+
.unadvertiseNativeCodexPeer?.(route.alias)
|
|
1471
|
+
.catch(() => {
|
|
1472
|
+
cleanupFailed = true;
|
|
1473
|
+
});
|
|
1474
|
+
}
|
|
1475
|
+
if (routeSelected) {
|
|
1476
|
+
await codexAdapter
|
|
1477
|
+
.releaseRoute?.(route.binding.routeHandle)
|
|
1478
|
+
.catch(() => {
|
|
1479
|
+
cleanupFailed = true;
|
|
1480
|
+
});
|
|
1481
|
+
}
|
|
1482
|
+
if (cleanupFailed || storeReactivated || deferredEndpointRefresh) {
|
|
1483
|
+
throw new BridgeError("CODEX_BOOT_REACTIVATION_CLEANUP_FAILED", "A Codex boot reactivation crossed its durable boundary without exact live authority.");
|
|
1484
|
+
}
|
|
1485
|
+
this.addRuntimeAlert("CODEX_BOOT_REACTIVATION_SKIPPED", "warning", { provider: "codex", host: route.binding.hostId, alias: route.alias });
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1488
|
+
return idleAliases;
|
|
1489
|
+
}
|
|
1033
1490
|
async completePendingCodexSuccessionRecoveryLocked(params) {
|
|
1034
1491
|
const recovery = this.pendingCodexSuccessionRecovery;
|
|
1035
1492
|
if (recovery === undefined)
|
|
@@ -1744,7 +2201,7 @@ export class GatewayService {
|
|
|
1744
2201
|
claude.rejectedInboundSettlements === 0 &&
|
|
1745
2202
|
codexQuiet &&
|
|
1746
2203
|
this.successionServiceStateClean(aliases) &&
|
|
1747
|
-
this.
|
|
2204
|
+
this.endpointCallbacksQuiet() &&
|
|
1748
2205
|
this.detachedReceiptWrites.size === 0);
|
|
1749
2206
|
}
|
|
1750
2207
|
async codexSuccessionBarrierCleanLocked(oldRegistration) {
|
|
@@ -1771,7 +2228,7 @@ export class GatewayService {
|
|
|
1771
2228
|
claude.monitorFrozen &&
|
|
1772
2229
|
codex.clean &&
|
|
1773
2230
|
this.successionServiceStateClean(aliases) &&
|
|
1774
|
-
this.
|
|
2231
|
+
this.endpointCallbacksQuiet() &&
|
|
1775
2232
|
this.detachedReceiptWrites.size === 0);
|
|
1776
2233
|
}
|
|
1777
2234
|
rescheduleIdleRoutesLocked() {
|
|
@@ -1780,13 +2237,48 @@ export class GatewayService {
|
|
|
1780
2237
|
this.scheduleDispatch(alias);
|
|
1781
2238
|
}
|
|
1782
2239
|
}
|
|
2240
|
+
codexSuccessionIsStable() {
|
|
2241
|
+
const state = this.codexSuccessionState;
|
|
2242
|
+
return (state === undefined ||
|
|
2243
|
+
state.phase === "active_old" ||
|
|
2244
|
+
(state.phase === "active_new" && state.retired === null));
|
|
2245
|
+
}
|
|
2246
|
+
async requireUnchangedStaleCodexOrphan(alias, expected) {
|
|
2247
|
+
let current;
|
|
2248
|
+
try {
|
|
2249
|
+
current = await this.store.inspectStaleCodexOrphan(alias);
|
|
2250
|
+
}
|
|
2251
|
+
catch {
|
|
2252
|
+
throw new BridgeError("CODEX_ORPHAN_RECOVERY_MISMATCH", "The stale Codex registration changed during recovery.");
|
|
2253
|
+
}
|
|
2254
|
+
if (bindingKey(current) !== bindingKey(expected)) {
|
|
2255
|
+
throw new BridgeError("CODEX_ORPHAN_RECOVERY_MISMATCH", "The stale Codex registration changed during recovery.");
|
|
2256
|
+
}
|
|
2257
|
+
}
|
|
2258
|
+
nativeCodexOrphanBarrierClean(observation, generation) {
|
|
2259
|
+
return (observation.clean &&
|
|
2260
|
+
observation.generation === generation &&
|
|
2261
|
+
observation.activeGenerationMatched &&
|
|
2262
|
+
observation.ingressQuiesced &&
|
|
2263
|
+
observation.monitorFrozen &&
|
|
2264
|
+
!observation.discoveryInFlight &&
|
|
2265
|
+
observation.pendingOutboundReceipts === 0 &&
|
|
2266
|
+
observation.pendingInboundReceipts === 0 &&
|
|
2267
|
+
observation.rejectedInboundSettlements === 0);
|
|
2268
|
+
}
|
|
2269
|
+
endpointCallbacksQuiet() {
|
|
2270
|
+
return (!this.hasPendingCallbackWork() &&
|
|
2271
|
+
this.activeEndpointRefreshCallbackKey === undefined);
|
|
2272
|
+
}
|
|
1783
2273
|
async registerCodex(params) {
|
|
1784
2274
|
if (!params.alias.startsWith("codex-")) {
|
|
1785
2275
|
throw new BridgeError("INVALID_CODEX_PEER_ALIAS", "A registered Codex alias must start with codex-.");
|
|
1786
2276
|
}
|
|
1787
2277
|
if (this.codexSuccessionPoisoned.has(params.alias) ||
|
|
2278
|
+
this.codexOrphanCleanupPending.has(params.alias) ||
|
|
1788
2279
|
(params.succeedsAlias !== undefined &&
|
|
1789
|
-
this.codexSuccessionPoisoned.has(params.succeedsAlias)
|
|
2280
|
+
(this.codexSuccessionPoisoned.has(params.succeedsAlias) ||
|
|
2281
|
+
this.codexOrphanCleanupPending.has(params.succeedsAlias)))) {
|
|
1790
2282
|
throw new BridgeError("CODEX_SUCCESSION_RECOVERY_REQUIRED", "Codex registration is offline until the incomplete succession is manually recovered.");
|
|
1791
2283
|
}
|
|
1792
2284
|
if (params.succeedsAlias !== undefined) {
|
|
@@ -1822,16 +2314,41 @@ export class GatewayService {
|
|
|
1822
2314
|
alias: params.alias,
|
|
1823
2315
|
routeHandle: params.threadId,
|
|
1824
2316
|
});
|
|
2317
|
+
await this.drainSelectorCallbackSnapshotLocked();
|
|
1825
2318
|
if (registered.routeHandle !== params.threadId) {
|
|
1826
2319
|
throw new BridgeError("ROUTE_MISMATCH", "The connector registered a different task.");
|
|
1827
2320
|
}
|
|
2321
|
+
let deferredEndpointRefresh = false;
|
|
2322
|
+
if (registered.endpointRefresh !== undefined) {
|
|
2323
|
+
try {
|
|
2324
|
+
deferredEndpointRefresh = await this.onCodexEndpointRefresh(adapter, registered.endpointRefresh, {
|
|
2325
|
+
deferActivation: true,
|
|
2326
|
+
deferredRoute: {
|
|
2327
|
+
alias: params.alias,
|
|
2328
|
+
threadId: params.threadId,
|
|
2329
|
+
state: registered.state,
|
|
2330
|
+
},
|
|
2331
|
+
});
|
|
2332
|
+
}
|
|
2333
|
+
catch (error) {
|
|
2334
|
+
await adapter.releaseRoute?.(params.threadId).catch(() => undefined);
|
|
2335
|
+
throw error;
|
|
2336
|
+
}
|
|
2337
|
+
}
|
|
1828
2338
|
const binding = {
|
|
1829
2339
|
...adapter.identity,
|
|
1830
2340
|
routeHandle: params.threadId,
|
|
1831
2341
|
ownerLease: stableLease("codex", `${params.hostId}\0${params.threadId}`),
|
|
1832
2342
|
};
|
|
1833
2343
|
try {
|
|
1834
|
-
await this.
|
|
2344
|
+
const afterRefresh = await this.store.inspectPrivateRoute(params.alias);
|
|
2345
|
+
const alreadyInstalledExactly = afterRefresh !== undefined &&
|
|
2346
|
+
afterRefresh.enabled &&
|
|
2347
|
+
afterRefresh.compatibility === "compatible" &&
|
|
2348
|
+
bindingKey(afterRefresh.binding) === bindingKey(binding);
|
|
2349
|
+
if (!alreadyInstalledExactly || !deferredEndpointRefresh) {
|
|
2350
|
+
await this.registerOrRebind(params.alias, binding, "endpoint_reobserved", registered.state);
|
|
2351
|
+
}
|
|
1835
2352
|
}
|
|
1836
2353
|
catch (error) {
|
|
1837
2354
|
await adapter.releaseRoute?.(params.threadId).catch(() => undefined);
|
|
@@ -1857,10 +2374,32 @@ export class GatewayService {
|
|
|
1857
2374
|
: "busy");
|
|
1858
2375
|
listenerGeneration = this.requireCurrentNativeCodexGeneration(claudeAdapter, params.alias);
|
|
1859
2376
|
await this.completePendingCodexSuccessionRecoveryLocked(params);
|
|
2377
|
+
this.lockCodexRegistration(params, listenerGeneration);
|
|
2378
|
+
if (deferredEndpointRefresh &&
|
|
2379
|
+
registered.endpointRefresh !== undefined) {
|
|
2380
|
+
await this.onCodexEndpointRefresh(adapter, registered.endpointRefresh, {
|
|
2381
|
+
requiredRoute: {
|
|
2382
|
+
alias: params.alias,
|
|
2383
|
+
threadId: params.threadId,
|
|
2384
|
+
state: registered.state,
|
|
2385
|
+
},
|
|
2386
|
+
});
|
|
2387
|
+
}
|
|
1860
2388
|
await this.changed();
|
|
1861
2389
|
}
|
|
1862
2390
|
catch (error) {
|
|
1863
2391
|
if (routeBeforeRegistration !== undefined) {
|
|
2392
|
+
if (deferredEndpointRefresh &&
|
|
2393
|
+
registered.endpointRefresh?.outcome === "compatible") {
|
|
2394
|
+
try {
|
|
2395
|
+
await this.store.markConnectorOffline(registered.endpointRefresh.current, "CODEX_ENDPOINT_ACTIVATION_FAILED", [], { preserveQueued: true });
|
|
2396
|
+
this.routeStates.set(params.alias, "stale");
|
|
2397
|
+
await this.changed();
|
|
2398
|
+
}
|
|
2399
|
+
catch {
|
|
2400
|
+
this.dashboardHealthy = false;
|
|
2401
|
+
}
|
|
2402
|
+
}
|
|
1864
2403
|
// A persisted or live exact route, its queue, and any advertisement
|
|
1865
2404
|
// predate this invocation, so a later status/dashboard failure cannot
|
|
1866
2405
|
// safely delete them as "rollback". A post-restart reactivation also
|
|
@@ -1971,6 +2510,220 @@ export class GatewayService {
|
|
|
1971
2510
|
});
|
|
1972
2511
|
await this.changed();
|
|
1973
2512
|
}
|
|
2513
|
+
finalizeStaleCodexOrphanRemovalLocked(alias) {
|
|
2514
|
+
const aliases = new Set([alias]);
|
|
2515
|
+
this.forgetBinding(alias);
|
|
2516
|
+
this.purgeSuccessionTransientState(aliases);
|
|
2517
|
+
const lock = this.codexRegistrationLocks.get(alias);
|
|
2518
|
+
if (lock !== undefined) {
|
|
2519
|
+
this.codexRegistrationLocks.delete(alias);
|
|
2520
|
+
const state = this.codexSuccessionState;
|
|
2521
|
+
if ((state?.phase === "active_old" && state.active.alias === alias) ||
|
|
2522
|
+
(state?.phase === "active_new" && state.active.alias === alias)) {
|
|
2523
|
+
this.codexSuccessionState = undefined;
|
|
2524
|
+
}
|
|
2525
|
+
}
|
|
2526
|
+
this.codexOrphanCleanupPending.delete(alias);
|
|
2527
|
+
this.codexSuccessionDispatchFrozen.delete(alias);
|
|
2528
|
+
}
|
|
2529
|
+
async removeStaleCodexRegistration(alias) {
|
|
2530
|
+
const pendingCleanup = this.codexOrphanCleanupPending.get(alias);
|
|
2531
|
+
if (pendingCleanup !== undefined &&
|
|
2532
|
+
(await this.store.wasStaleCodexOrphanRemovalCommitted({
|
|
2533
|
+
alias,
|
|
2534
|
+
binding: pendingCleanup.binding,
|
|
2535
|
+
previousSequence: pendingCleanup.previousSequence,
|
|
2536
|
+
}))) {
|
|
2537
|
+
this.finalizeStaleCodexOrphanRemovalLocked(alias);
|
|
2538
|
+
await this.changed();
|
|
2539
|
+
return;
|
|
2540
|
+
}
|
|
2541
|
+
const authority = await this.store.inspectStaleCodexOrphanRemovalAuthority(alias);
|
|
2542
|
+
const binding = authority.binding;
|
|
2543
|
+
const aliases = new Set([alias]);
|
|
2544
|
+
const resumesExactCleanup = pendingCleanup !== undefined &&
|
|
2545
|
+
bindingKey(pendingCleanup.binding) === bindingKey(binding) &&
|
|
2546
|
+
pendingCleanup.previousSequence === authority.previousSequence;
|
|
2547
|
+
if (pendingCleanup !== undefined && !resumesExactCleanup) {
|
|
2548
|
+
throw new BridgeError("CODEX_ORPHAN_RECOVERY_MISMATCH", "The stale Codex registration changed after native cleanup.");
|
|
2549
|
+
}
|
|
2550
|
+
if (!this.codexSuccessionIsStable() ||
|
|
2551
|
+
this.pendingCodexSuccessionRecovery !== undefined ||
|
|
2552
|
+
this.codexSuccessionPoisoned.has(alias) ||
|
|
2553
|
+
(this.codexSuccessionDispatchFrozen.has(alias) && !resumesExactCleanup)) {
|
|
2554
|
+
throw new BridgeError("CODEX_ORPHAN_RECOVERY_BLOCKED", "Codex orphan recovery cannot run during registration succession.");
|
|
2555
|
+
}
|
|
2556
|
+
const lock = this.codexRegistrationLocks.get(alias);
|
|
2557
|
+
if (lock !== undefined &&
|
|
2558
|
+
(lock.threadId !== binding.routeHandle ||
|
|
2559
|
+
lock.hostId !== binding.hostId ||
|
|
2560
|
+
lock.generation === "unconfirmed" ||
|
|
2561
|
+
!isCodexRegistrationGeneration(lock.generation))) {
|
|
2562
|
+
throw new BridgeError("CODEX_ORPHAN_NATIVE_IDENTITY_UNPROVEN", "The native Codex listener identity cannot be proven for orphan recovery.");
|
|
2563
|
+
}
|
|
2564
|
+
const claudeAdapter = this.adapter("claude", binding.hostId);
|
|
2565
|
+
const codexAdapter = this.adapter("codex", binding.hostId);
|
|
2566
|
+
const currentNative = claudeAdapter.currentNativeCodexPeerGeneration;
|
|
2567
|
+
if (currentNative === undefined) {
|
|
2568
|
+
throw new BridgeError("CODEX_ORPHAN_NATIVE_IDENTITY_UNPROVEN", "The Claude provider cannot prove whether the native Codex listener exists.");
|
|
2569
|
+
}
|
|
2570
|
+
this.codexSuccessionDispatchFrozen.add(alias);
|
|
2571
|
+
this.scheduledDispatchTargets.delete(alias);
|
|
2572
|
+
let nativeGeneration;
|
|
2573
|
+
let nativeQuiesceAttempted = false;
|
|
2574
|
+
let irreversibleCleanupStarted = resumesExactCleanup;
|
|
2575
|
+
try {
|
|
2576
|
+
try {
|
|
2577
|
+
const observed = currentNative.call(claudeAdapter, alias);
|
|
2578
|
+
if (!isCodexRegistrationGeneration(observed)) {
|
|
2579
|
+
throw new BridgeError("CODEX_ORPHAN_NATIVE_IDENTITY_UNPROVEN", "The native Codex listener generation is malformed.");
|
|
2580
|
+
}
|
|
2581
|
+
if (lock !== undefined && lock.generation !== observed) {
|
|
2582
|
+
throw new BridgeError("CODEX_ORPHAN_NATIVE_IDENTITY_UNPROVEN", "The native Codex listener generation changed before orphan recovery.");
|
|
2583
|
+
}
|
|
2584
|
+
nativeGeneration = observed;
|
|
2585
|
+
}
|
|
2586
|
+
catch (error) {
|
|
2587
|
+
const exactAbsence = error instanceof BridgeError &&
|
|
2588
|
+
error.code === "CODEX_PEER_GENERATION_MISMATCH";
|
|
2589
|
+
if (!exactAbsence || (lock !== undefined && !resumesExactCleanup)) {
|
|
2590
|
+
if (error instanceof BridgeError &&
|
|
2591
|
+
error.code === "CODEX_ORPHAN_NATIVE_IDENTITY_UNPROVEN") {
|
|
2592
|
+
throw error;
|
|
2593
|
+
}
|
|
2594
|
+
throw new BridgeError("CODEX_ORPHAN_NATIVE_IDENTITY_UNPROVEN", "The native Codex listener identity cannot be proven for orphan recovery.");
|
|
2595
|
+
}
|
|
2596
|
+
}
|
|
2597
|
+
let observeNative;
|
|
2598
|
+
if (nativeGeneration !== undefined) {
|
|
2599
|
+
const quiesce = this.requireSuccessionMethod(claudeAdapter, "quiesceNativeCodexPeerGeneration");
|
|
2600
|
+
// Require the inverse operation before acquiring the quiescence fence
|
|
2601
|
+
// so every pre-unadvertise failure has one exact cleanup path.
|
|
2602
|
+
this.requireSuccessionMethod(claudeAdapter, "resumeNativeCodexPeerGeneration");
|
|
2603
|
+
observeNative = this.requireSuccessionMethod(claudeAdapter, "observeNativeCodexSuccessionBarrier");
|
|
2604
|
+
nativeQuiesceAttempted = true;
|
|
2605
|
+
await quiesce.call(claudeAdapter, nativeGeneration);
|
|
2606
|
+
}
|
|
2607
|
+
// Join every native frame admitted before the exact generation fence,
|
|
2608
|
+
// and every detached terminal receipt write those frames create.
|
|
2609
|
+
await this.drainPreDeadlineDeliveryCallbacksLocked();
|
|
2610
|
+
await this.drainCallbackQueueLocked();
|
|
2611
|
+
await this.drainDetachedReceiptWritesLocked();
|
|
2612
|
+
await this.requireUnchangedStaleCodexOrphan(alias, binding);
|
|
2613
|
+
const observeCodex = codexAdapter.observeRouteSuccessionBarrier;
|
|
2614
|
+
if (observeCodex === undefined) {
|
|
2615
|
+
throw new BridgeError("CODEX_ORPHAN_RECOVERY_BUSY", "The Codex provider cannot prove an empty orphan route boundary.");
|
|
2616
|
+
}
|
|
2617
|
+
const beforeRelease = observeCodex.call(codexAdapter, binding.routeHandle);
|
|
2618
|
+
if (beforeRelease.routePresent ||
|
|
2619
|
+
!this.codexRouteSuccessionQuiet(beforeRelease) ||
|
|
2620
|
+
beforeRelease.routeCreationInFlight ||
|
|
2621
|
+
beforeRelease.routeReleaseInFlight) {
|
|
2622
|
+
throw new BridgeError("CODEX_ORPHAN_RECOVERY_BUSY", "The stale Codex registration still has a live provider or endpoint-refresh claim.");
|
|
2623
|
+
}
|
|
2624
|
+
if (nativeGeneration !== undefined) {
|
|
2625
|
+
const nativeBarrier = await observeNative.call(claudeAdapter, nativeGeneration);
|
|
2626
|
+
if (!this.nativeCodexOrphanBarrierClean(nativeBarrier, nativeGeneration)) {
|
|
2627
|
+
throw new BridgeError("CODEX_ORPHAN_RECOVERY_BUSY", "The native Codex listener still owns ingress or receipt work.");
|
|
2628
|
+
}
|
|
2629
|
+
}
|
|
2630
|
+
if (!this.successionServiceStateClean(aliases) ||
|
|
2631
|
+
!this.endpointCallbacksQuiet() ||
|
|
2632
|
+
this.detachedReceiptWrites.size !== 0) {
|
|
2633
|
+
throw new BridgeError("CODEX_ORPHAN_RECOVERY_BUSY", "The stale Codex registration still has a live service claim.");
|
|
2634
|
+
}
|
|
2635
|
+
// The awaited native barrier can admit a provider-side route recovery
|
|
2636
|
+
// claim. Re-sample the Codex boundary after that last await and directly
|
|
2637
|
+
// before the generation-unqualified release call.
|
|
2638
|
+
const releaseBoundary = observeCodex.call(codexAdapter, binding.routeHandle);
|
|
2639
|
+
if (releaseBoundary.routePresent ||
|
|
2640
|
+
!this.codexRouteSuccessionQuiet(releaseBoundary) ||
|
|
2641
|
+
releaseBoundary.routeCreationInFlight ||
|
|
2642
|
+
releaseBoundary.routeReleaseInFlight) {
|
|
2643
|
+
throw new BridgeError("CODEX_ORPHAN_RECOVERY_BUSY", "The stale Codex registration acquired a provider recovery claim before release.");
|
|
2644
|
+
}
|
|
2645
|
+
// The pre-barrier proves this release targets an absent stale handle and
|
|
2646
|
+
// cannot join an already-observed creation. Revalidate immediately after
|
|
2647
|
+
// the provider join before touching the native advertisement.
|
|
2648
|
+
await codexAdapter.releaseRoute?.(binding.routeHandle);
|
|
2649
|
+
await this.drainCallbackQueueLocked();
|
|
2650
|
+
await this.drainDetachedReceiptWritesLocked();
|
|
2651
|
+
await this.requireUnchangedStaleCodexOrphan(alias, binding);
|
|
2652
|
+
const afterRelease = observeCodex.call(codexAdapter, binding.routeHandle);
|
|
2653
|
+
if (afterRelease.routePresent ||
|
|
2654
|
+
!this.codexRouteSuccessionQuiet(afterRelease) ||
|
|
2655
|
+
afterRelease.routeCreationInFlight ||
|
|
2656
|
+
afterRelease.routeReleaseInFlight ||
|
|
2657
|
+
!this.successionServiceStateClean(aliases) ||
|
|
2658
|
+
!this.endpointCallbacksQuiet() ||
|
|
2659
|
+
this.detachedReceiptWrites.size !== 0) {
|
|
2660
|
+
throw new BridgeError("CODEX_ORPHAN_RECOVERY_BUSY", "The stale Codex registration changed at the provider cleanup boundary.");
|
|
2661
|
+
}
|
|
2662
|
+
if (nativeGeneration !== undefined) {
|
|
2663
|
+
const nativeBarrier = await observeNative.call(claudeAdapter, nativeGeneration);
|
|
2664
|
+
if (!this.nativeCodexOrphanBarrierClean(nativeBarrier, nativeGeneration)) {
|
|
2665
|
+
throw new BridgeError("CODEX_ORPHAN_RECOVERY_BUSY", "The native Codex listener changed during provider cleanup.");
|
|
2666
|
+
}
|
|
2667
|
+
if (currentNative.call(claudeAdapter, alias) !== nativeGeneration) {
|
|
2668
|
+
throw new BridgeError("CODEX_ORPHAN_NATIVE_IDENTITY_UNPROVEN", "The native Codex listener generation changed before removal.");
|
|
2669
|
+
}
|
|
2670
|
+
}
|
|
2671
|
+
await this.requireUnchangedStaleCodexOrphan(alias, binding);
|
|
2672
|
+
if (nativeGeneration !== undefined) {
|
|
2673
|
+
const unadvertise = claudeAdapter.unadvertiseNativeCodexPeer;
|
|
2674
|
+
if (unadvertise === undefined) {
|
|
2675
|
+
throw new BridgeError("CODEX_ORPHAN_NATIVE_IDENTITY_UNPROVEN", "The native Codex listener cannot be removed exactly.");
|
|
2676
|
+
}
|
|
2677
|
+
irreversibleCleanupStarted = true;
|
|
2678
|
+
this.codexOrphanCleanupPending.set(alias, {
|
|
2679
|
+
binding: { ...binding },
|
|
2680
|
+
previousSequence: authority.previousSequence,
|
|
2681
|
+
});
|
|
2682
|
+
// The supervisor may remove the exact helper and then report an
|
|
2683
|
+
// ambiguous close failure. Retain retry authority before invoking the
|
|
2684
|
+
// irreversible operation; a retry proves whether the listener remains.
|
|
2685
|
+
await unadvertise.call(claudeAdapter, alias);
|
|
2686
|
+
}
|
|
2687
|
+
// Exact native absence is also an irreversible cleanup boundary: if the
|
|
2688
|
+
// durable mutation fails, only this same binding may resume removal.
|
|
2689
|
+
this.codexOrphanCleanupPending.set(alias, {
|
|
2690
|
+
binding: { ...binding },
|
|
2691
|
+
previousSequence: authority.previousSequence,
|
|
2692
|
+
});
|
|
2693
|
+
irreversibleCleanupStarted = true;
|
|
2694
|
+
const removed = await this.store.removeStaleCodexOrphan({ alias });
|
|
2695
|
+
if (bindingKey(removed.binding) !== bindingKey(binding)) {
|
|
2696
|
+
throw new BridgeError("CODEX_ORPHAN_RECOVERY_MISMATCH", "The stale Codex registration changed during recovery.");
|
|
2697
|
+
}
|
|
2698
|
+
this.finalizeStaleCodexOrphanRemovalLocked(alias);
|
|
2699
|
+
await this.changed();
|
|
2700
|
+
}
|
|
2701
|
+
catch (error) {
|
|
2702
|
+
if (!irreversibleCleanupStarted) {
|
|
2703
|
+
let resumeFailed = false;
|
|
2704
|
+
if (nativeGeneration !== undefined && nativeQuiesceAttempted) {
|
|
2705
|
+
try {
|
|
2706
|
+
const resume = this.requireSuccessionMethod(claudeAdapter, "resumeNativeCodexPeerGeneration");
|
|
2707
|
+
await resume.call(claudeAdapter, nativeGeneration);
|
|
2708
|
+
}
|
|
2709
|
+
catch {
|
|
2710
|
+
resumeFailed = true;
|
|
2711
|
+
this.dashboardHealthy = false;
|
|
2712
|
+
}
|
|
2713
|
+
}
|
|
2714
|
+
if (!resumeFailed) {
|
|
2715
|
+
this.codexSuccessionDispatchFrozen.delete(alias);
|
|
2716
|
+
if (this.routeStates.get(alias) === "idle") {
|
|
2717
|
+
this.scheduleDispatch(alias);
|
|
2718
|
+
}
|
|
2719
|
+
}
|
|
2720
|
+
else {
|
|
2721
|
+
throw new BridgeError("CODEX_ORPHAN_RECOVERY_CLEANUP_FAILED", "The native Codex listener could not be resumed after failed orphan recovery.");
|
|
2722
|
+
}
|
|
2723
|
+
}
|
|
2724
|
+
throw error;
|
|
2725
|
+
}
|
|
2726
|
+
}
|
|
1974
2727
|
async discoveryPublicFingerprint() {
|
|
1975
2728
|
const base = await this.store.publicSnapshot();
|
|
1976
2729
|
const snapshot = projectGatewayPublicSnapshot({
|
|
@@ -2568,11 +3321,15 @@ export class GatewayService {
|
|
|
2568
3321
|
const previous = this.routeBindings.get(alias);
|
|
2569
3322
|
if (previous !== undefined &&
|
|
2570
3323
|
bindingKey(previous) !== bindingKey(binding)) {
|
|
3324
|
+
this.bindingAliases.delete(bindingKey(previous));
|
|
2571
3325
|
this.pendingClaudeReplies.delete(bindingKey(previous));
|
|
2572
3326
|
}
|
|
2573
3327
|
this.routeBindings.set(alias, binding);
|
|
2574
3328
|
this.routeStates.set(alias, state);
|
|
2575
3329
|
this.bindingAliases.set(bindingKey(binding), alias);
|
|
3330
|
+
if (binding.provider === "claude" && state === "idle") {
|
|
3331
|
+
this.claudePrewriteIdleWaitTargets.delete(alias);
|
|
3332
|
+
}
|
|
2576
3333
|
}
|
|
2577
3334
|
forgetBinding(alias) {
|
|
2578
3335
|
const heldTimer = this.heldRedispatchTimers.get(alias);
|
|
@@ -2587,6 +3344,7 @@ export class GatewayService {
|
|
|
2587
3344
|
}
|
|
2588
3345
|
this.routeBindings.delete(alias);
|
|
2589
3346
|
this.routeStates.delete(alias);
|
|
3347
|
+
this.claudePrewriteIdleWaitTargets.delete(alias);
|
|
2590
3348
|
for (const [messageId, continuation] of this.providerTurnContinuations) {
|
|
2591
3349
|
if (continuation.targetAlias !== alias)
|
|
2592
3350
|
continue;
|
|
@@ -3365,7 +4123,8 @@ export class GatewayService {
|
|
|
3365
4123
|
}
|
|
3366
4124
|
if (!isTerminalDeliveryMachine(tracker.machine) &&
|
|
3367
4125
|
tracker.machine.dispatchRetryAt !== null &&
|
|
3368
|
-
tracker.machine.dispatchRetryAt <= now
|
|
4126
|
+
tracker.machine.dispatchRetryAt <= now &&
|
|
4127
|
+
!this.claudePrewriteIdleWaitTargets.has(tracker.targetAlias)) {
|
|
3369
4128
|
if (!this.dispatchRunnerTargets.has(tracker.targetAlias)) {
|
|
3370
4129
|
this.scheduleDispatch(tracker.targetAlias);
|
|
3371
4130
|
}
|
|
@@ -3454,6 +4213,7 @@ export class GatewayService {
|
|
|
3454
4213
|
if (wakeups.deadlineAt !== undefined)
|
|
3455
4214
|
consider(wakeups.deadlineAt);
|
|
3456
4215
|
if (wakeups.dispatchRetryAt !== undefined &&
|
|
4216
|
+
!this.claudePrewriteIdleWaitTargets.has(tracker.targetAlias) &&
|
|
3457
4217
|
!this.scheduledDispatchTargets.has(tracker.targetAlias) &&
|
|
3458
4218
|
!this.dispatchRunnerTargets.has(tracker.targetAlias)) {
|
|
3459
4219
|
consider(wakeups.dispatchRetryAt);
|
|
@@ -3923,11 +4683,51 @@ export class GatewayService {
|
|
|
3923
4683
|
await this.publishNativeHeldLocked(afterCallbacks);
|
|
3924
4684
|
}
|
|
3925
4685
|
}
|
|
4686
|
+
restoreQueuedMessageContextLocked(item, binding) {
|
|
4687
|
+
this.pruneTransient();
|
|
4688
|
+
const generatedConversationId = createGatewayConversationId();
|
|
4689
|
+
const conversationId = item.conversationIdSuffix === undefined
|
|
4690
|
+
? generatedConversationId
|
|
4691
|
+
: `${generatedConversationId.slice(0, -8)}${item.conversationIdSuffix}`;
|
|
4692
|
+
const conversation = {
|
|
4693
|
+
id: conversationId,
|
|
4694
|
+
sourceAlias: item.sourceAlias,
|
|
4695
|
+
targetAlias: item.targetAlias,
|
|
4696
|
+
expectsReply: false,
|
|
4697
|
+
nextSequence: 1,
|
|
4698
|
+
lastHopCount: item.hopCount,
|
|
4699
|
+
lastActivityAt: this.now().toISOString(),
|
|
4700
|
+
...(item.pair === true ? { pair: true } : {}),
|
|
4701
|
+
};
|
|
4702
|
+
const context = {
|
|
4703
|
+
conversationId,
|
|
4704
|
+
isReply: false,
|
|
4705
|
+
expectsReply: false,
|
|
4706
|
+
hopCount: item.hopCount,
|
|
4707
|
+
sequence: 0,
|
|
4708
|
+
targetBindingKey: bindingKey(binding),
|
|
4709
|
+
authorization: "selected_route",
|
|
4710
|
+
targetAlias: item.targetAlias,
|
|
4711
|
+
deadlineAt: item.deadlineAt,
|
|
4712
|
+
recovered: true,
|
|
4713
|
+
};
|
|
4714
|
+
this.conversations.set(conversationId, conversation);
|
|
4715
|
+
this.messageContexts.set(item.messageId, context);
|
|
4716
|
+
this.armDeliveryTracker({
|
|
4717
|
+
messageId: item.messageId,
|
|
4718
|
+
conversationId,
|
|
4719
|
+
targetAlias: item.targetAlias,
|
|
4720
|
+
enqueuedAt: Date.parse(item.enqueuedAt),
|
|
4721
|
+
deadlineAt: item.deadlineAt,
|
|
4722
|
+
});
|
|
4723
|
+
return context;
|
|
4724
|
+
}
|
|
3926
4725
|
async dispatchOne(targetAlias) {
|
|
3927
4726
|
if (!this.running ||
|
|
3928
4727
|
this.closing ||
|
|
3929
4728
|
this.codexSuccessionDispatchFrozen.has(targetAlias) ||
|
|
3930
|
-
this.codexSuccessionPoisoned.has(targetAlias)
|
|
4729
|
+
this.codexSuccessionPoisoned.has(targetAlias) ||
|
|
4730
|
+
this.claudePrewriteIdleWaitTargets.has(targetAlias)) {
|
|
3931
4731
|
return;
|
|
3932
4732
|
}
|
|
3933
4733
|
if (await this.processLifecycleLocked()) {
|
|
@@ -3949,8 +4749,19 @@ export class GatewayService {
|
|
|
3949
4749
|
}
|
|
3950
4750
|
if (item === undefined)
|
|
3951
4751
|
return;
|
|
3952
|
-
const context = this.messageContexts.get(item.messageId);
|
|
3953
4752
|
const selectedBinding = this.routeBindings.get(targetAlias);
|
|
4753
|
+
let context = this.messageContexts.get(item.messageId);
|
|
4754
|
+
if (context === undefined && selectedBinding === undefined) {
|
|
4755
|
+
const result = await this.store.requeueInFlightMessage(item.messageId, item.body);
|
|
4756
|
+
if (result.status === "settled") {
|
|
4757
|
+
await this.applyTerminalSettlementLocked(result.settlement);
|
|
4758
|
+
}
|
|
4759
|
+
await this.changed();
|
|
4760
|
+
return;
|
|
4761
|
+
}
|
|
4762
|
+
if (context === undefined && selectedBinding !== undefined) {
|
|
4763
|
+
context = this.restoreQueuedMessageContextLocked(item, selectedBinding);
|
|
4764
|
+
}
|
|
3954
4765
|
const binding = context?.nativeReplyBinding ?? selectedBinding;
|
|
3955
4766
|
if (context === undefined) {
|
|
3956
4767
|
const result = await this.store.settleMessage({
|
|
@@ -3985,6 +4796,14 @@ export class GatewayService {
|
|
|
3985
4796
|
inspection?.state === "stale" ||
|
|
3986
4797
|
inspection?.safeErrorCode === "CODEX_ROUTE_STALE");
|
|
3987
4798
|
if (codexRouteStale) {
|
|
4799
|
+
if (context.recovered === true) {
|
|
4800
|
+
const result = await this.store.requeueInFlightMessage(item.messageId, item.body);
|
|
4801
|
+
if (result.status === "settled") {
|
|
4802
|
+
await this.applyTerminalSettlementLocked(result.settlement);
|
|
4803
|
+
}
|
|
4804
|
+
await this.changed();
|
|
4805
|
+
return;
|
|
4806
|
+
}
|
|
3988
4807
|
await this.advanceDeliveryLocked(item.messageId, {
|
|
3989
4808
|
type: "route_terminated",
|
|
3990
4809
|
at: this.now().getTime(),
|
|
@@ -4088,6 +4907,8 @@ export class GatewayService {
|
|
|
4088
4907
|
try {
|
|
4089
4908
|
result = await this.awaitDispatchWithNativeHeldLocked(tracker, this.adapter(binding.provider, binding.hostId).dispatch({
|
|
4090
4909
|
sourceAlias: item.sourceAlias,
|
|
4910
|
+
targetAlias: item.targetAlias,
|
|
4911
|
+
conversationId: context.conversationId,
|
|
4091
4912
|
binding,
|
|
4092
4913
|
authorization: context.authorization,
|
|
4093
4914
|
messageId: item.messageId,
|
|
@@ -4126,12 +4947,14 @@ export class GatewayService {
|
|
|
4126
4947
|
if (result.state === "deferred") {
|
|
4127
4948
|
const currentTargetAlias = this.bindingAliases.get(bindingKey(binding)) ?? targetAlias;
|
|
4128
4949
|
const now = this.now().getTime();
|
|
4950
|
+
const deferredSafeErrorCode = safeCode(result.safeErrorCode, "PROVIDER_DISPATCH_DEFERRED");
|
|
4129
4951
|
await this.advanceDeliveryLocked(item.messageId, {
|
|
4130
4952
|
type: "dispatch_clean_prewrite_failed",
|
|
4131
4953
|
at: now,
|
|
4132
4954
|
retryAt: now + 500,
|
|
4133
|
-
safeErrorCode:
|
|
4955
|
+
safeErrorCode: deferredSafeErrorCode,
|
|
4134
4956
|
});
|
|
4957
|
+
tracker.deliverySafeErrorCode = deferredSafeErrorCode;
|
|
4135
4958
|
const postDeferred = this.deliveryTrackers.get(item.messageId)?.machine;
|
|
4136
4959
|
if (postDeferred === undefined ||
|
|
4137
4960
|
isTerminalDeliveryMachine(postDeferred)) {
|
|
@@ -4150,13 +4973,18 @@ export class GatewayService {
|
|
|
4150
4973
|
await this.changed();
|
|
4151
4974
|
return;
|
|
4152
4975
|
}
|
|
4153
|
-
const requeued = await this.store.requeueInFlightMessage(item.messageId, item.body);
|
|
4976
|
+
const requeued = await this.store.requeueInFlightMessage(item.messageId, item.body, deferredSafeErrorCode);
|
|
4154
4977
|
if (this.activeDispatchByTarget.get(currentTargetAlias) === item.messageId) {
|
|
4155
4978
|
this.activeDispatchByTarget.delete(currentTargetAlias);
|
|
4156
4979
|
}
|
|
4157
4980
|
if (requeued.status === "requeued") {
|
|
4158
4981
|
if (context.authorization === "selected_route") {
|
|
4159
|
-
|
|
4982
|
+
const waitForClaudeIdle = binding.provider === "claude" &&
|
|
4983
|
+
CLAUDE_CLEAN_PREWRITE_ROUTE_CODES.has(deferredSafeErrorCode);
|
|
4984
|
+
if (waitForClaudeIdle) {
|
|
4985
|
+
this.claudePrewriteIdleWaitTargets.add(currentTargetAlias);
|
|
4986
|
+
}
|
|
4987
|
+
else if (!steeringWhileBusy) {
|
|
4160
4988
|
this.routeStates.set(currentTargetAlias, "idle");
|
|
4161
4989
|
await this.store.observeRoute({
|
|
4162
4990
|
binding,
|
|
@@ -4196,7 +5024,8 @@ export class GatewayService {
|
|
|
4196
5024
|
scheduleDispatch(targetAlias) {
|
|
4197
5025
|
if (this.closing ||
|
|
4198
5026
|
this.codexSuccessionDispatchFrozen.has(targetAlias) ||
|
|
4199
|
-
this.codexSuccessionPoisoned.has(targetAlias)
|
|
5027
|
+
this.codexSuccessionPoisoned.has(targetAlias) ||
|
|
5028
|
+
this.claudePrewriteIdleWaitTargets.has(targetAlias)) {
|
|
4200
5029
|
return;
|
|
4201
5030
|
}
|
|
4202
5031
|
if (this.scheduledDispatchTargets.has(targetAlias))
|
|
@@ -4275,7 +5104,9 @@ export class GatewayService {
|
|
|
4275
5104
|
safeErrorCode: safeCode(event.safeErrorCode, "DISPATCH_OUTCOME_AMBIGUOUS"),
|
|
4276
5105
|
}
|
|
4277
5106
|
: event.state === "transport_written"
|
|
4278
|
-
?
|
|
5107
|
+
? target.provider === "claude"
|
|
5108
|
+
? { type: "provider_released", observedAt: receivedAt }
|
|
5109
|
+
: { type: "transport_written", observedAt: receivedAt }
|
|
4279
5110
|
: event.state === "held"
|
|
4280
5111
|
? { type: "provider_held", observedAt: receivedAt }
|
|
4281
5112
|
: event.state === "released" || event.state === "completed"
|
|
@@ -4586,6 +5417,401 @@ export class GatewayService {
|
|
|
4586
5417
|
}
|
|
4587
5418
|
}
|
|
4588
5419
|
}
|
|
5420
|
+
async onCodexEndpointRefresh(adapter, event, options = {}) {
|
|
5421
|
+
const transitionKey = endpointRefreshCallbackKey(event);
|
|
5422
|
+
if (this.endpointRefreshCallbackPoisoned &&
|
|
5423
|
+
this.activeEndpointRefreshCallbackKey !== transitionKey) {
|
|
5424
|
+
throw new BridgeError("CODEX_ENDPOINT_REFRESH_CONFLICT", "Conflicting Codex endpoint transitions require a clean broker restart.");
|
|
5425
|
+
}
|
|
5426
|
+
const ownsActiveTransition = this.activeEndpointRefreshCallbackKey === undefined;
|
|
5427
|
+
if (!ownsActiveTransition &&
|
|
5428
|
+
this.activeEndpointRefreshCallbackKey !== transitionKey) {
|
|
5429
|
+
this.poisonEndpointRefreshCallbacks({
|
|
5430
|
+
type: "endpoint_refresh",
|
|
5431
|
+
adapter,
|
|
5432
|
+
value: event,
|
|
5433
|
+
});
|
|
5434
|
+
throw new BridgeError("CODEX_ENDPOINT_REFRESH_CONFLICT", "Conflicting Codex endpoint transitions require a clean broker restart.");
|
|
5435
|
+
}
|
|
5436
|
+
if (ownsActiveTransition) {
|
|
5437
|
+
this.activeEndpointRefreshCallback = {
|
|
5438
|
+
type: "endpoint_refresh",
|
|
5439
|
+
adapter,
|
|
5440
|
+
value: event,
|
|
5441
|
+
};
|
|
5442
|
+
this.activeEndpointRefreshCallbackKey = transitionKey;
|
|
5443
|
+
}
|
|
5444
|
+
let succeeded = false;
|
|
5445
|
+
let activationDeferred = false;
|
|
5446
|
+
try {
|
|
5447
|
+
activationDeferred = await this.applyCodexEndpointRefresh(adapter, event, transitionKey, options);
|
|
5448
|
+
succeeded = true;
|
|
5449
|
+
return activationDeferred;
|
|
5450
|
+
}
|
|
5451
|
+
finally {
|
|
5452
|
+
if (ownsActiveTransition &&
|
|
5453
|
+
this.activeEndpointRefreshCallbackKey === transitionKey) {
|
|
5454
|
+
this.activeEndpointRefreshCallback = undefined;
|
|
5455
|
+
this.activeEndpointRefreshCallbackKey = undefined;
|
|
5456
|
+
const requestedRetry = this.endpointRefreshRetryRequested;
|
|
5457
|
+
this.endpointRefreshRetryRequested = undefined;
|
|
5458
|
+
if ((!succeeded || activationDeferred) &&
|
|
5459
|
+
!this.endpointRefreshCallbackPoisoned &&
|
|
5460
|
+
requestedRetry !== undefined) {
|
|
5461
|
+
this.endpointRefreshCallback = requestedRetry;
|
|
5462
|
+
this.enqueueCallbackWorkerOnly();
|
|
5463
|
+
}
|
|
5464
|
+
}
|
|
5465
|
+
}
|
|
5466
|
+
}
|
|
5467
|
+
async applyCodexEndpointRefresh(adapter, event, transitionKey, options) {
|
|
5468
|
+
if (this.closing ||
|
|
5469
|
+
(event.outcome !== "compatible" && event.outcome !== "incompatible") ||
|
|
5470
|
+
!isCompatibilityAttestation(event.attestation) ||
|
|
5471
|
+
event.attestation.surface !== "codex" ||
|
|
5472
|
+
event.previous.provider !== "codex" ||
|
|
5473
|
+
!PRIVATE_HANDLE.test(event.previous.endpointGeneration) ||
|
|
5474
|
+
adapter.identity.provider !== "codex" ||
|
|
5475
|
+
adapter.identity.hostId !== event.previous.hostId ||
|
|
5476
|
+
!this.config.allowedHosts.includes(event.previous.hostId)) {
|
|
5477
|
+
throw new BridgeError("CODEX_ENDPOINT_REFRESH_MISMATCH", "The Codex endpoint transition failed closed runtime validation.");
|
|
5478
|
+
}
|
|
5479
|
+
if (event.outcome === "compatible") {
|
|
5480
|
+
const routeHandles = Array.isArray(event.routes)
|
|
5481
|
+
? event.routes.map((route) => route.routeHandle)
|
|
5482
|
+
: [];
|
|
5483
|
+
if (event.current.provider !== "codex" ||
|
|
5484
|
+
event.current.hostId !== event.previous.hostId ||
|
|
5485
|
+
!PRIVATE_HANDLE.test(event.current.endpointGeneration) ||
|
|
5486
|
+
event.current.endpointGeneration ===
|
|
5487
|
+
event.previous.endpointGeneration ||
|
|
5488
|
+
adapter.identity.endpointGeneration !==
|
|
5489
|
+
event.current.endpointGeneration ||
|
|
5490
|
+
event.attestation.version !== adapter.protocolVersion ||
|
|
5491
|
+
event.attestation.tier === "incompatible" ||
|
|
5492
|
+
!Array.isArray(event.routes) ||
|
|
5493
|
+
event.routes.length > this.config.limits.maxRoutes ||
|
|
5494
|
+
new Set(routeHandles).size !== routeHandles.length ||
|
|
5495
|
+
event.routes.some((route) => !PRIVATE_HANDLE.test(route.routeHandle) ||
|
|
5496
|
+
!["idle", "busy", "awaiting_approval"].includes(route.state))) {
|
|
5497
|
+
throw new BridgeError("CODEX_ENDPOINT_REFRESH_MISMATCH", "The refreshed Codex provider identity changed before activation.");
|
|
5498
|
+
}
|
|
5499
|
+
if (this.activatedEndpointRefreshes.get(event.current.hostId) ===
|
|
5500
|
+
transitionKey) {
|
|
5501
|
+
return false;
|
|
5502
|
+
}
|
|
5503
|
+
}
|
|
5504
|
+
else {
|
|
5505
|
+
if (event.candidate.provider !== "codex" ||
|
|
5506
|
+
event.candidate.hostId !== event.previous.hostId ||
|
|
5507
|
+
!PRIVATE_HANDLE.test(event.candidate.endpointGeneration) ||
|
|
5508
|
+
event.candidate.endpointGeneration ===
|
|
5509
|
+
event.previous.endpointGeneration ||
|
|
5510
|
+
adapter.identity.endpointGeneration !==
|
|
5511
|
+
event.previous.endpointGeneration ||
|
|
5512
|
+
event.attestation.tier !== "incompatible") {
|
|
5513
|
+
throw new BridgeError("CODEX_ENDPOINT_REFRESH_MISMATCH", "The incompatible Codex candidate no longer matches the active endpoint.");
|
|
5514
|
+
}
|
|
5515
|
+
if (this.handledIncompatibleEndpointRefreshes.get(event.previous.hostId) === transitionKey) {
|
|
5516
|
+
return false;
|
|
5517
|
+
}
|
|
5518
|
+
}
|
|
5519
|
+
let inventory = await this.store.inspectPrivateCodexRoutes();
|
|
5520
|
+
const previousRoutes = inventory.filter((route) => route.binding.hostId === event.previous.hostId &&
|
|
5521
|
+
route.binding.endpointGeneration === event.previous.endpointGeneration);
|
|
5522
|
+
const aliases = previousRoutes.map((route) => route.alias);
|
|
5523
|
+
await this.drainPreDeadlineDeliveryCallbacksLocked();
|
|
5524
|
+
const inFlightSettlements = aliases.length === 0
|
|
5525
|
+
? []
|
|
5526
|
+
: await this.planRoutesInFlightSettlementsLocked(aliases, {
|
|
5527
|
+
unwrittenOutcome: "failed",
|
|
5528
|
+
safeErrorCode: "ENDPOINT_GENERATION_CHANGED",
|
|
5529
|
+
});
|
|
5530
|
+
let settlements = [];
|
|
5531
|
+
try {
|
|
5532
|
+
settlements = await this.store.markConnectorOffline(event.previous, "ENDPOINT_GENERATION_CHANGED", inFlightSettlements, { preserveQueued: true });
|
|
5533
|
+
}
|
|
5534
|
+
catch (error) {
|
|
5535
|
+
const alreadyInvalidatedCompatibleTransition = event.outcome === "compatible" &&
|
|
5536
|
+
error instanceof BridgeError &&
|
|
5537
|
+
error.code === "CONNECTOR_NOT_FOUND" &&
|
|
5538
|
+
previousRoutes.every((route) => route.state === "stale" && route.compatibility === "expired");
|
|
5539
|
+
if (!alreadyInvalidatedCompatibleTransition)
|
|
5540
|
+
throw error;
|
|
5541
|
+
}
|
|
5542
|
+
for (const settlement of settlements) {
|
|
5543
|
+
await this.applyTerminalSettlementLocked(settlement);
|
|
5544
|
+
}
|
|
5545
|
+
for (const route of previousRoutes) {
|
|
5546
|
+
this.routeStates.set(route.alias, "stale");
|
|
5547
|
+
}
|
|
5548
|
+
await this.store.recordCompatibilityAttestation(event.attestation);
|
|
5549
|
+
if (event.outcome === "incompatible") {
|
|
5550
|
+
await this.requireEndpointRefreshUnconflicted();
|
|
5551
|
+
this.activeCodexIncompatibility = structuredClone(event.attestation);
|
|
5552
|
+
this.addRuntimeAlert(event.attestation.safeErrorCode ?? "CODEX_ENDPOINT_INCOMPATIBLE", "error", { provider: "codex", host: event.previous.hostId });
|
|
5553
|
+
this.handledIncompatibleEndpointRefreshes.set(event.previous.hostId, transitionKey);
|
|
5554
|
+
await this.changed();
|
|
5555
|
+
return false;
|
|
5556
|
+
}
|
|
5557
|
+
await this.store.observeConnector({
|
|
5558
|
+
identity: event.current,
|
|
5559
|
+
health: "healthy",
|
|
5560
|
+
compatibility: "compatible",
|
|
5561
|
+
protocol: adapter.protocol,
|
|
5562
|
+
protocolVersion: adapter.protocolVersion,
|
|
5563
|
+
});
|
|
5564
|
+
const refreshedByHandle = new Map(event.routes.map((route) => [route.routeHandle, route.state]));
|
|
5565
|
+
const pendingReanchors = previousRoutes.flatMap((route) => {
|
|
5566
|
+
const state = refreshedByHandle.get(route.binding.routeHandle);
|
|
5567
|
+
return state === undefined
|
|
5568
|
+
? []
|
|
5569
|
+
: [
|
|
5570
|
+
{
|
|
5571
|
+
alias: route.alias,
|
|
5572
|
+
threadId: route.binding.routeHandle,
|
|
5573
|
+
ownerLease: route.binding.ownerLease,
|
|
5574
|
+
state,
|
|
5575
|
+
},
|
|
5576
|
+
];
|
|
5577
|
+
});
|
|
5578
|
+
if (pendingReanchors.length > 0) {
|
|
5579
|
+
try {
|
|
5580
|
+
await this.store.reanchorCodexRoutes({
|
|
5581
|
+
oldEndpoint: event.previous,
|
|
5582
|
+
newEndpoint: event.current,
|
|
5583
|
+
routes: pendingReanchors,
|
|
5584
|
+
});
|
|
5585
|
+
this.durablyAppliedEndpointRefreshes.set(event.current.hostId, transitionKey);
|
|
5586
|
+
}
|
|
5587
|
+
catch (error) {
|
|
5588
|
+
// Persistence may commit and then report an unknown outcome. Because
|
|
5589
|
+
// service/store mutation is serialized, exact G2 ownership for every
|
|
5590
|
+
// requested G1 alias proves this transition durably applied and gives
|
|
5591
|
+
// a later identical retry authority to restore a fail-closed G2 route.
|
|
5592
|
+
const after = await this.store.inspectPrivateCodexRoutes();
|
|
5593
|
+
const committedExactly = pendingReanchors.every((expected) => {
|
|
5594
|
+
const matches = after.filter((route) => route.alias === expected.alias &&
|
|
5595
|
+
route.binding.provider === "codex" &&
|
|
5596
|
+
route.binding.hostId === event.current.hostId &&
|
|
5597
|
+
route.binding.endpointGeneration ===
|
|
5598
|
+
event.current.endpointGeneration &&
|
|
5599
|
+
route.binding.routeHandle === expected.threadId &&
|
|
5600
|
+
route.binding.ownerLease === expected.ownerLease &&
|
|
5601
|
+
route.enabled &&
|
|
5602
|
+
route.compatibility === "compatible");
|
|
5603
|
+
return matches.length === 1;
|
|
5604
|
+
});
|
|
5605
|
+
if (committedExactly) {
|
|
5606
|
+
this.durablyAppliedEndpointRefreshes.set(event.current.hostId, transitionKey);
|
|
5607
|
+
try {
|
|
5608
|
+
await this.store.markConnectorOffline(event.current, "CODEX_ENDPOINT_ACTIVATION_FAILED", [], { preserveQueued: true });
|
|
5609
|
+
for (const route of pendingReanchors) {
|
|
5610
|
+
this.routeStates.set(route.alias, "stale");
|
|
5611
|
+
}
|
|
5612
|
+
await this.changed();
|
|
5613
|
+
}
|
|
5614
|
+
catch {
|
|
5615
|
+
this.dashboardHealthy = false;
|
|
5616
|
+
}
|
|
5617
|
+
}
|
|
5618
|
+
throw error;
|
|
5619
|
+
}
|
|
5620
|
+
}
|
|
5621
|
+
// A retry may arrive after the store commit but before native status or
|
|
5622
|
+
// provider activation. Re-read exact durable authority and accept only one
|
|
5623
|
+
// unique current-generation owner for every claimed task.
|
|
5624
|
+
inventory = await this.store.inspectPrivateCodexRoutes();
|
|
5625
|
+
if (this.durablyAppliedEndpointRefreshes.get(event.current.hostId) ===
|
|
5626
|
+
transitionKey) {
|
|
5627
|
+
const observedStates = new Map(event.routes.map((route) => [route.routeHandle, route.state]));
|
|
5628
|
+
for (const route of inventory) {
|
|
5629
|
+
const state = observedStates.get(route.binding.routeHandle);
|
|
5630
|
+
if (state === undefined ||
|
|
5631
|
+
route.binding.hostId !== event.current.hostId ||
|
|
5632
|
+
route.binding.endpointGeneration !==
|
|
5633
|
+
event.current.endpointGeneration ||
|
|
5634
|
+
route.state !== "stale" ||
|
|
5635
|
+
route.compatibility !== "expired") {
|
|
5636
|
+
continue;
|
|
5637
|
+
}
|
|
5638
|
+
await this.registerOrRebind(route.alias, route.binding, "endpoint_reobserved", state, route.binding.ownerLease);
|
|
5639
|
+
}
|
|
5640
|
+
inventory = await this.store.inspectPrivateCodexRoutes();
|
|
5641
|
+
}
|
|
5642
|
+
const previousRouteHandles = new Set(previousRoutes.map((route) => route.binding.routeHandle));
|
|
5643
|
+
const activatedRoutes = event.routes.flatMap((observed) => {
|
|
5644
|
+
const matches = inventory.filter((route) => route.binding.provider === "codex" &&
|
|
5645
|
+
route.binding.hostId === event.current.hostId &&
|
|
5646
|
+
route.binding.endpointGeneration ===
|
|
5647
|
+
event.current.endpointGeneration &&
|
|
5648
|
+
route.binding.routeHandle === observed.routeHandle &&
|
|
5649
|
+
route.enabled &&
|
|
5650
|
+
route.compatibility === "compatible");
|
|
5651
|
+
if (matches.length > 1) {
|
|
5652
|
+
throw new BridgeError("CODEX_ENDPOINT_REFRESH_MISMATCH", "A refreshed Codex task has multiple durable owners.");
|
|
5653
|
+
}
|
|
5654
|
+
if (matches.length === 0) {
|
|
5655
|
+
// A selector-triggered refresh can report the freshly selected task
|
|
5656
|
+
// before registerRoute installs that new alias. It has no prior G1
|
|
5657
|
+
// owner to re-anchor, so activation covers only already-durable tasks;
|
|
5658
|
+
// the caller installs this one immediately after the provider ack.
|
|
5659
|
+
if (!previousRouteHandles.has(observed.routeHandle))
|
|
5660
|
+
return [];
|
|
5661
|
+
throw new BridgeError("CODEX_ENDPOINT_REFRESH_MISMATCH", "A previously durable Codex task was not re-anchored exactly.");
|
|
5662
|
+
}
|
|
5663
|
+
const match = matches[0];
|
|
5664
|
+
if (match === undefined)
|
|
5665
|
+
return [];
|
|
5666
|
+
return {
|
|
5667
|
+
alias: match.alias,
|
|
5668
|
+
threadId: observed.routeHandle,
|
|
5669
|
+
ownerLease: match.binding.ownerLease,
|
|
5670
|
+
state: observed.state,
|
|
5671
|
+
};
|
|
5672
|
+
});
|
|
5673
|
+
const pendingSelector = this.pendingSelectorEndpointActivations.get(event.current.hostId);
|
|
5674
|
+
const requiredRoute = options.deferActivation === true
|
|
5675
|
+
? undefined
|
|
5676
|
+
: options.requiredRoute ??
|
|
5677
|
+
(pendingSelector?.transitionKey === transitionKey
|
|
5678
|
+
? pendingSelector.route
|
|
5679
|
+
: undefined);
|
|
5680
|
+
if (requiredRoute !== undefined &&
|
|
5681
|
+
!activatedRoutes.some((route) => route.alias === requiredRoute.alias &&
|
|
5682
|
+
route.threadId === requiredRoute.threadId)) {
|
|
5683
|
+
const matches = inventory.filter((route) => route.alias === requiredRoute.alias &&
|
|
5684
|
+
route.binding.provider === "codex" &&
|
|
5685
|
+
route.binding.hostId === event.current.hostId &&
|
|
5686
|
+
route.binding.endpointGeneration ===
|
|
5687
|
+
event.current.endpointGeneration &&
|
|
5688
|
+
route.binding.routeHandle === requiredRoute.threadId &&
|
|
5689
|
+
route.enabled &&
|
|
5690
|
+
route.compatibility === "compatible");
|
|
5691
|
+
const match = matches[0];
|
|
5692
|
+
if (matches.length !== 1 || match === undefined) {
|
|
5693
|
+
throw new BridgeError("CODEX_ENDPOINT_REFRESH_MISMATCH", "The registering Codex task was not durably installed exactly.");
|
|
5694
|
+
}
|
|
5695
|
+
activatedRoutes.push({
|
|
5696
|
+
alias: requiredRoute.alias,
|
|
5697
|
+
threadId: requiredRoute.threadId,
|
|
5698
|
+
ownerLease: match.binding.ownerLease,
|
|
5699
|
+
state: requiredRoute.state,
|
|
5700
|
+
});
|
|
5701
|
+
}
|
|
5702
|
+
if (options.deferActivation === true) {
|
|
5703
|
+
const deferredRoute = options.deferredRoute;
|
|
5704
|
+
if (deferredRoute === undefined) {
|
|
5705
|
+
throw new BridgeError("CODEX_ENDPOINT_REFRESH_MISMATCH", "A deferred selector transition requires one exact registration.");
|
|
5706
|
+
}
|
|
5707
|
+
const existing = this.pendingSelectorEndpointActivations.get(event.current.hostId);
|
|
5708
|
+
if (existing !== undefined &&
|
|
5709
|
+
(existing.transitionKey !== transitionKey ||
|
|
5710
|
+
existing.route.alias !== deferredRoute.alias ||
|
|
5711
|
+
existing.route.threadId !== deferredRoute.threadId)) {
|
|
5712
|
+
throw new BridgeError("CODEX_ENDPOINT_REFRESH_CONFLICT", "A different selector transition is already awaiting native activation.");
|
|
5713
|
+
}
|
|
5714
|
+
this.pendingSelectorEndpointActivations.set(event.current.hostId, {
|
|
5715
|
+
transitionKey,
|
|
5716
|
+
route: { ...deferredRoute },
|
|
5717
|
+
});
|
|
5718
|
+
return true;
|
|
5719
|
+
}
|
|
5720
|
+
const requireNativeContinuity = (route) => {
|
|
5721
|
+
const registration = this.codexRegistrationLocks.get(route.alias);
|
|
5722
|
+
if (registration === undefined ||
|
|
5723
|
+
registration.hostId !== event.current.hostId ||
|
|
5724
|
+
registration.threadId !== route.threadId ||
|
|
5725
|
+
!isCodexRegistrationGeneration(registration.generation)) {
|
|
5726
|
+
throw new BridgeError("CODEX_ENDPOINT_REFRESH_NATIVE_LISTENER_MISSING", "The refreshed Codex route has no exact live native listener authority.");
|
|
5727
|
+
}
|
|
5728
|
+
const nativeGeneration = this.requireCurrentNativeCodexGeneration(this.adapter("claude", event.current.hostId), route.alias);
|
|
5729
|
+
if (nativeGeneration !== registration.generation) {
|
|
5730
|
+
throw new BridgeError("CODEX_ENDPOINT_REFRESH_NATIVE_LISTENER_MISSING", "The refreshed Codex route's native listener generation changed before activation.");
|
|
5731
|
+
}
|
|
5732
|
+
};
|
|
5733
|
+
try {
|
|
5734
|
+
for (const route of activatedRoutes) {
|
|
5735
|
+
const binding = {
|
|
5736
|
+
...event.current,
|
|
5737
|
+
routeHandle: route.threadId,
|
|
5738
|
+
ownerLease: route.ownerLease,
|
|
5739
|
+
};
|
|
5740
|
+
this.rememberBinding(route.alias, binding, route.state);
|
|
5741
|
+
}
|
|
5742
|
+
for (const route of activatedRoutes) {
|
|
5743
|
+
requireNativeContinuity(route);
|
|
5744
|
+
await this.setNativeCodexStatus(route.alias, route.state === "idle"
|
|
5745
|
+
? "idle"
|
|
5746
|
+
: route.state === "awaiting_approval"
|
|
5747
|
+
? "waiting"
|
|
5748
|
+
: "busy", true);
|
|
5749
|
+
requireNativeContinuity(route);
|
|
5750
|
+
}
|
|
5751
|
+
await this.requireEndpointRefreshUnconflicted();
|
|
5752
|
+
for (const route of activatedRoutes) {
|
|
5753
|
+
const activityKey = `${transitionKey}\0${route.alias}`;
|
|
5754
|
+
if (!this.endpointRefreshActivityKeys.has(activityKey)) {
|
|
5755
|
+
this.endpointRefreshActivityKeys.add(activityKey);
|
|
5756
|
+
while (this.endpointRefreshActivityKeys.size > RUNTIME_ACTIVITY_CAPACITY) {
|
|
5757
|
+
const oldest = this.endpointRefreshActivityKeys.values().next().value;
|
|
5758
|
+
if (oldest === undefined)
|
|
5759
|
+
break;
|
|
5760
|
+
this.endpointRefreshActivityKeys.delete(oldest);
|
|
5761
|
+
}
|
|
5762
|
+
this.activitySequence += 1;
|
|
5763
|
+
this.activityEvents.push({
|
|
5764
|
+
sequence: this.activitySequence,
|
|
5765
|
+
timestamp: this.now().toISOString(),
|
|
5766
|
+
kind: "endpoint",
|
|
5767
|
+
action: "endpoint_refreshed",
|
|
5768
|
+
outcome: "accepted",
|
|
5769
|
+
aliases: [route.alias],
|
|
5770
|
+
operatorAction: false,
|
|
5771
|
+
});
|
|
5772
|
+
while (this.activityEvents.length > RUNTIME_ACTIVITY_CAPACITY) {
|
|
5773
|
+
this.activityEvents.shift();
|
|
5774
|
+
}
|
|
5775
|
+
}
|
|
5776
|
+
if (route.state === "idle")
|
|
5777
|
+
this.scheduleDispatch(route.alias);
|
|
5778
|
+
}
|
|
5779
|
+
await this.changed();
|
|
5780
|
+
await this.requireEndpointRefreshUnconflicted();
|
|
5781
|
+
for (const route of activatedRoutes) {
|
|
5782
|
+
requireNativeContinuity(route);
|
|
5783
|
+
}
|
|
5784
|
+
// Provider writes remain closed until every durable and native transition
|
|
5785
|
+
// above is complete. This acknowledgement is deliberately the final
|
|
5786
|
+
// fallible activation step.
|
|
5787
|
+
const accept = adapter.acceptCompatibilityAttestation;
|
|
5788
|
+
if (accept === undefined) {
|
|
5789
|
+
throw new BridgeError("CODEX_ENDPOINT_REFRESH_ACTIVATION_UNAVAILABLE", "The Codex provider cannot acknowledge the validated endpoint transition.");
|
|
5790
|
+
}
|
|
5791
|
+
accept.call(adapter, event.attestation);
|
|
5792
|
+
}
|
|
5793
|
+
catch (error) {
|
|
5794
|
+
try {
|
|
5795
|
+
await this.store.markConnectorOffline(event.current, "CODEX_ENDPOINT_ACTIVATION_FAILED", [], { preserveQueued: true });
|
|
5796
|
+
for (const route of activatedRoutes) {
|
|
5797
|
+
this.routeStates.set(route.alias, "stale");
|
|
5798
|
+
}
|
|
5799
|
+
await this.changed();
|
|
5800
|
+
}
|
|
5801
|
+
catch {
|
|
5802
|
+
this.dashboardHealthy = false;
|
|
5803
|
+
}
|
|
5804
|
+
throw error;
|
|
5805
|
+
}
|
|
5806
|
+
this.activatedEndpointRefreshes.set(event.current.hostId, transitionKey);
|
|
5807
|
+
if (this.pendingSelectorEndpointActivations.get(event.current.hostId)
|
|
5808
|
+
?.transitionKey === transitionKey) {
|
|
5809
|
+
this.pendingSelectorEndpointActivations.delete(event.current.hostId);
|
|
5810
|
+
}
|
|
5811
|
+
this.handledIncompatibleEndpointRefreshes.delete(event.current.hostId);
|
|
5812
|
+
this.activeCodexIncompatibility = undefined;
|
|
5813
|
+
return false;
|
|
5814
|
+
}
|
|
4589
5815
|
async onRouteState(source, event) {
|
|
4590
5816
|
if (this.closing)
|
|
4591
5817
|
return;
|
|
@@ -4598,6 +5824,9 @@ export class GatewayService {
|
|
|
4598
5824
|
const [alias, binding] = entry;
|
|
4599
5825
|
const previousState = this.routeStates.get(alias);
|
|
4600
5826
|
this.routeStates.set(alias, event.state);
|
|
5827
|
+
if (binding.provider === "claude" && event.state === "idle") {
|
|
5828
|
+
this.claudePrewriteIdleWaitTargets.delete(alias);
|
|
5829
|
+
}
|
|
4601
5830
|
const staleSafeCode = binding.provider === "codex"
|
|
4602
5831
|
? "CODEX_ROUTE_STALE"
|
|
4603
5832
|
: safeCode(event.safeErrorCode, "ROUTE_STALE");
|
|
@@ -4655,15 +5884,27 @@ export class GatewayService {
|
|
|
4655
5884
|
this.scheduleDispatch(alias);
|
|
4656
5885
|
}
|
|
4657
5886
|
}
|
|
4658
|
-
async setNativeCodexStatus(alias, status) {
|
|
5887
|
+
async setNativeCodexStatus(alias, status, requireSuccess = false) {
|
|
4659
5888
|
const binding = this.routeBindings.get(alias);
|
|
4660
5889
|
if (binding?.provider !== "codex")
|
|
4661
5890
|
return;
|
|
4662
|
-
|
|
4663
|
-
|
|
4664
|
-
|
|
5891
|
+
const adapter = this.adapter("claude", binding.hostId);
|
|
5892
|
+
const update = adapter.updateNativeCodexPeerStatus;
|
|
5893
|
+
if (update === undefined) {
|
|
5894
|
+
if (requireSuccess) {
|
|
5895
|
+
throw new BridgeError("CODEX_ENDPOINT_REFRESH_NATIVE_STATUS_FAILED", "The native listener does not support refreshed Codex status activation.", true);
|
|
5896
|
+
}
|
|
5897
|
+
return;
|
|
5898
|
+
}
|
|
5899
|
+
try {
|
|
5900
|
+
await update.call(adapter, alias, status);
|
|
5901
|
+
}
|
|
5902
|
+
catch (error) {
|
|
4665
5903
|
this.dashboardHealthy = false;
|
|
4666
|
-
|
|
5904
|
+
if (requireSuccess) {
|
|
5905
|
+
throw new BridgeError("CODEX_ENDPOINT_REFRESH_NATIVE_STATUS_FAILED", "The refreshed Codex route could not be activated in the native listener.", true);
|
|
5906
|
+
}
|
|
5907
|
+
}
|
|
4667
5908
|
}
|
|
4668
5909
|
rejectDetachedNativeReceipt(hostId, receiptHandle, diagnosticCode) {
|
|
4669
5910
|
const work = this.settleDetachedNativeReceipt(hostId, receiptHandle, diagnosticCode).finally(() => {
|
|
@@ -4748,15 +5989,29 @@ export class GatewayService {
|
|
|
4748
5989
|
async publicSnapshotLocked() {
|
|
4749
5990
|
const base = await this.store.publicSnapshot();
|
|
4750
5991
|
const compatibilityChecks = [];
|
|
4751
|
-
for (const
|
|
4752
|
-
|
|
4753
|
-
|
|
5992
|
+
for (const surface of compatibilitySurfaces) {
|
|
5993
|
+
if (surface === "codex" && this.activeCodexIncompatibility !== undefined) {
|
|
5994
|
+
compatibilityChecks.push(structuredClone(this.activeCodexIncompatibility));
|
|
4754
5995
|
continue;
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
5996
|
+
}
|
|
5997
|
+
const currentVersions = [
|
|
5998
|
+
...new Set(this.adapters.flatMap((adapter) => {
|
|
5999
|
+
const observation = adapter.compatibilitySurface?.();
|
|
6000
|
+
return observation?.surface === surface ? [observation.version] : [];
|
|
6001
|
+
})),
|
|
6002
|
+
].sort((left, right) => left.localeCompare(right));
|
|
6003
|
+
const currentAttestations = (await Promise.all(currentVersions.map(async (version) => await this.store.inspectCompatibilityAttestation(surface, version))))
|
|
6004
|
+
.filter((attestation) => attestation !== undefined)
|
|
6005
|
+
.sort((left, right) => {
|
|
6006
|
+
const checkedAt = right.checkedAt.localeCompare(left.checkedAt);
|
|
6007
|
+
if (checkedAt !== 0)
|
|
6008
|
+
return checkedAt;
|
|
6009
|
+
return left.version.localeCompare(right.version);
|
|
6010
|
+
});
|
|
6011
|
+
const selected = currentAttestations[0];
|
|
6012
|
+
if (selected !== undefined)
|
|
6013
|
+
compatibilityChecks.push(selected);
|
|
4758
6014
|
}
|
|
4759
|
-
compatibilityChecks.sort((left, right) => left.surface.localeCompare(right.surface));
|
|
4760
6015
|
const peers = this.availablePeers.map((peer) => ({ ...peer }));
|
|
4761
6016
|
if (!arePublicAvailablePeerSnapshots(peers)) {
|
|
4762
6017
|
throw new BridgeError("INVALID_TRANSIENT_INVENTORY", "The transient peer inventory is unsafe.");
|