inngest 4.2.5-pr-1480.2 → 4.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api/schema.d.cts +2 -2
- package/api/schema.d.ts +2 -2
- package/components/connect/strategies/core/connection.cjs +8 -81
- package/components/connect/strategies/core/connection.cjs.map +1 -1
- package/components/connect/strategies/core/connection.js +8 -81
- package/components/connect/strategies/core/connection.js.map +1 -1
- package/components/connect/strategies/core/heartbeat.cjs +1 -2
- package/components/connect/strategies/core/heartbeat.cjs.map +1 -1
- package/components/connect/strategies/core/heartbeat.js +1 -2
- package/components/connect/strategies/core/heartbeat.js.map +1 -1
- package/components/connect/strategies/core/requestProcessor.cjs +3 -15
- package/components/connect/strategies/core/requestProcessor.cjs.map +1 -1
- package/components/connect/strategies/core/requestProcessor.js +3 -15
- package/components/connect/strategies/core/requestProcessor.js.map +1 -1
- package/components/connect/types.cjs.map +1 -1
- package/components/connect/types.d.cts +0 -18
- package/components/connect/types.d.cts.map +1 -1
- package/components/connect/types.d.ts +0 -18
- package/components/connect/types.d.ts.map +1 -1
- package/components/connect/types.js.map +1 -1
- package/components/execution/otel/middleware.d.cts +2 -2
- package/components/execution/otel/middleware.d.ts +2 -2
- package/components/realtime/types.d.cts +3 -3
- package/components/realtime/types.d.cts.map +1 -1
- package/components/realtime/types.d.ts +3 -3
- package/components/realtime/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/react.d.cts.map +1 -1
- package/react.d.ts.map +1 -1
- package/types.d.cts +10 -10
- package/types.d.ts +10 -10
- package/version.cjs +1 -1
- package/version.cjs.map +1 -1
- package/version.d.cts +1 -1
- package/version.d.ts +1 -1
- package/version.js +1 -1
- package/version.js.map +1 -1
- package/components/connect/strategies/core/types.cjs +0 -21
- package/components/connect/strategies/core/types.cjs.map +0 -1
- package/components/connect/strategies/core/types.js +0 -20
- package/components/connect/strategies/core/types.js.map +0 -1
package/api/schema.d.cts
CHANGED
|
@@ -27,8 +27,8 @@ declare const stepSchema: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnion<[z.Zo
|
|
|
27
27
|
error: z.ZodType<JsonError, z.ZodTypeDef, JsonError>;
|
|
28
28
|
}, "strict", z.ZodTypeAny, {
|
|
29
29
|
error: {
|
|
30
|
-
name?: string | undefined;
|
|
31
30
|
error?: string | undefined;
|
|
31
|
+
name?: string | undefined;
|
|
32
32
|
message?: string | undefined;
|
|
33
33
|
stack?: string | undefined;
|
|
34
34
|
} & {
|
|
@@ -39,8 +39,8 @@ declare const stepSchema: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnion<[z.Zo
|
|
|
39
39
|
type: "error";
|
|
40
40
|
}, {
|
|
41
41
|
error: {
|
|
42
|
-
name?: string | undefined;
|
|
43
42
|
error?: string | undefined;
|
|
43
|
+
name?: string | undefined;
|
|
44
44
|
message?: string | undefined;
|
|
45
45
|
stack?: string | undefined;
|
|
46
46
|
} & {
|
package/api/schema.d.ts
CHANGED
|
@@ -27,8 +27,8 @@ declare const stepSchema: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnion<[z.Zo
|
|
|
27
27
|
error: z.ZodType<JsonError, z.ZodTypeDef, JsonError>;
|
|
28
28
|
}, "strict", z.ZodTypeAny, {
|
|
29
29
|
error: {
|
|
30
|
-
name?: string | undefined;
|
|
31
30
|
error?: string | undefined;
|
|
31
|
+
name?: string | undefined;
|
|
32
32
|
message?: string | undefined;
|
|
33
33
|
stack?: string | undefined;
|
|
34
34
|
} & {
|
|
@@ -39,8 +39,8 @@ declare const stepSchema: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnion<[z.Zo
|
|
|
39
39
|
type: "error";
|
|
40
40
|
}, {
|
|
41
41
|
error: {
|
|
42
|
-
name?: string | undefined;
|
|
43
42
|
error?: string | undefined;
|
|
43
|
+
name?: string | undefined;
|
|
44
44
|
message?: string | undefined;
|
|
45
45
|
stack?: string | undefined;
|
|
46
46
|
} & {
|
|
@@ -6,7 +6,6 @@ const require_types = require('../../types.cjs');
|
|
|
6
6
|
const require_url = require('../../../../helpers/url.cjs');
|
|
7
7
|
const require_messages = require('../../messages.cjs');
|
|
8
8
|
const require_handshake = require('./handshake.cjs');
|
|
9
|
-
const require_types$1 = require('./types.cjs');
|
|
10
9
|
const require_heartbeat = require('./heartbeat.cjs');
|
|
11
10
|
const require_requestProcessor = require('./requestProcessor.cjs');
|
|
12
11
|
const require_statusReporter = require('./statusReporter.cjs');
|
|
@@ -36,7 +35,7 @@ let __jpwilliams_waitgroup = require("@jpwilliams/waitgroup");
|
|
|
36
35
|
* - Manages a single heartbeat interval targeting the active connection
|
|
37
36
|
* - Handles reconnection, drain, and shutdown as state changes
|
|
38
37
|
*/
|
|
39
|
-
var ConnectionCore = class
|
|
38
|
+
var ConnectionCore = class {
|
|
40
39
|
config;
|
|
41
40
|
callbacks;
|
|
42
41
|
_activeConnection;
|
|
@@ -52,10 +51,7 @@ var ConnectionCore = class ConnectionCore {
|
|
|
52
51
|
_lastMessageReceivedAt;
|
|
53
52
|
excludeGateways = /* @__PURE__ */ new Set();
|
|
54
53
|
wakeSignal;
|
|
55
|
-
pendingWakeReasons = [];
|
|
56
54
|
hasConnectedBefore = false;
|
|
57
|
-
shutdownDumpInterval;
|
|
58
|
-
static SHUTDOWN_DUMP_INTERVAL_MS = 6e4;
|
|
59
55
|
loopPromise;
|
|
60
56
|
resolveFirstReady;
|
|
61
57
|
rejectFirstReady;
|
|
@@ -91,7 +87,7 @@ var ConnectionCore = class ConnectionCore {
|
|
|
91
87
|
return self.config.appIds;
|
|
92
88
|
}
|
|
93
89
|
};
|
|
94
|
-
const wakeSignalRef = { wake: (
|
|
90
|
+
const wakeSignalRef = { wake: () => this.wake() };
|
|
95
91
|
const self = this;
|
|
96
92
|
this.heartbeatManager = new require_heartbeat.HeartbeatManager(accessor, wakeSignalRef, callbacks.logger);
|
|
97
93
|
this.heartbeatManager.onHeartbeatSent = () => {
|
|
@@ -150,13 +146,11 @@ var ConnectionCore = class ConnectionCore {
|
|
|
150
146
|
const inFlightCount = Object.keys(this._inProgressRequests.requestLeases).length;
|
|
151
147
|
this.callbacks.logger.info({ inFlightCount }, "Shutting down, waiting for in-flight requests");
|
|
152
148
|
this._shutdownRequested = true;
|
|
153
|
-
this.dumpInFlightForShutdown("drain-start");
|
|
154
|
-
this.startShutdownInFlightDumpTimer();
|
|
155
149
|
if (this._activeConnection?.ws.readyState === WebSocket.OPEN) {
|
|
156
150
|
this._activeConnection.ws.send(require_buffer.ensureUnsharedArrayBuffer(require_connect.ConnectMessage.encode(require_connect.ConnectMessage.create({ kind: require_connect.GatewayMessageType.WORKER_PAUSE })).finish()));
|
|
157
151
|
this.callbacks.logger.info({ connectionId: this._activeConnection.id }, "Sent WORKER_PAUSE, draining");
|
|
158
152
|
}
|
|
159
|
-
this.wake(
|
|
153
|
+
this.wake();
|
|
160
154
|
if (this.loopPromise) await this.loopPromise;
|
|
161
155
|
this.callbacks.logger.info("Connection closed");
|
|
162
156
|
}
|
|
@@ -175,8 +169,7 @@ var ConnectionCore = class ConnectionCore {
|
|
|
175
169
|
resolve
|
|
176
170
|
};
|
|
177
171
|
}
|
|
178
|
-
wake(
|
|
179
|
-
this.pendingWakeReasons.push(reason);
|
|
172
|
+
wake() {
|
|
180
173
|
this.wakeSignal.resolve();
|
|
181
174
|
this.resetWakeSignal();
|
|
182
175
|
}
|
|
@@ -188,63 +181,8 @@ var ConnectionCore = class ConnectionCore {
|
|
|
188
181
|
hasInFlightRequests() {
|
|
189
182
|
return Object.keys(this._inProgressRequests.requestLeases).length > 0;
|
|
190
183
|
}
|
|
191
|
-
/**
|
|
192
|
-
* Debug-level "still draining" dump emitted at drain start and periodically
|
|
193
|
-
* thereafter while in-flight requests are holding the shutdown. One summary
|
|
194
|
-
* line plus one line per request carrying `requestId`, `runId`, `stepId`,
|
|
195
|
-
* `functionSlug`, `ageMs`, and `sinceLastLeaseExtendMs`. Does not affect
|
|
196
|
-
* info/warn logs.
|
|
197
|
-
*
|
|
198
|
-
* `requestLeases` drives the reconcile-loop exit gate, so use it as the
|
|
199
|
-
* single source of truth for the in-flight set; `requestMeta` carries the
|
|
200
|
-
* enrichment fields and is kept in sync alongside the lease map.
|
|
201
|
-
*/
|
|
202
|
-
dumpInFlightForShutdown(reason) {
|
|
203
|
-
const leaseIds = Object.keys(this._inProgressRequests.requestLeases);
|
|
204
|
-
if (leaseIds.length === 0) return;
|
|
205
|
-
const now = Date.now();
|
|
206
|
-
const ages = [];
|
|
207
|
-
for (const id of leaseIds) {
|
|
208
|
-
const m = this._inProgressRequests.requestMeta[id];
|
|
209
|
-
if (m?.leaseAcquiredAt) ages.push(now - m.leaseAcquiredAt);
|
|
210
|
-
}
|
|
211
|
-
this.callbacks.logger.debug({
|
|
212
|
-
reason,
|
|
213
|
-
inFlightCount: leaseIds.length,
|
|
214
|
-
oldestAgeMs: ages.length > 0 ? Math.max(...ages) : void 0
|
|
215
|
-
}, "Shutdown: still draining");
|
|
216
|
-
for (const id of leaseIds) {
|
|
217
|
-
const m = this._inProgressRequests.requestMeta[id];
|
|
218
|
-
if (!m) continue;
|
|
219
|
-
this.callbacks.logger.debug({
|
|
220
|
-
reason,
|
|
221
|
-
requestId: m.requestId,
|
|
222
|
-
runId: m.runId,
|
|
223
|
-
stepId: m.stepId,
|
|
224
|
-
functionSlug: m.functionSlug,
|
|
225
|
-
appId: m.appId,
|
|
226
|
-
ageMs: m.leaseAcquiredAt ? now - m.leaseAcquiredAt : void 0,
|
|
227
|
-
sinceLastLeaseExtendMs: m.leaseLastExtendedAt ? now - m.leaseLastExtendedAt : void 0
|
|
228
|
-
}, "Shutdown: still draining in-flight request");
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
startShutdownInFlightDumpTimer() {
|
|
232
|
-
if (this.shutdownDumpInterval) return;
|
|
233
|
-
this.shutdownDumpInterval = setInterval(() => {
|
|
234
|
-
if (!this._shutdownRequested) return;
|
|
235
|
-
this.dumpInFlightForShutdown("periodic");
|
|
236
|
-
this.wake(require_types$1.WAKE_REASON.ShutdownStillPending);
|
|
237
|
-
}, ConnectionCore.SHUTDOWN_DUMP_INTERVAL_MS);
|
|
238
|
-
}
|
|
239
|
-
stopShutdownInFlightDumpTimer() {
|
|
240
|
-
if (this.shutdownDumpInterval) {
|
|
241
|
-
clearInterval(this.shutdownDumpInterval);
|
|
242
|
-
this.shutdownDumpInterval = void 0;
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
184
|
async reconcileLoop(initialAttempt) {
|
|
246
185
|
let attempt = initialAttempt;
|
|
247
|
-
this.callbacks.logger.debug({ initialAttempt }, "Reconcile loop entered");
|
|
248
186
|
while (true) {
|
|
249
187
|
if (this._shutdownRequested && !this.hasInFlightRequests()) break;
|
|
250
188
|
if (!this._activeConnection || this._activeConnection.dead) {
|
|
@@ -281,10 +219,7 @@ var ConnectionCore = class ConnectionCore {
|
|
|
281
219
|
if (this._shutdownRequested) {
|
|
282
220
|
conn.ws.send(require_buffer.ensureUnsharedArrayBuffer(require_connect.ConnectMessage.encode(require_connect.ConnectMessage.create({ kind: require_connect.GatewayMessageType.WORKER_PAUSE })).finish()));
|
|
283
221
|
this.callbacks.logger.info({ connectionId: conn.id }, "Sent WORKER_PAUSE on reconnect during shutdown");
|
|
284
|
-
} else {
|
|
285
|
-
conn.ws.send(require_buffer.ensureUnsharedArrayBuffer(require_connect.ConnectMessage.encode(require_connect.ConnectMessage.create({ kind: require_connect.GatewayMessageType.WORKER_READY })).finish()));
|
|
286
|
-
this.callbacks.logger.info({ connectionId: conn.id }, "Sent WORKER_READY");
|
|
287
|
-
}
|
|
222
|
+
} else conn.ws.send(require_buffer.ensureUnsharedArrayBuffer(require_connect.ConnectMessage.encode(require_connect.ConnectMessage.create({ kind: require_connect.GatewayMessageType.WORKER_READY })).finish()));
|
|
288
223
|
this.callbacks.onConnectionActive?.(this.useSigningKey);
|
|
289
224
|
this.resolveFirstReady?.();
|
|
290
225
|
this.resolveFirstReady = void 0;
|
|
@@ -318,22 +253,14 @@ var ConnectionCore = class ConnectionCore {
|
|
|
318
253
|
}
|
|
319
254
|
}
|
|
320
255
|
await this.wakeSignal.promise;
|
|
321
|
-
const reasons = this.pendingWakeReasons;
|
|
322
|
-
this.pendingWakeReasons = [];
|
|
323
256
|
this.callbacks.logger.debug({
|
|
324
|
-
reasons,
|
|
325
257
|
shutdownRequested: this._shutdownRequested,
|
|
326
258
|
hasActiveConnection: !!this._activeConnection,
|
|
327
259
|
activeConnectionDead: this._activeConnection?.dead
|
|
328
260
|
}, "Reconcile loop woken");
|
|
329
261
|
}
|
|
330
|
-
this.callbacks.logger.debug({
|
|
331
|
-
shutdownRequested: this._shutdownRequested,
|
|
332
|
-
inFlightCount: Object.keys(this._inProgressRequests.requestLeases).length
|
|
333
|
-
}, "Reconcile loop exiting");
|
|
334
262
|
this.heartbeatManager.stop();
|
|
335
263
|
this.statusReporter.stop();
|
|
336
|
-
this.stopShutdownInFlightDumpTimer();
|
|
337
264
|
this._activeConnection?.close();
|
|
338
265
|
this._activeConnection = void 0;
|
|
339
266
|
this._drainingConnection?.close();
|
|
@@ -357,7 +284,7 @@ var ConnectionCore = class ConnectionCore {
|
|
|
357
284
|
conn.dead = true;
|
|
358
285
|
this.excludeGateways.add(gatewayGroup);
|
|
359
286
|
if (this._activeConnection?.id === connectionId) this._activeConnection = void 0;
|
|
360
|
-
this.wake(
|
|
287
|
+
this.wake();
|
|
361
288
|
};
|
|
362
289
|
ws.onclose = (ev) => {
|
|
363
290
|
if (conn.dead) return;
|
|
@@ -372,7 +299,7 @@ var ConnectionCore = class ConnectionCore {
|
|
|
372
299
|
conn.dead = true;
|
|
373
300
|
this.excludeGateways.add(gatewayGroup);
|
|
374
301
|
if (this._activeConnection?.id === connectionId) this._activeConnection = void 0;
|
|
375
|
-
this.wake(
|
|
302
|
+
this.wake();
|
|
376
303
|
};
|
|
377
304
|
ws.onmessage = async (event) => {
|
|
378
305
|
this._lastMessageReceivedAt = Date.now();
|
|
@@ -386,7 +313,7 @@ var ConnectionCore = class ConnectionCore {
|
|
|
386
313
|
}, "Gateway draining, opening new connection");
|
|
387
314
|
this._drainingConnection = this._activeConnection;
|
|
388
315
|
this._activeConnection = void 0;
|
|
389
|
-
this.wake(
|
|
316
|
+
this.wake();
|
|
390
317
|
return;
|
|
391
318
|
}
|
|
392
319
|
if (connectMessage.kind === require_connect.GatewayMessageType.GATEWAY_HEARTBEAT) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"connection.cjs","names":["WaitGroup","resolve: () => void","HeartbeatManager","StatusReporter","RequestProcessor","ConnectionState","ensureUnsharedArrayBuffer","ConnectMessage","GatewayMessageType","WAKE_REASON","resolveApiBaseUrl","ages: number[]","establishConnection","ReconnectError","AuthError","ConnectionLimitError","waitWithCancel","expBackoff","parseConnectMessage","gatewayMessageTypeToJSON"],"sources":["../../../../../src/components/connect/strategies/core/connection.ts"],"sourcesContent":["/**\n * Shared connection core logic used by both SameThreadStrategy and\n * WorkerThreadStrategy.\n *\n * This module uses a **reconcile loop** that continuously ensures a live\n * WebSocket connection is open. Reconnection, drain, and shutdown are\n * expressed as state changes that wake the loop rather than recursive\n * calls or callback-driven control flow.\n *\n * Domain-specific logic is delegated to focused sub-modules:\n * - {@link HeartbeatManager} — periodic heartbeat pings\n * - {@link RequestProcessor} — executor requests, lease extensions, reply ACKs\n * - {@link establishConnection} — HTTP start + WebSocket handshake\n */\n\nimport { WaitGroup } from \"@jpwilliams/waitgroup\";\nimport { resolveApiBaseUrl } from \"../../../../helpers/url.ts\";\nimport type { Logger } from \"../../../../middleware/logger.ts\";\nimport type { GatewayExecutorRequestData } from \"../../../../proto/src/components/connect/protobuf/connect.ts\";\nimport {\n ConnectMessage,\n GatewayMessageType,\n gatewayMessageTypeToJSON,\n} from \"../../../../proto/src/components/connect/protobuf/connect.ts\";\nimport { ensureUnsharedArrayBuffer } from \"../../buffer.ts\";\nimport { parseConnectMessage } from \"../../messages.ts\";\nimport {\n type ConnectDebugState,\n ConnectionState,\n type InFlightRequest,\n} from \"../../types.ts\";\nimport {\n AuthError,\n ConnectionLimitError,\n expBackoff,\n ReconnectError,\n waitWithCancel,\n} from \"../../util.ts\";\nimport { establishConnection } from \"./handshake.ts\";\nimport { HeartbeatManager } from \"./heartbeat.ts\";\nimport { RequestProcessor } from \"./requestProcessor.ts\";\nimport { StatusReporter } from \"./statusReporter.ts\";\nimport type { BaseConnectionConfig } from \"./types.ts\";\nimport { WAKE_REASON, type WakeReason } from \"./types.ts\";\n\n/**\n * Connection object representing an active WebSocket connection.\n */\nexport interface Connection {\n id: string;\n ws: WebSocket;\n pendingHeartbeats: number;\n /** When true the connection is considered unusable and the reconcile loop\n * will establish a replacement. */\n dead: boolean;\n heartbeatIntervalMs: number;\n extendLeaseIntervalMs: number;\n statusIntervalMs: number;\n /** Timestamp (ms) when the connection was established. */\n connectedAt: number;\n /** Disable all handlers and close the underlying WebSocket. */\n close(): void;\n}\n\n/**\n * Configuration for the connection core.\n * Extends BaseConnectionConfig with connection-specific options.\n */\nexport interface ConnectionCoreConfig extends BaseConnectionConfig {\n instanceId?: string;\n maxWorkerConcurrency?: number;\n gatewayUrl?: string;\n appIds: string[];\n}\n\n/**\n * Callbacks for connection core events.\n */\nexport interface ConnectionCoreCallbacks {\n logger: Logger;\n onStateChange: (state: ConnectionState) => void;\n getState: () => ConnectionState;\n handleExecutionRequest: (\n request: GatewayExecutorRequestData,\n ) => Promise<Uint8Array>;\n onReplyAck?: (requestId: string) => void;\n onBufferResponse?: (requestId: string, responseBytes: Uint8Array) => void;\n onConnectionActive?: (signingKey: string | undefined) => void;\n}\n\n/**\n * Core connection manager that handles WebSocket connection lifecycle,\n * handshake, heartbeat, lease extension, and reconnection.\n *\n * Uses a reconcile loop that:\n * - Ensures a WebSocket connection is always open\n * - Manages a single heartbeat interval targeting the active connection\n * - Handles reconnection, drain, and shutdown as state changes\n */\nexport class ConnectionCore {\n private config: ConnectionCoreConfig;\n private callbacks: ConnectionCoreCallbacks;\n\n // Exposed via ConnectionAccessor for sub-modules\n private _activeConnection: Connection | undefined;\n private _drainingConnection: Connection | undefined;\n private _shutdownRequested = false;\n private _inProgressRequests: {\n wg: WaitGroup;\n requestLeases: Record<string, string>;\n requestMeta: Record<string, InFlightRequest>;\n } = {\n wg: new WaitGroup(),\n requestLeases: {},\n requestMeta: {},\n };\n\n private _lastHeartbeatSentAt: number | undefined;\n private _lastHeartbeatReceivedAt: number | undefined;\n private _lastMessageReceivedAt: number | undefined;\n\n private excludeGateways: Set<string> = new Set();\n\n // Wake signal for the reconcile loop\n private wakeSignal: { promise: Promise<void>; resolve: () => void };\n // Reasons accumulated since the last loop wake. Read + cleared by the loop.\n private pendingWakeReasons: WakeReason[] = [];\n\n // Whether we've ever successfully connected (used to distinguish\n // CONNECTING from RECONNECTING state transitions).\n private hasConnectedBefore = false;\n\n // Shutdown diagnostics: periodic \"still draining\" dump logged while the\n // drain is outstanding. Started in close(), cleared in teardown.\n private shutdownDumpInterval: ReturnType<typeof setInterval> | undefined;\n\n // Cadence for the periodic \"still draining\" debug dump.\n private static readonly SHUTDOWN_DUMP_INTERVAL_MS = 60_000;\n\n // Loop promise — resolved when the reconcile loop exits\n private loopPromise: Promise<void> | undefined;\n\n // First-ready resolution — resolves start() when first connection is ready\n private resolveFirstReady: (() => void) | undefined;\n private rejectFirstReady: ((err: unknown) => void) | undefined;\n\n // Signing key management\n private useSigningKey: string | undefined;\n\n // Sub-modules\n private readonly heartbeatManager: HeartbeatManager;\n private readonly statusReporter: StatusReporter;\n private readonly requestProcessor: RequestProcessor;\n\n constructor(\n config: ConnectionCoreConfig,\n callbacks: ConnectionCoreCallbacks,\n ) {\n this.config = config;\n this.callbacks = callbacks;\n this.useSigningKey = config.hashedSigningKey;\n\n // Initialize the wake signal\n let resolve: () => void;\n const promise = new Promise<void>((r) => {\n resolve = r;\n });\n this.wakeSignal = { promise, resolve: resolve! };\n\n // Build a ConnectionAccessor view for sub-modules\n const accessor = {\n get activeConnection() {\n return self._activeConnection;\n },\n get drainingConnection() {\n return self._drainingConnection;\n },\n get shutdownRequested() {\n return self._shutdownRequested;\n },\n get inProgressRequests() {\n return self._inProgressRequests;\n },\n get appIds() {\n return self.config.appIds;\n },\n };\n\n const wakeSignalRef = { wake: (reason?: WakeReason) => this.wake(reason) };\n\n const self = this;\n\n this.heartbeatManager = new HeartbeatManager(\n accessor,\n wakeSignalRef,\n callbacks.logger,\n );\n this.heartbeatManager.onHeartbeatSent = () => {\n this._lastHeartbeatSentAt = Date.now();\n };\n\n this.statusReporter = new StatusReporter(accessor, callbacks.logger);\n\n this.requestProcessor = new RequestProcessor(\n accessor,\n wakeSignalRef,\n callbacks,\n callbacks.logger,\n );\n }\n\n get connectionId(): string | undefined {\n return this._activeConnection?.id;\n }\n\n /**\n * Wait for all in-progress requests to complete.\n */\n async waitForInProgress(): Promise<void> {\n await this._inProgressRequests.wg.wait();\n }\n\n /**\n * Return a snapshot of debug/health information for this connection.\n */\n getDebugState(): ConnectDebugState {\n return {\n state: this.callbacks.getState(),\n activeConnectionId: this._activeConnection?.id,\n drainingConnectionId: this._drainingConnection?.id,\n lastHeartbeatSentAt: this._lastHeartbeatSentAt,\n lastHeartbeatReceivedAt: this._lastHeartbeatReceivedAt,\n lastMessageReceivedAt: this._lastMessageReceivedAt,\n shutdownRequested: this._shutdownRequested,\n inFlightRequestCount: Object.keys(this._inProgressRequests.requestLeases)\n .length,\n inFlightRequests: Object.values(this._inProgressRequests.requestMeta),\n };\n }\n\n /**\n * Start the reconcile loop. Resolves when the first connection is active.\n * The loop continues running in the background.\n */\n async start(attempt = 0): Promise<void> {\n if (typeof WebSocket === \"undefined\") {\n throw new Error(\"WebSockets not supported in current environment\");\n }\n\n const state = this.callbacks.getState();\n if (state === ConnectionState.CLOSED) {\n throw new Error(\"Connection already closed\");\n }\n\n const firstReadyPromise = new Promise<void>((resolve, reject) => {\n this.resolveFirstReady = resolve;\n this.rejectFirstReady = reject;\n });\n\n this.loopPromise = this.reconcileLoop(attempt);\n\n // If the loop ends before firstReady resolves, propagate any error\n this.loopPromise.catch((err) => {\n this.rejectFirstReady?.(err);\n });\n\n await firstReadyPromise;\n }\n\n /**\n * Request graceful shutdown. Resolves when fully closed (in-flight done,\n * connection closed).\n */\n async close(): Promise<void> {\n const inFlightCount = Object.keys(\n this._inProgressRequests.requestLeases,\n ).length;\n this.callbacks.logger.info(\n { inFlightCount },\n \"Shutting down, waiting for in-flight requests\",\n );\n // Flip the shutdown flag before starting any timers so the periodic\n // dump guard (`if (!this._shutdownRequested) return`) cannot observe a\n // stale `false` on its first tick.\n this._shutdownRequested = true;\n // Verbose per-request dump (debug-only) at drain start so operators can\n // immediately see which runs are holding the shutdown.\n this.dumpInFlightForShutdown(\"drain-start\");\n this.startShutdownInFlightDumpTimer();\n\n if (this._activeConnection?.ws.readyState === WebSocket.OPEN) {\n this._activeConnection.ws.send(\n ensureUnsharedArrayBuffer(\n ConnectMessage.encode(\n ConnectMessage.create({\n kind: GatewayMessageType.WORKER_PAUSE,\n }),\n ).finish(),\n ),\n );\n this.callbacks.logger.info(\n { connectionId: this._activeConnection.id },\n \"Sent WORKER_PAUSE, draining\",\n );\n }\n\n this.wake(WAKE_REASON.ShutdownRequested);\n\n if (this.loopPromise) {\n await this.loopPromise;\n }\n\n this.callbacks.logger.info(\"Connection closed\");\n }\n\n async getApiBaseUrl(): Promise<string> {\n return resolveApiBaseUrl({\n apiBaseUrl: this.config.apiBaseUrl,\n mode: this.config.mode,\n });\n }\n\n // ---------------------------------------------------------------------------\n // Wake signal\n // ---------------------------------------------------------------------------\n\n private resetWakeSignal(): void {\n let resolve: () => void;\n const promise = new Promise<void>((r) => {\n resolve = r;\n });\n this.wakeSignal = { promise, resolve: resolve! };\n }\n\n private wake(reason: WakeReason = WAKE_REASON.Unknown): void {\n this.pendingWakeReasons.push(reason);\n this.wakeSignal.resolve();\n this.resetWakeSignal();\n }\n\n // ---------------------------------------------------------------------------\n // Signing key management\n // ---------------------------------------------------------------------------\n\n private switchAuthKey(): void {\n const switchToFallback =\n this.useSigningKey === this.config.hashedSigningKey;\n if (switchToFallback) {\n this.callbacks.logger.debug(\"Switching to fallback signing key\");\n }\n this.useSigningKey = switchToFallback\n ? this.config.hashedFallbackKey\n : this.config.hashedSigningKey;\n }\n\n // ---------------------------------------------------------------------------\n // In-flight helpers\n // ---------------------------------------------------------------------------\n\n private hasInFlightRequests(): boolean {\n return Object.keys(this._inProgressRequests.requestLeases).length > 0;\n }\n\n /**\n * Debug-level \"still draining\" dump emitted at drain start and periodically\n * thereafter while in-flight requests are holding the shutdown. One summary\n * line plus one line per request carrying `requestId`, `runId`, `stepId`,\n * `functionSlug`, `ageMs`, and `sinceLastLeaseExtendMs`. Does not affect\n * info/warn logs.\n *\n * `requestLeases` drives the reconcile-loop exit gate, so use it as the\n * single source of truth for the in-flight set; `requestMeta` carries the\n * enrichment fields and is kept in sync alongside the lease map.\n */\n private dumpInFlightForShutdown(reason: string): void {\n const leaseIds = Object.keys(this._inProgressRequests.requestLeases);\n if (leaseIds.length === 0) return;\n const now = Date.now();\n const ages: number[] = [];\n for (const id of leaseIds) {\n const m = this._inProgressRequests.requestMeta[id];\n if (m?.leaseAcquiredAt) ages.push(now - m.leaseAcquiredAt);\n }\n\n this.callbacks.logger.debug(\n {\n reason,\n inFlightCount: leaseIds.length,\n oldestAgeMs: ages.length > 0 ? Math.max(...ages) : undefined,\n },\n \"Shutdown: still draining\",\n );\n\n for (const id of leaseIds) {\n const m = this._inProgressRequests.requestMeta[id];\n if (!m) continue;\n this.callbacks.logger.debug(\n {\n reason,\n requestId: m.requestId,\n runId: m.runId,\n stepId: m.stepId,\n functionSlug: m.functionSlug,\n appId: m.appId,\n ageMs: m.leaseAcquiredAt ? now - m.leaseAcquiredAt : undefined,\n sinceLastLeaseExtendMs: m.leaseLastExtendedAt\n ? now - m.leaseLastExtendedAt\n : undefined,\n },\n \"Shutdown: still draining in-flight request\",\n );\n }\n }\n\n private startShutdownInFlightDumpTimer(): void {\n if (this.shutdownDumpInterval) return;\n this.shutdownDumpInterval = setInterval(() => {\n if (!this._shutdownRequested) return;\n this.dumpInFlightForShutdown(\"periodic\");\n // Wake the loop so its \"Reconcile loop woken\" line emits a fresh\n // state snapshot alongside the in-flight dump. Loop will park again\n // immediately if nothing has changed.\n this.wake(WAKE_REASON.ShutdownStillPending);\n }, ConnectionCore.SHUTDOWN_DUMP_INTERVAL_MS);\n }\n\n private stopShutdownInFlightDumpTimer(): void {\n if (this.shutdownDumpInterval) {\n clearInterval(this.shutdownDumpInterval);\n this.shutdownDumpInterval = undefined;\n }\n }\n\n // ---------------------------------------------------------------------------\n // Reconcile loop\n // ---------------------------------------------------------------------------\n\n private async reconcileLoop(initialAttempt: number): Promise<void> {\n let attempt = initialAttempt;\n\n this.callbacks.logger.debug({ initialAttempt }, \"Reconcile loop entered\");\n\n while (true) {\n // Exit condition: shutdown requested + no in-flight requests\n if (this._shutdownRequested && !this.hasInFlightRequests()) {\n break;\n }\n\n // Ensure we have a live connection\n if (!this._activeConnection || this._activeConnection.dead) {\n this.callbacks.logger.debug(\n {\n hasActiveConnection: !!this._activeConnection,\n activeConnectionDead: this._activeConnection?.dead,\n hasDrainingConnection: !!this._drainingConnection,\n drainingConnectionId: this._drainingConnection?.id,\n },\n \"No active connection\",\n );\n\n if (this.hasConnectedBefore) {\n this.callbacks.logger.info({ attempt }, \"Reconnecting\");\n } else {\n this.callbacks.logger.info(\"Connecting\");\n }\n\n this.callbacks.onStateChange(\n this.hasConnectedBefore\n ? ConnectionState.RECONNECTING\n : ConnectionState.CONNECTING,\n );\n\n try {\n const { conn, gatewayGroup } = await establishConnection(\n this.config,\n this.useSigningKey,\n attempt,\n this.excludeGateways,\n this.callbacks.logger,\n );\n\n // Attach post-handshake handlers\n this.attachHandlers(conn, gatewayGroup);\n\n // Clean up draining connection after new one is ready\n if (this._drainingConnection) {\n this.callbacks.logger.info(\n {\n oldConnectionId: this._drainingConnection.id,\n newConnectionId: conn.id,\n },\n \"Replaced draining connection\",\n );\n this._drainingConnection.close();\n this._drainingConnection = undefined;\n }\n\n this._activeConnection = conn;\n this.heartbeatManager.updateInterval(conn.heartbeatIntervalMs);\n this.statusReporter.updateInterval(conn.statusIntervalMs);\n attempt = 0;\n this.hasConnectedBefore = true;\n this.callbacks.logger.info(\n { connectionId: conn.id, gatewayGroup },\n \"Connection active\",\n );\n this.callbacks.onStateChange(ConnectionState.ACTIVE);\n\n if (this._shutdownRequested) {\n // Reconnected during shutdown to keep in-flight requests alive.\n // Send WORKER_PAUSE instead of WORKER_READY so no new work is routed.\n conn.ws.send(\n ensureUnsharedArrayBuffer(\n ConnectMessage.encode(\n ConnectMessage.create({\n kind: GatewayMessageType.WORKER_PAUSE,\n }),\n ).finish(),\n ),\n );\n this.callbacks.logger.info(\n { connectionId: conn.id },\n \"Sent WORKER_PAUSE on reconnect during shutdown\",\n );\n } else {\n // Signal the gateway that we're ready to receive requests.\n // This must happen after ACTIVE so the gateway doesn't route\n // requests before handlers are fully attached.\n conn.ws.send(\n ensureUnsharedArrayBuffer(\n ConnectMessage.encode(\n ConnectMessage.create({\n kind: GatewayMessageType.WORKER_READY,\n }),\n ).finish(),\n ),\n );\n this.callbacks.logger.info(\n { connectionId: conn.id },\n \"Sent WORKER_READY\",\n );\n }\n\n // Flush any buffered responses via HTTP now that we're active.\n this.callbacks.onConnectionActive?.(this.useSigningKey);\n\n this.resolveFirstReady?.();\n this.resolveFirstReady = undefined;\n this.rejectFirstReady = undefined;\n } catch (err) {\n if (!(err instanceof ReconnectError)) throw err;\n\n attempt = err.attempt + 1;\n if (err instanceof AuthError) this.switchAuthKey();\n if (err instanceof ConnectionLimitError) {\n this.callbacks.logger.error(\"Max concurrent connections reached\");\n }\n\n // Gateway is draining, we should retry much faster\n if (err.message?.includes(\"connect_gateway_closing\")) {\n const jitter = 500 + Math.random() * 1000;\n this.callbacks.logger.info(\n { attempt, delay: Math.round(jitter), error: err.message },\n \"Gateway draining, retrying\",\n );\n const cancelled = await waitWithCancel(jitter, () => {\n return this._shutdownRequested && !this.hasInFlightRequests();\n });\n if (cancelled) break;\n continue;\n }\n\n const delay = expBackoff(attempt);\n this.callbacks.logger.info(\n { attempt, delay },\n \"Reconnecting after failure\",\n );\n\n const cancelled = await waitWithCancel(delay, () => {\n return this._shutdownRequested && !this.hasInFlightRequests();\n });\n if (cancelled) break;\n continue;\n }\n }\n\n // Wait for something to change\n await this.wakeSignal.promise;\n const reasons = this.pendingWakeReasons;\n this.pendingWakeReasons = [];\n this.callbacks.logger.debug(\n {\n reasons,\n shutdownRequested: this._shutdownRequested,\n hasActiveConnection: !!this._activeConnection,\n activeConnectionDead: this._activeConnection?.dead,\n },\n \"Reconcile loop woken\",\n );\n }\n\n this.callbacks.logger.debug(\n {\n shutdownRequested: this._shutdownRequested,\n inFlightCount: Object.keys(this._inProgressRequests.requestLeases)\n .length,\n },\n \"Reconcile loop exiting\",\n );\n\n // Teardown\n this.heartbeatManager.stop();\n this.statusReporter.stop();\n this.stopShutdownInFlightDumpTimer();\n this._activeConnection?.close();\n this._activeConnection = undefined;\n this._drainingConnection?.close();\n this._drainingConnection = undefined;\n }\n\n // ---------------------------------------------------------------------------\n // Post-handshake handler attachment\n // ---------------------------------------------------------------------------\n\n /**\n * Wire up error, close, and message handlers on a newly-handshaked connection.\n */\n private attachHandlers(conn: Connection, gatewayGroup: string): void {\n const { ws } = conn;\n const connectionId = conn.id;\n\n // Error/close handlers: mark connection as dead and wake the loop\n ws.onerror = (ev) => {\n if (conn.dead) return;\n const uptimeMs = Date.now() - conn.connectedAt;\n this.callbacks.logger.warn(\n {\n connectionId,\n gatewayGroup,\n uptimeMs,\n error: (ev as ErrorEvent)?.message,\n },\n \"Connection lost (error)\",\n );\n conn.dead = true;\n this.excludeGateways.add(gatewayGroup);\n if (this._activeConnection?.id === connectionId) {\n this._activeConnection = undefined;\n }\n this.wake(WAKE_REASON.WsError);\n };\n\n ws.onclose = (ev) => {\n if (conn.dead) return;\n const uptimeMs = Date.now() - conn.connectedAt;\n this.callbacks.logger.warn(\n {\n connectionId,\n gatewayGroup,\n uptimeMs,\n code: ev.code,\n reason: ev.reason,\n },\n \"Connection lost (close)\",\n );\n conn.dead = true;\n this.excludeGateways.add(gatewayGroup);\n if (this._activeConnection?.id === connectionId) {\n this._activeConnection = undefined;\n }\n this.wake(WAKE_REASON.WsClose);\n };\n\n // Message handler for post-handshake messages\n ws.onmessage = async (event) => {\n this._lastMessageReceivedAt = Date.now();\n\n const messageBytes = new Uint8Array(event.data as ArrayBuffer);\n const connectMessage = parseConnectMessage(messageBytes);\n\n if (connectMessage.kind === GatewayMessageType.GATEWAY_CLOSING) {\n const uptimeMs = Date.now() - conn.connectedAt;\n this.callbacks.logger.info(\n { connectionId: conn.id, gatewayGroup, uptimeMs },\n \"Gateway draining, opening new connection\",\n );\n // Move current connection to draining, clear active so the loop\n // establishes a replacement.\n this._drainingConnection = this._activeConnection;\n this._activeConnection = undefined;\n this.wake(WAKE_REASON.GatewayClosing);\n return;\n }\n\n if (connectMessage.kind === GatewayMessageType.GATEWAY_HEARTBEAT) {\n this._lastHeartbeatReceivedAt = Date.now();\n conn.pendingHeartbeats = 0;\n this.callbacks.logger.debug(\n { connectionId },\n \"Handled gateway heartbeat\",\n );\n return;\n }\n\n if (connectMessage.kind === GatewayMessageType.GATEWAY_EXECUTOR_REQUEST) {\n await this.requestProcessor.handleExecutorRequest(connectMessage, conn);\n return;\n }\n\n if (connectMessage.kind === GatewayMessageType.WORKER_REPLY_ACK) {\n this.requestProcessor.handleReplyAck(connectMessage, connectionId);\n return;\n }\n\n if (\n connectMessage.kind ===\n GatewayMessageType.WORKER_REQUEST_EXTEND_LEASE_ACK\n ) {\n this.requestProcessor.handleExtendLeaseAck(\n connectMessage,\n connectionId,\n );\n return;\n }\n\n this.callbacks.logger.warn(\n {\n kind: gatewayMessageTypeToJSON(connectMessage.kind),\n rawKind: connectMessage.kind,\n state: this.callbacks.getState(),\n connectionId,\n },\n \"Unexpected message type\",\n );\n };\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmGA,IAAa,iBAAb,MAAa,eAAe;CAC1B,AAAQ;CACR,AAAQ;CAGR,AAAQ;CACR,AAAQ;CACR,AAAQ,qBAAqB;CAC7B,AAAQ,sBAIJ;EACF,IAAI,IAAIA,kCAAW;EACnB,eAAe,EAAE;EACjB,aAAa,EAAE;EAChB;CAED,AAAQ;CACR,AAAQ;CACR,AAAQ;CAER,AAAQ,kCAA+B,IAAI,KAAK;CAGhD,AAAQ;CAER,AAAQ,qBAAmC,EAAE;CAI7C,AAAQ,qBAAqB;CAI7B,AAAQ;CAGR,OAAwB,4BAA4B;CAGpD,AAAQ;CAGR,AAAQ;CACR,AAAQ;CAGR,AAAQ;CAGR,AAAiB;CACjB,AAAiB;CACjB,AAAiB;CAEjB,YACE,QACA,WACA;AACA,OAAK,SAAS;AACd,OAAK,YAAY;AACjB,OAAK,gBAAgB,OAAO;EAG5B,IAAIC;AAIJ,OAAK,aAAa;GAAE,SAHJ,IAAI,SAAe,MAAM;AACvC,cAAU;KACV;GACoC;GAAU;EAGhD,MAAM,WAAW;GACf,IAAI,mBAAmB;AACrB,WAAO,KAAK;;GAEd,IAAI,qBAAqB;AACvB,WAAO,KAAK;;GAEd,IAAI,oBAAoB;AACtB,WAAO,KAAK;;GAEd,IAAI,qBAAqB;AACvB,WAAO,KAAK;;GAEd,IAAI,SAAS;AACX,WAAO,KAAK,OAAO;;GAEtB;EAED,MAAM,gBAAgB,EAAE,OAAO,WAAwB,KAAK,KAAK,OAAO,EAAE;EAE1E,MAAM,OAAO;AAEb,OAAK,mBAAmB,IAAIC,mCAC1B,UACA,eACA,UAAU,OACX;AACD,OAAK,iBAAiB,wBAAwB;AAC5C,QAAK,uBAAuB,KAAK,KAAK;;AAGxC,OAAK,iBAAiB,IAAIC,sCAAe,UAAU,UAAU,OAAO;AAEpE,OAAK,mBAAmB,IAAIC,0CAC1B,UACA,eACA,WACA,UAAU,OACX;;CAGH,IAAI,eAAmC;AACrC,SAAO,KAAK,mBAAmB;;;;;CAMjC,MAAM,oBAAmC;AACvC,QAAM,KAAK,oBAAoB,GAAG,MAAM;;;;;CAM1C,gBAAmC;AACjC,SAAO;GACL,OAAO,KAAK,UAAU,UAAU;GAChC,oBAAoB,KAAK,mBAAmB;GAC5C,sBAAsB,KAAK,qBAAqB;GAChD,qBAAqB,KAAK;GAC1B,yBAAyB,KAAK;GAC9B,uBAAuB,KAAK;GAC5B,mBAAmB,KAAK;GACxB,sBAAsB,OAAO,KAAK,KAAK,oBAAoB,cAAc,CACtE;GACH,kBAAkB,OAAO,OAAO,KAAK,oBAAoB,YAAY;GACtE;;;;;;CAOH,MAAM,MAAM,UAAU,GAAkB;AACtC,MAAI,OAAO,cAAc,YACvB,OAAM,IAAI,MAAM,kDAAkD;AAIpE,MADc,KAAK,UAAU,UAAU,KACzBC,8BAAgB,OAC5B,OAAM,IAAI,MAAM,4BAA4B;EAG9C,MAAM,oBAAoB,IAAI,SAAe,SAAS,WAAW;AAC/D,QAAK,oBAAoB;AACzB,QAAK,mBAAmB;IACxB;AAEF,OAAK,cAAc,KAAK,cAAc,QAAQ;AAG9C,OAAK,YAAY,OAAO,QAAQ;AAC9B,QAAK,mBAAmB,IAAI;IAC5B;AAEF,QAAM;;;;;;CAOR,MAAM,QAAuB;EAC3B,MAAM,gBAAgB,OAAO,KAC3B,KAAK,oBAAoB,cAC1B,CAAC;AACF,OAAK,UAAU,OAAO,KACpB,EAAE,eAAe,EACjB,gDACD;AAID,OAAK,qBAAqB;AAG1B,OAAK,wBAAwB,cAAc;AAC3C,OAAK,gCAAgC;AAErC,MAAI,KAAK,mBAAmB,GAAG,eAAe,UAAU,MAAM;AAC5D,QAAK,kBAAkB,GAAG,KACxBC,yCACEC,+BAAe,OACbA,+BAAe,OAAO,EACpB,MAAMC,mCAAmB,cAC1B,CAAC,CACH,CAAC,QAAQ,CACX,CACF;AACD,QAAK,UAAU,OAAO,KACpB,EAAE,cAAc,KAAK,kBAAkB,IAAI,EAC3C,8BACD;;AAGH,OAAK,KAAKC,4BAAY,kBAAkB;AAExC,MAAI,KAAK,YACP,OAAM,KAAK;AAGb,OAAK,UAAU,OAAO,KAAK,oBAAoB;;CAGjD,MAAM,gBAAiC;AACrC,SAAOC,8BAAkB;GACvB,YAAY,KAAK,OAAO;GACxB,MAAM,KAAK,OAAO;GACnB,CAAC;;CAOJ,AAAQ,kBAAwB;EAC9B,IAAIT;AAIJ,OAAK,aAAa;GAAE,SAHJ,IAAI,SAAe,MAAM;AACvC,cAAU;KACV;GACoC;GAAU;;CAGlD,AAAQ,KAAK,SAAqBQ,4BAAY,SAAe;AAC3D,OAAK,mBAAmB,KAAK,OAAO;AACpC,OAAK,WAAW,SAAS;AACzB,OAAK,iBAAiB;;CAOxB,AAAQ,gBAAsB;EAC5B,MAAM,mBACJ,KAAK,kBAAkB,KAAK,OAAO;AACrC,MAAI,iBACF,MAAK,UAAU,OAAO,MAAM,oCAAoC;AAElE,OAAK,gBAAgB,mBACjB,KAAK,OAAO,oBACZ,KAAK,OAAO;;CAOlB,AAAQ,sBAA+B;AACrC,SAAO,OAAO,KAAK,KAAK,oBAAoB,cAAc,CAAC,SAAS;;;;;;;;;;;;;CActE,AAAQ,wBAAwB,QAAsB;EACpD,MAAM,WAAW,OAAO,KAAK,KAAK,oBAAoB,cAAc;AACpE,MAAI,SAAS,WAAW,EAAG;EAC3B,MAAM,MAAM,KAAK,KAAK;EACtB,MAAME,OAAiB,EAAE;AACzB,OAAK,MAAM,MAAM,UAAU;GACzB,MAAM,IAAI,KAAK,oBAAoB,YAAY;AAC/C,OAAI,GAAG,gBAAiB,MAAK,KAAK,MAAM,EAAE,gBAAgB;;AAG5D,OAAK,UAAU,OAAO,MACpB;GACE;GACA,eAAe,SAAS;GACxB,aAAa,KAAK,SAAS,IAAI,KAAK,IAAI,GAAG,KAAK,GAAG;GACpD,EACD,2BACD;AAED,OAAK,MAAM,MAAM,UAAU;GACzB,MAAM,IAAI,KAAK,oBAAoB,YAAY;AAC/C,OAAI,CAAC,EAAG;AACR,QAAK,UAAU,OAAO,MACpB;IACE;IACA,WAAW,EAAE;IACb,OAAO,EAAE;IACT,QAAQ,EAAE;IACV,cAAc,EAAE;IAChB,OAAO,EAAE;IACT,OAAO,EAAE,kBAAkB,MAAM,EAAE,kBAAkB;IACrD,wBAAwB,EAAE,sBACtB,MAAM,EAAE,sBACR;IACL,EACD,6CACD;;;CAIL,AAAQ,iCAAuC;AAC7C,MAAI,KAAK,qBAAsB;AAC/B,OAAK,uBAAuB,kBAAkB;AAC5C,OAAI,CAAC,KAAK,mBAAoB;AAC9B,QAAK,wBAAwB,WAAW;AAIxC,QAAK,KAAKF,4BAAY,qBAAqB;KAC1C,eAAe,0BAA0B;;CAG9C,AAAQ,gCAAsC;AAC5C,MAAI,KAAK,sBAAsB;AAC7B,iBAAc,KAAK,qBAAqB;AACxC,QAAK,uBAAuB;;;CAQhC,MAAc,cAAc,gBAAuC;EACjE,IAAI,UAAU;AAEd,OAAK,UAAU,OAAO,MAAM,EAAE,gBAAgB,EAAE,yBAAyB;AAEzE,SAAO,MAAM;AAEX,OAAI,KAAK,sBAAsB,CAAC,KAAK,qBAAqB,CACxD;AAIF,OAAI,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,MAAM;AAC1D,SAAK,UAAU,OAAO,MACpB;KACE,qBAAqB,CAAC,CAAC,KAAK;KAC5B,sBAAsB,KAAK,mBAAmB;KAC9C,uBAAuB,CAAC,CAAC,KAAK;KAC9B,sBAAsB,KAAK,qBAAqB;KACjD,EACD,uBACD;AAED,QAAI,KAAK,mBACP,MAAK,UAAU,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe;QAEvD,MAAK,UAAU,OAAO,KAAK,aAAa;AAG1C,SAAK,UAAU,cACb,KAAK,qBACDJ,8BAAgB,eAChBA,8BAAgB,WACrB;AAED,QAAI;KACF,MAAM,EAAE,MAAM,iBAAiB,MAAMO,sCACnC,KAAK,QACL,KAAK,eACL,SACA,KAAK,iBACL,KAAK,UAAU,OAChB;AAGD,UAAK,eAAe,MAAM,aAAa;AAGvC,SAAI,KAAK,qBAAqB;AAC5B,WAAK,UAAU,OAAO,KACpB;OACE,iBAAiB,KAAK,oBAAoB;OAC1C,iBAAiB,KAAK;OACvB,EACD,+BACD;AACD,WAAK,oBAAoB,OAAO;AAChC,WAAK,sBAAsB;;AAG7B,UAAK,oBAAoB;AACzB,UAAK,iBAAiB,eAAe,KAAK,oBAAoB;AAC9D,UAAK,eAAe,eAAe,KAAK,iBAAiB;AACzD,eAAU;AACV,UAAK,qBAAqB;AAC1B,UAAK,UAAU,OAAO,KACpB;MAAE,cAAc,KAAK;MAAI;MAAc,EACvC,oBACD;AACD,UAAK,UAAU,cAAcP,8BAAgB,OAAO;AAEpD,SAAI,KAAK,oBAAoB;AAG3B,WAAK,GAAG,KACNC,yCACEC,+BAAe,OACbA,+BAAe,OAAO,EACpB,MAAMC,mCAAmB,cAC1B,CAAC,CACH,CAAC,QAAQ,CACX,CACF;AACD,WAAK,UAAU,OAAO,KACpB,EAAE,cAAc,KAAK,IAAI,EACzB,iDACD;YACI;AAIL,WAAK,GAAG,KACNF,yCACEC,+BAAe,OACbA,+BAAe,OAAO,EACpB,MAAMC,mCAAmB,cAC1B,CAAC,CACH,CAAC,QAAQ,CACX,CACF;AACD,WAAK,UAAU,OAAO,KACpB,EAAE,cAAc,KAAK,IAAI,EACzB,oBACD;;AAIH,UAAK,UAAU,qBAAqB,KAAK,cAAc;AAEvD,UAAK,qBAAqB;AAC1B,UAAK,oBAAoB;AACzB,UAAK,mBAAmB;aACjB,KAAK;AACZ,SAAI,EAAE,eAAeK,6BAAiB,OAAM;AAE5C,eAAU,IAAI,UAAU;AACxB,SAAI,eAAeC,uBAAW,MAAK,eAAe;AAClD,SAAI,eAAeC,kCACjB,MAAK,UAAU,OAAO,MAAM,qCAAqC;AAInE,SAAI,IAAI,SAAS,SAAS,0BAA0B,EAAE;MACpD,MAAM,SAAS,MAAM,KAAK,QAAQ,GAAG;AACrC,WAAK,UAAU,OAAO,KACpB;OAAE;OAAS,OAAO,KAAK,MAAM,OAAO;OAAE,OAAO,IAAI;OAAS,EAC1D,6BACD;AAID,UAHkB,MAAMC,4BAAe,cAAc;AACnD,cAAO,KAAK,sBAAsB,CAAC,KAAK,qBAAqB;QAC7D,CACa;AACf;;KAGF,MAAM,QAAQC,wBAAW,QAAQ;AACjC,UAAK,UAAU,OAAO,KACpB;MAAE;MAAS;MAAO,EAClB,6BACD;AAKD,SAHkB,MAAMD,4BAAe,aAAa;AAClD,aAAO,KAAK,sBAAsB,CAAC,KAAK,qBAAqB;OAC7D,CACa;AACf;;;AAKJ,SAAM,KAAK,WAAW;GACtB,MAAM,UAAU,KAAK;AACrB,QAAK,qBAAqB,EAAE;AAC5B,QAAK,UAAU,OAAO,MACpB;IACE;IACA,mBAAmB,KAAK;IACxB,qBAAqB,CAAC,CAAC,KAAK;IAC5B,sBAAsB,KAAK,mBAAmB;IAC/C,EACD,uBACD;;AAGH,OAAK,UAAU,OAAO,MACpB;GACE,mBAAmB,KAAK;GACxB,eAAe,OAAO,KAAK,KAAK,oBAAoB,cAAc,CAC/D;GACJ,EACD,yBACD;AAGD,OAAK,iBAAiB,MAAM;AAC5B,OAAK,eAAe,MAAM;AAC1B,OAAK,+BAA+B;AACpC,OAAK,mBAAmB,OAAO;AAC/B,OAAK,oBAAoB;AACzB,OAAK,qBAAqB,OAAO;AACjC,OAAK,sBAAsB;;;;;CAU7B,AAAQ,eAAe,MAAkB,cAA4B;EACnE,MAAM,EAAE,OAAO;EACf,MAAM,eAAe,KAAK;AAG1B,KAAG,WAAW,OAAO;AACnB,OAAI,KAAK,KAAM;GACf,MAAM,WAAW,KAAK,KAAK,GAAG,KAAK;AACnC,QAAK,UAAU,OAAO,KACpB;IACE;IACA;IACA;IACA,OAAQ,IAAmB;IAC5B,EACD,0BACD;AACD,QAAK,OAAO;AACZ,QAAK,gBAAgB,IAAI,aAAa;AACtC,OAAI,KAAK,mBAAmB,OAAO,aACjC,MAAK,oBAAoB;AAE3B,QAAK,KAAKP,4BAAY,QAAQ;;AAGhC,KAAG,WAAW,OAAO;AACnB,OAAI,KAAK,KAAM;GACf,MAAM,WAAW,KAAK,KAAK,GAAG,KAAK;AACnC,QAAK,UAAU,OAAO,KACpB;IACE;IACA;IACA;IACA,MAAM,GAAG;IACT,QAAQ,GAAG;IACZ,EACD,0BACD;AACD,QAAK,OAAO;AACZ,QAAK,gBAAgB,IAAI,aAAa;AACtC,OAAI,KAAK,mBAAmB,OAAO,aACjC,MAAK,oBAAoB;AAE3B,QAAK,KAAKA,4BAAY,QAAQ;;AAIhC,KAAG,YAAY,OAAO,UAAU;AAC9B,QAAK,yBAAyB,KAAK,KAAK;GAGxC,MAAM,iBAAiBS,qCADF,IAAI,WAAW,MAAM,KAAoB,CACN;AAExD,OAAI,eAAe,SAASV,mCAAmB,iBAAiB;IAC9D,MAAM,WAAW,KAAK,KAAK,GAAG,KAAK;AACnC,SAAK,UAAU,OAAO,KACpB;KAAE,cAAc,KAAK;KAAI;KAAc;KAAU,EACjD,2CACD;AAGD,SAAK,sBAAsB,KAAK;AAChC,SAAK,oBAAoB;AACzB,SAAK,KAAKC,4BAAY,eAAe;AACrC;;AAGF,OAAI,eAAe,SAASD,mCAAmB,mBAAmB;AAChE,SAAK,2BAA2B,KAAK,KAAK;AAC1C,SAAK,oBAAoB;AACzB,SAAK,UAAU,OAAO,MACpB,EAAE,cAAc,EAChB,4BACD;AACD;;AAGF,OAAI,eAAe,SAASA,mCAAmB,0BAA0B;AACvE,UAAM,KAAK,iBAAiB,sBAAsB,gBAAgB,KAAK;AACvE;;AAGF,OAAI,eAAe,SAASA,mCAAmB,kBAAkB;AAC/D,SAAK,iBAAiB,eAAe,gBAAgB,aAAa;AAClE;;AAGF,OACE,eAAe,SACfA,mCAAmB,iCACnB;AACA,SAAK,iBAAiB,qBACpB,gBACA,aACD;AACD;;AAGF,QAAK,UAAU,OAAO,KACpB;IACE,MAAMW,yCAAyB,eAAe,KAAK;IACnD,SAAS,eAAe;IACxB,OAAO,KAAK,UAAU,UAAU;IAChC;IACD,EACD,0BACD"}
|
|
1
|
+
{"version":3,"file":"connection.cjs","names":["WaitGroup","resolve: () => void","HeartbeatManager","StatusReporter","RequestProcessor","ConnectionState","ensureUnsharedArrayBuffer","ConnectMessage","GatewayMessageType","resolveApiBaseUrl","establishConnection","ReconnectError","AuthError","ConnectionLimitError","waitWithCancel","expBackoff","parseConnectMessage","gatewayMessageTypeToJSON"],"sources":["../../../../../src/components/connect/strategies/core/connection.ts"],"sourcesContent":["/**\n * Shared connection core logic used by both SameThreadStrategy and\n * WorkerThreadStrategy.\n *\n * This module uses a **reconcile loop** that continuously ensures a live\n * WebSocket connection is open. Reconnection, drain, and shutdown are\n * expressed as state changes that wake the loop rather than recursive\n * calls or callback-driven control flow.\n *\n * Domain-specific logic is delegated to focused sub-modules:\n * - {@link HeartbeatManager} — periodic heartbeat pings\n * - {@link RequestProcessor} — executor requests, lease extensions, reply ACKs\n * - {@link establishConnection} — HTTP start + WebSocket handshake\n */\n\nimport { WaitGroup } from \"@jpwilliams/waitgroup\";\nimport { resolveApiBaseUrl } from \"../../../../helpers/url.ts\";\nimport type { Logger } from \"../../../../middleware/logger.ts\";\nimport type { GatewayExecutorRequestData } from \"../../../../proto/src/components/connect/protobuf/connect.ts\";\nimport {\n ConnectMessage,\n GatewayMessageType,\n gatewayMessageTypeToJSON,\n} from \"../../../../proto/src/components/connect/protobuf/connect.ts\";\nimport { ensureUnsharedArrayBuffer } from \"../../buffer.ts\";\nimport { parseConnectMessage } from \"../../messages.ts\";\nimport {\n type ConnectDebugState,\n ConnectionState,\n type InFlightRequest,\n} from \"../../types.ts\";\nimport {\n AuthError,\n ConnectionLimitError,\n expBackoff,\n ReconnectError,\n waitWithCancel,\n} from \"../../util.ts\";\nimport { establishConnection } from \"./handshake.ts\";\nimport { HeartbeatManager } from \"./heartbeat.ts\";\nimport { RequestProcessor } from \"./requestProcessor.ts\";\nimport { StatusReporter } from \"./statusReporter.ts\";\nimport type { BaseConnectionConfig } from \"./types.ts\";\n\n/**\n * Connection object representing an active WebSocket connection.\n */\nexport interface Connection {\n id: string;\n ws: WebSocket;\n pendingHeartbeats: number;\n /** When true the connection is considered unusable and the reconcile loop\n * will establish a replacement. */\n dead: boolean;\n heartbeatIntervalMs: number;\n extendLeaseIntervalMs: number;\n statusIntervalMs: number;\n /** Timestamp (ms) when the connection was established. */\n connectedAt: number;\n /** Disable all handlers and close the underlying WebSocket. */\n close(): void;\n}\n\n/**\n * Configuration for the connection core.\n * Extends BaseConnectionConfig with connection-specific options.\n */\nexport interface ConnectionCoreConfig extends BaseConnectionConfig {\n instanceId?: string;\n maxWorkerConcurrency?: number;\n gatewayUrl?: string;\n appIds: string[];\n}\n\n/**\n * Callbacks for connection core events.\n */\nexport interface ConnectionCoreCallbacks {\n logger: Logger;\n onStateChange: (state: ConnectionState) => void;\n getState: () => ConnectionState;\n handleExecutionRequest: (\n request: GatewayExecutorRequestData,\n ) => Promise<Uint8Array>;\n onReplyAck?: (requestId: string) => void;\n onBufferResponse?: (requestId: string, responseBytes: Uint8Array) => void;\n onConnectionActive?: (signingKey: string | undefined) => void;\n}\n\n/**\n * Core connection manager that handles WebSocket connection lifecycle,\n * handshake, heartbeat, lease extension, and reconnection.\n *\n * Uses a reconcile loop that:\n * - Ensures a WebSocket connection is always open\n * - Manages a single heartbeat interval targeting the active connection\n * - Handles reconnection, drain, and shutdown as state changes\n */\nexport class ConnectionCore {\n private config: ConnectionCoreConfig;\n private callbacks: ConnectionCoreCallbacks;\n\n // Exposed via ConnectionAccessor for sub-modules\n private _activeConnection: Connection | undefined;\n private _drainingConnection: Connection | undefined;\n private _shutdownRequested = false;\n private _inProgressRequests: {\n wg: WaitGroup;\n requestLeases: Record<string, string>;\n requestMeta: Record<string, InFlightRequest>;\n } = {\n wg: new WaitGroup(),\n requestLeases: {},\n requestMeta: {},\n };\n\n private _lastHeartbeatSentAt: number | undefined;\n private _lastHeartbeatReceivedAt: number | undefined;\n private _lastMessageReceivedAt: number | undefined;\n\n private excludeGateways: Set<string> = new Set();\n\n // Wake signal for the reconcile loop\n private wakeSignal: { promise: Promise<void>; resolve: () => void };\n\n // Whether we've ever successfully connected (used to distinguish\n // CONNECTING from RECONNECTING state transitions).\n private hasConnectedBefore = false;\n\n // Loop promise — resolved when the reconcile loop exits\n private loopPromise: Promise<void> | undefined;\n\n // First-ready resolution — resolves start() when first connection is ready\n private resolveFirstReady: (() => void) | undefined;\n private rejectFirstReady: ((err: unknown) => void) | undefined;\n\n // Signing key management\n private useSigningKey: string | undefined;\n\n // Sub-modules\n private readonly heartbeatManager: HeartbeatManager;\n private readonly statusReporter: StatusReporter;\n private readonly requestProcessor: RequestProcessor;\n\n constructor(\n config: ConnectionCoreConfig,\n callbacks: ConnectionCoreCallbacks,\n ) {\n this.config = config;\n this.callbacks = callbacks;\n this.useSigningKey = config.hashedSigningKey;\n\n // Initialize the wake signal\n let resolve: () => void;\n const promise = new Promise<void>((r) => {\n resolve = r;\n });\n this.wakeSignal = { promise, resolve: resolve! };\n\n // Build a ConnectionAccessor view for sub-modules\n const accessor = {\n get activeConnection() {\n return self._activeConnection;\n },\n get drainingConnection() {\n return self._drainingConnection;\n },\n get shutdownRequested() {\n return self._shutdownRequested;\n },\n get inProgressRequests() {\n return self._inProgressRequests;\n },\n get appIds() {\n return self.config.appIds;\n },\n };\n\n const wakeSignalRef = { wake: () => this.wake() };\n\n const self = this;\n\n this.heartbeatManager = new HeartbeatManager(\n accessor,\n wakeSignalRef,\n callbacks.logger,\n );\n this.heartbeatManager.onHeartbeatSent = () => {\n this._lastHeartbeatSentAt = Date.now();\n };\n\n this.statusReporter = new StatusReporter(accessor, callbacks.logger);\n\n this.requestProcessor = new RequestProcessor(\n accessor,\n wakeSignalRef,\n callbacks,\n callbacks.logger,\n );\n }\n\n get connectionId(): string | undefined {\n return this._activeConnection?.id;\n }\n\n /**\n * Wait for all in-progress requests to complete.\n */\n async waitForInProgress(): Promise<void> {\n await this._inProgressRequests.wg.wait();\n }\n\n /**\n * Return a snapshot of debug/health information for this connection.\n */\n getDebugState(): ConnectDebugState {\n return {\n state: this.callbacks.getState(),\n activeConnectionId: this._activeConnection?.id,\n drainingConnectionId: this._drainingConnection?.id,\n lastHeartbeatSentAt: this._lastHeartbeatSentAt,\n lastHeartbeatReceivedAt: this._lastHeartbeatReceivedAt,\n lastMessageReceivedAt: this._lastMessageReceivedAt,\n shutdownRequested: this._shutdownRequested,\n inFlightRequestCount: Object.keys(this._inProgressRequests.requestLeases)\n .length,\n inFlightRequests: Object.values(this._inProgressRequests.requestMeta),\n };\n }\n\n /**\n * Start the reconcile loop. Resolves when the first connection is active.\n * The loop continues running in the background.\n */\n async start(attempt = 0): Promise<void> {\n if (typeof WebSocket === \"undefined\") {\n throw new Error(\"WebSockets not supported in current environment\");\n }\n\n const state = this.callbacks.getState();\n if (state === ConnectionState.CLOSED) {\n throw new Error(\"Connection already closed\");\n }\n\n const firstReadyPromise = new Promise<void>((resolve, reject) => {\n this.resolveFirstReady = resolve;\n this.rejectFirstReady = reject;\n });\n\n this.loopPromise = this.reconcileLoop(attempt);\n\n // If the loop ends before firstReady resolves, propagate any error\n this.loopPromise.catch((err) => {\n this.rejectFirstReady?.(err);\n });\n\n await firstReadyPromise;\n }\n\n /**\n * Request graceful shutdown. Resolves when fully closed (in-flight done,\n * connection closed).\n */\n async close(): Promise<void> {\n const inFlightCount = Object.keys(\n this._inProgressRequests.requestLeases,\n ).length;\n this.callbacks.logger.info(\n { inFlightCount },\n \"Shutting down, waiting for in-flight requests\",\n );\n this._shutdownRequested = true;\n\n if (this._activeConnection?.ws.readyState === WebSocket.OPEN) {\n this._activeConnection.ws.send(\n ensureUnsharedArrayBuffer(\n ConnectMessage.encode(\n ConnectMessage.create({\n kind: GatewayMessageType.WORKER_PAUSE,\n }),\n ).finish(),\n ),\n );\n this.callbacks.logger.info(\n { connectionId: this._activeConnection.id },\n \"Sent WORKER_PAUSE, draining\",\n );\n }\n\n this.wake();\n\n if (this.loopPromise) {\n await this.loopPromise;\n }\n\n this.callbacks.logger.info(\"Connection closed\");\n }\n\n async getApiBaseUrl(): Promise<string> {\n return resolveApiBaseUrl({\n apiBaseUrl: this.config.apiBaseUrl,\n mode: this.config.mode,\n });\n }\n\n // ---------------------------------------------------------------------------\n // Wake signal\n // ---------------------------------------------------------------------------\n\n private resetWakeSignal(): void {\n let resolve: () => void;\n const promise = new Promise<void>((r) => {\n resolve = r;\n });\n this.wakeSignal = { promise, resolve: resolve! };\n }\n\n private wake(): void {\n this.wakeSignal.resolve();\n this.resetWakeSignal();\n }\n\n // ---------------------------------------------------------------------------\n // Signing key management\n // ---------------------------------------------------------------------------\n\n private switchAuthKey(): void {\n const switchToFallback =\n this.useSigningKey === this.config.hashedSigningKey;\n if (switchToFallback) {\n this.callbacks.logger.debug(\"Switching to fallback signing key\");\n }\n this.useSigningKey = switchToFallback\n ? this.config.hashedFallbackKey\n : this.config.hashedSigningKey;\n }\n\n // ---------------------------------------------------------------------------\n // In-flight helpers\n // ---------------------------------------------------------------------------\n\n private hasInFlightRequests(): boolean {\n return Object.keys(this._inProgressRequests.requestLeases).length > 0;\n }\n\n // ---------------------------------------------------------------------------\n // Reconcile loop\n // ---------------------------------------------------------------------------\n\n private async reconcileLoop(initialAttempt: number): Promise<void> {\n let attempt = initialAttempt;\n\n while (true) {\n // Exit condition: shutdown requested + no in-flight requests\n if (this._shutdownRequested && !this.hasInFlightRequests()) {\n break;\n }\n\n // Ensure we have a live connection\n if (!this._activeConnection || this._activeConnection.dead) {\n this.callbacks.logger.debug(\n {\n hasActiveConnection: !!this._activeConnection,\n activeConnectionDead: this._activeConnection?.dead,\n hasDrainingConnection: !!this._drainingConnection,\n drainingConnectionId: this._drainingConnection?.id,\n },\n \"No active connection\",\n );\n\n if (this.hasConnectedBefore) {\n this.callbacks.logger.info({ attempt }, \"Reconnecting\");\n } else {\n this.callbacks.logger.info(\"Connecting\");\n }\n\n this.callbacks.onStateChange(\n this.hasConnectedBefore\n ? ConnectionState.RECONNECTING\n : ConnectionState.CONNECTING,\n );\n\n try {\n const { conn, gatewayGroup } = await establishConnection(\n this.config,\n this.useSigningKey,\n attempt,\n this.excludeGateways,\n this.callbacks.logger,\n );\n\n // Attach post-handshake handlers\n this.attachHandlers(conn, gatewayGroup);\n\n // Clean up draining connection after new one is ready\n if (this._drainingConnection) {\n this.callbacks.logger.info(\n {\n oldConnectionId: this._drainingConnection.id,\n newConnectionId: conn.id,\n },\n \"Replaced draining connection\",\n );\n this._drainingConnection.close();\n this._drainingConnection = undefined;\n }\n\n this._activeConnection = conn;\n this.heartbeatManager.updateInterval(conn.heartbeatIntervalMs);\n this.statusReporter.updateInterval(conn.statusIntervalMs);\n attempt = 0;\n this.hasConnectedBefore = true;\n this.callbacks.logger.info(\n { connectionId: conn.id, gatewayGroup },\n \"Connection active\",\n );\n this.callbacks.onStateChange(ConnectionState.ACTIVE);\n\n if (this._shutdownRequested) {\n // Reconnected during shutdown to keep in-flight requests alive.\n // Send WORKER_PAUSE instead of WORKER_READY so no new work is routed.\n conn.ws.send(\n ensureUnsharedArrayBuffer(\n ConnectMessage.encode(\n ConnectMessage.create({\n kind: GatewayMessageType.WORKER_PAUSE,\n }),\n ).finish(),\n ),\n );\n this.callbacks.logger.info(\n { connectionId: conn.id },\n \"Sent WORKER_PAUSE on reconnect during shutdown\",\n );\n } else {\n // Signal the gateway that we're ready to receive requests.\n // This must happen after ACTIVE so the gateway doesn't route\n // requests before handlers are fully attached.\n conn.ws.send(\n ensureUnsharedArrayBuffer(\n ConnectMessage.encode(\n ConnectMessage.create({\n kind: GatewayMessageType.WORKER_READY,\n }),\n ).finish(),\n ),\n );\n }\n\n // Flush any buffered responses via HTTP now that we're active.\n this.callbacks.onConnectionActive?.(this.useSigningKey);\n\n this.resolveFirstReady?.();\n this.resolveFirstReady = undefined;\n this.rejectFirstReady = undefined;\n } catch (err) {\n if (!(err instanceof ReconnectError)) throw err;\n\n attempt = err.attempt + 1;\n if (err instanceof AuthError) this.switchAuthKey();\n if (err instanceof ConnectionLimitError) {\n this.callbacks.logger.error(\"Max concurrent connections reached\");\n }\n\n // Gateway is draining, we should retry much faster\n if (err.message?.includes(\"connect_gateway_closing\")) {\n const jitter = 500 + Math.random() * 1000;\n this.callbacks.logger.info(\n { attempt, delay: Math.round(jitter), error: err.message },\n \"Gateway draining, retrying\",\n );\n const cancelled = await waitWithCancel(jitter, () => {\n return this._shutdownRequested && !this.hasInFlightRequests();\n });\n if (cancelled) break;\n continue;\n }\n\n const delay = expBackoff(attempt);\n this.callbacks.logger.info(\n { attempt, delay },\n \"Reconnecting after failure\",\n );\n\n const cancelled = await waitWithCancel(delay, () => {\n return this._shutdownRequested && !this.hasInFlightRequests();\n });\n if (cancelled) break;\n continue;\n }\n }\n\n // Wait for something to change\n await this.wakeSignal.promise;\n this.callbacks.logger.debug(\n {\n shutdownRequested: this._shutdownRequested,\n hasActiveConnection: !!this._activeConnection,\n activeConnectionDead: this._activeConnection?.dead,\n },\n \"Reconcile loop woken\",\n );\n }\n\n // Teardown\n this.heartbeatManager.stop();\n this.statusReporter.stop();\n this._activeConnection?.close();\n this._activeConnection = undefined;\n this._drainingConnection?.close();\n this._drainingConnection = undefined;\n }\n\n // ---------------------------------------------------------------------------\n // Post-handshake handler attachment\n // ---------------------------------------------------------------------------\n\n /**\n * Wire up error, close, and message handlers on a newly-handshaked connection.\n */\n private attachHandlers(conn: Connection, gatewayGroup: string): void {\n const { ws } = conn;\n const connectionId = conn.id;\n\n // Error/close handlers: mark connection as dead and wake the loop\n ws.onerror = (ev) => {\n if (conn.dead) return;\n const uptimeMs = Date.now() - conn.connectedAt;\n this.callbacks.logger.warn(\n {\n connectionId,\n gatewayGroup,\n uptimeMs,\n error: (ev as ErrorEvent)?.message,\n },\n \"Connection lost (error)\",\n );\n conn.dead = true;\n this.excludeGateways.add(gatewayGroup);\n if (this._activeConnection?.id === connectionId) {\n this._activeConnection = undefined;\n }\n this.wake();\n };\n\n ws.onclose = (ev) => {\n if (conn.dead) return;\n const uptimeMs = Date.now() - conn.connectedAt;\n this.callbacks.logger.warn(\n {\n connectionId,\n gatewayGroup,\n uptimeMs,\n code: ev.code,\n reason: ev.reason,\n },\n \"Connection lost (close)\",\n );\n conn.dead = true;\n this.excludeGateways.add(gatewayGroup);\n if (this._activeConnection?.id === connectionId) {\n this._activeConnection = undefined;\n }\n this.wake();\n };\n\n // Message handler for post-handshake messages\n ws.onmessage = async (event) => {\n this._lastMessageReceivedAt = Date.now();\n\n const messageBytes = new Uint8Array(event.data as ArrayBuffer);\n const connectMessage = parseConnectMessage(messageBytes);\n\n if (connectMessage.kind === GatewayMessageType.GATEWAY_CLOSING) {\n const uptimeMs = Date.now() - conn.connectedAt;\n this.callbacks.logger.info(\n { connectionId: conn.id, gatewayGroup, uptimeMs },\n \"Gateway draining, opening new connection\",\n );\n // Move current connection to draining, clear active so the loop\n // establishes a replacement.\n this._drainingConnection = this._activeConnection;\n this._activeConnection = undefined;\n this.wake();\n return;\n }\n\n if (connectMessage.kind === GatewayMessageType.GATEWAY_HEARTBEAT) {\n this._lastHeartbeatReceivedAt = Date.now();\n conn.pendingHeartbeats = 0;\n this.callbacks.logger.debug(\n { connectionId },\n \"Handled gateway heartbeat\",\n );\n return;\n }\n\n if (connectMessage.kind === GatewayMessageType.GATEWAY_EXECUTOR_REQUEST) {\n await this.requestProcessor.handleExecutorRequest(connectMessage, conn);\n return;\n }\n\n if (connectMessage.kind === GatewayMessageType.WORKER_REPLY_ACK) {\n this.requestProcessor.handleReplyAck(connectMessage, connectionId);\n return;\n }\n\n if (\n connectMessage.kind ===\n GatewayMessageType.WORKER_REQUEST_EXTEND_LEASE_ACK\n ) {\n this.requestProcessor.handleExtendLeaseAck(\n connectMessage,\n connectionId,\n );\n return;\n }\n\n this.callbacks.logger.warn(\n {\n kind: gatewayMessageTypeToJSON(connectMessage.kind),\n rawKind: connectMessage.kind,\n state: this.callbacks.getState(),\n connectionId,\n },\n \"Unexpected message type\",\n );\n };\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkGA,IAAa,iBAAb,MAA4B;CAC1B,AAAQ;CACR,AAAQ;CAGR,AAAQ;CACR,AAAQ;CACR,AAAQ,qBAAqB;CAC7B,AAAQ,sBAIJ;EACF,IAAI,IAAIA,kCAAW;EACnB,eAAe,EAAE;EACjB,aAAa,EAAE;EAChB;CAED,AAAQ;CACR,AAAQ;CACR,AAAQ;CAER,AAAQ,kCAA+B,IAAI,KAAK;CAGhD,AAAQ;CAIR,AAAQ,qBAAqB;CAG7B,AAAQ;CAGR,AAAQ;CACR,AAAQ;CAGR,AAAQ;CAGR,AAAiB;CACjB,AAAiB;CACjB,AAAiB;CAEjB,YACE,QACA,WACA;AACA,OAAK,SAAS;AACd,OAAK,YAAY;AACjB,OAAK,gBAAgB,OAAO;EAG5B,IAAIC;AAIJ,OAAK,aAAa;GAAE,SAHJ,IAAI,SAAe,MAAM;AACvC,cAAU;KACV;GACoC;GAAU;EAGhD,MAAM,WAAW;GACf,IAAI,mBAAmB;AACrB,WAAO,KAAK;;GAEd,IAAI,qBAAqB;AACvB,WAAO,KAAK;;GAEd,IAAI,oBAAoB;AACtB,WAAO,KAAK;;GAEd,IAAI,qBAAqB;AACvB,WAAO,KAAK;;GAEd,IAAI,SAAS;AACX,WAAO,KAAK,OAAO;;GAEtB;EAED,MAAM,gBAAgB,EAAE,YAAY,KAAK,MAAM,EAAE;EAEjD,MAAM,OAAO;AAEb,OAAK,mBAAmB,IAAIC,mCAC1B,UACA,eACA,UAAU,OACX;AACD,OAAK,iBAAiB,wBAAwB;AAC5C,QAAK,uBAAuB,KAAK,KAAK;;AAGxC,OAAK,iBAAiB,IAAIC,sCAAe,UAAU,UAAU,OAAO;AAEpE,OAAK,mBAAmB,IAAIC,0CAC1B,UACA,eACA,WACA,UAAU,OACX;;CAGH,IAAI,eAAmC;AACrC,SAAO,KAAK,mBAAmB;;;;;CAMjC,MAAM,oBAAmC;AACvC,QAAM,KAAK,oBAAoB,GAAG,MAAM;;;;;CAM1C,gBAAmC;AACjC,SAAO;GACL,OAAO,KAAK,UAAU,UAAU;GAChC,oBAAoB,KAAK,mBAAmB;GAC5C,sBAAsB,KAAK,qBAAqB;GAChD,qBAAqB,KAAK;GAC1B,yBAAyB,KAAK;GAC9B,uBAAuB,KAAK;GAC5B,mBAAmB,KAAK;GACxB,sBAAsB,OAAO,KAAK,KAAK,oBAAoB,cAAc,CACtE;GACH,kBAAkB,OAAO,OAAO,KAAK,oBAAoB,YAAY;GACtE;;;;;;CAOH,MAAM,MAAM,UAAU,GAAkB;AACtC,MAAI,OAAO,cAAc,YACvB,OAAM,IAAI,MAAM,kDAAkD;AAIpE,MADc,KAAK,UAAU,UAAU,KACzBC,8BAAgB,OAC5B,OAAM,IAAI,MAAM,4BAA4B;EAG9C,MAAM,oBAAoB,IAAI,SAAe,SAAS,WAAW;AAC/D,QAAK,oBAAoB;AACzB,QAAK,mBAAmB;IACxB;AAEF,OAAK,cAAc,KAAK,cAAc,QAAQ;AAG9C,OAAK,YAAY,OAAO,QAAQ;AAC9B,QAAK,mBAAmB,IAAI;IAC5B;AAEF,QAAM;;;;;;CAOR,MAAM,QAAuB;EAC3B,MAAM,gBAAgB,OAAO,KAC3B,KAAK,oBAAoB,cAC1B,CAAC;AACF,OAAK,UAAU,OAAO,KACpB,EAAE,eAAe,EACjB,gDACD;AACD,OAAK,qBAAqB;AAE1B,MAAI,KAAK,mBAAmB,GAAG,eAAe,UAAU,MAAM;AAC5D,QAAK,kBAAkB,GAAG,KACxBC,yCACEC,+BAAe,OACbA,+BAAe,OAAO,EACpB,MAAMC,mCAAmB,cAC1B,CAAC,CACH,CAAC,QAAQ,CACX,CACF;AACD,QAAK,UAAU,OAAO,KACpB,EAAE,cAAc,KAAK,kBAAkB,IAAI,EAC3C,8BACD;;AAGH,OAAK,MAAM;AAEX,MAAI,KAAK,YACP,OAAM,KAAK;AAGb,OAAK,UAAU,OAAO,KAAK,oBAAoB;;CAGjD,MAAM,gBAAiC;AACrC,SAAOC,8BAAkB;GACvB,YAAY,KAAK,OAAO;GACxB,MAAM,KAAK,OAAO;GACnB,CAAC;;CAOJ,AAAQ,kBAAwB;EAC9B,IAAIR;AAIJ,OAAK,aAAa;GAAE,SAHJ,IAAI,SAAe,MAAM;AACvC,cAAU;KACV;GACoC;GAAU;;CAGlD,AAAQ,OAAa;AACnB,OAAK,WAAW,SAAS;AACzB,OAAK,iBAAiB;;CAOxB,AAAQ,gBAAsB;EAC5B,MAAM,mBACJ,KAAK,kBAAkB,KAAK,OAAO;AACrC,MAAI,iBACF,MAAK,UAAU,OAAO,MAAM,oCAAoC;AAElE,OAAK,gBAAgB,mBACjB,KAAK,OAAO,oBACZ,KAAK,OAAO;;CAOlB,AAAQ,sBAA+B;AACrC,SAAO,OAAO,KAAK,KAAK,oBAAoB,cAAc,CAAC,SAAS;;CAOtE,MAAc,cAAc,gBAAuC;EACjE,IAAI,UAAU;AAEd,SAAO,MAAM;AAEX,OAAI,KAAK,sBAAsB,CAAC,KAAK,qBAAqB,CACxD;AAIF,OAAI,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,MAAM;AAC1D,SAAK,UAAU,OAAO,MACpB;KACE,qBAAqB,CAAC,CAAC,KAAK;KAC5B,sBAAsB,KAAK,mBAAmB;KAC9C,uBAAuB,CAAC,CAAC,KAAK;KAC9B,sBAAsB,KAAK,qBAAqB;KACjD,EACD,uBACD;AAED,QAAI,KAAK,mBACP,MAAK,UAAU,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe;QAEvD,MAAK,UAAU,OAAO,KAAK,aAAa;AAG1C,SAAK,UAAU,cACb,KAAK,qBACDI,8BAAgB,eAChBA,8BAAgB,WACrB;AAED,QAAI;KACF,MAAM,EAAE,MAAM,iBAAiB,MAAMK,sCACnC,KAAK,QACL,KAAK,eACL,SACA,KAAK,iBACL,KAAK,UAAU,OAChB;AAGD,UAAK,eAAe,MAAM,aAAa;AAGvC,SAAI,KAAK,qBAAqB;AAC5B,WAAK,UAAU,OAAO,KACpB;OACE,iBAAiB,KAAK,oBAAoB;OAC1C,iBAAiB,KAAK;OACvB,EACD,+BACD;AACD,WAAK,oBAAoB,OAAO;AAChC,WAAK,sBAAsB;;AAG7B,UAAK,oBAAoB;AACzB,UAAK,iBAAiB,eAAe,KAAK,oBAAoB;AAC9D,UAAK,eAAe,eAAe,KAAK,iBAAiB;AACzD,eAAU;AACV,UAAK,qBAAqB;AAC1B,UAAK,UAAU,OAAO,KACpB;MAAE,cAAc,KAAK;MAAI;MAAc,EACvC,oBACD;AACD,UAAK,UAAU,cAAcL,8BAAgB,OAAO;AAEpD,SAAI,KAAK,oBAAoB;AAG3B,WAAK,GAAG,KACNC,yCACEC,+BAAe,OACbA,+BAAe,OAAO,EACpB,MAAMC,mCAAmB,cAC1B,CAAC,CACH,CAAC,QAAQ,CACX,CACF;AACD,WAAK,UAAU,OAAO,KACpB,EAAE,cAAc,KAAK,IAAI,EACzB,iDACD;WAKD,MAAK,GAAG,KACNF,yCACEC,+BAAe,OACbA,+BAAe,OAAO,EACpB,MAAMC,mCAAmB,cAC1B,CAAC,CACH,CAAC,QAAQ,CACX,CACF;AAIH,UAAK,UAAU,qBAAqB,KAAK,cAAc;AAEvD,UAAK,qBAAqB;AAC1B,UAAK,oBAAoB;AACzB,UAAK,mBAAmB;aACjB,KAAK;AACZ,SAAI,EAAE,eAAeG,6BAAiB,OAAM;AAE5C,eAAU,IAAI,UAAU;AACxB,SAAI,eAAeC,uBAAW,MAAK,eAAe;AAClD,SAAI,eAAeC,kCACjB,MAAK,UAAU,OAAO,MAAM,qCAAqC;AAInE,SAAI,IAAI,SAAS,SAAS,0BAA0B,EAAE;MACpD,MAAM,SAAS,MAAM,KAAK,QAAQ,GAAG;AACrC,WAAK,UAAU,OAAO,KACpB;OAAE;OAAS,OAAO,KAAK,MAAM,OAAO;OAAE,OAAO,IAAI;OAAS,EAC1D,6BACD;AAID,UAHkB,MAAMC,4BAAe,cAAc;AACnD,cAAO,KAAK,sBAAsB,CAAC,KAAK,qBAAqB;QAC7D,CACa;AACf;;KAGF,MAAM,QAAQC,wBAAW,QAAQ;AACjC,UAAK,UAAU,OAAO,KACpB;MAAE;MAAS;MAAO,EAClB,6BACD;AAKD,SAHkB,MAAMD,4BAAe,aAAa;AAClD,aAAO,KAAK,sBAAsB,CAAC,KAAK,qBAAqB;OAC7D,CACa;AACf;;;AAKJ,SAAM,KAAK,WAAW;AACtB,QAAK,UAAU,OAAO,MACpB;IACE,mBAAmB,KAAK;IACxB,qBAAqB,CAAC,CAAC,KAAK;IAC5B,sBAAsB,KAAK,mBAAmB;IAC/C,EACD,uBACD;;AAIH,OAAK,iBAAiB,MAAM;AAC5B,OAAK,eAAe,MAAM;AAC1B,OAAK,mBAAmB,OAAO;AAC/B,OAAK,oBAAoB;AACzB,OAAK,qBAAqB,OAAO;AACjC,OAAK,sBAAsB;;;;;CAU7B,AAAQ,eAAe,MAAkB,cAA4B;EACnE,MAAM,EAAE,OAAO;EACf,MAAM,eAAe,KAAK;AAG1B,KAAG,WAAW,OAAO;AACnB,OAAI,KAAK,KAAM;GACf,MAAM,WAAW,KAAK,KAAK,GAAG,KAAK;AACnC,QAAK,UAAU,OAAO,KACpB;IACE;IACA;IACA;IACA,OAAQ,IAAmB;IAC5B,EACD,0BACD;AACD,QAAK,OAAO;AACZ,QAAK,gBAAgB,IAAI,aAAa;AACtC,OAAI,KAAK,mBAAmB,OAAO,aACjC,MAAK,oBAAoB;AAE3B,QAAK,MAAM;;AAGb,KAAG,WAAW,OAAO;AACnB,OAAI,KAAK,KAAM;GACf,MAAM,WAAW,KAAK,KAAK,GAAG,KAAK;AACnC,QAAK,UAAU,OAAO,KACpB;IACE;IACA;IACA;IACA,MAAM,GAAG;IACT,QAAQ,GAAG;IACZ,EACD,0BACD;AACD,QAAK,OAAO;AACZ,QAAK,gBAAgB,IAAI,aAAa;AACtC,OAAI,KAAK,mBAAmB,OAAO,aACjC,MAAK,oBAAoB;AAE3B,QAAK,MAAM;;AAIb,KAAG,YAAY,OAAO,UAAU;AAC9B,QAAK,yBAAyB,KAAK,KAAK;GAGxC,MAAM,iBAAiBE,qCADF,IAAI,WAAW,MAAM,KAAoB,CACN;AAExD,OAAI,eAAe,SAASR,mCAAmB,iBAAiB;IAC9D,MAAM,WAAW,KAAK,KAAK,GAAG,KAAK;AACnC,SAAK,UAAU,OAAO,KACpB;KAAE,cAAc,KAAK;KAAI;KAAc;KAAU,EACjD,2CACD;AAGD,SAAK,sBAAsB,KAAK;AAChC,SAAK,oBAAoB;AACzB,SAAK,MAAM;AACX;;AAGF,OAAI,eAAe,SAASA,mCAAmB,mBAAmB;AAChE,SAAK,2BAA2B,KAAK,KAAK;AAC1C,SAAK,oBAAoB;AACzB,SAAK,UAAU,OAAO,MACpB,EAAE,cAAc,EAChB,4BACD;AACD;;AAGF,OAAI,eAAe,SAASA,mCAAmB,0BAA0B;AACvE,UAAM,KAAK,iBAAiB,sBAAsB,gBAAgB,KAAK;AACvE;;AAGF,OAAI,eAAe,SAASA,mCAAmB,kBAAkB;AAC/D,SAAK,iBAAiB,eAAe,gBAAgB,aAAa;AAClE;;AAGF,OACE,eAAe,SACfA,mCAAmB,iCACnB;AACA,SAAK,iBAAiB,qBACpB,gBACA,aACD;AACD;;AAGF,QAAK,UAAU,OAAO,KACpB;IACE,MAAMS,yCAAyB,eAAe,KAAK;IACnD,SAAS,eAAe;IACxB,OAAO,KAAK,UAAU,UAAU;IAChC;IACD,EACD,0BACD"}
|
|
@@ -5,7 +5,6 @@ import { ConnectionState } from "../../types.js";
|
|
|
5
5
|
import { resolveApiBaseUrl } from "../../../../helpers/url.js";
|
|
6
6
|
import { parseConnectMessage } from "../../messages.js";
|
|
7
7
|
import { establishConnection } from "./handshake.js";
|
|
8
|
-
import { WAKE_REASON } from "./types.js";
|
|
9
8
|
import { HeartbeatManager } from "./heartbeat.js";
|
|
10
9
|
import { RequestProcessor } from "./requestProcessor.js";
|
|
11
10
|
import { StatusReporter } from "./statusReporter.js";
|
|
@@ -35,7 +34,7 @@ import { WaitGroup } from "@jpwilliams/waitgroup";
|
|
|
35
34
|
* - Manages a single heartbeat interval targeting the active connection
|
|
36
35
|
* - Handles reconnection, drain, and shutdown as state changes
|
|
37
36
|
*/
|
|
38
|
-
var ConnectionCore = class
|
|
37
|
+
var ConnectionCore = class {
|
|
39
38
|
config;
|
|
40
39
|
callbacks;
|
|
41
40
|
_activeConnection;
|
|
@@ -51,10 +50,7 @@ var ConnectionCore = class ConnectionCore {
|
|
|
51
50
|
_lastMessageReceivedAt;
|
|
52
51
|
excludeGateways = /* @__PURE__ */ new Set();
|
|
53
52
|
wakeSignal;
|
|
54
|
-
pendingWakeReasons = [];
|
|
55
53
|
hasConnectedBefore = false;
|
|
56
|
-
shutdownDumpInterval;
|
|
57
|
-
static SHUTDOWN_DUMP_INTERVAL_MS = 6e4;
|
|
58
54
|
loopPromise;
|
|
59
55
|
resolveFirstReady;
|
|
60
56
|
rejectFirstReady;
|
|
@@ -90,7 +86,7 @@ var ConnectionCore = class ConnectionCore {
|
|
|
90
86
|
return self.config.appIds;
|
|
91
87
|
}
|
|
92
88
|
};
|
|
93
|
-
const wakeSignalRef = { wake: (
|
|
89
|
+
const wakeSignalRef = { wake: () => this.wake() };
|
|
94
90
|
const self = this;
|
|
95
91
|
this.heartbeatManager = new HeartbeatManager(accessor, wakeSignalRef, callbacks.logger);
|
|
96
92
|
this.heartbeatManager.onHeartbeatSent = () => {
|
|
@@ -149,13 +145,11 @@ var ConnectionCore = class ConnectionCore {
|
|
|
149
145
|
const inFlightCount = Object.keys(this._inProgressRequests.requestLeases).length;
|
|
150
146
|
this.callbacks.logger.info({ inFlightCount }, "Shutting down, waiting for in-flight requests");
|
|
151
147
|
this._shutdownRequested = true;
|
|
152
|
-
this.dumpInFlightForShutdown("drain-start");
|
|
153
|
-
this.startShutdownInFlightDumpTimer();
|
|
154
148
|
if (this._activeConnection?.ws.readyState === WebSocket.OPEN) {
|
|
155
149
|
this._activeConnection.ws.send(ensureUnsharedArrayBuffer(ConnectMessage.encode(ConnectMessage.create({ kind: GatewayMessageType.WORKER_PAUSE })).finish()));
|
|
156
150
|
this.callbacks.logger.info({ connectionId: this._activeConnection.id }, "Sent WORKER_PAUSE, draining");
|
|
157
151
|
}
|
|
158
|
-
this.wake(
|
|
152
|
+
this.wake();
|
|
159
153
|
if (this.loopPromise) await this.loopPromise;
|
|
160
154
|
this.callbacks.logger.info("Connection closed");
|
|
161
155
|
}
|
|
@@ -174,8 +168,7 @@ var ConnectionCore = class ConnectionCore {
|
|
|
174
168
|
resolve
|
|
175
169
|
};
|
|
176
170
|
}
|
|
177
|
-
wake(
|
|
178
|
-
this.pendingWakeReasons.push(reason);
|
|
171
|
+
wake() {
|
|
179
172
|
this.wakeSignal.resolve();
|
|
180
173
|
this.resetWakeSignal();
|
|
181
174
|
}
|
|
@@ -187,63 +180,8 @@ var ConnectionCore = class ConnectionCore {
|
|
|
187
180
|
hasInFlightRequests() {
|
|
188
181
|
return Object.keys(this._inProgressRequests.requestLeases).length > 0;
|
|
189
182
|
}
|
|
190
|
-
/**
|
|
191
|
-
* Debug-level "still draining" dump emitted at drain start and periodically
|
|
192
|
-
* thereafter while in-flight requests are holding the shutdown. One summary
|
|
193
|
-
* line plus one line per request carrying `requestId`, `runId`, `stepId`,
|
|
194
|
-
* `functionSlug`, `ageMs`, and `sinceLastLeaseExtendMs`. Does not affect
|
|
195
|
-
* info/warn logs.
|
|
196
|
-
*
|
|
197
|
-
* `requestLeases` drives the reconcile-loop exit gate, so use it as the
|
|
198
|
-
* single source of truth for the in-flight set; `requestMeta` carries the
|
|
199
|
-
* enrichment fields and is kept in sync alongside the lease map.
|
|
200
|
-
*/
|
|
201
|
-
dumpInFlightForShutdown(reason) {
|
|
202
|
-
const leaseIds = Object.keys(this._inProgressRequests.requestLeases);
|
|
203
|
-
if (leaseIds.length === 0) return;
|
|
204
|
-
const now = Date.now();
|
|
205
|
-
const ages = [];
|
|
206
|
-
for (const id of leaseIds) {
|
|
207
|
-
const m = this._inProgressRequests.requestMeta[id];
|
|
208
|
-
if (m?.leaseAcquiredAt) ages.push(now - m.leaseAcquiredAt);
|
|
209
|
-
}
|
|
210
|
-
this.callbacks.logger.debug({
|
|
211
|
-
reason,
|
|
212
|
-
inFlightCount: leaseIds.length,
|
|
213
|
-
oldestAgeMs: ages.length > 0 ? Math.max(...ages) : void 0
|
|
214
|
-
}, "Shutdown: still draining");
|
|
215
|
-
for (const id of leaseIds) {
|
|
216
|
-
const m = this._inProgressRequests.requestMeta[id];
|
|
217
|
-
if (!m) continue;
|
|
218
|
-
this.callbacks.logger.debug({
|
|
219
|
-
reason,
|
|
220
|
-
requestId: m.requestId,
|
|
221
|
-
runId: m.runId,
|
|
222
|
-
stepId: m.stepId,
|
|
223
|
-
functionSlug: m.functionSlug,
|
|
224
|
-
appId: m.appId,
|
|
225
|
-
ageMs: m.leaseAcquiredAt ? now - m.leaseAcquiredAt : void 0,
|
|
226
|
-
sinceLastLeaseExtendMs: m.leaseLastExtendedAt ? now - m.leaseLastExtendedAt : void 0
|
|
227
|
-
}, "Shutdown: still draining in-flight request");
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
startShutdownInFlightDumpTimer() {
|
|
231
|
-
if (this.shutdownDumpInterval) return;
|
|
232
|
-
this.shutdownDumpInterval = setInterval(() => {
|
|
233
|
-
if (!this._shutdownRequested) return;
|
|
234
|
-
this.dumpInFlightForShutdown("periodic");
|
|
235
|
-
this.wake(WAKE_REASON.ShutdownStillPending);
|
|
236
|
-
}, ConnectionCore.SHUTDOWN_DUMP_INTERVAL_MS);
|
|
237
|
-
}
|
|
238
|
-
stopShutdownInFlightDumpTimer() {
|
|
239
|
-
if (this.shutdownDumpInterval) {
|
|
240
|
-
clearInterval(this.shutdownDumpInterval);
|
|
241
|
-
this.shutdownDumpInterval = void 0;
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
183
|
async reconcileLoop(initialAttempt) {
|
|
245
184
|
let attempt = initialAttempt;
|
|
246
|
-
this.callbacks.logger.debug({ initialAttempt }, "Reconcile loop entered");
|
|
247
185
|
while (true) {
|
|
248
186
|
if (this._shutdownRequested && !this.hasInFlightRequests()) break;
|
|
249
187
|
if (!this._activeConnection || this._activeConnection.dead) {
|
|
@@ -280,10 +218,7 @@ var ConnectionCore = class ConnectionCore {
|
|
|
280
218
|
if (this._shutdownRequested) {
|
|
281
219
|
conn.ws.send(ensureUnsharedArrayBuffer(ConnectMessage.encode(ConnectMessage.create({ kind: GatewayMessageType.WORKER_PAUSE })).finish()));
|
|
282
220
|
this.callbacks.logger.info({ connectionId: conn.id }, "Sent WORKER_PAUSE on reconnect during shutdown");
|
|
283
|
-
} else {
|
|
284
|
-
conn.ws.send(ensureUnsharedArrayBuffer(ConnectMessage.encode(ConnectMessage.create({ kind: GatewayMessageType.WORKER_READY })).finish()));
|
|
285
|
-
this.callbacks.logger.info({ connectionId: conn.id }, "Sent WORKER_READY");
|
|
286
|
-
}
|
|
221
|
+
} else conn.ws.send(ensureUnsharedArrayBuffer(ConnectMessage.encode(ConnectMessage.create({ kind: GatewayMessageType.WORKER_READY })).finish()));
|
|
287
222
|
this.callbacks.onConnectionActive?.(this.useSigningKey);
|
|
288
223
|
this.resolveFirstReady?.();
|
|
289
224
|
this.resolveFirstReady = void 0;
|
|
@@ -317,22 +252,14 @@ var ConnectionCore = class ConnectionCore {
|
|
|
317
252
|
}
|
|
318
253
|
}
|
|
319
254
|
await this.wakeSignal.promise;
|
|
320
|
-
const reasons = this.pendingWakeReasons;
|
|
321
|
-
this.pendingWakeReasons = [];
|
|
322
255
|
this.callbacks.logger.debug({
|
|
323
|
-
reasons,
|
|
324
256
|
shutdownRequested: this._shutdownRequested,
|
|
325
257
|
hasActiveConnection: !!this._activeConnection,
|
|
326
258
|
activeConnectionDead: this._activeConnection?.dead
|
|
327
259
|
}, "Reconcile loop woken");
|
|
328
260
|
}
|
|
329
|
-
this.callbacks.logger.debug({
|
|
330
|
-
shutdownRequested: this._shutdownRequested,
|
|
331
|
-
inFlightCount: Object.keys(this._inProgressRequests.requestLeases).length
|
|
332
|
-
}, "Reconcile loop exiting");
|
|
333
261
|
this.heartbeatManager.stop();
|
|
334
262
|
this.statusReporter.stop();
|
|
335
|
-
this.stopShutdownInFlightDumpTimer();
|
|
336
263
|
this._activeConnection?.close();
|
|
337
264
|
this._activeConnection = void 0;
|
|
338
265
|
this._drainingConnection?.close();
|
|
@@ -356,7 +283,7 @@ var ConnectionCore = class ConnectionCore {
|
|
|
356
283
|
conn.dead = true;
|
|
357
284
|
this.excludeGateways.add(gatewayGroup);
|
|
358
285
|
if (this._activeConnection?.id === connectionId) this._activeConnection = void 0;
|
|
359
|
-
this.wake(
|
|
286
|
+
this.wake();
|
|
360
287
|
};
|
|
361
288
|
ws.onclose = (ev) => {
|
|
362
289
|
if (conn.dead) return;
|
|
@@ -371,7 +298,7 @@ var ConnectionCore = class ConnectionCore {
|
|
|
371
298
|
conn.dead = true;
|
|
372
299
|
this.excludeGateways.add(gatewayGroup);
|
|
373
300
|
if (this._activeConnection?.id === connectionId) this._activeConnection = void 0;
|
|
374
|
-
this.wake(
|
|
301
|
+
this.wake();
|
|
375
302
|
};
|
|
376
303
|
ws.onmessage = async (event) => {
|
|
377
304
|
this._lastMessageReceivedAt = Date.now();
|
|
@@ -385,7 +312,7 @@ var ConnectionCore = class ConnectionCore {
|
|
|
385
312
|
}, "Gateway draining, opening new connection");
|
|
386
313
|
this._drainingConnection = this._activeConnection;
|
|
387
314
|
this._activeConnection = void 0;
|
|
388
|
-
this.wake(
|
|
315
|
+
this.wake();
|
|
389
316
|
return;
|
|
390
317
|
}
|
|
391
318
|
if (connectMessage.kind === GatewayMessageType.GATEWAY_HEARTBEAT) {
|