@copilotkit/runtime 1.79.1 → 1.79.2-canary.1792172552

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/dist/lib/runtime/copilot-runtime.cjs +1 -0
  2. package/dist/lib/runtime/copilot-runtime.cjs.map +1 -1
  3. package/dist/lib/runtime/copilot-runtime.d.cts +1 -0
  4. package/dist/lib/runtime/copilot-runtime.d.cts.map +1 -1
  5. package/dist/lib/runtime/copilot-runtime.d.mts +1 -0
  6. package/dist/lib/runtime/copilot-runtime.d.mts.map +1 -1
  7. package/dist/lib/runtime/copilot-runtime.mjs +1 -0
  8. package/dist/lib/runtime/copilot-runtime.mjs.map +1 -1
  9. package/dist/package.cjs +5 -5
  10. package/dist/package.mjs +5 -5
  11. package/dist/v2/index.d.cts +2 -2
  12. package/dist/v2/index.d.mts +2 -2
  13. package/dist/v2/runtime/core/package-info.cjs +1 -1
  14. package/dist/v2/runtime/core/package-info.mjs +1 -1
  15. package/dist/v2/runtime/core/runtime.cjs +2 -1
  16. package/dist/v2/runtime/core/runtime.cjs.map +1 -1
  17. package/dist/v2/runtime/core/runtime.d.cts +3 -0
  18. package/dist/v2/runtime/core/runtime.d.cts.map +1 -1
  19. package/dist/v2/runtime/core/runtime.d.mts +3 -0
  20. package/dist/v2/runtime/core/runtime.d.mts.map +1 -1
  21. package/dist/v2/runtime/core/runtime.mjs +2 -1
  22. package/dist/v2/runtime/core/runtime.mjs.map +1 -1
  23. package/dist/v2/runtime/index.d.cts +1 -1
  24. package/dist/v2/runtime/index.d.mts +1 -1
  25. package/dist/v2/runtime/runner/index.d.cts +1 -1
  26. package/dist/v2/runtime/runner/index.d.mts +1 -1
  27. package/dist/v2/runtime/runner/intelligence.cjs +83 -23
  28. package/dist/v2/runtime/runner/intelligence.cjs.map +1 -1
  29. package/dist/v2/runtime/runner/intelligence.d.cts +31 -2
  30. package/dist/v2/runtime/runner/intelligence.d.cts.map +1 -1
  31. package/dist/v2/runtime/runner/intelligence.d.mts +31 -2
  32. package/dist/v2/runtime/runner/intelligence.d.mts.map +1 -1
  33. package/dist/v2/runtime/runner/intelligence.mjs +84 -24
  34. package/dist/v2/runtime/runner/intelligence.mjs.map +1 -1
  35. package/package.json +9 -9
@@ -148,7 +148,8 @@ var IntelligenceAgentRunner = class extends require_agent_runner.AgentRunner {
148
148
  isRunning: true,
149
149
  stopRequested: false,
150
150
  agent,
151
- currentEvents: [],
151
+ finalizer: (0, _copilotkit_shared.createRunEventFinalizer)(),
152
+ runErrorEmitted: false,
152
153
  nextEventSeq: 1,
153
154
  hasRunStarted: false,
154
155
  hasJoined: false,
@@ -160,6 +161,9 @@ var IntelligenceAgentRunner = class extends require_agent_runner.AgentRunner {
160
161
  resolveCompletion,
161
162
  stopTimer: null,
162
163
  pendingEvents: /* @__PURE__ */ new Map(),
164
+ durableProgressAt: 0,
165
+ lastQueuedEventSeq: 0,
166
+ pendingEventsOutOfOrder: false,
163
167
  activeEventBatch: null,
164
168
  nextEventPushAttempt: 0,
165
169
  eventRetryTimer: null,
@@ -249,7 +253,6 @@ var IntelligenceAgentRunner = class extends require_agent_runner.AgentRunner {
249
253
  code: "CHANNEL_JOIN_ERROR"
250
254
  };
251
255
  observer.next(errorEvent);
252
- state.currentEvents.push(errorEvent);
253
256
  this.removeThread(threadId, state);
254
257
  startupBoundary?.rejectStartup(error);
255
258
  observer.complete();
@@ -263,7 +266,6 @@ var IntelligenceAgentRunner = class extends require_agent_runner.AgentRunner {
263
266
  code: "CHANNEL_JOIN_TIMEOUT"
264
267
  };
265
268
  observer.next(errorEvent);
266
- state.currentEvents.push(errorEvent);
267
269
  this.removeThread(threadId, state);
268
270
  startupBoundary?.rejectStartup(error);
269
271
  observer.complete();
@@ -338,12 +340,12 @@ var IntelligenceAgentRunner = class extends require_agent_runner.AgentRunner {
338
340
  });
339
341
  }
340
342
  async executeAgentRun(request, state, threadId, onRunError) {
341
- const { currentEvents } = state;
342
343
  const pushCanonicalEvent = (event) => {
343
344
  if (!this.isCurrentThreadState(threadId, state)) return;
344
345
  const canonicalEvent = this.stampRunnerMetadata(this.stampCanonicalRunOwnership(this.withoutAgentEventIdentity(event), request), state);
345
- currentEvents.push(canonicalEvent);
346
+ state.finalizer.observe(canonicalEvent);
346
347
  if (canonicalEvent.type === _ag_ui_client.EventType.RUN_STARTED) state.hasRunStarted = true;
348
+ if (canonicalEvent.type === _ag_ui_client.EventType.RUN_ERROR) state.runErrorEmitted = true;
347
349
  this.queueRunnerEvent(this.createRunnerEventPayload(canonicalEvent, request, state), state);
348
350
  if (canonicalEvent.type === _ag_ui_client.EventType.RUN_ERROR) onRunError(canonicalEvent);
349
351
  };
@@ -388,14 +390,14 @@ var IntelligenceAgentRunner = class extends require_agent_runner.AgentRunner {
388
390
  } catch (error) {
389
391
  if (state.stopRequested || !this.isCurrentThreadState(threadId, state)) return;
390
392
  ensureRunStarted();
391
- if (!currentEvents.find((event) => event.type === _ag_ui_client.EventType.RUN_ERROR)) pushCanonicalEvent({
393
+ if (!state.runErrorEmitted) pushCanonicalEvent({
392
394
  type: _ag_ui_client.EventType.RUN_ERROR,
393
395
  message: error instanceof Error ? error.message : String(error)
394
396
  });
395
397
  } finally {
396
398
  if (!this.isCurrentThreadState(threadId, state)) return;
397
399
  ensureRunStarted();
398
- const appended = (0, _copilotkit_shared.finalizeRunEvents)(currentEvents, {
400
+ const appended = state.finalizer.finalize({
399
401
  stopRequested: state.stopRequested,
400
402
  protocolVersion: request.input.protocolVersion
401
403
  });
@@ -404,20 +406,56 @@ var IntelligenceAgentRunner = class extends require_agent_runner.AgentRunner {
404
406
  this.completeWhenDurable(threadId, state);
405
407
  }
406
408
  }
407
- /** Queue one immutable event payload until Redis-backed gateway acknowledgement. */
409
+ /**
410
+ * Queue one immutable event payload until Redis-backed gateway acknowledgement.
411
+ *
412
+ * `pushCanonicalEvent` is the only producer and stamps a fresh, increasing
413
+ * `cpki_event_seq` on every event it queues, so the map's insertion order is
414
+ * sequence order. Batches are therefore read from the front of the map
415
+ * instead of sorting the whole queue on every event.
416
+ */
408
417
  queueRunnerEvent(payload, state) {
409
418
  if (!this.isCurrentThreadState(state.threadId, state)) return;
410
419
  const eventId = this.runnerEventId(payload);
411
- if (!state.pendingEvents.has(eventId)) state.pendingEvents.set(eventId, {
412
- payload: structuredClone(payload),
413
- queuedAt: Date.now()
414
- });
415
- this.scheduleEventDeadline(state);
420
+ if (!state.pendingEvents.has(eventId)) {
421
+ const queuedAt = Date.now();
422
+ const wasEmpty = state.pendingEvents.size === 0;
423
+ const eventSeq = this.runnerEventSeq(payload);
424
+ if (eventSeq < state.lastQueuedEventSeq && !state.pendingEventsOutOfOrder) {
425
+ state.pendingEventsOutOfOrder = true;
426
+ _copilotkit_shared.logger.warn({
427
+ threadId: state.threadId,
428
+ runId: state.runId,
429
+ eventSeq,
430
+ lastQueuedEventSeq: state.lastQueuedEventSeq
431
+ }, "Runner event queued out of sequence order; sorting pending events");
432
+ }
433
+ state.lastQueuedEventSeq = Math.max(state.lastQueuedEventSeq, eventSeq);
434
+ state.pendingEvents.set(eventId, {
435
+ payload: structuredClone(payload),
436
+ queuedAt
437
+ });
438
+ if (wasEmpty) {
439
+ state.durableProgressAt = queuedAt;
440
+ this.scheduleEventDeadline(state);
441
+ }
442
+ }
416
443
  this.replayPendingEvents(state);
417
444
  }
445
+ /** The oldest `limit` pending events, in sequence order. */
446
+ oldestPendingEvents(state, limit) {
447
+ if (state.pendingEventsOutOfOrder) return [...state.pendingEvents.entries()].sort(([, left], [, right]) => this.runnerEventSeq(left.payload) - this.runnerEventSeq(right.payload)).slice(0, limit);
448
+ const oldest = [];
449
+ for (const entry of state.pendingEvents) {
450
+ if (oldest.length >= limit) break;
451
+ oldest.push(entry);
452
+ }
453
+ return oldest;
454
+ }
418
455
  pushPendingEventBatch(eventIds, events, state) {
419
456
  if (!this.isCurrentThreadState(state.threadId, state) || this.failIfEventDeadlineExceeded(state) || state.channel.state !== "joined" || !state.socket.isConnected()) return;
420
457
  const attempt = ++state.nextEventPushAttempt;
458
+ const sentAt = Date.now();
421
459
  state.activeEventBatch = {
422
460
  eventIds,
423
461
  attempt
@@ -428,9 +466,12 @@ var IntelligenceAgentRunner = class extends require_agent_runner.AgentRunner {
428
466
  if (!this.isCurrentThreadState(state.threadId, state) || state.activeEventBatch?.attempt !== attempt) return;
429
467
  if (this.failIfEventDeadlineExceeded(state)) return;
430
468
  for (const eventId of eventIds) state.pendingEvents.delete(eventId);
469
+ state.durableProgressAt = Date.now();
470
+ this.reportEventDelivery(state, eventIds.length, state.durableProgressAt - sentAt);
431
471
  state.activeEventBatch = null;
432
472
  state.eventRetryAttempt = 0;
433
473
  if (state.pendingEvents.size === 0) {
474
+ state.pendingEventsOutOfOrder = false;
434
475
  this.clearPendingEventRetry(state);
435
476
  this.clearPendingEventFlush(state);
436
477
  }
@@ -439,6 +480,27 @@ var IntelligenceAgentRunner = class extends require_agent_runner.AgentRunner {
439
480
  this.replayPendingEvents(state);
440
481
  }).receive("error", (response) => this.handlePendingEventFailure(state, attempt, response)).receive("timeout", () => this.handlePendingEventFailure(state, attempt));
441
482
  }
483
+ reportEventDelivery(state, acknowledgedEvents, ackLatencyMs) {
484
+ const onStats = this.options.onEventDeliveryStats;
485
+ if (!onStats) return;
486
+ const [oldest] = this.oldestPendingEvents(state, 1);
487
+ try {
488
+ onStats({
489
+ threadId: state.threadId,
490
+ runId: state.runId,
491
+ acknowledgedEvents,
492
+ ackLatencyMs,
493
+ pendingEvents: state.pendingEvents.size,
494
+ oldestPendingAgeMs: oldest ? Date.now() - oldest[1].queuedAt : 0
495
+ });
496
+ } catch (error) {
497
+ _copilotkit_shared.logger.warn({
498
+ err: error,
499
+ threadId: state.threadId,
500
+ runId: state.runId
501
+ }, "onEventDeliveryStats callback failed");
502
+ }
503
+ }
442
504
  handlePendingEventFailure(state, attempt, response) {
443
505
  if (!this.isCurrentThreadState(state.threadId, state) || state.activeEventBatch?.attempt !== attempt) return;
444
506
  if (this.failIfEventDeadlineExceeded(state)) return;
@@ -467,8 +529,8 @@ var IntelligenceAgentRunner = class extends require_agent_runner.AgentRunner {
467
529
  }
468
530
  schedulePendingEventFlush(state) {
469
531
  if (!this.isCurrentThreadState(state.threadId, state) || state.pendingEvents.size === 0 || state.activeEventBatch !== null || state.eventFlushTimer !== null) return;
470
- const oldestQueuedAt = Math.min(...[...state.pendingEvents.values()].map((event) => event.queuedAt));
471
- const delay = Math.max(0, oldestQueuedAt + RUNNER_EVENT_BATCH_FLUSH_MS - Date.now());
532
+ const [[, oldest]] = this.oldestPendingEvents(state, 1);
533
+ const delay = Math.max(0, oldest.queuedAt + RUNNER_EVENT_BATCH_FLUSH_MS - Date.now());
472
534
  state.eventFlushTimer = setTimeout(() => {
473
535
  state.eventFlushTimer = null;
474
536
  if (!this.isCurrentThreadState(state.threadId, state)) return;
@@ -489,12 +551,11 @@ var IntelligenceAgentRunner = class extends require_agent_runner.AgentRunner {
489
551
  }
490
552
  replayPendingEvents(state) {
491
553
  if (!this.isCurrentThreadState(state.threadId, state) || state.activeEventBatch !== null) return;
492
- const pendingEvents = [...state.pendingEvents.entries()].sort(([, left], [, right]) => this.runnerEventSeq(left.payload) - this.runnerEventSeq(right.payload));
493
- if (pendingEvents.length === 0) return;
554
+ if (state.pendingEvents.size === 0) return;
494
555
  const batchSize = state.supportsRunnerEventBatch ? MAX_RUNNER_EVENT_BATCH_SIZE : 1;
495
- if (!state.supportsRunnerEventBatch || pendingEvents.length >= batchSize) {
556
+ if (!state.supportsRunnerEventBatch || state.pendingEvents.size >= batchSize) {
496
557
  this.clearPendingEventFlush(state);
497
- const batch = pendingEvents.slice(0, batchSize);
558
+ const batch = this.oldestPendingEvents(state, batchSize);
498
559
  this.pushPendingEventBatch(batch.map(([eventId]) => eventId), batch.map(([, event]) => event), state);
499
560
  return;
500
561
  }
@@ -502,7 +563,7 @@ var IntelligenceAgentRunner = class extends require_agent_runner.AgentRunner {
502
563
  }
503
564
  flushPendingEventBatch(state) {
504
565
  if (!this.isCurrentThreadState(state.threadId, state) || state.activeEventBatch !== null || state.pendingEvents.size === 0) return;
505
- const batch = [...state.pendingEvents.entries()].sort(([, left], [, right]) => this.runnerEventSeq(left.payload) - this.runnerEventSeq(right.payload)).slice(0, MAX_RUNNER_EVENT_BATCH_SIZE);
566
+ const batch = this.oldestPendingEvents(state, MAX_RUNNER_EVENT_BATCH_SIZE);
506
567
  this.pushPendingEventBatch(batch.map(([eventId]) => eventId), batch.map(([, event]) => event), state);
507
568
  }
508
569
  retryActiveEventBatch(state) {
@@ -552,7 +613,7 @@ var IntelligenceAgentRunner = class extends require_agent_runner.AgentRunner {
552
613
  state.eventDeadlineTimer = null;
553
614
  }
554
615
  if (!this.isCurrentThreadState(state.threadId, state) || state.pendingEvents.size === 0) return;
555
- const deadline = Math.min(...[...state.pendingEvents.values()].map((event) => event.queuedAt)) + EVENT_DURABILITY_DEADLINE_MS;
616
+ const deadline = state.durableProgressAt + EVENT_DURABILITY_DEADLINE_MS;
556
617
  state.eventDeadlineTimer = setTimeout(() => {
557
618
  state.eventDeadlineTimer = null;
558
619
  if (!this.isCurrentThreadState(state.threadId, state)) return;
@@ -569,8 +630,7 @@ var IntelligenceAgentRunner = class extends require_agent_runner.AgentRunner {
569
630
  }
570
631
  failIfEventDeadlineExceeded(state) {
571
632
  if (state.pendingEvents.size === 0) return false;
572
- const oldestQueuedAt = Math.min(...[...state.pendingEvents.values()].map((event) => event.queuedAt));
573
- if (Date.now() < oldestQueuedAt + EVENT_DURABILITY_DEADLINE_MS) return false;
633
+ if (Date.now() < state.durableProgressAt + EVENT_DURABILITY_DEADLINE_MS) return false;
574
634
  this.failThread(state.threadId, state, /* @__PURE__ */ new Error("Timed out trying to durably deliver runner events"));
575
635
  return true;
576
636
  }
@@ -1 +1 @@
1
- {"version":3,"file":"intelligence.cjs","names":["AgentRunner","Socket","Observable","AG_UI_CHANNEL_EVENT","EventType"],"sources":["../../../../src/v2/runtime/runner/intelligence.ts"],"sourcesContent":["import type {\n AgentRunnerConnectRequest,\n AgentRunnerIsRunningRequest,\n AgentRunnerRunRequest,\n} from \"./agent-runner\";\nimport { AgentRunner } from \"./agent-runner\";\nimport type { AgentRunnerStopRequest } from \"./agent-runner\";\nimport { Observable } from \"rxjs\";\nimport type { AbstractAgent, BaseEvent, RunStartedEvent } from \"@ag-ui/client\";\nimport { EventType } from \"@ag-ui/client\";\nimport {\n finalizeRunEvents,\n stripIntelligenceRoutingFields,\n AG_UI_CHANNEL_EVENT,\n phoenixExponentialBackoff,\n logger,\n} from \"@copilotkit/shared\";\nimport type { Channel } from \"phoenix\";\nimport { Socket } from \"phoenix\";\nimport { randomUUID } from \"node:crypto\";\n\nexport interface IntelligenceAgentRunnerOptions {\n /** Phoenix runner websocket URL, e.g. \"ws://localhost:4000/runner\" */\n url: string;\n /** Optional Phoenix socket auth token used during websocket connect. */\n authToken?: string;\n /** Max delay (ms) for WebSocket reconnect backoff. @default 10_000 */\n maxReconnectMs?: number;\n /** Max delay (ms) for channel rejoin backoff. @default 30_000 */\n maxRejoinMs?: number;\n /**\n * Interval (ms) between Phoenix heartbeats on the runner WebSocket. Keep it\n * below the idle timeout of any proxy between the runtime and Intelligence.\n * @default 15_000\n */\n heartbeatIntervalMs?: number;\n}\n\nexport interface RunnerStartupBoundary {\n events: Observable<BaseEvent>;\n startup: Promise<void>;\n}\n\ninterface ThreadState {\n threadId: string;\n runId: string;\n socket: Socket;\n channel: Channel;\n isRunning: boolean;\n stopRequested: boolean;\n agent: AbstractAgent | null;\n currentEvents: BaseEvent[];\n nextEventSeq: number;\n hasRunStarted: boolean;\n hasJoined: boolean;\n supportsRunnerEventBatch: boolean;\n producerFinished: boolean;\n cancellation: Promise<void>;\n cancelRun: () => void;\n completion: Promise<boolean>;\n resolveCompletion: (completed: boolean) => void;\n stopTimer: ReturnType<typeof setTimeout> | null;\n pendingEvents: Map<\n string,\n { payload: Record<string, unknown>; queuedAt: number }\n >;\n activeEventBatch: { eventIds: string[]; attempt: number } | null;\n nextEventPushAttempt: number;\n eventRetryTimer: ReturnType<typeof setTimeout> | null;\n eventFlushTimer: ReturnType<typeof setTimeout> | null;\n eventDeadlineTimer: ReturnType<typeof setTimeout> | null;\n socketReconnectWatchdog: ReturnType<typeof setTimeout> | null;\n eventRetryAttempt: number;\n completeRun: () => void;\n failRun: (error: Error) => void;\n}\n\nconst MAX_CONSECUTIVE_SOCKET_ERRORS = 5;\nconst EVENT_RETRY_BASE_MS = 100;\nconst EVENT_RETRY_MAX_MS = 2_000;\nconst RUNNER_EVENT_BATCH_CAPABILITY = \"runner_event_batch_v1\";\nconst MAX_RUNNER_EVENT_BATCH_SIZE = 32;\nconst RUNNER_EVENT_BATCH_FLUSH_MS = 5;\nconst EVENT_DURABILITY_DEADLINE_MS = 60_000;\n\nexport class IntelligenceAgentRunner extends AgentRunner {\n private options: IntelligenceAgentRunnerOptions;\n private threads = new Map<string, ThreadState>();\n\n constructor(options: IntelligenceAgentRunnerOptions) {\n super();\n // Store config — sockets are created per-run, not eagerly.\n this.options = options;\n }\n\n /**\n * Create a new Phoenix socket with explicit exponential backoff.\n *\n * Each run/connect gets its own socket so that:\n * - A socket failure only affects a single thread, not all threads.\n * - Cleanup is simple: channel.leave() + socket.disconnect() tears\n * down everything for that run with no shared-state concerns.\n * - Each run gets its own independent retry budget.\n *\n * reconnectAfterMs — delay before Phoenix reconnects the WebSocket\n * after an unclean close. 100ms base, doubling up to maxReconnectMs (default 10s).\n *\n * rejoinAfterMs — delay before Phoenix re-joins a channel that\n * entered the \"errored\" state. 1s base, doubling up to maxRejoinMs (default 30s).\n *\n * These are set explicitly because Phoenix's default schedule is a\n * fixed stepped array (not exponential), and any code that calls\n * socket.disconnect() in an onError handler will set\n * closeWasClean = true and reset the reconnect timer — permanently\n * killing retries.\n *\n * heartbeatIntervalMs — how often Phoenix pings the server\n * (default 15s). Phoenix's own default is 30s, which equals a common\n * reverse-proxy WebSocket idle timeout (e.g. Azure Application Gateway\n * behind AGIC), so a run that emits no events for 30s would have its\n * socket dropped by the proxy. The proxy timeout must exceed this value.\n */\n private createSocket(authToken = this.options.authToken): Socket {\n const socket = new Socket(this.options.url, {\n ...(authToken ? { authToken } : {}),\n reconnectAfterMs: phoenixExponentialBackoff(\n 100,\n this.options.maxReconnectMs ?? 10_000,\n ),\n rejoinAfterMs: phoenixExponentialBackoff(\n 1_000,\n this.options.maxRejoinMs ?? 30_000,\n ),\n heartbeatIntervalMs: this.options.heartbeatIntervalMs ?? 15_000,\n });\n socket.connect();\n return socket;\n }\n\n private createRunnerEventPayload(\n event: BaseEvent,\n request: AgentRunnerRunRequest,\n state: ThreadState,\n ): Record<string, unknown> {\n const canonicalEvent = this.stampRunnerMetadata(\n this.stampCanonicalRunOwnership(event, request),\n state,\n );\n const payload = {\n ...(canonicalEvent as Record<string, unknown>),\n };\n\n payload.threadId = request.threadId;\n payload.runId = request.input.runId;\n payload.thread_id = request.threadId;\n payload.run_id = request.input.runId;\n\n return payload;\n }\n\n private stampCanonicalRunOwnership(\n event: BaseEvent,\n request: AgentRunnerRunRequest,\n ): BaseEvent {\n const eventRecord = event as BaseEvent & Record<string, unknown>;\n eventRecord.threadId = request.threadId;\n eventRecord.runId = request.input.runId;\n return eventRecord;\n }\n\n /** Remove agent-controlled durable identity before the runtime assigns it. */\n private withoutAgentEventIdentity(event: BaseEvent): BaseEvent {\n const source = event as BaseEvent & { metadata?: Record<string, unknown> };\n const metadata = { ...source.metadata };\n delete metadata.cpki_event_id;\n delete metadata.cpki_event_seq;\n return { ...source, metadata };\n }\n\n private stampRunnerMetadata(event: BaseEvent, state: ThreadState): BaseEvent {\n const eventRecord = event as BaseEvent & {\n metadata?: Record<string, unknown>;\n };\n\n const existingMetadata = eventRecord.metadata ?? {};\n const hasEventId = typeof existingMetadata.cpki_event_id === \"string\";\n const hasEventSeq = typeof existingMetadata.cpki_event_seq === \"number\";\n\n if (hasEventId && hasEventSeq) {\n const eventSeq = existingMetadata.cpki_event_seq as number;\n state.nextEventSeq = Math.max(state.nextEventSeq, eventSeq + 1);\n return eventRecord;\n }\n\n const eventSeq = state.nextEventSeq++;\n\n eventRecord.metadata = {\n ...existingMetadata,\n cpki_event_id:\n typeof existingMetadata.cpki_event_id === \"string\"\n ? existingMetadata.cpki_event_id\n : randomUUID(),\n cpki_event_seq: eventSeq,\n };\n return eventRecord;\n }\n\n run(request: AgentRunnerRunRequest): Observable<BaseEvent> {\n return this.createRunObservable(request);\n }\n\n runWithStartupBoundary(\n request: AgentRunnerRunRequest,\n ): RunnerStartupBoundary {\n let resolveStartup: (() => void) | undefined;\n let rejectStartup: ((reason: Error) => void) | undefined;\n const startup = new Promise<void>((resolve, reject) => {\n resolveStartup = resolve;\n rejectStartup = reject;\n });\n\n return {\n events: this.createRunObservable(request, {\n resolveStartup: () => resolveStartup?.(),\n rejectStartup: (error) => rejectStartup?.(error),\n }),\n startup,\n };\n }\n\n private createRunObservable(\n request: AgentRunnerRunRequest,\n startupBoundary?: {\n resolveStartup: () => void;\n rejectStartup: (error: Error) => void;\n },\n ): Observable<BaseEvent> {\n const { threadId, agent, input } = request;\n\n const existing = this.threads.get(threadId);\n if (existing?.isRunning) {\n throw new Error(\"Thread already running\");\n }\n\n return new Observable((observer) => {\n if (this.threads.get(threadId)?.isRunning) {\n observer.error(new Error(\"Thread already running\"));\n return;\n }\n\n const socket = this.createSocket(request.authToken);\n\n const channel = socket.channel(`ingestion:${input.runId}`, {\n thread_id: threadId,\n run_id: input.runId,\n });\n\n let cancelRun!: () => void;\n const cancellation = new Promise<void>((resolve) => {\n cancelRun = resolve;\n });\n let resolveCompletion!: (completed: boolean) => void;\n const completion = new Promise<boolean>((resolve) => {\n resolveCompletion = resolve;\n });\n const state: ThreadState = {\n threadId,\n runId: input.runId,\n socket,\n channel,\n isRunning: true,\n stopRequested: false,\n agent,\n currentEvents: [],\n nextEventSeq: 1,\n hasRunStarted: false,\n hasJoined: false,\n supportsRunnerEventBatch: false,\n producerFinished: false,\n cancellation,\n cancelRun,\n completion,\n resolveCompletion,\n stopTimer: null,\n pendingEvents: new Map(),\n activeEventBatch: null,\n nextEventPushAttempt: 0,\n eventRetryTimer: null,\n eventFlushTimer: null,\n eventDeadlineTimer: null,\n socketReconnectWatchdog: null,\n eventRetryAttempt: 0,\n completeRun: () => observer.complete(),\n failRun: (error) => {\n startupBoundary?.rejectStartup(error);\n observer.error(error);\n },\n };\n this.threads.set(threadId, state);\n\n let consecutiveSocketErrors = 0;\n let plannedRestart = false;\n\n socket.onClose((event) => {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n plannedRestart = plannedRestart || event?.code === 1012;\n // Phoenix does not retry a normal (1000) close. A live run still\n // needs its transport; removeThread marks intentional cleanup inactive.\n if (state.socketReconnectWatchdog === null) {\n state.socketReconnectWatchdog = setTimeout(() => {\n state.socketReconnectWatchdog = null;\n if (!state.isRunning || socket.isConnected()) {\n return;\n }\n socket.disconnect(() => {\n if (state.isRunning && !socket.isConnected()) {\n socket.connect();\n }\n });\n }, 1_000);\n }\n });\n socket.onOpen(() => {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n if (state.socketReconnectWatchdog !== null) {\n clearTimeout(state.socketReconnectWatchdog);\n state.socketReconnectWatchdog = null;\n }\n consecutiveSocketErrors = 0;\n plannedRestart = false;\n });\n socket.onError(() => {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n // Once the gateway has accepted the run, transport recovery is not a\n // terminal condition. The agent may still be producing the\n // authoritative answer while Phoenix reconnects and replays durable\n // events, so an arbitrary socket-error count must not abort it.\n if (plannedRestart || state.hasJoined) {\n return;\n }\n\n consecutiveSocketErrors += 1;\n if (consecutiveSocketErrors >= MAX_CONSECUTIVE_SOCKET_ERRORS) {\n state.agent?.abortRun();\n }\n });\n\n // Listen for custom \"stop\" events pushed by the client over the\n // channel. This must be registered before channel.join() so the\n // handler is in place by the time the server starts relaying messages.\n // The client sends the stop event before leaving the channel, so the\n // runner is guaranteed to receive it while still joined.\n channel.on(AG_UI_CHANNEL_EVENT, (payload: BaseEvent) => {\n if (\n this.isCurrentThreadState(threadId, state) &&\n payload.type === EventType.CUSTOM &&\n (payload as BaseEvent & { name?: string }).name === \"stop\"\n ) {\n this.stop({ threadId, runId: state.runId }).catch((error) => {\n logger.error(\n { err: error, threadId, runId: state.runId },\n \"Failed to stop Intelligence run\",\n );\n });\n }\n });\n\n channel\n .join()\n .receive(\"ok\", (response) => {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n const supportsRunnerEventBatch =\n this.supportsRunnerEventBatch(response);\n const activeEventBatch =\n state.supportsRunnerEventBatch === supportsRunnerEventBatch\n ? state.activeEventBatch\n : null;\n state.supportsRunnerEventBatch = supportsRunnerEventBatch;\n if (state.hasJoined) {\n this.resetPendingEventRetry(state);\n if (activeEventBatch !== null) {\n state.activeEventBatch = activeEventBatch;\n this.retryActiveEventBatch(state);\n } else {\n this.replayPendingEvents(state);\n }\n return;\n }\n\n state.hasJoined = true;\n startupBoundary?.resolveStartup();\n void this.executeAgentRun(request, state, threadId, (event) => {\n observer.next(stripIntelligenceRoutingFields(event));\n });\n })\n .receive(\"error\", (resp) => {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n if (state.hasJoined) {\n if (this.isPermanentEventFailure(resp)) {\n const reason =\n typeof (resp as { reason?: unknown }).reason === \"string\"\n ? `: ${(resp as { reason: string }).reason}`\n : \"\";\n this.failThread(\n threadId,\n state,\n new Error(\n `Gateway permanently rejected channel rejoin${reason}`,\n ),\n );\n }\n return;\n }\n if (this.isRetryableJoinError(resp)) {\n return;\n }\n\n const error = new Error(\n `Failed to join channel: ${JSON.stringify(resp)}`,\n );\n const errorEvent = {\n type: EventType.RUN_ERROR,\n message: error.message,\n code: \"CHANNEL_JOIN_ERROR\",\n } as BaseEvent;\n observer.next(errorEvent);\n state.currentEvents.push(errorEvent);\n this.removeThread(threadId, state);\n startupBoundary?.rejectStartup(error);\n observer.complete();\n })\n .receive(\"timeout\", () => {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n if (state.hasJoined) {\n return;\n }\n\n const error = new Error(\"Timed out joining channel\");\n const errorEvent = {\n type: EventType.RUN_ERROR,\n message: error.message,\n code: \"CHANNEL_JOIN_TIMEOUT\",\n } as BaseEvent;\n observer.next(errorEvent);\n state.currentEvents.push(errorEvent);\n this.removeThread(threadId, state);\n startupBoundary?.rejectStartup(error);\n observer.complete();\n });\n\n return () => {\n this.removeThread(threadId, state);\n };\n });\n }\n\n connect(request: AgentRunnerConnectRequest): Observable<BaseEvent> {\n const { threadId } = request;\n\n return new Observable((observer) => {\n const socket = this.createSocket();\n\n const channel = socket.channel(`thread:${threadId}`);\n\n channel.on(\"ag_ui_event\", (payload: BaseEvent) => {\n observer.next(payload);\n\n if (\n payload.type === EventType.RUN_FINISHED ||\n payload.type === EventType.RUN_ERROR\n ) {\n observer.complete();\n }\n });\n\n const cleanup = () => {\n channel.leave();\n socket.disconnect();\n };\n\n channel\n .join()\n .receive(\"ok\", () => undefined)\n .receive(\"error\", (resp) => {\n observer.error(\n new Error(`Failed to join channel: ${JSON.stringify(resp)}`),\n );\n cleanup();\n })\n .receive(\"timeout\", () => {\n observer.error(new Error(\"Timed out joining channel\"));\n cleanup();\n });\n\n return () => {\n cleanup();\n };\n });\n }\n\n isRunning(request: AgentRunnerIsRunningRequest): Promise<boolean> {\n const state = this.threads.get(request.threadId);\n return Promise.resolve(state?.isRunning ?? false);\n }\n\n /** Stops this run and waits until its terminal events have been acknowledged. */\n stop(request: AgentRunnerStopRequest): Promise<boolean | undefined> {\n const state = this.threads.get(request.threadId);\n if (!state || !state.isRunning || state.stopRequested) {\n return Promise.resolve(false);\n }\n if (request.runId !== undefined && state.runId !== request.runId) {\n return Promise.resolve(false);\n }\n\n state.stopRequested = true;\n\n // Fence output before abort: adapters may emit synchronously, throw, or\n // ignore cancellation. Finalization must not depend on their cooperation.\n state.cancelRun();\n state.stopTimer = setTimeout(() => {\n this.failThread(\n state.threadId,\n state,\n new Error(\"Timed out stopping Intelligence run\"),\n );\n }, EVENT_DURABILITY_DEADLINE_MS);\n\n // Direct local abort — the runtime is the authority.\n if (state.agent) {\n try {\n state.agent.abortRun();\n } catch {\n // The local run is still fenced and must deliver its terminal events.\n }\n // Older AG-UI agents may not expose detachActiveRun. The cancellation\n // race still finalizes their run without waiting for the producer.\n if (typeof state.agent.detachActiveRun === \"function\") {\n try {\n Promise.resolve(state.agent.detachActiveRun()).catch((error) => {\n logger.warn(\n { err: error, threadId: state.threadId, runId: state.runId },\n \"Failed to detach stopped agent\",\n );\n });\n } catch (error) {\n logger.warn(\n { err: error, threadId: state.threadId, runId: state.runId },\n \"Failed to detach stopped agent\",\n );\n }\n }\n }\n\n return state.completion.then((completed) => {\n if (!completed) {\n throw new Error(\n \"Intelligence run stopped before terminal events were acknowledged\",\n );\n }\n return true;\n });\n }\n\n private async executeAgentRun(\n request: AgentRunnerRunRequest,\n state: ThreadState,\n threadId: string,\n onRunError: (event: BaseEvent) => void,\n ): Promise<void> {\n const { currentEvents } = state;\n const pushCanonicalEvent = (event: BaseEvent): void => {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n const canonicalEvent = this.stampRunnerMetadata(\n this.stampCanonicalRunOwnership(\n this.withoutAgentEventIdentity(event),\n request,\n ),\n state,\n );\n currentEvents.push(canonicalEvent);\n\n if (canonicalEvent.type === EventType.RUN_STARTED) {\n state.hasRunStarted = true;\n }\n\n this.queueRunnerEvent(\n this.createRunnerEventPayload(canonicalEvent, request, state),\n state,\n );\n // Notify the request handler without publishing the persisted error twice.\n // An agent may emit RUN_ERROR and complete normally instead of throwing.\n if (canonicalEvent.type === EventType.RUN_ERROR) {\n onRunError(canonicalEvent);\n }\n };\n\n const getPersistedInputMessages = () =>\n request.persistedInputMessages ?? request.input.messages;\n\n const buildRunStartedEvent = (\n source?: RunStartedEvent,\n ): RunStartedEvent => {\n const baseInput = source?.input ?? request.input;\n const persistedInputMessages = getPersistedInputMessages();\n const event =\n source ??\n ({\n type: EventType.RUN_STARTED,\n threadId: request.threadId,\n runId: request.input.runId,\n } as RunStartedEvent);\n event.threadId = request.threadId;\n event.runId = request.input.runId;\n event.input = {\n ...baseInput,\n threadId: request.threadId,\n runId: request.input.runId,\n ...(persistedInputMessages !== undefined\n ? { messages: persistedInputMessages }\n : {}),\n };\n return event;\n };\n\n const ensureRunStarted = (): void => {\n if (!state.hasRunStarted) {\n state.hasRunStarted = true;\n pushCanonicalEvent(buildRunStartedEvent());\n }\n };\n\n try {\n if (state.stopRequested) return;\n const backendThreadId = request.backendThreadId ?? request.threadId;\n request.agent.threadId = backendThreadId;\n await Promise.race([\n request.agent.runAgent(request.input, {\n onEvent: ({ event }: { event: BaseEvent }) => {\n if (state.stopRequested || state.producerFinished) return;\n if (event.type === EventType.RUN_STARTED) {\n pushCanonicalEvent(\n buildRunStartedEvent(event as RunStartedEvent),\n );\n return;\n }\n\n ensureRunStarted();\n pushCanonicalEvent(event);\n },\n }),\n state.cancellation,\n ]);\n } catch (error) {\n if (state.stopRequested || !this.isCurrentThreadState(threadId, state)) {\n return;\n }\n ensureRunStarted();\n const existingError = currentEvents.find(\n (event) => event.type === EventType.RUN_ERROR,\n );\n if (!existingError) {\n const errorEvent = {\n type: EventType.RUN_ERROR,\n message: error instanceof Error ? error.message : String(error),\n } as BaseEvent;\n pushCanonicalEvent(errorEvent);\n }\n } finally {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n ensureRunStarted();\n const appended = finalizeRunEvents(currentEvents, {\n stopRequested: state.stopRequested,\n protocolVersion: request.input.protocolVersion,\n });\n for (const event of appended) {\n pushCanonicalEvent(event);\n }\n state.producerFinished = true;\n this.completeWhenDurable(threadId, state);\n }\n }\n\n /** Queue one immutable event payload until Redis-backed gateway acknowledgement. */\n private queueRunnerEvent(\n payload: Record<string, unknown>,\n state: ThreadState,\n ): void {\n if (!this.isCurrentThreadState(state.threadId, state)) {\n return;\n }\n const eventId = this.runnerEventId(payload);\n if (!state.pendingEvents.has(eventId)) {\n state.pendingEvents.set(eventId, {\n payload: structuredClone(payload),\n queuedAt: Date.now(),\n });\n }\n this.scheduleEventDeadline(state);\n this.replayPendingEvents(state);\n }\n\n private pushPendingEventBatch(\n eventIds: string[],\n events: Array<{ payload: Record<string, unknown>; queuedAt: number }>,\n state: ThreadState,\n ): void {\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n this.failIfEventDeadlineExceeded(state) ||\n state.channel.state !== \"joined\" ||\n !state.socket.isConnected()\n ) {\n return;\n }\n\n const attempt = ++state.nextEventPushAttempt;\n state.activeEventBatch = { eventIds, attempt };\n const payloads = events.map((event) => event.payload);\n const isBatch = state.supportsRunnerEventBatch;\n\n state.channel\n .push(\n isBatch ? \"events\" : \"event\",\n isBatch ? { events: payloads } : payloads[0],\n )\n .receive(\"ok\", () => {\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n state.activeEventBatch?.attempt !== attempt\n ) {\n return;\n }\n if (this.failIfEventDeadlineExceeded(state)) {\n return;\n }\n for (const eventId of eventIds) {\n state.pendingEvents.delete(eventId);\n }\n state.activeEventBatch = null;\n state.eventRetryAttempt = 0;\n if (state.pendingEvents.size === 0) {\n this.clearPendingEventRetry(state);\n this.clearPendingEventFlush(state);\n }\n this.scheduleEventDeadline(state);\n this.completeWhenDurable(state.threadId, state);\n this.replayPendingEvents(state);\n })\n .receive(\"error\", (response) =>\n this.handlePendingEventFailure(state, attempt, response),\n )\n .receive(\"timeout\", () => this.handlePendingEventFailure(state, attempt));\n }\n\n private handlePendingEventFailure(\n state: ThreadState,\n attempt: number,\n response?: unknown,\n ): void {\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n state.activeEventBatch?.attempt !== attempt\n ) {\n return;\n }\n\n if (this.failIfEventDeadlineExceeded(state)) {\n return;\n }\n if (this.isPermanentEventFailure(response)) {\n const reason =\n typeof response === \"object\" &&\n response !== null &&\n typeof (response as { reason?: unknown }).reason === \"string\"\n ? (response as { reason: string }).reason\n : \"permanent_gateway_rejection\";\n this.failThread(\n state.threadId,\n state,\n new Error(`Runner event durability failed: ${reason}`),\n );\n return;\n }\n this.schedulePendingEventRetry(state);\n }\n\n private schedulePendingEventRetry(state: ThreadState): void {\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n state.pendingEvents.size === 0 ||\n state.eventRetryTimer !== null\n ) {\n return;\n }\n\n const delay = Math.min(\n EVENT_RETRY_BASE_MS * 2 ** state.eventRetryAttempt,\n EVENT_RETRY_MAX_MS,\n );\n state.eventRetryAttempt += 1;\n state.eventRetryTimer = setTimeout(() => {\n state.eventRetryTimer = null;\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n state.pendingEvents.size === 0\n ) {\n return;\n }\n this.retryActiveEventBatch(state);\n }, delay);\n }\n\n private clearPendingEventRetry(state: ThreadState): void {\n if (state.eventRetryTimer !== null) {\n clearTimeout(state.eventRetryTimer);\n state.eventRetryTimer = null;\n }\n }\n\n private schedulePendingEventFlush(state: ThreadState): void {\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n state.pendingEvents.size === 0 ||\n state.activeEventBatch !== null ||\n state.eventFlushTimer !== null\n ) {\n return;\n }\n\n const oldestQueuedAt = Math.min(\n ...[...state.pendingEvents.values()].map((event) => event.queuedAt),\n );\n const delay = Math.max(\n 0,\n oldestQueuedAt + RUNNER_EVENT_BATCH_FLUSH_MS - Date.now(),\n );\n state.eventFlushTimer = setTimeout(() => {\n state.eventFlushTimer = null;\n if (!this.isCurrentThreadState(state.threadId, state)) {\n return;\n }\n this.flushPendingEventBatch(state);\n }, delay);\n }\n\n private clearPendingEventFlush(state: ThreadState): void {\n if (state.eventFlushTimer !== null) {\n clearTimeout(state.eventFlushTimer);\n state.eventFlushTimer = null;\n }\n }\n\n private resetPendingEventRetry(state: ThreadState): void {\n this.clearPendingEventRetry(state);\n this.clearPendingEventFlush(state);\n state.eventRetryAttempt = 0;\n state.activeEventBatch = null;\n }\n\n private replayPendingEvents(state: ThreadState): void {\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n state.activeEventBatch !== null\n ) {\n return;\n }\n\n const pendingEvents = [...state.pendingEvents.entries()].sort(\n ([, left], [, right]) =>\n this.runnerEventSeq(left.payload) - this.runnerEventSeq(right.payload),\n );\n if (pendingEvents.length === 0) {\n return;\n }\n\n const batchSize = state.supportsRunnerEventBatch\n ? MAX_RUNNER_EVENT_BATCH_SIZE\n : 1;\n if (!state.supportsRunnerEventBatch || pendingEvents.length >= batchSize) {\n this.clearPendingEventFlush(state);\n const batch = pendingEvents.slice(0, batchSize);\n this.pushPendingEventBatch(\n batch.map(([eventId]) => eventId),\n batch.map(([, event]) => event),\n state,\n );\n return;\n }\n\n this.schedulePendingEventFlush(state);\n }\n\n private flushPendingEventBatch(state: ThreadState): void {\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n state.activeEventBatch !== null ||\n state.pendingEvents.size === 0\n ) {\n return;\n }\n\n const batch = [...state.pendingEvents.entries()]\n .sort(\n ([, left], [, right]) =>\n this.runnerEventSeq(left.payload) -\n this.runnerEventSeq(right.payload),\n )\n .slice(0, MAX_RUNNER_EVENT_BATCH_SIZE);\n this.pushPendingEventBatch(\n batch.map(([eventId]) => eventId),\n batch.map(([, event]) => event),\n state,\n );\n }\n\n private retryActiveEventBatch(state: ThreadState): void {\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n this.failIfEventDeadlineExceeded(state)\n ) {\n return;\n }\n const activeBatch = state.activeEventBatch;\n if (activeBatch === null) {\n this.replayPendingEvents(state);\n return;\n }\n\n const events = activeBatch.eventIds\n .map((eventId) => state.pendingEvents.get(eventId))\n .filter(\n (\n event,\n ): event is { payload: Record<string, unknown>; queuedAt: number } =>\n event !== undefined,\n );\n if (events.length !== activeBatch.eventIds.length) {\n state.activeEventBatch = null;\n this.replayPendingEvents(state);\n return;\n }\n\n this.pushPendingEventBatch(activeBatch.eventIds, events, state);\n }\n\n private completeWhenDurable(threadId: string, state: ThreadState): void {\n if (\n !this.isCurrentThreadState(threadId, state) ||\n !state.producerFinished ||\n state.pendingEvents.size !== 0\n ) {\n return;\n }\n if (this.threads.get(threadId) !== state) {\n return;\n }\n\n this.removeThread(threadId, state, true);\n state.completeRun();\n }\n\n private runnerEventId(payload: Record<string, unknown>): string {\n const metadata = payload.metadata as Record<string, unknown>;\n return metadata.cpki_event_id as string;\n }\n\n private runnerEventSeq(payload: Record<string, unknown>): number {\n const metadata = payload.metadata as Record<string, unknown>;\n return metadata.cpki_event_seq as number;\n }\n\n private supportsRunnerEventBatch(response: unknown): boolean {\n if (typeof response !== \"object\" || response === null) {\n return false;\n }\n const capabilities = (response as { capabilities?: unknown }).capabilities;\n return (\n Array.isArray(capabilities) &&\n capabilities.includes(RUNNER_EVENT_BATCH_CAPABILITY)\n );\n }\n\n private isRetryableJoinError(response: unknown): boolean {\n if (typeof response !== \"object\" || response === null) {\n return false;\n }\n const value = response as { reason?: unknown; retryable?: unknown };\n if (value.retryable === false) {\n return false;\n }\n return value.retryable === true || value.reason === \"gateway_draining\";\n }\n\n private isPermanentEventFailure(response: unknown): boolean {\n return (\n typeof response === \"object\" &&\n response !== null &&\n (response as { retryable?: unknown }).retryable === false\n );\n }\n\n private scheduleEventDeadline(state: ThreadState): void {\n if (state.eventDeadlineTimer !== null) {\n clearTimeout(state.eventDeadlineTimer);\n state.eventDeadlineTimer = null;\n }\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n state.pendingEvents.size === 0\n ) {\n return;\n }\n\n const oldestQueuedAt = Math.min(\n ...[...state.pendingEvents.values()].map((event) => event.queuedAt),\n );\n const deadline = oldestQueuedAt + EVENT_DURABILITY_DEADLINE_MS;\n state.eventDeadlineTimer = setTimeout(\n () => {\n state.eventDeadlineTimer = null;\n if (!this.isCurrentThreadState(state.threadId, state)) {\n return;\n }\n\n if (!this.failIfEventDeadlineExceeded(state)) {\n this.scheduleEventDeadline(state);\n }\n },\n Math.max(0, deadline - Date.now()),\n );\n }\n\n private failThread(threadId: string, state: ThreadState, error: Error): void {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n this.removeThread(threadId, state);\n try {\n state.agent?.abortRun();\n } catch {\n // The terminal durability error must still reach the subscriber.\n }\n state.failRun(error);\n }\n\n private failIfEventDeadlineExceeded(state: ThreadState): boolean {\n if (state.pendingEvents.size === 0) {\n return false;\n }\n const oldestQueuedAt = Math.min(\n ...[...state.pendingEvents.values()].map((event) => event.queuedAt),\n );\n if (Date.now() < oldestQueuedAt + EVENT_DURABILITY_DEADLINE_MS) {\n return false;\n }\n this.failThread(\n state.threadId,\n state,\n new Error(\"Timed out trying to durably deliver runner events\"),\n );\n return true;\n }\n\n private isCurrentThreadState(threadId: string, state: ThreadState): boolean {\n return state.isRunning && this.threads.get(threadId) === state;\n }\n\n /**\n * Tear down all resources for a thread: leave the channel,\n * disconnect the per-run socket, and remove the thread state.\n *\n * Idempotent — safe to call multiple times for the same threadId\n * (e.g. from join error handlers, finalize, and Observable teardown).\n */\n private removeThread(\n threadId: string,\n state: ThreadState,\n completed = false,\n ): void {\n if (this.threads.get(threadId) !== state) {\n return;\n }\n\n // Delete first so concurrent calls see the entry as already removed.\n this.threads.delete(threadId);\n state.isRunning = false;\n state.resolveCompletion(completed);\n if (state.stopTimer !== null) {\n clearTimeout(state.stopTimer);\n state.stopTimer = null;\n }\n this.clearPendingEventRetry(state);\n this.clearPendingEventFlush(state);\n if (state.eventDeadlineTimer !== null) {\n clearTimeout(state.eventDeadlineTimer);\n state.eventDeadlineTimer = null;\n }\n state.activeEventBatch = null;\n if (state.socketReconnectWatchdog !== null) {\n clearTimeout(state.socketReconnectWatchdog);\n state.socketReconnectWatchdog = null;\n }\n\n try {\n state.channel.leave();\n } catch {\n // Channel may already be closed/left.\n }\n try {\n state.socket.disconnect();\n } catch {\n // Socket may already be disconnected.\n }\n }\n}\n"],"mappings":";;;;;;;;;;AA6EA,MAAM,gCAAgC;AACtC,MAAM,sBAAsB;AAC5B,MAAM,qBAAqB;AAC3B,MAAM,gCAAgC;AACtC,MAAM,8BAA8B;AACpC,MAAM,8BAA8B;AACpC,MAAM,+BAA+B;AAErC,IAAa,0BAAb,cAA6CA,iCAAY;CAIvD,YAAY,SAAyC;AACnD,SAAO;iCAHS,IAAI,KAA0B;AAK9C,OAAK,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8BjB,AAAQ,aAAa,YAAY,KAAK,QAAQ,WAAmB;EAC/D,MAAM,SAAS,IAAIC,eAAO,KAAK,QAAQ,KAAK;GAC1C,GAAI,YAAY,EAAE,WAAW,GAAG,EAAE;GAClC,oEACE,KACA,KAAK,QAAQ,kBAAkB,IAChC;GACD,iEACE,KACA,KAAK,QAAQ,eAAe,IAC7B;GACD,qBAAqB,KAAK,QAAQ,uBAAuB;GAC1D,CAAC;AACF,SAAO,SAAS;AAChB,SAAO;;CAGT,AAAQ,yBACN,OACA,SACA,OACyB;EAKzB,MAAM,UAAU,EACd,GALqB,KAAK,oBAC1B,KAAK,2BAA2B,OAAO,QAAQ,EAC/C,MACD,EAGA;AAED,UAAQ,WAAW,QAAQ;AAC3B,UAAQ,QAAQ,QAAQ,MAAM;AAC9B,UAAQ,YAAY,QAAQ;AAC5B,UAAQ,SAAS,QAAQ,MAAM;AAE/B,SAAO;;CAGT,AAAQ,2BACN,OACA,SACW;EACX,MAAM,cAAc;AACpB,cAAY,WAAW,QAAQ;AAC/B,cAAY,QAAQ,QAAQ,MAAM;AAClC,SAAO;;;CAIT,AAAQ,0BAA0B,OAA6B;EAC7D,MAAM,SAAS;EACf,MAAM,WAAW,EAAE,GAAG,OAAO,UAAU;AACvC,SAAO,SAAS;AAChB,SAAO,SAAS;AAChB,SAAO;GAAE,GAAG;GAAQ;GAAU;;CAGhC,AAAQ,oBAAoB,OAAkB,OAA+B;EAC3E,MAAM,cAAc;EAIpB,MAAM,mBAAmB,YAAY,YAAY,EAAE;EACnD,MAAM,aAAa,OAAO,iBAAiB,kBAAkB;EAC7D,MAAM,cAAc,OAAO,iBAAiB,mBAAmB;AAE/D,MAAI,cAAc,aAAa;GAC7B,MAAM,WAAW,iBAAiB;AAClC,SAAM,eAAe,KAAK,IAAI,MAAM,cAAc,WAAW,EAAE;AAC/D,UAAO;;EAGT,MAAM,WAAW,MAAM;AAEvB,cAAY,WAAW;GACrB,GAAG;GACH,eACE,OAAO,iBAAiB,kBAAkB,WACtC,iBAAiB,6CACL;GAClB,gBAAgB;GACjB;AACD,SAAO;;CAGT,IAAI,SAAuD;AACzD,SAAO,KAAK,oBAAoB,QAAQ;;CAG1C,uBACE,SACuB;EACvB,IAAI;EACJ,IAAI;EACJ,MAAM,UAAU,IAAI,SAAe,SAAS,WAAW;AACrD,oBAAiB;AACjB,mBAAgB;IAChB;AAEF,SAAO;GACL,QAAQ,KAAK,oBAAoB,SAAS;IACxC,sBAAsB,kBAAkB;IACxC,gBAAgB,UAAU,gBAAgB,MAAM;IACjD,CAAC;GACF;GACD;;CAGH,AAAQ,oBACN,SACA,iBAIuB;EACvB,MAAM,EAAE,UAAU,OAAO,UAAU;AAGnC,MADiB,KAAK,QAAQ,IAAI,SAAS,EAC7B,UACZ,OAAM,IAAI,MAAM,yBAAyB;AAG3C,SAAO,IAAIC,iBAAY,aAAa;AAClC,OAAI,KAAK,QAAQ,IAAI,SAAS,EAAE,WAAW;AACzC,aAAS,sBAAM,IAAI,MAAM,yBAAyB,CAAC;AACnD;;GAGF,MAAM,SAAS,KAAK,aAAa,QAAQ,UAAU;GAEnD,MAAM,UAAU,OAAO,QAAQ,aAAa,MAAM,SAAS;IACzD,WAAW;IACX,QAAQ,MAAM;IACf,CAAC;GAEF,IAAI;GACJ,MAAM,eAAe,IAAI,SAAe,YAAY;AAClD,gBAAY;KACZ;GACF,IAAI;GACJ,MAAM,aAAa,IAAI,SAAkB,YAAY;AACnD,wBAAoB;KACpB;GACF,MAAM,QAAqB;IACzB;IACA,OAAO,MAAM;IACb;IACA;IACA,WAAW;IACX,eAAe;IACf;IACA,eAAe,EAAE;IACjB,cAAc;IACd,eAAe;IACf,WAAW;IACX,0BAA0B;IAC1B,kBAAkB;IAClB;IACA;IACA;IACA;IACA,WAAW;IACX,+BAAe,IAAI,KAAK;IACxB,kBAAkB;IAClB,sBAAsB;IACtB,iBAAiB;IACjB,iBAAiB;IACjB,oBAAoB;IACpB,yBAAyB;IACzB,mBAAmB;IACnB,mBAAmB,SAAS,UAAU;IACtC,UAAU,UAAU;AAClB,sBAAiB,cAAc,MAAM;AACrC,cAAS,MAAM,MAAM;;IAExB;AACD,QAAK,QAAQ,IAAI,UAAU,MAAM;GAEjC,IAAI,0BAA0B;GAC9B,IAAI,iBAAiB;AAErB,UAAO,SAAS,UAAU;AACxB,QAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;AAEF,qBAAiB,kBAAkB,OAAO,SAAS;AAGnD,QAAI,MAAM,4BAA4B,KACpC,OAAM,0BAA0B,iBAAiB;AAC/C,WAAM,0BAA0B;AAChC,SAAI,CAAC,MAAM,aAAa,OAAO,aAAa,CAC1C;AAEF,YAAO,iBAAiB;AACtB,UAAI,MAAM,aAAa,CAAC,OAAO,aAAa,CAC1C,QAAO,SAAS;OAElB;OACD,IAAM;KAEX;AACF,UAAO,aAAa;AAClB,QAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;AAEF,QAAI,MAAM,4BAA4B,MAAM;AAC1C,kBAAa,MAAM,wBAAwB;AAC3C,WAAM,0BAA0B;;AAElC,8BAA0B;AAC1B,qBAAiB;KACjB;AACF,UAAO,cAAc;AACnB,QAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;AAMF,QAAI,kBAAkB,MAAM,UAC1B;AAGF,+BAA2B;AAC3B,QAAI,2BAA2B,8BAC7B,OAAM,OAAO,UAAU;KAEzB;AAOF,WAAQ,GAAGC,yCAAsB,YAAuB;AACtD,QACE,KAAK,qBAAqB,UAAU,MAAM,IAC1C,QAAQ,SAASC,wBAAU,UAC1B,QAA0C,SAAS,OAEpD,MAAK,KAAK;KAAE;KAAU,OAAO,MAAM;KAAO,CAAC,CAAC,OAAO,UAAU;AAC3D,+BAAO,MACL;MAAE,KAAK;MAAO;MAAU,OAAO,MAAM;MAAO,EAC5C,kCACD;MACD;KAEJ;AAEF,WACG,MAAM,CACN,QAAQ,OAAO,aAAa;AAC3B,QAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;IAEF,MAAM,2BACJ,KAAK,yBAAyB,SAAS;IACzC,MAAM,mBACJ,MAAM,6BAA6B,2BAC/B,MAAM,mBACN;AACN,UAAM,2BAA2B;AACjC,QAAI,MAAM,WAAW;AACnB,UAAK,uBAAuB,MAAM;AAClC,SAAI,qBAAqB,MAAM;AAC7B,YAAM,mBAAmB;AACzB,WAAK,sBAAsB,MAAM;WAEjC,MAAK,oBAAoB,MAAM;AAEjC;;AAGF,UAAM,YAAY;AAClB,qBAAiB,gBAAgB;AACjC,IAAK,KAAK,gBAAgB,SAAS,OAAO,WAAW,UAAU;AAC7D,cAAS,4DAAoC,MAAM,CAAC;MACpD;KACF,CACD,QAAQ,UAAU,SAAS;AAC1B,QAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;AAEF,QAAI,MAAM,WAAW;AACnB,SAAI,KAAK,wBAAwB,KAAK,EAAE;MACtC,MAAM,SACJ,OAAQ,KAA8B,WAAW,WAC7C,KAAM,KAA4B,WAClC;AACN,WAAK,WACH,UACA,uBACA,IAAI,MACF,8CAA8C,SAC/C,CACF;;AAEH;;AAEF,QAAI,KAAK,qBAAqB,KAAK,CACjC;IAGF,MAAM,wBAAQ,IAAI,MAChB,2BAA2B,KAAK,UAAU,KAAK,GAChD;IACD,MAAM,aAAa;KACjB,MAAMA,wBAAU;KAChB,SAAS,MAAM;KACf,MAAM;KACP;AACD,aAAS,KAAK,WAAW;AACzB,UAAM,cAAc,KAAK,WAAW;AACpC,SAAK,aAAa,UAAU,MAAM;AAClC,qBAAiB,cAAc,MAAM;AACrC,aAAS,UAAU;KACnB,CACD,QAAQ,iBAAiB;AACxB,QAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;AAEF,QAAI,MAAM,UACR;IAGF,MAAM,wBAAQ,IAAI,MAAM,4BAA4B;IACpD,MAAM,aAAa;KACjB,MAAMA,wBAAU;KAChB,SAAS,MAAM;KACf,MAAM;KACP;AACD,aAAS,KAAK,WAAW;AACzB,UAAM,cAAc,KAAK,WAAW;AACpC,SAAK,aAAa,UAAU,MAAM;AAClC,qBAAiB,cAAc,MAAM;AACrC,aAAS,UAAU;KACnB;AAEJ,gBAAa;AACX,SAAK,aAAa,UAAU,MAAM;;IAEpC;;CAGJ,QAAQ,SAA2D;EACjE,MAAM,EAAE,aAAa;AAErB,SAAO,IAAIF,iBAAY,aAAa;GAClC,MAAM,SAAS,KAAK,cAAc;GAElC,MAAM,UAAU,OAAO,QAAQ,UAAU,WAAW;AAEpD,WAAQ,GAAG,gBAAgB,YAAuB;AAChD,aAAS,KAAK,QAAQ;AAEtB,QACE,QAAQ,SAASE,wBAAU,gBAC3B,QAAQ,SAASA,wBAAU,UAE3B,UAAS,UAAU;KAErB;GAEF,MAAM,gBAAgB;AACpB,YAAQ,OAAO;AACf,WAAO,YAAY;;AAGrB,WACG,MAAM,CACN,QAAQ,YAAY,OAAU,CAC9B,QAAQ,UAAU,SAAS;AAC1B,aAAS,sBACP,IAAI,MAAM,2BAA2B,KAAK,UAAU,KAAK,GAAG,CAC7D;AACD,aAAS;KACT,CACD,QAAQ,iBAAiB;AACxB,aAAS,sBAAM,IAAI,MAAM,4BAA4B,CAAC;AACtD,aAAS;KACT;AAEJ,gBAAa;AACX,aAAS;;IAEX;;CAGJ,UAAU,SAAwD;EAChE,MAAM,QAAQ,KAAK,QAAQ,IAAI,QAAQ,SAAS;AAChD,SAAO,QAAQ,QAAQ,OAAO,aAAa,MAAM;;;CAInD,KAAK,SAA+D;EAClE,MAAM,QAAQ,KAAK,QAAQ,IAAI,QAAQ,SAAS;AAChD,MAAI,CAAC,SAAS,CAAC,MAAM,aAAa,MAAM,cACtC,QAAO,QAAQ,QAAQ,MAAM;AAE/B,MAAI,QAAQ,UAAU,UAAa,MAAM,UAAU,QAAQ,MACzD,QAAO,QAAQ,QAAQ,MAAM;AAG/B,QAAM,gBAAgB;AAItB,QAAM,WAAW;AACjB,QAAM,YAAY,iBAAiB;AACjC,QAAK,WACH,MAAM,UACN,uBACA,IAAI,MAAM,sCAAsC,CACjD;KACA,6BAA6B;AAGhC,MAAI,MAAM,OAAO;AACf,OAAI;AACF,UAAM,MAAM,UAAU;WAChB;AAKR,OAAI,OAAO,MAAM,MAAM,oBAAoB,WACzC,KAAI;AACF,YAAQ,QAAQ,MAAM,MAAM,iBAAiB,CAAC,CAAC,OAAO,UAAU;AAC9D,+BAAO,KACL;MAAE,KAAK;MAAO,UAAU,MAAM;MAAU,OAAO,MAAM;MAAO,EAC5D,iCACD;MACD;YACK,OAAO;AACd,8BAAO,KACL;KAAE,KAAK;KAAO,UAAU,MAAM;KAAU,OAAO,MAAM;KAAO,EAC5D,iCACD;;;AAKP,SAAO,MAAM,WAAW,MAAM,cAAc;AAC1C,OAAI,CAAC,UACH,OAAM,IAAI,MACR,oEACD;AAEH,UAAO;IACP;;CAGJ,MAAc,gBACZ,SACA,OACA,UACA,YACe;EACf,MAAM,EAAE,kBAAkB;EAC1B,MAAM,sBAAsB,UAA2B;AACrD,OAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;GAEF,MAAM,iBAAiB,KAAK,oBAC1B,KAAK,2BACH,KAAK,0BAA0B,MAAM,EACrC,QACD,EACD,MACD;AACD,iBAAc,KAAK,eAAe;AAElC,OAAI,eAAe,SAASA,wBAAU,YACpC,OAAM,gBAAgB;AAGxB,QAAK,iBACH,KAAK,yBAAyB,gBAAgB,SAAS,MAAM,EAC7D,MACD;AAGD,OAAI,eAAe,SAASA,wBAAU,UACpC,YAAW,eAAe;;EAI9B,MAAM,kCACJ,QAAQ,0BAA0B,QAAQ,MAAM;EAElD,MAAM,wBACJ,WACoB;GACpB,MAAM,YAAY,QAAQ,SAAS,QAAQ;GAC3C,MAAM,yBAAyB,2BAA2B;GAC1D,MAAM,QACJ,UACC;IACC,MAAMA,wBAAU;IAChB,UAAU,QAAQ;IAClB,OAAO,QAAQ,MAAM;IACtB;AACH,SAAM,WAAW,QAAQ;AACzB,SAAM,QAAQ,QAAQ,MAAM;AAC5B,SAAM,QAAQ;IACZ,GAAG;IACH,UAAU,QAAQ;IAClB,OAAO,QAAQ,MAAM;IACrB,GAAI,2BAA2B,SAC3B,EAAE,UAAU,wBAAwB,GACpC,EAAE;IACP;AACD,UAAO;;EAGT,MAAM,yBAA+B;AACnC,OAAI,CAAC,MAAM,eAAe;AACxB,UAAM,gBAAgB;AACtB,uBAAmB,sBAAsB,CAAC;;;AAI9C,MAAI;AACF,OAAI,MAAM,cAAe;GACzB,MAAM,kBAAkB,QAAQ,mBAAmB,QAAQ;AAC3D,WAAQ,MAAM,WAAW;AACzB,SAAM,QAAQ,KAAK,CACjB,QAAQ,MAAM,SAAS,QAAQ,OAAO,EACpC,UAAU,EAAE,YAAkC;AAC5C,QAAI,MAAM,iBAAiB,MAAM,iBAAkB;AACnD,QAAI,MAAM,SAASA,wBAAU,aAAa;AACxC,wBACE,qBAAqB,MAAyB,CAC/C;AACD;;AAGF,sBAAkB;AAClB,uBAAmB,MAAM;MAE5B,CAAC,EACF,MAAM,aACP,CAAC;WACK,OAAO;AACd,OAAI,MAAM,iBAAiB,CAAC,KAAK,qBAAqB,UAAU,MAAM,CACpE;AAEF,qBAAkB;AAIlB,OAAI,CAHkB,cAAc,MACjC,UAAU,MAAM,SAASA,wBAAU,UACrC,CAMC,oBAJmB;IACjB,MAAMA,wBAAU;IAChB,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,MAAM;IAChE,CAC6B;YAExB;AACR,OAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;AAEF,qBAAkB;GAClB,MAAM,qDAA6B,eAAe;IAChD,eAAe,MAAM;IACrB,iBAAiB,QAAQ,MAAM;IAChC,CAAC;AACF,QAAK,MAAM,SAAS,SAClB,oBAAmB,MAAM;AAE3B,SAAM,mBAAmB;AACzB,QAAK,oBAAoB,UAAU,MAAM;;;;CAK7C,AAAQ,iBACN,SACA,OACM;AACN,MAAI,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,CACnD;EAEF,MAAM,UAAU,KAAK,cAAc,QAAQ;AAC3C,MAAI,CAAC,MAAM,cAAc,IAAI,QAAQ,CACnC,OAAM,cAAc,IAAI,SAAS;GAC/B,SAAS,gBAAgB,QAAQ;GACjC,UAAU,KAAK,KAAK;GACrB,CAAC;AAEJ,OAAK,sBAAsB,MAAM;AACjC,OAAK,oBAAoB,MAAM;;CAGjC,AAAQ,sBACN,UACA,QACA,OACM;AACN,MACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,KAAK,4BAA4B,MAAM,IACvC,MAAM,QAAQ,UAAU,YACxB,CAAC,MAAM,OAAO,aAAa,CAE3B;EAGF,MAAM,UAAU,EAAE,MAAM;AACxB,QAAM,mBAAmB;GAAE;GAAU;GAAS;EAC9C,MAAM,WAAW,OAAO,KAAK,UAAU,MAAM,QAAQ;EACrD,MAAM,UAAU,MAAM;AAEtB,QAAM,QACH,KACC,UAAU,WAAW,SACrB,UAAU,EAAE,QAAQ,UAAU,GAAG,SAAS,GAC3C,CACA,QAAQ,YAAY;AACnB,OACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,MAAM,kBAAkB,YAAY,QAEpC;AAEF,OAAI,KAAK,4BAA4B,MAAM,CACzC;AAEF,QAAK,MAAM,WAAW,SACpB,OAAM,cAAc,OAAO,QAAQ;AAErC,SAAM,mBAAmB;AACzB,SAAM,oBAAoB;AAC1B,OAAI,MAAM,cAAc,SAAS,GAAG;AAClC,SAAK,uBAAuB,MAAM;AAClC,SAAK,uBAAuB,MAAM;;AAEpC,QAAK,sBAAsB,MAAM;AACjC,QAAK,oBAAoB,MAAM,UAAU,MAAM;AAC/C,QAAK,oBAAoB,MAAM;IAC/B,CACD,QAAQ,UAAU,aACjB,KAAK,0BAA0B,OAAO,SAAS,SAAS,CACzD,CACA,QAAQ,iBAAiB,KAAK,0BAA0B,OAAO,QAAQ,CAAC;;CAG7E,AAAQ,0BACN,OACA,SACA,UACM;AACN,MACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,MAAM,kBAAkB,YAAY,QAEpC;AAGF,MAAI,KAAK,4BAA4B,MAAM,CACzC;AAEF,MAAI,KAAK,wBAAwB,SAAS,EAAE;GAC1C,MAAM,SACJ,OAAO,aAAa,YACpB,aAAa,QACb,OAAQ,SAAkC,WAAW,WAChD,SAAgC,SACjC;AACN,QAAK,WACH,MAAM,UACN,uBACA,IAAI,MAAM,mCAAmC,SAAS,CACvD;AACD;;AAEF,OAAK,0BAA0B,MAAM;;CAGvC,AAAQ,0BAA0B,OAA0B;AAC1D,MACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,MAAM,cAAc,SAAS,KAC7B,MAAM,oBAAoB,KAE1B;EAGF,MAAM,QAAQ,KAAK,IACjB,sBAAsB,KAAK,MAAM,mBACjC,mBACD;AACD,QAAM,qBAAqB;AAC3B,QAAM,kBAAkB,iBAAiB;AACvC,SAAM,kBAAkB;AACxB,OACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,MAAM,cAAc,SAAS,EAE7B;AAEF,QAAK,sBAAsB,MAAM;KAChC,MAAM;;CAGX,AAAQ,uBAAuB,OAA0B;AACvD,MAAI,MAAM,oBAAoB,MAAM;AAClC,gBAAa,MAAM,gBAAgB;AACnC,SAAM,kBAAkB;;;CAI5B,AAAQ,0BAA0B,OAA0B;AAC1D,MACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,MAAM,cAAc,SAAS,KAC7B,MAAM,qBAAqB,QAC3B,MAAM,oBAAoB,KAE1B;EAGF,MAAM,iBAAiB,KAAK,IAC1B,GAAG,CAAC,GAAG,MAAM,cAAc,QAAQ,CAAC,CAAC,KAAK,UAAU,MAAM,SAAS,CACpE;EACD,MAAM,QAAQ,KAAK,IACjB,GACA,iBAAiB,8BAA8B,KAAK,KAAK,CAC1D;AACD,QAAM,kBAAkB,iBAAiB;AACvC,SAAM,kBAAkB;AACxB,OAAI,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,CACnD;AAEF,QAAK,uBAAuB,MAAM;KACjC,MAAM;;CAGX,AAAQ,uBAAuB,OAA0B;AACvD,MAAI,MAAM,oBAAoB,MAAM;AAClC,gBAAa,MAAM,gBAAgB;AACnC,SAAM,kBAAkB;;;CAI5B,AAAQ,uBAAuB,OAA0B;AACvD,OAAK,uBAAuB,MAAM;AAClC,OAAK,uBAAuB,MAAM;AAClC,QAAM,oBAAoB;AAC1B,QAAM,mBAAmB;;CAG3B,AAAQ,oBAAoB,OAA0B;AACpD,MACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,MAAM,qBAAqB,KAE3B;EAGF,MAAM,gBAAgB,CAAC,GAAG,MAAM,cAAc,SAAS,CAAC,CAAC,MACtD,GAAG,OAAO,GAAG,WACZ,KAAK,eAAe,KAAK,QAAQ,GAAG,KAAK,eAAe,MAAM,QAAQ,CACzE;AACD,MAAI,cAAc,WAAW,EAC3B;EAGF,MAAM,YAAY,MAAM,2BACpB,8BACA;AACJ,MAAI,CAAC,MAAM,4BAA4B,cAAc,UAAU,WAAW;AACxE,QAAK,uBAAuB,MAAM;GAClC,MAAM,QAAQ,cAAc,MAAM,GAAG,UAAU;AAC/C,QAAK,sBACH,MAAM,KAAK,CAAC,aAAa,QAAQ,EACjC,MAAM,KAAK,GAAG,WAAW,MAAM,EAC/B,MACD;AACD;;AAGF,OAAK,0BAA0B,MAAM;;CAGvC,AAAQ,uBAAuB,OAA0B;AACvD,MACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,MAAM,qBAAqB,QAC3B,MAAM,cAAc,SAAS,EAE7B;EAGF,MAAM,QAAQ,CAAC,GAAG,MAAM,cAAc,SAAS,CAAC,CAC7C,MACE,GAAG,OAAO,GAAG,WACZ,KAAK,eAAe,KAAK,QAAQ,GACjC,KAAK,eAAe,MAAM,QAAQ,CACrC,CACA,MAAM,GAAG,4BAA4B;AACxC,OAAK,sBACH,MAAM,KAAK,CAAC,aAAa,QAAQ,EACjC,MAAM,KAAK,GAAG,WAAW,MAAM,EAC/B,MACD;;CAGH,AAAQ,sBAAsB,OAA0B;AACtD,MACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,KAAK,4BAA4B,MAAM,CAEvC;EAEF,MAAM,cAAc,MAAM;AAC1B,MAAI,gBAAgB,MAAM;AACxB,QAAK,oBAAoB,MAAM;AAC/B;;EAGF,MAAM,SAAS,YAAY,SACxB,KAAK,YAAY,MAAM,cAAc,IAAI,QAAQ,CAAC,CAClD,QAEG,UAEA,UAAU,OACb;AACH,MAAI,OAAO,WAAW,YAAY,SAAS,QAAQ;AACjD,SAAM,mBAAmB;AACzB,QAAK,oBAAoB,MAAM;AAC/B;;AAGF,OAAK,sBAAsB,YAAY,UAAU,QAAQ,MAAM;;CAGjE,AAAQ,oBAAoB,UAAkB,OAA0B;AACtE,MACE,CAAC,KAAK,qBAAqB,UAAU,MAAM,IAC3C,CAAC,MAAM,oBACP,MAAM,cAAc,SAAS,EAE7B;AAEF,MAAI,KAAK,QAAQ,IAAI,SAAS,KAAK,MACjC;AAGF,OAAK,aAAa,UAAU,OAAO,KAAK;AACxC,QAAM,aAAa;;CAGrB,AAAQ,cAAc,SAA0C;AAE9D,SADiB,QAAQ,SACT;;CAGlB,AAAQ,eAAe,SAA0C;AAE/D,SADiB,QAAQ,SACT;;CAGlB,AAAQ,yBAAyB,UAA4B;AAC3D,MAAI,OAAO,aAAa,YAAY,aAAa,KAC/C,QAAO;EAET,MAAM,eAAgB,SAAwC;AAC9D,SACE,MAAM,QAAQ,aAAa,IAC3B,aAAa,SAAS,8BAA8B;;CAIxD,AAAQ,qBAAqB,UAA4B;AACvD,MAAI,OAAO,aAAa,YAAY,aAAa,KAC/C,QAAO;EAET,MAAM,QAAQ;AACd,MAAI,MAAM,cAAc,MACtB,QAAO;AAET,SAAO,MAAM,cAAc,QAAQ,MAAM,WAAW;;CAGtD,AAAQ,wBAAwB,UAA4B;AAC1D,SACE,OAAO,aAAa,YACpB,aAAa,QACZ,SAAqC,cAAc;;CAIxD,AAAQ,sBAAsB,OAA0B;AACtD,MAAI,MAAM,uBAAuB,MAAM;AACrC,gBAAa,MAAM,mBAAmB;AACtC,SAAM,qBAAqB;;AAE7B,MACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,MAAM,cAAc,SAAS,EAE7B;EAMF,MAAM,WAHiB,KAAK,IAC1B,GAAG,CAAC,GAAG,MAAM,cAAc,QAAQ,CAAC,CAAC,KAAK,UAAU,MAAM,SAAS,CACpE,GACiC;AAClC,QAAM,qBAAqB,iBACnB;AACJ,SAAM,qBAAqB;AAC3B,OAAI,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,CACnD;AAGF,OAAI,CAAC,KAAK,4BAA4B,MAAM,CAC1C,MAAK,sBAAsB,MAAM;KAGrC,KAAK,IAAI,GAAG,WAAW,KAAK,KAAK,CAAC,CACnC;;CAGH,AAAQ,WAAW,UAAkB,OAAoB,OAAoB;AAC3E,MAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;AAEF,OAAK,aAAa,UAAU,MAAM;AAClC,MAAI;AACF,SAAM,OAAO,UAAU;UACjB;AAGR,QAAM,QAAQ,MAAM;;CAGtB,AAAQ,4BAA4B,OAA6B;AAC/D,MAAI,MAAM,cAAc,SAAS,EAC/B,QAAO;EAET,MAAM,iBAAiB,KAAK,IAC1B,GAAG,CAAC,GAAG,MAAM,cAAc,QAAQ,CAAC,CAAC,KAAK,UAAU,MAAM,SAAS,CACpE;AACD,MAAI,KAAK,KAAK,GAAG,iBAAiB,6BAChC,QAAO;AAET,OAAK,WACH,MAAM,UACN,uBACA,IAAI,MAAM,oDAAoD,CAC/D;AACD,SAAO;;CAGT,AAAQ,qBAAqB,UAAkB,OAA6B;AAC1E,SAAO,MAAM,aAAa,KAAK,QAAQ,IAAI,SAAS,KAAK;;;;;;;;;CAU3D,AAAQ,aACN,UACA,OACA,YAAY,OACN;AACN,MAAI,KAAK,QAAQ,IAAI,SAAS,KAAK,MACjC;AAIF,OAAK,QAAQ,OAAO,SAAS;AAC7B,QAAM,YAAY;AAClB,QAAM,kBAAkB,UAAU;AAClC,MAAI,MAAM,cAAc,MAAM;AAC5B,gBAAa,MAAM,UAAU;AAC7B,SAAM,YAAY;;AAEpB,OAAK,uBAAuB,MAAM;AAClC,OAAK,uBAAuB,MAAM;AAClC,MAAI,MAAM,uBAAuB,MAAM;AACrC,gBAAa,MAAM,mBAAmB;AACtC,SAAM,qBAAqB;;AAE7B,QAAM,mBAAmB;AACzB,MAAI,MAAM,4BAA4B,MAAM;AAC1C,gBAAa,MAAM,wBAAwB;AAC3C,SAAM,0BAA0B;;AAGlC,MAAI;AACF,SAAM,QAAQ,OAAO;UACf;AAGR,MAAI;AACF,SAAM,OAAO,YAAY;UACnB"}
1
+ {"version":3,"file":"intelligence.cjs","names":["AgentRunner","Socket","Observable","AG_UI_CHANNEL_EVENT","EventType"],"sources":["../../../../src/v2/runtime/runner/intelligence.ts"],"sourcesContent":["import type {\n AgentRunnerConnectRequest,\n AgentRunnerIsRunningRequest,\n AgentRunnerRunRequest,\n} from \"./agent-runner\";\nimport { AgentRunner } from \"./agent-runner\";\nimport type { AgentRunnerStopRequest } from \"./agent-runner\";\nimport { Observable } from \"rxjs\";\nimport type { AbstractAgent, BaseEvent, RunStartedEvent } from \"@ag-ui/client\";\nimport { EventType } from \"@ag-ui/client\";\nimport {\n createRunEventFinalizer,\n stripIntelligenceRoutingFields,\n AG_UI_CHANNEL_EVENT,\n phoenixExponentialBackoff,\n logger,\n} from \"@copilotkit/shared\";\nimport type { RunEventFinalizer } from \"@copilotkit/shared\";\nimport type { Channel } from \"phoenix\";\nimport { Socket } from \"phoenix\";\nimport { randomUUID } from \"node:crypto\";\n\nexport interface IntelligenceAgentRunnerOptions {\n /** Phoenix runner websocket URL, e.g. \"ws://localhost:4000/runner\" */\n url: string;\n /** Optional Phoenix socket auth token used during websocket connect. */\n authToken?: string;\n /** Max delay (ms) for WebSocket reconnect backoff. @default 10_000 */\n maxReconnectMs?: number;\n /** Max delay (ms) for channel rejoin backoff. @default 30_000 */\n maxRejoinMs?: number;\n /**\n * Interval (ms) between Phoenix heartbeats on the runner WebSocket. Keep it\n * below the idle timeout of any proxy between the runtime and Intelligence.\n * @default 15_000\n */\n heartbeatIntervalMs?: number;\n /**\n * Called each time Intelligence acknowledges a batch of runner events.\n * Use it to watch the delivery queue: a growing `pendingEvents` or\n * `oldestPendingAgeMs` means events are produced faster than Intelligence\n * accepts them. Errors thrown by the callback are logged and ignored.\n */\n onEventDeliveryStats?: (stats: RunnerEventDeliveryStats) => void;\n}\n\nexport interface RunnerEventDeliveryStats {\n threadId: string;\n runId: string;\n /** Events in the batch Intelligence just acknowledged. */\n acknowledgedEvents: number;\n /** Time from sending the batch to its acknowledgement. */\n ackLatencyMs: number;\n /** Events still waiting for acknowledgement. */\n pendingEvents: number;\n /** Age of the oldest event still waiting, or 0 when none is. */\n oldestPendingAgeMs: number;\n}\n\nexport interface RunnerStartupBoundary {\n events: Observable<BaseEvent>;\n startup: Promise<void>;\n}\n\ninterface ThreadState {\n threadId: string;\n runId: string;\n socket: Socket;\n channel: Channel;\n isRunning: boolean;\n stopRequested: boolean;\n agent: AbstractAgent | null;\n /** Tracks only open lifecycles, so a run's events are never retained. */\n finalizer: RunEventFinalizer;\n runErrorEmitted: boolean;\n nextEventSeq: number;\n hasRunStarted: boolean;\n hasJoined: boolean;\n supportsRunnerEventBatch: boolean;\n producerFinished: boolean;\n cancellation: Promise<void>;\n cancelRun: () => void;\n completion: Promise<boolean>;\n resolveCompletion: (completed: boolean) => void;\n stopTimer: ReturnType<typeof setTimeout> | null;\n pendingEvents: Map<\n string,\n { payload: Record<string, unknown>; queuedAt: number }\n >;\n /**\n * When the queue last became non-empty or last had a batch acknowledged.\n *\n * The deadline runs from here, so it measures a stalled gateway, not a long\n * queue. While the gateway keeps acknowledging, `pendingEvents` can hold up\n * to the rest of the current run: at most the run's own events, which the\n * runner used to retain in full anyway. Nothing limits how large a single\n * run gets; that is a separate question from delivery. Watch the queue\n * through `onEventDeliveryStats`.\n */\n durableProgressAt: number;\n /** Highest `cpki_event_seq` queued so far in this run. */\n lastQueuedEventSeq: number;\n /** Set when an event was queued behind a higher sequence number. */\n pendingEventsOutOfOrder: boolean;\n activeEventBatch: { eventIds: string[]; attempt: number } | null;\n nextEventPushAttempt: number;\n eventRetryTimer: ReturnType<typeof setTimeout> | null;\n eventFlushTimer: ReturnType<typeof setTimeout> | null;\n eventDeadlineTimer: ReturnType<typeof setTimeout> | null;\n socketReconnectWatchdog: ReturnType<typeof setTimeout> | null;\n eventRetryAttempt: number;\n completeRun: () => void;\n failRun: (error: Error) => void;\n}\n\nconst MAX_CONSECUTIVE_SOCKET_ERRORS = 5;\nconst EVENT_RETRY_BASE_MS = 100;\nconst EVENT_RETRY_MAX_MS = 2_000;\nconst RUNNER_EVENT_BATCH_CAPABILITY = \"runner_event_batch_v1\";\nconst MAX_RUNNER_EVENT_BATCH_SIZE = 32;\nconst RUNNER_EVENT_BATCH_FLUSH_MS = 5;\nconst EVENT_DURABILITY_DEADLINE_MS = 60_000;\n\nexport class IntelligenceAgentRunner extends AgentRunner {\n private options: IntelligenceAgentRunnerOptions;\n private threads = new Map<string, ThreadState>();\n\n constructor(options: IntelligenceAgentRunnerOptions) {\n super();\n // Store config — sockets are created per-run, not eagerly.\n this.options = options;\n }\n\n /**\n * Create a new Phoenix socket with explicit exponential backoff.\n *\n * Each run/connect gets its own socket so that:\n * - A socket failure only affects a single thread, not all threads.\n * - Cleanup is simple: channel.leave() + socket.disconnect() tears\n * down everything for that run with no shared-state concerns.\n * - Each run gets its own independent retry budget.\n *\n * reconnectAfterMs — delay before Phoenix reconnects the WebSocket\n * after an unclean close. 100ms base, doubling up to maxReconnectMs (default 10s).\n *\n * rejoinAfterMs — delay before Phoenix re-joins a channel that\n * entered the \"errored\" state. 1s base, doubling up to maxRejoinMs (default 30s).\n *\n * These are set explicitly because Phoenix's default schedule is a\n * fixed stepped array (not exponential), and any code that calls\n * socket.disconnect() in an onError handler will set\n * closeWasClean = true and reset the reconnect timer — permanently\n * killing retries.\n *\n * heartbeatIntervalMs — how often Phoenix pings the server\n * (default 15s). Phoenix's own default is 30s, which equals a common\n * reverse-proxy WebSocket idle timeout (e.g. Azure Application Gateway\n * behind AGIC), so a run that emits no events for 30s would have its\n * socket dropped by the proxy. The proxy timeout must exceed this value.\n */\n private createSocket(authToken = this.options.authToken): Socket {\n const socket = new Socket(this.options.url, {\n ...(authToken ? { authToken } : {}),\n reconnectAfterMs: phoenixExponentialBackoff(\n 100,\n this.options.maxReconnectMs ?? 10_000,\n ),\n rejoinAfterMs: phoenixExponentialBackoff(\n 1_000,\n this.options.maxRejoinMs ?? 30_000,\n ),\n heartbeatIntervalMs: this.options.heartbeatIntervalMs ?? 15_000,\n });\n socket.connect();\n return socket;\n }\n\n private createRunnerEventPayload(\n event: BaseEvent,\n request: AgentRunnerRunRequest,\n state: ThreadState,\n ): Record<string, unknown> {\n const canonicalEvent = this.stampRunnerMetadata(\n this.stampCanonicalRunOwnership(event, request),\n state,\n );\n const payload = {\n ...(canonicalEvent as Record<string, unknown>),\n };\n\n payload.threadId = request.threadId;\n payload.runId = request.input.runId;\n payload.thread_id = request.threadId;\n payload.run_id = request.input.runId;\n\n return payload;\n }\n\n private stampCanonicalRunOwnership(\n event: BaseEvent,\n request: AgentRunnerRunRequest,\n ): BaseEvent {\n const eventRecord = event as BaseEvent & Record<string, unknown>;\n eventRecord.threadId = request.threadId;\n eventRecord.runId = request.input.runId;\n return eventRecord;\n }\n\n /** Remove agent-controlled durable identity before the runtime assigns it. */\n private withoutAgentEventIdentity(event: BaseEvent): BaseEvent {\n const source = event as BaseEvent & { metadata?: Record<string, unknown> };\n const metadata = { ...source.metadata };\n delete metadata.cpki_event_id;\n delete metadata.cpki_event_seq;\n return { ...source, metadata };\n }\n\n private stampRunnerMetadata(event: BaseEvent, state: ThreadState): BaseEvent {\n const eventRecord = event as BaseEvent & {\n metadata?: Record<string, unknown>;\n };\n\n const existingMetadata = eventRecord.metadata ?? {};\n const hasEventId = typeof existingMetadata.cpki_event_id === \"string\";\n const hasEventSeq = typeof existingMetadata.cpki_event_seq === \"number\";\n\n if (hasEventId && hasEventSeq) {\n const eventSeq = existingMetadata.cpki_event_seq as number;\n state.nextEventSeq = Math.max(state.nextEventSeq, eventSeq + 1);\n return eventRecord;\n }\n\n const eventSeq = state.nextEventSeq++;\n\n eventRecord.metadata = {\n ...existingMetadata,\n cpki_event_id:\n typeof existingMetadata.cpki_event_id === \"string\"\n ? existingMetadata.cpki_event_id\n : randomUUID(),\n cpki_event_seq: eventSeq,\n };\n return eventRecord;\n }\n\n run(request: AgentRunnerRunRequest): Observable<BaseEvent> {\n return this.createRunObservable(request);\n }\n\n runWithStartupBoundary(\n request: AgentRunnerRunRequest,\n ): RunnerStartupBoundary {\n let resolveStartup: (() => void) | undefined;\n let rejectStartup: ((reason: Error) => void) | undefined;\n const startup = new Promise<void>((resolve, reject) => {\n resolveStartup = resolve;\n rejectStartup = reject;\n });\n\n return {\n events: this.createRunObservable(request, {\n resolveStartup: () => resolveStartup?.(),\n rejectStartup: (error) => rejectStartup?.(error),\n }),\n startup,\n };\n }\n\n private createRunObservable(\n request: AgentRunnerRunRequest,\n startupBoundary?: {\n resolveStartup: () => void;\n rejectStartup: (error: Error) => void;\n },\n ): Observable<BaseEvent> {\n const { threadId, agent, input } = request;\n\n const existing = this.threads.get(threadId);\n if (existing?.isRunning) {\n throw new Error(\"Thread already running\");\n }\n\n return new Observable((observer) => {\n if (this.threads.get(threadId)?.isRunning) {\n observer.error(new Error(\"Thread already running\"));\n return;\n }\n\n const socket = this.createSocket(request.authToken);\n\n const channel = socket.channel(`ingestion:${input.runId}`, {\n thread_id: threadId,\n run_id: input.runId,\n });\n\n let cancelRun!: () => void;\n const cancellation = new Promise<void>((resolve) => {\n cancelRun = resolve;\n });\n let resolveCompletion!: (completed: boolean) => void;\n const completion = new Promise<boolean>((resolve) => {\n resolveCompletion = resolve;\n });\n const state: ThreadState = {\n threadId,\n runId: input.runId,\n socket,\n channel,\n isRunning: true,\n stopRequested: false,\n agent,\n finalizer: createRunEventFinalizer(),\n runErrorEmitted: false,\n nextEventSeq: 1,\n hasRunStarted: false,\n hasJoined: false,\n supportsRunnerEventBatch: false,\n producerFinished: false,\n cancellation,\n cancelRun,\n completion,\n resolveCompletion,\n stopTimer: null,\n pendingEvents: new Map(),\n durableProgressAt: 0,\n lastQueuedEventSeq: 0,\n pendingEventsOutOfOrder: false,\n activeEventBatch: null,\n nextEventPushAttempt: 0,\n eventRetryTimer: null,\n eventFlushTimer: null,\n eventDeadlineTimer: null,\n socketReconnectWatchdog: null,\n eventRetryAttempt: 0,\n completeRun: () => observer.complete(),\n failRun: (error) => {\n startupBoundary?.rejectStartup(error);\n observer.error(error);\n },\n };\n this.threads.set(threadId, state);\n\n let consecutiveSocketErrors = 0;\n let plannedRestart = false;\n\n socket.onClose((event) => {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n plannedRestart = plannedRestart || event?.code === 1012;\n // Phoenix does not retry a normal (1000) close. A live run still\n // needs its transport; removeThread marks intentional cleanup inactive.\n if (state.socketReconnectWatchdog === null) {\n state.socketReconnectWatchdog = setTimeout(() => {\n state.socketReconnectWatchdog = null;\n if (!state.isRunning || socket.isConnected()) {\n return;\n }\n socket.disconnect(() => {\n if (state.isRunning && !socket.isConnected()) {\n socket.connect();\n }\n });\n }, 1_000);\n }\n });\n socket.onOpen(() => {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n if (state.socketReconnectWatchdog !== null) {\n clearTimeout(state.socketReconnectWatchdog);\n state.socketReconnectWatchdog = null;\n }\n consecutiveSocketErrors = 0;\n plannedRestart = false;\n });\n socket.onError(() => {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n // Once the gateway has accepted the run, transport recovery is not a\n // terminal condition. The agent may still be producing the\n // authoritative answer while Phoenix reconnects and replays durable\n // events, so an arbitrary socket-error count must not abort it.\n if (plannedRestart || state.hasJoined) {\n return;\n }\n\n consecutiveSocketErrors += 1;\n if (consecutiveSocketErrors >= MAX_CONSECUTIVE_SOCKET_ERRORS) {\n state.agent?.abortRun();\n }\n });\n\n // Listen for custom \"stop\" events pushed by the client over the\n // channel. This must be registered before channel.join() so the\n // handler is in place by the time the server starts relaying messages.\n // The client sends the stop event before leaving the channel, so the\n // runner is guaranteed to receive it while still joined.\n channel.on(AG_UI_CHANNEL_EVENT, (payload: BaseEvent) => {\n if (\n this.isCurrentThreadState(threadId, state) &&\n payload.type === EventType.CUSTOM &&\n (payload as BaseEvent & { name?: string }).name === \"stop\"\n ) {\n this.stop({ threadId, runId: state.runId }).catch((error) => {\n logger.error(\n { err: error, threadId, runId: state.runId },\n \"Failed to stop Intelligence run\",\n );\n });\n }\n });\n\n channel\n .join()\n .receive(\"ok\", (response) => {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n const supportsRunnerEventBatch =\n this.supportsRunnerEventBatch(response);\n const activeEventBatch =\n state.supportsRunnerEventBatch === supportsRunnerEventBatch\n ? state.activeEventBatch\n : null;\n state.supportsRunnerEventBatch = supportsRunnerEventBatch;\n if (state.hasJoined) {\n this.resetPendingEventRetry(state);\n if (activeEventBatch !== null) {\n state.activeEventBatch = activeEventBatch;\n this.retryActiveEventBatch(state);\n } else {\n this.replayPendingEvents(state);\n }\n return;\n }\n\n state.hasJoined = true;\n startupBoundary?.resolveStartup();\n void this.executeAgentRun(request, state, threadId, (event) => {\n observer.next(stripIntelligenceRoutingFields(event));\n });\n })\n .receive(\"error\", (resp) => {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n if (state.hasJoined) {\n if (this.isPermanentEventFailure(resp)) {\n const reason =\n typeof (resp as { reason?: unknown }).reason === \"string\"\n ? `: ${(resp as { reason: string }).reason}`\n : \"\";\n this.failThread(\n threadId,\n state,\n new Error(\n `Gateway permanently rejected channel rejoin${reason}`,\n ),\n );\n }\n return;\n }\n if (this.isRetryableJoinError(resp)) {\n return;\n }\n\n const error = new Error(\n `Failed to join channel: ${JSON.stringify(resp)}`,\n );\n const errorEvent = {\n type: EventType.RUN_ERROR,\n message: error.message,\n code: \"CHANNEL_JOIN_ERROR\",\n } as BaseEvent;\n observer.next(errorEvent);\n this.removeThread(threadId, state);\n startupBoundary?.rejectStartup(error);\n observer.complete();\n })\n .receive(\"timeout\", () => {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n if (state.hasJoined) {\n return;\n }\n\n const error = new Error(\"Timed out joining channel\");\n const errorEvent = {\n type: EventType.RUN_ERROR,\n message: error.message,\n code: \"CHANNEL_JOIN_TIMEOUT\",\n } as BaseEvent;\n observer.next(errorEvent);\n this.removeThread(threadId, state);\n startupBoundary?.rejectStartup(error);\n observer.complete();\n });\n\n return () => {\n this.removeThread(threadId, state);\n };\n });\n }\n\n connect(request: AgentRunnerConnectRequest): Observable<BaseEvent> {\n const { threadId } = request;\n\n return new Observable((observer) => {\n const socket = this.createSocket();\n\n const channel = socket.channel(`thread:${threadId}`);\n\n channel.on(\"ag_ui_event\", (payload: BaseEvent) => {\n observer.next(payload);\n\n if (\n payload.type === EventType.RUN_FINISHED ||\n payload.type === EventType.RUN_ERROR\n ) {\n observer.complete();\n }\n });\n\n const cleanup = () => {\n channel.leave();\n socket.disconnect();\n };\n\n channel\n .join()\n .receive(\"ok\", () => undefined)\n .receive(\"error\", (resp) => {\n observer.error(\n new Error(`Failed to join channel: ${JSON.stringify(resp)}`),\n );\n cleanup();\n })\n .receive(\"timeout\", () => {\n observer.error(new Error(\"Timed out joining channel\"));\n cleanup();\n });\n\n return () => {\n cleanup();\n };\n });\n }\n\n isRunning(request: AgentRunnerIsRunningRequest): Promise<boolean> {\n const state = this.threads.get(request.threadId);\n return Promise.resolve(state?.isRunning ?? false);\n }\n\n /** Stops this run and waits until its terminal events have been acknowledged. */\n stop(request: AgentRunnerStopRequest): Promise<boolean | undefined> {\n const state = this.threads.get(request.threadId);\n if (!state || !state.isRunning || state.stopRequested) {\n return Promise.resolve(false);\n }\n if (request.runId !== undefined && state.runId !== request.runId) {\n return Promise.resolve(false);\n }\n\n state.stopRequested = true;\n\n // Fence output before abort: adapters may emit synchronously, throw, or\n // ignore cancellation. Finalization must not depend on their cooperation.\n state.cancelRun();\n state.stopTimer = setTimeout(() => {\n this.failThread(\n state.threadId,\n state,\n new Error(\"Timed out stopping Intelligence run\"),\n );\n }, EVENT_DURABILITY_DEADLINE_MS);\n\n // Direct local abort — the runtime is the authority.\n if (state.agent) {\n try {\n state.agent.abortRun();\n } catch {\n // The local run is still fenced and must deliver its terminal events.\n }\n // Older AG-UI agents may not expose detachActiveRun. The cancellation\n // race still finalizes their run without waiting for the producer.\n if (typeof state.agent.detachActiveRun === \"function\") {\n try {\n Promise.resolve(state.agent.detachActiveRun()).catch((error) => {\n logger.warn(\n { err: error, threadId: state.threadId, runId: state.runId },\n \"Failed to detach stopped agent\",\n );\n });\n } catch (error) {\n logger.warn(\n { err: error, threadId: state.threadId, runId: state.runId },\n \"Failed to detach stopped agent\",\n );\n }\n }\n }\n\n return state.completion.then((completed) => {\n if (!completed) {\n throw new Error(\n \"Intelligence run stopped before terminal events were acknowledged\",\n );\n }\n return true;\n });\n }\n\n private async executeAgentRun(\n request: AgentRunnerRunRequest,\n state: ThreadState,\n threadId: string,\n onRunError: (event: BaseEvent) => void,\n ): Promise<void> {\n const pushCanonicalEvent = (event: BaseEvent): void => {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n const canonicalEvent = this.stampRunnerMetadata(\n this.stampCanonicalRunOwnership(\n this.withoutAgentEventIdentity(event),\n request,\n ),\n state,\n );\n state.finalizer.observe(canonicalEvent);\n\n if (canonicalEvent.type === EventType.RUN_STARTED) {\n state.hasRunStarted = true;\n }\n if (canonicalEvent.type === EventType.RUN_ERROR) {\n state.runErrorEmitted = true;\n }\n\n this.queueRunnerEvent(\n this.createRunnerEventPayload(canonicalEvent, request, state),\n state,\n );\n // Notify the request handler without publishing the persisted error twice.\n // An agent may emit RUN_ERROR and complete normally instead of throwing.\n if (canonicalEvent.type === EventType.RUN_ERROR) {\n onRunError(canonicalEvent);\n }\n };\n\n const getPersistedInputMessages = () =>\n request.persistedInputMessages ?? request.input.messages;\n\n const buildRunStartedEvent = (\n source?: RunStartedEvent,\n ): RunStartedEvent => {\n const baseInput = source?.input ?? request.input;\n const persistedInputMessages = getPersistedInputMessages();\n const event =\n source ??\n ({\n type: EventType.RUN_STARTED,\n threadId: request.threadId,\n runId: request.input.runId,\n } as RunStartedEvent);\n event.threadId = request.threadId;\n event.runId = request.input.runId;\n event.input = {\n ...baseInput,\n threadId: request.threadId,\n runId: request.input.runId,\n ...(persistedInputMessages !== undefined\n ? { messages: persistedInputMessages }\n : {}),\n };\n return event;\n };\n\n const ensureRunStarted = (): void => {\n if (!state.hasRunStarted) {\n state.hasRunStarted = true;\n pushCanonicalEvent(buildRunStartedEvent());\n }\n };\n\n try {\n if (state.stopRequested) return;\n const backendThreadId = request.backendThreadId ?? request.threadId;\n request.agent.threadId = backendThreadId;\n await Promise.race([\n request.agent.runAgent(request.input, {\n onEvent: ({ event }: { event: BaseEvent }) => {\n if (state.stopRequested || state.producerFinished) return;\n if (event.type === EventType.RUN_STARTED) {\n pushCanonicalEvent(\n buildRunStartedEvent(event as RunStartedEvent),\n );\n return;\n }\n\n ensureRunStarted();\n pushCanonicalEvent(event);\n },\n }),\n state.cancellation,\n ]);\n } catch (error) {\n if (state.stopRequested || !this.isCurrentThreadState(threadId, state)) {\n return;\n }\n ensureRunStarted();\n if (!state.runErrorEmitted) {\n const errorEvent = {\n type: EventType.RUN_ERROR,\n message: error instanceof Error ? error.message : String(error),\n } as BaseEvent;\n pushCanonicalEvent(errorEvent);\n }\n } finally {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n ensureRunStarted();\n const appended = state.finalizer.finalize({\n stopRequested: state.stopRequested,\n protocolVersion: request.input.protocolVersion,\n });\n for (const event of appended) {\n pushCanonicalEvent(event);\n }\n state.producerFinished = true;\n this.completeWhenDurable(threadId, state);\n }\n }\n\n /**\n * Queue one immutable event payload until Redis-backed gateway acknowledgement.\n *\n * `pushCanonicalEvent` is the only producer and stamps a fresh, increasing\n * `cpki_event_seq` on every event it queues, so the map's insertion order is\n * sequence order. Batches are therefore read from the front of the map\n * instead of sorting the whole queue on every event.\n */\n private queueRunnerEvent(\n payload: Record<string, unknown>,\n state: ThreadState,\n ): void {\n if (!this.isCurrentThreadState(state.threadId, state)) {\n return;\n }\n const eventId = this.runnerEventId(payload);\n if (!state.pendingEvents.has(eventId)) {\n const queuedAt = Date.now();\n const wasEmpty = state.pendingEvents.size === 0;\n const eventSeq = this.runnerEventSeq(payload);\n if (\n eventSeq < state.lastQueuedEventSeq &&\n !state.pendingEventsOutOfOrder\n ) {\n state.pendingEventsOutOfOrder = true;\n logger.warn(\n {\n threadId: state.threadId,\n runId: state.runId,\n eventSeq,\n lastQueuedEventSeq: state.lastQueuedEventSeq,\n },\n \"Runner event queued out of sequence order; sorting pending events\",\n );\n }\n state.lastQueuedEventSeq = Math.max(state.lastQueuedEventSeq, eventSeq);\n state.pendingEvents.set(eventId, {\n payload: structuredClone(payload),\n queuedAt,\n });\n if (wasEmpty) {\n state.durableProgressAt = queuedAt;\n // An empty queue has no deadline timer; its first event starts one.\n this.scheduleEventDeadline(state);\n }\n }\n this.replayPendingEvents(state);\n }\n\n /** The oldest `limit` pending events, in sequence order. */\n private oldestPendingEvents(\n state: ThreadState,\n limit: number,\n ): Array<[string, { payload: Record<string, unknown>; queuedAt: number }]> {\n // Guard for the invariant in queueRunnerEvent: should another path ever\n // queue an event behind a higher sequence number, fall back to sorting\n // until the queue drains rather than send out of order.\n if (state.pendingEventsOutOfOrder) {\n return [...state.pendingEvents.entries()]\n .sort(\n ([, left], [, right]) =>\n this.runnerEventSeq(left.payload) -\n this.runnerEventSeq(right.payload),\n )\n .slice(0, limit);\n }\n const oldest: Array<\n [string, { payload: Record<string, unknown>; queuedAt: number }]\n > = [];\n for (const entry of state.pendingEvents) {\n if (oldest.length >= limit) break;\n oldest.push(entry);\n }\n return oldest;\n }\n\n private pushPendingEventBatch(\n eventIds: string[],\n events: Array<{ payload: Record<string, unknown>; queuedAt: number }>,\n state: ThreadState,\n ): void {\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n this.failIfEventDeadlineExceeded(state) ||\n state.channel.state !== \"joined\" ||\n !state.socket.isConnected()\n ) {\n return;\n }\n\n const attempt = ++state.nextEventPushAttempt;\n const sentAt = Date.now();\n state.activeEventBatch = { eventIds, attempt };\n const payloads = events.map((event) => event.payload);\n const isBatch = state.supportsRunnerEventBatch;\n\n state.channel\n .push(\n isBatch ? \"events\" : \"event\",\n isBatch ? { events: payloads } : payloads[0],\n )\n .receive(\"ok\", () => {\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n state.activeEventBatch?.attempt !== attempt\n ) {\n return;\n }\n if (this.failIfEventDeadlineExceeded(state)) {\n return;\n }\n for (const eventId of eventIds) {\n state.pendingEvents.delete(eventId);\n }\n // The gateway is accepting events, so the deadline measures from here.\n // A slow but healthy gateway must not fail a run that is still draining.\n state.durableProgressAt = Date.now();\n this.reportEventDelivery(\n state,\n eventIds.length,\n state.durableProgressAt - sentAt,\n );\n state.activeEventBatch = null;\n state.eventRetryAttempt = 0;\n if (state.pendingEvents.size === 0) {\n state.pendingEventsOutOfOrder = false;\n this.clearPendingEventRetry(state);\n this.clearPendingEventFlush(state);\n }\n this.scheduleEventDeadline(state);\n this.completeWhenDurable(state.threadId, state);\n this.replayPendingEvents(state);\n })\n .receive(\"error\", (response) =>\n this.handlePendingEventFailure(state, attempt, response),\n )\n .receive(\"timeout\", () => this.handlePendingEventFailure(state, attempt));\n }\n\n private reportEventDelivery(\n state: ThreadState,\n acknowledgedEvents: number,\n ackLatencyMs: number,\n ): void {\n const onStats = this.options.onEventDeliveryStats;\n if (!onStats) return;\n const [oldest] = this.oldestPendingEvents(state, 1);\n try {\n onStats({\n threadId: state.threadId,\n runId: state.runId,\n acknowledgedEvents,\n ackLatencyMs,\n pendingEvents: state.pendingEvents.size,\n oldestPendingAgeMs: oldest ? Date.now() - oldest[1].queuedAt : 0,\n });\n } catch (error) {\n logger.warn(\n { err: error, threadId: state.threadId, runId: state.runId },\n \"onEventDeliveryStats callback failed\",\n );\n }\n }\n\n private handlePendingEventFailure(\n state: ThreadState,\n attempt: number,\n response?: unknown,\n ): void {\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n state.activeEventBatch?.attempt !== attempt\n ) {\n return;\n }\n\n if (this.failIfEventDeadlineExceeded(state)) {\n return;\n }\n if (this.isPermanentEventFailure(response)) {\n const reason =\n typeof response === \"object\" &&\n response !== null &&\n typeof (response as { reason?: unknown }).reason === \"string\"\n ? (response as { reason: string }).reason\n : \"permanent_gateway_rejection\";\n this.failThread(\n state.threadId,\n state,\n new Error(`Runner event durability failed: ${reason}`),\n );\n return;\n }\n this.schedulePendingEventRetry(state);\n }\n\n private schedulePendingEventRetry(state: ThreadState): void {\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n state.pendingEvents.size === 0 ||\n state.eventRetryTimer !== null\n ) {\n return;\n }\n\n const delay = Math.min(\n EVENT_RETRY_BASE_MS * 2 ** state.eventRetryAttempt,\n EVENT_RETRY_MAX_MS,\n );\n state.eventRetryAttempt += 1;\n state.eventRetryTimer = setTimeout(() => {\n state.eventRetryTimer = null;\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n state.pendingEvents.size === 0\n ) {\n return;\n }\n this.retryActiveEventBatch(state);\n }, delay);\n }\n\n private clearPendingEventRetry(state: ThreadState): void {\n if (state.eventRetryTimer !== null) {\n clearTimeout(state.eventRetryTimer);\n state.eventRetryTimer = null;\n }\n }\n\n private schedulePendingEventFlush(state: ThreadState): void {\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n state.pendingEvents.size === 0 ||\n state.activeEventBatch !== null ||\n state.eventFlushTimer !== null\n ) {\n return;\n }\n\n const [[, oldest]] = this.oldestPendingEvents(state, 1);\n const delay = Math.max(\n 0,\n oldest.queuedAt + RUNNER_EVENT_BATCH_FLUSH_MS - Date.now(),\n );\n state.eventFlushTimer = setTimeout(() => {\n state.eventFlushTimer = null;\n if (!this.isCurrentThreadState(state.threadId, state)) {\n return;\n }\n this.flushPendingEventBatch(state);\n }, delay);\n }\n\n private clearPendingEventFlush(state: ThreadState): void {\n if (state.eventFlushTimer !== null) {\n clearTimeout(state.eventFlushTimer);\n state.eventFlushTimer = null;\n }\n }\n\n private resetPendingEventRetry(state: ThreadState): void {\n this.clearPendingEventRetry(state);\n this.clearPendingEventFlush(state);\n state.eventRetryAttempt = 0;\n state.activeEventBatch = null;\n }\n\n private replayPendingEvents(state: ThreadState): void {\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n state.activeEventBatch !== null\n ) {\n return;\n }\n\n if (state.pendingEvents.size === 0) {\n return;\n }\n\n const batchSize = state.supportsRunnerEventBatch\n ? MAX_RUNNER_EVENT_BATCH_SIZE\n : 1;\n if (\n !state.supportsRunnerEventBatch ||\n state.pendingEvents.size >= batchSize\n ) {\n this.clearPendingEventFlush(state);\n const batch = this.oldestPendingEvents(state, batchSize);\n this.pushPendingEventBatch(\n batch.map(([eventId]) => eventId),\n batch.map(([, event]) => event),\n state,\n );\n return;\n }\n\n this.schedulePendingEventFlush(state);\n }\n\n private flushPendingEventBatch(state: ThreadState): void {\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n state.activeEventBatch !== null ||\n state.pendingEvents.size === 0\n ) {\n return;\n }\n\n const batch = this.oldestPendingEvents(state, MAX_RUNNER_EVENT_BATCH_SIZE);\n this.pushPendingEventBatch(\n batch.map(([eventId]) => eventId),\n batch.map(([, event]) => event),\n state,\n );\n }\n\n private retryActiveEventBatch(state: ThreadState): void {\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n this.failIfEventDeadlineExceeded(state)\n ) {\n return;\n }\n const activeBatch = state.activeEventBatch;\n if (activeBatch === null) {\n this.replayPendingEvents(state);\n return;\n }\n\n const events = activeBatch.eventIds\n .map((eventId) => state.pendingEvents.get(eventId))\n .filter(\n (\n event,\n ): event is { payload: Record<string, unknown>; queuedAt: number } =>\n event !== undefined,\n );\n if (events.length !== activeBatch.eventIds.length) {\n state.activeEventBatch = null;\n this.replayPendingEvents(state);\n return;\n }\n\n this.pushPendingEventBatch(activeBatch.eventIds, events, state);\n }\n\n private completeWhenDurable(threadId: string, state: ThreadState): void {\n if (\n !this.isCurrentThreadState(threadId, state) ||\n !state.producerFinished ||\n state.pendingEvents.size !== 0\n ) {\n return;\n }\n if (this.threads.get(threadId) !== state) {\n return;\n }\n\n this.removeThread(threadId, state, true);\n state.completeRun();\n }\n\n private runnerEventId(payload: Record<string, unknown>): string {\n const metadata = payload.metadata as Record<string, unknown>;\n return metadata.cpki_event_id as string;\n }\n\n private runnerEventSeq(payload: Record<string, unknown>): number {\n const metadata = payload.metadata as Record<string, unknown>;\n return metadata.cpki_event_seq as number;\n }\n\n private supportsRunnerEventBatch(response: unknown): boolean {\n if (typeof response !== \"object\" || response === null) {\n return false;\n }\n const capabilities = (response as { capabilities?: unknown }).capabilities;\n return (\n Array.isArray(capabilities) &&\n capabilities.includes(RUNNER_EVENT_BATCH_CAPABILITY)\n );\n }\n\n private isRetryableJoinError(response: unknown): boolean {\n if (typeof response !== \"object\" || response === null) {\n return false;\n }\n const value = response as { reason?: unknown; retryable?: unknown };\n if (value.retryable === false) {\n return false;\n }\n return value.retryable === true || value.reason === \"gateway_draining\";\n }\n\n private isPermanentEventFailure(response: unknown): boolean {\n return (\n typeof response === \"object\" &&\n response !== null &&\n (response as { retryable?: unknown }).retryable === false\n );\n }\n\n private scheduleEventDeadline(state: ThreadState): void {\n if (state.eventDeadlineTimer !== null) {\n clearTimeout(state.eventDeadlineTimer);\n state.eventDeadlineTimer = null;\n }\n if (\n !this.isCurrentThreadState(state.threadId, state) ||\n state.pendingEvents.size === 0\n ) {\n return;\n }\n\n const deadline = state.durableProgressAt + EVENT_DURABILITY_DEADLINE_MS;\n state.eventDeadlineTimer = setTimeout(\n () => {\n state.eventDeadlineTimer = null;\n if (!this.isCurrentThreadState(state.threadId, state)) {\n return;\n }\n\n if (!this.failIfEventDeadlineExceeded(state)) {\n this.scheduleEventDeadline(state);\n }\n },\n Math.max(0, deadline - Date.now()),\n );\n }\n\n private failThread(threadId: string, state: ThreadState, error: Error): void {\n if (!this.isCurrentThreadState(threadId, state)) {\n return;\n }\n this.removeThread(threadId, state);\n try {\n state.agent?.abortRun();\n } catch {\n // The terminal durability error must still reach the subscriber.\n }\n state.failRun(error);\n }\n\n private failIfEventDeadlineExceeded(state: ThreadState): boolean {\n if (state.pendingEvents.size === 0) {\n return false;\n }\n if (Date.now() < state.durableProgressAt + EVENT_DURABILITY_DEADLINE_MS) {\n return false;\n }\n this.failThread(\n state.threadId,\n state,\n new Error(\"Timed out trying to durably deliver runner events\"),\n );\n return true;\n }\n\n private isCurrentThreadState(threadId: string, state: ThreadState): boolean {\n return state.isRunning && this.threads.get(threadId) === state;\n }\n\n /**\n * Tear down all resources for a thread: leave the channel,\n * disconnect the per-run socket, and remove the thread state.\n *\n * Idempotent — safe to call multiple times for the same threadId\n * (e.g. from join error handlers, finalize, and Observable teardown).\n */\n private removeThread(\n threadId: string,\n state: ThreadState,\n completed = false,\n ): void {\n if (this.threads.get(threadId) !== state) {\n return;\n }\n\n // Delete first so concurrent calls see the entry as already removed.\n this.threads.delete(threadId);\n state.isRunning = false;\n state.resolveCompletion(completed);\n if (state.stopTimer !== null) {\n clearTimeout(state.stopTimer);\n state.stopTimer = null;\n }\n this.clearPendingEventRetry(state);\n this.clearPendingEventFlush(state);\n if (state.eventDeadlineTimer !== null) {\n clearTimeout(state.eventDeadlineTimer);\n state.eventDeadlineTimer = null;\n }\n state.activeEventBatch = null;\n if (state.socketReconnectWatchdog !== null) {\n clearTimeout(state.socketReconnectWatchdog);\n state.socketReconnectWatchdog = null;\n }\n\n try {\n state.channel.leave();\n } catch {\n // Channel may already be closed/left.\n }\n try {\n state.socket.disconnect();\n } catch {\n // Socket may already be disconnected.\n }\n }\n}\n"],"mappings":";;;;;;;;;;AAmHA,MAAM,gCAAgC;AACtC,MAAM,sBAAsB;AAC5B,MAAM,qBAAqB;AAC3B,MAAM,gCAAgC;AACtC,MAAM,8BAA8B;AACpC,MAAM,8BAA8B;AACpC,MAAM,+BAA+B;AAErC,IAAa,0BAAb,cAA6CA,iCAAY;CAIvD,YAAY,SAAyC;AACnD,SAAO;iCAHS,IAAI,KAA0B;AAK9C,OAAK,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8BjB,AAAQ,aAAa,YAAY,KAAK,QAAQ,WAAmB;EAC/D,MAAM,SAAS,IAAIC,eAAO,KAAK,QAAQ,KAAK;GAC1C,GAAI,YAAY,EAAE,WAAW,GAAG,EAAE;GAClC,oEACE,KACA,KAAK,QAAQ,kBAAkB,IAChC;GACD,iEACE,KACA,KAAK,QAAQ,eAAe,IAC7B;GACD,qBAAqB,KAAK,QAAQ,uBAAuB;GAC1D,CAAC;AACF,SAAO,SAAS;AAChB,SAAO;;CAGT,AAAQ,yBACN,OACA,SACA,OACyB;EAKzB,MAAM,UAAU,EACd,GALqB,KAAK,oBAC1B,KAAK,2BAA2B,OAAO,QAAQ,EAC/C,MACD,EAGA;AAED,UAAQ,WAAW,QAAQ;AAC3B,UAAQ,QAAQ,QAAQ,MAAM;AAC9B,UAAQ,YAAY,QAAQ;AAC5B,UAAQ,SAAS,QAAQ,MAAM;AAE/B,SAAO;;CAGT,AAAQ,2BACN,OACA,SACW;EACX,MAAM,cAAc;AACpB,cAAY,WAAW,QAAQ;AAC/B,cAAY,QAAQ,QAAQ,MAAM;AAClC,SAAO;;;CAIT,AAAQ,0BAA0B,OAA6B;EAC7D,MAAM,SAAS;EACf,MAAM,WAAW,EAAE,GAAG,OAAO,UAAU;AACvC,SAAO,SAAS;AAChB,SAAO,SAAS;AAChB,SAAO;GAAE,GAAG;GAAQ;GAAU;;CAGhC,AAAQ,oBAAoB,OAAkB,OAA+B;EAC3E,MAAM,cAAc;EAIpB,MAAM,mBAAmB,YAAY,YAAY,EAAE;EACnD,MAAM,aAAa,OAAO,iBAAiB,kBAAkB;EAC7D,MAAM,cAAc,OAAO,iBAAiB,mBAAmB;AAE/D,MAAI,cAAc,aAAa;GAC7B,MAAM,WAAW,iBAAiB;AAClC,SAAM,eAAe,KAAK,IAAI,MAAM,cAAc,WAAW,EAAE;AAC/D,UAAO;;EAGT,MAAM,WAAW,MAAM;AAEvB,cAAY,WAAW;GACrB,GAAG;GACH,eACE,OAAO,iBAAiB,kBAAkB,WACtC,iBAAiB,6CACL;GAClB,gBAAgB;GACjB;AACD,SAAO;;CAGT,IAAI,SAAuD;AACzD,SAAO,KAAK,oBAAoB,QAAQ;;CAG1C,uBACE,SACuB;EACvB,IAAI;EACJ,IAAI;EACJ,MAAM,UAAU,IAAI,SAAe,SAAS,WAAW;AACrD,oBAAiB;AACjB,mBAAgB;IAChB;AAEF,SAAO;GACL,QAAQ,KAAK,oBAAoB,SAAS;IACxC,sBAAsB,kBAAkB;IACxC,gBAAgB,UAAU,gBAAgB,MAAM;IACjD,CAAC;GACF;GACD;;CAGH,AAAQ,oBACN,SACA,iBAIuB;EACvB,MAAM,EAAE,UAAU,OAAO,UAAU;AAGnC,MADiB,KAAK,QAAQ,IAAI,SAAS,EAC7B,UACZ,OAAM,IAAI,MAAM,yBAAyB;AAG3C,SAAO,IAAIC,iBAAY,aAAa;AAClC,OAAI,KAAK,QAAQ,IAAI,SAAS,EAAE,WAAW;AACzC,aAAS,sBAAM,IAAI,MAAM,yBAAyB,CAAC;AACnD;;GAGF,MAAM,SAAS,KAAK,aAAa,QAAQ,UAAU;GAEnD,MAAM,UAAU,OAAO,QAAQ,aAAa,MAAM,SAAS;IACzD,WAAW;IACX,QAAQ,MAAM;IACf,CAAC;GAEF,IAAI;GACJ,MAAM,eAAe,IAAI,SAAe,YAAY;AAClD,gBAAY;KACZ;GACF,IAAI;GACJ,MAAM,aAAa,IAAI,SAAkB,YAAY;AACnD,wBAAoB;KACpB;GACF,MAAM,QAAqB;IACzB;IACA,OAAO,MAAM;IACb;IACA;IACA,WAAW;IACX,eAAe;IACf;IACA,4DAAoC;IACpC,iBAAiB;IACjB,cAAc;IACd,eAAe;IACf,WAAW;IACX,0BAA0B;IAC1B,kBAAkB;IAClB;IACA;IACA;IACA;IACA,WAAW;IACX,+BAAe,IAAI,KAAK;IACxB,mBAAmB;IACnB,oBAAoB;IACpB,yBAAyB;IACzB,kBAAkB;IAClB,sBAAsB;IACtB,iBAAiB;IACjB,iBAAiB;IACjB,oBAAoB;IACpB,yBAAyB;IACzB,mBAAmB;IACnB,mBAAmB,SAAS,UAAU;IACtC,UAAU,UAAU;AAClB,sBAAiB,cAAc,MAAM;AACrC,cAAS,MAAM,MAAM;;IAExB;AACD,QAAK,QAAQ,IAAI,UAAU,MAAM;GAEjC,IAAI,0BAA0B;GAC9B,IAAI,iBAAiB;AAErB,UAAO,SAAS,UAAU;AACxB,QAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;AAEF,qBAAiB,kBAAkB,OAAO,SAAS;AAGnD,QAAI,MAAM,4BAA4B,KACpC,OAAM,0BAA0B,iBAAiB;AAC/C,WAAM,0BAA0B;AAChC,SAAI,CAAC,MAAM,aAAa,OAAO,aAAa,CAC1C;AAEF,YAAO,iBAAiB;AACtB,UAAI,MAAM,aAAa,CAAC,OAAO,aAAa,CAC1C,QAAO,SAAS;OAElB;OACD,IAAM;KAEX;AACF,UAAO,aAAa;AAClB,QAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;AAEF,QAAI,MAAM,4BAA4B,MAAM;AAC1C,kBAAa,MAAM,wBAAwB;AAC3C,WAAM,0BAA0B;;AAElC,8BAA0B;AAC1B,qBAAiB;KACjB;AACF,UAAO,cAAc;AACnB,QAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;AAMF,QAAI,kBAAkB,MAAM,UAC1B;AAGF,+BAA2B;AAC3B,QAAI,2BAA2B,8BAC7B,OAAM,OAAO,UAAU;KAEzB;AAOF,WAAQ,GAAGC,yCAAsB,YAAuB;AACtD,QACE,KAAK,qBAAqB,UAAU,MAAM,IAC1C,QAAQ,SAASC,wBAAU,UAC1B,QAA0C,SAAS,OAEpD,MAAK,KAAK;KAAE;KAAU,OAAO,MAAM;KAAO,CAAC,CAAC,OAAO,UAAU;AAC3D,+BAAO,MACL;MAAE,KAAK;MAAO;MAAU,OAAO,MAAM;MAAO,EAC5C,kCACD;MACD;KAEJ;AAEF,WACG,MAAM,CACN,QAAQ,OAAO,aAAa;AAC3B,QAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;IAEF,MAAM,2BACJ,KAAK,yBAAyB,SAAS;IACzC,MAAM,mBACJ,MAAM,6BAA6B,2BAC/B,MAAM,mBACN;AACN,UAAM,2BAA2B;AACjC,QAAI,MAAM,WAAW;AACnB,UAAK,uBAAuB,MAAM;AAClC,SAAI,qBAAqB,MAAM;AAC7B,YAAM,mBAAmB;AACzB,WAAK,sBAAsB,MAAM;WAEjC,MAAK,oBAAoB,MAAM;AAEjC;;AAGF,UAAM,YAAY;AAClB,qBAAiB,gBAAgB;AACjC,IAAK,KAAK,gBAAgB,SAAS,OAAO,WAAW,UAAU;AAC7D,cAAS,4DAAoC,MAAM,CAAC;MACpD;KACF,CACD,QAAQ,UAAU,SAAS;AAC1B,QAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;AAEF,QAAI,MAAM,WAAW;AACnB,SAAI,KAAK,wBAAwB,KAAK,EAAE;MACtC,MAAM,SACJ,OAAQ,KAA8B,WAAW,WAC7C,KAAM,KAA4B,WAClC;AACN,WAAK,WACH,UACA,uBACA,IAAI,MACF,8CAA8C,SAC/C,CACF;;AAEH;;AAEF,QAAI,KAAK,qBAAqB,KAAK,CACjC;IAGF,MAAM,wBAAQ,IAAI,MAChB,2BAA2B,KAAK,UAAU,KAAK,GAChD;IACD,MAAM,aAAa;KACjB,MAAMA,wBAAU;KAChB,SAAS,MAAM;KACf,MAAM;KACP;AACD,aAAS,KAAK,WAAW;AACzB,SAAK,aAAa,UAAU,MAAM;AAClC,qBAAiB,cAAc,MAAM;AACrC,aAAS,UAAU;KACnB,CACD,QAAQ,iBAAiB;AACxB,QAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;AAEF,QAAI,MAAM,UACR;IAGF,MAAM,wBAAQ,IAAI,MAAM,4BAA4B;IACpD,MAAM,aAAa;KACjB,MAAMA,wBAAU;KAChB,SAAS,MAAM;KACf,MAAM;KACP;AACD,aAAS,KAAK,WAAW;AACzB,SAAK,aAAa,UAAU,MAAM;AAClC,qBAAiB,cAAc,MAAM;AACrC,aAAS,UAAU;KACnB;AAEJ,gBAAa;AACX,SAAK,aAAa,UAAU,MAAM;;IAEpC;;CAGJ,QAAQ,SAA2D;EACjE,MAAM,EAAE,aAAa;AAErB,SAAO,IAAIF,iBAAY,aAAa;GAClC,MAAM,SAAS,KAAK,cAAc;GAElC,MAAM,UAAU,OAAO,QAAQ,UAAU,WAAW;AAEpD,WAAQ,GAAG,gBAAgB,YAAuB;AAChD,aAAS,KAAK,QAAQ;AAEtB,QACE,QAAQ,SAASE,wBAAU,gBAC3B,QAAQ,SAASA,wBAAU,UAE3B,UAAS,UAAU;KAErB;GAEF,MAAM,gBAAgB;AACpB,YAAQ,OAAO;AACf,WAAO,YAAY;;AAGrB,WACG,MAAM,CACN,QAAQ,YAAY,OAAU,CAC9B,QAAQ,UAAU,SAAS;AAC1B,aAAS,sBACP,IAAI,MAAM,2BAA2B,KAAK,UAAU,KAAK,GAAG,CAC7D;AACD,aAAS;KACT,CACD,QAAQ,iBAAiB;AACxB,aAAS,sBAAM,IAAI,MAAM,4BAA4B,CAAC;AACtD,aAAS;KACT;AAEJ,gBAAa;AACX,aAAS;;IAEX;;CAGJ,UAAU,SAAwD;EAChE,MAAM,QAAQ,KAAK,QAAQ,IAAI,QAAQ,SAAS;AAChD,SAAO,QAAQ,QAAQ,OAAO,aAAa,MAAM;;;CAInD,KAAK,SAA+D;EAClE,MAAM,QAAQ,KAAK,QAAQ,IAAI,QAAQ,SAAS;AAChD,MAAI,CAAC,SAAS,CAAC,MAAM,aAAa,MAAM,cACtC,QAAO,QAAQ,QAAQ,MAAM;AAE/B,MAAI,QAAQ,UAAU,UAAa,MAAM,UAAU,QAAQ,MACzD,QAAO,QAAQ,QAAQ,MAAM;AAG/B,QAAM,gBAAgB;AAItB,QAAM,WAAW;AACjB,QAAM,YAAY,iBAAiB;AACjC,QAAK,WACH,MAAM,UACN,uBACA,IAAI,MAAM,sCAAsC,CACjD;KACA,6BAA6B;AAGhC,MAAI,MAAM,OAAO;AACf,OAAI;AACF,UAAM,MAAM,UAAU;WAChB;AAKR,OAAI,OAAO,MAAM,MAAM,oBAAoB,WACzC,KAAI;AACF,YAAQ,QAAQ,MAAM,MAAM,iBAAiB,CAAC,CAAC,OAAO,UAAU;AAC9D,+BAAO,KACL;MAAE,KAAK;MAAO,UAAU,MAAM;MAAU,OAAO,MAAM;MAAO,EAC5D,iCACD;MACD;YACK,OAAO;AACd,8BAAO,KACL;KAAE,KAAK;KAAO,UAAU,MAAM;KAAU,OAAO,MAAM;KAAO,EAC5D,iCACD;;;AAKP,SAAO,MAAM,WAAW,MAAM,cAAc;AAC1C,OAAI,CAAC,UACH,OAAM,IAAI,MACR,oEACD;AAEH,UAAO;IACP;;CAGJ,MAAc,gBACZ,SACA,OACA,UACA,YACe;EACf,MAAM,sBAAsB,UAA2B;AACrD,OAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;GAEF,MAAM,iBAAiB,KAAK,oBAC1B,KAAK,2BACH,KAAK,0BAA0B,MAAM,EACrC,QACD,EACD,MACD;AACD,SAAM,UAAU,QAAQ,eAAe;AAEvC,OAAI,eAAe,SAASA,wBAAU,YACpC,OAAM,gBAAgB;AAExB,OAAI,eAAe,SAASA,wBAAU,UACpC,OAAM,kBAAkB;AAG1B,QAAK,iBACH,KAAK,yBAAyB,gBAAgB,SAAS,MAAM,EAC7D,MACD;AAGD,OAAI,eAAe,SAASA,wBAAU,UACpC,YAAW,eAAe;;EAI9B,MAAM,kCACJ,QAAQ,0BAA0B,QAAQ,MAAM;EAElD,MAAM,wBACJ,WACoB;GACpB,MAAM,YAAY,QAAQ,SAAS,QAAQ;GAC3C,MAAM,yBAAyB,2BAA2B;GAC1D,MAAM,QACJ,UACC;IACC,MAAMA,wBAAU;IAChB,UAAU,QAAQ;IAClB,OAAO,QAAQ,MAAM;IACtB;AACH,SAAM,WAAW,QAAQ;AACzB,SAAM,QAAQ,QAAQ,MAAM;AAC5B,SAAM,QAAQ;IACZ,GAAG;IACH,UAAU,QAAQ;IAClB,OAAO,QAAQ,MAAM;IACrB,GAAI,2BAA2B,SAC3B,EAAE,UAAU,wBAAwB,GACpC,EAAE;IACP;AACD,UAAO;;EAGT,MAAM,yBAA+B;AACnC,OAAI,CAAC,MAAM,eAAe;AACxB,UAAM,gBAAgB;AACtB,uBAAmB,sBAAsB,CAAC;;;AAI9C,MAAI;AACF,OAAI,MAAM,cAAe;GACzB,MAAM,kBAAkB,QAAQ,mBAAmB,QAAQ;AAC3D,WAAQ,MAAM,WAAW;AACzB,SAAM,QAAQ,KAAK,CACjB,QAAQ,MAAM,SAAS,QAAQ,OAAO,EACpC,UAAU,EAAE,YAAkC;AAC5C,QAAI,MAAM,iBAAiB,MAAM,iBAAkB;AACnD,QAAI,MAAM,SAASA,wBAAU,aAAa;AACxC,wBACE,qBAAqB,MAAyB,CAC/C;AACD;;AAGF,sBAAkB;AAClB,uBAAmB,MAAM;MAE5B,CAAC,EACF,MAAM,aACP,CAAC;WACK,OAAO;AACd,OAAI,MAAM,iBAAiB,CAAC,KAAK,qBAAqB,UAAU,MAAM,CACpE;AAEF,qBAAkB;AAClB,OAAI,CAAC,MAAM,gBAKT,oBAJmB;IACjB,MAAMA,wBAAU;IAChB,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,MAAM;IAChE,CAC6B;YAExB;AACR,OAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;AAEF,qBAAkB;GAClB,MAAM,WAAW,MAAM,UAAU,SAAS;IACxC,eAAe,MAAM;IACrB,iBAAiB,QAAQ,MAAM;IAChC,CAAC;AACF,QAAK,MAAM,SAAS,SAClB,oBAAmB,MAAM;AAE3B,SAAM,mBAAmB;AACzB,QAAK,oBAAoB,UAAU,MAAM;;;;;;;;;;;CAY7C,AAAQ,iBACN,SACA,OACM;AACN,MAAI,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,CACnD;EAEF,MAAM,UAAU,KAAK,cAAc,QAAQ;AAC3C,MAAI,CAAC,MAAM,cAAc,IAAI,QAAQ,EAAE;GACrC,MAAM,WAAW,KAAK,KAAK;GAC3B,MAAM,WAAW,MAAM,cAAc,SAAS;GAC9C,MAAM,WAAW,KAAK,eAAe,QAAQ;AAC7C,OACE,WAAW,MAAM,sBACjB,CAAC,MAAM,yBACP;AACA,UAAM,0BAA0B;AAChC,8BAAO,KACL;KACE,UAAU,MAAM;KAChB,OAAO,MAAM;KACb;KACA,oBAAoB,MAAM;KAC3B,EACD,oEACD;;AAEH,SAAM,qBAAqB,KAAK,IAAI,MAAM,oBAAoB,SAAS;AACvE,SAAM,cAAc,IAAI,SAAS;IAC/B,SAAS,gBAAgB,QAAQ;IACjC;IACD,CAAC;AACF,OAAI,UAAU;AACZ,UAAM,oBAAoB;AAE1B,SAAK,sBAAsB,MAAM;;;AAGrC,OAAK,oBAAoB,MAAM;;;CAIjC,AAAQ,oBACN,OACA,OACyE;AAIzE,MAAI,MAAM,wBACR,QAAO,CAAC,GAAG,MAAM,cAAc,SAAS,CAAC,CACtC,MACE,GAAG,OAAO,GAAG,WACZ,KAAK,eAAe,KAAK,QAAQ,GACjC,KAAK,eAAe,MAAM,QAAQ,CACrC,CACA,MAAM,GAAG,MAAM;EAEpB,MAAM,SAEF,EAAE;AACN,OAAK,MAAM,SAAS,MAAM,eAAe;AACvC,OAAI,OAAO,UAAU,MAAO;AAC5B,UAAO,KAAK,MAAM;;AAEpB,SAAO;;CAGT,AAAQ,sBACN,UACA,QACA,OACM;AACN,MACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,KAAK,4BAA4B,MAAM,IACvC,MAAM,QAAQ,UAAU,YACxB,CAAC,MAAM,OAAO,aAAa,CAE3B;EAGF,MAAM,UAAU,EAAE,MAAM;EACxB,MAAM,SAAS,KAAK,KAAK;AACzB,QAAM,mBAAmB;GAAE;GAAU;GAAS;EAC9C,MAAM,WAAW,OAAO,KAAK,UAAU,MAAM,QAAQ;EACrD,MAAM,UAAU,MAAM;AAEtB,QAAM,QACH,KACC,UAAU,WAAW,SACrB,UAAU,EAAE,QAAQ,UAAU,GAAG,SAAS,GAC3C,CACA,QAAQ,YAAY;AACnB,OACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,MAAM,kBAAkB,YAAY,QAEpC;AAEF,OAAI,KAAK,4BAA4B,MAAM,CACzC;AAEF,QAAK,MAAM,WAAW,SACpB,OAAM,cAAc,OAAO,QAAQ;AAIrC,SAAM,oBAAoB,KAAK,KAAK;AACpC,QAAK,oBACH,OACA,SAAS,QACT,MAAM,oBAAoB,OAC3B;AACD,SAAM,mBAAmB;AACzB,SAAM,oBAAoB;AAC1B,OAAI,MAAM,cAAc,SAAS,GAAG;AAClC,UAAM,0BAA0B;AAChC,SAAK,uBAAuB,MAAM;AAClC,SAAK,uBAAuB,MAAM;;AAEpC,QAAK,sBAAsB,MAAM;AACjC,QAAK,oBAAoB,MAAM,UAAU,MAAM;AAC/C,QAAK,oBAAoB,MAAM;IAC/B,CACD,QAAQ,UAAU,aACjB,KAAK,0BAA0B,OAAO,SAAS,SAAS,CACzD,CACA,QAAQ,iBAAiB,KAAK,0BAA0B,OAAO,QAAQ,CAAC;;CAG7E,AAAQ,oBACN,OACA,oBACA,cACM;EACN,MAAM,UAAU,KAAK,QAAQ;AAC7B,MAAI,CAAC,QAAS;EACd,MAAM,CAAC,UAAU,KAAK,oBAAoB,OAAO,EAAE;AACnD,MAAI;AACF,WAAQ;IACN,UAAU,MAAM;IAChB,OAAO,MAAM;IACb;IACA;IACA,eAAe,MAAM,cAAc;IACnC,oBAAoB,SAAS,KAAK,KAAK,GAAG,OAAO,GAAG,WAAW;IAChE,CAAC;WACK,OAAO;AACd,6BAAO,KACL;IAAE,KAAK;IAAO,UAAU,MAAM;IAAU,OAAO,MAAM;IAAO,EAC5D,uCACD;;;CAIL,AAAQ,0BACN,OACA,SACA,UACM;AACN,MACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,MAAM,kBAAkB,YAAY,QAEpC;AAGF,MAAI,KAAK,4BAA4B,MAAM,CACzC;AAEF,MAAI,KAAK,wBAAwB,SAAS,EAAE;GAC1C,MAAM,SACJ,OAAO,aAAa,YACpB,aAAa,QACb,OAAQ,SAAkC,WAAW,WAChD,SAAgC,SACjC;AACN,QAAK,WACH,MAAM,UACN,uBACA,IAAI,MAAM,mCAAmC,SAAS,CACvD;AACD;;AAEF,OAAK,0BAA0B,MAAM;;CAGvC,AAAQ,0BAA0B,OAA0B;AAC1D,MACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,MAAM,cAAc,SAAS,KAC7B,MAAM,oBAAoB,KAE1B;EAGF,MAAM,QAAQ,KAAK,IACjB,sBAAsB,KAAK,MAAM,mBACjC,mBACD;AACD,QAAM,qBAAqB;AAC3B,QAAM,kBAAkB,iBAAiB;AACvC,SAAM,kBAAkB;AACxB,OACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,MAAM,cAAc,SAAS,EAE7B;AAEF,QAAK,sBAAsB,MAAM;KAChC,MAAM;;CAGX,AAAQ,uBAAuB,OAA0B;AACvD,MAAI,MAAM,oBAAoB,MAAM;AAClC,gBAAa,MAAM,gBAAgB;AACnC,SAAM,kBAAkB;;;CAI5B,AAAQ,0BAA0B,OAA0B;AAC1D,MACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,MAAM,cAAc,SAAS,KAC7B,MAAM,qBAAqB,QAC3B,MAAM,oBAAoB,KAE1B;EAGF,MAAM,CAAC,GAAG,WAAW,KAAK,oBAAoB,OAAO,EAAE;EACvD,MAAM,QAAQ,KAAK,IACjB,GACA,OAAO,WAAW,8BAA8B,KAAK,KAAK,CAC3D;AACD,QAAM,kBAAkB,iBAAiB;AACvC,SAAM,kBAAkB;AACxB,OAAI,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,CACnD;AAEF,QAAK,uBAAuB,MAAM;KACjC,MAAM;;CAGX,AAAQ,uBAAuB,OAA0B;AACvD,MAAI,MAAM,oBAAoB,MAAM;AAClC,gBAAa,MAAM,gBAAgB;AACnC,SAAM,kBAAkB;;;CAI5B,AAAQ,uBAAuB,OAA0B;AACvD,OAAK,uBAAuB,MAAM;AAClC,OAAK,uBAAuB,MAAM;AAClC,QAAM,oBAAoB;AAC1B,QAAM,mBAAmB;;CAG3B,AAAQ,oBAAoB,OAA0B;AACpD,MACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,MAAM,qBAAqB,KAE3B;AAGF,MAAI,MAAM,cAAc,SAAS,EAC/B;EAGF,MAAM,YAAY,MAAM,2BACpB,8BACA;AACJ,MACE,CAAC,MAAM,4BACP,MAAM,cAAc,QAAQ,WAC5B;AACA,QAAK,uBAAuB,MAAM;GAClC,MAAM,QAAQ,KAAK,oBAAoB,OAAO,UAAU;AACxD,QAAK,sBACH,MAAM,KAAK,CAAC,aAAa,QAAQ,EACjC,MAAM,KAAK,GAAG,WAAW,MAAM,EAC/B,MACD;AACD;;AAGF,OAAK,0BAA0B,MAAM;;CAGvC,AAAQ,uBAAuB,OAA0B;AACvD,MACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,MAAM,qBAAqB,QAC3B,MAAM,cAAc,SAAS,EAE7B;EAGF,MAAM,QAAQ,KAAK,oBAAoB,OAAO,4BAA4B;AAC1E,OAAK,sBACH,MAAM,KAAK,CAAC,aAAa,QAAQ,EACjC,MAAM,KAAK,GAAG,WAAW,MAAM,EAC/B,MACD;;CAGH,AAAQ,sBAAsB,OAA0B;AACtD,MACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,KAAK,4BAA4B,MAAM,CAEvC;EAEF,MAAM,cAAc,MAAM;AAC1B,MAAI,gBAAgB,MAAM;AACxB,QAAK,oBAAoB,MAAM;AAC/B;;EAGF,MAAM,SAAS,YAAY,SACxB,KAAK,YAAY,MAAM,cAAc,IAAI,QAAQ,CAAC,CAClD,QAEG,UAEA,UAAU,OACb;AACH,MAAI,OAAO,WAAW,YAAY,SAAS,QAAQ;AACjD,SAAM,mBAAmB;AACzB,QAAK,oBAAoB,MAAM;AAC/B;;AAGF,OAAK,sBAAsB,YAAY,UAAU,QAAQ,MAAM;;CAGjE,AAAQ,oBAAoB,UAAkB,OAA0B;AACtE,MACE,CAAC,KAAK,qBAAqB,UAAU,MAAM,IAC3C,CAAC,MAAM,oBACP,MAAM,cAAc,SAAS,EAE7B;AAEF,MAAI,KAAK,QAAQ,IAAI,SAAS,KAAK,MACjC;AAGF,OAAK,aAAa,UAAU,OAAO,KAAK;AACxC,QAAM,aAAa;;CAGrB,AAAQ,cAAc,SAA0C;AAE9D,SADiB,QAAQ,SACT;;CAGlB,AAAQ,eAAe,SAA0C;AAE/D,SADiB,QAAQ,SACT;;CAGlB,AAAQ,yBAAyB,UAA4B;AAC3D,MAAI,OAAO,aAAa,YAAY,aAAa,KAC/C,QAAO;EAET,MAAM,eAAgB,SAAwC;AAC9D,SACE,MAAM,QAAQ,aAAa,IAC3B,aAAa,SAAS,8BAA8B;;CAIxD,AAAQ,qBAAqB,UAA4B;AACvD,MAAI,OAAO,aAAa,YAAY,aAAa,KAC/C,QAAO;EAET,MAAM,QAAQ;AACd,MAAI,MAAM,cAAc,MACtB,QAAO;AAET,SAAO,MAAM,cAAc,QAAQ,MAAM,WAAW;;CAGtD,AAAQ,wBAAwB,UAA4B;AAC1D,SACE,OAAO,aAAa,YACpB,aAAa,QACZ,SAAqC,cAAc;;CAIxD,AAAQ,sBAAsB,OAA0B;AACtD,MAAI,MAAM,uBAAuB,MAAM;AACrC,gBAAa,MAAM,mBAAmB;AACtC,SAAM,qBAAqB;;AAE7B,MACE,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,IACjD,MAAM,cAAc,SAAS,EAE7B;EAGF,MAAM,WAAW,MAAM,oBAAoB;AAC3C,QAAM,qBAAqB,iBACnB;AACJ,SAAM,qBAAqB;AAC3B,OAAI,CAAC,KAAK,qBAAqB,MAAM,UAAU,MAAM,CACnD;AAGF,OAAI,CAAC,KAAK,4BAA4B,MAAM,CAC1C,MAAK,sBAAsB,MAAM;KAGrC,KAAK,IAAI,GAAG,WAAW,KAAK,KAAK,CAAC,CACnC;;CAGH,AAAQ,WAAW,UAAkB,OAAoB,OAAoB;AAC3E,MAAI,CAAC,KAAK,qBAAqB,UAAU,MAAM,CAC7C;AAEF,OAAK,aAAa,UAAU,MAAM;AAClC,MAAI;AACF,SAAM,OAAO,UAAU;UACjB;AAGR,QAAM,QAAQ,MAAM;;CAGtB,AAAQ,4BAA4B,OAA6B;AAC/D,MAAI,MAAM,cAAc,SAAS,EAC/B,QAAO;AAET,MAAI,KAAK,KAAK,GAAG,MAAM,oBAAoB,6BACzC,QAAO;AAET,OAAK,WACH,MAAM,UACN,uBACA,IAAI,MAAM,oDAAoD,CAC/D;AACD,SAAO;;CAGT,AAAQ,qBAAqB,UAAkB,OAA6B;AAC1E,SAAO,MAAM,aAAa,KAAK,QAAQ,IAAI,SAAS,KAAK;;;;;;;;;CAU3D,AAAQ,aACN,UACA,OACA,YAAY,OACN;AACN,MAAI,KAAK,QAAQ,IAAI,SAAS,KAAK,MACjC;AAIF,OAAK,QAAQ,OAAO,SAAS;AAC7B,QAAM,YAAY;AAClB,QAAM,kBAAkB,UAAU;AAClC,MAAI,MAAM,cAAc,MAAM;AAC5B,gBAAa,MAAM,UAAU;AAC7B,SAAM,YAAY;;AAEpB,OAAK,uBAAuB,MAAM;AAClC,OAAK,uBAAuB,MAAM;AAClC,MAAI,MAAM,uBAAuB,MAAM;AACrC,gBAAa,MAAM,mBAAmB;AACtC,SAAM,qBAAqB;;AAE7B,QAAM,mBAAmB;AACzB,MAAI,MAAM,4BAA4B,MAAM;AAC1C,gBAAa,MAAM,wBAAwB;AAC3C,SAAM,0BAA0B;;AAGlC,MAAI;AACF,SAAM,QAAQ,OAAO;UACf;AAGR,MAAI;AACF,SAAM,OAAO,YAAY;UACnB"}
@@ -19,6 +19,25 @@ interface IntelligenceAgentRunnerOptions {
19
19
  * @default 15_000
20
20
  */
21
21
  heartbeatIntervalMs?: number;
22
+ /**
23
+ * Called each time Intelligence acknowledges a batch of runner events.
24
+ * Use it to watch the delivery queue: a growing `pendingEvents` or
25
+ * `oldestPendingAgeMs` means events are produced faster than Intelligence
26
+ * accepts them. Errors thrown by the callback are logged and ignored.
27
+ */
28
+ onEventDeliveryStats?: (stats: RunnerEventDeliveryStats) => void;
29
+ }
30
+ interface RunnerEventDeliveryStats {
31
+ threadId: string;
32
+ runId: string;
33
+ /** Events in the batch Intelligence just acknowledged. */
34
+ acknowledgedEvents: number;
35
+ /** Time from sending the batch to its acknowledgement. */
36
+ ackLatencyMs: number;
37
+ /** Events still waiting for acknowledgement. */
38
+ pendingEvents: number;
39
+ /** Age of the oldest event still waiting, or 0 when none is. */
40
+ oldestPendingAgeMs: number;
22
41
  }
23
42
  interface RunnerStartupBoundary {
24
43
  events: Observable<BaseEvent>;
@@ -69,9 +88,19 @@ declare class IntelligenceAgentRunner extends AgentRunner {
69
88
  /** Stops this run and waits until its terminal events have been acknowledged. */
70
89
  stop(request: AgentRunnerStopRequest): Promise<boolean | undefined>;
71
90
  private executeAgentRun;
72
- /** Queue one immutable event payload until Redis-backed gateway acknowledgement. */
91
+ /**
92
+ * Queue one immutable event payload until Redis-backed gateway acknowledgement.
93
+ *
94
+ * `pushCanonicalEvent` is the only producer and stamps a fresh, increasing
95
+ * `cpki_event_seq` on every event it queues, so the map's insertion order is
96
+ * sequence order. Batches are therefore read from the front of the map
97
+ * instead of sorting the whole queue on every event.
98
+ */
73
99
  private queueRunnerEvent;
100
+ /** The oldest `limit` pending events, in sequence order. */
101
+ private oldestPendingEvents;
74
102
  private pushPendingEventBatch;
103
+ private reportEventDelivery;
75
104
  private handlePendingEventFailure;
76
105
  private schedulePendingEventRetry;
77
106
  private clearPendingEventRetry;
@@ -101,5 +130,5 @@ declare class IntelligenceAgentRunner extends AgentRunner {
101
130
  private removeThread;
102
131
  }
103
132
  //#endregion
104
- export { IntelligenceAgentRunner, IntelligenceAgentRunnerOptions, RunnerStartupBoundary };
133
+ export { IntelligenceAgentRunner, IntelligenceAgentRunnerOptions, RunnerEventDeliveryStats, RunnerStartupBoundary };
105
134
  //# sourceMappingURL=intelligence.d.cts.map
@@ -1 +1 @@
1
- {"version":3,"file":"intelligence.d.cts","names":[],"sources":["../../../../src/v2/runtime/runner/intelligence.ts"],"mappings":";;;;;;UAqBiB,8BAAA;EAA8B;EAE7C,GAAA;EAF6C;EAI7C,SAAA;EAAA;EAEA,cAAA;EAEA;EAAA,WAAA;EAMmB;;AAGrB;;;EAHE,mBAAA;AAAA;AAAA,UAGe,qBAAA;EACf,MAAA,EAAQ,UAAA,CAAW,SAAA;EACnB,OAAA,EAAS,OAAA;AAAA;AAAA,cA6CE,uBAAA,SAAgC,WAAA;EAAA,QACnC,OAAA;EAAA,QACA,OAAA;cAEI,OAAA,EAAS,8BAAA;EAjDL;;AA6ClB;;;;;;;;;;;;;;;;;;;;;;;;;EA7CkB,QAkFR,YAAA;EAAA,QAiBA,wBAAA;EAAA,QAqBA,0BAAA;EAWA;EAAA,QAAA,yBAAA;EAAA,QAQA,mBAAA;EA4BR,GAAA,CAAI,OAAA,EAAS,qBAAA,GAAwB,UAAA,CAAW,SAAA;EAIhD,sBAAA,CACE,OAAA,EAAS,qBAAA,GACR,qBAAA;EAAA,QAiBK,mBAAA;EA8OR,OAAA,CAAQ,OAAA,EAAS,yBAAA,GAA4B,UAAA,CAAW,SAAA;EA4CxD,SAAA,CAAU,OAAA,EAAS,2BAAA,GAA8B,OAAA;EA5StC;EAkTX,IAAA,CAAK,OAAA,EAAS,sBAAA,GAAyB,OAAA;EAAA,QA0DzB,eAAA;EA1VN;EAAA,QAsdA,gBAAA;EAAA,QAkBA,qBAAA;EAAA,QAqDA,yBAAA;EAAA,QAgCA,yBAAA;EAAA,QA0BA,sBAAA;EAAA,QAOA,yBAAA;EAAA,QA0BA,sBAAA;EAAA,QAOA,sBAAA;EAAA,QAOA,mBAAA;EAAA,QAiCA,sBAAA;EAAA,QAuBA,qBAAA;EAAA,QA8BA,mBAAA;EAAA,QAgBA,aAAA;EAAA,QAKA,cAAA;EAAA,QAKA,wBAAA;EAAA,QAWA,oBAAA;EAAA,QAWA,uBAAA;EAAA,QAQA,qBAAA;EAAA,QA+BA,UAAA;EAAA,QAaA,2BAAA;EAAA,QAkBA,oBAAA;EAnNA;;;;;;;EAAA,QA8NA,YAAA;AAAA"}
1
+ {"version":3,"file":"intelligence.d.cts","names":[],"sources":["../../../../src/v2/runtime/runner/intelligence.ts"],"mappings":";;;;;;UAsBiB,8BAAA;EAA8B;EAE7C,GAAA;EAmBuD;EAjBvD,SAAA;EAAA;EAEA,cAAA;EAEA;EAAA,WAAA;EAaA;;;;;EAPA,mBAAA;EAUuC;;;;;;EAHvC,oBAAA,IAAwB,KAAA,EAAO,wBAAA;AAAA;AAAA,UAGhB,wBAAA;EACf,QAAA;EACA,KAAA;EAWe;EATf,kBAAA;;EAEA,YAAA;EAQQ;EANR,aAAA;EAOgB;EALhB,kBAAA;AAAA;AAAA,UAGe,qBAAA;EACf,MAAA,EAAQ,UAAA,CAAW,SAAA;EACnB,OAAA,EAAS,OAAA;AAAA;AAAA,cA8DE,uBAAA,SAAgC,WAAA;EAAA,QACnC,OAAA;EAAA,QACA,OAAA;cAEI,OAAA,EAAS,8BAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;UAiCb,YAAA;EAAA,QAiBA,wBAAA;EAAA,QAqBA,0BAAA;EA+CJ;EAAA,QApCI,yBAAA;EAAA,QAQA,mBAAA;EA4BR,GAAA,CAAI,OAAA,EAAS,qBAAA,GAAwB,UAAA,CAAW,SAAA;EAIhD,sBAAA,CACE,OAAA,EAAS,qBAAA,GACR,qBAAA;EAAA,QAiBK,mBAAA;EAgPR,OAAA,CAAQ,OAAA,EAAS,yBAAA,GAA4B,UAAA,CAAW,SAAA;EA4CxD,SAAA,CAAU,OAAA,EAAS,2BAAA,GAA8B,OAAA;EA5CjD;EAkDA,IAAA,CAAK,OAAA,EAAS,sBAAA,GAAyB,OAAA;EAAA,QA0DzB,eAAA;EA5G+B;;;;;;;;EAAA,QA8OrC,gBAAA;EA5L+B;EAAA,QAsO/B,mBAAA;EAAA,QA0BA,qBAAA;EAAA,QA+DA,mBAAA;EAAA,QAyBA,yBAAA;EAAA,QAgCA,yBAAA;EAAA,QA0BA,sBAAA;EAAA,QAOA,yBAAA;EAAA,QAwBA,sBAAA;EAAA,QAOA,sBAAA;EAAA,QAOA,mBAAA;EAAA,QAgCA,sBAAA;EAAA,QAiBA,qBAAA;EAAA,QA8BA,mBAAA;EAAA,QAgBA,aAAA;EAAA,QAKA,cAAA;EAAA,QAKA,wBAAA;EAAA,QAWA,oBAAA;EAAA,QAWA,uBAAA;EAAA,QAQA,qBAAA;EAAA,QA4BA,UAAA;EAAA,QAaA,2BAAA;EAAA,QAeA,oBAAA;EAfA;;;;;;;EAAA,QA0BA,YAAA;AAAA"}
@@ -19,6 +19,25 @@ interface IntelligenceAgentRunnerOptions {
19
19
  * @default 15_000
20
20
  */
21
21
  heartbeatIntervalMs?: number;
22
+ /**
23
+ * Called each time Intelligence acknowledges a batch of runner events.
24
+ * Use it to watch the delivery queue: a growing `pendingEvents` or
25
+ * `oldestPendingAgeMs` means events are produced faster than Intelligence
26
+ * accepts them. Errors thrown by the callback are logged and ignored.
27
+ */
28
+ onEventDeliveryStats?: (stats: RunnerEventDeliveryStats) => void;
29
+ }
30
+ interface RunnerEventDeliveryStats {
31
+ threadId: string;
32
+ runId: string;
33
+ /** Events in the batch Intelligence just acknowledged. */
34
+ acknowledgedEvents: number;
35
+ /** Time from sending the batch to its acknowledgement. */
36
+ ackLatencyMs: number;
37
+ /** Events still waiting for acknowledgement. */
38
+ pendingEvents: number;
39
+ /** Age of the oldest event still waiting, or 0 when none is. */
40
+ oldestPendingAgeMs: number;
22
41
  }
23
42
  interface RunnerStartupBoundary {
24
43
  events: Observable<BaseEvent>;
@@ -69,9 +88,19 @@ declare class IntelligenceAgentRunner extends AgentRunner {
69
88
  /** Stops this run and waits until its terminal events have been acknowledged. */
70
89
  stop(request: AgentRunnerStopRequest): Promise<boolean | undefined>;
71
90
  private executeAgentRun;
72
- /** Queue one immutable event payload until Redis-backed gateway acknowledgement. */
91
+ /**
92
+ * Queue one immutable event payload until Redis-backed gateway acknowledgement.
93
+ *
94
+ * `pushCanonicalEvent` is the only producer and stamps a fresh, increasing
95
+ * `cpki_event_seq` on every event it queues, so the map's insertion order is
96
+ * sequence order. Batches are therefore read from the front of the map
97
+ * instead of sorting the whole queue on every event.
98
+ */
73
99
  private queueRunnerEvent;
100
+ /** The oldest `limit` pending events, in sequence order. */
101
+ private oldestPendingEvents;
74
102
  private pushPendingEventBatch;
103
+ private reportEventDelivery;
75
104
  private handlePendingEventFailure;
76
105
  private schedulePendingEventRetry;
77
106
  private clearPendingEventRetry;
@@ -101,5 +130,5 @@ declare class IntelligenceAgentRunner extends AgentRunner {
101
130
  private removeThread;
102
131
  }
103
132
  //#endregion
104
- export { IntelligenceAgentRunner, IntelligenceAgentRunnerOptions, RunnerStartupBoundary };
133
+ export { IntelligenceAgentRunner, IntelligenceAgentRunnerOptions, RunnerEventDeliveryStats, RunnerStartupBoundary };
105
134
  //# sourceMappingURL=intelligence.d.mts.map
@@ -1 +1 @@
1
- {"version":3,"file":"intelligence.d.mts","names":[],"sources":["../../../../src/v2/runtime/runner/intelligence.ts"],"mappings":";;;;;;UAqBiB,8BAAA;EAA8B;EAE7C,GAAA;EAF6C;EAI7C,SAAA;EAAA;EAEA,cAAA;EAEA;EAAA,WAAA;EAMmB;;AAGrB;;;EAHE,mBAAA;AAAA;AAAA,UAGe,qBAAA;EACf,MAAA,EAAQ,UAAA,CAAW,SAAA;EACnB,OAAA,EAAS,OAAA;AAAA;AAAA,cA6CE,uBAAA,SAAgC,WAAA;EAAA,QACnC,OAAA;EAAA,QACA,OAAA;cAEI,OAAA,EAAS,8BAAA;EAjDL;;AA6ClB;;;;;;;;;;;;;;;;;;;;;;;;;EA7CkB,QAkFR,YAAA;EAAA,QAiBA,wBAAA;EAAA,QAqBA,0BAAA;EAWA;EAAA,QAAA,yBAAA;EAAA,QAQA,mBAAA;EA4BR,GAAA,CAAI,OAAA,EAAS,qBAAA,GAAwB,UAAA,CAAW,SAAA;EAIhD,sBAAA,CACE,OAAA,EAAS,qBAAA,GACR,qBAAA;EAAA,QAiBK,mBAAA;EA8OR,OAAA,CAAQ,OAAA,EAAS,yBAAA,GAA4B,UAAA,CAAW,SAAA;EA4CxD,SAAA,CAAU,OAAA,EAAS,2BAAA,GAA8B,OAAA;EA5StC;EAkTX,IAAA,CAAK,OAAA,EAAS,sBAAA,GAAyB,OAAA;EAAA,QA0DzB,eAAA;EA1VN;EAAA,QAsdA,gBAAA;EAAA,QAkBA,qBAAA;EAAA,QAqDA,yBAAA;EAAA,QAgCA,yBAAA;EAAA,QA0BA,sBAAA;EAAA,QAOA,yBAAA;EAAA,QA0BA,sBAAA;EAAA,QAOA,sBAAA;EAAA,QAOA,mBAAA;EAAA,QAiCA,sBAAA;EAAA,QAuBA,qBAAA;EAAA,QA8BA,mBAAA;EAAA,QAgBA,aAAA;EAAA,QAKA,cAAA;EAAA,QAKA,wBAAA;EAAA,QAWA,oBAAA;EAAA,QAWA,uBAAA;EAAA,QAQA,qBAAA;EAAA,QA+BA,UAAA;EAAA,QAaA,2BAAA;EAAA,QAkBA,oBAAA;EAnNA;;;;;;;EAAA,QA8NA,YAAA;AAAA"}
1
+ {"version":3,"file":"intelligence.d.mts","names":[],"sources":["../../../../src/v2/runtime/runner/intelligence.ts"],"mappings":";;;;;;UAsBiB,8BAAA;EAA8B;EAE7C,GAAA;EAmBuD;EAjBvD,SAAA;EAAA;EAEA,cAAA;EAEA;EAAA,WAAA;EAaA;;;;;EAPA,mBAAA;EAUuC;;;;;;EAHvC,oBAAA,IAAwB,KAAA,EAAO,wBAAA;AAAA;AAAA,UAGhB,wBAAA;EACf,QAAA;EACA,KAAA;EAWe;EATf,kBAAA;;EAEA,YAAA;EAQQ;EANR,aAAA;EAOgB;EALhB,kBAAA;AAAA;AAAA,UAGe,qBAAA;EACf,MAAA,EAAQ,UAAA,CAAW,SAAA;EACnB,OAAA,EAAS,OAAA;AAAA;AAAA,cA8DE,uBAAA,SAAgC,WAAA;EAAA,QACnC,OAAA;EAAA,QACA,OAAA;cAEI,OAAA,EAAS,8BAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;UAiCb,YAAA;EAAA,QAiBA,wBAAA;EAAA,QAqBA,0BAAA;EA+CJ;EAAA,QApCI,yBAAA;EAAA,QAQA,mBAAA;EA4BR,GAAA,CAAI,OAAA,EAAS,qBAAA,GAAwB,UAAA,CAAW,SAAA;EAIhD,sBAAA,CACE,OAAA,EAAS,qBAAA,GACR,qBAAA;EAAA,QAiBK,mBAAA;EAgPR,OAAA,CAAQ,OAAA,EAAS,yBAAA,GAA4B,UAAA,CAAW,SAAA;EA4CxD,SAAA,CAAU,OAAA,EAAS,2BAAA,GAA8B,OAAA;EA5CjD;EAkDA,IAAA,CAAK,OAAA,EAAS,sBAAA,GAAyB,OAAA;EAAA,QA0DzB,eAAA;EA5G+B;;;;;;;;EAAA,QA8OrC,gBAAA;EA5L+B;EAAA,QAsO/B,mBAAA;EAAA,QA0BA,qBAAA;EAAA,QA+DA,mBAAA;EAAA,QAyBA,yBAAA;EAAA,QAgCA,yBAAA;EAAA,QA0BA,sBAAA;EAAA,QAOA,yBAAA;EAAA,QAwBA,sBAAA;EAAA,QAOA,sBAAA;EAAA,QAOA,mBAAA;EAAA,QAgCA,sBAAA;EAAA,QAiBA,qBAAA;EAAA,QA8BA,mBAAA;EAAA,QAgBA,aAAA;EAAA,QAKA,cAAA;EAAA,QAKA,wBAAA;EAAA,QAWA,oBAAA;EAAA,QAWA,uBAAA;EAAA,QAQA,qBAAA;EAAA,QA4BA,UAAA;EAAA,QAaA,2BAAA;EAAA,QAeA,oBAAA;EAfA;;;;;;;EAAA,QA0BA,YAAA;AAAA"}