@parall/agent-core 1.44.0 → 1.46.0

package/src/gateway-lane-flow.ts

@@ -1,3 +1,4 @@
+import { ApiError } from '@parall/sdk';
 import type { ParallClient } from '@parall/sdk';
 import type { DispatchAdapter, GatewayLogger } from './dispatch-adapter.js';
 import type { DispatchableMessage, MessageDispatchDecision } from './gateway-base.js';
@@ -16,8 +17,15 @@ import type { ParallEvent } from './types.js';
 export interface LaneFlowHost {
   laneLedger?: LaneLedger;
   ledgerDisabled: boolean;
+  /**
+   * Sticky: the server answered a by-id complete with 400 (predates the
+   * form, v1.44). Typed resolution falls back to the legacy ack for the
+   * rest of the process; claims stay on the ledger (that face is older).
+   */
+  typedByIdCompleteUnsupported: boolean;
   shuttingDown: boolean;
   dispatchedMessages: Set<string>;
+  dispatchedTasks: Set<string>;
   /**
    * Per-WorkItem failure backoff for typed dispatch consumption. A consume
    * that ends without an ack re-arms the entry; the next attempt for the
@@ -173,19 +181,208 @@ export async function dispatchLaneGroup(
   return 'dispatched';
 }
 
+/**
+ * The WorkItem ids of a typed event group whose lifecycle the ledger owns
+ * (claimed into dsp lanes by consumeTypedDispatch), or null when the group
+ * must stay on the legacy received/ack surface — ledger unavailable, an id
+ * missing, or a member that rides a message lane. Non-null means: skip
+ * legacy received+ack, resolve via the by-id complete, and fold the
+ * turn-error signal into the outcome.
+ */
+export function typedLedgerEventIds(host: LaneFlowHost, events: ParallEvent[]): string[] | null {
+  if (!host.laneLedger || host.ledgerDisabled) return null;
+  const ids: string[] = [];
+  for (const ev of events) {
+    if (!ev.dispatchEventId || host.usesLaneLedger(ev)) return null;
+    ids.push(ev.dispatchEventId);
+  }
+  return ids.length > 0 ? ids : null;
+}
+
+/** Outcome of a by-id complete attempt. */
+export type TypedResolveOutcome = 'ok' | 'stale' | 'unsupported' | 'failed';
+
+/**
+ * A pre-by-id server (v1.44) routes a by-id body into its lane-form
+ * validation and answers 400 INVALID_REQUEST. Any 400 here means the server
+ * did not understand the form — a well-formed by-id call never 400s on a
+ * current server — so the caller falls back to the legacy typed ack for the
+ * rest of the process (sticky; the bridge restarts on the next update).
+ */
+function isByIDCompleteUnsupported(err: unknown): boolean {
+  return err instanceof ApiError && err.status === 400;
+}
+
+/**
+ * By-id complete (turn_outcome ok): terminal resolution of ONE WorkItem —
+ * exact where the source pair is ambiguous across task siblings. `lane`
+ * fences the call while the row is claim-owned (a dethroned caller gets
+ * 'stale' and must leave the row to its successor); omit it for the
+ * wrapper-less pending close (buffered drain, administrative drops).
+ */
+export async function resolveDispatchByID(
+  host: LaneFlowHost,
+  dispatchEventId: string,
+  lane?: string,
+): Promise<TypedResolveOutcome> {
+  try {
+    await host.opts.client.completeDispatch(host.opts.config.org_id, {
+      dispatch_event_id: dispatchEventId,
+      ...(lane ? { lane } : {}),
+      turn_outcome: 'ok',
+    });
+    return 'ok';
+  } catch (err) {
+    if (err instanceof ApiError && err.status === 409) return 'stale';
+    if (isByIDCompleteUnsupported(err)) return 'unsupported';
+    host.opts.log?.warn(
+      `by-id complete failed for ${dispatchEventId} — leaving for re-drive: ${String(err)}`,
+    );
+    return 'failed';
+  }
+}
+
+/**
+ * Free a typed event's in-memory hot-path dedupe claim by its source pair —
+ * the ParallEvent-keyed mirror of the gateway's clearTypedDispatchDedupe
+ * (which keys on the wire DispatchNewData). Used when a buffered typed
+ * dispatch ran but its resolution failed: the member re-drives, and a stale
+ * local claim would make this pod reject the retry forever.
+ *
+ * PARITY: the switch below and clearTypedDispatchDedupe's must handle the
+ * same typed source families — when adding a new typed event type, extend
+ * BOTH (they key the same dedupe entries from different event shapes).
+ */
+export function clearTypedDedupeForEvent(host: LaneFlowHost, event: ParallEvent): void {
+  const sourceId = event.ackSourceId;
+  if (!sourceId) return;
+  switch (event.ackSourceType) {
+    case 'task_activity': {
+      // Task dedupe keys are `${task_id}:${updated_at}`; the task id is the
+      // event target. Prefix-clear mirrors clearTypedDispatchDedupe.
+      const prefix = `${event.targetId}:`;
+      for (const key of host.dispatchedTasks) {
+        if (key.startsWith(prefix)) host.dispatchedTasks.delete(key);
+      }
+      break;
+    }
+    case 'comment':
+      host.dispatchedTasks.delete(`comment:${sourceId}`);
+      break;
+    case 'schedule_run':
+      host.dispatchedTasks.delete(`schedule_run:${sourceId}`);
+      break;
+    case 'external_trigger_run':
+      host.dispatchedTasks.delete(`external_trigger_run:${sourceId}`);
+      break;
+    case 'channel_message':
+      host.dispatchedMessages.delete(`channel_message:${sourceId}`);
+      break;
+    case 'approval':
+      host.dispatchedTasks.delete(`approval:${sourceId}`);
+      break;
+  }
+}
+
+/**
+ * Settle a drained typed group — the one wrapper-less dispatch path: these
+ * events buffered behind a busy main (fork-less adapter), their consume
+ * guards returned false and released the claims, so the drain site owns the
+ * terminal resolution. Rows are pending — the by-id close resolves them
+ * without a fence; a row a successor lane meanwhile claimed answers stale and
+ * is that owner's to finish. `turnErrored` (consumed at the call site before
+ * the drain can start another turn) leaves the rows for the renotify pacing
+ * instead. Either way a row left live gets its local dedupe claim freed, or
+ * this pod would reject the retry forever.
+ */
+export async function settleDrainedTypedGroup(
+  host: LaneFlowHost,
+  events: ParallEvent[],
+  ids: string[],
+  turnErrored: boolean,
+): Promise<void> {
+  if (turnErrored) {
+    host.opts.log?.info(
+      `buffered typed turn for ${events[events.length - 1]?.messageId} surfaced a runtime error — leaving for re-drive`,
+    );
+    for (const event of events) clearTypedDedupeForEvent(host, event);
+    return;
+  }
+  // The legacy ack BY ID, never by source — task siblings share a source
+  // pair, and a by-source ack would sweep the undispatched one. Awaited, and
+  // the dedupe claim is freed on EVERY outcome: a claim's lifetime is the
+  // buffer stay (#1149). On failure the row is still pending and its retry
+  // must not be self-rejected; on success the row is terminal — but a
+  // shared-key sibling WorkItem (task assign+update fetch the same
+  // `task:updated_at` pair) must not be rejected by a leftover claim either.
+  const legacyAckFrom = async (start: number) => {
+    for (const [j, id] of ids.slice(start).entries()) {
+      try {
+        await host.opts.client.ackDispatchByID(host.opts.config.org_id, id);
+      } catch {
+        // Transient ack failure: the row stays live server-side and its
+        // re-drive re-converges — keep settling the REST of the group (a
+        // propagated error would strand every later id's retained claim).
+      } finally {
+        clearTypedDedupeForEvent(host, events[start + j]);
+      }
+    }
+  };
+  if (host.typedByIdCompleteUnsupported) {
+    // Sticky: the server already answered one by-id probe with 400 — go
+    // straight to the legacy acks instead of re-probing per group.
+    await legacyAckFrom(0);
+    return;
+  }
+  for (const [i, id] of ids.entries()) {
+    const outcome = await resolveDispatchByID(host, id);
+    if (outcome === 'unsupported') {
+      host.typedByIdCompleteUnsupported = true;
+      host.opts.log?.warn(
+        'server predates the by-id dispatch complete — falling back to legacy typed acks',
+      );
+      await legacyAckFrom(i);
+      return;
+    }
+    // Freed on every outcome — see legacyAckFrom's contract note.
+    clearTypedDedupeForEvent(host, events[i]);
+  }
+}
+
 /** Failure backoff pacing for typed dispatch retries (base 2s, cap 5min). */
 const TYPED_BACKOFF_BASE_MS = 2_000;
 const TYPED_BACKOFF_CAP_MS = 5 * 60_000;
 const TYPED_BACKOFF_MAP_CAP = 512;
 
+/** Consumption hooks for one typed dispatch. */
+export interface TypedConsumeHooks {
+  /**
+   * Legacy administrative ack — invoked ONLY on the ledger-disabled fallback
+   * path (old server). On the ledger path the WorkItem resolves through the
+   * sources-form complete instead; this callback never fires there.
+   */
+  legacyAck: (dispatchEventId?: string) => boolean | void | Promise<boolean | void>;
+  /**
+   * Free the item's in-memory hot-path dedupe claim. Invoked when the handler
+   * ran but the resolution could not commit (sources-form complete failed) —
+   * the member is released for re-drive, and a stale local claim would make
+   * this pod reject its own retry forever.
+   */
+  clearDedupe?: () => void;
+}
+
 /**
  * Consume one typed dispatch (task/comment/schedule/trigger/approval) under
  * its typed-lane occupancy guard: claim the dsp:<id> lane (skip when another
- * pod holds it or the WorkItem is already resolved), run the handler, ack on
- * success (the doc's option (b): notification delivered, tracked elsewhere),
- * then release the lane. Legacy run+ack flow when the ledger is unavailable.
+ * pod holds it or the WorkItem is already resolved), run the handler, then
+ * settle. A successful run resolves the WorkItem terminally via the
+ * sources-form complete (turn_outcome ok — the server sweeps it, or lets a
+ * typed Effect committed during the turn stand, and drops the vacated lane);
+ * a failed / errored / unresolved run releases the member back to pending on
+ * the redrive budget via the lane complete. Legacy run+ack flow when the
+ * ledger is unavailable.
  *
- * Repeated failures back off: a consume that ends un-acked re-arms the
+ * Repeated failures back off: a consume that ends unresolved re-arms the
  * WorkItem's backoff entry, and the next attempt sleeps out the remaining
  * window before claiming. Without this, complete's release re-drives the
  * item instantly and a persistently-failing consume (e.g. buffered behind a
@@ -196,7 +393,7 @@ export async function consumeTypedDispatch(
   host: LaneFlowHost,
   ref: { dispatchEventId?: string; sourceType?: string; sourceId?: string },
   run: (dispatchEventId?: string) => Promise<boolean>,
-  ack: (dispatchEventId?: string) => boolean | void | Promise<boolean | void>,
+  hooks: TypedConsumeHooks,
 ): Promise<void> {
   const backoffKey = ref.dispatchEventId ?? `${ref.sourceType}:${ref.sourceId}`;
   const armed = host.typedRedriveBackoff.get(backoffKey);
@@ -221,8 +418,8 @@ export async function consumeTypedDispatch(
   // consume — clearing backoff there would let an ack outage re-create the
   // wire-speed release/re-drive loop against a pre-budget server.
   const settleAck = (ackResult: boolean | void) => ackResult !== false;
-  const settle = (acked: boolean) => {
-    if (acked) {
+  const settle = (resolved: boolean) => {
+    if (resolved) {
       host.typedRedriveBackoff.delete(backoffKey);
       return;
     }
@@ -249,7 +446,7 @@ export async function consumeTypedDispatch(
     let acked = false;
     try {
       if (await run(ref.dispatchEventId)) {
-        acked = settleAck(await ack(ref.dispatchEventId));
+        acked = settleAck(await hooks.legacyAck(ref.dispatchEventId));
       }
     } finally {
       settle(acked);
@@ -279,20 +476,65 @@ export async function consumeTypedDispatch(
     );
     return;
   }
-  let acked = false;
+  let resolved = false;
+  let viaLegacyAck = false;
   try {
-    // Ack must settle before Complete. A fire-and-forget ack races the lane
-    // release: Complete can return the still-received typed item to pending
-    // and publish a re-drive while its successful ack is still in flight.
+    // A successful run resolves the member terminally through the by-id
+    // complete — one fenced call that sweeps exactly this WorkItem (a typed
+    // Effect committed during the turn already made it terminal — idempotent
+    // no-op) AND drops the vacated lane. Resolution must settle inside this
+    // guard: a fire-and-forget resolve would race the finally's release and
+    // spuriously re-drive handled work.
     if (await run(lane.typedDispatchEventId)) {
-      acked = settleAck(await ack(lane.typedDispatchEventId));
+      if (host.typedByIdCompleteUnsupported || !lane.typedDispatchEventId) {
+        // Pre-by-id server (or a defensive claim shape without the id):
+        // the legacy administrative ack is still live there.
+        viaLegacyAck = true;
+        resolved = settleAck(await hooks.legacyAck(lane.typedDispatchEventId));
+      } else {
+        const outcome = await resolveDispatchByID(host, lane.typedDispatchEventId, lane.lane);
+        if (outcome === 'unsupported') {
+          host.typedByIdCompleteUnsupported = true;
+          host.opts.log?.warn(
+            'server predates the by-id dispatch complete — falling back to the legacy typed ack',
+          );
+          viaLegacyAck = true;
+          resolved = settleAck(await hooks.legacyAck(lane.typedDispatchEventId));
+        } else if (outcome === 'stale') {
+          // A successor lane owns the row — its turn resolves it. Nothing to
+          // spin on locally; our claim record is already dethroned. Free the
+          // local dedupe claim though: if the successor later FAILS and
+          // releases the row, the re-drive may land back on this pod, and a
+          // stale local claim would self-reject the retry into the budget.
+          hooks.clearDedupe?.();
+          resolved = true;
+        } else {
+          resolved = outcome === 'ok';
+          if (!resolved) {
+            // The member is about to be released for re-drive — free its
+            // hot-path dedupe claim first, or this pod rejects its own retry
+            // forever (same contract as the legacy ack-failure path).
+            hooks.clearDedupe?.();
+          }
+        }
+      }
     }
   } finally {
-    settle(acked);
-    // Release the occupancy row. A buffered dispatch may outlive this guard
-    // (lane TTL) — acceptable at-least-once; the ledger's resolution paths
-    // still dedupe the persistent side effects.
-    await host.laneLedger.completeIfIdle(lane.laneKey, false).catch(() => {});
+    settle(resolved);
+    if (resolved && !viaLegacyAck) {
+      // The by-id complete already dropped (or a takeover already owns) the
+      // server-side lane row; only the local record and context file remain.
+      host.laneLedger.dropLocal(lane.laneKey);
+    } else {
+      // Two jobs, one call: an unresolved turn (unconsumed / failed /
+      // errored / resolution-failed) releases the member back to pending on
+      // the redrive budget; a legacy-acked turn still needs its server-side
+      // lane row dropped (the ack resolves the row but not the occupancy).
+      // A buffered dispatch may outlive this guard (lane TTL) — acceptable
+      // at-least-once; the ledger's resolution paths still dedupe the
+      // persistent side effects.
+      await host.laneLedger.completeIfIdle(lane.laneKey, false).catch(() => {});
+    }
   }
 }
 
@@ -310,7 +552,22 @@ export async function consumeMessageWorkItem(
 ): Promise<void> {
   if (host.shuttingDown) return;
   if (!host.tryClaimMessage(item.source_id)) return;
+  // Administrative drop (deleted source / self-sender / skip decision): on
+  // the ledger path the by-id complete closes the pending row terminally and
+  // refuses (409) a row a live lane owns — the owner's turn resolves it. The
+  // legacy ack remains the ledger-disabled / pre-by-id fallback.
+  // Fire-and-forget either way: a failed drop leaves the row live and the
+  // next re-drive converges on the same drop.
   const ackItem = () => {
+    if (host.laneLedger && !host.ledgerDisabled && !host.typedByIdCompleteUnsupported) {
+      void resolveDispatchByID(host, item.id).then((outcome) => {
+        if (outcome === 'unsupported') {
+          host.typedByIdCompleteUnsupported = true;
+          host.opts.client.ackDispatchByID(host.opts.config.org_id, item.id).catch(() => {});
+        }
+      });
+      return;
+    }
     host.opts.client.ackDispatchByID(host.opts.config.org_id, item.id).catch(() => {});
   };
 

package/src/http-keepalive.ts

@@ -0,0 +1,36 @@
+import { Agent, setGlobalDispatcher } from 'undici';
+
+/**
+ * Keep bridge→api HTTP connections long-lived.
+ *
+ * Node's built-in fetch closes idle pooled connections after ~4s when the
+ * server sends no Keep-Alive hint (our NLB+Envoy edge sends none), so any two
+ * bridge API calls spaced more than a few seconds apart pay a fresh TCP+TLS
+ * handshake. On a degraded long-haul link that handshake is exactly what
+ * fails — established connections (the WS, the runtime's own model stream)
+ * ride through such windows while every bridge call times out (2026-07-10:
+ * step creation losses). A 2-minute idle timeout keeps one warm connection
+ * across an active turn's step cadence, comfortably below the NLB's 350s
+ * idle cutoff.
+ *
+ * Uses undici's global-dispatcher registry, which Node's built-in fetch
+ * shares (verified on Node 22 / undici 7) — so this covers every fetch in
+ * the process: ParallClient, lane-ledger heartbeats, OTLP export.
+ * Node-only; call once from a bridge/daemon entrypoint before any fetch.
+ */
+let installed = false;
+
+export function configureHttpKeepAlive(opts?: { keepAliveTimeoutMs?: number }): void {
+  // Idempotent: a second call (e.g. a process hosting several bridge
+  // entrypoints) must not replace — and leak — the already-installed Agent.
+  if (installed) return;
+  installed = true;
+  const keepAliveTimeout = opts?.keepAliveTimeoutMs ?? 120_000;
+  setGlobalDispatcher(
+    new Agent({
+      keepAliveTimeout,
+      // Also cap the server-hinted value: never idle past the edge's cutoff.
+      keepAliveMaxTimeout: keepAliveTimeout,
+    }),
+  );
+}

package/src/index.ts

@@ -2,7 +2,7 @@ export * from './provider-config.js';
 export * from './types.js';
 export * from './lane-key.js';
 export type { LaneFlowHost } from './gateway-lane-flow.js';
-export { consumeTypedDispatch } from './gateway-lane-flow.js';
+export { consumeTypedDispatch, settleDrainedTypedGroup } from './gateway-lane-flow.js';
 export * from './session-state.js';
 export * from './routing.js';
 export * from './event-format.js';
@@ -11,6 +11,12 @@ export * from './bridge-workspace.js';
 export * from './dispatch-adapter.js';
 export { createLogger, childLogger } from './logger.js';
 export * from './gateway-base.js';
+export { configureHttpKeepAlive } from './http-keepalive.js';
+// StepPersister / StepRetryQueue / SessionLifecycleCoordinator /
+// ForkSessionFinalizer are internal gateway collaborators — deliberately NOT
+// re-exported from the package root (no external consumer; a root export
+// would become an accidental long-term compat promise). Tests import their
+// built modules directly (e.g. `../dist/step-retry-queue.js`).
 export * from './platform-config.js';
 export * from './channel-capability.js';
 export * from './channel-token.js';

package/src/lane-ledger.ts

@@ -393,6 +393,20 @@ export class LaneLedger {
     return lane;
   }
 
+  /**
+   * Drop a lane's local record without any server call — for a typed lane
+   * whose member the by-id complete just resolved (the server dropped the
+   * vacated lane row in the same transaction). Calling completeIfIdle
+   * instead would fire a lane-form Complete at a lane that no longer exists
+   * and burn an RPC on the guaranteed STALE_LANE answer.
+   */
+  dropLocal(laneKey: string): void {
+    const lane = this.lanes.get(laneKey);
+    if (!lane) return;
+    this.lanes.delete(laneKey);
+    this.removeLaneContext(lane);
+  }
+
   /**
    * Remove the per-lane context file (and its CLI sidecar) when the lane
    * ends. A leftover file would make a later cross-context send to the same