@parall/agent-core 1.45.0 → 1.47.0

package/src/gateway-base.ts

@@ -1,6 +1,7 @@
 import * as os from 'node:os';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
+import { randomUUID } from 'node:crypto';
 import { ApiError, MENTION_ALL_USER_ID, ParallClient, ParallWs } from '@parall/sdk';
 import type {
   AgentConfigUpdateData,
@@ -28,6 +29,7 @@ import {
   buildForkResultPrefix,
   buildForkScopePrefix,
 } from './event-format.js';
+import { CAPABILITY_SLACK_SEND, channelCapabilityKeyFor } from './channel-capability.js';
 import {
   buildErrorStepContent,
   type CleanupForkOpts,
@@ -38,6 +40,7 @@ import {
   type RuntimeEvent,
 } from './dispatch-adapter.js';
 import {
+  clearTypedDedupeForEvent,
   consumeMessageWorkItem,
   consumeTypedDispatch,
   dispatchLaneGroup,
@@ -48,6 +51,9 @@ import {
 import type { LaneFlowHost, TypedConsumeHooks } from './gateway-lane-flow.js';
 import { LaneLedger } from './lane-ledger.js';
 import { routeTrigger } from './routing.js';
+import { StepPersister, isRetryableStepError } from './step-persister.js';
+import { SessionLifecycleCoordinator, type TurnHandle } from './session-lifecycle.js';
+import { ForkSessionFinalizer } from './fork-session-finalizer.js';
 import {
   clearDispatchMessageId,
   clearDispatchMetrics,
@@ -141,6 +147,10 @@ export type ParallGatewayOptions = {
   shutdownDeadlineMs?: number;
   forkDeadlineMs?: number;
   dispatchDeadlineMs?: number;
+  /** Test port: step retry-queue backoff override (StepRetryQueue schedule). */
+  stepRetryDelaysMs?: number[];
+  /** Test port: session lifecycle reconcile backoff override. */
+  lifecycleRetryDelaysMs?: number[];
   contextFilePathForSession?: (sessionKey: string) => string | undefined;
   /** @deprecated Use contextFilePathForSession. Kept for runtimes that haven't migrated. */
   stepIdFilePathForSession?: (sessionKey: string) => string | undefined;
@@ -213,6 +223,36 @@ export function parseDispatchDeadlineMs(raw: string | undefined): number | undef
   return Math.floor(n);
 }
 
+/**
+ * Stable input-step idempotency key: a re-delivered event (catch-up replay,
+ * lane re-drive) replays to the same row instead of duplicating it.
+ *
+ * Message-shaped events key on the message id — it is the logical identity
+ * and stays stable across delivery paths (a live WS delivery carries no
+ * dispatchEventId; its catch-up replay does — keying on the WorkItem there
+ * would split the two into different keys and duplicate the input step).
+ *
+ * Typed events key on the WorkItem id instead: their messageId can be
+ * REUSED across distinct work items — task events carry the task id for
+ * both task_assign and a later task_update, so keying on messageId would
+ * silently dedupe the second, legitimate, input step away. For the same
+ * reason a typed event WITHOUT a WorkItem id (legacy server) must NOT fall
+ * back to messageId — it gets a random UUID per logical step instead. The
+ * key rides the CreateAgentStepRequest object, so the same UUID is reused
+ * across that step's HTTP retries and queue redrives (replay-safe), while
+ * a redelivered legacy event writes a fresh row (rare duplicate beats
+ * silently losing a legitimate step). See protocol-vectors/agent-steps.json.
+ */
+export function inputStepIdempotencyKey(
+  event: Pick<ParallEvent, 'type' | 'dispatchEventId' | 'messageId'>,
+): string {
+  if (event.type === 'message' || event.type === 'channel_message') {
+    return event.messageId ? `input:${event.messageId}` : randomUUID();
+  }
+  if (event.dispatchEventId) return `input:${event.dispatchEventId}`;
+  return randomUUID();
+}
+
 function resolveStepTarget(event: ParallEvent): { target_type: string; target_id?: string } {
   if (event.type === 'task' || event.targetId.startsWith('tsk_')) {
     return { target_type: 'task', target_id: event.targetId };
@@ -322,6 +362,13 @@ export class ParallAgentGateway {
 
   private lastHeartbeatAt = Date.now();
   private draining = false;
+  /**
+   * Typed WorkItem ids whose drain group left the buffer but has not settled
+   * yet. isBufferedTypedWorkItem treats them as still buffered — a re-drive
+   * claim taken inside this window would turn the drain's fence-less close
+   * stale (its dedupe cleared, its release re-driving handled work) (#1149).
+   */
+  private drainingTypedIds = new Set<string>();
 
   // Graceful shutdown state. When SIGTERM / abort fires, `shuttingDown` flips
   // to true so no new dispatches start, and `inFlightDispatches` counts runs
@@ -333,6 +380,16 @@ export class ParallAgentGateway {
   private pendingRestartNotification: string | null = null;
 
   private readonly laneLedger?: LaneLedger;
+  private readonly stepPersister: StepPersister;
+  // Single entry point for session active/idle writes — serialized
+  // desired-state reconciler (see session-lifecycle.ts). The gateway only
+  // declares turn boundaries; ordering, retries and stale-finish rejection
+  // live in the coordinator.
+  private readonly sessionLifecycle: SessionLifecycleCoordinator;
+  // Normal fork teardown use case: seal → drain → close → release. The
+  // gateway only triggers it; ordering and ownership live in the finalizer
+  // (see fork-session-finalizer.ts).
+  private readonly forkFinalizer: ForkSessionFinalizer;
   // Sticky fallback: flipped when the server predates the ledger (claim
   // endpoint 404) so every subsequent dispatch uses the legacy flow.
   private ledgerDisabled = false;
@@ -361,6 +418,35 @@ export class ParallAgentGateway {
     this.SHUTDOWN_DEADLINE_MS = opts.shutdownDeadlineMs ?? 60_000;
     this.FORK_DEADLINE_MS = opts.forkDeadlineMs ?? 2 * 60 * 60_000;
     this.DISPATCH_DEADLINE_MS = opts.dispatchDeadlineMs ?? 20 * 60_000;
+    this.stepPersister = new StepPersister({
+      client: opts.client,
+      orgId: opts.config.org_id,
+      agentUserId: opts.agentUserId,
+      log: { warn: (msg) => this.opts.log?.warn(msg) },
+      isSessionStale: (err) => this.isSessionNotLiveError(err),
+      retryDelaysMs: opts.stepRetryDelaysMs,
+    });
+    this.sessionLifecycle = new SessionLifecycleCoordinator({
+      write: (sessionId, payload) =>
+        this.opts.client
+          .updateAgentSession(this.opts.config.org_id, this.opts.agentUserId, sessionId, payload)
+          .then(() => undefined),
+      // Same classification as step writes: transient (timeout/network/5xx)
+      // retries with backoff; a 4xx is permanent (never loop against it);
+      // a terminal-session 409 drops the session for good.
+      isRetryable: isRetryableStepError,
+      isSessionStale: (err) => this.isSessionNotLiveError(err),
+      log: { warn: (msg) => this.opts.log?.warn(msg) },
+      retryDelaysMs: opts.lifecycleRetryDelaysMs,
+    });
+    this.forkFinalizer = new ForkSessionFinalizer({
+      steps: this.stepPersister,
+      lifecycle: this.sessionLifecycle,
+      log: {
+        warn: (msg) => this.opts.log?.warn(msg),
+        error: (msg) => this.opts.log?.error(msg),
+      },
+    });
     if (opts.coldStartWindowMs != null) {
       opts.log?.warn?.(
         'coldStartWindowMs is deprecated and ignored — cold-start time filter has been removed',
@@ -420,6 +506,14 @@ export class ParallAgentGateway {
       const prevId = data.previous_session_id ?? '';
       this.opts.log?.info(`new session signal received (previous=${prevId})`);
       this.sessionBindings.clear();
+      // The server closed EVERY open session — not just `previous_session_id`
+      // (active forks have their own ase_ ids, and sessionBindings was just
+      // cleared, so they would otherwise keep retrying against closed rows
+      // until a 409 or the age budget). Any parked lifecycle write or step
+      // write would now 409 against a terminal row: drop them all rather than
+      // burn retry budget and log noise discovering it.
+      this.sessionLifecycle.dropAllSessions();
+      this.stepPersister.dropAllSessions();
       if (prevId) {
         this.pendingRestartNotification = `[Harness Notification] This is a fresh session. Your previous session (${prevId}) was ended by the user and you have been restarted.`;
       }
@@ -485,6 +579,15 @@ export class ParallAgentGateway {
     });
 
     ws.on('dispatch.new', async (data: DispatchNewData) => {
+      // Re-drive of a WorkItem whose event copy is already buffered: skip
+      // BEFORE the typed claim (see isBufferedTypedWorkItem). The renotify
+      // pacing re-checks after the drain settles.
+      if (data.event_type !== 'message' && this.isBufferedTypedWorkItem(data.id)) {
+        this.opts.log?.info(
+          `typed dispatch ${data.id} already buffered for the drain — skipping re-claim`,
+        );
+        return;
+      }
       if (data.event_type === 'task_assign') {
         if (!data.task_id) return;
         try {
@@ -780,7 +883,14 @@ export class ParallAgentGateway {
       // into one turn.
       return this.laneLedger!.laneKeyFor(event);
     }
-    return event.targetId;
+    // Typed events never share a drain group with messages. On the
+    // ledger-disabled fallback both group by target id, and a typed event
+    // whose target is a chat (approval) could batch behind that chat's
+    // messages — the drain body carries only the trailing message while the
+    // legacy ack sweeps the whole group, silently dropping the typed body
+    // (typed events are never steer-injected, so the drain body is their
+    // only route to the model). Split the groups instead (#1149).
+    return event.type === 'message' ? event.targetId : `typed:${event.targetId}`;
   }
 
   // Lane-flow protocols live in gateway-lane-flow.ts; these thin delegates
@@ -831,6 +941,28 @@ export class ParallAgentGateway {
     );
   }
 
+  /**
+   * True while a typed WorkItem's event copy sits in the main buffer waiting
+   * for the drain. A buffered copy keeps its hot-path dedupe claim: the drain
+   * owns its settlement, and the released row's re-drive must short-circuit
+   * BEFORE claiming (a duplicate claim opens a lane that races the drain's
+   * fence-less by-id close — 409 STALE, and the claimant's release would
+   * re-drive already-handled work) (#1149). The claim's lifetime equals the
+   * buffer stay: settlement clears it on every drain outcome — success,
+   * failure, or a thrown turn.
+   */
+  private isBufferedTypedWorkItem(dispatchEventId: string | undefined | null): boolean {
+    if (!dispatchEventId) return false;
+    return (
+      this.drainingTypedIds.has(dispatchEventId) ||
+      this.dispatchState.mainBuffer.some((e) => e.dispatchEventId === dispatchEventId)
+    );
+  }
+
+  private isTypedEventBuffered(event: ParallEvent): boolean {
+    return this.isBufferedTypedWorkItem(event.dispatchEventId);
+  }
+
   // PARITY: this switch and gateway-lane-flow's clearTypedDedupeForEvent must
   // handle the same typed source families — extend BOTH when adding a typed
   // event type (same dedupe entries, keyed from different event shapes).
@@ -896,69 +1028,60 @@ export class ParallAgentGateway {
 
   private async createInputStep(sessionId: string, event: ParallEvent) {
     const target = resolveStepTarget(event);
-    try {
-      await this.opts.client.createAgentStep(
-        this.opts.config.org_id,
-        this.opts.agentUserId,
-        sessionId,
-        {
-          step_type: 'input',
-          target_type: target.target_type,
-          target_id: target.target_id,
-          content: {
-            trigger_type:
-              event.type === 'task'
-                ? 'task_assign'
-                : event.type === 'task_comment'
-                  ? 'task_comment'
-                  : event.type === 'wiki_comment'
-                    ? 'wiki_comment'
-                    : event.type === 'schedule'
-                      ? 'schedule_fire'
-                      : event.type === 'external_trigger'
-                        ? 'external_trigger'
-                        : event.type === 'channel_message'
-                          ? 'channel_message'
-                          : event.type === 'approval'
-                            ? 'approval_decided'
-                            : 'mention',
-            trigger_ref:
-              event.type === 'task'
-                ? { task_id: event.targetId }
-                : event.type === 'task_comment'
-                  ? { comment_id: event.messageId, task_id: event.targetId }
-                  : event.type === 'wiki_comment'
-                    ? { comment_id: event.messageId, target_uri: event.replyTargetUri }
-                    : event.type === 'schedule'
-                      ? { schedule_id: event.targetId, run_id: event.messageId }
-                      : event.type === 'external_trigger'
-                        ? {
-                            trigger_id: event.targetId,
-                            run_id: event.messageId,
-                            connection_id: event.externalConnectionId,
-                            ingress_event_id: event.externalIngressEventId,
-                          }
-                        : event.type === 'channel_message'
-                          ? {
-                              conversation_id: event.targetId,
-                              channel_message_id: event.messageId,
-                              provider: event.channelProvider,
-                              external_conversation_id: event.channelExternalConversationId,
-                            }
-                          : event.type === 'approval'
-                            ? { approval_id: event.messageId }
-                            : { message_id: event.messageId },
-            sender_id: event.senderId,
-            sender_name: event.senderName,
-            summary: event.body.substring(0, 200),
-            ...(event.sentAt ? { sent_at: event.sentAt } : {}),
-          },
-        },
-      );
-    } catch (err) {
-      if (this.isSessionNotLiveError(err)) throw err;
-      this.opts.log?.warn(`failed to create input step: ${String(err)}`);
-    }
+    await this.stepPersister.persist(sessionId, 'input', {
+      step_type: 'input',
+      target_type: target.target_type,
+      target_id: target.target_id,
+      idempotency_key: inputStepIdempotencyKey(event),
+      content: {
+        trigger_type:
+          event.type === 'task'
+            ? 'task_assign'
+            : event.type === 'task_comment'
+              ? 'task_comment'
+              : event.type === 'wiki_comment'
+                ? 'wiki_comment'
+                : event.type === 'schedule'
+                  ? 'schedule_fire'
+                  : event.type === 'external_trigger'
+                    ? 'external_trigger'
+                    : event.type === 'channel_message'
+                      ? 'channel_message'
+                      : event.type === 'approval'
+                        ? 'approval_decided'
+                        : 'mention',
+        trigger_ref:
+          event.type === 'task'
+            ? { task_id: event.targetId }
+            : event.type === 'task_comment'
+              ? { comment_id: event.messageId, task_id: event.targetId }
+              : event.type === 'wiki_comment'
+                ? { comment_id: event.messageId, target_uri: event.replyTargetUri }
+                : event.type === 'schedule'
+                  ? { schedule_id: event.targetId, run_id: event.messageId }
+                  : event.type === 'external_trigger'
+                    ? {
+                        trigger_id: event.targetId,
+                        run_id: event.messageId,
+                        connection_id: event.externalConnectionId,
+                        ingress_event_id: event.externalIngressEventId,
+                      }
+                    : event.type === 'channel_message'
+                      ? {
+                          conversation_id: event.targetId,
+                          channel_message_id: event.messageId,
+                          provider: event.channelProvider,
+                          external_conversation_id: event.channelExternalConversationId,
+                        }
+                      : event.type === 'approval'
+                        ? { approval_id: event.messageId }
+                        : { message_id: event.messageId },
+        sender_id: event.senderId,
+        sender_name: event.senderName,
+        summary: event.body.substring(0, 200),
+        ...(event.sentAt ? { sent_at: event.sentAt } : {}),
+      },
+    });
   }
 
   private async createRuntimeStep(
@@ -970,62 +1093,57 @@ export class ParallAgentGateway {
     laneContextFilePath?: string,
   ) {
     const target = resolveStepTarget(event);
-    try {
-      switch (runtimeEvent.type) {
-        case 'thinking':
-          await this.opts.client.createAgentStep(
-            this.opts.config.org_id,
-            this.opts.agentUserId,
-            sessionId,
-            {
-              step_type: 'thinking',
-              target_type: target.target_type,
-              target_id: target.target_id,
-              content: { text: runtimeEvent.text },
-              group_key: runtimeEvent.groupKey,
-            },
-          );
-          break;
+    switch (runtimeEvent.type) {
+      case 'thinking':
+        await this.stepPersister.persist(sessionId, 'thinking', {
+          step_type: 'thinking',
+          target_type: target.target_type,
+          target_id: target.target_id,
+          idempotency_key: randomUUID(),
+          content: { text: runtimeEvent.text },
+          group_key: runtimeEvent.groupKey,
+        });
+        break;
 
-        case 'text':
-          await this.opts.client.createAgentStep(
-            this.opts.config.org_id,
-            this.opts.agentUserId,
-            sessionId,
-            {
-              step_type: 'text',
-              target_type: target.target_type,
-              target_id: target.target_id,
-              content: {
-                text: runtimeEvent.text,
-                suppressed: runtimeEvent.project !== true,
-              },
-              projection: runtimeEvent.project === true,
-              group_key: runtimeEvent.groupKey,
-            },
-          );
-          break;
+      case 'text':
+        await this.stepPersister.persist(sessionId, 'text', {
+          step_type: 'text',
+          target_type: target.target_type,
+          target_id: target.target_id,
+          idempotency_key: randomUUID(),
+          content: {
+            text: runtimeEvent.text,
+            suppressed: runtimeEvent.project !== true,
+          },
+          projection: runtimeEvent.project === true,
+          group_key: runtimeEvent.groupKey,
+        });
+        break;
 
-        case 'tool_call': {
-          const step = await this.opts.client.createAgentStep(
-            this.opts.config.org_id,
-            this.opts.agentUserId,
-            sessionId,
-            {
-              step_type: 'tool_call',
-              target_type: target.target_type,
-              target_id: target.target_id,
-              content: {
-                call_id: runtimeEvent.callId,
-                tool_name: runtimeEvent.toolName,
-                tool_input: runtimeEvent.input,
-                status: 'running',
-                started_at: runtimeEvent.startedAt ?? new Date().toISOString(),
-              },
-              group_key: runtimeEvent.groupKey,
-              runtime_key: runtimeEvent.callId,
-            },
-          );
+      case 'tool_call': {
+        const step = await this.stepPersister.persist(sessionId, 'tool_call', {
+          step_type: 'tool_call',
+          target_type: target.target_type,
+          target_id: target.target_id,
+          // call_id is session-unique for bridge runtimes (server-enforced),
+          // so the bare form anchors the tool step pair across retries —
+          // unlike parel's turn-scoped `tc:{turnId}:{callId}` (see
+          // protocol-vectors/agent-steps.json).
+          idempotency_key: `tc:${runtimeEvent.callId}`,
+          content: {
+            call_id: runtimeEvent.callId,
+            tool_name: runtimeEvent.toolName,
+            tool_input: runtimeEvent.input,
+            status: 'running',
+            started_at: runtimeEvent.startedAt ?? new Date().toISOString(),
+          },
+          group_key: runtimeEvent.groupKey,
+          runtime_key: runtimeEvent.callId,
+        });
+        // step is null when the write was queued for background retry — the
+        // CLI step-id linkage window has then passed, same as a failed write
+        // before the queue existed.
+        if (step) {
           if (contextFilePath) {
             this.updateContextFileStepId(contextFilePath, step.id);
           } else if (stepIdFilePath) {
@@ -1034,57 +1152,48 @@ export class ParallAgentGateway {
           if (laneContextFilePath) {
             this.updateContextFileStepId(laneContextFilePath, step.id);
           }
-          break;
         }
+        break;
+      }
 
-        case 'tool_result':
-          await this.opts.client.createAgentStep(
-            this.opts.config.org_id,
-            this.opts.agentUserId,
-            sessionId,
-            {
-              step_type: 'tool_result',
-              target_type: target.target_type,
-              target_id: target.target_id,
-              content: {
-                call_id: runtimeEvent.callId,
-                tool_name: runtimeEvent.toolName,
-                status: runtimeEvent.error ? 'error' : 'success',
-                output: runtimeEvent.output,
-                duration_ms: runtimeEvent.durationMs ?? 0,
-                collapsible: true,
-              },
-              group_key: runtimeEvent.groupKey,
-            },
-          );
-          if (contextFilePath) {
-            this.updateContextFileStepId(contextFilePath, null);
-          } else if (stepIdFilePath) {
-            this.clearStepIdFile(stepIdFilePath);
-          }
-          if (laneContextFilePath) {
-            this.updateContextFileStepId(laneContextFilePath, null);
-          }
-          break;
+      case 'tool_result':
+        await this.stepPersister.persist(sessionId, 'tool_result', {
+          step_type: 'tool_result',
+          target_type: target.target_type,
+          target_id: target.target_id,
+          idempotency_key: `tr:${runtimeEvent.callId}`,
+          content: {
+            call_id: runtimeEvent.callId,
+            tool_name: runtimeEvent.toolName,
+            status: runtimeEvent.error ? 'error' : 'success',
+            output: runtimeEvent.output,
+            duration_ms: runtimeEvent.durationMs ?? 0,
+            collapsible: true,
+          },
+          group_key: runtimeEvent.groupKey,
+        });
+        // The tool has finished regardless of whether the step write landed
+        // inline or was queued — always clear the step-id linkage.
+        if (contextFilePath) {
+          this.updateContextFileStepId(contextFilePath, null);
+        } else if (stepIdFilePath) {
+          this.clearStepIdFile(stepIdFilePath);
+        }
+        if (laneContextFilePath) {
+          this.updateContextFileStepId(laneContextFilePath, null);
+        }
+        break;
 
-        case 'error':
-          await this.opts.client.createAgentStep(
-            this.opts.config.org_id,
-            this.opts.agentUserId,
-            sessionId,
-            {
-              step_type: 'text',
-              target_type: target.target_type,
-              target_id: target.target_id,
-              content: buildErrorStepContent(runtimeEvent.message),
-              projection: false,
-            },
-          );
-          break;
-      }
-    } catch (err) {
-      if (this.isSessionNotLiveError(err)) throw err;
-      this.opts.log?.warn(`failed to create ${runtimeEvent.type} step: ${String(err)}`);
+      case 'error':
+        await this.stepPersister.persist(sessionId, 'error', {
+          step_type: 'text',
+          target_type: target.target_type,
+          target_id: target.target_id,
+          idempotency_key: randomUUID(),
+          content: buildErrorStepContent(runtimeEvent.message),
+          projection: false,
+        });
+        break;
     }
   }
 
@@ -1307,9 +1416,19 @@ export class ParallAgentGateway {
 
       let binding = this.sessionBindings.get(sessionKey);
       let inputStepsCreated = false;
-      let triggerMessageSet = false;
+      let turnHandle: TurnHandle | undefined;
       let dispatchError: unknown;
       const pendingSendCallIds = new Set<string>();
+      // Turn boundary: must complete one bounded active reconciliation
+      // BEFORE this turn's first AgentStep persists — a reused idle session
+      // otherwise races the status write and the server presence guard
+      // swallows the turn's first activity. Resolves even when the write
+      // fails (warned; the coordinator keeps reconciling in the background)
+      // so a degraded link never blocks the step flow indefinitely.
+      const ensureTurnBegun = async (): Promise<void> => {
+        if (turnHandle || !binding) return;
+        turnHandle = await this.sessionLifecycle.beginTurn(binding.agentSessionId, event.messageId);
+      };
       try {
         dispatchSpan = startDispatchSpan(event, this.opts.runtimeType, sessionKey);
         for await (const runtimeEvent of this.opts.dispatchAdapter.dispatch({
@@ -1351,6 +1470,7 @@ export class ParallAgentGateway {
               // while a dispatch was in flight) inside the in-flight window so a
               // shutdown short-circuit BEFORE this point cannot leave orphan input
               // steps that the replacement pod would duplicate on replay.
+              await ensureTurnBegun();
               if (earlierEvents.length > 0) {
                 await this.createInputStepsForEarlierEvents(binding.agentSessionId, earlierEvents);
               }
@@ -1364,17 +1484,7 @@ export class ParallAgentGateway {
             const detail = runtimeEvent.type === 'error' ? `: ${runtimeEvent.message}` : '';
             throw new Error(`runtime emitted ${runtimeEvent.type} before runtime_session${detail}`);
           }
-          if (!triggerMessageSet) {
-            triggerMessageSet = true;
-            this.opts.client
-              .updateAgentSession(
-                this.opts.config.org_id,
-                this.opts.agentUserId,
-                binding.agentSessionId,
-                { status: 'active', trigger_message_id: event.messageId },
-              )
-              .catch((err) => this.opts.log?.warn?.(`failed to set session active: ${err}`));
-          }
+          await ensureTurnBegun();
           if (!inputStepsCreated) {
             if (earlierEvents.length > 0) {
               await this.createInputStepsForEarlierEvents(binding.agentSessionId, earlierEvents);
@@ -1426,6 +1536,7 @@ export class ParallAgentGateway {
           throw new Error('runtime completed without runtime_session');
         }
         if (!inputStepsCreated) {
+          await ensureTurnBegun();
           if (earlierEvents.length > 0) {
             await this.createInputStepsForEarlierEvents(binding.agentSessionId, earlierEvents);
           }
@@ -1436,6 +1547,7 @@ export class ParallAgentGateway {
         let staleDetected = this.isSessionNotLiveError(err);
         if (!staleDetected && binding) {
           try {
+            await ensureTurnBegun();
             await this.createRuntimeStep(
               binding.agentSessionId,
               event,
@@ -1456,6 +1568,8 @@ export class ParallAgentGateway {
             `session ${binding.agentSessionId} is stale (mid-dispatch), triggering recovery for ${sessionKey}`,
           );
           this.sessionBindings.delete(sessionKey);
+          this.stepPersister.dropSession(binding.agentSessionId);
+          this.sessionLifecycle.dropSession(binding.agentSessionId);
           if (sessionKey === this.opts.runtimeKey) {
             this.activeSessionId = undefined;
           }
@@ -1492,15 +1606,10 @@ export class ParallAgentGateway {
         }
 
         clearDispatchMetrics(sessionKey);
-        if (triggerMessageSet && binding) {
-          this.opts.client
-            .updateAgentSession(
-              this.opts.config.org_id,
-              this.opts.agentUserId,
-              binding.agentSessionId,
-              { status: 'idle' },
-            )
-            .catch((err) => this.opts.log?.warn?.(`failed to set session idle: ${err}`));
+        if (turnHandle) {
+          // Stale-handle safe: if a newer turn already began on this
+          // session, the coordinator ignores this finish outright.
+          this.sessionLifecycle.finishTurn(turnHandle);
         }
         clearSessionMessageId(sessionKey);
         clearDispatchMessageId(sessionKey);
@@ -1749,15 +1858,18 @@ export class ParallAgentGateway {
         }
       }
       if (forkBinding) {
-        this.opts.client
-          .updateAgentSession(
-            this.opts.config.org_id,
-            this.opts.agentUserId,
-            forkBinding.agentSessionId,
-            { status: 'closed' },
-          )
-          .catch(() => {});
-        this.sessionBindings.delete(fork.fork.sessionKey);
+        // Normal fork teardown is the ForkSessionFinalizer use case:
+        // seal → drain parked/in-flight step writes → close (serialized
+        // behind the turn's idle, ownership held through close retries) →
+        // release the binding. The gateway only triggers it. Notably this
+        // must NOT drop the session's step queue — parked steps drain to
+        // the server before the close is issued (the identity check on the
+        // release keeps a replacement fork's binding intact).
+        await this.forkFinalizer.finalize(forkBinding.agentSessionId, () => {
+          if (this.sessionBindings.get(fork.fork.sessionKey) === forkBinding) {
+            this.sessionBindings.delete(fork.fork.sessionKey);
+          }
+        });
       }
     }
   }
@@ -1844,55 +1956,110 @@ export class ParallAgentGateway {
         // false) and released the claims, so THIS site owns their resolution.
         // Message groups here are the ledger-disabled legacy flow.
         const typedRefs = this.typedLedgerEventIds(events);
-        if (!typedRefs) {
+        // Guard the settlement window: from the moment the group leaves the
+        // buffer until settlement finishes, a re-drive must still be
+        // absorbed by isBufferedTypedWorkItem — see drainingTypedIds. Keyed
+        // on the events' own WorkItem ids, NOT typedRefs: the legacy
+        // (ledger-disabled) fallback settles through per-event acks but its
+        // re-drives converge through the same buffered-WorkItem check.
+        const drainingIds = events
+          .map((ev) => ev.dispatchEventId)
+          .filter((id): id is string => !!id);
+        for (const id of drainingIds) this.drainingTypedIds.add(id);
+        try {
+          if (!typedRefs) {
+            try {
+              await this.emitDispatchReceived(event);
+            } catch (err) {
+              this.opts.log?.warn?.(
+                `mark-received failed for buffered dispatch, leaving unacked for retry: ${String(err)}`,
+              );
+              this.dispatchState.mainBuffer.unshift(...events);
+              this.dispatchState.pendingForkResults.unshift(...pendingFork);
+              break;
+            }
+          }
+          // A typed group dispatches ALL buffered bodies, not just the
+          // newest: typed events are never steer-injected (unlike messages,
+          // which the model already saw mid-turn), so a comment burst folded
+          // into one drain turn would otherwise surface only its last member
+          // to the LLM — earlier ones exist solely as input steps the model
+          // never reads. Keyed on the event kind, NOT on typedRefs: the
+          // ledger-disabled fallback buffers the same bursts and owes the
+          // model the same visibility (groups are homogeneous — see
+          // dispatchGroupKey). Adapters that present earlierEvents natively
+          // (OpenClaw InboundHistory) are exempt — concatenating would show
+          // every earlier member twice.
+          const isTypedGroup = events.every((ev) => ev.type !== 'message');
+          const body =
+            isTypedGroup &&
+            events.length > 1 &&
+            this.opts.dispatchAdapter.earlierEventsInPrompt !== true
+              ? events.map((ev) => buildEventBody(ev)).join('\n\n')
+              : buildEventBody(event);
+          let dispatched: boolean;
           try {
-            await this.emitDispatchReceived(event);
+            dispatched = await this.runDispatch(
+              event,
+              this.opts.runtimeKey,
+              forkPrefix + body,
+              earlier,
+            );
           } catch (err) {
-            this.opts.log?.warn?.(
-              `mark-received failed for buffered dispatch, leaving unacked for retry: ${String(err)}`,
+            if (!isTypedGroup) throw err;
+            // The retained dedupe claims live exactly as long as the buffer
+            // stay — a thrown turn dropped these events without settlement,
+            // so free the claims here or every re-drive is rejected at the
+            // dedupe gate until restart (#1149).
+            this.opts.log?.error(
+              `typed drain dispatch failed for ${event.messageId} (group of ${events.length}, claims freed for re-drive): ${String(err)}`,
             );
+            for (const ev of events) clearTypedDedupeForEvent(this.laneFlowHost(), ev);
+            this.dispatchState.pendingForkResults.unshift(...pendingFork);
+            continue;
+          }
+          if (!dispatched) {
+            // Shutdown: skip the ack so the server redelivers these buffered
+            // events to the replacement pod via dispatch catch-up. Put both
+            // the buffered events and the fork results back so nothing is
+            // lost.
             this.dispatchState.mainBuffer.unshift(...events);
             this.dispatchState.pendingForkResults.unshift(...pendingFork);
             break;
           }
-        }
-        const dispatched = await this.runDispatch(
-          event,
-          this.opts.runtimeKey,
-          forkPrefix + buildEventBody(event),
-          earlier,
-        );
-        if (!dispatched) {
-          // Shutdown: skip the ack so the server redelivers these buffered
-          // events to the replacement pod via dispatch catch-up. Put both the
-          // buffered events and the fork results back so nothing is lost.
-          this.dispatchState.mainBuffer.unshift(...events);
-          this.dispatchState.pendingForkResults.unshift(...pendingFork);
-          break;
-        }
-        if (typedRefs) {
-          // Wrapper-less resolution of the buffered typed group — protocol
-          // lives in gateway-lane-flow.ts. The turn-error marker is consumed
-          // HERE, before this loop can start another turn on the session.
-          await settleDrainedTypedGroup(
-            this.laneFlowHost(),
-            events,
-            typedRefs,
-            this.consumeTurnError(this.opts.runtimeKey),
-          );
-          continue;
-        }
-        for (const bufferedEvent of events) {
-          const sourceType =
-            bufferedEvent.ackSourceType ??
-            (bufferedEvent.type === 'task' ? 'task_activity' : 'message');
-          const sourceId = bufferedEvent.ackSourceId ?? bufferedEvent.messageId;
-          this.opts.client
-            .ackDispatch(this.opts.config.org_id, {
-              source_type: sourceType,
-              source_id: sourceId,
-            })
-            .catch(() => {});
+          if (typedRefs) {
+            // Wrapper-less resolution of the buffered typed group — protocol
+            // lives in gateway-lane-flow.ts. The turn-error marker is
+            // consumed HERE, before this loop can start another turn on the
+            // session.
+            await settleDrainedTypedGroup(
+              this.laneFlowHost(),
+              events,
+              typedRefs,
+              this.consumeTurnError(this.opts.runtimeKey),
+            );
+            continue;
+          }
+          for (const bufferedEvent of events) {
+            const sourceType =
+              bufferedEvent.ackSourceType ??
+              (bufferedEvent.type === 'task' ? 'task_activity' : 'message');
+            const sourceId = bufferedEvent.ackSourceId ?? bufferedEvent.messageId;
+            this.opts.client
+              .ackDispatch(this.opts.config.org_id, {
+                source_type: sourceType,
+                source_id: sourceId,
+              })
+              .catch(() => {});
+            // Retained-claim lifetime is the buffer stay on the legacy face
+            // too: the fire-and-forget ack may fail (the row re-drives and
+            // must not be self-rejected), and a shared-key task sibling must
+            // not be blocked by this drained copy's claim. No-op for message
+            // events (no typed dedupe entry).
+            clearTypedDedupeForEvent(this.laneFlowHost(), bufferedEvent);
+          }
+        } finally {
+          for (const id of drainingIds) this.drainingTypedIds.delete(id);
         }
       }
     } finally {
@@ -2021,10 +2188,27 @@ export class ParallAgentGateway {
         if (this.shuttingDown) {
           return false;
         }
+        // A re-driven typed WorkItem may already be buffered from a prior
+        // consume attempt (claim → busy main → buffer → release → server
+        // re-drive): the buffered copy is the one the drain settles, so a
+        // second copy would double the drain group's input steps and prompt
+        // content. Drop the duplicate; the caller releases the row again and
+        // the re-drive keeps converging on the buffered copy (#1149).
+        if (this.isBufferedTypedWorkItem(event.dispatchEventId)) {
+          return false;
+        }
         // Push synchronously BEFORE the (possibly async) steer attempt so
         // arrival order is preserved and the event cannot be orphaned in a
         // gap between the steer await and the push.
         this.dispatchState.mainBuffer.push(event);
+        // FIFO fence for BOTH injection branches below: a buffered typed
+        // event is never injected, but the adapters track pending injections
+        // as a COUNT, not by identity — if a message injected behind a
+        // buffered typed event, the drain (typed group first, FIFO) would
+        // consume the message's steer output as the typed group's turn: the
+        // typed body never reaches the model yet resolves, and the message
+        // replays. When anything un-injected sits ahead, buffer only.
+        const typedAheadInBuffer = this.dispatchState.mainBuffer.some((e) => e.type !== 'message');
         if (this.usesLaneLedger(event)) {
           // Ledger flow: fold into the live lane server-side FIRST, then
           // inject. An un-folded injection is forbidden (the pending WorkItem
@@ -2039,6 +2223,7 @@ export class ParallAgentGateway {
           // the event un-folded keeps it buffered; the drain claims it as
           // its own turn and folds it there.
           if (
+            !typedAheadInBuffer &&
             this.mainCurrentGroupKey === this.dispatchGroupKey(event) &&
             this.opts.dispatchAdapter.enqueueDuringDispatch != null &&
             (await this.laneLedger?.steerLive(event)) &&
@@ -2052,6 +2237,15 @@ export class ParallAgentGateway {
             );
           }
         } else if (
+          // Message events only. A typed event (task_comment/schedule/…)
+          // rides the typed-consume contract — buffer-main resolves false and
+          // the claim releases for re-drive — so an injection here is exactly
+          // the forbidden un-folded injection: the LLM sees the content while
+          // the WorkItem stays live, and every re-drive injects it AGAIN (the
+          // 7/16 watcher duplicate-delivery loop, #1149). Typed events stay
+          // buffered; the drain claims them as their own turn.
+          event.type === 'message' &&
+          !typedAheadInBuffer &&
           this.dispatchState.mainCurrentTargetId === event.targetId &&
           (await this.opts.dispatchAdapter.enqueueDuringDispatch?.(
             this.opts.runtimeKey,
@@ -2362,7 +2556,7 @@ export class ParallAgentGateway {
     };
 
     const dispatched = await this.handleInboundEvent(event);
-    if (!dispatched) {
+    if (!dispatched && !this.isTypedEventBuffered(event)) {
       this.dispatchedTasks.delete(dedupeKey);
     }
     return dispatched;
@@ -2473,7 +2667,7 @@ export class ParallAgentGateway {
       this.dispatchedTasks.delete(dedupeKey);
       throw err;
     }
-    if (!dispatched) {
+    if (!dispatched && !this.isTypedEventBuffered(event)) {
       this.dispatchedTasks.delete(dedupeKey);
     }
     return dispatched;
@@ -2550,7 +2744,7 @@ export class ParallAgentGateway {
       this.dispatchedTasks.delete(dedupeKey);
       throw err;
     }
-    if (!dispatched) {
+    if (!dispatched && !this.isTypedEventBuffered(event)) {
       this.dispatchedTasks.delete(dedupeKey);
     }
     return dispatched;
@@ -2629,7 +2823,7 @@ export class ParallAgentGateway {
       this.dispatchedTasks.delete(dedupeKey);
       throw err;
     }
-    if (!dispatched) {
+    if (!dispatched && !this.isTypedEventBuffered(event)) {
       this.dispatchedTasks.delete(dedupeKey);
     }
     return dispatched;
@@ -2721,18 +2915,20 @@ export class ParallAgentGateway {
       }
     }
 
-    // The reply hint routes on the live capability grant: `<provider>-cli`
-    // present → the vendor CLI is on PATH (broker shim) and is THE reply
-    // path; absent → outbound is disabled for this org (flag/connection off)
-    // and the hint must say so instead of pointing at a retired clip. The
-    // provider label lookup above is best-effort/cosmetic — when it fails,
-    // ANY granted `*-cli` capability keeps the hint on the CLI path: a
-    // transient metadata miss must not flip an actively granted agent's
-    // hint to "outbound disabled" and strand a valid external message.
+    // The reply hint routes on the live capability grant, keyed PER
+    // PROVIDER: feishu's affordance is the vendor CLI on PATH (`feishu-cli`
+    // → lark-cli, tier A) and slack's is the platform verb (`slack-send` →
+    // `parall slack send`, tier B — there is no `slack-cli`). Absent →
+    // outbound is disabled for this org (flag/connection off) and the hint
+    // must say so instead of pointing at a retired clip. The provider label
+    // lookup above is best-effort/cosmetic — when it fails, ANY granted
+    // channel capability keeps the hint on the capability path: a transient
+    // metadata miss must not flip an actively granted agent's hint to
+    // "outbound disabled" and strand a valid external message.
     const keys = this.opts.getCapabilityKeys?.() ?? [];
     const cliCapable = provider
-      ? keys.includes(`${provider}-cli`)
-      : keys.some((k) => k.endsWith('-cli'));
+      ? keys.includes(channelCapabilityKeyFor(provider))
+      : keys.some((k) => k.endsWith('-cli') || k === CAPABILITY_SLACK_SEND);
 
     const event: ParallEvent = {
       type: 'channel_message',
@@ -2763,7 +2959,7 @@ export class ParallAgentGateway {
       this.dispatchedMessages.delete(claimKey);
       throw err;
     }
-    if (!dispatched) {
+    if (!dispatched && !this.isTypedEventBuffered(event)) {
       this.dispatchedMessages.delete(claimKey);
     }
     return dispatched;
@@ -2810,7 +3006,7 @@ export class ParallAgentGateway {
       this.dispatchedTasks.delete(dedupeKey);
       throw err;
     }
-    if (!dispatched) {
+    if (!dispatched && !this.isTypedEventBuffered(event)) {
       this.dispatchedTasks.delete(dedupeKey);
     }
     return dispatched;
@@ -2871,7 +3067,7 @@ export class ParallAgentGateway {
       this.dispatchedTasks.delete(dedupeKey);
       throw err;
     }
-    if (!dispatched) {
+    if (!dispatched && !this.isTypedEventBuffered(event)) {
       this.dispatchedTasks.delete(dedupeKey);
     }
     return dispatched;
@@ -2947,6 +3143,16 @@ export class ParallAgentGateway {
           continue;
         }
 
+        // Same pre-claim guard as the dispatch.new handler: a WorkItem whose
+        // event copy is already buffered belongs to the drain — claiming it
+        // here would race the drain's fence-less settlement.
+        if (item.event_type !== 'message' && this.isBufferedTypedWorkItem(item.id)) {
+          this.opts.log?.info(
+            `typed dispatch ${item.id} already buffered for the drain — skipping catch-up claim`,
+          );
+          continue;
+        }
+
         processed++;
         try {
           const typedHooks: TypedConsumeHooks = {
@@ -3254,6 +3460,32 @@ export class ParallAgentGateway {
 
     await this.opts.onBeforeDisconnect?.();
 
+    // Parked step writes are process-local and their WorkItems are already
+    // resolved — restart catch-up will NOT re-drive them, so anything still
+    // parked at exit is permanently lost. Spend a slice of the shutdown
+    // budget on one flush pass first: the common shutdown (idle-stop,
+    // deploy) happens on a healthy network where these writes just succeed.
+    // The 10s cap is hard — a write still in flight at the deadline is
+    // abandoned to the background (see StepRetryQueue.flush).
+    if (this.stepPersister.pendingTotal() > 0) {
+      const remaining = await this.stepPersister.flush(10_000);
+      if (remaining > 0) {
+        this.opts.log?.warn(
+          `${remaining} parked step write(s) could not be flushed at shutdown; they are permanently lost`,
+        );
+      }
+    }
+    this.stepPersister.dispose();
+    // Lifecycle last: idle writes land after the flushed steps, so the
+    // server clears activity once and no flushed step can relight it.
+    const lifecycleRemaining = await this.sessionLifecycle.flush(5_000);
+    if (lifecycleRemaining > 0) {
+      this.opts.log?.warn(
+        `${lifecycleRemaining} session lifecycle write(s) unreconciled at shutdown`,
+      );
+    }
+    this.sessionLifecycle.dispose();
+
     this.opts.ws.disconnect();
     this.opts.log?.info(`disconnected`);
   }