@parall/agent-core 1.45.0 → 1.46.0

package/dist/gateway-base.js

@@ -1,12 +1,17 @@
 import * as os from 'node:os';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
+import { randomUUID } from 'node:crypto';
 import { ApiError, MENTION_ALL_USER_ID } from '@parall/sdk';
 import { buildEventBody, buildEventBodyForForkResult, buildForkResultPrefix, buildForkScopePrefix, } from './event-format.js';
+import { CAPABILITY_SLACK_SEND, channelCapabilityKeyFor } from './channel-capability.js';
 import { buildErrorStepContent, } from './dispatch-adapter.js';
-import { consumeMessageWorkItem, consumeTypedDispatch, dispatchLaneGroup, resolveDispatchByID, settleDrainedTypedGroup, typedLedgerEventIds, } from './gateway-lane-flow.js';
+import { clearTypedDedupeForEvent, consumeMessageWorkItem, consumeTypedDispatch, dispatchLaneGroup, resolveDispatchByID, settleDrainedTypedGroup, typedLedgerEventIds, } from './gateway-lane-flow.js';
 import { LaneLedger } from './lane-ledger.js';
 import { routeTrigger } from './routing.js';
+import { StepPersister, isRetryableStepError } from './step-persister.js';
+import { SessionLifecycleCoordinator } from './session-lifecycle.js';
+import { ForkSessionFinalizer } from './fork-session-finalizer.js';
 import { clearDispatchMessageId, clearDispatchMetrics, clearDispatchNoReply, clearSessionMessageId, getDispatchMetrics, recordDeliverText, recordMessageSend, recordNoReply, recordToolCall, resetDispatchMetrics, setDispatchMessageId, setDispatchNoReply, setSessionChatId, setSessionMessageId, } from './session-state.js';
 import { isParallSendCommand, isParallNoReplyCommand, extractShellCommand, } from './bridge-workspace.js';
 import { startDispatchSpan, endDispatchSpan, recordDispatchMetric, recordMissingReply, runWithSessionKey, } from './telemetry.js';
@@ -43,6 +48,34 @@ export function parseDispatchDeadlineMs(raw) {
         return undefined;
     return Math.floor(n);
 }
+/**
+ * Stable input-step idempotency key: a re-delivered event (catch-up replay,
+ * lane re-drive) replays to the same row instead of duplicating it.
+ *
+ * Message-shaped events key on the message id — it is the logical identity
+ * and stays stable across delivery paths (a live WS delivery carries no
+ * dispatchEventId; its catch-up replay does — keying on the WorkItem there
+ * would split the two into different keys and duplicate the input step).
+ *
+ * Typed events key on the WorkItem id instead: their messageId can be
+ * REUSED across distinct work items — task events carry the task id for
+ * both task_assign and a later task_update, so keying on messageId would
+ * silently dedupe the second, legitimate, input step away. For the same
+ * reason a typed event WITHOUT a WorkItem id (legacy server) must NOT fall
+ * back to messageId — it gets a random UUID per logical step instead. The
+ * key rides the CreateAgentStepRequest object, so the same UUID is reused
+ * across that step's HTTP retries and queue redrives (replay-safe), while
+ * a redelivered legacy event writes a fresh row (rare duplicate beats
+ * silently losing a legitimate step). See protocol-vectors/agent-steps.json.
+ */
+export function inputStepIdempotencyKey(event) {
+    if (event.type === 'message' || event.type === 'channel_message') {
+        return event.messageId ? `input:${event.messageId}` : randomUUID();
+    }
+    if (event.dispatchEventId)
+        return `input:${event.dispatchEventId}`;
+    return randomUUID();
+}
 function resolveStepTarget(event) {
     if (event.type === 'task' || event.targetId.startsWith('tsk_')) {
         return { target_type: 'task', target_id: event.targetId };
@@ -141,6 +174,13 @@ export class ParallAgentGateway {
     heartbeatTimer = null;
     lastHeartbeatAt = Date.now();
     draining = false;
+    /**
+     * Typed WorkItem ids whose drain group left the buffer but has not settled
+     * yet. isBufferedTypedWorkItem treats them as still buffered — a re-drive
+     * claim taken inside this window would turn the drain's fence-less close
+     * stale (its dedupe cleared, its release re-driving handled work) (#1149).
+     */
+    drainingTypedIds = new Set();
     // Graceful shutdown state. When SIGTERM / abort fires, `shuttingDown` flips
     // to true so no new dispatches start, and `inFlightDispatches` counts runs
     // still in progress. `shutdown()` awaits drain up to SHUTDOWN_DEADLINE_MS
@@ -150,6 +190,16 @@ export class ParallAgentGateway {
     drainResolvers = [];
     pendingRestartNotification = null;
     laneLedger;
+    stepPersister;
+    // Single entry point for session active/idle writes — serialized
+    // desired-state reconciler (see session-lifecycle.ts). The gateway only
+    // declares turn boundaries; ordering, retries and stale-finish rejection
+    // live in the coordinator.
+    sessionLifecycle;
+    // Normal fork teardown use case: seal → drain → close → release. The
+    // gateway only triggers it; ordering and ownership live in the finalizer
+    // (see fork-session-finalizer.ts).
+    forkFinalizer;
     // Sticky fallback: flipped when the server predates the ledger (claim
     // endpoint 404) so every subsequent dispatch uses the legacy flow.
     ledgerDisabled = false;
@@ -177,6 +227,34 @@ export class ParallAgentGateway {
         this.SHUTDOWN_DEADLINE_MS = opts.shutdownDeadlineMs ?? 60_000;
         this.FORK_DEADLINE_MS = opts.forkDeadlineMs ?? 2 * 60 * 60_000;
         this.DISPATCH_DEADLINE_MS = opts.dispatchDeadlineMs ?? 20 * 60_000;
+        this.stepPersister = new StepPersister({
+            client: opts.client,
+            orgId: opts.config.org_id,
+            agentUserId: opts.agentUserId,
+            log: { warn: (msg) => this.opts.log?.warn(msg) },
+            isSessionStale: (err) => this.isSessionNotLiveError(err),
+            retryDelaysMs: opts.stepRetryDelaysMs,
+        });
+        this.sessionLifecycle = new SessionLifecycleCoordinator({
+            write: (sessionId, payload) => this.opts.client
+                .updateAgentSession(this.opts.config.org_id, this.opts.agentUserId, sessionId, payload)
+                .then(() => undefined),
+            // Same classification as step writes: transient (timeout/network/5xx)
+            // retries with backoff; a 4xx is permanent (never loop against it);
+            // a terminal-session 409 drops the session for good.
+            isRetryable: isRetryableStepError,
+            isSessionStale: (err) => this.isSessionNotLiveError(err),
+            log: { warn: (msg) => this.opts.log?.warn(msg) },
+            retryDelaysMs: opts.lifecycleRetryDelaysMs,
+        });
+        this.forkFinalizer = new ForkSessionFinalizer({
+            steps: this.stepPersister,
+            lifecycle: this.sessionLifecycle,
+            log: {
+                warn: (msg) => this.opts.log?.warn(msg),
+                error: (msg) => this.opts.log?.error(msg),
+            },
+        });
         if (opts.coldStartWindowMs != null) {
             opts.log?.warn?.('coldStartWindowMs is deprecated and ignored — cold-start time filter has been removed');
         }
@@ -230,6 +308,14 @@ export class ParallAgentGateway {
             const prevId = data.previous_session_id ?? '';
             this.opts.log?.info(`new session signal received (previous=${prevId})`);
             this.sessionBindings.clear();
+            // The server closed EVERY open session — not just `previous_session_id`
+            // (active forks have their own ase_ ids, and sessionBindings was just
+            // cleared, so they would otherwise keep retrying against closed rows
+            // until a 409 or the age budget). Any parked lifecycle write or step
+            // write would now 409 against a terminal row: drop them all rather than
+            // burn retry budget and log noise discovering it.
+            this.sessionLifecycle.dropAllSessions();
+            this.stepPersister.dropAllSessions();
             if (prevId) {
                 this.pendingRestartNotification = `[Harness Notification] This is a fresh session. Your previous session (${prevId}) was ended by the user and you have been restarted.`;
             }
@@ -285,6 +371,13 @@ export class ParallAgentGateway {
             }
         });
         ws.on('dispatch.new', async (data) => {
+            // Re-drive of a WorkItem whose event copy is already buffered: skip
+            // BEFORE the typed claim (see isBufferedTypedWorkItem). The renotify
+            // pacing re-checks after the drain settles.
+            if (data.event_type !== 'message' && this.isBufferedTypedWorkItem(data.id)) {
+                this.opts.log?.info(`typed dispatch ${data.id} already buffered for the drain — skipping re-claim`);
+                return;
+            }
             if (data.event_type === 'task_assign') {
                 if (!data.task_id)
                     return;
@@ -518,7 +611,14 @@ export class ParallAgentGateway {
             // into one turn.
             return this.laneLedger.laneKeyFor(event);
         }
-        return event.targetId;
+        // Typed events never share a drain group with messages. On the
+        // ledger-disabled fallback both group by target id, and a typed event
+        // whose target is a chat (approval) could batch behind that chat's
+        // messages — the drain body carries only the trailing message while the
+        // legacy ack sweeps the whole group, silently dropping the typed body
+        // (typed events are never steer-injected, so the drain body is their
+        // only route to the model). Split the groups instead (#1149).
+        return event.type === 'message' ? event.targetId : `typed:${event.targetId}`;
     }
     // Lane-flow protocols live in gateway-lane-flow.ts; these thin delegates
     // keep call sites and tests on the class surface.
@@ -548,6 +648,25 @@ export class ParallAgentGateway {
             return false;
         });
     }
+    /**
+     * True while a typed WorkItem's event copy sits in the main buffer waiting
+     * for the drain. A buffered copy keeps its hot-path dedupe claim: the drain
+     * owns its settlement, and the released row's re-drive must short-circuit
+     * BEFORE claiming (a duplicate claim opens a lane that races the drain's
+     * fence-less by-id close — 409 STALE, and the claimant's release would
+     * re-drive already-handled work) (#1149). The claim's lifetime equals the
+     * buffer stay: settlement clears it on every drain outcome — success,
+     * failure, or a thrown turn.
+     */
+    isBufferedTypedWorkItem(dispatchEventId) {
+        if (!dispatchEventId)
+            return false;
+        return (this.drainingTypedIds.has(dispatchEventId) ||
+            this.dispatchState.mainBuffer.some((e) => e.dispatchEventId === dispatchEventId));
+    }
+    isTypedEventBuffered(event) {
+        return this.isBufferedTypedWorkItem(event.dispatchEventId);
+    }
     // PARITY: this switch and gateway-lane-flow's clearTypedDedupeForEvent must
     // handle the same typed source families — extend BOTH when adding a typed
     // event type (same dedupe entries, keyed from different event shapes).
@@ -614,106 +733,110 @@ export class ParallAgentGateway {
     }
     async createInputStep(sessionId, event) {
         const target = resolveStepTarget(event);
-        try {
-            await this.opts.client.createAgentStep(this.opts.config.org_id, this.opts.agentUserId, sessionId, {
-                step_type: 'input',
-                target_type: target.target_type,
-                target_id: target.target_id,
-                content: {
-                    trigger_type: event.type === 'task'
-                        ? 'task_assign'
-                        : event.type === 'task_comment'
-                            ? 'task_comment'
-                            : event.type === 'wiki_comment'
-                                ? 'wiki_comment'
-                                : event.type === 'schedule'
-                                    ? 'schedule_fire'
-                                    : event.type === 'external_trigger'
-                                        ? 'external_trigger'
-                                        : event.type === 'channel_message'
-                                            ? 'channel_message'
-                                            : event.type === 'approval'
-                                                ? 'approval_decided'
-                                                : 'mention',
-                    trigger_ref: event.type === 'task'
-                        ? { task_id: event.targetId }
-                        : event.type === 'task_comment'
-                            ? { comment_id: event.messageId, task_id: event.targetId }
-                            : event.type === 'wiki_comment'
-                                ? { comment_id: event.messageId, target_uri: event.replyTargetUri }
-                                : event.type === 'schedule'
-                                    ? { schedule_id: event.targetId, run_id: event.messageId }
-                                    : event.type === 'external_trigger'
+        await this.stepPersister.persist(sessionId, 'input', {
+            step_type: 'input',
+            target_type: target.target_type,
+            target_id: target.target_id,
+            idempotency_key: inputStepIdempotencyKey(event),
+            content: {
+                trigger_type: event.type === 'task'
+                    ? 'task_assign'
+                    : event.type === 'task_comment'
+                        ? 'task_comment'
+                        : event.type === 'wiki_comment'
+                            ? 'wiki_comment'
+                            : event.type === 'schedule'
+                                ? 'schedule_fire'
+                                : event.type === 'external_trigger'
+                                    ? 'external_trigger'
+                                    : event.type === 'channel_message'
+                                        ? 'channel_message'
+                                        : event.type === 'approval'
+                                            ? 'approval_decided'
+                                            : 'mention',
+                trigger_ref: event.type === 'task'
+                    ? { task_id: event.targetId }
+                    : event.type === 'task_comment'
+                        ? { comment_id: event.messageId, task_id: event.targetId }
+                        : event.type === 'wiki_comment'
+                            ? { comment_id: event.messageId, target_uri: event.replyTargetUri }
+                            : event.type === 'schedule'
+                                ? { schedule_id: event.targetId, run_id: event.messageId }
+                                : event.type === 'external_trigger'
+                                    ? {
+                                        trigger_id: event.targetId,
+                                        run_id: event.messageId,
+                                        connection_id: event.externalConnectionId,
+                                        ingress_event_id: event.externalIngressEventId,
+                                    }
+                                    : event.type === 'channel_message'
                                         ? {
-                                            trigger_id: event.targetId,
-                                            run_id: event.messageId,
-                                            connection_id: event.externalConnectionId,
-                                            ingress_event_id: event.externalIngressEventId,
+                                            conversation_id: event.targetId,
+                                            channel_message_id: event.messageId,
+                                            provider: event.channelProvider,
+                                            external_conversation_id: event.channelExternalConversationId,
                                         }
-                                        : event.type === 'channel_message'
-                                            ? {
-                                                conversation_id: event.targetId,
-                                                channel_message_id: event.messageId,
-                                                provider: event.channelProvider,
-                                                external_conversation_id: event.channelExternalConversationId,
-                                            }
-                                            : event.type === 'approval'
-                                                ? { approval_id: event.messageId }
-                                                : { message_id: event.messageId },
-                    sender_id: event.senderId,
-                    sender_name: event.senderName,
-                    summary: event.body.substring(0, 200),
-                    ...(event.sentAt ? { sent_at: event.sentAt } : {}),
-                },
-            });
-        }
-        catch (err) {
-            if (this.isSessionNotLiveError(err))
-                throw err;
-            this.opts.log?.warn(`failed to create input step: ${String(err)}`);
-        }
+                                        : event.type === 'approval'
+                                            ? { approval_id: event.messageId }
+                                            : { message_id: event.messageId },
+                sender_id: event.senderId,
+                sender_name: event.senderName,
+                summary: event.body.substring(0, 200),
+                ...(event.sentAt ? { sent_at: event.sentAt } : {}),
+            },
+        });
     }
     async createRuntimeStep(sessionId, event, runtimeEvent, stepIdFilePath, contextFilePath, laneContextFilePath) {
         const target = resolveStepTarget(event);
-        try {
-            switch (runtimeEvent.type) {
-                case 'thinking':
-                    await this.opts.client.createAgentStep(this.opts.config.org_id, this.opts.agentUserId, sessionId, {
-                        step_type: 'thinking',
-                        target_type: target.target_type,
-                        target_id: target.target_id,
-                        content: { text: runtimeEvent.text },
-                        group_key: runtimeEvent.groupKey,
-                    });
-                    break;
-                case 'text':
-                    await this.opts.client.createAgentStep(this.opts.config.org_id, this.opts.agentUserId, sessionId, {
-                        step_type: 'text',
-                        target_type: target.target_type,
-                        target_id: target.target_id,
-                        content: {
-                            text: runtimeEvent.text,
-                            suppressed: runtimeEvent.project !== true,
-                        },
-                        projection: runtimeEvent.project === true,
-                        group_key: runtimeEvent.groupKey,
-                    });
-                    break;
-                case 'tool_call': {
-                    const step = await this.opts.client.createAgentStep(this.opts.config.org_id, this.opts.agentUserId, sessionId, {
-                        step_type: 'tool_call',
-                        target_type: target.target_type,
-                        target_id: target.target_id,
-                        content: {
-                            call_id: runtimeEvent.callId,
-                            tool_name: runtimeEvent.toolName,
-                            tool_input: runtimeEvent.input,
-                            status: 'running',
-                            started_at: runtimeEvent.startedAt ?? new Date().toISOString(),
-                        },
-                        group_key: runtimeEvent.groupKey,
-                        runtime_key: runtimeEvent.callId,
-                    });
+        switch (runtimeEvent.type) {
+            case 'thinking':
+                await this.stepPersister.persist(sessionId, 'thinking', {
+                    step_type: 'thinking',
+                    target_type: target.target_type,
+                    target_id: target.target_id,
+                    idempotency_key: randomUUID(),
+                    content: { text: runtimeEvent.text },
+                    group_key: runtimeEvent.groupKey,
+                });
+                break;
+            case 'text':
+                await this.stepPersister.persist(sessionId, 'text', {
+                    step_type: 'text',
+                    target_type: target.target_type,
+                    target_id: target.target_id,
+                    idempotency_key: randomUUID(),
+                    content: {
+                        text: runtimeEvent.text,
+                        suppressed: runtimeEvent.project !== true,
+                    },
+                    projection: runtimeEvent.project === true,
+                    group_key: runtimeEvent.groupKey,
+                });
+                break;
+            case 'tool_call': {
+                const step = await this.stepPersister.persist(sessionId, 'tool_call', {
+                    step_type: 'tool_call',
+                    target_type: target.target_type,
+                    target_id: target.target_id,
+                    // call_id is session-unique for bridge runtimes (server-enforced),
+                    // so the bare form anchors the tool step pair across retries —
+                    // unlike parel's turn-scoped `tc:{turnId}:{callId}` (see
+                    // protocol-vectors/agent-steps.json).
+                    idempotency_key: `tc:${runtimeEvent.callId}`,
+                    content: {
+                        call_id: runtimeEvent.callId,
+                        tool_name: runtimeEvent.toolName,
+                        tool_input: runtimeEvent.input,
+                        status: 'running',
+                        started_at: runtimeEvent.startedAt ?? new Date().toISOString(),
+                    },
+                    group_key: runtimeEvent.groupKey,
+                    runtime_key: runtimeEvent.callId,
+                });
+                // step is null when the write was queued for background retry — the
+                // CLI step-id linkage window has then passed, same as a failed write
+                // before the queue existed.
+                if (step) {
                     if (contextFilePath) {
                         this.updateContextFileStepId(contextFilePath, step.id);
                     }
@@ -723,48 +846,47 @@ export class ParallAgentGateway {
                     if (laneContextFilePath) {
                         this.updateContextFileStepId(laneContextFilePath, step.id);
                     }
-                    break;
                 }
-                case 'tool_result':
-                    await this.opts.client.createAgentStep(this.opts.config.org_id, this.opts.agentUserId, sessionId, {
-                        step_type: 'tool_result',
-                        target_type: target.target_type,
-                        target_id: target.target_id,
-                        content: {
-                            call_id: runtimeEvent.callId,
-                            tool_name: runtimeEvent.toolName,
-                            status: runtimeEvent.error ? 'error' : 'success',
-                            output: runtimeEvent.output,
-                            duration_ms: runtimeEvent.durationMs ?? 0,
-                            collapsible: true,
-                        },
-                        group_key: runtimeEvent.groupKey,
-                    });
-                    if (contextFilePath) {
-                        this.updateContextFileStepId(contextFilePath, null);
-                    }
-                    else if (stepIdFilePath) {
-                        this.clearStepIdFile(stepIdFilePath);
-                    }
-                    if (laneContextFilePath) {
-                        this.updateContextFileStepId(laneContextFilePath, null);
-                    }
-                    break;
-                case 'error':
-                    await this.opts.client.createAgentStep(this.opts.config.org_id, this.opts.agentUserId, sessionId, {
-                        step_type: 'text',
-                        target_type: target.target_type,
-                        target_id: target.target_id,
-                        content: buildErrorStepContent(runtimeEvent.message),
-                        projection: false,
-                    });
-                    break;
+                break;
             }
-        }
-        catch (err) {
-            if (this.isSessionNotLiveError(err))
-                throw err;
-            this.opts.log?.warn(`failed to create ${runtimeEvent.type} step: ${String(err)}`);
+            case 'tool_result':
+                await this.stepPersister.persist(sessionId, 'tool_result', {
+                    step_type: 'tool_result',
+                    target_type: target.target_type,
+                    target_id: target.target_id,
+                    idempotency_key: `tr:${runtimeEvent.callId}`,
+                    content: {
+                        call_id: runtimeEvent.callId,
+                        tool_name: runtimeEvent.toolName,
+                        status: runtimeEvent.error ? 'error' : 'success',
+                        output: runtimeEvent.output,
+                        duration_ms: runtimeEvent.durationMs ?? 0,
+                        collapsible: true,
+                    },
+                    group_key: runtimeEvent.groupKey,
+                });
+                // The tool has finished regardless of whether the step write landed
+                // inline or was queued — always clear the step-id linkage.
+                if (contextFilePath) {
+                    this.updateContextFileStepId(contextFilePath, null);
+                }
+                else if (stepIdFilePath) {
+                    this.clearStepIdFile(stepIdFilePath);
+                }
+                if (laneContextFilePath) {
+                    this.updateContextFileStepId(laneContextFilePath, null);
+                }
+                break;
+            case 'error':
+                await this.stepPersister.persist(sessionId, 'error', {
+                    step_type: 'text',
+                    target_type: target.target_type,
+                    target_id: target.target_id,
+                    idempotency_key: randomUUID(),
+                    content: buildErrorStepContent(runtimeEvent.message),
+                    projection: false,
+                });
+                break;
         }
     }
     writeContextFile(filePath, ctx) {
@@ -948,9 +1070,20 @@ export class ParallAgentGateway {
                 : null;
             let binding = this.sessionBindings.get(sessionKey);
             let inputStepsCreated = false;
-            let triggerMessageSet = false;
+            let turnHandle;
             let dispatchError;
             const pendingSendCallIds = new Set();
+            // Turn boundary: must complete one bounded active reconciliation
+            // BEFORE this turn's first AgentStep persists — a reused idle session
+            // otherwise races the status write and the server presence guard
+            // swallows the turn's first activity. Resolves even when the write
+            // fails (warned; the coordinator keeps reconciling in the background)
+            // so a degraded link never blocks the step flow indefinitely.
+            const ensureTurnBegun = async () => {
+                if (turnHandle || !binding)
+                    return;
+                turnHandle = await this.sessionLifecycle.beginTurn(binding.agentSessionId, event.messageId);
+            };
             try {
                 dispatchSpan = startDispatchSpan(event, this.opts.runtimeType, sessionKey);
                 for await (const runtimeEvent of this.opts.dispatchAdapter.dispatch({
@@ -980,6 +1113,7 @@ export class ParallAgentGateway {
                             // while a dispatch was in flight) inside the in-flight window so a
                             // shutdown short-circuit BEFORE this point cannot leave orphan input
                             // steps that the replacement pod would duplicate on replay.
+                            await ensureTurnBegun();
                             if (earlierEvents.length > 0) {
                                 await this.createInputStepsForEarlierEvents(binding.agentSessionId, earlierEvents);
                             }
@@ -992,12 +1126,7 @@ export class ParallAgentGateway {
                         const detail = runtimeEvent.type === 'error' ? `: ${runtimeEvent.message}` : '';
                         throw new Error(`runtime emitted ${runtimeEvent.type} before runtime_session${detail}`);
                     }
-                    if (!triggerMessageSet) {
-                        triggerMessageSet = true;
-                        this.opts.client
-                            .updateAgentSession(this.opts.config.org_id, this.opts.agentUserId, binding.agentSessionId, { status: 'active', trigger_message_id: event.messageId })
-                            .catch((err) => this.opts.log?.warn?.(`failed to set session active: ${err}`));
-                    }
+                    await ensureTurnBegun();
                     if (!inputStepsCreated) {
                         if (earlierEvents.length > 0) {
                             await this.createInputStepsForEarlierEvents(binding.agentSessionId, earlierEvents);
@@ -1043,6 +1172,7 @@ export class ParallAgentGateway {
                     throw new Error('runtime completed without runtime_session');
                 }
                 if (!inputStepsCreated) {
+                    await ensureTurnBegun();
                     if (earlierEvents.length > 0) {
                         await this.createInputStepsForEarlierEvents(binding.agentSessionId, earlierEvents);
                     }
@@ -1054,6 +1184,7 @@ export class ParallAgentGateway {
                 let staleDetected = this.isSessionNotLiveError(err);
                 if (!staleDetected && binding) {
                     try {
+                        await ensureTurnBegun();
                         await this.createRuntimeStep(binding.agentSessionId, event, {
                             type: 'error',
                             message: `Dispatch failed: ${String(err)}`,
@@ -1067,6 +1198,8 @@ export class ParallAgentGateway {
                 if (staleDetected && binding) {
                     this.opts.log?.warn?.(`session ${binding.agentSessionId} is stale (mid-dispatch), triggering recovery for ${sessionKey}`);
                     this.sessionBindings.delete(sessionKey);
+                    this.stepPersister.dropSession(binding.agentSessionId);
+                    this.sessionLifecycle.dropSession(binding.agentSessionId);
                     if (sessionKey === this.opts.runtimeKey) {
                         this.activeSessionId = undefined;
                     }
@@ -1099,10 +1232,10 @@ export class ParallAgentGateway {
                     recordMissingReply(this.opts.runtimeType);
                 }
                 clearDispatchMetrics(sessionKey);
-                if (triggerMessageSet && binding) {
-                    this.opts.client
-                        .updateAgentSession(this.opts.config.org_id, this.opts.agentUserId, binding.agentSessionId, { status: 'idle' })
-                        .catch((err) => this.opts.log?.warn?.(`failed to set session idle: ${err}`));
+                if (turnHandle) {
+                    // Stale-handle safe: if a newer turn already began on this
+                    // session, the coordinator ignores this finish outright.
+                    this.sessionLifecycle.finishTurn(turnHandle);
                 }
                 clearSessionMessageId(sessionKey);
                 clearDispatchMessageId(sessionKey);
@@ -1344,10 +1477,18 @@ export class ParallAgentGateway {
                 }
             }
             if (forkBinding) {
-                this.opts.client
-                    .updateAgentSession(this.opts.config.org_id, this.opts.agentUserId, forkBinding.agentSessionId, { status: 'closed' })
-                    .catch(() => { });
-                this.sessionBindings.delete(fork.fork.sessionKey);
+                // Normal fork teardown is the ForkSessionFinalizer use case:
+                // seal → drain parked/in-flight step writes → close (serialized
+                // behind the turn's idle, ownership held through close retries) →
+                // release the binding. The gateway only triggers it. Notably this
+                // must NOT drop the session's step queue — parked steps drain to
+                // the server before the close is issued (the identity check on the
+                // release keeps a replacement fork's binding intact).
+                await this.forkFinalizer.finalize(forkBinding.agentSessionId, () => {
+                    if (this.sessionBindings.get(fork.fork.sessionKey) === forkBinding) {
+                        this.sessionBindings.delete(fork.fork.sessionKey);
+                    }
+                });
             }
         }
     }
@@ -1426,43 +1567,101 @@ export class ParallAgentGateway {
                 // false) and released the claims, so THIS site owns their resolution.
                 // Message groups here are the ledger-disabled legacy flow.
                 const typedRefs = this.typedLedgerEventIds(events);
-                if (!typedRefs) {
+                // Guard the settlement window: from the moment the group leaves the
+                // buffer until settlement finishes, a re-drive must still be
+                // absorbed by isBufferedTypedWorkItem — see drainingTypedIds. Keyed
+                // on the events' own WorkItem ids, NOT typedRefs: the legacy
+                // (ledger-disabled) fallback settles through per-event acks but its
+                // re-drives converge through the same buffered-WorkItem check.
+                const drainingIds = events
+                    .map((ev) => ev.dispatchEventId)
+                    .filter((id) => !!id);
+                for (const id of drainingIds)
+                    this.drainingTypedIds.add(id);
+                try {
+                    if (!typedRefs) {
+                        try {
+                            await this.emitDispatchReceived(event);
+                        }
+                        catch (err) {
+                            this.opts.log?.warn?.(`mark-received failed for buffered dispatch, leaving unacked for retry: ${String(err)}`);
+                            this.dispatchState.mainBuffer.unshift(...events);
+                            this.dispatchState.pendingForkResults.unshift(...pendingFork);
+                            break;
+                        }
+                    }
+                    // A typed group dispatches ALL buffered bodies, not just the
+                    // newest: typed events are never steer-injected (unlike messages,
+                    // which the model already saw mid-turn), so a comment burst folded
+                    // into one drain turn would otherwise surface only its last member
+                    // to the LLM — earlier ones exist solely as input steps the model
+                    // never reads. Keyed on the event kind, NOT on typedRefs: the
+                    // ledger-disabled fallback buffers the same bursts and owes the
+                    // model the same visibility (groups are homogeneous — see
+                    // dispatchGroupKey). Adapters that present earlierEvents natively
+                    // (OpenClaw InboundHistory) are exempt — concatenating would show
+                    // every earlier member twice.
+                    const isTypedGroup = events.every((ev) => ev.type !== 'message');
+                    const body = isTypedGroup &&
+                        events.length > 1 &&
+                        this.opts.dispatchAdapter.earlierEventsInPrompt !== true
+                        ? events.map((ev) => buildEventBody(ev)).join('\n\n')
+                        : buildEventBody(event);
+                    let dispatched;
                     try {
-                        await this.emitDispatchReceived(event);
+                        dispatched = await this.runDispatch(event, this.opts.runtimeKey, forkPrefix + body, earlier);
                     }
                     catch (err) {
-                        this.opts.log?.warn?.(`mark-received failed for buffered dispatch, leaving unacked for retry: ${String(err)}`);
+                        if (!isTypedGroup)
+                            throw err;
+                        // The retained dedupe claims live exactly as long as the buffer
+                        // stay — a thrown turn dropped these events without settlement,
+                        // so free the claims here or every re-drive is rejected at the
+                        // dedupe gate until restart (#1149).
+                        this.opts.log?.error(`typed drain dispatch failed for ${event.messageId} (group of ${events.length}, claims freed for re-drive): ${String(err)}`);
+                        for (const ev of events)
+                            clearTypedDedupeForEvent(this.laneFlowHost(), ev);
+                        this.dispatchState.pendingForkResults.unshift(...pendingFork);
+                        continue;
+                    }
+                    if (!dispatched) {
+                        // Shutdown: skip the ack so the server redelivers these buffered
+                        // events to the replacement pod via dispatch catch-up. Put both
+                        // the buffered events and the fork results back so nothing is
+                        // lost.
                         this.dispatchState.mainBuffer.unshift(...events);
                         this.dispatchState.pendingForkResults.unshift(...pendingFork);
                         break;
                     }
+                    if (typedRefs) {
+                        // Wrapper-less resolution of the buffered typed group — protocol
+                        // lives in gateway-lane-flow.ts. The turn-error marker is
+                        // consumed HERE, before this loop can start another turn on the
+                        // session.
+                        await settleDrainedTypedGroup(this.laneFlowHost(), events, typedRefs, this.consumeTurnError(this.opts.runtimeKey));
+                        continue;
+                    }
+                    for (const bufferedEvent of events) {
+                        const sourceType = bufferedEvent.ackSourceType ??
+                            (bufferedEvent.type === 'task' ? 'task_activity' : 'message');
+                        const sourceId = bufferedEvent.ackSourceId ?? bufferedEvent.messageId;
+                        this.opts.client
+                            .ackDispatch(this.opts.config.org_id, {
+                            source_type: sourceType,
+                            source_id: sourceId,
+                        })
+                            .catch(() => { });
+                        // Retained-claim lifetime is the buffer stay on the legacy face
+                        // too: the fire-and-forget ack may fail (the row re-drives and
+                        // must not be self-rejected), and a shared-key task sibling must
+                        // not be blocked by this drained copy's claim. No-op for message
+                        // events (no typed dedupe entry).
+                        clearTypedDedupeForEvent(this.laneFlowHost(), bufferedEvent);
+                    }
                 }
-                const dispatched = await this.runDispatch(event, this.opts.runtimeKey, forkPrefix + buildEventBody(event), earlier);
-                if (!dispatched) {
-                    // Shutdown: skip the ack so the server redelivers these buffered
-                    // events to the replacement pod via dispatch catch-up. Put both the
-                    // buffered events and the fork results back so nothing is lost.
-                    this.dispatchState.mainBuffer.unshift(...events);
-                    this.dispatchState.pendingForkResults.unshift(...pendingFork);
-                    break;
-                }
-                if (typedRefs) {
-                    // Wrapper-less resolution of the buffered typed group — protocol
-                    // lives in gateway-lane-flow.ts. The turn-error marker is consumed
-                    // HERE, before this loop can start another turn on the session.
-                    await settleDrainedTypedGroup(this.laneFlowHost(), events, typedRefs, this.consumeTurnError(this.opts.runtimeKey));
-                    continue;
-                }
-                for (const bufferedEvent of events) {
-                    const sourceType = bufferedEvent.ackSourceType ??
-                        (bufferedEvent.type === 'task' ? 'task_activity' : 'message');
-                    const sourceId = bufferedEvent.ackSourceId ?? bufferedEvent.messageId;
-                    this.opts.client
-                        .ackDispatch(this.opts.config.org_id, {
-                        source_type: sourceType,
-                        source_id: sourceId,
-                    })
-                        .catch(() => { });
+                finally {
+                    for (const id of drainingIds)
+                        this.drainingTypedIds.delete(id);
                 }
             }
         }
@@ -1578,10 +1777,27 @@ export class ParallAgentGateway {
                 if (this.shuttingDown) {
                     return false;
                 }
+                // A re-driven typed WorkItem may already be buffered from a prior
+                // consume attempt (claim → busy main → buffer → release → server
+                // re-drive): the buffered copy is the one the drain settles, so a
+                // second copy would double the drain group's input steps and prompt
+                // content. Drop the duplicate; the caller releases the row again and
+                // the re-drive keeps converging on the buffered copy (#1149).
+                if (this.isBufferedTypedWorkItem(event.dispatchEventId)) {
+                    return false;
+                }
                 // Push synchronously BEFORE the (possibly async) steer attempt so
                 // arrival order is preserved and the event cannot be orphaned in a
                 // gap between the steer await and the push.
                 this.dispatchState.mainBuffer.push(event);
+                // FIFO fence for BOTH injection branches below: a buffered typed
+                // event is never injected, but the adapters track pending injections
+                // as a COUNT, not by identity — if a message injected behind a
+                // buffered typed event, the drain (typed group first, FIFO) would
+                // consume the message's steer output as the typed group's turn: the
+                // typed body never reaches the model yet resolves, and the message
+                // replays. When anything un-injected sits ahead, buffer only.
+                const typedAheadInBuffer = this.dispatchState.mainBuffer.some((e) => e.type !== 'message');
                 if (this.usesLaneLedger(event)) {
                     // Ledger flow: fold into the live lane server-side FIRST, then
                     // inject. An un-folded injection is forbidden (the pending WorkItem
@@ -1595,14 +1811,25 @@ export class ParallAgentGateway {
                     // current turn's reply without the model ever seeing it. Leaving
                     // the event un-folded keeps it buffered; the drain claims it as
                     // its own turn and folds it there.
-                    if (this.mainCurrentGroupKey === this.dispatchGroupKey(event) &&
+                    if (!typedAheadInBuffer &&
+                        this.mainCurrentGroupKey === this.dispatchGroupKey(event) &&
                         this.opts.dispatchAdapter.enqueueDuringDispatch != null &&
                         (await this.laneLedger?.steerLive(event)) &&
                         (await this.opts.dispatchAdapter.enqueueDuringDispatch(this.opts.runtimeKey, buildEventBody(event)))) {
                         this.opts.log?.info(`steer folded+injected for ${event.messageId} (will drain for bookkeeping)`);
                     }
                 }
-                else if (this.dispatchState.mainCurrentTargetId === event.targetId &&
+                else if (
+                // Message events only. A typed event (task_comment/schedule/…)
+                // rides the typed-consume contract — buffer-main resolves false and
+                // the claim releases for re-drive — so an injection here is exactly
+                // the forbidden un-folded injection: the LLM sees the content while
+                // the WorkItem stays live, and every re-drive injects it AGAIN (the
+                // 7/16 watcher duplicate-delivery loop, #1149). Typed events stay
+                // buffered; the drain claims them as their own turn.
+                event.type === 'message' &&
+                    !typedAheadInBuffer &&
+                    this.dispatchState.mainCurrentTargetId === event.targetId &&
                     (await this.opts.dispatchAdapter.enqueueDuringDispatch?.(this.opts.runtimeKey, buildEventBody(event)))) {
                     this.opts.log?.info(`steer injected for ${event.messageId} (will drain for bookkeeping)`);
                 }
@@ -1874,7 +2101,7 @@ export class ParallAgentGateway {
             dispatchEventId,
         };
         const dispatched = await this.handleInboundEvent(event);
-        if (!dispatched) {
+        if (!dispatched && !this.isTypedEventBuffered(event)) {
             this.dispatchedTasks.delete(dedupeKey);
         }
         return dispatched;
@@ -1971,7 +2198,7 @@ export class ParallAgentGateway {
             this.dispatchedTasks.delete(dedupeKey);
             throw err;
         }
-        if (!dispatched) {
+        if (!dispatched && !this.isTypedEventBuffered(event)) {
             this.dispatchedTasks.delete(dedupeKey);
         }
         return dispatched;
@@ -2040,7 +2267,7 @@ export class ParallAgentGateway {
             this.dispatchedTasks.delete(dedupeKey);
             throw err;
         }
-        if (!dispatched) {
+        if (!dispatched && !this.isTypedEventBuffered(event)) {
             this.dispatchedTasks.delete(dedupeKey);
         }
         return dispatched;
@@ -2110,7 +2337,7 @@ export class ParallAgentGateway {
             this.dispatchedTasks.delete(dedupeKey);
             throw err;
         }
-        if (!dispatched) {
+        if (!dispatched && !this.isTypedEventBuffered(event)) {
             this.dispatchedTasks.delete(dedupeKey);
         }
         return dispatched;
@@ -2183,18 +2410,20 @@ export class ParallAgentGateway {
                 provider = undefined; // label degrades; reply hint still names the clip generically
             }
         }
-        // The reply hint routes on the live capability grant: `<provider>-cli`
-        // present → the vendor CLI is on PATH (broker shim) and is THE reply
-        // path; absent → outbound is disabled for this org (flag/connection off)
-        // and the hint must say so instead of pointing at a retired clip. The
-        // provider label lookup above is best-effort/cosmetic — when it fails,
-        // ANY granted `*-cli` capability keeps the hint on the CLI path: a
-        // transient metadata miss must not flip an actively granted agent's
-        // hint to "outbound disabled" and strand a valid external message.
+        // The reply hint routes on the live capability grant, keyed PER
+        // PROVIDER: feishu's affordance is the vendor CLI on PATH (`feishu-cli`
+        // → lark-cli, tier A) and slack's is the platform verb (`slack-send` →
+        // `parall slack send`, tier B — there is no `slack-cli`). Absent →
+        // outbound is disabled for this org (flag/connection off) and the hint
+        // must say so instead of pointing at a retired clip. The provider label
+        // lookup above is best-effort/cosmetic — when it fails, ANY granted
+        // channel capability keeps the hint on the capability path: a transient
+        // metadata miss must not flip an actively granted agent's hint to
+        // "outbound disabled" and strand a valid external message.
         const keys = this.opts.getCapabilityKeys?.() ?? [];
         const cliCapable = provider
-            ? keys.includes(`${provider}-cli`)
-            : keys.some((k) => k.endsWith('-cli'));
+            ? keys.includes(channelCapabilityKeyFor(provider))
+            : keys.some((k) => k.endsWith('-cli') || k === CAPABILITY_SLACK_SEND);
         const event = {
             type: 'channel_message',
             targetId: conv.id,
@@ -2224,7 +2453,7 @@ export class ParallAgentGateway {
             this.dispatchedMessages.delete(claimKey);
             throw err;
         }
-        if (!dispatched) {
+        if (!dispatched && !this.isTypedEventBuffered(event)) {
             this.dispatchedMessages.delete(claimKey);
         }
         return dispatched;
@@ -2267,7 +2496,7 @@ export class ParallAgentGateway {
             this.dispatchedTasks.delete(dedupeKey);
             throw err;
         }
-        if (!dispatched) {
+        if (!dispatched && !this.isTypedEventBuffered(event)) {
             this.dispatchedTasks.delete(dedupeKey);
         }
         return dispatched;
@@ -2319,7 +2548,7 @@ export class ParallAgentGateway {
             this.dispatchedTasks.delete(dedupeKey);
             throw err;
         }
-        if (!dispatched) {
+        if (!dispatched && !this.isTypedEventBuffered(event)) {
             this.dispatchedTasks.delete(dedupeKey);
         }
         return dispatched;
@@ -2389,6 +2618,13 @@ export class ParallAgentGateway {
                     }
                     continue;
                 }
+                // Same pre-claim guard as the dispatch.new handler: a WorkItem whose
+                // event copy is already buffered belongs to the drain — claiming it
+                // here would race the drain's fence-less settlement.
+                if (item.event_type !== 'message' && this.isBufferedTypedWorkItem(item.id)) {
+                    this.opts.log?.info(`typed dispatch ${item.id} already buffered for the drain — skipping catch-up claim`);
+                    continue;
+                }
                 processed++;
                 try {
                     const typedHooks = {
@@ -2611,6 +2847,27 @@ export class ParallAgentGateway {
             await this.laneLedger.releaseAll();
         }
         await this.opts.onBeforeDisconnect?.();
+        // Parked step writes are process-local and their WorkItems are already
+        // resolved — restart catch-up will NOT re-drive them, so anything still
+        // parked at exit is permanently lost. Spend a slice of the shutdown
+        // budget on one flush pass first: the common shutdown (idle-stop,
+        // deploy) happens on a healthy network where these writes just succeed.
+        // The 10s cap is hard — a write still in flight at the deadline is
+        // abandoned to the background (see StepRetryQueue.flush).
+        if (this.stepPersister.pendingTotal() > 0) {
+            const remaining = await this.stepPersister.flush(10_000);
+            if (remaining > 0) {
+                this.opts.log?.warn(`${remaining} parked step write(s) could not be flushed at shutdown; they are permanently lost`);
+            }
+        }
+        this.stepPersister.dispose();
+        // Lifecycle last: idle writes land after the flushed steps, so the
+        // server clears activity once and no flushed step can relight it.
+        const lifecycleRemaining = await this.sessionLifecycle.flush(5_000);
+        if (lifecycleRemaining > 0) {
+            this.opts.log?.warn(`${lifecycleRemaining} session lifecycle write(s) unreconciled at shutdown`);
+        }
+        this.sessionLifecycle.dispose();
         this.opts.ws.disconnect();
         this.opts.log?.info(`disconnected`);
     }