@tickernelz/paperclip-pro-server 2026.930.2 → 2026.930.4

package/dist/routes/issues.js

@@ -8,7 +8,7 @@ import { releaseDependencyGateRecoveryHold } from "../services/dependency-gate-r
 import { documentExportFileName, extractIssueReferenceIdentifiers, requiresExecutionReconciliation } from "@tickernelz/paperclip-pro-shared";
 import { renderDocumentPdf } from "../services/document-pdf.js";
 import { validateExecutionReconciliation, markExecutionReconciliation, } from "../services/execution-recovery-resolution.js";
-import { storedSteeringAcknowledgement, reconcileSteeredIdentity, reserveSteeredIdentity, acceptSteeredIdentity, rejectSteeredIdentity, } from "../services/run-identity.js";
+import { storedSteeringAcknowledgement, reserveSteeredIdentity, acceptSteeredIdentity, rejectSteeredIdentity, } from "../services/run-identity.js";
 import { createHash, randomUUID } from "node:crypto";
 import { Router } from "express";
 import multer from "multer";
@@ -74,8 +74,17 @@ import { evaluateIssueThreadInteractionResolverAudience, issueThreadInteractionA
 import { resolveSelectedSuggestedTasks } from "../services/issue-thread-interactions.js";
 import { crossIssueInfluenceLimitError, crossIssueInfluenceRunContextError, observeCrossIssueInfluence, } from "../services/cross-issue-influence-limit.js";
 import { getNativeSessionSteeringState, hasLiveAdapterSteering, NativeSessionSteeringError, steerNativeSession, } from "../services/native-runtime/native-session-executor.js";
-import { buildQueuedCommentQueueSnapshot, decideQueuedCommentQueueSteering, queuedCommentIdsFromWakePayload, withQueuedCommentIdsInWakePayload, } from "../services/issue-queued-comment-queue.js";
+import { buildQueuedCommentQueueSnapshot, decideQueuedCommentQueueSteering, queuedCommentIdsFromWakePayload, withQueuedCommentIdsInRunContext, withQueuedCommentIdsInWakePayload, } from "../services/issue-queued-comment-queue.js";
+import { resultJsonWithSteeringAcknowledgement } from "../services/steering-acknowledgements.js";
+import { setTimeout as sleep } from "node:timers/promises";
 const MAX_ISSUE_COMMENT_LIMIT = 500;
+const DEFAULT_STEERING_RETRY = { delaysMs: [1_000, 3_000], budgetMs: 10_000, minAttemptMs: 2_000 };
+const RETRYABLE_STEERING_CODES = {
+    steering_rejected: true,
+    steering_stale_turn: true,
+    steering_temporarily_unavailable: true,
+    steering_identity_pending: true,
+};
 const updateIssueRouteSchema = updateIssueSchema.extend({
     interrupt: z.boolean().optional(),
 });
@@ -4814,13 +4823,15 @@ export function issueRoutes(db, storage, opts = {}) {
         const activeRunId = state === "deferred"
             ? (input.targetRunId ?? input.issue.executionRunId ?? null)
             : null;
-        const activeRun = activeRunId
-            ? await input.tx
+        const activeRunQuery = activeRunId
+            ? input.tx
                 .select()
                 .from(heartbeatRuns)
                 .where(and(eq(heartbeatRuns.id, activeRunId), eq(heartbeatRuns.companyId, input.issue.companyId), input.allowStoppedTarget ? undefined : eq(heartbeatRuns.status, "running")))
-                .for("update")
                 .limit(1)
+            : null;
+        const activeRun = activeRunQuery
+            ? await (input.lockActiveRun === false ? activeRunQuery : activeRunQuery.for("update"))
                 .then((rows) => rows[0] ?? null)
             : null;
         if (input.targetRunId) {
@@ -11549,6 +11560,128 @@ export function issueRoutes(db, storage, opts = {}) {
         });
         res.json(await runRedactions.redactForIssue(issue.companyId, issue.id, queue));
     });
+    const steeringRetry = opts.steeringRetry ?? DEFAULT_STEERING_RETRY;
+    async function withdrawSteeredComment(input) {
+        const { tx, wake, queueRun, commentId } = input;
+        const remainingIds = input.queue.entries
+            .map((candidate) => candidate.comment.id)
+            .filter((candidateId) => candidateId !== commentId);
+        const now = new Date();
+        if (remainingIds.length === 0) {
+            await tx
+                .update(agentWakeupRequests)
+                .set({ status: "cancelled", finishedAt: now, updatedAt: now })
+                .where(eq(agentWakeupRequests.id, wake.id));
+            if (queueRun) {
+                await tx
+                    .update(heartbeatRuns)
+                    .set({
+                    status: "cancelled",
+                    finishedAt: now,
+                    error: "Queued message was steered into the running turn",
+                    errorCode: "queued_comment_steered",
+                    updatedAt: now,
+                })
+                    .where(and(eq(heartbeatRuns.id, queueRun.id), eq(heartbeatRuns.status, "queued")));
+                await tx
+                    .update(issueRows)
+                    .set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: now })
+                    .where(and(eq(issueRows.id, input.issueId), eq(issueRows.executionRunId, queueRun.id)));
+            }
+            return null;
+        }
+        if (queueRun) {
+            await tx
+                .update(heartbeatRuns)
+                .set({
+                contextSnapshot: withQueuedCommentIdsInRunContext(queueRun.contextSnapshot, remainingIds),
+                updatedAt: now,
+            })
+                .where(and(eq(heartbeatRuns.id, queueRun.id), eq(heartbeatRuns.status, "queued")));
+        }
+        return tx
+            .update(agentWakeupRequests)
+            .set({
+            payload: withQueuedCommentIdsInWakePayload(wake.payload, remainingIds),
+            updatedAt: now,
+        })
+            .where(eq(agentWakeupRequests.id, wake.id))
+            .returning()
+            .then((rows) => rows[0] ?? wake);
+    }
+    async function recordSteeringAcknowledgement(input) {
+        const now = new Date();
+        await input.tx
+            .update(heartbeatRuns)
+            .set({
+            resultJson: resultJsonWithSteeringAcknowledgement(input.commentId, {
+                status: "acknowledged",
+                queueId: input.queueId,
+                turnId: input.turnId,
+                acknowledgedAt: now.toISOString(),
+            }),
+            updatedAt: now,
+        })
+            .where(eq(heartbeatRuns.id, input.runId));
+    }
+    async function settleLateSteeringAcknowledgement(input) {
+        await db.transaction(async (tx) => {
+            await tx
+                .select({ id: issueRows.id })
+                .from(issueRows)
+                .where(and(eq(issueRows.id, input.issue.id), eq(issueRows.companyId, input.issue.companyId)))
+                .for("update");
+            const run = await tx
+                .select({ resultJson: heartbeatRuns.resultJson })
+                .from(heartbeatRuns)
+                .where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.issue.companyId)))
+                .then((rows) => rows[0] ?? null);
+            if (!run)
+                return;
+            if (input.identity)
+                await acceptSteeredIdentity(tx, input.identity);
+            const recorded = readObject(readObject(readObject(run.resultJson).queuedSteeringAcknowledgements)[input.commentId]);
+            if (recorded.status === "acknowledged")
+                return;
+            const pendingWakes = await tx
+                .select({ id: agentWakeupRequests.id, payload: agentWakeupRequests.payload })
+                .from(agentWakeupRequests)
+                .where(and(eq(agentWakeupRequests.companyId, input.issue.companyId), inArray(agentWakeupRequests.status, ["deferred_issue_execution", "queued"]), sql `${agentWakeupRequests.payload}->>'issueId' = ${input.issue.id}`));
+            for (const pending of pendingWakes) {
+                if (!queuedCommentIdsFromWakePayload(pending.payload).includes(input.commentId) &&
+                    queuedInteractionId(pending.payload) !== input.commentId)
+                    continue;
+                const locked = await lockQueuedCommentState({
+                    tx,
+                    issue: input.issue,
+                    actor: input.actor,
+                    queueId: pending.id,
+                    lockActiveRun: false,
+                }).catch((error) => {
+                    if (error instanceof HttpError)
+                        return null;
+                    throw error;
+                });
+                if (!locked?.queue.entries.some((entry) => entry.comment.id === input.commentId))
+                    continue;
+                await withdrawSteeredComment({
+                    tx,
+                    issueId: input.issue.id,
+                    queue: locked.queue,
+                    wake: locked.wake,
+                    queueRun: locked.queueRun,
+                    commentId: input.commentId,
+                });
+            }
+            await recordSteeringAcknowledgement({
+                tx,
+                runId: input.runId,
+                commentId: input.commentId,
+                queueId: input.queueId,
+                turnId: input.turnId,
+            });
+        });
+    }
     async function steerQueuedCommentInTransaction(input) {
         const { issue, actor, commentId } = input;
         const responseWake = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.id, input.queueId), eq(agentWakeupRequests.companyId, issue.companyId))).then(rows => rows[0]);
@@ -11604,7 +11737,6 @@ export function issueRoutes(db, storage, opts = {}) {
                         .select()
                         .from(heartbeatRuns)
                         .where(and(eq(heartbeatRuns.id, input.targetRunId), eq(heartbeatRuns.companyId, issue.companyId), eq(heartbeatRuns.agentId, retryWake.agentId)))
-                        .for("update")
                         .limit(1)
                         .then((rows) => rows[0] ?? null)
                     : null;
@@ -11635,6 +11767,7 @@ export function issueRoutes(db, storage, opts = {}) {
                     actor,
                     queueId: input.queueId,
                     targetRunId: input.targetRunId,
+                    lockActiveRun: false,
                 });
                 if (!locked.activeRun) {
                     throw conflict("The queued message targets a stale run", {
@@ -11681,11 +11814,12 @@ export function issueRoutes(db, storage, opts = {}) {
                     });
                 }
                 steeringDeliveryAttempted = true;
+                const activeRunId = locked.activeRun.id;
                 const acknowledgement = (await storedSteeringAcknowledgement(tx, steeringIdentity ?? {
-                    companyId: issue.companyId, runId: locked.activeRun.id, messageId: commentId,
+                    companyId: issue.companyId, runId: activeRunId, messageId: commentId,
                 })) ??
                     (await steerNativeSession({
-                        runId: locked.activeRun.id,
+                        runId: activeRunId,
                         message: steeringExposesLowTrustRaw
                             ? entry.comment.body
                             : sanitizeQuarantinedCommentForHigherTrust({
@@ -11693,51 +11827,39 @@ export function issueRoutes(db, storage, opts = {}) {
                                 sourceTrust: entry.comment.sourceTrust ?? null,
                             }).body,
                         correlationId: commentId,
-                        onAcknowledged: steeringIdentity
-                            ? () => reconcileSteeredIdentity(db, steeringIdentity)
-                            : undefined,
+                        timeoutMs: input.timeoutMs,
+                        onAcknowledged: ({ turnId }) => settleLateSteeringAcknowledgement({
+                            issue,
+                            actor,
+                            commentId,
+                            queueId: input.queueId,
+                            runId: activeRunId,
+                            turnId,
+                            identity: steeringIdentity,
+                        }),
+                        onUnacknowledged: async () => {
+                            if (steeringIdentity)
+                                await rejectSteeredIdentity(db, steeringIdentity);
+                        },
                     }));
                 if (steeringIdentity)
                     await acceptSteeredIdentity(tx, steeringIdentity);
                 acknowledgedTurnId = acknowledgement.turnId;
-                const remainingIds = locked.queue.entries
-                    .map((candidate) => candidate.comment.id)
-                    .filter((candidateId) => candidateId !== commentId);
-                const now = new Date();
-                const nextWake = remainingIds.length === 0
-                    ? await tx
-                        .update(agentWakeupRequests)
-                        .set({ status: "cancelled", finishedAt: now, updatedAt: now })
-                        .where(eq(agentWakeupRequests.id, locked.wake.id))
-                        .returning()
-                        .then(() => null)
-                    : await tx
-                        .update(agentWakeupRequests)
-                        .set({
-                        payload: withQueuedCommentIdsInWakePayload(locked.wake.payload, remainingIds),
-                        updatedAt: now,
-                    })
-                        .where(eq(agentWakeupRequests.id, locked.wake.id))
-                        .returning()
-                        .then((rows) => rows[0] ?? locked.wake);
-                await tx
-                    .update(heartbeatRuns)
-                    .set({
-                    resultJson: {
-                        ...runResult,
-                        queuedSteeringAcknowledgements: {
-                            ...acknowledgements,
-                            [commentId]: {
-                                status: "acknowledged",
-                                queueId: input.queueId,
-                                turnId: acknowledgement.turnId,
-                                acknowledgedAt: now.toISOString(),
-                            },
-                        },
-                    },
-                    updatedAt: now,
-                })
-                    .where(eq(heartbeatRuns.id, locked.activeRun.id));
+                const nextWake = await withdrawSteeredComment({
+                    tx,
+                    issueId: issue.id,
+                    queue: locked.queue,
+                    wake: locked.wake,
+                    queueRun: null,
+                    commentId,
+                });
+                await recordSteeringAcknowledgement({
+                    tx,
+                    runId: activeRunId,
+                    commentId,
+                    queueId: input.queueId,
+                    turnId: acknowledgement.turnId,
+                });
                 return buildQueuedCommentQueue({
                     executor: tx,
                     issue,
@@ -11756,20 +11878,93 @@ export function issueRoutes(db, storage, opts = {}) {
             if (steeringIdentity && !uncertain)
                 await rejectSteeredIdentity(db, steeringIdentity);
             if (error instanceof NativeSessionSteeringError) {
-                throw conflict(error.message, { code: error.code, retryable: true });
+                throw conflict(error.message, {
+                    code: error.code,
+                    retryable: true,
+                    ...(error.detail ? { detail: error.detail } : {}),
+                });
             }
             throw error;
         }
         return { queue, duplicate, acknowledgedTurnId };
     }
+    function steeringErrorCode(error) {
+        if (!(error instanceof HttpError))
+            return null;
+        const details = readObject(error.details);
+        return typeof details.code === "string" ? details.code : null;
+    }
+    async function resolveQueuedSteeringTarget(input) {
+        try {
+            const issue = await svc.getById(input.issue.id);
+            const activeRun = issue ? await resolveActiveIssueRun(issue) : null;
+            const queue = await buildQueuedCommentQueue({
+                executor: db,
+                issue: issue ?? input.issue,
+                activeRun,
+                actor: input.actor,
+                steeringDisposition: "temporarily_unavailable",
+            });
+            const entry = queue.entries.find((candidate) => candidate.comment.id === input.commentId);
+            if (entry && queue.protocol !== "paperclip_runner_v1") {
+                return { kind: "unavailable", reason: "legacy_protocol" };
+            }
+            return entry && queue.queueId && queue.targetRunId
+                ? {
+                    kind: "target",
+                    target: { queueId: queue.queueId, targetRunId: queue.targetRunId, revision: queue.revision },
+                }
+                : { kind: "none" };
+        }
+        catch (err) {
+            logger.warn({ err, issueId: input.issue.id, commentId: input.commentId }, "failed to resolve the steering target for a posted comment");
+            return { kind: "unavailable", reason: "steering_failed" };
+        }
+    }
+    async function steerQueuedCommentWithRetry(input) {
+        const deadline = Date.now() + steeringRetry.budgetMs;
+        let target = input.target;
+        for (let attempt = 0;; attempt += 1) {
+            try {
+                return await steerQueuedCommentInTransaction({
+                    issue: input.issue,
+                    actor: input.actor,
+                    commentId: input.commentId,
+                    queueId: target.queueId,
+                    targetRunId: target.targetRunId,
+                    revision: target.revision,
+                    timeoutMs: Math.max(steeringRetry.minAttemptMs, deadline - Date.now()),
+                });
+            }
+            catch (error) {
+                const code = steeringErrorCode(error);
+                const delayMs = steeringRetry.delaysMs[attempt];
+                const retryable = code !== null &&
+                    (RETRYABLE_STEERING_CODES[code] === true ||
+                        (input.targetIsServerResolved && code === "queued_comment_revision_conflict"));
+                if (!retryable ||
+                    delayMs === undefined ||
+                    deadline - Date.now() - delayMs < steeringRetry.minAttemptMs) {
+                    throw error;
+                }
+                logger.warn({
+                    issueId: input.issue.id,
+                    commentId: input.commentId,
+                    targetRunId: target.targetRunId,
+                    attempt: attempt + 1,
+                    code,
+                }, "retrying a transiently refused steer");
+                await sleep(delayMs);
+                const next = await resolveQueuedSteeringTarget(input);
+                if (next.kind !== "target" || next.target.targetRunId !== target.targetRunId)
+                    throw error;
+                if (input.targetIsServerResolved)
+                    target = next.target;
+            }
+        }
+    }
     function steeringUnavailableFromError(error) {
-        const code = error instanceof HttpError &&
-            error.details &&
-            typeof error.details === "object" &&
-            !Array.isArray(error.details) &&
-            typeof error.details.code === "string"
-            ? error.details.code
-            : null;
+        const code = steeringErrorCode(error);
         if (code === "steering_unsupported")
             return "legacy_protocol";
         if (code === "steering_identity_mismatch")
@@ -11853,30 +12048,11 @@ export function issueRoutes(db, storage, opts = {}) {
         catch (err) {
             logger.warn({ err, issueId: input.issue.id, commentId: input.commentId }, "failed to complete the comment wake dispatch before steering");
         }
-        let target = null;
-        try {
-            const issue = await svc.getById(input.issue.id);
-            const activeRun = issue ? await resolveActiveIssueRun(issue) : null;
-            const queue = await buildQueuedCommentQueue({
-                executor: db,
-                issue: issue ?? input.issue,
-                activeRun,
-                actor: input.actor,
-                steeringDisposition: "temporarily_unavailable",
-            });
-            const entry = queue.entries.find((candidate) => candidate.comment.id === input.commentId);
-            if (entry && queue.protocol !== "paperclip_runner_v1") {
-                return { deliveredAs: "queued", steeringUnavailable: "legacy_protocol" };
-            }
-            target =
-                entry && queue.queueId && queue.targetRunId
-                    ? { queueId: queue.queueId, targetRunId: queue.targetRunId, revision: queue.revision }
-                    : null;
-        }
-        catch (err) {
-            logger.warn({ err, issueId: input.issue.id, commentId: input.commentId }, "failed to resolve the steering target for a posted comment");
-            return { deliveredAs: "queued", steeringUnavailable: "steering_failed" };
+        const resolved = await resolveQueuedSteeringTarget(input);
+        if (resolved.kind === "unavailable") {
+            return { deliveredAs: "queued", steeringUnavailable: resolved.reason };
         }
+        const target = resolved.kind === "target" ? resolved.target : null;
         if (!target) {
             return {
                 deliveredAs: "queued",
@@ -11895,13 +12071,12 @@ export function issueRoutes(db, storage, opts = {}) {
         }
         let steered;
         try {
-            steered = await steerQueuedCommentInTransaction({
+            steered = await steerQueuedCommentWithRetry({
                 issue: input.issue,
                 actor: input.actor,
                 commentId: input.commentId,
-                queueId: target.queueId,
-                targetRunId: target.targetRunId,
-                revision: target.revision,
+                target,
+                targetIsServerResolved: true,
             });
         }
         catch (error) {
@@ -11962,13 +12137,16 @@ export function issueRoutes(db, storage, opts = {}) {
         if (!decision.allowed)
             throw forbidden(decision.explanation, authorizationDeniedDetails(decision));
         const actor = getActorInfo(req);
-        const { queue, duplicate, acknowledgedTurnId } = await steerQueuedCommentInTransaction({
+        const { queue, duplicate, acknowledgedTurnId } = await steerQueuedCommentWithRetry({
             issue,
             actor,
             commentId,
-            queueId: req.body.queueId,
-            targetRunId: req.body.targetRunId,
-            revision: req.body.revision,
+            target: {
+                queueId: req.body.queueId,
+                targetRunId: req.body.targetRunId,
+                revision: req.body.revision,
+            },
+            targetIsServerResolved: false,
         });
         await logQueuedCommentSteered({
             issue,