@dsh-enhanced/assistant-delivery 0.1.62 → 0.1.64

package/lib/agent-runtime.js

@@ -13,6 +13,8 @@ import { externalPrincipalId } from './canonical.js';
 import { feedbackSignalInput, feedbackUsage, parseFeedbackCommand, } from './feedback-command.js';
 import { objectiveStateMessage } from './feedback-command.js';
 import { learningCommandUsage, parseLearningCommand, serializeLearningExport, } from './learning-command.js';
+import { attachRestartRun, restartDigest, restartContinuationText } from './restart-runtime.js';
+import { admitRestartContinuation, createRestartContinuation } from './restart-native.js';
 import { renderOwnerApprovalStatus } from './approval-status-context.js';
 import { isExactDeliveryCommand, learningDispatchRecoveryCode, parseDeliveryCommand, permissionDispatchRecoveryCode, } from './session-commands.js';
 import { DeliveryStoreError } from './store.js';
@@ -20,6 +22,12 @@ import { SessionLeaseUnavailable } from './session-lease-runtime.js';
 import { isDeliveryGoalWakeDeadline } from './goal-wake-types.js';
 import { parseWorkflowCommand, workflowCommandUsage, } from './workflow-command.js';
 const SAFE_NATIVE_COMMANDS = new Set(['compact']);
+class AgentTaskTimeoutError extends Error {
+    constructor() {
+        super('assistant-delivery: foreground task exceeded its execution deadline');
+        this.name = 'AgentTaskTimeoutError';
+    }
+}
 class UserSessionCancellation extends Error {
     command;
     constructor(command) {
@@ -1116,6 +1124,24 @@ export class DshDeliveryRuntime {
     ctx;
     policy;
     options;
+    restartRuns = new WeakMap();
+    canRequestRestart(agent) { return this.restartRuns.get(agent)?.canRequest() === true; }
+    requestRestart(agent, reason) {
+        const run = this.restartRuns.get(agent);
+        if (run === undefined)
+            throw new Error('assistant-delivery: this foreground task has no restart authority');
+        const record = run.request(reason);
+        return { id: record.id, status: record.status };
+    }
+    async resumeRestart(record, binding, envelope, signal) {
+        const restart = this.options.restart;
+        if (restart === undefined || !restart.authorize(record))
+            throw new Error('assistant-delivery: restart continuation is not authorized');
+        return await this.process(binding, envelope, signal, undefined, () => {
+            if (!restart.authorize(record))
+                throw new Error('assistant-delivery: restart continuation authority changed');
+        }, record);
+    }
     dispatchControl = 'explicit';
     activeSessionControls = new Map();
     sessionLeases;
@@ -1561,11 +1587,15 @@ export class DshDeliveryRuntime {
         return true;
     }
     /** Wait for only the turn that claims Delivery's exact message identity. */
-    followupTurn(agent, message, onClaimed, onSettled) {
+    followupTurn(agent, message, onClaimed, onSettled, admit, signal) {
         return new Promise((resolve, reject) => {
             let claimedTurn;
             let settled = false;
+            let abortDeadline;
             const cleanup = () => {
+                if (abortDeadline !== undefined)
+                    clearTimeout(abortDeadline);
+                signal?.removeEventListener('abort', aborted);
                 removeClaimed();
                 removeDiscarded();
                 removeSessionEvent();
@@ -1584,6 +1614,13 @@ export class DshDeliveryRuntime {
                 cleanup();
                 reject(error);
             };
+            const aborted = () => {
+                // Let cooperative native cancellation persist its tool result and
+                // turn/end before disposing. Uncooperative tools get only this grace.
+                abortDeadline ??= setTimeout(() => fail(signal?.reason
+                    ?? new Error('assistant-delivery: execution aborted')), 1_000);
+                abortDeadline.unref?.();
+            };
             const removeClaimed = this.ctx.on('agent/inbox/claimed', event => {
                 if (event.agent === agent && event.message.id === message.id) {
                     claimedTurn = event.turn;
@@ -1614,8 +1651,21 @@ export class DshDeliveryRuntime {
                     settle(claimedTurn);
                 }
             });
+            signal?.addEventListener('abort', aborted, { once: true });
+            if (signal?.aborted) {
+                fail(signal.reason);
+                return;
+            }
             try {
-                agent.followup(message);
+                if (admit === undefined)
+                    agent.followup(message);
+                else
+                    void admit().then(state => {
+                        if (state.kind === 'settled')
+                            settle(state.turn);
+                        else if (state.kind !== 'pending')
+                            fail(new Error('assistant-delivery: continuation was already dispatched without a settled result'));
+                    }, fail);
             }
             catch (error) {
                 fail(error);
@@ -1728,7 +1778,7 @@ export class DshDeliveryRuntime {
             return await this.followupTurn(agent, message, onClaimed, () => {
                 exactTurnSettled = true;
                 releaseToolBarriers();
-            });
+            }, undefined, signal);
         }
         catch (error) {
             if (signal.aborted)
@@ -3341,7 +3391,7 @@ export class DshDeliveryRuntime {
             await this.disposeAfterReplyBoundary(binding.sessionId, handle);
         }
     }
-    async process(binding, envelope, outerSignal, prepared, markDispatching) {
+    async process(binding, envelope, outerSignal, prepared, markDispatching, restartContinuation) {
         const sessionCommand = parseDeliveryCommand(envelope);
         if (envelope.kind === 'command' && sessionCommand === undefined) {
             return this.replySessionCommand(binding, envelope, {
@@ -3386,14 +3436,21 @@ export class DshDeliveryRuntime {
             throw error;
         }
         let control;
+        let taskDeadline;
         try {
             control = this.beginSessionControl(binding.sessionId, lease.signal);
+            taskDeadline = setTimeout(() => {
+                // Cancel the durable lease as well as the local wait: late tools and
+                // model requests must remain fenced until real teardown is observed.
+                lease.cancel(new AgentTaskTimeoutError());
+            }, this.options.taskTimeoutMs);
+            taskDeadline.unref?.();
             lease.dispatch();
             return await this.processControlled(binding, envelope, control.controller.signal, sessionCommand, prepared, markDispatching === undefined ? undefined : () => {
                 lease.assert();
                 markDispatching();
                 lease.assert();
-            });
+            }, restartContinuation);
         }
         catch (error) {
             if (control?.command !== undefined && causedByUserCancellation(error)) {
@@ -3408,12 +3465,14 @@ export class DshDeliveryRuntime {
             throw error;
         }
         finally {
+            if (taskDeadline !== undefined)
+                clearTimeout(taskDeadline);
             if (control !== undefined)
                 this.endSessionControl(binding.sessionId, control);
             lease.close();
         }
     }
-    async processControlled(binding, envelope, signal, sessionCommand, prepared, markDispatching) {
+    async processControlled(binding, envelope, signal, sessionCommand, prepared, markDispatching, restartContinuation) {
         if (sessionCommand?.name === 'feedback') {
             return await this.runFeedbackCommand(sessionCommand, binding, envelope, signal, markDispatching);
         }
@@ -3516,9 +3575,13 @@ export class DshDeliveryRuntime {
         if (agents === undefined || sessions === undefined) {
             return { outcome: 'not-processed', failureCode: 'agent-runtime-unavailable', retryable: true };
         }
-        const persistedSelection = this.options.getModelSelection(envelope.conversation);
+        const restart = this.options.restart;
+        const continuationTask = restartContinuation === undefined ? undefined : restart?.store.getTask(restartContinuation.inboxId);
+        if (restartContinuation !== undefined && continuationTask?.model === undefined)
+            throw new Error('assistant-delivery: restart model is not frozen');
+        const persistedSelection = restartContinuation === undefined ? this.options.getModelSelection(envelope.conversation) : undefined;
         const requestedRoute = toModelRoute(persistedSelection
-            ?? prepared?.modelRoute
+            ?? continuationTask?.model ?? prepared?.modelRoute
             ?? { provider: this.options.provider, model: this.options.model });
         const resolvedRoute = await this.resolveExecutionRoute(envelope.conversation, persistedSelection, requestedRoute, signal);
         if ('retry' in resolvedRoute) {
@@ -3532,21 +3595,45 @@ export class DshDeliveryRuntime {
         let foregroundExecution;
         let removeAcceptanceRequest;
         let acceptanceSucceeded = false;
+        let completionPending = false;
         let goalContinuation;
         let goalContinuationQuiescent = true;
         let removeAbort;
         let removeProgress;
         let progressOpen = true;
+        let restartRun;
+        let restartCheckpoint;
         let progressQueue = Promise.resolve();
+        let progressTerminal = false;
+        const drainProgress = async () => {
+            let timeout;
+            try {
+                await Promise.race([progressQueue, new Promise(resolve => {
+                        timeout = setTimeout(resolve, 5_000);
+                        timeout.unref?.();
+                    })]);
+            }
+            finally {
+                if (timeout !== undefined)
+                    clearTimeout(timeout);
+            }
+        };
         const publishProgress = (update) => {
-            if (!progressOpen)
+            const terminal = update.kind === 'completed' || update.kind === 'failed' || update.kind === 'paused';
+            if ((!progressOpen && !terminal) || progressTerminal)
                 return;
-            progressQueue = progressQueue
-                .then(async () => {
-                if (checkAuthorization() !== 'authorized')
+            if (terminal)
+                progressTerminal = true;
+            const publish = async () => {
+                if ((!terminal && progressTerminal) || checkAuthorization() !== 'authorized')
                     return;
                 await this.options.progress(binding, envelope.eventId, update);
-            })
+            };
+            // Terminal journaling must not wait behind a slow provider's step updates.
+            // The adapter preserves provider ordering after staging the durable result.
+            progressQueue = (terminal
+                ? Promise.all([progressQueue, publish()]).then(() => { })
+                : progressQueue.then(publish))
                 // Progress is presentation-only: its provider failure must never retry an Agent turn.
                 .catch(() => { });
         };
@@ -3627,8 +3714,27 @@ export class DshDeliveryRuntime {
             // an owner tool can create/resume a goal in that very first turn and the
             // host driver is otherwise free to claim its first round at idle.
             goalContinuation = this.waitForNativeGoalContinuation(binding, envelope, agent, signal);
-            acceptance = this.options.prepareForegroundTaskAcceptance(binding, envelope);
-            foregroundExecution = this.options.beginForegroundTaskExecution(binding, envelope);
+            acceptance = restartContinuation === undefined ? this.options.prepareForegroundTaskAcceptance(binding, envelope) : undefined;
+            foregroundExecution = restartContinuation?.inboxId ?? this.options.beginForegroundTaskExecution(binding, envelope);
+            if (restart !== undefined && foregroundExecution !== undefined && restart.store.getTask(foregroundExecution) !== undefined) {
+                if (restartContinuation !== undefined) {
+                    const checkpoint = restartContinuation.checkpoint;
+                    if (checkpoint === undefined || restartDigest(agent.session.snapshotEvents().slice(0, checkpoint.sessionEventCount)) !== checkpoint.sessionDigest
+                        || checkpoint.continuationTextDigest !== restartDigest(restartContinuationText)
+                        || createRestartContinuation({ continuationId: restartContinuation.id, text: restartContinuationText }).id !== checkpoint.continuationMessageId) {
+                        throw new Error('assistant-delivery: restart checkpoint does not match the persisted Session');
+                    }
+                }
+                restartRun = attachRestartRun({ agent, inboxId: foregroundExecution, binding, envelope,
+                    store: restart.store, config: restart.config,
+                    isRestartSafeTool: restart.isRestartSafeTool,
+                    ...(restartContinuation === undefined ? {} : { continuation: restartContinuation }),
+                    authorize: () => checkAuthorization() === 'authorized' && restart.authorizeTask(foregroundExecution),
+                    authorizeRestart: () => acceptance === undefined && !goalContinuation?.pinActiveArmedGoal()
+                        && restart.authorizeTask(foregroundExecution),
+                });
+                this.restartRuns.set(agent, restartRun);
+            }
             if (acceptance !== undefined || foregroundExecution !== undefined)
                 removeAcceptanceRequest = this.ctx.on('session/event', (session, event) => {
                     if (session !== agent.session || event.type !== 'request/header')
@@ -3646,11 +3752,32 @@ export class DshDeliveryRuntime {
             // Once the durable marker exists, even a synchronous followup failure is ambiguous:
             // implementations may enqueue before throwing, so no retry is safe.
             dispatched = true;
-            const message = createUserMessage({ content, source: {
+            const message = restartContinuation === undefined ? createUserMessage({ content, source: {
                     kind: 'delivery', channel: envelope.channel, account: envelope.account, eventId: envelope.eventId,
                     trust: 'untrusted',
-                } });
-            let deliveryTurn = await this.followupTurn(agent, message);
+                } }) : createRestartContinuation({ continuationId: restartContinuation.id, text: restartContinuationText });
+            let deliveryTurn = await this.followupTurn(agent, message, undefined, undefined, restartContinuation === undefined ? undefined
+                : () => admitRestartContinuation(agent, message, { signal,
+                    authorize: () => {
+                        this.sessionLeases.assert(String(agent.session.id));
+                        if (!restart.authorize(restartContinuation))
+                            throw new Error('restart authority changed');
+                    },
+                    flush: () => sessions.flush(agent.session),
+                    onAdmitted: () => {
+                        this.sessionLeases.assert(String(agent.session.id));
+                        restart.onContinuationAdmitted(restartContinuation, String(agent.session.id));
+                    },
+                }), signal);
+            if (restartRun?.requested() !== undefined) {
+                await this.waitForAgentIdle(agent, signal);
+                restartCheckpoint = await restartRun.checkpoint();
+                this.options.replyCommand(binding, envelope.eventId, {
+                    text: '已保存当前任务，准备进行维护重启；服务就绪后会自动继续，无需重新发送请求。', format: 'plain',
+                }, `inbound:${foregroundExecution}:maintenance`);
+                // Publish the maintenance handoff only after teardown and ready commit.
+                return { outcome: 'processed' };
+            }
             let completionKind;
             let automaticTurns = 0;
             let assembledText = '';
@@ -3844,7 +3971,7 @@ export class DshDeliveryRuntime {
                 goalContinuationQuiescent = await goalContinuation.settle();
             }
             signal.throwIfAborted();
-            publishProgress({ kind: 'completed' });
+            completionPending = true;
             acceptanceSucceeded = goalContinuationQuiescent;
             return { outcome: 'processed' };
         }
@@ -3858,7 +3985,13 @@ export class DshDeliveryRuntime {
                 return { outcome: 'processed' };
             }
             if (dispatched) {
-                publishProgress({ kind: 'failed' });
+                const timedOut = signal.reason instanceof AgentTaskTimeoutError;
+                publishProgress({ kind: 'failed', ...(timedOut
+                        ? { code: 'agent-task-timeout' } : {}) });
+                // Preserve an exact durable diagnosis so the existing interruption
+                // Outbox can notify the owner even when progress presentation failed.
+                if (timedOut)
+                    return { outcome: 'not-processed', failureCode: 'agent-task-timeout', retryable: false };
                 throw new Error(`assistant-delivery: Agent turn became ambiguous: ${String(error)}`, { cause: error });
             }
             if (causedByDurableIdentity(error)) {
@@ -3906,23 +4039,49 @@ export class DshDeliveryRuntime {
             removeProgress?.();
             progressOpen = false;
             let disposed = false;
+            let maintenanceReady = false;
             try {
-                disposed = await this.disposeAfterReplyBoundary(binding.sessionId, handle, () => goalContinuation?.dispose(), !goalContinuationQuiescent, goalContinuation?.teardownTimeoutMs());
+                disposed = await this.disposeAfterReplyBoundary(binding.sessionId, handle, () => goalContinuation?.dispose(), !goalContinuationQuiescent, restartCheckpoint === undefined && restartContinuation === undefined
+                    ? goalContinuation?.teardownTimeoutMs() ?? 10_000 : 10_000);
             }
             finally {
                 acceptanceSucceeded = acceptanceSucceeded && disposed && !signal.aborted
                     && (goalContinuation?.isQuiescent() ?? true);
-                if (foregroundExecution !== undefined) {
+                if (foregroundExecution !== undefined && restartCheckpoint === undefined) {
                     try {
                         this.options.completeForegroundTaskExecution(foregroundExecution, {
                             status: acceptanceSucceeded ? 'succeeded' : 'unknown', quiescent: acceptanceSucceeded,
                         });
                     }
                     catch (error) {
+                        acceptanceSucceeded = false;
                         this.ctx.logger.warn(`assistant-delivery: foreground execution completion failed: ${String(error)}`);
                     }
                 }
-                if (acceptance !== undefined) {
+                if (restart !== undefined && restartContinuation === undefined && restartCheckpoint === undefined) {
+                    const unfinished = restartRun?.requested();
+                    if (unfinished !== undefined)
+                        restart.store.fail(unfinished.id, 'restart-checkpoint-failed');
+                }
+                if (restart !== undefined && restartCheckpoint !== undefined) {
+                    if (disposed && !signal.aborted && restart.authorize(restartCheckpoint)) {
+                        restart.store.ready(restartCheckpoint.id);
+                        maintenanceReady = true;
+                    }
+                    else
+                        restart.store.fail(restartCheckpoint.id, 'restart-teardown-not-quiescent');
+                }
+                if (restart !== undefined && restartContinuation !== undefined) {
+                    const current = restart.store.getRequest(restartContinuation.id);
+                    if (current?.status === 'resuming') {
+                        if (acceptanceSucceeded)
+                            restart.store.complete(restartContinuation.id);
+                        else
+                            restart.store.fail(restartContinuation.id, 'restart-continuation-outcome-unknown');
+                    }
+                }
+                restartRun?.dispose();
+                if (acceptance !== undefined && restartCheckpoint === undefined) {
                     try {
                         await this.options.completeForegroundTaskAcceptance(acceptance, {
                             status: acceptanceSucceeded ? 'succeeded' : 'unknown', quiescent: acceptanceSucceeded,
@@ -3932,6 +4091,16 @@ export class DshDeliveryRuntime {
                         this.ctx.logger.warn(`assistant-delivery: acceptance completion failed: ${String(error)}`);
                     }
                 }
+                // A provider-confirmed success cannot precede the durable execution
+                // result: teardown can still fail after the final reply was queued.
+                if (restartCheckpoint !== undefined)
+                    publishProgress(maintenanceReady
+                        ? { kind: 'paused' } : { kind: 'failed', code: 'INTERRUPTED_RESULT_UNKNOWN' });
+                else if (completionPending)
+                    publishProgress(acceptanceSucceeded
+                        ? { kind: 'completed' } : { kind: 'failed', code: 'INTERRUPTED_RESULT_UNKNOWN' });
+                // Terminal journal admission precedes the bounded provider drain.
+                await drainProgress();
             }
         }
     }