@taskforcehq/taskforce 0.3.326 → 0.3.328

package/dist/sync/coordinator/localWorkspaceSyncRunner.js

@@ -0,0 +1,981 @@
+import { canonicalWorkspaceSyncJson } from '../v3/canonicalWorkspaceSyncJson.js';
+import { isLocalWorkspaceSyncPullFairnessDue, LocalWorkspaceSyncRunnerStateError, } from './localWorkspaceSyncRunnerState.js';
+export class LocalWorkspaceSyncRunnerError extends Error {
+    code;
+    constructor(message, code) {
+        super(message);
+        this.code = code;
+        this.name = 'LocalWorkspaceSyncRunnerError';
+    }
+}
+const RUNNER_COMMANDS = new Set([
+    'EnableSync',
+    'DisableSync',
+    'Sync',
+    'Retry',
+    'Repair',
+]);
+const SCHEDULED_RETRY_BASE_DELAY_MS = 1_000;
+const SCHEDULED_RETRY_MAX_DELAY_MS = 5 * 60_000;
+function cloneCheckpoint(checkpoint) {
+    return JSON.parse(canonicalWorkspaceSyncJson(checkpoint));
+}
+function hasEmptyArgs(args) {
+    return args === null
+        || (Boolean(args)
+            && typeof args === 'object'
+            && !Array.isArray(args)
+            && Object.keys(args).length === 0);
+}
+function isAuthError(error) {
+    const code = String(error?.code || '');
+    return code.includes('AUTH_REQUIRED') || code.includes('AUTH_INVALID');
+}
+function isStaleRunnerError(error) {
+    const code = String(error?.code || '');
+    return code.includes('STALE_GENERATION')
+        || code.includes('STALE_REVISION')
+        || code === 'SYNC_RUNNER_COORDINATOR_FENCE_INVALID';
+}
+export class LocalWorkspaceSyncRunner {
+    options;
+    context = null;
+    queueTail = Promise.resolve();
+    commandExecutions = new Map();
+    retryTimer = null;
+    acceptedCommandWatchdogTimer = null;
+    lifecycleToken = 0;
+    prepared = false;
+    activated = false;
+    acceptingWork = false;
+    terminated = false;
+    constructor(options) {
+        this.options = options;
+    }
+    async start(context) {
+        await this.prepare(context);
+        const coordinator = this.options.coordinatorStore.readOrCreateState();
+        if (coordinator.activeWorkspaceId === context.workspaceId
+            && coordinator.coordinatorGeneration === context.coordinatorGeneration
+            && coordinator.ownershipMode === 'server'
+            && coordinator.transitionState === 'idle') {
+            this.activate();
+        }
+    }
+    async prepare(context) {
+        if (this.context && !this.terminated) {
+            throw new LocalWorkspaceSyncRunnerError('The local workspace sync runner is already started.', 'SYNC_RUNNER_ALREADY_STARTED');
+        }
+        this.assertPreparationFence(context);
+        this.clearRunnerTimers();
+        this.context = { ...context };
+        this.prepared = false;
+        this.activated = false;
+        this.terminated = false;
+        this.acceptingWork = false;
+        ++this.lifecycleToken;
+        try {
+            await this.options.recoverBeforeStart?.(context);
+            this.options.runnerStateStore.initialize({
+                tenantId: context.tenantId,
+                workspaceId: context.workspaceId,
+                coordinatorGeneration: context.coordinatorGeneration,
+            });
+            this.prepared = true;
+        }
+        catch (error) {
+            this.resetAfterPreparationFailure();
+            throw error;
+        }
+    }
+    activate() {
+        const context = this.requirePrepared();
+        this.assertCoordinatorFence(context);
+        if (this.activated)
+            return;
+        try {
+            this.options.coordinatorStore.recoverRunnerCommandsForRecreation({
+                workspaceId: context.workspaceId,
+                coordinatorGeneration: context.coordinatorGeneration,
+            });
+            const state = this.readRunnerState(context);
+            this.terminated = false;
+            this.acceptingWork = true;
+            this.activated = true;
+            const token = ++this.lifecycleToken;
+            this.schedulePersistedRetry(state, token);
+            this.scheduleAcceptedCommandWatchdog(token);
+            this.wakeAcceptedCommands();
+        }
+        catch (error) {
+            this.acceptingWork = false;
+            this.activated = false;
+            this.clearRunnerTimers();
+            throw error;
+        }
+    }
+    executeCommand(commandId) {
+        if (!this.context || this.terminated || !this.acceptingWork) {
+            return Promise.resolve(this.options.coordinatorStore.readCommand(commandId));
+        }
+        const existing = this.commandExecutions.get(commandId);
+        if (existing)
+            return existing;
+        const execution = this.enqueue(async () => this.drainAcceptedCommands(commandId))
+            .finally(() => {
+            if (this.commandExecutions.get(commandId) === execution) {
+                this.commandExecutions.delete(commandId);
+            }
+        });
+        this.commandExecutions.set(commandId, execution);
+        return execution;
+    }
+    inspect() {
+        if (!this.context || this.terminated || !this.acceptingWork) {
+            return Promise.resolve();
+        }
+        return this.enqueue(async () => this.runInspection());
+    }
+    pullDue() {
+        if (!this.context || this.terminated || !this.acceptingWork) {
+            return Promise.resolve();
+        }
+        return this.enqueue(async () => this.runInspection(this.options.operations.pull));
+    }
+    retryDue() {
+        if (!this.context || this.terminated || !this.acceptingWork) {
+            return Promise.resolve();
+        }
+        const token = this.lifecycleToken;
+        return this.enqueue(async () => this.runScheduledRetry(token));
+    }
+    pushDue() {
+        if (!this.context || this.terminated || !this.acceptingWork) {
+            return Promise.resolve();
+        }
+        return this.enqueue(async () => this.runInspection(this.options.operations.push));
+    }
+    pushContentPrerequisiteDue() {
+        if (!this.context || this.terminated || !this.acceptingWork) {
+            return Promise.resolve();
+        }
+        return this.enqueue(async () => this.runInspection(this.options.operations.pushContentPrerequisite, { bypassRetryKind: 'repair' }));
+    }
+    observeRemoteHint(observedAt = this.now()) {
+        if (!this.context || this.terminated || !this.acceptingWork) {
+            return Promise.resolve();
+        }
+        return this.enqueue(async () => {
+            const context = this.requireStarted();
+            const state = this.readRunnerState(context);
+            if (!state.checkpoint.syncEnabled
+                || state.checkpoint.syncPhase === 'provision-local')
+                return;
+            const currentObservedAt = state.checkpoint.remoteHintObservedAt;
+            const nextObservedAt = currentObservedAt
+                && Date.parse(currentObservedAt) <= Date.parse(observedAt)
+                ? currentObservedAt
+                : observedAt;
+            if (nextObservedAt === currentObservedAt)
+                return;
+            this.options.runnerStateStore.compareAndSet({
+                tenantId: context.tenantId,
+                workspaceId: context.workspaceId,
+                expectedCoordinatorGeneration: context.coordinatorGeneration,
+                expectedCheckpointRevision: state.checkpointRevision,
+                checkpoint: {
+                    ...cloneCheckpoint(state.checkpoint),
+                    remoteHintObservedAt: nextObservedAt,
+                    consecutivePushesWhilePullDue: 0,
+                },
+            });
+        });
+    }
+    dispatchPushWithFairness(dispatch, options) {
+        if (!this.context || this.terminated || !this.acceptingWork) {
+            return Promise.resolve({ kind: 'disabled' });
+        }
+        return this.enqueue(async () => {
+            const context = this.requireStarted();
+            let state = this.readRunnerState(context);
+            if (!state.checkpoint.syncEnabled)
+                return { kind: 'disabled' };
+            const hasAcceptedCommand = () => this.options.coordinatorStore.listAcceptedRunnerCommands({
+                workspaceId: context.workspaceId,
+                coordinatorGeneration: context.coordinatorGeneration,
+            }).some((command) => command.commandId !== options?.ignoreAcceptedCommandId);
+            if (hasAcceptedCommand())
+                return { kind: 'command-pending' };
+            if (!options?.ignorePullFairness
+                && isLocalWorkspaceSyncPullFairnessDue(state.checkpoint, this.now())) {
+                return { kind: 'pull-required' };
+            }
+            let pullRequired = false;
+            let commandPending = false;
+            const result = await dispatch(() => {
+                state = this.readRunnerState(context);
+                if (hasAcceptedCommand()) {
+                    commandPending = true;
+                    throw new Error('A durable sync command took priority before outbox transport.');
+                }
+                if (!options?.ignorePullFairness
+                    && isLocalWorkspaceSyncPullFairnessDue(state.checkpoint, this.now())) {
+                    pullRequired = true;
+                    throw new Error('A cloud pull became due before outbox transport.');
+                }
+                if (options?.ignorePullFairness
+                    || !state.checkpoint.remoteHintObservedAt)
+                    return;
+                state = this.options.runnerStateStore.compareAndSet({
+                    tenantId: context.tenantId,
+                    workspaceId: context.workspaceId,
+                    expectedCoordinatorGeneration: context.coordinatorGeneration,
+                    expectedCheckpointRevision: state.checkpointRevision,
+                    checkpoint: {
+                        ...cloneCheckpoint(state.checkpoint),
+                        consecutivePushesWhilePullDue: Math.min(Number.MAX_SAFE_INTEGER, state.checkpoint.consecutivePushesWhilePullDue + 1),
+                    },
+                });
+            });
+            if (commandPending)
+                return { kind: 'command-pending' };
+            if (pullRequired)
+                return { kind: 'pull-required' };
+            return { kind: 'dispatched', result };
+        });
+    }
+    notifyCredentialAvailable() {
+        if (!this.context || !this.acceptingWork || this.terminated)
+            return;
+        const state = this.options.runnerStateStore.read(this.context.tenantId, this.context.workspaceId);
+        if (state)
+            this.schedulePersistedRetry(state, this.lifecycleToken);
+        for (const command of this.options.coordinatorStore.listAcceptedRunnerCommands({
+            workspaceId: this.context.workspaceId,
+            coordinatorGeneration: this.context.coordinatorGeneration,
+        })) {
+            this.settleInternal(this.executeCommand(command.commandId), 'credential-wake');
+        }
+    }
+    async drain(graceMs) {
+        if (!this.context)
+            return;
+        this.acceptingWork = false;
+        ++this.lifecycleToken;
+        this.clearRunnerTimers();
+        const timeoutMs = Math.max(0, Math.floor(graceMs));
+        let timer = null;
+        const drained = await Promise.race([
+            this.queueTail.then(() => true, () => true),
+            new Promise((resolve) => {
+                timer = this.setTimer(() => resolve(false), timeoutMs);
+                timer.unref?.();
+            }),
+        ]);
+        if (timer)
+            this.clearTimer(timer);
+        if (!drained) {
+            throw new LocalWorkspaceSyncRunnerError('The local workspace sync runner did not drain before its deadline.', 'SYNC_RUNNER_DRAIN_TIMEOUT');
+        }
+        await this.options.drainBarrier?.(this.context, timeoutMs);
+    }
+    async stop() {
+        if (!this.context)
+            return;
+        await this.drain(this.options.defaultDrainGraceMs ?? 15_000);
+        this.terminated = true;
+        this.prepared = false;
+        this.activated = false;
+        this.context = null;
+    }
+    readStatus() {
+        const context = this.context;
+        return {
+            running: Boolean(context) && this.acceptingWork && !this.terminated,
+            prepared: this.prepared,
+            acceptingWork: this.acceptingWork,
+            lifecycleToken: this.lifecycleToken,
+            queuedCommandCount: this.commandExecutions.size,
+            checkpoint: context
+                ? this.options.runnerStateStore.read(context.tenantId, context.workspaceId)
+                : null,
+        };
+    }
+    async executeClaimedCommandWithFailureBoundary(context, command) {
+        try {
+            this.validateCommand(command);
+            return await this.executeClaimedCommand(context, command);
+        }
+        catch (error) {
+            if (isStaleRunnerError(error)) {
+                this.terminateForStaleFence();
+                return this.options.coordinatorStore.finishCommand({
+                    commandId: command.commandId,
+                    status: 'superseded',
+                    errorCode: String(error?.code || 'SYNC_RUNNER_STALE'),
+                    errorMessage: String(error?.message || error),
+                });
+            }
+            return this.completeFailure(context, command, error);
+        }
+    }
+    async drainAcceptedCommands(requestedCommandId) {
+        if (!this.context || !this.acceptingWork || this.terminated) {
+            return this.options.coordinatorStore.readCommand(requestedCommandId);
+        }
+        const context = this.requireStarted();
+        while (this.acceptingWork) {
+            let command;
+            try {
+                command = this.options.coordinatorStore.claimNextRunnerCommand({
+                    workspaceId: context.workspaceId,
+                    coordinatorGeneration: context.coordinatorGeneration,
+                    cloudCapabilityAvailable: this.hasExactCapability(context),
+                });
+            }
+            catch (error) {
+                if (!isStaleRunnerError(error))
+                    throw error;
+                this.terminateForStaleFence();
+                const requested = this.options.coordinatorStore.readCommand(requestedCommandId);
+                if (requested && ['accepted', 'running'].includes(requested.status)) {
+                    this.options.coordinatorStore.finishCommand({
+                        commandId: requestedCommandId,
+                        status: 'superseded',
+                        errorCode: String(error?.code || 'SYNC_RUNNER_STALE'),
+                        errorMessage: String(error?.message || error),
+                    });
+                }
+                break;
+            }
+            if (!command)
+                break;
+            await this.executeClaimedCommandWithFailureBoundary(context, command);
+        }
+        return this.options.coordinatorStore.readCommand(requestedCommandId);
+    }
+    async executeClaimedCommand(context, command) {
+        let state = this.readRunnerState(context);
+        if (command.type === 'EnableSync' || command.type === 'DisableSync') {
+            const enabled = command.type === 'EnableSync';
+            if (!enabled)
+                this.clearRetryTimer();
+            const checkpoint = {
+                ...cloneCheckpoint(state.checkpoint),
+                syncEnabled: enabled,
+                controlState: 'idle',
+                ...(!enabled
+                    ? {
+                        remoteHintObservedAt: null,
+                        consecutivePushesWhilePullDue: 0,
+                    }
+                    : {}),
+                lastError: null,
+            };
+            this.options.runnerStateStore.compareAndSetWithCommandTransition({
+                tenantId: context.tenantId,
+                workspaceId: context.workspaceId,
+                expectedCoordinatorGeneration: context.coordinatorGeneration,
+                expectedCheckpointRevision: state.checkpointRevision,
+                checkpoint,
+                commandId: command.commandId,
+                nextCommandStatus: 'succeeded',
+                result: { syncEnabled: enabled },
+            });
+            const completed = this.options.coordinatorStore.readCommand(command.commandId);
+            if (enabled) {
+                const latest = this.readRunnerState(context);
+                this.schedulePersistedRetry(latest, this.lifecycleToken);
+            }
+            return completed;
+        }
+        if (!state.checkpoint.syncEnabled) {
+            throw new LocalWorkspaceSyncRunnerError(`${command.type} requires workspace sync to be enabled.`, 'SYNC_RUNNER_COMMAND_DISABLED');
+        }
+        const controlState = command.type === 'Repair'
+            ? 'repairing'
+            : command.type === 'Retry'
+                ? 'starting'
+                : 'starting';
+        state = this.options.runnerStateStore.compareAndSet({
+            tenantId: context.tenantId,
+            workspaceId: context.workspaceId,
+            expectedCoordinatorGeneration: context.coordinatorGeneration,
+            expectedCheckpointRevision: state.checkpointRevision,
+            checkpoint: {
+                ...cloneCheckpoint(state.checkpoint),
+                controlState,
+                lastError: null,
+            },
+        });
+        if (!this.hasExactCapability(context)) {
+            return this.deferForAuth(context, command, state);
+        }
+        const operation = command.type === 'Sync'
+            ? this.options.operations.sync
+            : command.type === 'Retry'
+                ? this.options.operations.retry
+                : this.options.operations.repair;
+        const checkpointPort = this.createOperationCheckpointPort(context, state);
+        try {
+            const outcome = await operation({
+                command,
+                coordinatorGeneration: context.coordinatorGeneration,
+                checkpoint: cloneCheckpoint(state.checkpoint),
+                checkpointRevision: state.checkpointRevision,
+                persistCheckpoint: checkpointPort.persist,
+                now: this.now(),
+            });
+            checkpointPort.close();
+            state = checkpointPort.assertOutcome(outcome.checkpoint);
+            const checkpoint = {
+                ...cloneCheckpoint(outcome.checkpoint),
+                controlState: 'idle',
+                lastError: null,
+            };
+            const completedState = this.options.runnerStateStore.compareAndSetWithCommandTransition({
+                tenantId: context.tenantId,
+                workspaceId: context.workspaceId,
+                expectedCoordinatorGeneration: context.coordinatorGeneration,
+                expectedCheckpointRevision: state.checkpointRevision,
+                checkpoint,
+                commandId: command.commandId,
+                nextCommandStatus: 'succeeded',
+                result: outcome.result ?? { outcome: 'completed' },
+            });
+            this.schedulePersistedRetry(completedState, this.lifecycleToken);
+            return this.options.coordinatorStore.readCommand(command.commandId);
+        }
+        catch (error) {
+            checkpointPort.close();
+            const failureState = checkpointPort.readFailureState();
+            if (isStaleRunnerError(error))
+                throw error;
+            if (isAuthError(error)
+                || this.options.credentialBroker.readSnapshot().status === 'blocked-auth') {
+                return this.deferForAuth(context, command, failureState);
+            }
+            return this.completeFailure(context, command, error, failureState);
+        }
+    }
+    async runInspection(operation = this.options.operations.sync, options = {}) {
+        if (!this.context || !this.acceptingWork || this.terminated)
+            return;
+        const context = this.requireStarted();
+        try {
+            this.assertCoordinatorFence(context);
+        }
+        catch (error) {
+            if (isStaleRunnerError(error)) {
+                this.terminateForStaleFence();
+                return;
+            }
+            throw error;
+        }
+        let state = this.readRunnerState(context);
+        if (!state.checkpoint.syncEnabled || !this.hasExactCapability(context))
+            return;
+        if (state.checkpoint.retry
+            && state.checkpoint.retry.kind !== options.bypassRetryKind
+            && Date.parse(state.checkpoint.retry.retryAt) > Date.parse(this.now())) {
+            this.schedulePersistedRetry(state, this.lifecycleToken);
+            return;
+        }
+        state = this.options.runnerStateStore.compareAndSet({
+            tenantId: context.tenantId,
+            workspaceId: context.workspaceId,
+            expectedCoordinatorGeneration: context.coordinatorGeneration,
+            expectedCheckpointRevision: state.checkpointRevision,
+            checkpoint: {
+                ...cloneCheckpoint(state.checkpoint),
+                controlState: 'starting',
+                lastError: null,
+            },
+        });
+        const checkpointPort = this.createOperationCheckpointPort(context, state);
+        try {
+            const outcome = await operation({
+                command: null,
+                coordinatorGeneration: context.coordinatorGeneration,
+                checkpoint: cloneCheckpoint(state.checkpoint),
+                checkpointRevision: state.checkpointRevision,
+                persistCheckpoint: checkpointPort.persist,
+                now: this.now(),
+            });
+            checkpointPort.close();
+            state = checkpointPort.assertOutcome(outcome.checkpoint);
+            const completed = this.options.runnerStateStore.compareAndSet({
+                tenantId: context.tenantId,
+                workspaceId: context.workspaceId,
+                expectedCoordinatorGeneration: context.coordinatorGeneration,
+                expectedCheckpointRevision: state.checkpointRevision,
+                checkpoint: {
+                    ...cloneCheckpoint(outcome.checkpoint),
+                    controlState: 'idle',
+                    lastError: null,
+                },
+            });
+            this.schedulePersistedRetry(completed, this.lifecycleToken);
+        }
+        catch (error) {
+            checkpointPort.close();
+            if (isStaleRunnerError(error)) {
+                this.terminateForStaleFence();
+                return;
+            }
+            try {
+                state = checkpointPort.readFailureState();
+            }
+            catch (stateError) {
+                if (isStaleRunnerError(stateError)) {
+                    this.terminateForStaleFence();
+                    return;
+                }
+                throw stateError;
+            }
+            const authBlocked = isAuthError(error)
+                || this.options.credentialBroker.readSnapshot().status === 'blocked-auth';
+            const retry = state.checkpoint.retry;
+            const nextAttempt = Math.min(Number.MAX_SAFE_INTEGER, Math.max(1, retry?.attempt ?? 1) + 1);
+            const retryDelayMs = Math.min(SCHEDULED_RETRY_MAX_DELAY_MS, SCHEDULED_RETRY_BASE_DELAY_MS
+                * (2 ** Math.min(18, Math.max(0, nextAttempt - 1))));
+            const failed = this.options.runnerStateStore.compareAndSet({
+                tenantId: context.tenantId,
+                workspaceId: context.workspaceId,
+                expectedCoordinatorGeneration: context.coordinatorGeneration,
+                expectedCheckpointRevision: state.checkpointRevision,
+                checkpoint: {
+                    ...cloneCheckpoint(state.checkpoint),
+                    controlState: authBlocked ? 'idle' : 'retry-wait',
+                    retry: authBlocked || !retry
+                        ? retry
+                        : {
+                            ...retry,
+                            attempt: nextAttempt,
+                            retryAt: new Date(Date.parse(this.now()) + retryDelayMs).toISOString(),
+                        },
+                    lastError: String(error?.message || error),
+                },
+            });
+            if (!authBlocked)
+                this.schedulePersistedRetry(failed, this.lifecycleToken);
+        }
+    }
+    deferForAuth(context, command, state) {
+        this.options.runnerStateStore.compareAndSetWithCommandTransition({
+            tenantId: context.tenantId,
+            workspaceId: context.workspaceId,
+            expectedCoordinatorGeneration: context.coordinatorGeneration,
+            expectedCheckpointRevision: state.checkpointRevision,
+            checkpoint: {
+                ...cloneCheckpoint(state.checkpoint),
+                controlState: 'idle',
+                lastError: 'Cloud sync is waiting for an authenticated Taskforce session.',
+            },
+            commandId: command.commandId,
+            nextCommandStatus: 'accepted',
+            errorCode: 'SYNC_COORDINATOR_AUTH_REQUIRED',
+            errorMessage: 'Cloud sync is waiting for an authenticated Taskforce session.',
+        });
+        return this.options.coordinatorStore.readCommand(command.commandId);
+    }
+    completeFailure(context, command, error, expectedState) {
+        const state = expectedState ?? this.readRunnerState(context);
+        this.options.runnerStateStore.compareAndSetWithCommandTransition({
+            tenantId: context.tenantId,
+            workspaceId: context.workspaceId,
+            expectedCoordinatorGeneration: context.coordinatorGeneration,
+            expectedCheckpointRevision: state.checkpointRevision,
+            checkpoint: {
+                ...cloneCheckpoint(state.checkpoint),
+                controlState: 'error',
+                lastError: String(error?.message || error),
+            },
+            commandId: command.commandId,
+            nextCommandStatus: 'failed',
+            errorCode: String(error?.code || 'SYNC_RUNNER_COMMAND_FAILED'),
+            errorMessage: String(error?.message || error),
+        });
+        return this.options.coordinatorStore.readCommand(command.commandId);
+    }
+    schedulePersistedRetry(state, token) {
+        this.clearRetryTimer();
+        if (!this.acceptingWork
+            || this.terminated
+            || !state.checkpoint.syncEnabled
+            || !state.checkpoint.retry
+            || !this.hasExactCapability(this.context))
+            return;
+        const delayMs = Math.max(0, Math.min(2_147_483_647, Date.parse(state.checkpoint.retry.retryAt) - Date.parse(this.now())));
+        this.retryTimer = this.setTimer(() => {
+            this.retryTimer = null;
+            if (token !== this.lifecycleToken || !this.acceptingWork || this.terminated)
+                return;
+            if (this.options.requestScheduledRetryWake && this.context) {
+                try {
+                    this.options.requestScheduledRetryWake({ ...this.context });
+                }
+                catch (error) {
+                    this.options.reportInternalError?.(error, 'scheduled-retry');
+                }
+                return;
+            }
+            this.settleInternal(this.enqueue(async () => {
+                if (token !== this.lifecycleToken
+                    || !this.acceptingWork
+                    || this.terminated
+                    || !this.context)
+                    return;
+                await this.runScheduledRetry(token);
+            }), 'scheduled-retry');
+        }, delayMs);
+        this.retryTimer.unref?.();
+    }
+    scheduleAcceptedCommandWatchdog(token) {
+        this.clearAcceptedCommandWatchdogTimer();
+        if (!this.acceptingWork || this.terminated)
+            return;
+        const configuredDelay = this.options.acceptedCommandWatchdogMs ?? 5_000;
+        const delayMs = Math.max(250, Math.min(60_000, Math.floor(configuredDelay)));
+        this.acceptedCommandWatchdogTimer = this.setTimer(() => {
+            this.acceptedCommandWatchdogTimer = null;
+            if (token !== this.lifecycleToken || !this.acceptingWork || this.terminated)
+                return;
+            const watchdog = this.enqueue(async () => {
+                if (token !== this.lifecycleToken || !this.acceptingWork || this.terminated)
+                    return;
+                await this.drainAcceptedCommandsWithoutRequestedCommand();
+            }).finally(() => {
+                if (token === this.lifecycleToken && this.acceptingWork && !this.terminated) {
+                    this.scheduleAcceptedCommandWatchdog(token);
+                }
+            });
+            this.settleInternal(watchdog, 'command-watchdog');
+        }, delayMs);
+        this.acceptedCommandWatchdogTimer.unref?.();
+    }
+    async drainAcceptedCommandsWithoutRequestedCommand() {
+        if (!this.context || !this.acceptingWork || this.terminated)
+            return;
+        const context = this.requireStarted();
+        while (this.acceptingWork) {
+            let command;
+            try {
+                command = this.options.coordinatorStore.claimNextRunnerCommand({
+                    workspaceId: context.workspaceId,
+                    coordinatorGeneration: context.coordinatorGeneration,
+                    cloudCapabilityAvailable: this.hasExactCapability(context),
+                });
+            }
+            catch (error) {
+                if (!isStaleRunnerError(error))
+                    throw error;
+                this.terminateForStaleFence();
+                break;
+            }
+            if (!command)
+                break;
+            await this.executeClaimedCommandWithFailureBoundary(context, command);
+        }
+    }
+    wakeAcceptedCommands() {
+        if (!this.context || !this.acceptingWork || this.terminated)
+            return;
+        for (const command of this.options.coordinatorStore.listAcceptedRunnerCommands({
+            workspaceId: this.context.workspaceId,
+            coordinatorGeneration: this.context.coordinatorGeneration,
+        })) {
+            this.settleInternal(this.executeCommand(command.commandId), 'activation-wake');
+        }
+    }
+    async runScheduledRetry(token) {
+        if (token !== this.lifecycleToken
+            || !this.context
+            || !this.acceptingWork
+            || this.terminated)
+            return;
+        const context = this.requireStarted();
+        try {
+            this.assertCoordinatorFence(context);
+        }
+        catch (error) {
+            if (isStaleRunnerError(error)) {
+                this.terminateForStaleFence();
+                return;
+            }
+            throw error;
+        }
+        let state = this.readRunnerState(context);
+        if (!state.checkpoint.syncEnabled
+            || !state.checkpoint.retry
+            || Date.parse(state.checkpoint.retry.retryAt) > Date.parse(this.now())
+            || !this.hasExactCapability(context)) {
+            this.schedulePersistedRetry(state, token);
+            return;
+        }
+        state = this.options.runnerStateStore.compareAndSet({
+            tenantId: context.tenantId,
+            workspaceId: context.workspaceId,
+            expectedCoordinatorGeneration: context.coordinatorGeneration,
+            expectedCheckpointRevision: state.checkpointRevision,
+            checkpoint: {
+                ...cloneCheckpoint(state.checkpoint),
+                controlState: 'starting',
+                lastError: null,
+            },
+        });
+        const checkpointPort = this.createOperationCheckpointPort(context, state);
+        try {
+            const outcome = await this.options.operations.retry({
+                command: null,
+                coordinatorGeneration: context.coordinatorGeneration,
+                checkpoint: cloneCheckpoint(state.checkpoint),
+                checkpointRevision: state.checkpointRevision,
+                persistCheckpoint: checkpointPort.persist,
+                now: this.now(),
+            });
+            checkpointPort.close();
+            state = checkpointPort.assertOutcome(outcome.checkpoint);
+            const completed = this.options.runnerStateStore.compareAndSet({
+                tenantId: context.tenantId,
+                workspaceId: context.workspaceId,
+                expectedCoordinatorGeneration: context.coordinatorGeneration,
+                expectedCheckpointRevision: state.checkpointRevision,
+                checkpoint: {
+                    ...cloneCheckpoint(outcome.checkpoint),
+                    controlState: 'idle',
+                    lastError: null,
+                },
+            });
+            this.schedulePersistedRetry(completed, token);
+        }
+        catch (error) {
+            checkpointPort.close();
+            if (isStaleRunnerError(error)) {
+                this.terminateForStaleFence();
+                return;
+            }
+            try {
+                state = checkpointPort.readFailureState();
+            }
+            catch (stateError) {
+                if (isStaleRunnerError(stateError)) {
+                    this.terminateForStaleFence();
+                    return;
+                }
+                throw stateError;
+            }
+            const authBlocked = isAuthError(error)
+                || this.options.credentialBroker.readSnapshot().status === 'blocked-auth';
+            const retry = state.checkpoint.retry;
+            const nextAttempt = Math.min(Number.MAX_SAFE_INTEGER, Math.max(1, retry?.attempt ?? 1) + 1);
+            const retryDelayMs = Math.min(SCHEDULED_RETRY_MAX_DELAY_MS, SCHEDULED_RETRY_BASE_DELAY_MS
+                * (2 ** Math.min(18, Math.max(0, nextAttempt - 1))));
+            const failed = this.options.runnerStateStore.compareAndSet({
+                tenantId: context.tenantId,
+                workspaceId: context.workspaceId,
+                expectedCoordinatorGeneration: context.coordinatorGeneration,
+                expectedCheckpointRevision: state.checkpointRevision,
+                checkpoint: {
+                    ...cloneCheckpoint(state.checkpoint),
+                    controlState: authBlocked ? 'idle' : 'retry-wait',
+                    retry: authBlocked || !retry
+                        ? retry
+                        : {
+                            ...retry,
+                            attempt: nextAttempt,
+                            retryAt: new Date(Date.parse(this.now()) + retryDelayMs).toISOString(),
+                        },
+                    lastError: String(error?.message || error),
+                },
+            });
+            if (!authBlocked)
+                this.schedulePersistedRetry(failed, token);
+        }
+    }
+    createOperationCheckpointPort(context, initialState) {
+        let open = true;
+        let expectedRevision = initialState.checkpointRevision;
+        let persisted = false;
+        return {
+            persist: (checkpoint, options) => {
+                if (!open) {
+                    throw new LocalWorkspaceSyncRunnerError('The runner operation checkpoint port is closed.', 'SYNC_RUNNER_COMMAND_INVALID');
+                }
+                this.assertCoordinatorFence(context);
+                const state = this.options.runnerStateStore.compareAndSet({
+                    tenantId: context.tenantId,
+                    workspaceId: context.workspaceId,
+                    expectedCoordinatorGeneration: context.coordinatorGeneration,
+                    expectedCheckpointRevision: expectedRevision,
+                    checkpoint: cloneCheckpoint(checkpoint),
+                    beforeCommit: options?.beforeCommit,
+                });
+                expectedRevision = state.checkpointRevision;
+                persisted = true;
+                return {
+                    checkpoint: cloneCheckpoint(state.checkpoint),
+                    checkpointRevision: state.checkpointRevision,
+                };
+            },
+            assertOutcome: (checkpoint) => {
+                const current = this.readRunnerState(context);
+                if (current.checkpointRevision !== expectedRevision
+                    || (persisted
+                        && canonicalWorkspaceSyncJson(current.checkpoint)
+                            !== canonicalWorkspaceSyncJson(checkpoint))) {
+                    throw new LocalWorkspaceSyncRunnerStateError('The runner operation result does not match its persisted checkpoint.', 'SYNC_RUNNER_STATE_STALE_REVISION');
+                }
+                return current;
+            },
+            readFailureState: () => {
+                const current = this.readRunnerState(context);
+                if (current.checkpointRevision > expectedRevision) {
+                    throw new LocalWorkspaceSyncRunnerStateError('The runner checkpoint advanced outside the active operation.', 'SYNC_RUNNER_STATE_STALE_REVISION');
+                }
+                // A lower revision is the expected result when persistence
+                // occurred inside an apply transaction that later rolled
+                // back. The checkpoint port is closed before this read, so
+                // failure handling resumes from the durable revision rather
+                // than the transaction-local revision.
+                return current;
+            },
+            close: () => {
+                open = false;
+            },
+        };
+    }
+    hasExactCapability(context) {
+        try {
+            this.options.credentialBroker.requireCapability({
+                workspaceId: context.workspaceId,
+                coordinatorGeneration: context.coordinatorGeneration,
+            });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    validateCommand(command) {
+        if (!RUNNER_COMMANDS.has(command.type)
+            || command.version !== 1
+            || !hasEmptyArgs(command.args)) {
+            throw new LocalWorkspaceSyncRunnerError(`Runner command ${command.commandId} has an unsupported type, version, or argument shape.`, 'SYNC_RUNNER_COMMAND_INVALID');
+        }
+        const context = this.requireStarted();
+        if (command.workspaceId !== context.workspaceId
+            || command.expectedCoordinatorGeneration !== context.coordinatorGeneration
+            || command.requestedBy.tenantId !== context.tenantId) {
+            throw new LocalWorkspaceSyncRunnerError(`Runner command ${command.commandId} belongs to a stale coordinator scope.`, 'SYNC_RUNNER_COORDINATOR_FENCE_INVALID');
+        }
+    }
+    assertCoordinatorFence(context) {
+        const coordinator = this.options.coordinatorStore.readOrCreateState();
+        if (coordinator.activeWorkspaceId !== context.workspaceId
+            || coordinator.coordinatorGeneration !== context.coordinatorGeneration
+            || coordinator.ownershipMode !== 'server'
+            || coordinator.transitionState !== 'idle') {
+            throw new LocalWorkspaceSyncRunnerError('The server runner does not own the requested coordinator scope.', 'SYNC_RUNNER_COORDINATOR_FENCE_INVALID');
+        }
+    }
+    assertPreparationFence(context) {
+        const coordinator = this.options.coordinatorStore.readOrCreateState();
+        const isActiveServer = coordinator.ownershipMode === 'server'
+            && coordinator.transitionState === 'idle';
+        const isStartingTransferTarget = coordinator.ownershipMode === 'transitioning-to-server'
+            && coordinator.transitionState === 'transferring-ownership'
+            && coordinator.transferPhase === 'starting-target';
+        const isRestartingDesiredServer = coordinator.ownershipMode === 'stopped'
+            && coordinator.desiredOwnershipMode === 'server'
+            && coordinator.transitionState === 'idle';
+        if (coordinator.activeWorkspaceId !== context.workspaceId
+            || coordinator.coordinatorGeneration !== context.coordinatorGeneration
+            || (!isActiveServer && !isStartingTransferTarget && !isRestartingDesiredServer)) {
+            throw new LocalWorkspaceSyncRunnerError('The server runner cannot be prepared for the requested coordinator scope.', 'SYNC_RUNNER_COORDINATOR_FENCE_INVALID');
+        }
+    }
+    readRunnerState(context) {
+        const state = this.options.runnerStateStore.read(context.tenantId, context.workspaceId);
+        if (!state) {
+            throw new LocalWorkspaceSyncRunnerStateError('The local workspace sync runner state is missing.', 'SYNC_RUNNER_STATE_INVALID');
+        }
+        if (state.coordinatorGeneration !== context.coordinatorGeneration) {
+            throw new LocalWorkspaceSyncRunnerStateError('The local workspace sync runner state belongs to a stale generation.', 'SYNC_RUNNER_STATE_STALE_GENERATION');
+        }
+        return state;
+    }
+    enqueue(operation) {
+        const result = this.queueTail.then(operation, operation);
+        this.queueTail = result.then(() => undefined, () => undefined);
+        return result;
+    }
+    settleInternal(promise, source) {
+        void promise.catch((error) => {
+            try {
+                this.options.reportInternalError?.(error, source);
+            }
+            catch {
+                // Diagnostics must never turn a contained runner failure into an
+                // unhandled rejection.
+            }
+        });
+    }
+    terminateForStaleFence() {
+        this.acceptingWork = false;
+        this.terminated = true;
+        this.prepared = false;
+        this.activated = false;
+        ++this.lifecycleToken;
+        this.clearRunnerTimers();
+    }
+    requireStarted() {
+        if (!this.context || this.terminated || !this.acceptingWork) {
+            throw new LocalWorkspaceSyncRunnerError('The local workspace sync runner is not started.', 'SYNC_RUNNER_NOT_STARTED');
+        }
+        return this.context;
+    }
+    requirePrepared() {
+        if (!this.context || this.terminated || !this.prepared) {
+            throw new LocalWorkspaceSyncRunnerError('The local workspace sync runner is not prepared.', 'SYNC_RUNNER_NOT_STARTED');
+        }
+        return this.context;
+    }
+    resetAfterPreparationFailure() {
+        this.acceptingWork = false;
+        this.prepared = false;
+        this.activated = false;
+        this.terminated = true;
+        this.context = null;
+        ++this.lifecycleToken;
+        this.clearRunnerTimers();
+    }
+    now() {
+        return this.options.now?.() || new Date().toISOString();
+    }
+    setTimer(callback, delayMs) {
+        return this.options.setTimer?.(callback, delayMs) || setTimeout(callback, delayMs);
+    }
+    clearTimer(timer) {
+        if (this.options.clearTimer)
+            this.options.clearTimer(timer);
+        else
+            clearTimeout(timer);
+    }
+    clearRetryTimer() {
+        if (!this.retryTimer)
+            return;
+        this.clearTimer(this.retryTimer);
+        this.retryTimer = null;
+    }
+    clearAcceptedCommandWatchdogTimer() {
+        if (!this.acceptedCommandWatchdogTimer)
+            return;
+        this.clearTimer(this.acceptedCommandWatchdogTimer);
+        this.acceptedCommandWatchdogTimer = null;
+    }
+    clearRunnerTimers() {
+        this.clearRetryTimer();
+        this.clearAcceptedCommandWatchdogTimer();
+    }
+}