@powerhousedao/reactor 5.2.0-staging.9 → 5.3.0-staging.1

package/dist/src/executor/simple-job-executor.js

@@ -1,9 +1,17 @@
+import { deriveOperationId, isUndoRedo } from "document-model/core";
 import { driveCollectionId } from "../cache/operation-index-types.js";
 import { OperationEventTypes, } from "../events/types.js";
 import { DocumentDeletedError, InvalidSignatureError, } from "../shared/errors.js";
 import { reshuffleByTimestampAndIndex } from "../utils/reshuffle.js";
 import { applyDeleteDocumentAction, applyUpgradeDocumentAction, createDocumentFromAction, getNextIndexForScope, } from "./util.js";
 const MAX_SKIP_THRESHOLD = 100;
+const documentScopeActions = [
+    "CREATE_DOCUMENT",
+    "DELETE_DOCUMENT",
+    "UPGRADE_DOCUMENT",
+    "ADD_RELATIONSHIP",
+    "REMOVE_RELATIONSHIP",
+];
 /**
  * Simple job executor that processes a job by applying actions through document model reducers.
  *
@@ -12,6 +20,7 @@ const MAX_SKIP_THRESHOLD = 100;
  * @see docs/planning/Jobs/reshuffle.md for skip mechanism details
  */
 export class SimpleJobExecutor {
+    logger;
     registry;
     documentStorage;
     operationStorage;
@@ -22,7 +31,8 @@ export class SimpleJobExecutor {
     documentMetaCache;
     signatureVerifier;
     config;
-    constructor(registry, documentStorage, operationStorage, operationStore, eventBus, writeCache, operationIndex, documentMetaCache, config, signatureVerifier) {
+    constructor(logger, registry, documentStorage, operationStorage, operationStore, eventBus, writeCache, operationIndex, documentMetaCache, config, signatureVerifier) {
+        this.logger = logger;
         this.registry = registry;
         this.documentStorage = documentStorage;
         this.operationStorage = operationStorage;
@@ -33,6 +43,7 @@ export class SimpleJobExecutor {
         this.documentMetaCache = documentMetaCache;
         this.signatureVerifier = signatureVerifier;
         this.config = {
+            maxSkipThreshold: config.maxSkipThreshold ?? MAX_SKIP_THRESHOLD,
             maxConcurrency: config.maxConcurrency ?? 1,
             jobTimeoutMs: config.jobTimeoutMs ?? 30000,
             retryBaseDelayMs: config.retryBaseDelayMs ?? 100,
@@ -58,6 +69,7 @@ export class SimpleJobExecutor {
                     const event = {
                         jobId: job.id,
                         operations: result.operationsWithContext,
+                        jobMeta: job.meta,
                     };
                     this.eventBus
                         .emit(OperationEventTypes.OPERATION_WRITTEN, event)
@@ -85,6 +97,7 @@ export class SimpleJobExecutor {
             const event = {
                 jobId: job.id,
                 operations: result.operationsWithContext,
+                jobMeta: job.meta,
             };
             this.eventBus
                 .emit(OperationEventTypes.OPERATION_WRITTEN, event)
@@ -100,7 +113,7 @@ export class SimpleJobExecutor {
             duration: Date.now() - startTime,
         };
     }
-    async processActions(job, actions, startTime, indexTxn, skipValues) {
+    async processActions(job, actions, startTime, indexTxn, skipValues, sourceOperations) {
         const generatedOperations = [];
         const operationsWithContext = [];
         try {
@@ -114,207 +127,23 @@ export class SimpleJobExecutor {
                 error: error instanceof Error ? error : new Error(String(error)),
             };
         }
-        let actionIndex = 0;
-        for (const action of actions) {
-            if (action.type === "CREATE_DOCUMENT") {
-                const result = await this.executeCreateDocumentAction(job, action, startTime, indexTxn, skipValues?.[actionIndex]);
-                const error = this.accumulateResultOrReturnError(result, generatedOperations, operationsWithContext);
-                if (error !== null) {
-                    return {
-                        success: false,
-                        generatedOperations,
-                        operationsWithContext,
-                        error: error.error,
-                    };
-                }
-                actionIndex++;
-                continue;
-            }
-            if (action.type === "DELETE_DOCUMENT") {
-                const result = await this.executeDeleteDocumentAction(job, action, startTime, indexTxn);
-                const error = this.accumulateResultOrReturnError(result, generatedOperations, operationsWithContext);
-                if (error !== null) {
-                    return {
-                        success: false,
-                        generatedOperations,
-                        operationsWithContext,
-                        error: error.error,
-                    };
-                }
-                actionIndex++;
-                continue;
-            }
-            if (action.type === "UPGRADE_DOCUMENT") {
-                const result = await this.executeUpgradeDocumentAction(job, action, startTime, indexTxn, skipValues?.[actionIndex]);
-                const error = this.accumulateResultOrReturnError(result, generatedOperations, operationsWithContext);
-                if (error !== null) {
-                    return {
-                        success: false,
-                        generatedOperations,
-                        operationsWithContext,
-                        error: error.error,
-                    };
-                }
-                actionIndex++;
-                continue;
-            }
-            if (action.type === "ADD_RELATIONSHIP") {
-                const result = await this.executeAddRelationshipAction(job, action, startTime, indexTxn);
-                const error = this.accumulateResultOrReturnError(result, generatedOperations, operationsWithContext);
-                if (error !== null) {
-                    return {
-                        success: false,
-                        generatedOperations,
-                        operationsWithContext,
-                        error: error.error,
-                    };
-                }
-                actionIndex++;
-                continue;
-            }
-            if (action.type === "REMOVE_RELATIONSHIP") {
-                const result = await this.executeRemoveRelationshipAction(job, action, startTime, indexTxn);
-                const error = this.accumulateResultOrReturnError(result, generatedOperations, operationsWithContext);
-                if (error !== null) {
-                    return {
-                        success: false,
-                        generatedOperations,
-                        operationsWithContext,
-                        error: error.error,
-                    };
-                }
-                actionIndex++;
-                continue;
-            }
-            let docMeta;
-            try {
-                docMeta = await this.documentMetaCache.getDocumentMeta(job.documentId, job.branch);
-            }
-            catch (error) {
+        for (let actionIndex = 0; actionIndex < actions.length; actionIndex++) {
+            const action = actions[actionIndex];
+            const skip = skipValues?.[actionIndex] ?? 0;
+            const sourceOperation = sourceOperations?.[actionIndex];
+            const isDocumentAction = documentScopeActions.includes(action.type);
+            const result = isDocumentAction
+                ? await this.executeDocumentAction(job, action, startTime, indexTxn, skip)
+                : await this.executeRegularAction(job, action, startTime, indexTxn, skip, sourceOperation);
+            const error = this.accumulateResultOrReturnError(result, generatedOperations, operationsWithContext);
+            if (error !== null) {
                 return {
                     success: false,
                     generatedOperations,
                     operationsWithContext,
-                    error: error instanceof Error ? error : new Error(String(error)),
+                    error: error.error,
                 };
             }
-            if (docMeta.state.isDeleted) {
-                return {
-                    success: false,
-                    generatedOperations,
-                    operationsWithContext,
-                    error: new DocumentDeletedError(job.documentId, docMeta.state.deletedAtUtcIso),
-                };
-            }
-            let document;
-            try {
-                document = await this.writeCache.getState(job.documentId, job.scope, job.branch);
-            }
-            catch (error) {
-                return {
-                    success: false,
-                    generatedOperations,
-                    operationsWithContext,
-                    error: error instanceof Error ? error : new Error(String(error)),
-                };
-            }
-            let module;
-            try {
-                module = this.registry.getModule(document.header.documentType);
-            }
-            catch (error) {
-                return {
-                    success: false,
-                    generatedOperations,
-                    operationsWithContext,
-                    error: error instanceof Error ? error : new Error(String(error)),
-                };
-            }
-            let updatedDocument;
-            try {
-                updatedDocument = module.reducer(document, action);
-            }
-            catch (error) {
-                const contextMessage = `Failed to apply action to document:\n  Action type: ${action.type}\n  Document ID: ${job.documentId}\n  Document type: ${document.header.documentType}\n  Scope: ${job.scope}\n  Original error: ${error instanceof Error ? error.message : String(error)}`;
-                const enhancedError = new Error(contextMessage);
-                if (error instanceof Error && error.stack) {
-                    enhancedError.stack = `${contextMessage}\n\nOriginal stack trace:\n${error.stack}`;
-                }
-                return {
-                    success: false,
-                    generatedOperations,
-                    operationsWithContext,
-                    error: enhancedError,
-                };
-            }
-            const scope = job.scope;
-            const operations = updatedDocument.operations[scope];
-            if (operations.length === 0) {
-                return {
-                    success: false,
-                    generatedOperations,
-                    operationsWithContext,
-                    error: new Error("No operation generated from action"),
-                };
-            }
-            const newOperation = operations[operations.length - 1];
-            if (skipValues && actionIndex < skipValues.length) {
-                newOperation.skip = skipValues[actionIndex];
-            }
-            generatedOperations.push(newOperation);
-            if (this.config.legacyStorageEnabled) {
-                try {
-                    await this.operationStorage.addDocumentOperations(job.documentId, [newOperation], updatedDocument);
-                }
-                catch (error) {
-                    return {
-                        success: false,
-                        generatedOperations,
-                        operationsWithContext,
-                        error: error instanceof Error ? error : new Error(String(error)),
-                    };
-                }
-            }
-            const resultingState = JSON.stringify(updatedDocument.state);
-            try {
-                await this.operationStore.apply(job.documentId, document.header.documentType, scope, job.branch, newOperation.index, (txn) => {
-                    txn.addOperations(newOperation);
-                });
-            }
-            catch (error) {
-                return {
-                    success: false,
-                    generatedOperations,
-                    operationsWithContext,
-                    error: new Error(`Failed to write operation to IOperationStore: ${error instanceof Error ? error.message : String(error)}`),
-                };
-            }
-            updatedDocument.header.revision = {
-                ...updatedDocument.header.revision,
-                [scope]: newOperation.index + 1,
-            };
-            this.writeCache.putState(job.documentId, scope, job.branch, newOperation.index, updatedDocument);
-            indexTxn.write([
-                {
-                    ...newOperation,
-                    documentId: job.documentId,
-                    documentType: document.header.documentType,
-                    branch: job.branch,
-                    scope,
-                },
-            ]);
-            operationsWithContext.push({
-                operation: newOperation,
-                context: {
-                    documentId: job.documentId,
-                    scope,
-                    branch: job.branch,
-                    documentType: document.header.documentType,
-                    resultingState,
-                    ordinal: 0,
-                },
-            });
-            actionIndex++;
         }
         return {
             success: true,
@@ -322,6 +151,26 @@ export class SimpleJobExecutor {
             operationsWithContext,
         };
     }
+    /**
+     * Execute a document scope action (CREATE_DOCUMENT, DELETE_DOCUMENT, UPGRADE_DOCUMENT,
+     * ADD_RELATIONSHIP, REMOVE_RELATIONSHIP) by dispatching to the appropriate handler.
+     */
+    async executeDocumentAction(job, action, startTime, indexTxn, skip = 0) {
+        switch (action.type) {
+            case "CREATE_DOCUMENT":
+                return this.executeCreateDocumentAction(job, action, startTime, indexTxn, skip);
+            case "DELETE_DOCUMENT":
+                return this.executeDeleteDocumentAction(job, action, startTime, indexTxn);
+            case "UPGRADE_DOCUMENT":
+                return this.executeUpgradeDocumentAction(job, action, startTime, indexTxn, skip);
+            case "ADD_RELATIONSHIP":
+                return this.executeAddRelationshipAction(job, action, startTime, indexTxn);
+            case "REMOVE_RELATIONSHIP":
+                return this.executeRemoveRelationshipAction(job, action, startTime, indexTxn);
+            default:
+                return this.buildErrorResult(job, new Error(`Unknown document action type: ${action.type}`), startTime);
+        }
+    }
     /**
      * Execute a CREATE_DOCUMENT system action.
      * This creates a new document in storage along with its initial operation.
@@ -346,7 +195,11 @@ export class SimpleJobExecutor {
                 return this.buildErrorResult(job, new Error(`Failed to create document in storage: ${error instanceof Error ? error.message : String(error)}`), startTime);
             }
         }
-        const operation = this.createOperation(action, 0, skip);
+        const operation = this.createOperation(action, 0, skip, {
+            documentId: document.header.id,
+            scope: job.scope,
+            branch: job.branch,
+        });
         // Legacy: Write the CREATE_DOCUMENT operation to legacy storage
         if (this.config.legacyStorageEnabled) {
             try {
@@ -417,7 +270,11 @@ export class SimpleJobExecutor {
             return this.buildErrorResult(job, new DocumentDeletedError(documentId, documentState.deletedAtUtcIso), startTime);
         }
         const nextIndex = getNextIndexForScope(document, job.scope);
-        const operation = this.createOperation(action, nextIndex);
+        const operation = this.createOperation(action, nextIndex, 0, {
+            documentId,
+            scope: job.scope,
+            branch: job.branch,
+        });
         if (this.config.legacyStorageEnabled) {
             try {
                 await this.documentStorage.delete(documentId);
@@ -457,7 +314,7 @@ export class SimpleJobExecutor {
     }
     /**
      * Execute an UPGRADE_DOCUMENT system action.
-     * This sets the document's initial state from the upgrade action.
+     * Handles initial upgrades (version 0 to N), same-version no-ops, and multi-step upgrade chains.
      * The operation index is determined from the document's current operation count.
      */
     async executeUpgradeDocumentAction(job, action, startTime, indexTxn, skip = 0) {
@@ -466,6 +323,8 @@ export class SimpleJobExecutor {
             return this.buildErrorResult(job, new Error("UPGRADE_DOCUMENT action requires a documentId in input"), startTime);
         }
         const documentId = input.documentId;
+        const fromVersion = input.fromVersion;
+        const toVersion = input.toVersion;
         let document;
         try {
             document = await this.writeCache.getState(documentId, job.scope, job.branch);
@@ -478,8 +337,35 @@ export class SimpleJobExecutor {
             return this.buildErrorResult(job, new DocumentDeletedError(documentId, documentState.deletedAtUtcIso), startTime);
         }
         const nextIndex = getNextIndexForScope(document, job.scope);
-        applyUpgradeDocumentAction(document, action);
-        const operation = this.createOperation(action, nextIndex, skip);
+        let upgradePath;
+        if (fromVersion > 0 && fromVersion < toVersion) {
+            try {
+                upgradePath = this.registry.computeUpgradePath(document.header.documentType, fromVersion, toVersion);
+            }
+            catch (error) {
+                return this.buildErrorResult(job, error instanceof Error ? error : new Error(String(error)), startTime);
+            }
+        }
+        if (fromVersion === toVersion && fromVersion > 0) {
+            return {
+                job,
+                success: true,
+                operations: [],
+                operationsWithContext: [],
+                duration: Date.now() - startTime,
+            };
+        }
+        try {
+            document = applyUpgradeDocumentAction(document, action, upgradePath);
+        }
+        catch (error) {
+            return this.buildErrorResult(job, error instanceof Error ? error : new Error(String(error)), startTime);
+        }
+        const operation = this.createOperation(action, nextIndex, skip, {
+            documentId,
+            scope: job.scope,
+            branch: job.branch,
+        });
         // Write the updated document to legacy storage
         if (this.config.legacyStorageEnabled) {
             try {
@@ -535,19 +421,12 @@ export class SimpleJobExecutor {
         catch (error) {
             return this.buildErrorResult(job, new Error(`ADD_RELATIONSHIP: source document ${input.sourceId} not found: ${error instanceof Error ? error.message : String(error)}`), startTime);
         }
-        let targetDoc;
-        try {
-            targetDoc = await this.writeCache.getState(input.targetId, "document", job.branch);
-        }
-        catch (error) {
-            return this.buildErrorResult(job, new Error(`ADD_RELATIONSHIP: target document ${input.targetId} not found: ${error instanceof Error ? error.message : String(error)}`), startTime);
-        }
-        const targetDocState = targetDoc.state.document;
-        if (targetDocState.isDeleted) {
-            return this.buildErrorResult(job, new Error(`ADD_RELATIONSHIP: target document ${input.targetId} is deleted`), startTime);
-        }
         const nextIndex = getNextIndexForScope(sourceDoc, job.scope);
-        const operation = this.createOperation(action, nextIndex);
+        const operation = this.createOperation(action, nextIndex, 0, {
+            documentId: input.sourceId,
+            scope: job.scope,
+            branch: job.branch,
+        });
         const writeError = await this.writeOperationToStore(input.sourceId, sourceDoc.header.documentType, job.scope, job.branch, operation, job, startTime);
         if (writeError !== null) {
             return writeError;
@@ -581,6 +460,11 @@ export class SimpleJobExecutor {
             const collectionId = driveCollectionId(job.branch, input.sourceId);
             indexTxn.addToCollection(collectionId, input.targetId);
         }
+        this.documentMetaCache.putDocumentMeta(input.sourceId, job.branch, {
+            state: sourceDoc.state.document,
+            documentType: sourceDoc.header.documentType,
+            documentScopeRevision: operation.index + 1,
+        });
         return this.buildSuccessResult(job, operation, input.sourceId, sourceDoc.header.documentType, resultingState, startTime);
     }
     async executeRemoveRelationshipAction(job, action, startTime, indexTxn) {
@@ -599,7 +483,11 @@ export class SimpleJobExecutor {
             return this.buildErrorResult(job, new Error(`REMOVE_RELATIONSHIP: source document ${input.sourceId} not found: ${error instanceof Error ? error.message : String(error)}`), startTime);
         }
         const nextIndex = getNextIndexForScope(sourceDoc, job.scope);
-        const operation = this.createOperation(action, nextIndex);
+        const operation = this.createOperation(action, nextIndex, 0, {
+            documentId: input.sourceId,
+            scope: job.scope,
+            branch: job.branch,
+        });
         const writeError = await this.writeOperationToStore(input.sourceId, sourceDoc.header.documentType, job.scope, job.branch, operation, job, startTime);
         if (writeError !== null) {
             return writeError;
@@ -633,10 +521,127 @@ export class SimpleJobExecutor {
             const collectionId = driveCollectionId(job.branch, input.sourceId);
             indexTxn.removeFromCollection(collectionId, input.targetId);
         }
+        this.documentMetaCache.putDocumentMeta(input.sourceId, job.branch, {
+            state: sourceDoc.state.document,
+            documentType: sourceDoc.header.documentType,
+            documentScopeRevision: operation.index + 1,
+        });
         return this.buildSuccessResult(job, operation, input.sourceId, sourceDoc.header.documentType, resultingState, startTime);
     }
-    createOperation(action, index, skip = 0) {
+    /**
+     * Execute a regular document action by applying it through the document model reducer.
+     * If sourceOperation is provided (for load jobs), its id and timestamp are preserved.
+     */
+    async executeRegularAction(job, action, startTime, indexTxn, skip = 0, sourceOperation) {
+        let docMeta;
+        try {
+            docMeta = await this.documentMetaCache.getDocumentMeta(job.documentId, job.branch);
+        }
+        catch (error) {
+            return this.buildErrorResult(job, error instanceof Error ? error : new Error(String(error)), startTime);
+        }
+        if (docMeta.state.isDeleted) {
+            return this.buildErrorResult(job, new DocumentDeletedError(job.documentId, docMeta.state.deletedAtUtcIso), startTime);
+        }
+        let document;
+        try {
+            document = await this.writeCache.getState(job.documentId, job.scope, job.branch);
+        }
+        catch (error) {
+            return this.buildErrorResult(job, error instanceof Error ? error : new Error(String(error)), startTime);
+        }
+        let module;
+        try {
+            // Use document version to get the correct module
+            // Version 0 means not yet upgraded - use latest version
+            const moduleVersion = docMeta.state.version === 0 ? undefined : docMeta.state.version;
+            module = this.registry.getModule(document.header.documentType, moduleVersion);
+        }
+        catch (error) {
+            return this.buildErrorResult(job, error instanceof Error ? error : new Error(String(error)), startTime);
+        }
+        let updatedDocument;
+        try {
+            const reducerOptions = sourceOperation
+                ? {
+                    skip,
+                    branch: job.branch,
+                    replayOptions: { operation: sourceOperation },
+                }
+                : { skip, branch: job.branch };
+            updatedDocument = module.reducer(document, action, undefined, reducerOptions);
+        }
+        catch (error) {
+            const contextMessage = `Failed to apply action to document:\n  Action type: ${action.type}\n  Document ID: ${job.documentId}\n  Document type: ${document.header.documentType}\n  Scope: ${job.scope}\n  Original error: ${error instanceof Error ? error.message : String(error)}`;
+            const enhancedError = new Error(contextMessage);
+            if (error instanceof Error && error.stack) {
+                enhancedError.stack = `${contextMessage}\n\nOriginal stack trace:\n${error.stack}`;
+            }
+            return this.buildErrorResult(job, enhancedError, startTime);
+        }
+        const scope = job.scope;
+        const operations = updatedDocument.operations[scope];
+        if (operations.length === 0) {
+            return this.buildErrorResult(job, new Error("No operation generated from action"), startTime);
+        }
+        const newOperation = operations[operations.length - 1];
+        if (!isUndoRedo(action)) {
+            newOperation.skip = skip;
+        }
+        if (this.config.legacyStorageEnabled) {
+            try {
+                await this.operationStorage.addDocumentOperations(job.documentId, [newOperation], updatedDocument);
+            }
+            catch (error) {
+                return this.buildErrorResult(job, error instanceof Error ? error : new Error(String(error)), startTime);
+            }
+        }
+        const resultingState = JSON.stringify({
+            ...updatedDocument.state,
+            header: updatedDocument.header,
+        });
+        const writeFailResult = await this.writeOperationToStore(job.documentId, document.header.documentType, scope, job.branch, newOperation, job, startTime);
+        if (writeFailResult !== null) {
+            return writeFailResult;
+        }
+        updatedDocument.header.revision = {
+            ...updatedDocument.header.revision,
+            [scope]: newOperation.index + 1,
+        };
+        this.writeCache.putState(job.documentId, scope, job.branch, newOperation.index, updatedDocument);
+        indexTxn.write([
+            {
+                ...newOperation,
+                documentId: job.documentId,
+                documentType: document.header.documentType,
+                branch: job.branch,
+                scope,
+            },
+        ]);
+        return {
+            job,
+            success: true,
+            operations: [newOperation],
+            operationsWithContext: [
+                {
+                    operation: newOperation,
+                    context: {
+                        documentId: job.documentId,
+                        scope,
+                        branch: job.branch,
+                        documentType: document.header.documentType,
+                        resultingState,
+                        ordinal: 0,
+                    },
+                },
+            ],
+            duration: Date.now() - startTime,
+        };
+    }
+    createOperation(action, index, skip = 0, context) {
+        const id = deriveOperationId(context.documentId, context.scope, context.branch, action.id);
         return {
+            id,
             index: index,
             timestampUtcMs: action.timestampUtcMs || new Date().toISOString(),
             hash: "",
@@ -657,29 +662,81 @@ export class SimpleJobExecutor {
         catch {
             latestRevision = 0;
         }
-        const minIncomingIndex = job.operations.reduce((min, operation) => Math.min(min, operation.index), Number.POSITIVE_INFINITY);
-        const skipCount = minIncomingIndex === Number.POSITIVE_INFINITY
-            ? 0
-            : Math.max(0, latestRevision - minIncomingIndex);
-        if (skipCount > MAX_SKIP_THRESHOLD) {
+        let minIncomingIndex = Number.POSITIVE_INFINITY;
+        let minIncomingTimestamp = job.operations[0]?.timestampUtcMs || "";
+        for (const operation of job.operations) {
+            minIncomingIndex = Math.min(minIncomingIndex, operation.index);
+            const ts = operation.timestampUtcMs || "";
+            if (ts < minIncomingTimestamp) {
+                minIncomingTimestamp = ts;
+            }
+        }
+        let conflictingOps = [];
+        try {
+            const conflictingResult = await this.operationStore.getConflicting(job.documentId, scope, job.branch, minIncomingTimestamp, { limit: this.config.maxSkipThreshold + 1 });
+            if (conflictingResult.hasMore) {
+                return {
+                    job,
+                    success: false,
+                    error: new Error(`Excessive reshuffle detected: more than ${this.config.maxSkipThreshold} conflicting operations found. ` +
+                        `This indicates a significant divergence between local and incoming operations.`),
+                    duration: Date.now() - startTime,
+                };
+            }
+            conflictingOps = conflictingResult.items;
+        }
+        catch {
+            conflictingOps = [];
+        }
+        // Filter out operations that have been superseded by later operations with skip values.
+        // An operation at index N is superseded if there exists an operation at index M > N
+        // where (M - skip_M) <= N, meaning the later operation's logical index covers N.
+        const nonSupersededOps = conflictingOps.filter((op) => {
+            for (const laterOp of conflictingOps) {
+                if (laterOp.index > op.index && laterOp.skip > 0) {
+                    const logicalIndex = laterOp.index - laterOp.skip;
+                    if (logicalIndex <= op.index) {
+                        return false;
+                    }
+                }
+            }
+            return true;
+        });
+        // All non-superseded conflicting operations need to be reshuffled
+        const existingOpsToReshuffle = nonSupersededOps;
+        // Skip count is the number of existing operations that need to be rewound
+        const skipCount = existingOpsToReshuffle.length;
+        if (skipCount > this.config.maxSkipThreshold) {
             return {
                 job,
                 success: false,
-                error: new Error(`Excessive reshuffle detected: skip count of ${skipCount} exceeds threshold of ${MAX_SKIP_THRESHOLD}. ` +
-                    `This indicates an attempt to insert an operation at index ${minIncomingIndex} when the latest revision is ${latestRevision}.`),
+                error: new Error(`Excessive reshuffle detected: skip count of ${skipCount} exceeds threshold of ${this.config.maxSkipThreshold}. ` +
+                    `This indicates a significant divergence between local and incoming operations.`),
                 duration: Date.now() - startTime,
             };
         }
+        // Filter out incoming operations that are duplicates (action already exists locally
+        // or appears multiple times in incoming)
+        const existingActionIds = new Set(nonSupersededOps.map((op) => op.action.id));
+        const seenIncomingActionIds = new Set();
+        const incomingOpsToApply = job.operations.filter((op) => {
+            if (existingActionIds.has(op.action.id))
+                return false;
+            if (seenIncomingActionIds.has(op.action.id))
+                return false;
+            seenIncomingActionIds.add(op.action.id);
+            return true;
+        });
         const reshuffledOperations = reshuffleByTimestampAndIndex({
             index: latestRevision,
             skip: skipCount,
-        }, [], job.operations.map((operation) => ({
+        }, existingOpsToReshuffle, incomingOpsToApply.map((operation) => ({
             ...operation,
             id: operation.id,
         })));
         const actions = reshuffledOperations.map((operation) => operation.action);
         const skipValues = reshuffledOperations.map((operation) => operation.skip);
-        const result = await this.processActions(job, actions, startTime, indexTxn, skipValues);
+        const result = await this.processActions(job, actions, startTime, indexTxn, skipValues, reshuffledOperations);
         if (!result.success) {
             return {
                 job,
@@ -708,6 +765,8 @@ export class SimpleJobExecutor {
             return null;
         }
         catch (error) {
+            this.logger.error("Error writing @Operation to IOperationStore: @Error", operation, error);
+            this.writeCache.invalidate(documentId, scope, branch);
             return {
                 job,
                 success: false,
@@ -765,7 +824,7 @@ export class SimpleJobExecutor {
                 continue;
             }
             if (signer.signatures.length === 0) {
-                throw new InvalidSignatureError(job.documentId, `Operation ${operation.id ?? "unknown"} at index ${operation.index} has signer but no signatures`);
+                throw new InvalidSignatureError(job.documentId, `Operation ${operation.id} at index ${operation.index} has signer but no signatures`);
             }
             const publicKey = signer.app.key;
             let isValid = false;
@@ -774,10 +833,10 @@ export class SimpleJobExecutor {
             }
             catch (error) {
                 const errorMessage = error instanceof Error ? error.message : String(error);
-                throw new InvalidSignatureError(job.documentId, `Operation ${operation.id ?? "unknown"} at index ${operation.index} verification failed: ${errorMessage}`);
+                throw new InvalidSignatureError(job.documentId, `Operation ${operation.id} at index ${operation.index} verification failed: ${errorMessage}`);
             }
             if (!isValid) {
-                throw new InvalidSignatureError(job.documentId, `Operation ${operation.id ?? "unknown"} at index ${operation.index} signature verification returned false`);
+                throw new InvalidSignatureError(job.documentId, `Operation ${operation.id} at index ${operation.index} signature verification returned false`);
             }
         }
     }
@@ -797,6 +856,7 @@ export class SimpleJobExecutor {
             let isValid = false;
             try {
                 const tempOperation = {
+                    id: deriveOperationId(job.documentId, action.scope, job.branch, action.id),
                     index: 0,
                     timestampUtcMs: action.timestampUtcMs || new Date().toISOString(),
                     hash: "",