document-drive 4.1.0-dev.7 → 4.1.0-dev.70

package/dist/src/server/base-server.js

@@ -1,19 +1,8 @@
-import { removeListener, removeTrigger, setSharingType, } from "#drive-document-model/gen/creators";
-import { createDocument } from "#drive-document-model/gen/utils";
-import { isActionJob, isDocumentJob, isOperationJob, } from "#queue/types";
-import { ReadModeServer } from "#read-mode/server";
-import { DefaultDrivesManager, } from "#utils/default-drives-manager";
-import { requestPublicDriveWithTokenFromReactor } from "#utils/graphql";
-import { isDocumentDrive, runAsapAsync } from "#utils/misc";
-import { RunAsap } from "#utils/run-asap";
-import { DocumentAlreadyExistsError, childLogger, } from "document-drive";
-import { attachBranch, createPresignedHeader, garbageCollect, garbageCollectDocumentOperations, groupOperationsByScope, merge, precedes, removeExistingOperations, replayDocument, reshuffleByTimestamp, skipHeaderOperations, sortOperations, validateHeader, } from "document-model";
+import { ConflictOperationError, DefaultDrivesManager, DefaultListenerManagerOptions, DocumentAlreadyExistsError, OperationError, PullResponderTransmitter, ReadModeServer, SwitchboardPushTransmitter, childLogger, driveCreateDocument, filterOperationsByRevision, isActionJob, isAtRevision, isDocumentDrive, isDocumentJob, isOperationJob, removeListener, removeTrigger, requestPublicDriveWithTokenFromReactor, resolveCreateDocumentInput, setSharingType, } from "document-drive";
+import { runAsap, runAsapAsync } from "document-drive/run-asap";
+import {} from "document-model";
+import { attachBranch, createPresignedHeader, defaultBaseState, garbageCollect, garbageCollectDocumentOperations, groupOperationsByScope, hashDocumentStateForScope, merge, precedes, removeExistingOperations, replayDocument, reshuffleByTimestamp, skipHeaderOperations, sortOperations, validateHeader, } from "document-model/core";
 import { ClientError } from "graphql-request";
-import { ConflictOperationError, OperationError, } from "./error.js";
-import { PullResponderTransmitter, } from "./listener/transmitter/pull-responder.js";
-import { SwitchboardPushTransmitter } from "./listener/transmitter/switchboard-push.js";
-import { DefaultListenerManagerOptions, } from "./types.js";
-import { filterOperationsByRevision, isAtRevision, resolveCreateDocumentInput, } from "./utils.js";
 export class BaseDocumentDriveServer {
     logger = childLogger(["BaseDocumentDriveServer"]);
     // external dependencies
@@ -47,18 +36,19 @@ export class BaseDocumentDriveServer {
                 ? this.processDriveActions(documentId, actions, options)
                 : this.processActions(documentId, actions, options);
         },
-        processDocumentJob: async ({ documentId, documentType, initialState, options, }) => {
+        processDocumentJob: async ({ documentId, documentType, header: inputHeader, initialState, options, }) => {
             const documentModelModule = this.getDocumentModelModule(documentType);
-            const document = documentModelModule.utils.createDocument({
-                ...initialState,
-            });
+            const document = documentModelModule.utils.createDocument(initialState);
             // TODO: header must be included
             const header = createPresignedHeader(documentId, documentType);
             document.header.id = documentId;
             document.header.sig = header.sig;
             document.header.documentType = documentType;
+            if (inputHeader) {
+                document.header.meta = inputHeader.meta;
+            }
             try {
-                const createdDocument = await this.createDocument({ document }, options?.source ?? { type: "local" }, initialState?.header.meta);
+                const createdDocument = await this.createDocument({ document }, options?.source ?? { type: "local" }, document.header.meta);
                 return {
                     status: "SUCCESS",
                     operations: [],
@@ -95,6 +85,7 @@ export class BaseDocumentDriveServer {
     // internal state
     triggerMap = new Map();
     initializePromise;
+    enableDualActionCreate;
     constructor(documentModelModules, storage, documentStorage, cache, queueManager, eventEmitter, synchronizationManager, listenerManager, options) {
         this.documentModelModules = documentModelModules;
         this.legacyStorage = storage;
@@ -117,9 +108,17 @@ export class BaseDocumentDriveServer {
                 ? () => Promise.resolve("")
                 : options.jwtHandler,
             taskQueueMethod: options?.taskQueueMethod === undefined
-                ? RunAsap.runAsap
+                ? runAsap
                 : options.taskQueueMethod,
+            featureFlags: {
+                ...options?.featureFlags,
+            },
         };
+        this.enableDualActionCreate =
+            options?.featureFlags?.enableDualActionCreate ?? false;
+        if (this.enableDualActionCreate) {
+            this.logger.warn("Dual action create is enabled.");
+        }
         // todo: move to external dependencies
         this.defaultDrivesManager = new DefaultDrivesManager(this, this.defaultDrivesManagerDelegate, options);
         this.initializePromise = this._initialize();
@@ -136,7 +135,7 @@ export class BaseDocumentDriveServer {
         await this.listenerManager.initialize(this.handleListenerError.bind(this));
         await this.queueManager.init(this.queueDelegate, (error) => {
             this.logger.error(`Error initializing queue manager`, error);
-            errors.push(error);
+            // errors.push(error);
         });
         try {
             await this.defaultDrivesManager.removeOldremoteDrives();
@@ -328,7 +327,7 @@ export class BaseDocumentDriveServer {
         }
     }
     getDocumentModelModule(documentType) {
-        const documentModelModule = this.documentModelModules.find((module) => module.documentModel.id === documentType);
+        const documentModelModule = this.documentModelModules.find((module) => module.documentModel.global.id === documentType);
         if (!documentModelModule) {
             throw new Error(`Document type ${documentType} not supported`);
         }
@@ -337,17 +336,25 @@ export class BaseDocumentDriveServer {
     getDocumentModelModules() {
         return [...this.documentModelModules];
     }
-    addDocument(document, meta) {
-        return this.createDocument({ document }, { type: "local" }, meta);
+    addDocument(documentOrType, meta) {
+        const input = typeof documentOrType === "string"
+            ? { documentType: documentOrType }
+            : { document: documentOrType };
+        return this.createDocument(input, { type: "local" }, meta);
     }
     async addDrive(input, preferredEditor) {
-        const document = createDocument({
-            state: {
-                global: {
-                    icon: input.global.icon ?? null,
-                    name: input.global.name,
-                },
-                local: input.local ?? {},
+        // Create document with custom global and local state
+        const document = driveCreateDocument({
+            global: {
+                nodes: [],
+                icon: null,
+                ...input.global,
+            },
+            local: {
+                availableOffline: input.local?.availableOffline ?? false,
+                sharingType: input.local?.sharingType ?? "public",
+                listeners: input.local?.listeners ?? [],
+                triggers: input.local?.triggers ?? [],
             },
         });
         if (input.id && input.id.length > 0) {
@@ -356,9 +363,10 @@ export class BaseDocumentDriveServer {
         if (input.slug && input.slug.length > 0) {
             document.header.slug = input.slug;
         }
-        if (preferredEditor) {
+        const editorToUse = input.preferredEditor || preferredEditor;
+        if (editorToUse) {
             document.header.meta = {
-                preferredEditor: preferredEditor,
+                preferredEditor: editorToUse,
             };
         }
         await this.documentStorage.create(document);
@@ -545,6 +553,14 @@ export class BaseDocumentDriveServer {
         }
     }
     async createDocument(input, source, meta) {
+        if (this.enableDualActionCreate) {
+            return this.createDocumentDualAction(input, source, meta);
+        }
+        else {
+            return this.createDocumentLegacy(input, source, meta);
+        }
+    }
+    async createDocumentLegacy(input, source, meta) {
         const { documentType, document: inputDocument } = resolveCreateDocumentInput(input);
         // if a document was provided then checks if it's valid
         let state = undefined;
@@ -558,9 +574,7 @@ export class BaseDocumentDriveServer {
         }
         // if no document was provided then create a new one
         const document = inputDocument ??
-            this.getDocumentModelModule(documentType).utils.createDocument({
-                state,
-            });
+            this.getDocumentModelModule(documentType).utils.createDocument(state);
         // get the header
         let header;
         // handle the legacy case where an id is provided
@@ -609,11 +623,9 @@ export class BaseDocumentDriveServer {
         // stores document information
         const documentStorage = {
             header,
-            history: document.history,
             operations: { global: [], local: [] },
             initialState: document.initialState,
             clipboard: [],
-            attachments: document.attachments,
             state: state ?? document.state,
         };
         await this.documentStorage.create(documentStorage);
@@ -646,7 +658,172 @@ export class BaseDocumentDriveServer {
                 await this.legacyStorage.addDocumentOperations(header.id, operations, document);
             }
         }
-        return document;
+        return await this.getDocument(documentStorage.header.id);
+    }
+    async createDocumentDualAction(input, source, meta) {
+        const { documentType, document: inputDocument } = resolveCreateDocumentInput(input);
+        // if a document was provided then checks if it's valid
+        let state = undefined;
+        if (inputDocument) {
+            if ("documentType" in input &&
+                documentType !== inputDocument.header.documentType) {
+                throw new Error(`Provided document is not ${documentType}`);
+            }
+            const doc = this._buildDocument(inputDocument);
+            state = doc.state;
+        }
+        // if no document was provided then create a new one
+        const document = inputDocument ??
+            this.getDocumentModelModule(documentType).utils.createDocument(state);
+        // get the header
+        let header;
+        // handle the legacy case where an id is provided
+        let isSigned = false;
+        if ("id" in input && input.id) {
+            if (inputDocument) {
+                header = document.header;
+                document.header.id = input.id;
+                this.logger.warn("Assigning an id to a document is deprecated. Use the header field instead.");
+            }
+            else {
+                this.logger.warn("Creating a document with an id is deprecated. Use the header field instead.");
+                header = createPresignedHeader(input.id, documentType);
+            }
+        }
+        else if ("header" in input) {
+            // validate the header passed in
+            await validateHeader(input.header);
+            isSigned = true;
+            header = input.header;
+        }
+        else if (inputDocument?.header) {
+            if (!inputDocument.header.id) {
+                throw new Error("Document header id is required");
+            }
+            if (!inputDocument.header.documentType) {
+                throw new Error("Document header documentType is required");
+            }
+            if (!inputDocument.header.createdAtUtcIso) {
+                throw new Error("Document header createdAtUtcIso is required");
+            }
+            if (!inputDocument.header.sig.nonce) {
+                this.logger.warn("Creating a document with an unsigned id is deprecated. Use createSignedHeaderForSigner.");
+                // throw new Error("Document header sig nonce is required"); TODO: uncomment when ready to enforce signed documents
+            }
+            else {
+                await validateHeader(inputDocument.header);
+                isSigned = true;
+            }
+            header = inputDocument.header;
+        }
+        else {
+            // otherwise, generate a header
+            header = createPresignedHeader(undefined, documentType);
+            isSigned = false;
+        }
+        if (meta) {
+            header.meta = { ...header.meta, ...meta };
+        }
+        const currentVersion = "0.1.0";
+        // Get initial state from input or model's defaultState
+        const initialState = state ?? document.state;
+        // Check if the input document already has operations
+        const existingOperations = Object.values(document.operations).flat();
+        const shouldCreateOperations = existingOperations.length === 0;
+        let operations = [];
+        if (shouldCreateOperations) {
+            const timestampUtcMs = new Date().toISOString();
+            // Determine if this is a signed document
+            const signing = isSigned
+                ? {
+                    signature: header.id, // The document ID is the signature
+                    publicKey: header.sig.publicKey,
+                    nonce: header.sig.nonce,
+                    createdAtUtcIso: header.createdAtUtcIso,
+                    documentType: header.documentType,
+                }
+                : undefined;
+            // Create actions for CREATE_DOCUMENT and UPGRADE_DOCUMENT
+            const createDocumentInput = {
+                model: documentType,
+                version: "0.0.0",
+                documentId: header.id,
+                signing,
+            };
+            const createDocumentAction = {
+                id: `${header.id}-create`,
+                type: "CREATE_DOCUMENT",
+                timestampUtcMs,
+                input: createDocumentInput,
+                scope: "global",
+            };
+            const upgradeDocumentInput = {
+                model: documentType,
+                fromVersion: "0.0.0",
+                toVersion: currentVersion,
+                documentId: header.id,
+                initialState,
+            };
+            const upgradeDocumentAction = {
+                id: `${header.id}-upgrade`,
+                type: "UPGRADE_DOCUMENT",
+                timestampUtcMs,
+                input: upgradeDocumentInput,
+                scope: "global",
+            };
+            // we need to create hashes for later verification
+            const baseState = defaultBaseState();
+            const createStateForHash = {
+                state: baseState,
+            };
+            const createHash = hashDocumentStateForScope(createStateForHash, "document");
+            const upgradeStateForHash = {
+                state: initialState,
+            };
+            const upgradeHash = hashDocumentStateForScope(upgradeStateForHash, 
+            // this is bad! there may not be a global scope
+            "global");
+            // Create operations from actions with computed hashes
+            operations = [
+                {
+                    index: 0,
+                    skip: 0,
+                    hash: createHash,
+                    timestampUtcMs,
+                    action: createDocumentAction,
+                },
+                {
+                    index: 1,
+                    skip: 0,
+                    hash: upgradeHash,
+                    timestampUtcMs,
+                    action: upgradeDocumentAction,
+                },
+            ];
+        }
+        else {
+            // Use existing operations from the input document
+            operations = existingOperations;
+        }
+        // stores document information with empty operations initially
+        const documentToStore = {
+            header,
+            operations: { global: [], local: [] },
+            initialState: document.initialState,
+            clipboard: [],
+            state: initialState,
+        };
+        await this.documentStorage.create(documentToStore);
+        // Store operations separately (if any)
+        if (operations.length > 0) {
+            if (isDocumentDrive(documentToStore)) {
+                await this.legacyStorage.addDriveOperations(header.id, operations, documentToStore);
+            }
+            else {
+                await this.legacyStorage.addDocumentOperations(header.id, operations, documentToStore);
+            }
+        }
+        return await this.getDocument(documentToStore.header.id);
     }
     async deleteDocument(documentId) {
         try {
@@ -741,8 +918,8 @@ export class BaseDocumentDriveServer {
             // to retrieve it from the db to avoid rerunning all the operations
             if (lastRemainingOperation && !lastRemainingOperation.resultingState) {
                 lastRemainingOperation.resultingState = await (isDocumentDrive(document)
-                    ? this.legacyStorage.getOperationResultingState?.(documentId, lastRemainingOperation.index, lastRemainingOperation.scope, "main")
-                    : this.legacyStorage.getDriveOperationResultingState?.(documentId, lastRemainingOperation.index, lastRemainingOperation.scope, "main"));
+                    ? this.legacyStorage.getOperationResultingState?.(documentId, lastRemainingOperation.index, lastRemainingOperation.action.scope, "main")
+                    : this.legacyStorage.getDriveOperationResultingState?.(documentId, lastRemainingOperation.index, lastRemainingOperation.action.scope, "main"));
             }
         }
         return {
@@ -761,17 +938,49 @@ export class BaseDocumentDriveServer {
             ? filterOperationsByRevision(documentStorage.operations, options.revisions)
             : documentStorage.operations;
         const operations = garbageCollectDocumentOperations(revisionOperations);
-        return replayDocument(documentStorage.initialState, operations, documentModelModule.reducer, undefined, documentStorage.header, undefined, {
+        // for backward compatibility, we need to add global and local
+        const headerOperations = { global: [], local: [] };
+        const operationsToReplay = { global: [], local: [] };
+        // Filter out CREATE_DOCUMENT and UPGRADE_DOCUMENT operations
+        // (these don't currently have reducers and should not be replayed)
+        for (const [scope, scopeOps] of Object.entries(operations)) {
+            for (const op of scopeOps) {
+                if (op.action.type === "CREATE_DOCUMENT" ||
+                    op.action.type === "UPGRADE_DOCUMENT") {
+                    headerOperations[scope].push(op);
+                }
+                else {
+                    operationsToReplay[scope].push(op);
+                }
+            }
+        }
+        const replayed = replayDocument(documentStorage.initialState, operationsToReplay, documentModelModule.reducer, undefined, documentStorage.header, undefined, {
             ...options,
             checkHashes: options?.checkHashes ?? true,
             reuseOperationResultingState: options?.checkHashes ?? true,
         });
+        // merge header operations back into the result
+        const allScopes = new Set([
+            ...Object.keys(headerOperations),
+            ...Object.keys(replayed.operations),
+        ]);
+        const finalOperations = {};
+        for (const scope of allScopes) {
+            finalOperations[scope] = [
+                ...headerOperations[scope],
+                ...replayed.operations[scope],
+            ];
+        }
+        return {
+            ...replayed,
+            operations: finalOperations,
+        };
     }
     async _performOperation(documentId, document, operation, skipHashValidation = false) {
         const documentModelModule = this.getDocumentModelModule(document.header.documentType);
         const signalResults = [];
         let newDocument = document;
-        const scope = operation.scope;
+        const scope = operation.action.scope;
         const documentOperations = garbageCollectDocumentOperations({
             ...document.operations,
             [scope]: skipHeaderOperations(document.operations[scope], operation),
@@ -781,11 +990,11 @@ export class BaseDocumentDriveServer {
         // to retrieve it from the db to avoid rerunning all the operations
         if (lastRemainingOperation && !lastRemainingOperation.resultingState) {
             lastRemainingOperation.resultingState = await (isDocumentDrive(document)
-                ? this.legacyStorage.getOperationResultingState?.(documentId, lastRemainingOperation.index, lastRemainingOperation.scope, "main")
-                : this.legacyStorage.getDriveOperationResultingState?.(documentId, lastRemainingOperation.index, lastRemainingOperation.scope, "main"));
+                ? this.legacyStorage.getOperationResultingState?.(documentId, lastRemainingOperation.index, lastRemainingOperation.action.scope, "main")
+                : this.legacyStorage.getDriveOperationResultingState?.(documentId, lastRemainingOperation.index, lastRemainingOperation.action.scope, "main"));
         }
         const operationSignals = [];
-        newDocument = documentModelModule.reducer(newDocument, operation, (signal) => {
+        newDocument = documentModelModule.reducer(newDocument, operation.action, (signal) => {
             let handler = undefined;
             switch (signal.type) {
                 case "CREATE_CHILD_DOCUMENT":
@@ -801,8 +1010,12 @@ export class BaseDocumentDriveServer {
             if (handler) {
                 operationSignals.push(() => handler().then((result) => ({ signal, result })));
             }
-        }, { skip: operation.skip, reuseOperationResultingState: true });
-        const appliedOperations = newDocument.operations[operation.scope].filter((op) => op.index == operation.index && op.skip == operation.skip);
+        }, {
+            skip: operation.skip,
+            reuseOperationResultingState: true,
+            replayOptions: { operation },
+        });
+        const appliedOperations = newDocument.operations[operation.action.scope].filter((op) => op.index == operation.index && op.skip == operation.skip);
         const appliedOperation = appliedOperations.at(0);
         if (!appliedOperation) {
             throw new OperationError("ERROR", operation, `Operation with index ${operation.index}:${operation.skip} was not applied.`);
@@ -889,7 +1102,8 @@ export class BaseDocumentDriveServer {
             jobId = await this.queueManager.addJob({
                 documentId: id,
                 documentType,
-                initialState: document,
+                initialState: document?.state,
+                header: document?.header,
                 options,
             });
         }
@@ -921,9 +1135,9 @@ export class BaseDocumentDriveServer {
         try {
             const document = await this.getDocument(id);
             const newOperation = operations.find((op) => !op.id ||
-                !document.operations[op.scope].find((existingOp) => existingOp.id === op.id &&
+                !document.operations[op.action.scope].find((existingOp) => existingOp.id === op.id &&
                     existingOp.index === op.index &&
-                    existingOp.type === op.type &&
+                    existingOp.action.type === op.action.type &&
                     existingOp.hash === op.hash));
             if (!newOperation) {
                 return {
@@ -1155,10 +1369,10 @@ export class BaseDocumentDriveServer {
                     const syncUnit = {
                         documentId,
                         documentType: document.header.documentType,
-                        scope: operation.scope,
+                        scope: operation.action.scope,
                         branch: "main", // TODO: handle branches
                         revision: operation.index + 1,
-                        lastUpdated: operation.timestamp,
+                        lastUpdated: operation.timestampUtcMs,
                     };
                     // checks if this sync unit was already added
                     const exists = syncUnits.some((unit) => unit.documentId === syncUnit.documentId &&
@@ -1277,9 +1491,9 @@ export class BaseDocumentDriveServer {
         try {
             const drive = await this.getDrive(driveId);
             const newOperation = operations.find((op) => !op.id ||
-                !drive.operations[op.scope].find((existingOp) => existingOp.id === op.id &&
+                !drive.operations[op.action.scope].find((existingOp) => existingOp.id === op.id &&
                     existingOp.index === op.index &&
-                    existingOp.type === op.type &&
+                    existingOp.action.type === op.action.type &&
                     existingOp.hash === op.hash));
             if (!newOperation) {
                 return {
@@ -1370,7 +1584,7 @@ export class BaseDocumentDriveServer {
             this.cache.setDrive(driveId, document).catch(this.logger.error);
             // update listener cache
             const lastOperation = operationsApplied
-                .filter((op) => op.scope === "global")
+                .filter((op) => op.action.scope === "global")
                 .slice()
                 .pop();
             if (lastOperation) {
@@ -1393,7 +1607,7 @@ export class BaseDocumentDriveServer {
                         documentType: document.header.documentType,
                         scope: "global",
                         branch: "main",
-                        lastUpdated: lastOperation.timestamp,
+                        lastUpdated: lastOperation.timestampUtcMs,
                         revision: lastOperation.index,
                     },
                 ], source, () => {
@@ -1516,7 +1730,7 @@ export class BaseDocumentDriveServer {
     }
     async _legacyAddFileAction(driveId, action, options) {
         // create document before adding it to the drive
-        const document = this.getDocumentModelModule(action.input.documentType).utils.createDocument({ ...action.input.document });
+        const document = this.getDocumentModelModule(action.input.documentType).utils.createDocument(action.input.document?.state || undefined);
         document.header.id = action.input.id;
         document.header.name = action.input.name;
         document.header.documentType = action.input.documentType;
@@ -1587,6 +1801,14 @@ export class BaseDocumentDriveServer {
         }
         const operations = strand.operations.map((op) => ({
             ...op,
+            action: {
+                id: op.actionId,
+                timestampUtcMs: op.timestampUtcMs,
+                type: op.type,
+                input: op.input,
+                context: op.context,
+                scope: strand.scope,
+            },
             scope: strand.scope,
             branch: strand.branch,
         }));