document-drive 0.0.26 → 0.0.28

package/src/server/listener/transmitter/switchboard-push.ts

@@ -0,0 +1,63 @@
+import stringify from 'json-stringify-deterministic';
+import { gql, requestGraphql } from '../../../utils/graphql';
+import {
+    BaseDocumentDriveServer,
+    Listener,
+    ListenerRevision,
+    StrandUpdate
+} from '../../types';
+import { ITransmitter } from './types';
+
+export class SwitchboardPushTransmitter implements ITransmitter {
+    private drive: BaseDocumentDriveServer;
+    private listener: Listener;
+    private targetURL: string;
+
+    constructor(listener: Listener, drive: BaseDocumentDriveServer) {
+        this.listener = listener;
+        this.drive = drive;
+        this.targetURL = listener.callInfo!.data!;
+    }
+
+    async transmit(strands: StrandUpdate[]): Promise<ListenerRevision[]> {
+        // Send Graphql mutation to switchboard
+        try {
+            const { pushUpdates } = await requestGraphql<{
+                pushUpdates: ListenerRevision[];
+            }>(
+                this.targetURL,
+                gql`
+                    mutation pushUpdates($strands: [InputStrandUpdate!]) {
+                        pushUpdates(strands: $strands) {
+                            driveId
+                            documentId
+                            scope
+                            branch
+                            status
+                            revision
+                        }
+                    }
+                `,
+                {
+                    strands: strands.map(strand => ({
+                        ...strand,
+                        operations: strand.operations.map(op => ({
+                            ...op,
+                            input: stringify(op.input)
+                        }))
+                    }))
+                }
+            );
+
+            if (!pushUpdates) {
+                throw new Error("Couldn't update listener revision");
+            }
+
+            return pushUpdates;
+        } catch (e) {
+            console.error(e);
+            throw e;
+        }
+        return [];
+    }
+}

package/src/server/listener/transmitter/types.ts

@@ -0,0 +1,18 @@
+import {
+    PullResponderTriggerData,
+    Trigger
+} from 'document-model-libs/document-drive';
+import { ListenerRevision, StrandUpdate } from '../..';
+
+export interface ITransmitter {
+    transmit(strands: StrandUpdate[]): Promise<ListenerRevision[]>;
+}
+
+export interface InternalTransmitterService extends ITransmitter {
+    getName(): string;
+}
+
+export type PullResponderTrigger = Omit<Trigger, 'data' | 'type'> & {
+    data: PullResponderTriggerData;
+    type: 'PullResponder';
+};

package/src/server/types.ts

@@ -2,68 +2,254 @@ import type {
     DocumentDriveAction,
     DocumentDriveDocument,
     DocumentDriveLocalState,
-    DocumentDriveState
+    DocumentDriveState,
+    ListenerCallInfo,
+    ListenerFilter
 } from 'document-model-libs/document-drive';
 import type {
     BaseAction,
+    CreateChildDocumentInput,
     Document,
     Operation,
+    OperationScope,
     Signal,
     State
 } from 'document-model/document';
+import { OperationError } from './error';
+import { ITransmitter } from './listener/transmitter/types';
 
 export type DriveInput = State<
-    Omit<DocumentDriveState, '__typename' | 'nodes'>,
+    Omit<DocumentDriveState, '__typename' | 'id' | 'nodes'> & { id?: string },
     DocumentDriveLocalState
 >;
 
-export type CreateDocumentInput = {
-    id: string;
-    documentType: string;
-    document?: Document;
+export type RemoteDriveOptions = DocumentDriveLocalState & {
+    // TODO make local state optional
+    pullFilter?: ListenerFilter;
+    pullInterval?: number;
 };
 
+export type CreateDocumentInput = CreateChildDocumentInput;
+
 export type SignalResult = {
     signal: Signal;
     result: unknown; // infer from return types on document-model
 };
 
 export type IOperationResult<T extends Document = Document> = {
-    success: boolean;
-    error?: Error;
+    status: UpdateStatus;
+    error?: OperationError;
     operations: Operation[];
     document: T | undefined;
     signals: SignalResult[];
 };
 
-export interface IDocumentDriveServer {
-    getDrives(): Promise<string[]>;
-    addDrive(drive: DriveInput): Promise<void>;
-    deleteDrive(id: string): Promise<void>;
-    getDrive(id: string): Promise<DocumentDriveDocument>;
+export type SynchronizationUnit = {
+    syncId: string;
+    driveId: string;
+    documentId: string;
+    documentType: string;
+    scope: string;
+    branch: string;
+    lastUpdated: string;
+    revision: number;
+};
+
+export type Listener = {
+    driveId: string;
+    listenerId: string;
+    label?: string;
+    block: boolean;
+    system: boolean;
+    filter: ListenerFilter;
+    callInfo?: ListenerCallInfo;
+};
+
+export type CreateListenerInput = {
+    driveId: string;
+    label?: string;
+    block: boolean;
+    system: boolean;
+    filter: ListenerFilter;
+    callInfo?: ListenerCallInfo;
+};
+
+export enum TransmitterType {
+    Internal,
+    SwitchboardPush,
+    PullResponder,
+    SecureConnect,
+    MatrixConnect,
+    RESTWebhook
+}
+
+export type ListenerRevision = {
+    driveId: string;
+    documentId: string;
+    scope: string;
+    branch: string;
+    status: UpdateStatus;
+    revision: number;
+};
+
+export type UpdateStatus = 'SUCCESS' | 'CONFLICT' | 'MISSING' | 'ERROR';
+export type ErrorStatus = Exclude<UpdateStatus, 'SUCCESS'>;
+
+export type OperationUpdate = {
+    timestamp: string;
+    index: number;
+    skip: number;
+    type: string;
+    input: object;
+    hash: string;
+};
+
+export type StrandUpdate = {
+    driveId: string;
+    documentId: string;
+    scope: OperationScope;
+    branch: string;
+    operations: OperationUpdate[];
+};
+
+export type SyncStatus = 'SYNCING' | UpdateStatus;
+
+export abstract class BaseDocumentDriveServer {
+    /** Public methods **/
+    abstract getDrives(): Promise<string[]>;
+    abstract addDrive(drive: DriveInput): Promise<void>;
+    abstract addRemoteDrive(
+        url: string,
+        options: RemoteDriveOptions
+    ): Promise<void>;
+    abstract deleteDrive(id: string): Promise<void>;
+    abstract getDrive(id: string): Promise<DocumentDriveDocument>;
 
-    getDocuments: (drive: string) => Promise<string[]>;
-    getDocument: (drive: string, id: string) => Promise<Document>;
-    createDocument(drive: string, document: CreateDocumentInput): Promise<void>;
-    deleteDocument(drive: string, id: string): Promise<void>;
+    abstract getDocuments(drive: string): Promise<string[]>;
+    abstract getDocument(drive: string, id: string): Promise<Document>;
 
-    addOperation(
+    abstract addOperation(
         drive: string,
         id: string,
         operation: Operation
-    ): Promise<IOperationResult>;
-    addOperations(
+    ): Promise<IOperationResult<Document>>;
+    abstract addOperations(
         drive: string,
         id: string,
         operations: Operation[]
-    ): Promise<IOperationResult>;
+    ): Promise<IOperationResult<Document>>;
 
-    addDriveOperation(
+    abstract addDriveOperation(
         drive: string,
         operation: Operation<DocumentDriveAction | BaseAction>
     ): Promise<IOperationResult<DocumentDriveDocument>>;
-    addDriveOperations(
+    abstract addDriveOperations(
         drive: string,
         operations: Operation<DocumentDriveAction | BaseAction>[]
     ): Promise<IOperationResult<DocumentDriveDocument>>;
+
+    abstract getSyncStatus(drive: string): SyncStatus;
+
+    /** Synchronization methods */
+    abstract getSynchronizationUnits(
+        driveId: string,
+        documentId?: string[],
+        scope?: string[],
+        branch?: string[]
+    ): Promise<SynchronizationUnit[]>;
+
+    abstract getSynchronizationUnit(
+        driveId: string,
+        syncId: string
+    ): Promise<SynchronizationUnit>;
+
+    abstract getOperationData(
+        driveId: string,
+        syncId: string,
+        filter: {
+            since?: string;
+            fromRevision?: number;
+        }
+    ): Promise<OperationUpdate[]>;
+
+    /** Internal methods **/
+    protected abstract createDocument(
+        drive: string,
+        document: CreateDocumentInput
+    ): Promise<Document>;
+    protected abstract deleteDocument(drive: string, id: string): Promise<void>;
+
+    abstract getTransmitter(
+        driveId: string,
+        listenerId: string
+    ): Promise<ITransmitter | undefined>;
+}
+
+export abstract class BaseListenerManager {
+    protected drive: BaseDocumentDriveServer;
+    protected listenerState: Map<string, Map<string, ListenerState>> =
+        new Map();
+    protected transmitters: Record<
+        DocumentDriveState['id'],
+        Record<Listener['listenerId'], ITransmitter>
+    > = {};
+
+    constructor(
+        drive: BaseDocumentDriveServer,
+        listenerState: Map<string, Map<string, ListenerState>> = new Map()
+    ) {
+        this.drive = drive;
+        this.listenerState = listenerState;
+    }
+
+    abstract init(): Promise<void>;
+    abstract addListener(listener: Listener): Promise<ITransmitter>;
+    abstract removeListener(
+        driveId: string,
+        listenerId: string
+    ): Promise<boolean>;
+    abstract getTransmitter(
+        driveId: string,
+        listenerId: string
+    ): Promise<ITransmitter | undefined>;
+    abstract updateSynchronizationRevision(
+        driveId: string,
+        syncId: string,
+        syncRev: number,
+        lastUpdated: string
+    ): Promise<void>;
+
+    abstract updateListenerRevision(
+        listenerId: string,
+        driveId: string,
+        syncId: string,
+        listenerRev: number
+    ): Promise<void>;
+}
+
+export type IDocumentDriveServer = Pick<
+    BaseDocumentDriveServer,
+    keyof BaseDocumentDriveServer
+>;
+
+export type ListenerStatus =
+    | 'CREATED'
+    | 'PENDING'
+    | 'SUCCESS'
+    | 'MISSING'
+    | 'CONFLICT'
+    | 'ERROR';
+
+export interface ListenerState {
+    driveId: string;
+    block: boolean;
+    pendingTimeout: string;
+    listener: Listener;
+    syncUnits: SyncronizationUnitState[];
+    listenerStatus: ListenerStatus;
+}
+
+export interface SyncronizationUnitState extends SynchronizationUnit {
+    listenerRev: number;
+    syncRev: number;
 }

package/src/storage/filesystem.ts

@@ -9,6 +9,7 @@ import {
     writeFileSync
 } from 'fs';
 import fs from 'fs/promises';
+import stringify from 'json-stringify-deterministic';
 import path from 'path';
 import sanitize from 'sanitize-filename';
 import { mergeOperations } from '..';
@@ -83,7 +84,6 @@ export class FilesystemStorage implements IDriveStorage {
             });
             return JSON.parse(content) as Promise<DocumentStorage>;
         } catch (error) {
-            console.error(error);
             throw new Error(`Document with id ${id} not found`);
         }
     }
@@ -91,7 +91,7 @@ export class FilesystemStorage implements IDriveStorage {
     async createDocument(drive: string, id: string, document: DocumentStorage) {
         const documentPath = this._buildDocumentPath(drive, id);
         await ensureDir(path.dirname(documentPath));
-        await writeFileSync(documentPath, JSON.stringify(document), {
+        await writeFileSync(documentPath, stringify(document), {
             encoding: 'utf-8'
         });
     }

package/src/storage/index.ts

@@ -1,5 +1 @@
-export { BrowserStorage } from './browser';
-export { FilesystemStorage } from './filesystem';
-export { MemoryStorage } from './memory';
-export { PrismaStorage } from './prisma';
 export type * from './types';

package/src/storage/memory.ts

@@ -30,6 +30,7 @@ export class MemoryStorage implements IDriveStorage {
         if (!document) {
             throw new Error(`Document with id ${id} not found`);
         }
+
         return document;
     }
 
@@ -47,7 +48,8 @@ export class MemoryStorage implements IDriveStorage {
             revision,
             documentType,
             created,
-            lastModified
+            lastModified,
+            clipboard
         } = document;
         this.documents[drive]![id] = {
             operations,
@@ -56,7 +58,8 @@ export class MemoryStorage implements IDriveStorage {
             revision,
             documentType,
             created,
-            lastModified
+            lastModified,
+            clipboard
         };
     }
 

package/src/storage/prisma.ts

@@ -17,9 +17,11 @@ export class PrismaStorage implements IDriveStorage {
     constructor(db: Prisma.TransactionClient) {
         this.db = db;
     }
+
     async createDrive(id: string, drive: DocumentDriveStorage): Promise<void> {
         await this.createDocument('drives', id, drive as DocumentStorage);
     }
+
     async addDriveOperations(
         id: string,
         operations: Operation[],
@@ -27,6 +29,7 @@ export class PrismaStorage implements IDriveStorage {
     ): Promise<void> {
         await this.addDocumentOperations('drives', id, operations, header);
     }
+
     async createDocument(
         drive: string,
         id: string,
@@ -84,7 +87,8 @@ export class PrismaStorage implements IDriveStorage {
                             timestamp: op.timestamp,
                             type: op.type,
                             scope: op.scope,
-                            branch: 'main'
+                            branch: 'main',
+                            skip: op.skip
                         },
                         update: {
                             driveId: drive,
@@ -95,18 +99,17 @@ export class PrismaStorage implements IDriveStorage {
                             timestamp: op.timestamp,
                             type: op.type,
                             scope: op.scope,
-                            branch: 'main'
+                            branch: 'main',
+                            skip: op.skip
                         }
                     });
                 })
             );
 
-            await this.db.document.update({
+            await this.db.document.updateMany({
                 where: {
-                    id_driveId: {
-                        id,
-                        driveId: 'drives'
-                    }
+                    id,
+                    driveId: drive
                 },
                 data: {
                     lastModified: header.lastModified,
@@ -175,7 +178,6 @@ export class PrismaStorage implements IDriveStorage {
         }
 
         const dbDoc = result;
-
         const doc = {
             created: dbDoc.created.toISOString(),
             name: dbDoc.name ? dbDoc.name : '',
@@ -184,11 +186,12 @@ export class PrismaStorage implements IDriveStorage {
                 DocumentDriveState,
                 DocumentDriveLocalState
             >,
-            lastModified: dbDoc.lastModified.toISOString(),
+            lastModified: new Date(dbDoc.lastModified).toISOString(),
             operations: {
                 global: dbDoc.operations
-                    .filter(op => op.scope === 'global')
+                    .filter(op => op.scope === 'global' && !op.clipboard)
                     .map(op => ({
+                        skip: op.skip,
                         hash: op.hash,
                         index: op.index,
                         timestamp: new Date(op.timestamp).toISOString(),
@@ -198,8 +201,9 @@ export class PrismaStorage implements IDriveStorage {
                         // attachments: fileRegistry
                     })),
                 local: dbDoc.operations
-                    .filter(op => op.scope === 'local')
+                    .filter(op => op.scope === 'local' && !op.clipboard)
                     .map(op => ({
+                        skip: op.skip,
                         hash: op.hash,
                         index: op.index,
                         timestamp: new Date(op.timestamp).toISOString(),
@@ -209,6 +213,18 @@ export class PrismaStorage implements IDriveStorage {
                         // attachments: fileRegistry
                     }))
             },
+            clipboard: dbDoc.operations
+                .filter(op => op.clipboard)
+                .map(op => ({
+                    skip: op.skip,
+                    hash: op.hash,
+                    index: op.index,
+                    timestamp: new Date(op.timestamp).toISOString(),
+                    input: op.input,
+                    type: op.type,
+                    scope: op.scope as OperationScope
+                    // attachments: fileRegistry
+                })),
             revision: dbDoc.revision as Required<Record<OperationScope, number>>
         };
 
@@ -216,12 +232,48 @@ export class PrismaStorage implements IDriveStorage {
     }
 
     async deleteDocument(drive: string, id: string) {
-        await this.db.document.deleteMany({
+        await this.db.attachment.deleteMany({
             where: {
                 driveId: drive,
-                id: id
+                documentId: id
+            }
+        });
+
+        await this.db.operation.deleteMany({
+            where: {
+                driveId: drive,
+                documentId: id
+            }
+        });
+
+        await this.db.document.delete({
+            where: {
+                id_driveId: {
+                    driveId: drive,
+                    id: id
+                }
             }
         });
+
+        if (drive === 'drives') {
+            await this.db.attachment.deleteMany({
+                where: {
+                    driveId: id
+                }
+            });
+
+            await this.db.operation.deleteMany({
+                where: {
+                    driveId: id
+                }
+            });
+
+            await this.db.document.deleteMany({
+                where: {
+                    driveId: id
+                }
+            });
+        }
     }
 
     async getDrives() {
@@ -234,10 +286,5 @@ export class PrismaStorage implements IDriveStorage {
 
     async deleteDrive(id: string) {
         await this.deleteDocument('drives', id);
-        await this.db.document.deleteMany({
-            where: {
-                driveId: id
-            }
-        });
     }
 }

package/src/utils/graphql.ts

@@ -0,0 +1,46 @@
+import request, { GraphQLClient, gql } from 'graphql-request';
+
+export { gql } from 'graphql-request';
+
+export type DriveInfo = {
+    id: string;
+    name: string;
+    slug: string;
+    icon?: string;
+};
+
+// replaces fetch so it can be used in Node and Browser envs
+export async function requestGraphql<T>(...args: Parameters<typeof request>) {
+    const [url, ...requestArgs] = args;
+    const client = new GraphQLClient(url, { fetch });
+    return client.request<T>(...requestArgs);
+}
+
+export async function requestPublicDrive(url: string): Promise<DriveInfo> {
+    let drive: DriveInfo;
+    try {
+        const result = await requestGraphql<{ drive: DriveInfo }>(
+            url,
+            gql`
+                query getDrive {
+                    drive {
+                        id
+                        name
+                        icon
+                        slug
+                    }
+                }
+            `
+        );
+        drive = result.drive;
+    } catch (e) {
+        console.error(e);
+        throw new Error("Couldn't find drive info");
+    }
+
+    if (!drive) {
+        throw new Error('Drive not found');
+    }
+
+    return drive;
+}

package/src/{utils.ts → utils/index.ts}

@@ -30,3 +30,11 @@ export function mergeOperations<A extends Action = Action>(
         return acc;
     }, currentOperations);
 }
+
+export function generateUUID(): string {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+    const crypto =
+        typeof window !== 'undefined' ? window.crypto : require('crypto');
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+    return crypto.randomUUID() as string;
+}