document-drive 1.0.0-experimental.1 → 1.0.0-experimental.2

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "document-drive",
-    "version": "1.0.0-experimental.1",
+    "version": "1.0.0-experimental.2",
     "license": "AGPL-3.0-only",
     "type": "module",
     "module": "./src/index.ts",

package/src/queue/base.ts

@@ -0,0 +1,273 @@
+import { IJob, IJobQueue, IQueue, IQueueManager, IServerDelegate, JobId, OperationJob, QueueEvents } from "./types";
+import { generateUUID } from "../utils";
+import { IOperationResult } from "../server";
+import { createNanoEvents, Unsubscribe } from 'nanoevents';
+import { Operation } from "document-model/document";
+import { AddFileInput, DeleteNodeInput } from "document-model-libs/document-drive";
+
+export class MemoryQueue<T, R> implements IQueue<T, R> {
+    private id: string;
+    private blocked = false;
+    private deleted = false;
+    private items: IJob<T>[] = [];
+    private results = new Map<JobId, R>();
+    private dependencies = new Array<IJob<OperationJob>>();
+
+    constructor(id: string) {
+        this.id = id;
+    }
+
+    async setDeleted(deleted: boolean) {
+        this.deleted = deleted;
+    }
+
+    async isDeleted() {
+        return this.deleted;
+    }
+
+    async setResult(jobId: string, result: R): Promise<void> {
+        this.results.set(jobId, result);
+        return Promise.resolve();
+    }
+
+    async getResult(jobId: string): Promise<R | undefined> {
+        return Promise.resolve(this.results.get(jobId));
+    }
+
+    async addJob(data: IJob<T>) {
+        this.items.push(data);
+        return Promise.resolve();
+    }
+
+    async getNextJob() {
+        const job = this.items.shift();
+        return Promise.resolve(job);
+    }
+
+    async amountOfJobs() {
+        return Promise.resolve(this.items.length);
+    }
+
+    getId() {
+        return this.id;
+    }
+
+    async setBlocked(blocked: boolean) {
+        this.blocked = blocked;
+    }
+
+    async isBlocked() {
+        return this.blocked;
+    }
+
+    async getJobs() {
+        return this.items;
+    }
+
+    async addDependencies(job: IJob<OperationJob>) {
+        if (!this.dependencies.find(j => j.jobId === job.jobId)) {
+            this.dependencies.push(job);
+        }
+        if (!this.isBlocked()) {
+            this.setBlocked(true);
+        }
+    }
+
+    async removeDependencies(job: IJob<OperationJob>) {
+        this.dependencies = this.dependencies.filter((j) => j.jobId !== job.jobId && j.driveId !== job.driveId);
+        if (this.dependencies.length === 0) {
+            await this.setBlocked(false);
+        }
+    }
+}
+
+export class BaseQueueManager implements IQueueManager {
+    protected emitter = createNanoEvents<QueueEvents>();
+    protected ticker = 0;
+    protected queues: IJobQueue[] = [];
+    protected workers: number;
+    protected timeout: number;
+    private delegate: IServerDelegate | undefined;
+
+    constructor(workers = 3, timeout = 0) {
+        this.workers = workers;
+        this.timeout = timeout;
+    }
+
+    async init(delegate: IServerDelegate, onError: (error: Error) => void): Promise<void> {
+        this.delegate = delegate;
+        for (let i = 0; i < this.workers; i++) {
+            setTimeout(() => this.processNextJob.bind(this)().catch(onError), 100 * i);
+        }
+        return Promise.resolve()
+    }
+
+    async addJob(job: OperationJob): Promise<JobId> {
+        if (!this.delegate) {
+            throw new Error("No server delegate defined");
+        }
+
+        const jobId = generateUUID();
+        const queue = this.getQueue(job.driveId, job.documentId);
+
+        if (await queue.isDeleted()) {
+            throw new Error("Queue is deleted")
+        }
+
+        // checks if the job is for a document that doesn't exist in storage yet
+        const newDocument = job.documentId && !(await this.delegate.checkDocumentExists(job.driveId, job.documentId));
+        // if it is a new document and queue is not yet blocked then 
+        // blocks it so the jobs are not processed until it's ready
+        if (newDocument && !(await queue.isBlocked())) {
+            await queue.setBlocked(true);
+
+            // checks if there any job in the queue adding the file and adds as dependency
+            const driveQueue = this.getQueue(job.driveId);
+            const jobs = await driveQueue.getJobs();
+            for (const driveJob of jobs) {
+                const op = driveJob.operations.find((j: Operation) => {
+                    const input = j.input as AddFileInput;
+                    return j.type === "ADD_FILE" && input.id === job.documentId
+                })
+                if (op) {
+                    await queue.addDependencies(driveJob);
+                }
+            }
+        }
+
+        // if it has ADD_FILE operations then adds the job as
+        // a dependency to the corresponding document queues
+        const addFileOps = job.operations.filter((j: Operation) => j.type === "ADD_FILE");
+        for (const addFileOp of addFileOps) {
+            const input = addFileOp.input as AddFileInput;
+            const q = this.getQueue(job.driveId, input.id)
+            await q.addDependencies({ jobId, ...job });
+        }
+
+        // remove document if operations contains delete_node
+        const removeFileOps = job.operations.filter((j: Operation) => j.type === "DELETE_NODE");
+        for (const removeFileOp of removeFileOps) {
+            const input = removeFileOp.input as DeleteNodeInput;
+            const queue = this.getQueue(job.driveId, input.id);
+            await queue.setDeleted(true);
+        }
+
+        await queue.addJob({ jobId, ...job });
+
+        return jobId;
+    }
+
+    async getResult(driveId: string, documentId: string, jobId: JobId): Promise<IOperationResult | undefined> {
+        const queue = this.getQueue(driveId, documentId);
+        return queue.getResult(jobId);
+    }
+
+    getQueue(driveId: string, documentId?: string) {
+        const queueId = this.getQueueId(driveId, documentId);
+        let queue = this.queues.find((q) => q.getId() === queueId);
+
+        if (!queue) {
+            queue = new MemoryQueue(queueId);
+            this.queues.push(queue);
+        }
+
+        return queue;
+    }
+
+    removeQueue(driveId: string, documentId?: string) {
+        const queueId = this.getQueueId(driveId, documentId);
+        this.queues = this.queues.filter((q) => q.getId() !== queueId);
+        this.emit("queueRemoved", queueId)
+    }
+
+    getQueueByIndex(index: number) {
+        const queue = this.queues[index];
+        if (queue) {
+            return queue;
+        }
+
+        return null;
+    }
+
+    getQueues() {
+        return Object.keys(new Array(this.queues));
+    }
+
+    private retryNextJob() {
+        const retry = this.timeout === 0 && typeof setImmediate !== "undefined" ? setImmediate : (fn: () => void) => setTimeout(fn, this.timeout);
+        return retry(() => this.processNextJob());
+    }
+
+    private async processNextJob() {
+        if (!this.delegate) {
+            throw new Error("No server delegate defined");
+        }
+
+        if (this.queues.length === 0) {
+            this.retryNextJob();
+            return;
+        }
+
+        const queue = this.queues[this.ticker];
+        this.ticker = this.ticker === this.queues.length - 1 ? 0 : this.ticker + 1;
+        if (!queue) {
+            this.ticker = 0;
+            this.retryNextJob();
+            return;
+        }
+
+        const amountOfJobs = await queue.amountOfJobs();
+        if (amountOfJobs === 0) {
+            this.retryNextJob();
+            return;
+        }
+
+        const isBlocked = await queue.isBlocked();
+        if (isBlocked) {
+            this.retryNextJob();
+            return;
+        }
+
+        await queue.setBlocked(true);
+        const nextJob = await queue.getNextJob();
+        if (!nextJob) {
+            this.retryNextJob();
+            return;
+        }
+
+        try {
+            const result = await this.delegate.processOperationJob(nextJob);
+            await queue.setResult(nextJob.jobId, result);
+
+            // unblock the document queues of each add_file operation
+            const addFileOperations = nextJob.operations.filter((op) => op.type === "ADD_FILE");
+            if (addFileOperations.length > 0) {
+                addFileOperations.map(async (addFileOp) => {
+                    const documentQueue = this.getQueue(nextJob.driveId, (addFileOp.input as AddFileInput).id);
+                    await documentQueue.removeDependencies(nextJob);
+                });
+            }
+
+            this.emit("jobCompleted", nextJob, result);
+        } catch (e) {
+            this.emit("jobFailed", nextJob, e as Error);
+        } finally {
+            queue.setBlocked(false);
+            await this.processNextJob();
+        }
+    }
+
+    protected emit<K extends keyof QueueEvents>(
+        event: K,
+        ...args: Parameters<QueueEvents[K]>
+    ) {
+        this.emitter.emit(event, ...args);
+    }
+    on<K extends keyof QueueEvents>(this: this, event: K, cb: QueueEvents[K]): Unsubscribe {
+        return this.emitter.on(event, cb);
+    }
+
+    protected getQueueId(driveId: string, documentId?: string) {
+        return `${driveId}${documentId ? `:${documentId}` : ''}`;
+    }
+}

package/src/queue/redis.ts

@@ -0,0 +1,143 @@
+import { RedisClientType } from "redis";
+import { IJob, IQueue, IQueueManager, OperationJob } from "./types";
+import { BaseQueueManager } from "./base";
+
+export class RedisQueue<T, R> implements IQueue<T, R> {
+    private id: string;
+    private client: RedisClientType;
+
+    constructor(id: string, client: RedisClientType) {
+        this.client = client;
+        this.id = id;
+        this.client.hSet("queues", id, "true");
+
+    }
+
+    async setResult(jobId: string, result: any): Promise<void> {
+        await this.client.hSet(this.id + "-results", jobId, JSON.stringify(result));
+    }
+    async getResult(jobId: string): Promise<any> {
+        const results = await this.client.hGet(this.id + "-results", jobId);
+        if (!results) {
+            return null;
+        }
+        return JSON.parse(results);
+    }
+
+    async addJob(data: any) {
+        await this.client.lPush(this.id + "-jobs", JSON.stringify(data));
+    }
+
+    async getNextJob() {
+        const job = await this.client.rPop(this.id + "-jobs");
+        if (!job) {
+            return null;
+        }
+        return JSON.parse(job);
+    }
+
+    async amountOfJobs() {
+        return this.client.lLen(this.id + "-jobs");
+    }
+
+    async setBlocked(blocked: boolean) {
+        if (blocked) {
+            await this.client.hSet(this.id, "blocked", "true");
+        } else {
+            await this.client.hDel(this.id, "blocked");
+        }
+    }
+
+    async isBlocked() {
+        const blockedResult = await this.client.hGet(this.id, "blocked");
+        if (blockedResult) {
+            return true;
+        }
+
+        return false;
+    }
+
+    getId() {
+        return this.id;
+    }
+
+    async getJobs() {
+        const entries = await this.client.lRange(this.id + "-jobs", 0, -1)
+        return entries.map(e => JSON.parse(e));
+    }
+
+    async addDependencies(job: IJob<OperationJob>) {
+        if (await this.hasDependency(job)) {
+            return;
+        }
+        await this.client.lPush(this.id + "-deps", JSON.stringify(job));
+        await this.setBlocked(true);
+    }
+
+    async hasDependency(job: IJob<OperationJob>) {
+        const deps = await this.client.lRange(this.id + "-deps", 0, -1);
+        return deps.some(d => d === JSON.stringify(job));
+    }
+
+    async removeDependencies(job: IJob<OperationJob>) {
+        const allDeps1 = await this.client.lLen(this.id + "-deps");
+        await this.client.lRem(this.id + "-deps", 1, JSON.stringify(job));
+        const allDeps = await this.client.lLen(this.id + "-deps");
+        if (allDeps > 0) {
+            await this.setBlocked(true);
+        } else {
+            await this.setBlocked(false);
+        }
+    }
+
+    async isDeleted() {
+        const deleted = await this.client.hGet(this.id, "deleted");
+        return deleted === "true";
+    }
+
+    async setDeleted(deleted: boolean) {
+        if (deleted) {
+            await this.client.hSet(this.id, "deleted", "true");
+        } else {
+            await this.client.hDel(this.id, "deleted");
+        }
+    }
+}
+
+export class RedisQueueManager extends BaseQueueManager implements IQueueManager {
+
+    private client: RedisClientType;
+
+    constructor(workers = 3, timeout = 0, client: RedisClientType) {
+        super(workers, timeout);
+        this.client = client;
+    }
+
+    async init(delegate: IServerDelegate, onError: (error: Error) => void): Promise<void> {
+        await super.init(delegate, onError);
+        // load all queues
+        const queues = await this.client.hGetAll("queues");
+        for (const queueId in queues) {
+            this.queues.push(new RedisQueue(queueId, this.client));
+        }
+    }
+
+    getQueue(driveId: string, documentId?: string) {
+        const queueId = this.getQueueId(driveId, documentId);
+        let queue = this.queues.find((q) => q.getId() === queueId);
+
+        if (!queue) {
+            queue = new RedisQueue(queueId, this.client);
+            this.queues.push(queue);
+        }
+
+        return queue;
+    }
+
+    removeQueue(driveId: string, documentId?: string | undefined): void {
+        super.removeQueue(driveId, documentId);
+
+        const queueId = this.getQueueId(driveId, documentId);
+        this.client.hDel("queues", queueId);
+    }
+}

package/src/queue/types.ts

@@ -0,0 +1,57 @@
+import { Operation } from "document-model/document";
+import { IOperationResult } from "../server";
+import type { Unsubscribe } from "nanoevents";
+
+export type OperationJob = {
+    driveId: string;
+    documentId?: string
+    operations: Operation[]
+    forceSync?: boolean
+}
+
+export type JobId = string;
+
+export interface QueueEvents {
+    jobCompleted: (job: IJob<OperationJob>, result: IOperationResult) => void;
+    jobFailed: (job: IJob<OperationJob>, error: Error) => void;
+}
+
+export interface IServerDelegate {
+    checkDocumentExists: (driveId: string, documentId: string) => Promise<boolean>;
+    processOperationJob: (job: OperationJob) => Promise<IOperationResult>;
+}
+
+export interface IQueueManager {
+    addJob(job: OperationJob): Promise<JobId>;
+    getResult(driveId: string, documentId: string, jobId: JobId): Promise<IOperationResult | undefined>;
+    getQueue(driveId: string, document?: string): IQueue<OperationJob, IOperationResult>;
+    removeQueue(driveId: string, documentId?: string): void;
+    getQueueByIndex(index: number): IQueue<OperationJob, IOperationResult> | null;
+    getQueues(): string[];
+    init(delegate: IServerDelegate, onError: (error: Error) => void): Promise<void>;
+    on<K extends keyof QueueEvents>(
+        this: this,
+        event: K,
+        cb: QueueEvents[K]
+    ): Unsubscribe;
+}
+
+export type IJob<T> = { jobId: JobId } & T;
+
+export interface IQueue<T, R> {
+    addJob(data: IJob<T>): Promise<void>;
+    getNextJob(): Promise<IJob<T> | undefined>;
+    amountOfJobs(): Promise<number>;
+    getId(): string;
+    setBlocked(blocked: boolean): Promise<void>;
+    isBlocked(): Promise<boolean>;
+    isDeleted(): Promise<boolean>;
+    setDeleted(deleted: boolean): Promise<void>;
+    setResult(jobId: JobId, result: R): Promise<void>;
+    getResult(jobId: JobId): Promise<R | undefined>;
+    getJobs(): Promise<IJob<T>[]>;
+    addDependencies(job: IJob<OperationJob>): Promise<void>;
+    removeDependencies(job: IJob<OperationJob>): Promise<void>;
+}
+
+export type IJobQueue = IQueue<OperationJob, IOperationResult>;

package/src/server/index.ts

@@ -68,6 +68,8 @@ import {
     type SynchronizationUnit
 } from './types';
 import { filterOperationsByRevision } from './utils';
+import { BaseQueueManager } from '../queue/base';
+import { IQueueManager } from '../queue/types';
 
 export * from './listener';
 export type * from './types';
@@ -86,16 +88,20 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
     >();
     private syncStatus = new Map<DocumentDriveState['id'], SyncStatus>();
 
+    private queueManager: IQueueManager;
+
     constructor(
         documentModels: DocumentModel[],
         storage: IDriveStorage = new MemoryStorage(),
-        cache: ICache = new InMemoryCache()
+        cache: ICache = new InMemoryCache(),
+        queueManager: IQueueManager = new BaseQueueManager(),
     ) {
         super();
         this.listenerStateManager = new ListenerManager(this);
         this.documentModels = documentModels;
         this.storage = storage;
         this.cache = cache;
+        this.queueManager = queueManager;
     }
 
     private updateSyncStatus(
@@ -223,6 +229,17 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
             });
         }
 
+        await this.queueManager.init({
+            checkDocumentExists: (driveId: string, documentId: string): Promise<boolean> => this.storage.checkDocumentExists(driveId, documentId),
+            processOperationJob: ({ driveId, documentId, operations, forceSync }) => documentId ?
+                this.addOperations(driveId, documentId, operations, forceSync)
+                : this.addDriveOperations(driveId, operations as Operation<DocumentDriveAction | BaseAction>[], forceSync)
+
+        }, error => {
+            logger.error(`Error initializing queue manager`, error);
+            errors.push(error);
+        })
+
         // if network connect comes online then
         // triggers the listeners update
         if (typeof window !== 'undefined') {
@@ -438,6 +455,7 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
 
         await this.storage.createDrive(id, document);
         await this._initializeDrive(id);
+
         return document;
     }
 
@@ -805,6 +823,44 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         }
     }
 
+
+    async queueOperations(drive: string,
+        id: string,
+        operations: Operation[],
+        forceSync = true) {
+        // try {
+        //     await this.getDocument(drive, id);
+        // } catch (error) {
+        //     logger.error('Error getting document', error);
+        //     throw error;
+        // }
+
+        try {
+            const jobId = await this.queueManager.addJob({ driveId: drive, documentId: id, operations, forceSync });
+
+            return new Promise((resolve, reject) => {
+                const unsubscribe = this.queueManager.on('jobCompleted', (job, result) => {
+                    if (job.jobId === jobId) {
+                        unsubscribe();
+                        unsubscribeError();
+                        resolve(result);
+                    }
+                });
+                const unsubscribeError = this.queueManager.on('jobFailed', (job, error) => {
+                    console.log("test")
+                    if (job.jobId === jobId) {
+                        unsubscribe();
+                        unsubscribeError();
+                        reject(error);
+                    }
+                });
+            })
+        } catch (error) {
+            logger.error('Error adding job', error);
+            throw error;
+        }
+    }
+
     async addOperations(
         drive: string,
         id: string,
@@ -958,6 +1014,31 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         }
     }
 
+    async queueDriveOperations(
+        drive: string,
+        operations: Operation<DocumentDriveAction | BaseAction>[],
+        forceSync = true
+    ): Promise<IOperationResult> {
+        const jobId = await this.queueManager.addJob({ driveId: drive, operations, forceSync });
+        return new Promise((resolve, reject) => {
+            const unsubscribe = this.queueManager.on('jobCompleted', (job, result) => {
+                if (job.jobId === jobId) {
+                    unsubscribe();
+                    unsubscribeError();
+                    resolve(result);
+                }
+            });
+            const unsubscribeError = this.queueManager.on('jobFailed', (job, error) => {
+                if (job.jobId === jobId) {
+                    unsubscribe();
+                    unsubscribeError();
+                    reject(error);
+                }
+            });
+
+        })
+    }
+
     async addDriveOperations(
         drive: string,
         operations: Operation<DocumentDriveAction | BaseAction>[],

package/src/server/utils.ts

@@ -32,3 +32,4 @@ export function filterOperationsByRevision(
         return acc;
     }, operations);
 }
+

package/src/storage/browser.ts

@@ -27,6 +27,13 @@ export class BrowserStorage implements IDriveStorage {
         return args.join(BrowserStorage.SEP);
     }
 
+    async checkDocumentExists(drive: string, id: string): Promise<boolean> {
+        const document = await (
+            await this.db
+        ).getItem<Document>(this.buildKey(drive, id));
+        return document !== undefined;
+    }
+
     async getDocuments(drive: string) {
         const keys = await (await this.db).keys();
         const driveKey = `${drive}${BrowserStorage.SEP}`;

package/src/storage/filesystem.ts

@@ -77,6 +77,11 @@ export class FilesystemStorage implements IDriveStorage {
         return documents;
     }
 
+    checkDocumentExists(drive: string, id: string): Promise<boolean> {
+        const documentExists = existsSync(this._buildDocumentPath(drive, id));
+        return Promise.resolve(documentExists);
+    }
+
     async getDocument(drive: string, id: string) {
         try {
             const content = readFileSync(this._buildDocumentPath(drive, id), {

package/src/storage/memory.ts

@@ -18,6 +18,10 @@ export class MemoryStorage implements IDriveStorage {
         this.drives = {};
     }
 
+    checkDocumentExists(drive: string, id: string): Promise<boolean> {
+        return Promise.resolve(this.documents[drive]?.[id] !== undefined)
+    }
+
     async getDocuments(drive: string) {
         return Object.keys(this.documents[drive] ?? {});
     }
@@ -121,6 +125,7 @@ export class MemoryStorage implements IDriveStorage {
 
     async createDrive(id: string, drive: DocumentDriveStorage) {
         this.drives[id] = drive;
+        this.documents[id] = {};
         const { slug } = drive.initialState.state.global;
         if (slug) {
             this.slugToDriveId[slug] = id;

package/src/storage/prisma.ts

@@ -285,6 +285,16 @@ export class PrismaStorage implements IDriveStorage {
         return docs.map(doc => doc.id);
     }
 
+    async checkDocumentExists(driveId: string, id: string) {
+        const count = await this.db.document.count({
+            where: {
+                id: id,
+                driveId: driveId
+            },
+        });
+        return count > 0;
+    }
+
     async getDocument(driveId: string, id: string, tx?: Transaction) {
         const result = await (tx ?? this.db).document.findFirst({
             where: {

package/src/storage/sequelize.ts

@@ -275,6 +275,21 @@ export class SequelizeStorage implements IDriveStorage {
         return ids;
     }
 
+    async checkDocumentExists(driveId: string, id: string): Promise<boolean> {
+        const Document = this.db.models.document;
+        if (!Document) {
+            throw new Error('Document model not found');
+        }
+        const count = await Document.count({
+            where: {
+                id: id,
+                driveId: driveId
+            },
+        })
+
+        return count > 0;
+    }
+
     async getDocument(driveId: string, id: string) {
         const Document = this.db.models.document;
         if (!Document) {

package/src/storage/types.ts

@@ -17,6 +17,7 @@ export type DocumentStorage<D extends Document = Document> = Omit<
 export type DocumentDriveStorage = DocumentStorage<DocumentDriveDocument>;
 
 export interface IStorage {
+    checkDocumentExists(drive: string, id: string): Promise<boolean>;
     getDocuments: (drive: string) => Promise<string[]>;
     getDocument(drive: string, id: string): Promise<DocumentStorage>;
     createDocument(