document-drive 1.0.0-alpha.9 → 1.0.0-alpha.91

package/LICENSE

@@ -56,6 +56,7 @@ this license.
   The precise terms and conditions for copying, distribution and
 modification follow.
 
+
                        TERMS AND CONDITIONS
 
   0. Definitions.

package/README.md

@@ -0,0 +1 @@
+# Document Drive

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "document-drive",
-    "version": "1.0.0-alpha.9",
+    "version": "1.0.0-alpha.91",
     "license": "AGPL-3.0-only",
     "type": "module",
     "module": "./src/index.ts",
@@ -13,8 +13,14 @@
         "./storage/filesystem": "./src/storage/filesystem.ts",
         "./storage/memory": "./src/storage/memory.ts",
         "./storage/prisma": "./src/storage/prisma.ts",
+        "./cache/redis": "./src/cache/redis.ts",
+        "./cache/memory": "./src/cache/memory.ts",
+        "./queue/redis": "./src/queue/redis.ts",
+        "./queue/base": "./src/queue/base.ts",
         "./utils": "./src/utils/index.ts",
-        "./utils/graphql": "./src/utils/graphql.ts"
+        "./utils/graphql": "./src/utils/graphql.ts",
+        "./utils/migrations": "./src/utils/migrations.ts",
+        "./logger": "./src/utils/logger.ts"
     },
     "files": [
         "./src"
@@ -25,48 +31,56 @@
         "lint:fix": "eslint src --ext .js,.jsx,.ts,.tsx --fix",
         "format": "prettier . --write",
         "release": "semantic-release",
-        "test": "vitest run --coverage",
+        "test": "vitest run --coverage --exclude \"test/flaky/**\"",
         "test:watch": "vitest watch"
     },
     "peerDependencies": {
-        "@prisma/client": "5.8.1",
-        "document-model": "^1.0.29",
-        "document-model-libs": "^1.1.44",
+        "document-model": "^1.7.0",
+        "document-model-libs": "^1.57.0"
+    },
+    "optionalDependencies": {
+        "@prisma/client": "^5.18.0",
         "localforage": "^1.10.0",
-        "sequelize": "^6.35.2",
+        "redis": "^4.6.15",
+        "sequelize": "^6.37.3",
         "sqlite3": "^5.1.7"
     },
     "dependencies": {
-        "graphql": "^16.8.1",
+        "exponential-backoff": "^3.1.1",
+        "graphql": "^16.9.0",
         "graphql-request": "^6.1.0",
         "json-stringify-deterministic": "^1.0.12",
         "nanoevents": "^9.0.0",
-        "sanitize-filename": "^1.6.3"
+        "sanitize-filename": "^1.6.3",
+        "uuid": "^9.0.1"
     },
     "devDependencies": {
         "@commitlint/cli": "^18.6.1",
-        "@commitlint/config-conventional": "^18.6.2",
-        "@prisma/client": "5.8.1",
+        "@commitlint/config-conventional": "^18.6.3",
+        "@prisma/client": "5.17.0",
         "@semantic-release/changelog": "^6.0.3",
         "@semantic-release/git": "^10.0.1",
         "@total-typescript/ts-reset": "^0.5.1",
-        "@types/node": "^20.11.16",
-        "@typescript-eslint/eslint-plugin": "^6.18.1",
-        "@typescript-eslint/parser": "^6.18.1",
-        "@vitest/coverage-v8": "^0.34.6",
-        "document-model": "^1.0.30",
-        "document-model-libs": "^1.1.51",
-        "eslint": "^8.56.0",
+        "@types/node": "^20.14.11",
+        "@types/uuid": "^9.0.8",
+        "@typescript-eslint/eslint-plugin": "^6.21.0",
+        "@typescript-eslint/parser": "^6.21.0",
+        "@vitest/coverage-v8": "^2.0.5",
+        "document-model": "^1.7.0",
+        "document-model-libs": "^1.70.0",
+        "eslint": "^8.57.0",
         "eslint-config-prettier": "^9.1.0",
-        "fake-indexeddb": "^5.0.1",
+        "fake-indexeddb": "^5.0.2",
         "localforage": "^1.10.0",
-        "msw": "^2.1.2",
-        "prettier": "^3.1.1",
+        "msw": "^2.3.1",
+        "prettier": "^3.3.3",
         "prettier-plugin-organize-imports": "^3.2.4",
-        "semantic-release": "^23.0.2",
-        "sequelize": "^6.35.2",
+        "prisma": "^5.18.0",
+        "semantic-release": "^23.1.1",
+        "sequelize": "^6.37.2",
         "sqlite3": "^5.1.7",
-        "typescript": "^5.3.2",
-        "vitest": "^1.2.2"
-    }
+        "typescript": "^5.5.3",
+        "vitest": "^2.0.5"
+    },
+    "packageManager": "pnpm@9.1.4+sha256.30a1801ac4e723779efed13a21f4c39f9eb6c9fbb4ced101bce06b422593d7c9"
 }

package/src/cache/index.ts

@@ -0,0 +1,2 @@
+export * from './memory';
+export * from './types';

package/src/cache/memory.ts

@@ -0,0 +1,33 @@
+import { Document } from 'document-model/document';
+import { ICache } from './types';
+
+class InMemoryCache implements ICache {
+    private cache = new Map<string, Map<string, Document>>();
+
+    async setDocument(drive: string, id: string, document: Document) {
+        const global = document.operations.global.map(e => {
+            delete e.resultingState;
+            return e;
+        });
+        const local = document.operations.local.map(e => {
+            delete e.resultingState;
+            return e;
+        });
+        const doc = { ...document, operations: { global, local } };
+        if (!this.cache.has(drive)) {
+            this.cache.set(drive, new Map());
+        }
+        this.cache.get(drive)?.set(id, doc);
+        return true;
+    }
+
+    async deleteDocument(drive: string, id: string) {
+        return this.cache.get(drive)?.delete(id) ?? false;
+    }
+
+    async getDocument(drive: string, id: string) {
+        return this.cache.get(drive)?.get(id);
+    }
+}
+
+export default InMemoryCache;

package/src/cache/redis.ts

@@ -0,0 +1,56 @@
+import { Document } from 'document-model/document';
+import type { RedisClientType } from 'redis';
+import { ICache } from './types';
+
+class RedisCache implements ICache {
+    private redis: RedisClientType;
+    private timeoutInSeconds: number;
+
+    constructor(
+        redis: RedisClientType,
+        timeoutInSeconds: number | undefined = 5 * 60
+    ) {
+        this.redis = redis;
+        this.timeoutInSeconds = timeoutInSeconds;
+    }
+
+    private static _getId(drive: string, id: string) {
+        return `cache:${drive}:${id}`;
+    }
+
+    async setDocument(drive: string, id: string, document: Document) {
+        const global = document.operations.global.map(e => {
+            delete e.resultingState;
+            return e;
+        });
+        const local = document.operations.local.map(e => {
+            delete e.resultingState;
+            return e;
+        });
+        const doc = { ...document, operations: { global, local } };
+        const redisId = RedisCache._getId(drive, id);
+        const result = await this.redis.set(redisId, JSON.stringify(doc), {
+            EX: this.timeoutInSeconds ? this.timeoutInSeconds : undefined
+        });
+
+        if (result === 'OK') {
+            return true;
+        }
+
+        return false;
+    }
+
+    async getDocument(drive: string, id: string) {
+        const redisId = RedisCache._getId(drive, id);
+        const doc = await this.redis.get(redisId);
+
+        return doc ? (JSON.parse(doc) as Document) : undefined;
+    }
+
+    async deleteDocument(drive: string, id: string) {
+        const redisId = RedisCache._getId(drive, id);
+        return (await this.redis.del(redisId)) > 0;
+    }
+}
+
+export default RedisCache;

package/src/cache/types.ts

@@ -0,0 +1,13 @@
+import type { Document } from 'document-model/document';
+
+export interface ICache {
+    setDocument(
+        drive: string,
+        id: string,
+        document: Document
+    ): Promise<boolean>;
+    getDocument(drive: string, id: string): Promise<Document | undefined>;
+
+    // @returns — true if a document existed and has been removed, or false if the document is not cached.
+    deleteDocument(drive: string, id: string): Promise<boolean>;
+}

package/src/index.ts

@@ -1,3 +1,4 @@
 export * from './server';
+export * from './server/error';
 export * from './storage';
 export * from './utils';

package/src/queue/base.ts

@@ -0,0 +1,328 @@
+import {
+    AddFileInput,
+    DeleteNodeInput
+} from 'document-model-libs/document-drive';
+import { Action } from 'document-model/document';
+import { Unsubscribe, createNanoEvents } from 'nanoevents';
+import { generateUUID } from '../utils';
+import { logger } from '../utils/logger';
+import {
+    IJob,
+    IJobQueue,
+    IQueue,
+    IQueueManager,
+    IServerDelegate,
+    Job,
+    JobId,
+    QueueEvents,
+    isOperationJob
+} from './types';
+
+export class MemoryQueue<T, R> implements IQueue<T, R> {
+    private id: string;
+    private blocked = false;
+    private deleted = false;
+    private items: IJob<T>[] = [];
+    private dependencies = new Array<IJob<Job>>();
+
+    constructor(id: string) {
+        this.id = id;
+    }
+
+    async setDeleted(deleted: boolean) {
+        this.deleted = deleted;
+    }
+
+    async isDeleted() {
+        return this.deleted;
+    }
+
+    async addJob(data: IJob<T>) {
+        this.items.push(data);
+        return Promise.resolve();
+    }
+
+    async getNextJob() {
+        const job = this.items.shift();
+        return Promise.resolve(job);
+    }
+
+    async amountOfJobs() {
+        return Promise.resolve(this.items.length);
+    }
+
+    getId() {
+        return this.id;
+    }
+
+    async setBlocked(blocked: boolean) {
+        this.blocked = blocked;
+    }
+
+    async isBlocked() {
+        return this.blocked;
+    }
+
+    async getJobs() {
+        return this.items;
+    }
+
+    async addDependencies(job: IJob<Job>) {
+        if (!this.dependencies.find(j => j.jobId === job.jobId)) {
+            this.dependencies.push(job);
+        }
+        if (!this.isBlocked()) {
+            this.setBlocked(true);
+        }
+    }
+
+    async removeDependencies(job: IJob<Job>) {
+        this.dependencies = this.dependencies.filter(
+            j => j.jobId !== job.jobId && j.driveId !== job.driveId
+        );
+        if (this.dependencies.length === 0) {
+            await this.setBlocked(false);
+        }
+    }
+}
+
+export class BaseQueueManager implements IQueueManager {
+    protected emitter = createNanoEvents<QueueEvents>();
+    protected ticker = 0;
+    protected queues: IJobQueue[] = [];
+    protected workers: number;
+    protected timeout: number;
+    private delegate: IServerDelegate | undefined;
+
+    constructor(workers = 3, timeout = 0) {
+        this.workers = workers;
+        this.timeout = timeout;
+    }
+
+    async init(
+        delegate: IServerDelegate,
+        onError: (error: Error) => void
+    ): Promise<void> {
+        this.delegate = delegate;
+        for (let i = 0; i < this.workers; i++) {
+            setTimeout(
+                () => this.processNextJob.bind(this)().catch(onError),
+                100 * i
+            );
+        }
+        return Promise.resolve();
+    }
+
+    async addJob(job: Job): Promise<JobId> {
+        if (!this.delegate) {
+            throw new Error('No server delegate defined');
+        }
+
+        const jobId = generateUUID();
+        const queue = this.getQueue(job.driveId, job.documentId);
+
+        if (await queue.isDeleted()) {
+            throw new Error('Queue is deleted');
+        }
+
+        // checks if the job is for a document that doesn't exist in storage yet
+        const newDocument =
+            job.documentId &&
+            !(await this.delegate.checkDocumentExists(
+                job.driveId,
+                job.documentId
+            ));
+        // if it is a new document and queue is not yet blocked then
+        // blocks it so the jobs are not processed until it's ready
+        if (newDocument && !(await queue.isBlocked())) {
+            await queue.setBlocked(true);
+
+            // checks if there any job in the queue adding the file and adds as dependency
+            const driveQueue = this.getQueue(job.driveId);
+            const jobs = await driveQueue.getJobs();
+            for (const driveJob of jobs) {
+                const actions = isOperationJob(driveJob)
+                    ? driveJob.operations
+                    : driveJob.actions;
+                const op = actions.find((j: Action) => {
+                    const input = j.input as AddFileInput;
+                    return j.type === 'ADD_FILE' && input.id === job.documentId;
+                });
+                if (op) {
+                    await queue.addDependencies(driveJob);
+                }
+            }
+        }
+
+        // if it has ADD_FILE operations then adds the job as
+        // a dependency to the corresponding document queues
+        const actions = isOperationJob(job) ? job.operations : job.actions;
+        const addFileOps = actions.filter((j: Action) => j.type === 'ADD_FILE');
+        for (const addFileOp of addFileOps) {
+            const input = addFileOp.input as AddFileInput;
+            const q = this.getQueue(job.driveId, input.id);
+            await q.addDependencies({ jobId, ...job });
+        }
+
+        // remove document if operations contains delete_node
+        const removeFileOps = actions.filter(
+            (j: Action) => j.type === 'DELETE_NODE'
+        );
+        for (const removeFileOp of removeFileOps) {
+            const input = removeFileOp.input as DeleteNodeInput;
+            const queue = this.getQueue(job.driveId, input.id);
+            await queue.setDeleted(true);
+        }
+        await queue.addJob({ jobId, ...job });
+
+        return jobId;
+    }
+
+    getQueue(driveId: string, documentId?: string) {
+        const queueId = this.getQueueId(driveId, documentId);
+        let queue = this.queues.find(q => q.getId() === queueId);
+
+        if (!queue) {
+            queue = new MemoryQueue(queueId);
+            this.queues.push(queue);
+        }
+
+        return queue;
+    }
+
+    removeQueue(driveId: string, documentId?: string) {
+        const queueId = this.getQueueId(driveId, documentId);
+        this.queues = this.queues.filter(q => q.getId() !== queueId);
+        this.emit('queueRemoved', queueId);
+    }
+
+    getQueueByIndex(index: number) {
+        const queue = this.queues[index];
+        if (queue) {
+            return queue;
+        }
+
+        return null;
+    }
+
+    getQueues() {
+        return this.queues.map(q => q.getId());
+    }
+
+    private retryNextJob(timeout?: number) {
+        const _timeout = timeout !== undefined ? timeout : this.timeout;
+        const retry =
+            _timeout === 0 && typeof setImmediate !== 'undefined'
+                ? setImmediate
+                : (fn: () => void) => setTimeout(fn, _timeout);
+        return retry(() => this.processNextJob());
+    }
+
+    private async findFirstNonEmptyQueue(
+        ticker: number
+    ): Promise<number | null> {
+        const numQueues = this.queues.length;
+
+        for (let i = 0; i < numQueues; i++) {
+            const index = (ticker + i) % numQueues;
+            const queue = this.queues[index];
+            if (queue && (await queue.amountOfJobs()) > 0) {
+                return index;
+            }
+        }
+        return null;
+    }
+
+    private async processNextJob() {
+        if (!this.delegate) {
+            throw new Error('No server delegate defined');
+        }
+
+        if (this.queues.length === 0) {
+            this.retryNextJob();
+            return;
+        }
+
+        const queue = this.queues[this.ticker];
+        if (!queue) {
+            this.ticker = 0;
+            this.retryNextJob();
+            return;
+        }
+
+        // if no jobs in the current queue then looks for the
+        // next queue with jobs. If no jobs in any queue then
+        // retries after a timeout
+        const amountOfJobs = await queue.amountOfJobs();
+        if (amountOfJobs === 0) {
+            const nextTicker = await this.findFirstNonEmptyQueue(this.ticker);
+            if (nextTicker !== null) {
+                this.ticker = nextTicker;
+                this.retryNextJob(0);
+            } else {
+                this.retryNextJob();
+            }
+            return;
+        }
+
+        this.ticker =
+            this.ticker === this.queues.length - 1 ? 0 : this.ticker + 1;
+
+        const isBlocked = await queue.isBlocked();
+        if (isBlocked) {
+            this.retryNextJob();
+            return;
+        }
+
+        await queue.setBlocked(true);
+        const nextJob = await queue.getNextJob();
+        if (!nextJob) {
+            this.retryNextJob();
+            return;
+        }
+
+        try {
+            const result = await this.delegate.processJob(nextJob);
+
+            // unblock the document queues of each add_file operation
+            const actions = isOperationJob(nextJob)
+                ? nextJob.operations
+                : nextJob.actions;
+            const addFileActions = actions.filter(op => op.type === 'ADD_FILE');
+            if (addFileActions.length > 0) {
+                for (const addFile of addFileActions) {
+                    const documentQueue = this.getQueue(
+                        nextJob.driveId,
+                        (addFile.input as AddFileInput).id
+                    );
+                    await documentQueue.removeDependencies(nextJob);
+                }
+            }
+            this.emit('jobCompleted', nextJob, result);
+        } catch (e) {
+            logger.error(`job failed`, e);
+            this.emit('jobFailed', nextJob, e as Error);
+        } finally {
+            await queue.setBlocked(false);
+            this.retryNextJob(0);
+        }
+    }
+
+    protected emit<K extends keyof QueueEvents>(
+        event: K,
+        ...args: Parameters<QueueEvents[K]>
+    ) {
+        this.emitter.emit(event, ...args);
+    }
+    on<K extends keyof QueueEvents>(
+        this: this,
+        event: K,
+        cb: QueueEvents[K]
+    ): Unsubscribe {
+        return this.emitter.on(event, cb);
+    }
+
+    protected getQueueId(driveId: string, documentId?: string) {
+        return `queue:${driveId}${documentId ? `:${documentId}` : ''}`;
+    }
+}

package/src/queue/index.ts

@@ -0,0 +1,2 @@
+export * from './base';
+export * from './types';