document-drive 1.0.0-alpha.9 → 1.0.0-experimental.2

package/LICENSE

@@ -56,6 +56,7 @@ this license.
   The precise terms and conditions for copying, distribution and
 modification follow.
 
+
                        TERMS AND CONDITIONS
 
   0. Definitions.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "document-drive",
-    "version": "1.0.0-alpha.9",
+    "version": "1.0.0-experimental.2",
     "license": "AGPL-3.0-only",
     "type": "module",
     "module": "./src/index.ts",
@@ -13,8 +13,11 @@
         "./storage/filesystem": "./src/storage/filesystem.ts",
         "./storage/memory": "./src/storage/memory.ts",
         "./storage/prisma": "./src/storage/prisma.ts",
+        "./cache/redis": "./src/cache/redis.ts",
+        "./cache/memory": "./src/cache/memory.ts",
         "./utils": "./src/utils/index.ts",
-        "./utils/graphql": "./src/utils/graphql.ts"
+        "./utils/graphql": "./src/utils/graphql.ts",
+        "./logger": "./src/utils/logger.ts"
     },
     "files": [
         "./src"
@@ -29,14 +32,18 @@
         "test:watch": "vitest watch"
     },
     "peerDependencies": {
-        "@prisma/client": "5.8.1",
-        "document-model": "^1.0.29",
-        "document-model-libs": "^1.1.44",
+        "document-model": "1.1.0-experimental.1",
+        "document-model-libs": "^1.37.0"
+    },
+    "optionalDependencies": {
+        "@prisma/client": "5.11.0",
         "localforage": "^1.10.0",
+        "redis": "^4.6.13",
         "sequelize": "^6.35.2",
         "sqlite3": "^5.1.7"
     },
     "dependencies": {
+        "exponential-backoff": "^3.1.1",
         "graphql": "^16.8.1",
         "graphql-request": "^6.1.0",
         "json-stringify-deterministic": "^1.0.12",
@@ -45,28 +52,29 @@
     },
     "devDependencies": {
         "@commitlint/cli": "^18.6.1",
-        "@commitlint/config-conventional": "^18.6.2",
-        "@prisma/client": "5.8.1",
+        "@commitlint/config-conventional": "^18.6.3",
+        "@prisma/client": "5.11.0",
         "@semantic-release/changelog": "^6.0.3",
         "@semantic-release/git": "^10.0.1",
         "@total-typescript/ts-reset": "^0.5.1",
-        "@types/node": "^20.11.16",
-        "@typescript-eslint/eslint-plugin": "^6.18.1",
-        "@typescript-eslint/parser": "^6.18.1",
-        "@vitest/coverage-v8": "^0.34.6",
-        "document-model": "^1.0.30",
-        "document-model-libs": "^1.1.51",
-        "eslint": "^8.56.0",
+        "@types/node": "^20.12.7",
+        "@typescript-eslint/eslint-plugin": "^6.21.0",
+        "@typescript-eslint/parser": "^6.21.0",
+        "@vitest/coverage-v8": "^1.4.0",
+        "document-model": "1.1.0-experimental.1",
+        "document-model-libs": "^1.37.0",
+        "eslint": "^8.57.0",
         "eslint-config-prettier": "^9.1.0",
-        "fake-indexeddb": "^5.0.1",
+        "fake-indexeddb": "^5.0.2",
         "localforage": "^1.10.0",
-        "msw": "^2.1.2",
-        "prettier": "^3.1.1",
+        "msw": "^2.2.13",
+        "prettier": "^3.2.5",
         "prettier-plugin-organize-imports": "^3.2.4",
-        "semantic-release": "^23.0.2",
-        "sequelize": "^6.35.2",
+        "prisma": "^5.12.1",
+        "semantic-release": "^23.0.8",
+        "sequelize": "^6.37.2",
         "sqlite3": "^5.1.7",
-        "typescript": "^5.3.2",
-        "vitest": "^1.2.2"
+        "typescript": "^5.4.4",
+        "vitest": "^1.5.0"
     }
 }

package/src/cache/index.ts

@@ -0,0 +1,2 @@
+export * from './memory';
+export * from './types';

package/src/cache/memory.ts

@@ -0,0 +1,24 @@
+import { Document } from "document-model/document";
+import { ICache } from "./types";
+
+class InMemoryCache implements ICache {
+    private cache = new Map<string, Map<string, Document>>();
+
+    async setDocument(drive: string, id: string, document: Document) {
+        if (!this.cache.has(drive)) {
+            this.cache.set(drive, new Map());
+        }
+        this.cache.get(drive)?.set(id, document);
+        return true;
+    }
+
+    async deleteDocument(drive: string, id: string) {
+        return this.cache.get(drive)?.delete(id) ?? false;
+    }
+
+    async getDocument(drive: string, id: string) {
+        return this.cache.get(drive)?.get(id);
+    }
+}
+
+export default InMemoryCache;

package/src/cache/redis.ts

@@ -0,0 +1,29 @@
+import { Document } from "document-model/document";
+import { ICache } from "./types";
+import type { RedisClientType } from "redis";
+import { logger } from "../utils/logger";
+
+class RedisCache implements ICache {
+    private redis: RedisClientType;
+
+    constructor(redis: RedisClientType) {
+        this.redis = redis;
+        this.redis.flushAll().catch(logger.error);
+    }
+
+    async setDocument(drive: string, id: string, document: Document) {
+        return (await this.redis.hSet(drive, id, JSON.stringify(document))) > 0;
+    }
+
+    async getDocument(drive: string, id: string) {
+        const doc = await this.redis.hGet(drive, id);
+
+        return doc ? JSON.parse(doc) as Document : undefined;
+    }
+
+    async deleteDocument(drive: string, id: string) {
+        return (await this.redis.hDel(drive, id)) > 0;
+    }
+}
+
+export default RedisCache;

package/src/cache/types.ts

@@ -0,0 +1,9 @@
+import type { Document } from "document-model/document";
+
+export interface ICache {
+    setDocument(drive: string, id: string, document: Document): Promise<boolean>
+    getDocument(drive: string, id: string): Promise<Document | undefined>
+
+    // @returns — true if a document existed and has been removed, or false if the document is not cached.
+    deleteDocument(drive: string, id: string): Promise<boolean>
+}

package/src/queue/base.ts

@@ -0,0 +1,273 @@
+import { IJob, IJobQueue, IQueue, IQueueManager, IServerDelegate, JobId, OperationJob, QueueEvents } from "./types";
+import { generateUUID } from "../utils";
+import { IOperationResult } from "../server";
+import { createNanoEvents, Unsubscribe } from 'nanoevents';
+import { Operation } from "document-model/document";
+import { AddFileInput, DeleteNodeInput } from "document-model-libs/document-drive";
+
+export class MemoryQueue<T, R> implements IQueue<T, R> {
+    private id: string;
+    private blocked = false;
+    private deleted = false;
+    private items: IJob<T>[] = [];
+    private results = new Map<JobId, R>();
+    private dependencies = new Array<IJob<OperationJob>>();
+
+    constructor(id: string) {
+        this.id = id;
+    }
+
+    async setDeleted(deleted: boolean) {
+        this.deleted = deleted;
+    }
+
+    async isDeleted() {
+        return this.deleted;
+    }
+
+    async setResult(jobId: string, result: R): Promise<void> {
+        this.results.set(jobId, result);
+        return Promise.resolve();
+    }
+
+    async getResult(jobId: string): Promise<R | undefined> {
+        return Promise.resolve(this.results.get(jobId));
+    }
+
+    async addJob(data: IJob<T>) {
+        this.items.push(data);
+        return Promise.resolve();
+    }
+
+    async getNextJob() {
+        const job = this.items.shift();
+        return Promise.resolve(job);
+    }
+
+    async amountOfJobs() {
+        return Promise.resolve(this.items.length);
+    }
+
+    getId() {
+        return this.id;
+    }
+
+    async setBlocked(blocked: boolean) {
+        this.blocked = blocked;
+    }
+
+    async isBlocked() {
+        return this.blocked;
+    }
+
+    async getJobs() {
+        return this.items;
+    }
+
+    async addDependencies(job: IJob<OperationJob>) {
+        if (!this.dependencies.find(j => j.jobId === job.jobId)) {
+            this.dependencies.push(job);
+        }
+        if (!this.isBlocked()) {
+            this.setBlocked(true);
+        }
+    }
+
+    async removeDependencies(job: IJob<OperationJob>) {
+        this.dependencies = this.dependencies.filter((j) => j.jobId !== job.jobId && j.driveId !== job.driveId);
+        if (this.dependencies.length === 0) {
+            await this.setBlocked(false);
+        }
+    }
+}
+
+export class BaseQueueManager implements IQueueManager {
+    protected emitter = createNanoEvents<QueueEvents>();
+    protected ticker = 0;
+    protected queues: IJobQueue[] = [];
+    protected workers: number;
+    protected timeout: number;
+    private delegate: IServerDelegate | undefined;
+
+    constructor(workers = 3, timeout = 0) {
+        this.workers = workers;
+        this.timeout = timeout;
+    }
+
+    async init(delegate: IServerDelegate, onError: (error: Error) => void): Promise<void> {
+        this.delegate = delegate;
+        for (let i = 0; i < this.workers; i++) {
+            setTimeout(() => this.processNextJob.bind(this)().catch(onError), 100 * i);
+        }
+        return Promise.resolve()
+    }
+
+    async addJob(job: OperationJob): Promise<JobId> {
+        if (!this.delegate) {
+            throw new Error("No server delegate defined");
+        }
+
+        const jobId = generateUUID();
+        const queue = this.getQueue(job.driveId, job.documentId);
+
+        if (await queue.isDeleted()) {
+            throw new Error("Queue is deleted")
+        }
+
+        // checks if the job is for a document that doesn't exist in storage yet
+        const newDocument = job.documentId && !(await this.delegate.checkDocumentExists(job.driveId, job.documentId));
+        // if it is a new document and queue is not yet blocked then 
+        // blocks it so the jobs are not processed until it's ready
+        if (newDocument && !(await queue.isBlocked())) {
+            await queue.setBlocked(true);
+
+            // checks if there any job in the queue adding the file and adds as dependency
+            const driveQueue = this.getQueue(job.driveId);
+            const jobs = await driveQueue.getJobs();
+            for (const driveJob of jobs) {
+                const op = driveJob.operations.find((j: Operation) => {
+                    const input = j.input as AddFileInput;
+                    return j.type === "ADD_FILE" && input.id === job.documentId
+                })
+                if (op) {
+                    await queue.addDependencies(driveJob);
+                }
+            }
+        }
+
+        // if it has ADD_FILE operations then adds the job as
+        // a dependency to the corresponding document queues
+        const addFileOps = job.operations.filter((j: Operation) => j.type === "ADD_FILE");
+        for (const addFileOp of addFileOps) {
+            const input = addFileOp.input as AddFileInput;
+            const q = this.getQueue(job.driveId, input.id)
+            await q.addDependencies({ jobId, ...job });
+        }
+
+        // remove document if operations contains delete_node
+        const removeFileOps = job.operations.filter((j: Operation) => j.type === "DELETE_NODE");
+        for (const removeFileOp of removeFileOps) {
+            const input = removeFileOp.input as DeleteNodeInput;
+            const queue = this.getQueue(job.driveId, input.id);
+            await queue.setDeleted(true);
+        }
+
+        await queue.addJob({ jobId, ...job });
+
+        return jobId;
+    }
+
+    async getResult(driveId: string, documentId: string, jobId: JobId): Promise<IOperationResult | undefined> {
+        const queue = this.getQueue(driveId, documentId);
+        return queue.getResult(jobId);
+    }
+
+    getQueue(driveId: string, documentId?: string) {
+        const queueId = this.getQueueId(driveId, documentId);
+        let queue = this.queues.find((q) => q.getId() === queueId);
+
+        if (!queue) {
+            queue = new MemoryQueue(queueId);
+            this.queues.push(queue);
+        }
+
+        return queue;
+    }
+
+    removeQueue(driveId: string, documentId?: string) {
+        const queueId = this.getQueueId(driveId, documentId);
+        this.queues = this.queues.filter((q) => q.getId() !== queueId);
+        this.emit("queueRemoved", queueId)
+    }
+
+    getQueueByIndex(index: number) {
+        const queue = this.queues[index];
+        if (queue) {
+            return queue;
+        }
+
+        return null;
+    }
+
+    getQueues() {
+        return Object.keys(new Array(this.queues));
+    }
+
+    private retryNextJob() {
+        const retry = this.timeout === 0 && typeof setImmediate !== "undefined" ? setImmediate : (fn: () => void) => setTimeout(fn, this.timeout);
+        return retry(() => this.processNextJob());
+    }
+
+    private async processNextJob() {
+        if (!this.delegate) {
+            throw new Error("No server delegate defined");
+        }
+
+        if (this.queues.length === 0) {
+            this.retryNextJob();
+            return;
+        }
+
+        const queue = this.queues[this.ticker];
+        this.ticker = this.ticker === this.queues.length - 1 ? 0 : this.ticker + 1;
+        if (!queue) {
+            this.ticker = 0;
+            this.retryNextJob();
+            return;
+        }
+
+        const amountOfJobs = await queue.amountOfJobs();
+        if (amountOfJobs === 0) {
+            this.retryNextJob();
+            return;
+        }
+
+        const isBlocked = await queue.isBlocked();
+        if (isBlocked) {
+            this.retryNextJob();
+            return;
+        }
+
+        await queue.setBlocked(true);
+        const nextJob = await queue.getNextJob();
+        if (!nextJob) {
+            this.retryNextJob();
+            return;
+        }
+
+        try {
+            const result = await this.delegate.processOperationJob(nextJob);
+            await queue.setResult(nextJob.jobId, result);
+
+            // unblock the document queues of each add_file operation
+            const addFileOperations = nextJob.operations.filter((op) => op.type === "ADD_FILE");
+            if (addFileOperations.length > 0) {
+                addFileOperations.map(async (addFileOp) => {
+                    const documentQueue = this.getQueue(nextJob.driveId, (addFileOp.input as AddFileInput).id);
+                    await documentQueue.removeDependencies(nextJob);
+                });
+            }
+
+            this.emit("jobCompleted", nextJob, result);
+        } catch (e) {
+            this.emit("jobFailed", nextJob, e as Error);
+        } finally {
+            queue.setBlocked(false);
+            await this.processNextJob();
+        }
+    }
+
+    protected emit<K extends keyof QueueEvents>(
+        event: K,
+        ...args: Parameters<QueueEvents[K]>
+    ) {
+        this.emitter.emit(event, ...args);
+    }
+    on<K extends keyof QueueEvents>(this: this, event: K, cb: QueueEvents[K]): Unsubscribe {
+        return this.emitter.on(event, cb);
+    }
+
+    protected getQueueId(driveId: string, documentId?: string) {
+        return `${driveId}${documentId ? `:${documentId}` : ''}`;
+    }
+}

package/src/queue/redis.ts

@@ -0,0 +1,143 @@
+import { RedisClientType } from "redis";
+import { IJob, IQueue, IQueueManager, OperationJob } from "./types";
+import { BaseQueueManager } from "./base";
+
+export class RedisQueue<T, R> implements IQueue<T, R> {
+    private id: string;
+    private client: RedisClientType;
+
+    constructor(id: string, client: RedisClientType) {
+        this.client = client;
+        this.id = id;
+        this.client.hSet("queues", id, "true");
+
+    }
+
+    async setResult(jobId: string, result: any): Promise<void> {
+        await this.client.hSet(this.id + "-results", jobId, JSON.stringify(result));
+    }
+    async getResult(jobId: string): Promise<any> {
+        const results = await this.client.hGet(this.id + "-results", jobId);
+        if (!results) {
+            return null;
+        }
+        return JSON.parse(results);
+    }
+
+    async addJob(data: any) {
+        await this.client.lPush(this.id + "-jobs", JSON.stringify(data));
+    }
+
+    async getNextJob() {
+        const job = await this.client.rPop(this.id + "-jobs");
+        if (!job) {
+            return null;
+        }
+        return JSON.parse(job);
+    }
+
+    async amountOfJobs() {
+        return this.client.lLen(this.id + "-jobs");
+    }
+
+    async setBlocked(blocked: boolean) {
+        if (blocked) {
+            await this.client.hSet(this.id, "blocked", "true");
+        } else {
+            await this.client.hDel(this.id, "blocked");
+        }
+    }
+
+    async isBlocked() {
+        const blockedResult = await this.client.hGet(this.id, "blocked");
+        if (blockedResult) {
+            return true;
+        }
+
+        return false;
+    }
+
+    getId() {
+        return this.id;
+    }
+
+    async getJobs() {
+        const entries = await this.client.lRange(this.id + "-jobs", 0, -1)
+        return entries.map(e => JSON.parse(e));
+    }
+
+    async addDependencies(job: IJob<OperationJob>) {
+        if (await this.hasDependency(job)) {
+            return;
+        }
+        await this.client.lPush(this.id + "-deps", JSON.stringify(job));
+        await this.setBlocked(true);
+    }
+
+    async hasDependency(job: IJob<OperationJob>) {
+        const deps = await this.client.lRange(this.id + "-deps", 0, -1);
+        return deps.some(d => d === JSON.stringify(job));
+    }
+
+    async removeDependencies(job: IJob<OperationJob>) {
+        const allDeps1 = await this.client.lLen(this.id + "-deps");
+        await this.client.lRem(this.id + "-deps", 1, JSON.stringify(job));
+        const allDeps = await this.client.lLen(this.id + "-deps");
+        if (allDeps > 0) {
+            await this.setBlocked(true);
+        } else {
+            await this.setBlocked(false);
+        }
+    }
+
+    async isDeleted() {
+        const deleted = await this.client.hGet(this.id, "deleted");
+        return deleted === "true";
+    }
+
+    async setDeleted(deleted: boolean) {
+        if (deleted) {
+            await this.client.hSet(this.id, "deleted", "true");
+        } else {
+            await this.client.hDel(this.id, "deleted");
+        }
+    }
+}
+
+export class RedisQueueManager extends BaseQueueManager implements IQueueManager {
+
+    private client: RedisClientType;
+
+    constructor(workers = 3, timeout = 0, client: RedisClientType) {
+        super(workers, timeout);
+        this.client = client;
+    }
+
+    async init(delegate: IServerDelegate, onError: (error: Error) => void): Promise<void> {
+        await super.init(delegate, onError);
+        // load all queues
+        const queues = await this.client.hGetAll("queues");
+        for (const queueId in queues) {
+            this.queues.push(new RedisQueue(queueId, this.client));
+        }
+    }
+
+    getQueue(driveId: string, documentId?: string) {
+        const queueId = this.getQueueId(driveId, documentId);
+        let queue = this.queues.find((q) => q.getId() === queueId);
+
+        if (!queue) {
+            queue = new RedisQueue(queueId, this.client);
+            this.queues.push(queue);
+        }
+
+        return queue;
+    }
+
+    removeQueue(driveId: string, documentId?: string | undefined): void {
+        super.removeQueue(driveId, documentId);
+
+        const queueId = this.getQueueId(driveId, documentId);
+        this.client.hDel("queues", queueId);
+    }
+}

package/src/queue/types.ts

@@ -0,0 +1,57 @@
+import { Operation } from "document-model/document";
+import { IOperationResult } from "../server";
+import type { Unsubscribe } from "nanoevents";
+
+export type OperationJob = {
+    driveId: string;
+    documentId?: string
+    operations: Operation[]
+    forceSync?: boolean
+}
+
+export type JobId = string;
+
+export interface QueueEvents {
+    jobCompleted: (job: IJob<OperationJob>, result: IOperationResult) => void;
+    jobFailed: (job: IJob<OperationJob>, error: Error) => void;
+}
+
+export interface IServerDelegate {
+    checkDocumentExists: (driveId: string, documentId: string) => Promise<boolean>;
+    processOperationJob: (job: OperationJob) => Promise<IOperationResult>;
+}
+
+export interface IQueueManager {
+    addJob(job: OperationJob): Promise<JobId>;
+    getResult(driveId: string, documentId: string, jobId: JobId): Promise<IOperationResult | undefined>;
+    getQueue(driveId: string, document?: string): IQueue<OperationJob, IOperationResult>;
+    removeQueue(driveId: string, documentId?: string): void;
+    getQueueByIndex(index: number): IQueue<OperationJob, IOperationResult> | null;
+    getQueues(): string[];
+    init(delegate: IServerDelegate, onError: (error: Error) => void): Promise<void>;
+    on<K extends keyof QueueEvents>(
+        this: this,
+        event: K,
+        cb: QueueEvents[K]
+    ): Unsubscribe;
+}
+
+export type IJob<T> = { jobId: JobId } & T;
+
+export interface IQueue<T, R> {
+    addJob(data: IJob<T>): Promise<void>;
+    getNextJob(): Promise<IJob<T> | undefined>;
+    amountOfJobs(): Promise<number>;
+    getId(): string;
+    setBlocked(blocked: boolean): Promise<void>;
+    isBlocked(): Promise<boolean>;
+    isDeleted(): Promise<boolean>;
+    setDeleted(deleted: boolean): Promise<void>;
+    setResult(jobId: JobId, result: R): Promise<void>;
+    getResult(jobId: JobId): Promise<R | undefined>;
+    getJobs(): Promise<IJob<T>[]>;
+    addDependencies(job: IJob<OperationJob>): Promise<void>;
+    removeDependencies(job: IJob<OperationJob>): Promise<void>;
+}
+
+export type IJobQueue = IQueue<OperationJob, IOperationResult>;

package/src/server/error.ts

@@ -16,3 +16,20 @@ export class OperationError extends Error {
         this.operation = operation;
     }
 }
+
+export class ConflictOperationError extends OperationError {
+    constructor(existingOperation: Operation, newOperation: Operation) {
+        super(
+            'CONFLICT',
+            newOperation,
+            `Conflicting operation on index ${newOperation.index}`,
+            { existingOperation, newOperation }
+        );
+    }
+}
+
+export class MissingOperationError extends OperationError {
+    constructor(index: number, operation: Operation) {
+        super('MISSING', operation, `Missing operation on index ${index}`);
+    }
+}