document-drive 1.19.1 → 1.20.1

package/dist/vitest.config.d.ts

@@ -0,0 +1,3 @@
+declare const _default: import("vitest/config.js").UserConfigFnObject;
+export default _default;
+//# sourceMappingURL=vitest.config.d.ts.map

package/dist/vitest.config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vitest.config.d.ts","sourceRoot":"","sources":["../vitest.config.ts"],"names":[],"mappings":";AAGA,wBAiBI"}

package/dist/vitest.config.js

@@ -0,0 +1,20 @@
+import { loadEnv } from "vite";
+import { defineConfig } from "vitest/config";
+export default defineConfig(({ mode }) => ({
+    test: {
+        environment: "node",
+        env: loadEnv(mode, process.cwd(), ""),
+        testTimeout: 5000,
+        server: {
+            deps: {
+                inline: ["document-model-libs"],
+            },
+        },
+        setupFiles: "./test/vitest-setup.ts",
+        poolOptions: {
+            forks: {
+                singleFork: true,
+            },
+        },
+    },
+}));

package/package.json

@@ -1,42 +1,20 @@
 {
   "name": "document-drive",
-  "version": "1.19.1",
+  "version": "1.20.1",
   "license": "AGPL-3.0-only",
   "type": "module",
-  "module": "./src/index.ts",
-  "types": "./src/index.ts",
+  "types": "./dist/index.d.ts",
   "exports": {
-    ".": "./src/index.ts",
-    "./server": "./src/server/index.ts",
-    "./storage": "./src/storage/index.ts",
-    "./storage/browser": "./src/storage/browser.ts",
-    "./storage/filesystem": "./src/storage/filesystem.ts",
-    "./storage/memory": "./src/storage/memory.ts",
-    "./storage/prisma": "./src/storage/prisma.ts",
-    "./cache/redis": "./src/cache/redis.ts",
-    "./cache/memory": "./src/cache/memory.ts",
-    "./queue/redis": "./src/queue/redis.ts",
-    "./queue/base": "./src/queue/base.ts",
-    "./utils": "./src/utils/index.ts",
-    "./utils/graphql": "./src/utils/graphql.ts",
-    "./utils/migrations": "./src/utils/migrations.ts",
-    "./logger": "./src/utils/logger.ts"
+    ".": "./dist/index.js",
+    "./*": "./dist/src/*.js"
+  },
+  "imports": {
+    "#*": "./dist/src/*.js"
   },
   "files": [
-    "./src"
+    "./dist"
   ],
-  "peerDependencies": {
-    "change-case": "^5.4.4",
-    "graphql": "^16.9.0",
-    "graphql-request": "^6.1.0",
-    "json-stringify-deterministic": "^1.0.12",
-    "nanoevents": "^9.0.0",
-    "prisma": "^5.18.0",
-    "sanitize-filename": "^1.6.3",
-    "@powerhousedao/scalars": "1.24.0",
-    "document-model": "2.20.0",
-    "document-model-libs": "1.133.0"
-  },
+  "sideEffects": false,
   "optionalDependencies": {
     "@prisma/client": "^5.18.0",
     "exponential-backoff": "^3.1.1",
@@ -44,9 +22,9 @@
     "redis": "^4.6.15",
     "sequelize": "^6.37.3",
     "sqlite3": "^5.1.7",
-    "@powerhousedao/scalars": "1.24.0"
+    "@powerhousedao/scalars": "1.25.0"
   },
-  "devDependencies": {
+  "dependencies": {
     "@prisma/client": "5.17.0",
     "@types/node": "^20.14.11",
     "@types/uuid": "^9.0.8",
@@ -63,20 +41,24 @@
     "sanitize-filename": "^1.6.3",
     "sequelize": "^6.37.2",
     "sqlite3": "^5.1.7",
+    "uuid": "^11.0.5",
     "vitest-fetch-mock": "^0.3.0",
     "webdriverio": "^9.0.9",
-    "document-model": "2.20.0",
-    "document-model-libs": "1.133.0"
+    "zod": "^3.24.1",
+    "@powerhousedao/scalars": "1.25.0",
+    "document-model": "2.21.0",
+    "@powerhousedao/codegen": "0.37.1"
   },
   "scripts": {
-    "check-types": "tsc --build",
-    "check-types:watch": "tsc --build --watch",
-    "postlint": "npm run check-types",
+    "build:tsc": "tsc --build",
+    "build:tsc:watch": "tsc --build --watch",
     "lint": "eslint .",
     "lint:nx": "eslint . --fix --quiet",
+    "build": "tsc --build",
     "release": "semantic-release",
     "test": "vitest run --coverage --exclude \"test/flaky/**\"",
     "test:watch": "vitest --coverage --exclude \"test/flaky/**\"",
+    "bench": "vitest bench",
     "clean": "rimraf dist",
     "clean:node_modules": "rimraf node_modules",
     "migrate": "prisma generate && prisma db push"

package/src/cache/index.ts

@@ -1,2 +0,0 @@
-export * from "./memory";
-export * from "./types";

package/src/cache/memory.ts

@@ -1,33 +0,0 @@
-import { Document } from "document-model/document";
-import { ICache } from "./types";
-
-class InMemoryCache implements ICache {
-  private cache = new Map<string, Map<string, Document>>();
-
-  async setDocument(drive: string, id: string, document: Document) {
-    const global = document.operations.global.map((e) => {
-      delete e.resultingState;
-      return e;
-    });
-    const local = document.operations.local.map((e) => {
-      delete e.resultingState;
-      return e;
-    });
-    const doc = { ...document, operations: { global, local } };
-    if (!this.cache.has(drive)) {
-      this.cache.set(drive, new Map());
-    }
-    this.cache.get(drive)?.set(id, doc);
-    return true;
-  }
-
-  async deleteDocument(drive: string, id: string) {
-    return this.cache.get(drive)?.delete(id) ?? false;
-  }
-
-  async getDocument(drive: string, id: string) {
-    return this.cache.get(drive)?.get(id);
-  }
-}
-
-export default InMemoryCache;

package/src/cache/redis.ts

@@ -1,56 +0,0 @@
-import { Document } from "document-model/document";
-import type { RedisClientType } from "redis";
-import { ICache } from "./types";
-
-class RedisCache implements ICache {
-  private redis: RedisClientType;
-  private timeoutInSeconds: number;
-
-  constructor(
-    redis: RedisClientType,
-    timeoutInSeconds: number | undefined = 5 * 60,
-  ) {
-    this.redis = redis;
-    this.timeoutInSeconds = timeoutInSeconds;
-  }
-
-  private static _getId(drive: string, id: string) {
-    return `cache:${drive}:${id}`;
-  }
-
-  async setDocument(drive: string, id: string, document: Document) {
-    const global = document.operations.global.map((e) => {
-      delete e.resultingState;
-      return e;
-    });
-    const local = document.operations.local.map((e) => {
-      delete e.resultingState;
-      return e;
-    });
-    const doc = { ...document, operations: { global, local } };
-    const redisId = RedisCache._getId(drive, id);
-    const result = await this.redis.set(redisId, JSON.stringify(doc), {
-      EX: this.timeoutInSeconds ? this.timeoutInSeconds : undefined,
-    });
-
-    if (result === "OK") {
-      return true;
-    }
-
-    return false;
-  }
-
-  async getDocument(drive: string, id: string) {
-    const redisId = RedisCache._getId(drive, id);
-    const doc = await this.redis.get(redisId);
-
-    return doc ? (JSON.parse(doc) as Document) : undefined;
-  }
-
-  async deleteDocument(drive: string, id: string) {
-    const redisId = RedisCache._getId(drive, id);
-    return (await this.redis.del(redisId)) > 0;
-  }
-}
-
-export default RedisCache;

package/src/cache/types.ts

@@ -1,9 +0,0 @@
-import type { Document } from "document-model/document";
-
-export interface ICache {
-  setDocument(drive: string, id: string, document: Document): Promise<boolean>;
-  getDocument(drive: string, id: string): Promise<Document | undefined>;
-
-  // @returns — true if a document existed and has been removed, or false if the document is not cached.
-  deleteDocument(drive: string, id: string): Promise<boolean>;
-}

package/src/index.ts

@@ -1,4 +0,0 @@
-export * from "./server";
-export * from "./server/error";
-export * from "./storage";
-export * from "./utils";

package/src/queue/base.ts

@@ -1,320 +0,0 @@
-import {
-  AddFileInput,
-  DeleteNodeInput,
-} from "document-model-libs/document-drive";
-import { Action } from "document-model/document";
-import { Unsubscribe, createNanoEvents } from "nanoevents";
-import { generateUUID, runAsap } from "../utils";
-import { logger } from "../utils/logger";
-import {
-  IJob,
-  IJobQueue,
-  IQueue,
-  IQueueManager,
-  IServerDelegate,
-  Job,
-  JobId,
-  QueueEvents,
-  isOperationJob,
-} from "./types";
-
-export class MemoryQueue<T, R> implements IQueue<T, R> {
-  private id: string;
-  private blocked = false;
-  private deleted = false;
-  private items: IJob<T>[] = [];
-  private dependencies = new Array<IJob<Job>>();
-
-  constructor(id: string) {
-    this.id = id;
-  }
-
-  async setDeleted(deleted: boolean) {
-    this.deleted = deleted;
-  }
-
-  async isDeleted() {
-    return this.deleted;
-  }
-
-  async addJob(data: IJob<T>) {
-    this.items.push(data);
-    return Promise.resolve();
-  }
-
-  async getNextJob() {
-    const job = this.items.shift();
-    return Promise.resolve(job);
-  }
-
-  async amountOfJobs() {
-    return Promise.resolve(this.items.length);
-  }
-
-  getId() {
-    return this.id;
-  }
-
-  async setBlocked(blocked: boolean) {
-    this.blocked = blocked;
-  }
-
-  async isBlocked() {
-    return this.blocked;
-  }
-
-  async getJobs() {
-    return this.items;
-  }
-
-  async addDependencies(job: IJob<Job>) {
-    if (!this.dependencies.find((j) => j.jobId === job.jobId)) {
-      this.dependencies.push(job);
-    }
-    if (!this.isBlocked()) {
-      this.setBlocked(true);
-    }
-  }
-
-  async removeDependencies(job: IJob<Job>) {
-    this.dependencies = this.dependencies.filter(
-      (j) => j.jobId !== job.jobId && j.driveId !== job.driveId,
-    );
-    if (this.dependencies.length === 0) {
-      await this.setBlocked(false);
-    }
-  }
-}
-
-export class BaseQueueManager implements IQueueManager {
-  protected emitter = createNanoEvents<QueueEvents>();
-  protected ticker = 0;
-  protected queues: IJobQueue[] = [];
-  protected workers: number;
-  protected timeout: number;
-  private delegate: IServerDelegate | undefined;
-
-  constructor(workers = 3, timeout = 0) {
-    this.workers = workers;
-    this.timeout = timeout;
-  }
-
-  async init(
-    delegate: IServerDelegate,
-    onError: (error: Error) => void,
-  ): Promise<void> {
-    this.delegate = delegate;
-    for (let i = 0; i < this.workers; i++) {
-      setTimeout(
-        () => this.processNextJob.bind(this)().catch(onError),
-        100 * i,
-      );
-    }
-    return Promise.resolve();
-  }
-
-  async addJob(job: Job): Promise<JobId> {
-    if (!this.delegate) {
-      throw new Error("No server delegate defined");
-    }
-
-    const jobId = generateUUID();
-    const queue = this.getQueue(job.driveId, job.documentId);
-
-    if (await queue.isDeleted()) {
-      throw new Error("Queue is deleted");
-    }
-
-    // checks if the job is for a document that doesn't exist in storage yet
-    const newDocument =
-      job.documentId &&
-      !(await this.delegate.checkDocumentExists(job.driveId, job.documentId));
-    // if it is a new document and queue is not yet blocked then
-    // blocks it so the jobs are not processed until it's ready
-    if (newDocument && !(await queue.isBlocked())) {
-      await queue.setBlocked(true);
-
-      // checks if there any job in the queue adding the file and adds as dependency
-      const driveQueue = this.getQueue(job.driveId);
-      const jobs = await driveQueue.getJobs();
-      for (const driveJob of jobs) {
-        const actions = isOperationJob(driveJob)
-          ? driveJob.operations
-          : driveJob.actions;
-        const op = actions.find((j: Action) => {
-          const input = j.input as AddFileInput;
-          return j.type === "ADD_FILE" && input.id === job.documentId;
-        });
-        if (op) {
-          await queue.addDependencies(driveJob);
-        }
-      }
-    }
-
-    // if it has ADD_FILE operations then adds the job as
-    // a dependency to the corresponding document queues
-    const actions = isOperationJob(job) ? job.operations : job.actions;
-    const addFileOps = actions.filter((j: Action) => j.type === "ADD_FILE");
-    for (const addFileOp of addFileOps) {
-      const input = addFileOp.input as AddFileInput;
-      const q = this.getQueue(job.driveId, input.id);
-      await q.addDependencies({ jobId, ...job });
-    }
-
-    // remove document if operations contains delete_node
-    const removeFileOps = actions.filter(
-      (j: Action) => j.type === "DELETE_NODE",
-    );
-    for (const removeFileOp of removeFileOps) {
-      const input = removeFileOp.input as DeleteNodeInput;
-      const queue = this.getQueue(job.driveId, input.id);
-      await queue.setDeleted(true);
-    }
-    await queue.addJob({ jobId, ...job });
-
-    return jobId;
-  }
-
-  getQueue(driveId: string, documentId?: string) {
-    const queueId = this.getQueueId(driveId, documentId);
-    let queue = this.queues.find((q) => q.getId() === queueId);
-
-    if (!queue) {
-      queue = new MemoryQueue(queueId);
-      this.queues.push(queue);
-    }
-
-    return queue;
-  }
-
-  removeQueue(driveId: string, documentId?: string) {
-    const queueId = this.getQueueId(driveId, documentId);
-    this.queues = this.queues.filter((q) => q.getId() !== queueId);
-    this.emit("queueRemoved", queueId);
-  }
-
-  getQueueByIndex(index: number) {
-    const queue = this.queues[index];
-    if (queue) {
-      return queue;
-    }
-
-    return null;
-  }
-
-  getQueues() {
-    return this.queues.map((q) => q.getId());
-  }
-
-  private retryNextJob(timeout?: number) {
-    const _timeout = timeout !== undefined ? timeout : this.timeout;
-    const retry =
-      _timeout > 0 ? (fn: () => void) => setTimeout(fn, _timeout) : runAsap;
-    retry(() => this.processNextJob());
-  }
-
-  private async findFirstNonEmptyQueue(ticker: number): Promise<number | null> {
-    const numQueues = this.queues.length;
-
-    for (let i = 0; i < numQueues; i++) {
-      const index = (ticker + i) % numQueues;
-      const queue = this.queues[index];
-      if (queue && (await queue.amountOfJobs()) > 0) {
-        return index;
-      }
-    }
-    return null;
-  }
-
-  private async processNextJob() {
-    if (!this.delegate) {
-      throw new Error("No server delegate defined");
-    }
-
-    if (this.queues.length === 0) {
-      this.retryNextJob();
-      return;
-    }
-
-    const queue = this.queues[this.ticker];
-    if (!queue) {
-      this.ticker = 0;
-      this.retryNextJob();
-      return;
-    }
-
-    // if no jobs in the current queue then looks for the
-    // next queue with jobs. If no jobs in any queue then
-    // retries after a timeout
-    const amountOfJobs = await queue.amountOfJobs();
-    if (amountOfJobs === 0) {
-      const nextTicker = await this.findFirstNonEmptyQueue(this.ticker);
-      if (nextTicker !== null) {
-        this.ticker = nextTicker;
-        this.retryNextJob(0);
-      } else {
-        this.retryNextJob();
-      }
-      return;
-    }
-
-    this.ticker = this.ticker === this.queues.length - 1 ? 0 : this.ticker + 1;
-
-    const isBlocked = await queue.isBlocked();
-    if (isBlocked) {
-      this.retryNextJob();
-      return;
-    }
-
-    await queue.setBlocked(true);
-    const nextJob = await queue.getNextJob();
-    if (!nextJob) {
-      this.retryNextJob();
-      return;
-    }
-
-    try {
-      const result = await this.delegate.processJob(nextJob);
-
-      // unblock the document queues of each add_file operation
-      const actions = isOperationJob(nextJob)
-        ? nextJob.operations
-        : nextJob.actions;
-      const addFileActions = actions.filter((op) => op.type === "ADD_FILE");
-      if (addFileActions.length > 0) {
-        for (const addFile of addFileActions) {
-          const documentQueue = this.getQueue(
-            nextJob.driveId,
-            (addFile.input as AddFileInput).id,
-          );
-          await documentQueue.removeDependencies(nextJob);
-        }
-      }
-      this.emit("jobCompleted", nextJob, result);
-    } catch (e) {
-      logger.error(`job failed`, e);
-      this.emit("jobFailed", nextJob, e as Error);
-    } finally {
-      await queue.setBlocked(false);
-      this.retryNextJob(0);
-    }
-  }
-
-  protected emit<K extends keyof QueueEvents>(
-    event: K,
-    ...args: Parameters<QueueEvents[K]>
-  ) {
-    this.emitter.emit(event, ...args);
-  }
-  on<K extends keyof QueueEvents>(
-    this: this,
-    event: K,
-    cb: QueueEvents[K],
-  ): Unsubscribe {
-    return this.emitter.on(event, cb);
-  }
-
-  protected getQueueId(driveId: string, documentId?: string) {
-    return `queue:${driveId}${documentId ? `:${documentId}` : ""}`;
-  }
-}

package/src/queue/index.ts

@@ -1,2 +0,0 @@
-export * from "./base";
-export * from "./types";

package/src/queue/redis.ts

@@ -1,144 +0,0 @@
-import { RedisClientType } from "redis";
-import { BaseQueueManager } from "./base";
-import {
-  IJob,
-  IQueue,
-  IQueueManager,
-  IServerDelegate,
-  OperationJob,
-} from "./types";
-
-export class RedisQueue<T, R> implements IQueue<T, R> {
-  private id: string;
-  private client: RedisClientType;
-
-  constructor(id: string, client: RedisClientType) {
-    this.client = client;
-    this.id = id;
-    this.client.hSet("queues", id, "true");
-    this.client.hSet(this.id, "blocked", "false");
-  }
-
-  async addJob(data: any) {
-    await this.client.lPush(this.id + "-jobs", JSON.stringify(data));
-  }
-
-  async getNextJob() {
-    const job = await this.client.rPop(this.id + "-jobs");
-    if (!job) {
-      return undefined;
-    }
-    return JSON.parse(job) as IJob<T>;
-  }
-
-  async amountOfJobs() {
-    return this.client.lLen(this.id + "-jobs");
-  }
-
-  async setBlocked(blocked: boolean) {
-    if (blocked) {
-      await this.client.hSet(this.id, "blocked", "true");
-    } else {
-      await this.client.hSet(this.id, "blocked", "false");
-    }
-  }
-
-  async isBlocked() {
-    const blockedResult = await this.client.hGet(this.id, "blocked");
-    if (blockedResult === "true") {
-      return true;
-    }
-
-    return false;
-  }
-
-  getId() {
-    return this.id;
-  }
-
-  async getJobs() {
-    const entries = await this.client.lRange(this.id + "-jobs", 0, -1);
-    return entries.map((e) => JSON.parse(e) as IJob<T>);
-  }
-
-  async addDependencies(job: IJob<OperationJob>) {
-    if (await this.hasDependency(job)) {
-      return;
-    }
-    await this.client.lPush(this.id + "-deps", JSON.stringify(job));
-    await this.setBlocked(true);
-  }
-
-  async hasDependency(job: IJob<OperationJob>) {
-    const deps = await this.client.lRange(this.id + "-deps", 0, -1);
-    return deps.some((d) => d === JSON.stringify(job));
-  }
-
-  async removeDependencies(job: IJob<OperationJob>) {
-    await this.client.lRem(this.id + "-deps", 1, JSON.stringify(job));
-    const allDeps = await this.client.lLen(this.id + "-deps");
-    if (allDeps > 0) {
-      await this.setBlocked(true);
-    } else {
-      await this.setBlocked(false);
-    }
-  }
-
-  async isDeleted() {
-    const active = await this.client.hGet("queues", this.id);
-    return active === "false";
-  }
-
-  async setDeleted(deleted: boolean) {
-    if (deleted) {
-      await this.client.hSet("queues", this.id, "false");
-    } else {
-      await this.client.hSet("queues", this.id, "true");
-    }
-  }
-}
-
-export class RedisQueueManager
-  extends BaseQueueManager
-  implements IQueueManager
-{
-  private client: RedisClientType;
-
-  constructor(workers = 3, timeout = 0, client: RedisClientType) {
-    super(workers, timeout);
-    this.client = client;
-  }
-
-  async init(
-    delegate: IServerDelegate,
-    onError: (error: Error) => void,
-  ): Promise<void> {
-    await super.init(delegate, onError);
-    const queues = await this.client.hGetAll("queues");
-    for (const queueId in queues) {
-      const active = await this.client.hGet("queues", queueId);
-      if (active === "true") {
-        this.queues.push(new RedisQueue(queueId, this.client));
-      }
-    }
-  }
-
-  getQueue(driveId: string, documentId?: string) {
-    const queueId = this.getQueueId(driveId, documentId);
-    let queue = this.queues.find((q) => q.getId() === queueId);
-
-    if (!queue) {
-      queue = new RedisQueue(queueId, this.client);
-      this.queues.push(queue);
-    }
-
-    return queue;
-  }
-
-  removeQueue(driveId: string, documentId?: string | undefined): void {
-    super.removeQueue(driveId, documentId);
-
-    const queueId = this.getQueueId(driveId, documentId);
-    this.client.hDel("queues", queueId);
-  }
-}