document-drive 1.0.0-alpha.83 → 1.0.0-alpha.85

package/README.md

@@ -1 +1 @@
-# Document Drive
+# Document Drive

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "document-drive",
-    "version": "1.0.0-alpha.83",
+    "version": "1.0.0-alpha.85",
     "license": "AGPL-3.0-only",
     "type": "module",
     "module": "./src/index.ts",

package/src/cache/memory.ts

@@ -1,5 +1,5 @@
-import { Document } from "document-model/document";
-import { ICache } from "./types";
+import { Document } from 'document-model/document';
+import { ICache } from './types';
 
 class InMemoryCache implements ICache {
     private cache = new Map<string, Map<string, Document>>();
@@ -13,7 +13,7 @@ class InMemoryCache implements ICache {
             delete e.resultingState;
             return e;
         });
-        const doc = { ...document, operations: { global, local } }
+        const doc = { ...document, operations: { global, local } };
         if (!this.cache.has(drive)) {
             this.cache.set(drive, new Map());
         }

package/src/cache/redis.ts

@@ -1,12 +1,15 @@
-import { Document } from "document-model/document";
-import { ICache } from "./types";
-import type { RedisClientType } from "redis";
+import { Document } from 'document-model/document';
+import type { RedisClientType } from 'redis';
+import { ICache } from './types';
 
 class RedisCache implements ICache {
     private redis: RedisClientType;
     private timeoutInSeconds: number;
 
-    constructor(redis: RedisClientType, timeoutInSeconds: number | undefined = 5 * 60) {
+    constructor(
+        redis: RedisClientType,
+        timeoutInSeconds: number | undefined = 5 * 60
+    ) {
         this.redis = redis;
         this.timeoutInSeconds = timeoutInSeconds;
     }
@@ -24,7 +27,7 @@ class RedisCache implements ICache {
             delete e.resultingState;
             return e;
         });
-        const doc = { ...document, operations: { global, local } }
+        const doc = { ...document, operations: { global, local } };
         const redisId = RedisCache._getId(drive, id);
         const result = await this.redis.set(redisId, JSON.stringify(doc), {
             EX: this.timeoutInSeconds ? this.timeoutInSeconds : undefined
@@ -41,7 +44,7 @@ class RedisCache implements ICache {
         const redisId = RedisCache._getId(drive, id);
         const doc = await this.redis.get(redisId);
 
-        return doc ? JSON.parse(doc) as Document : undefined;
+        return doc ? (JSON.parse(doc) as Document) : undefined;
     }
 
     async deleteDocument(drive: string, id: string) {

package/src/cache/types.ts

@@ -1,9 +1,13 @@
-import type { Document } from "document-model/document";
+import type { Document } from 'document-model/document';
 
 export interface ICache {
-    setDocument(drive: string, id: string, document: Document): Promise<boolean>
-    getDocument(drive: string, id: string): Promise<Document | undefined>
+    setDocument(
+        drive: string,
+        id: string,
+        document: Document
+    ): Promise<boolean>;
+    getDocument(drive: string, id: string): Promise<Document | undefined>;
 
     // @returns — true if a document existed and has been removed, or false if the document is not cached.
-    deleteDocument(drive: string, id: string): Promise<boolean>
+    deleteDocument(drive: string, id: string): Promise<boolean>;
 }

package/src/queue/base.ts

@@ -1,8 +1,22 @@
-import { IJob, IJobQueue, IQueue, IQueueManager, IServerDelegate, JobId, Job, QueueEvents, isOperationJob } from "./types";
-import { generateUUID } from "../utils";
-import { createNanoEvents, Unsubscribe } from 'nanoevents';
-import { Action } from "document-model/document";
-import { AddFileInput, DeleteNodeInput } from "document-model-libs/document-drive";
+import {
+    AddFileInput,
+    DeleteNodeInput
+} from 'document-model-libs/document-drive';
+import { Action } from 'document-model/document';
+import { Unsubscribe, createNanoEvents } from 'nanoevents';
+import { generateUUID } from '../utils';
+import { logger } from '../utils/logger';
+import {
+    IJob,
+    IJobQueue,
+    IQueue,
+    IQueueManager,
+    IServerDelegate,
+    Job,
+    JobId,
+    QueueEvents,
+    isOperationJob
+} from './types';
 
 export class MemoryQueue<T, R> implements IQueue<T, R> {
     private id: string;
@@ -63,7 +77,9 @@ export class MemoryQueue<T, R> implements IQueue<T, R> {
     }
 
     async removeDependencies(job: IJob<Job>) {
-        this.dependencies = this.dependencies.filter((j) => j.jobId !== job.jobId && j.driveId !== job.driveId);
+        this.dependencies = this.dependencies.filter(
+            j => j.jobId !== job.jobId && j.driveId !== job.driveId
+        );
         if (this.dependencies.length === 0) {
             await this.setBlocked(false);
         }
@@ -83,29 +99,40 @@ export class BaseQueueManager implements IQueueManager {
         this.timeout = timeout;
     }
 
-    async init(delegate: IServerDelegate, onError: (error: Error) => void): Promise<void> {
+    async init(
+        delegate: IServerDelegate,
+        onError: (error: Error) => void
+    ): Promise<void> {
         this.delegate = delegate;
         for (let i = 0; i < this.workers; i++) {
-            setTimeout(() => this.processNextJob.bind(this)().catch(onError), 100 * i);
+            setTimeout(
+                () => this.processNextJob.bind(this)().catch(onError),
+                100 * i
+            );
         }
-        return Promise.resolve()
+        return Promise.resolve();
     }
 
     async addJob(job: Job): Promise<JobId> {
         if (!this.delegate) {
-            throw new Error("No server delegate defined");
+            throw new Error('No server delegate defined');
         }
 
         const jobId = generateUUID();
         const queue = this.getQueue(job.driveId, job.documentId);
 
         if (await queue.isDeleted()) {
-            throw new Error("Queue is deleted")
+            throw new Error('Queue is deleted');
         }
 
         // checks if the job is for a document that doesn't exist in storage yet
-        const newDocument = job.documentId && !(await this.delegate.checkDocumentExists(job.driveId, job.documentId));
-        // if it is a new document and queue is not yet blocked then 
+        const newDocument =
+            job.documentId &&
+            !(await this.delegate.checkDocumentExists(
+                job.driveId,
+                job.documentId
+            ));
+        // if it is a new document and queue is not yet blocked then
         // blocks it so the jobs are not processed until it's ready
         if (newDocument && !(await queue.isBlocked())) {
             await queue.setBlocked(true);
@@ -114,11 +141,13 @@ export class BaseQueueManager implements IQueueManager {
             const driveQueue = this.getQueue(job.driveId);
             const jobs = await driveQueue.getJobs();
             for (const driveJob of jobs) {
-                const actions = isOperationJob(driveJob) ? driveJob.operations : driveJob.actions;
+                const actions = isOperationJob(driveJob)
+                    ? driveJob.operations
+                    : driveJob.actions;
                 const op = actions.find((j: Action) => {
                     const input = j.input as AddFileInput;
-                    return j.type === "ADD_FILE" && input.id === job.documentId
-                })
+                    return j.type === 'ADD_FILE' && input.id === job.documentId;
+                });
                 if (op) {
                     await queue.addDependencies(driveJob);
                 }
@@ -128,21 +157,22 @@ export class BaseQueueManager implements IQueueManager {
         // if it has ADD_FILE operations then adds the job as
         // a dependency to the corresponding document queues
         const actions = isOperationJob(job) ? job.operations : job.actions;
-        const addFileOps = actions.filter((j: Action) => j.type === "ADD_FILE");
+        const addFileOps = actions.filter((j: Action) => j.type === 'ADD_FILE');
         for (const addFileOp of addFileOps) {
             const input = addFileOp.input as AddFileInput;
-            const q = this.getQueue(job.driveId, input.id)
+            const q = this.getQueue(job.driveId, input.id);
             await q.addDependencies({ jobId, ...job });
         }
 
         // remove document if operations contains delete_node
-        const removeFileOps = actions.filter((j: Action) => j.type === "DELETE_NODE");
+        const removeFileOps = actions.filter(
+            (j: Action) => j.type === 'DELETE_NODE'
+        );
         for (const removeFileOp of removeFileOps) {
             const input = removeFileOp.input as DeleteNodeInput;
             const queue = this.getQueue(job.driveId, input.id);
             await queue.setDeleted(true);
         }
-
         await queue.addJob({ jobId, ...job });
 
         return jobId;
@@ -150,7 +180,7 @@ export class BaseQueueManager implements IQueueManager {
 
     getQueue(driveId: string, documentId?: string) {
         const queueId = this.getQueueId(driveId, documentId);
-        let queue = this.queues.find((q) => q.getId() === queueId);
+        let queue = this.queues.find(q => q.getId() === queueId);
 
         if (!queue) {
             queue = new MemoryQueue(queueId);
@@ -162,8 +192,8 @@ export class BaseQueueManager implements IQueueManager {
 
     removeQueue(driveId: string, documentId?: string) {
         const queueId = this.getQueueId(driveId, documentId);
-        this.queues = this.queues.filter((q) => q.getId() !== queueId);
-        this.emit("queueRemoved", queueId)
+        this.queues = this.queues.filter(q => q.getId() !== queueId);
+        this.emit('queueRemoved', queueId);
     }
 
     getQueueByIndex(index: number) {
@@ -176,17 +206,36 @@ export class BaseQueueManager implements IQueueManager {
     }
 
     getQueues() {
-        return Object.keys(new Array(this.queues));
+        return this.queues.map(q => q.getId());
     }
 
-    private retryNextJob() {
-        const retry = this.timeout === 0 && typeof setImmediate !== "undefined" ? setImmediate : (fn: () => void) => setTimeout(fn, this.timeout);
+    private retryNextJob(timeout?: number) {
+        const _timeout = timeout !== undefined ? timeout : this.timeout;
+        const retry =
+            _timeout === 0 && typeof setImmediate !== 'undefined'
+                ? setImmediate
+                : (fn: () => void) => setTimeout(fn, _timeout);
         return retry(() => this.processNextJob());
     }
 
+    private async findFirstNonEmptyQueue(
+        ticker: number
+    ): Promise<number | null> {
+        const numQueues = this.queues.length;
+
+        for (let i = 0; i < numQueues; i++) {
+            const index = (ticker + i) % numQueues;
+            const queue = this.queues[index];
+            if (queue && (await queue.amountOfJobs()) > 0) {
+                return index;
+            }
+        }
+        return null;
+    }
+
     private async processNextJob() {
         if (!this.delegate) {
-            throw new Error("No server delegate defined");
+            throw new Error('No server delegate defined');
         }
 
         if (this.queues.length === 0) {
@@ -195,19 +244,30 @@ export class BaseQueueManager implements IQueueManager {
         }
 
         const queue = this.queues[this.ticker];
-        this.ticker = this.ticker === this.queues.length - 1 ? 0 : this.ticker + 1;
         if (!queue) {
             this.ticker = 0;
             this.retryNextJob();
             return;
         }
 
+        // if no jobs in the current queue then looks for the
+        // next queue with jobs. If no jobs in any queue then
+        // retries after a timeout
         const amountOfJobs = await queue.amountOfJobs();
         if (amountOfJobs === 0) {
-            this.retryNextJob();
+            const nextTicker = await this.findFirstNonEmptyQueue(this.ticker);
+            if (nextTicker !== null) {
+                this.ticker = nextTicker;
+                this.retryNextJob(0);
+            } else {
+                this.retryNextJob();
+            }
             return;
         }
 
+        this.ticker =
+            this.ticker === this.queues.length - 1 ? 0 : this.ticker + 1;
+
         const isBlocked = await queue.isBlocked();
         if (isBlocked) {
             this.retryNextJob();
@@ -225,20 +285,26 @@ export class BaseQueueManager implements IQueueManager {
             const result = await this.delegate.processJob(nextJob);
 
             // unblock the document queues of each add_file operation
-            const actions = isOperationJob(nextJob) ? nextJob.operations : nextJob.actions;
-            const addFileActions = actions.filter((op) => op.type === "ADD_FILE");
+            const actions = isOperationJob(nextJob)
+                ? nextJob.operations
+                : nextJob.actions;
+            const addFileActions = actions.filter(op => op.type === 'ADD_FILE');
             if (addFileActions.length > 0) {
                 for (const addFile of addFileActions) {
-                    const documentQueue = this.getQueue(nextJob.driveId, (addFile.input as AddFileInput).id);
+                    const documentQueue = this.getQueue(
+                        nextJob.driveId,
+                        (addFile.input as AddFileInput).id
+                    );
                     await documentQueue.removeDependencies(nextJob);
-                };
+                }
             }
-            this.emit("jobCompleted", nextJob, result);
+            this.emit('jobCompleted', nextJob, result);
         } catch (e) {
-            this.emit("jobFailed", nextJob, e as Error);
+            logger.error(`job failed`, e);
+            this.emit('jobFailed', nextJob, e as Error);
         } finally {
             await queue.setBlocked(false);
-            await this.processNextJob();
+            this.retryNextJob(0);
         }
     }
 
@@ -248,11 +314,15 @@ export class BaseQueueManager implements IQueueManager {
     ) {
         this.emitter.emit(event, ...args);
     }
-    on<K extends keyof QueueEvents>(this: this, event: K, cb: QueueEvents[K]): Unsubscribe {
+    on<K extends keyof QueueEvents>(
+        this: this,
+        event: K,
+        cb: QueueEvents[K]
+    ): Unsubscribe {
         return this.emitter.on(event, cb);
     }
 
     protected getQueueId(driveId: string, documentId?: string) {
         return `queue:${driveId}${documentId ? `:${documentId}` : ''}`;
     }
-}
+}

package/src/queue/index.ts

@@ -1,2 +1,2 @@
 export * from './base';
-export * from './types';
+export * from './types';

package/src/queue/redis.ts

@@ -1,6 +1,12 @@
-import { RedisClientType } from "redis";
-import { IJob, IQueue, IQueueManager, IServerDelegate, OperationJob } from "./types";
-import { BaseQueueManager } from "./base";
+import { RedisClientType } from 'redis';
+import { BaseQueueManager } from './base';
+import {
+    IJob,
+    IQueue,
+    IQueueManager,
+    IServerDelegate,
+    OperationJob
+} from './types';
 
 export class RedisQueue<T, R> implements IQueue<T, R> {
     private id: string;
@@ -9,16 +15,16 @@ export class RedisQueue<T, R> implements IQueue<T, R> {
     constructor(id: string, client: RedisClientType) {
         this.client = client;
         this.id = id;
-        this.client.hSet("queues", id, "true");
-        this.client.hSet(this.id, "blocked", "false");
+        this.client.hSet('queues', id, 'true');
+        this.client.hSet(this.id, 'blocked', 'false');
     }
 
     async addJob(data: any) {
-        await this.client.lPush(this.id + "-jobs", JSON.stringify(data));
+        await this.client.lPush(this.id + '-jobs', JSON.stringify(data));
     }
 
     async getNextJob() {
-        const job = await this.client.rPop(this.id + "-jobs");
+        const job = await this.client.rPop(this.id + '-jobs');
         if (!job) {
             return undefined;
         }
@@ -26,20 +32,20 @@ export class RedisQueue<T, R> implements IQueue<T, R> {
     }
 
     async amountOfJobs() {
-        return this.client.lLen(this.id + "-jobs");
+        return this.client.lLen(this.id + '-jobs');
     }
 
     async setBlocked(blocked: boolean) {
         if (blocked) {
-            await this.client.hSet(this.id, "blocked", "true");
+            await this.client.hSet(this.id, 'blocked', 'true');
         } else {
-            await this.client.hSet(this.id, "blocked", "false");
+            await this.client.hSet(this.id, 'blocked', 'false');
         }
     }
 
     async isBlocked() {
-        const blockedResult = await this.client.hGet(this.id, "blocked");
-        if (blockedResult === "true") {
+        const blockedResult = await this.client.hGet(this.id, 'blocked');
+        if (blockedResult === 'true') {
             return true;
         }
 
@@ -51,7 +57,7 @@ export class RedisQueue<T, R> implements IQueue<T, R> {
     }
 
     async getJobs() {
-        const entries = await this.client.lRange(this.id + "-jobs", 0, -1)
+        const entries = await this.client.lRange(this.id + '-jobs', 0, -1);
         return entries.map(e => JSON.parse(e) as IJob<T>);
     }
 
@@ -59,18 +65,18 @@ export class RedisQueue<T, R> implements IQueue<T, R> {
         if (await this.hasDependency(job)) {
             return;
         }
-        await this.client.lPush(this.id + "-deps", JSON.stringify(job));
+        await this.client.lPush(this.id + '-deps', JSON.stringify(job));
         await this.setBlocked(true);
     }
 
     async hasDependency(job: IJob<OperationJob>) {
-        const deps = await this.client.lRange(this.id + "-deps", 0, -1);
+        const deps = await this.client.lRange(this.id + '-deps', 0, -1);
         return deps.some(d => d === JSON.stringify(job));
     }
 
     async removeDependencies(job: IJob<OperationJob>) {
-        await this.client.lRem(this.id + "-deps", 1, JSON.stringify(job));
-        const allDeps = await this.client.lLen(this.id + "-deps");
+        await this.client.lRem(this.id + '-deps', 1, JSON.stringify(job));
+        const allDeps = await this.client.lLen(this.id + '-deps');
         if (allDeps > 0) {
             await this.setBlocked(true);
         } else {
@@ -79,21 +85,23 @@ export class RedisQueue<T, R> implements IQueue<T, R> {
     }
 
     async isDeleted() {
-        const active = await this.client.hGet("queues", this.id);
-        return active === "false";
+        const active = await this.client.hGet('queues', this.id);
+        return active === 'false';
     }
 
     async setDeleted(deleted: boolean) {
         if (deleted) {
-            await this.client.hSet("queues", this.id, "false");
+            await this.client.hSet('queues', this.id, 'false');
         } else {
-            await this.client.hSet("queues", this.id, "true");
+            await this.client.hSet('queues', this.id, 'true');
         }
     }
 }
 
-export class RedisQueueManager extends BaseQueueManager implements IQueueManager {
-
+export class RedisQueueManager
+    extends BaseQueueManager
+    implements IQueueManager
+{
     private client: RedisClientType;
 
     constructor(workers = 3, timeout = 0, client: RedisClientType) {
@@ -101,12 +109,15 @@ export class RedisQueueManager extends BaseQueueManager implements IQueueManager
         this.client = client;
     }
 
-    async init(delegate: IServerDelegate, onError: (error: Error) => void): Promise<void> {
+    async init(
+        delegate: IServerDelegate,
+        onError: (error: Error) => void
+    ): Promise<void> {
         await super.init(delegate, onError);
-        const queues = await this.client.hGetAll("queues");
+        const queues = await this.client.hGetAll('queues');
         for (const queueId in queues) {
-            const active = await this.client.hGet("queues", queueId);
-            if (active === "true") {
+            const active = await this.client.hGet('queues', queueId);
+            if (active === 'true') {
                 this.queues.push(new RedisQueue(queueId, this.client));
             }
         }
@@ -114,7 +125,7 @@ export class RedisQueueManager extends BaseQueueManager implements IQueueManager
 
     getQueue(driveId: string, documentId?: string) {
         const queueId = this.getQueueId(driveId, documentId);
-        let queue = this.queues.find((q) => q.getId() === queueId);
+        let queue = this.queues.find(q => q.getId() === queueId);
 
         if (!queue) {
             queue = new RedisQueue(queueId, this.client);
@@ -128,6 +139,6 @@ export class RedisQueueManager extends BaseQueueManager implements IQueueManager
         super.removeQueue(driveId, documentId);
 
         const queueId = this.getQueueId(driveId, documentId);
-        this.client.hDel("queues", queueId);
+        this.client.hDel('queues', queueId);
     }
-}
+}

package/src/queue/types.ts

@@ -1,20 +1,20 @@
-import { Action, Operation } from "document-model/document";
-import { AddOperationOptions, IOperationResult } from "../server";
-import type { Unsubscribe } from "nanoevents";
+import { Action, Operation } from 'document-model/document';
+import type { Unsubscribe } from 'nanoevents';
+import { AddOperationOptions, IOperationResult } from '../server';
 
 export interface BaseJob {
     driveId: string;
-    documentId?: string
-    actions?: Action[]
-    options?: AddOperationOptions;    
+    documentId?: string;
+    actions?: Action[];
+    options?: AddOperationOptions;
 }
 
 export interface OperationJob extends BaseJob {
-    operations: Operation[]
+    operations: Operation[];
 }
 
 export interface ActionJob extends BaseJob {
-    actions: Action[]
+    actions: Action[];
 }
 
 export type Job = OperationJob | ActionJob;
@@ -28,9 +28,12 @@ export interface QueueEvents {
 }
 
 export interface IServerDelegate {
-    checkDocumentExists: (driveId: string, documentId: string) => Promise<boolean>;
+    checkDocumentExists: (
+        driveId: string,
+        documentId: string
+    ) => Promise<boolean>;
     processJob: (job: Job) => Promise<IOperationResult>;
-};
+}
 
 export interface IQueueManager {
     addJob(job: Job): Promise<JobId>;
@@ -38,7 +41,10 @@ export interface IQueueManager {
     removeQueue(driveId: string, documentId?: string): void;
     getQueueByIndex(index: number): IQueue<Job, IOperationResult> | null;
     getQueues(): string[];
-    init(delegate: IServerDelegate, onError: (error: Error) => void): Promise<void>;
+    init(
+        delegate: IServerDelegate,
+        onError: (error: Error) => void
+    ): Promise<void>;
     on<K extends keyof QueueEvents>(
         this: this,
         event: K,
@@ -65,9 +71,9 @@ export interface IQueue<T, R> {
 export type IJobQueue = IQueue<Job, IOperationResult>;
 
 export function isOperationJob(job: Job): job is OperationJob {
-    return "operations" in job;
+    return 'operations' in job;
 }
 
 export function isActionJob(job: Job): job is ActionJob {
-    return "actions" in job;
-}
+    return 'actions' in job;
+}

package/src/server/index.ts

@@ -220,6 +220,7 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
             }
 
             if (PullResponderTransmitter.isPullResponderTrigger(trigger)) {
+                let firstPull = true;
                 const cancelPullLoop = PullResponderTransmitter.setupPull(
                     driveId,
                     trigger,
@@ -269,6 +270,37 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                                 );
                             }
                         }
+
+                        // if it is the first pull and returns empty
+                        // then updates corresponding push transmitter
+                        if (firstPull) {
+                            firstPull = false;
+                            const pushListener =
+                                drive.state.local.listeners.find(
+                                    listener =>
+                                        trigger.data.url ===
+                                        listener.callInfo?.data
+                                );
+                            if (pushListener) {
+                                this.getSynchronizationUnitsRevision(
+                                    driveId,
+                                    syncUnits
+                                )
+                                    .then(syncUnitRevisions => {
+                                        for (const revision of syncUnitRevisions) {
+                                            this.listenerStateManager
+                                                .updateListenerRevision(
+                                                    pushListener.listenerId,
+                                                    driveId,
+                                                    revision.syncId,
+                                                    revision.revision
+                                                )
+                                                .catch(logger.error);
+                                        }
+                                    })
+                                    .catch(logger.error);
+                            }
+                        }
                     }
                 );
                 driveTriggers.set(trigger.id, cancelPullLoop);
@@ -405,16 +437,30 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
             documentType,
             drive
         );
-        const revisions = await this.storage.getSynchronizationUnitsRevision(
-            synchronizationUnitsQuery
+        return this.getSynchronizationUnitsRevision(
+            driveId,
+            synchronizationUnitsQuery,
+            drive
         );
+    }
+
+    public async getSynchronizationUnitsRevision(
+        driveId: string,
+        syncUnitsQuery: SynchronizationUnitQuery[],
+        loadedDrive?: DocumentDriveDocument
+    ): Promise<SynchronizationUnit[]> {
+        const drive = loadedDrive || (await this.getDrive(driveId));
 
-        const synchronizationUnits: SynchronizationUnit[] =
-            synchronizationUnitsQuery.map(s => ({
+        const revisions =
+            await this.storage.getSynchronizationUnitsRevision(syncUnitsQuery);
+
+        const synchronizationUnits: SynchronizationUnit[] = syncUnitsQuery.map(
+            s => ({
                 ...s,
                 lastUpdated: drive.created,
                 revision: -1
-            }));
+            })
+        );
         for (const revision of revisions) {
             const syncUnit = synchronizationUnits.find(
                 s =>
@@ -1368,37 +1414,42 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         actions: (DocumentDriveAction | BaseAction)[],
         options?: AddOperationOptions
     ): Promise<IOperationResult<DocumentDriveDocument>> {
-        const jobId = await this.queueManager.addJob({
-            driveId: drive,
-            actions,
-            options
-        });
-        return new Promise<IOperationResult<DocumentDriveDocument>>(
-            (resolve, reject) => {
-                const unsubscribe = this.queueManager.on(
-                    'jobCompleted',
-                    (job, result) => {
-                        if (job.jobId === jobId) {
-                            unsubscribe();
-                            unsubscribeError();
-                            resolve(
-                                result as IOperationResult<DocumentDriveDocument>
-                            );
+        try {
+            const jobId = await this.queueManager.addJob({
+                driveId: drive,
+                actions,
+                options
+            });
+            return new Promise<IOperationResult<DocumentDriveDocument>>(
+                (resolve, reject) => {
+                    const unsubscribe = this.queueManager.on(
+                        'jobCompleted',
+                        (job, result) => {
+                            if (job.jobId === jobId) {
+                                unsubscribe();
+                                unsubscribeError();
+                                resolve(
+                                    result as IOperationResult<DocumentDriveDocument>
+                                );
+                            }
                         }
-                    }
-                );
-                const unsubscribeError = this.queueManager.on(
-                    'jobFailed',
-                    (job, error) => {
-                        if (job.jobId === jobId) {
-                            unsubscribe();
-                            unsubscribeError();
-                            reject(error);
+                    );
+                    const unsubscribeError = this.queueManager.on(
+                        'jobFailed',
+                        (job, error) => {
+                            if (job.jobId === jobId) {
+                                unsubscribe();
+                                unsubscribeError();
+                                reject(error);
+                            }
                         }
-                    }
-                );
-            }
-        );
+                    );
+                }
+            );
+        } catch (error) {
+            logger.error('Error adding drive job', error);
+            throw error;
+        }
     }
 
     async addOperations(
@@ -1657,38 +1708,42 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         if (result) {
             return result;
         }
-
-        const jobId = await this.queueManager.addJob({
-            driveId: drive,
-            operations,
-            options
-        });
-        return new Promise<IOperationResult<DocumentDriveDocument>>(
-            (resolve, reject) => {
-                const unsubscribe = this.queueManager.on(
-                    'jobCompleted',
-                    (job, result) => {
-                        if (job.jobId === jobId) {
-                            unsubscribe();
-                            unsubscribeError();
-                            resolve(
-                                result as IOperationResult<DocumentDriveDocument>
-                            );
+        try {
+            const jobId = await this.queueManager.addJob({
+                driveId: drive,
+                operations,
+                options
+            });
+            return new Promise<IOperationResult<DocumentDriveDocument>>(
+                (resolve, reject) => {
+                    const unsubscribe = this.queueManager.on(
+                        'jobCompleted',
+                        (job, result) => {
+                            if (job.jobId === jobId) {
+                                unsubscribe();
+                                unsubscribeError();
+                                resolve(
+                                    result as IOperationResult<DocumentDriveDocument>
+                                );
+                            }
                         }
-                    }
-                );
-                const unsubscribeError = this.queueManager.on(
-                    'jobFailed',
-                    (job, error) => {
-                        if (job.jobId === jobId) {
-                            unsubscribe();
-                            unsubscribeError();
-                            reject(error);
+                    );
+                    const unsubscribeError = this.queueManager.on(
+                        'jobFailed',
+                        (job, error) => {
+                            if (job.jobId === jobId) {
+                                unsubscribe();
+                                unsubscribeError();
+                                reject(error);
+                            }
                         }
-                    }
-                );
-            }
-        );
+                    );
+                }
+            );
+        } catch (error) {
+            logger.error('Error adding drive job', error);
+            throw error;
+        }
     }
 
     async addDriveOperations(

package/src/server/listener/manager.ts

@@ -160,6 +160,15 @@ export class ListenerManager extends BaseListenerManager {
             ) {
                 continue;
             }
+
+            const transmitter = await this.getTransmitter(
+                driveId,
+                listener.listener.listenerId
+            );
+            if (!transmitter?.transmit) {
+                continue;
+            }
+
             for (const syncUnit of syncUnits) {
                 if (!this._checkFilter(listener.listener.filter, syncUnit)) {
                     continue;
@@ -229,7 +238,7 @@ export class ListenerManager extends BaseListenerManager {
         for (const [driveId, drive] of this.listenerState) {
             for (const [id, listener] of drive) {
                 const transmitter = await this.getTransmitter(driveId, id);
-                if (!transmitter) {
+                if (!transmitter?.transmit) {
                     continue;
                 }
 
@@ -239,6 +248,8 @@ export class ListenerManager extends BaseListenerManager {
                 );
 
                 const strandUpdates: StrandUpdate[] = [];
+
+                // TODO change to push one after the other, reusing operation data
                 await Promise.all(
                     syncUnits.map(async syncUnit => {
                         const unitState = listener.syncUnits.get(
@@ -292,7 +303,7 @@ export class ListenerManager extends BaseListenerManager {
 
                 // TODO update listeners in parallel, blocking for listeners with block=true
                 try {
-                    const listenerRevisions = await transmitter?.transmit(
+                    const listenerRevisions = await transmitter.transmit(
                         strandUpdates,
                         source
                     );
@@ -321,22 +332,31 @@ export class ListenerManager extends BaseListenerManager {
                             );
                         }
                     }
-                    const revisionError = listenerRevisions.find(
-                        l => l.status !== 'SUCCESS'
-                    );
-                    if (revisionError) {
-                        throw new OperationError(
-                            revisionError.status as ErrorStatus,
-                            undefined,
-                            revisionError.error,
-                            revisionError.error
-                        );
+
+                    for (const revision of listenerRevisions) {
+                        const error = revision.status === 'ERROR';
+                        if (revision.error?.includes('Missing operations')) {
+                            const updates = await this._triggerUpdate(
+                                source,
+                                onError
+                            );
+                            listenerUpdates.push(...updates);
+                        } else {
+                            listenerUpdates.push({
+                                listenerId: listener.listener.listenerId,
+                                listenerRevisions
+                            });
+                            if (error) {
+                                throw new OperationError(
+                                    revision.status as ErrorStatus,
+                                    undefined,
+                                    revision.error,
+                                    revision.error
+                                );
+                            }
+                        }
                     }
                     listener.listenerStatus = 'SUCCESS';
-                    listenerUpdates.push({
-                        listenerId: listener.listener.listenerId,
-                        listenerRevisions
-                    });
                 } catch (e) {
                     // TODO: Handle error based on listener params (blocking, retry, etc)
                     onError?.(e as Error, driveId, listener);

package/src/server/listener/transmitter/index.ts

@@ -1,4 +1,4 @@
+export * from './internal';
 export * from './pull-responder';
 export * from './switchboard-push';
-export * from './internal';
 export * from './types';

package/src/server/listener/transmitter/internal.ts

@@ -1,4 +1,5 @@
 import { Document, OperationScope } from 'document-model/document';
+import { logger } from '../../../utils/logger';
 import {
     BaseDocumentDriveServer,
     Listener,
@@ -8,7 +9,6 @@ import {
 } from '../../types';
 import { buildRevisionsFilter } from '../../utils';
 import { ITransmitter } from './types';
-import { logger } from '../../../utils/logger';
 
 export interface IReceiver {
     transmit: (strands: InternalTransmitterUpdate[]) => Promise<void>;
@@ -56,10 +56,10 @@ export class InternalTransmitter implements ITransmitter {
                 );
                 document = await (strand.documentId
                     ? this.drive.getDocument(
-                        strand.driveId,
-                        strand.documentId,
-                        { revisions }
-                    )
+                          strand.driveId,
+                          strand.documentId,
+                          { revisions }
+                      )
                     : this.drive.getDrive(strand.driveId, { revisions }));
                 retrievedDocuments.set(
                     `${strand.driveId}:${strand.documentId}`,

package/src/server/listener/transmitter/pull-responder.ts

@@ -59,10 +59,6 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
         this.manager = manager;
     }
 
-    async transmit(): Promise<ListenerRevision[]> {
-        return [];
-    }
-
     getStrands(since?: string | undefined): Promise<StrandUpdate[]> {
         return this.manager.getStrands(
             this.listener.driveId,

package/src/server/listener/transmitter/types.ts

@@ -11,7 +11,7 @@ export type StrandUpdateSource =
     | { type: 'trigger'; trigger: Trigger };
 
 export interface ITransmitter {
-    transmit(
+    transmit?(
         strands: StrandUpdate[],
         source: StrandUpdateSource
     ): Promise<ListenerRevision[]>;

package/src/server/utils.ts

@@ -32,4 +32,3 @@ export function filterOperationsByRevision(
         return acc;
     }, operations);
 }
-

package/src/storage/browser.ts

@@ -7,11 +7,7 @@ import {
     OperationScope
 } from 'document-model/document';
 import { mergeOperations, type SynchronizationUnitQuery } from '..';
-import {
-    DocumentDriveStorage,
-    DocumentStorage,
-    IDriveStorage,
-} from './types';
+import { DocumentDriveStorage, DocumentStorage, IDriveStorage } from './types';
 
 export class BrowserStorage implements IDriveStorage {
     private db: Promise<LocalForage>;

package/src/storage/filesystem.ts

@@ -1,5 +1,10 @@
 import { DocumentDriveAction } from 'document-model-libs/document-drive';
-import { BaseAction, DocumentHeader, Operation, OperationScope } from 'document-model/document';
+import {
+    BaseAction,
+    DocumentHeader,
+    Operation,
+    OperationScope
+} from 'document-model/document';
 import type { Dirent } from 'fs';
 import {
     existsSync,
@@ -202,7 +207,13 @@ export class FilesystemStorage implements IDriveStorage {
         // get oldes drives first
         const drives = (await this.getDrives()).reverse();
         for (const drive of drives) {
-            const { initialState: { state: { global: { slug: driveSlug } } } } = await this.getDrive(drive);
+            const {
+                initialState: {
+                    state: {
+                        global: { slug: driveSlug }
+                    }
+                }
+            } = await this.getDrive(drive);
             if (driveSlug === slug) {
                 return this.getDrive(drive);
             }

package/src/storage/memory.ts

@@ -6,13 +6,9 @@ import {
     Operation,
     OperationScope
 } from 'document-model/document';
-import {
-    DocumentDriveStorage,
-    DocumentStorage,
-    IDriveStorage,
-} from './types';
 import type { SynchronizationUnitQuery } from '../server/types';
 import { mergeOperations } from '../utils';
+import { DocumentDriveStorage, DocumentStorage, IDriveStorage } from './types';
 
 export class MemoryStorage implements IDriveStorage {
     private documents: Record<string, Record<string, DocumentStorage>>;

package/src/storage/prisma.ts

@@ -31,14 +31,14 @@ import {
 
 type Transaction =
     | Omit<
-        PrismaClient<Prisma.PrismaClientOptions, never>,
-        | '$connect'
-        | '$disconnect'
-        | '$on'
-        | '$transaction'
-        | '$use'
-        | '$extends'
-    >
+          PrismaClient<Prisma.PrismaClientOptions, never>,
+          | '$connect'
+          | '$disconnect'
+          | '$on'
+          | '$transaction'
+          | '$use'
+          | '$extends'
+      >
     | ExtendedPrismaClient;
 
 function storageToOperation(
@@ -80,9 +80,13 @@ function getRetryTransactionsClient<T extends PrismaClient>(
                 // eslint-disable-next-line prefer-spread
                 return backOff(() => prisma.$transaction.apply(prisma, args), {
                     retry: e => {
+                        const code = (e as { code: string }).code;
                         // Retry the transaction only if the error was due to a write conflict or deadlock
                         // See: https://www.prisma.io/docs/reference/api-reference/error-reference#p2034
-                        return (e as { code: string }).code === 'P2034';
+                        if (code !== 'P2034') {
+                            logger.error('TRANSACTION ERROR', e);
+                        }
+                        return code === 'P2034';
                     },
                     ...backOffOptions
                 });
@@ -502,9 +506,16 @@ export class PrismaStorage implements IDriveStorage {
                     id: id
                 }
             });
-        } catch (e: any) {
+        } catch (e: unknown) {
+            const prismaError = e as { code?: string; message?: string };
             // Ignore Error: P2025: An operation failed because it depends on one or more records that were required but not found.
-            if ((e.code && e.code === "P2025") || (e.message && e.message.includes("An operation failed because it depends on one or more records that were required but not found."))) {
+            if (
+                (prismaError.code && prismaError.code === 'P2025') ||
+                (prismaError.message &&
+                    prismaError.message.includes(
+                        'An operation failed because it depends on one or more records that were required but not found.'
+                    ))
+            ) {
                 return;
             }
 

package/src/storage/sequelize.ts

@@ -10,8 +10,8 @@ import {
     OperationScope
 } from 'document-model/document';
 import { DataTypes, Options, Sequelize } from 'sequelize';
-import { DocumentDriveStorage, DocumentStorage, IDriveStorage } from './types';
 import type { SynchronizationUnitQuery } from '../server/types';
+import { DocumentDriveStorage, DocumentStorage, IDriveStorage } from './types';
 
 export class SequelizeStorage implements IDriveStorage {
     private db: Sequelize;

package/src/storage/types.ts

@@ -1,6 +1,6 @@
 import type {
     DocumentDriveAction,
-    DocumentDriveDocument,
+    DocumentDriveDocument
 } from 'document-model-libs/document-drive';
 import type {
     Action,
@@ -8,7 +8,7 @@ import type {
     Document,
     DocumentHeader,
     DocumentOperations,
-    Operation,
+    Operation
 } from 'document-model/document';
 import type { SynchronizationUnitQuery } from '../server/types';
 
@@ -20,7 +20,10 @@ export type DocumentStorage<D extends Document = Document> = Omit<
 export type DocumentDriveStorage = DocumentStorage<DocumentDriveDocument>;
 
 export interface IStorageDelegate {
-    getCachedOperations(drive: string, id: string): Promise<DocumentOperations<Action> | undefined>;
+    getCachedOperations(
+        drive: string,
+        id: string
+    ): Promise<DocumentOperations<Action> | undefined>;
 }
 
 export interface IStorage {
@@ -36,7 +39,7 @@ export interface IStorage {
         drive: string,
         id: string,
         operations: Operation[],
-        header: DocumentHeader,
+        header: DocumentHeader
     ): Promise<void>;
     addDocumentOperationsWithTransaction?(
         drive: string,
@@ -47,16 +50,24 @@ export interface IStorage {
         }>
     ): Promise<void>;
     deleteDocument(drive: string, id: string): Promise<void>;
-    getOperationResultingState?(drive: string, id: string, index: number, scope: string, branch: string): Promise<unknown>;
+    getOperationResultingState?(
+        drive: string,
+        id: string,
+        index: number,
+        scope: string,
+        branch: string
+    ): Promise<unknown>;
     setStorageDelegate?(delegate: IStorageDelegate): void;
-    getSynchronizationUnitsRevision(units: SynchronizationUnitQuery[]): Promise<{
-        driveId: string;
-        documentId: string;
-        scope: string;
-        branch: string;
-        lastUpdated: string;
-        revision: number;
-    }[]>
+    getSynchronizationUnitsRevision(units: SynchronizationUnitQuery[]): Promise<
+        {
+            driveId: string;
+            documentId: string;
+            scope: string;
+            branch: string;
+            lastUpdated: string;
+            revision: number;
+        }[]
+    >;
 }
 export interface IDriveStorage extends IStorage {
     getDrives(): Promise<string[]>;
@@ -77,5 +88,10 @@ export interface IDriveStorage extends IStorage {
             header: DocumentHeader;
         }>
     ): Promise<void>;
-    getDriveOperationResultingState?(drive: string, index: number, scope: string, branch: string): Promise<unknown>;
-}
+    getDriveOperationResultingState?(
+        drive: string,
+        index: number,
+        scope: string,
+        branch: string
+    ): Promise<unknown>;
+}

package/src/utils/document-helpers.ts

@@ -1,6 +1,7 @@
 import { utils } from 'document-model/document';
 
-export const { attachBranch,
+export const {
+    attachBranch,
     garbageCollect,
     groupOperationsByScope,
     merge,

package/src/utils/index.ts

@@ -1,8 +1,6 @@
-import { v4 as uuidv4 } from 'uuid';
 import {
     DocumentDriveDocument,
-    documentModel as DocumentDriveModel,
-    z
+    documentModel as DocumentDriveModel
 } from 'document-model-libs/document-drive';
 import {
     Action,
@@ -12,49 +10,51 @@ import {
     Operation,
     OperationScope
 } from 'document-model/document';
+import { v4 as uuidv4 } from 'uuid';
 import { OperationError } from '../server/error';
 import { DocumentDriveStorage, DocumentStorage } from '../storage';
 
 export function isDocumentDriveStorage(
     document: DocumentStorage
 ): document is DocumentDriveStorage {
-    return (
-        document.documentType === DocumentDriveModel.id
-    );
+    return document.documentType === DocumentDriveModel.id;
 }
 
 export function isDocumentDrive(
     document: Document
 ): document is DocumentDriveDocument {
-    return (
-        document.documentType === DocumentDriveModel.id
-    );
+    return document.documentType === DocumentDriveModel.id;
 }
 
 export function mergeOperations<A extends Action = Action>(
     currentOperations: DocumentOperations<A>,
     newOperations: Operation<A | BaseAction>[]
 ): DocumentOperations<A> {
-    const minIndexByScope = Object.keys(currentOperations).reduce<Partial<Record<OperationScope, number>>>((acc, curr) => {
+    const minIndexByScope = Object.keys(currentOperations).reduce<
+        Partial<Record<OperationScope, number>>
+    >((acc, curr) => {
         const scope = curr as OperationScope;
-        acc[scope] = currentOperations[scope].at(-1)?.index ?? 0
+        acc[scope] = currentOperations[scope].at(-1)?.index ?? 0;
         return acc;
     }, {});
 
-    const conflictOp = newOperations.find(op => op.index < (minIndexByScope[op.scope] ?? 0));
+    const conflictOp = newOperations.find(
+        op => op.index < (minIndexByScope[op.scope] ?? 0)
+    );
     if (conflictOp) {
         throw new OperationError(
-            "ERROR",
+            'ERROR',
             conflictOp,
             `Tried to add operation with index ${conflictOp.index} and document is at index ${minIndexByScope[conflictOp.scope]}`
         );
     }
 
-    return newOperations.sort((a, b) => a.index - b.index
-    ).reduce<DocumentOperations<A>>((acc, curr) => {
-        const existingOperations = acc[curr.scope] || [];
-        return { ...acc, [curr.scope]: [...existingOperations, curr] };
-    }, currentOperations);
+    return newOperations
+        .sort((a, b) => a.index - b.index)
+        .reduce<DocumentOperations<A>>((acc, curr) => {
+            const existingOperations = acc[curr.scope] || [];
+            return { ...acc, [curr.scope]: [...existingOperations, curr] };
+        }, currentOperations);
 }
 
 export function generateUUID(): string {
@@ -86,4 +86,3 @@ export function isNoopUpdate(
 export function isBefore(dateA: Date | string, dateB: Date | string) {
     return new Date(dateA) < new Date(dateB);
 }
-

package/src/utils/logger.ts

@@ -1,5 +1,8 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
-export type ILogger = Pick<Console, 'log' | 'info' | 'warn' | 'error' | 'debug' | 'trace'>;
+export type ILogger = Pick<
+    Console,
+    'log' | 'info' | 'warn' | 'error' | 'debug' | 'trace'
+>;
 class Logger implements ILogger {
     #logger: ILogger = console;