document-drive 1.0.0-alpha.84 → 1.0.0-alpha.85

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "document-drive",
-    "version": "1.0.0-alpha.84",
+    "version": "1.0.0-alpha.85",
     "license": "AGPL-3.0-only",
     "type": "module",
     "module": "./src/index.ts",

package/src/queue/base.ts

@@ -5,6 +5,7 @@ import {
 import { Action } from 'document-model/document';
 import { Unsubscribe, createNanoEvents } from 'nanoevents';
 import { generateUUID } from '../utils';
+import { logger } from '../utils/logger';
 import {
     IJob,
     IJobQueue,
@@ -172,7 +173,6 @@ export class BaseQueueManager implements IQueueManager {
             const queue = this.getQueue(job.driveId, input.id);
             await queue.setDeleted(true);
         }
-
         await queue.addJob({ jobId, ...job });
 
         return jobId;
@@ -209,14 +209,30 @@ export class BaseQueueManager implements IQueueManager {
         return this.queues.map(q => q.getId());
     }
 
-    private retryNextJob() {
+    private retryNextJob(timeout?: number) {
+        const _timeout = timeout !== undefined ? timeout : this.timeout;
         const retry =
-            this.timeout === 0 && typeof setImmediate !== 'undefined'
+            _timeout === 0 && typeof setImmediate !== 'undefined'
                 ? setImmediate
-                : (fn: () => void) => setTimeout(fn, this.timeout);
+                : (fn: () => void) => setTimeout(fn, _timeout);
         return retry(() => this.processNextJob());
     }
 
+    private async findFirstNonEmptyQueue(
+        ticker: number
+    ): Promise<number | null> {
+        const numQueues = this.queues.length;
+
+        for (let i = 0; i < numQueues; i++) {
+            const index = (ticker + i) % numQueues;
+            const queue = this.queues[index];
+            if (queue && (await queue.amountOfJobs()) > 0) {
+                return index;
+            }
+        }
+        return null;
+    }
+
     private async processNextJob() {
         if (!this.delegate) {
             throw new Error('No server delegate defined');
@@ -228,20 +244,30 @@ export class BaseQueueManager implements IQueueManager {
         }
 
         const queue = this.queues[this.ticker];
-        this.ticker =
-            this.ticker === this.queues.length - 1 ? 0 : this.ticker + 1;
         if (!queue) {
             this.ticker = 0;
             this.retryNextJob();
             return;
         }
 
+        // if no jobs in the current queue then looks for the
+        // next queue with jobs. If no jobs in any queue then
+        // retries after a timeout
         const amountOfJobs = await queue.amountOfJobs();
         if (amountOfJobs === 0) {
-            this.retryNextJob();
+            const nextTicker = await this.findFirstNonEmptyQueue(this.ticker);
+            if (nextTicker !== null) {
+                this.ticker = nextTicker;
+                this.retryNextJob(0);
+            } else {
+                this.retryNextJob();
+            }
             return;
         }
 
+        this.ticker =
+            this.ticker === this.queues.length - 1 ? 0 : this.ticker + 1;
+
         const isBlocked = await queue.isBlocked();
         if (isBlocked) {
             this.retryNextJob();
@@ -274,10 +300,11 @@ export class BaseQueueManager implements IQueueManager {
             }
             this.emit('jobCompleted', nextJob, result);
         } catch (e) {
+            logger.error(`job failed`, e);
             this.emit('jobFailed', nextJob, e as Error);
         } finally {
             await queue.setBlocked(false);
-            await this.processNextJob();
+            this.retryNextJob(0);
         }
     }
 

package/src/server/index.ts

@@ -220,6 +220,7 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
             }
 
             if (PullResponderTransmitter.isPullResponderTrigger(trigger)) {
+                let firstPull = true;
                 const cancelPullLoop = PullResponderTransmitter.setupPull(
                     driveId,
                     trigger,
@@ -269,6 +270,37 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                                 );
                             }
                         }
+
+                        // if it is the first pull and returns empty
+                        // then updates corresponding push transmitter
+                        if (firstPull) {
+                            firstPull = false;
+                            const pushListener =
+                                drive.state.local.listeners.find(
+                                    listener =>
+                                        trigger.data.url ===
+                                        listener.callInfo?.data
+                                );
+                            if (pushListener) {
+                                this.getSynchronizationUnitsRevision(
+                                    driveId,
+                                    syncUnits
+                                )
+                                    .then(syncUnitRevisions => {
+                                        for (const revision of syncUnitRevisions) {
+                                            this.listenerStateManager
+                                                .updateListenerRevision(
+                                                    pushListener.listenerId,
+                                                    driveId,
+                                                    revision.syncId,
+                                                    revision.revision
+                                                )
+                                                .catch(logger.error);
+                                        }
+                                    })
+                                    .catch(logger.error);
+                            }
+                        }
                     }
                 );
                 driveTriggers.set(trigger.id, cancelPullLoop);
@@ -405,16 +437,30 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
             documentType,
             drive
         );
-        const revisions = await this.storage.getSynchronizationUnitsRevision(
-            synchronizationUnitsQuery
+        return this.getSynchronizationUnitsRevision(
+            driveId,
+            synchronizationUnitsQuery,
+            drive
         );
+    }
 
-        const synchronizationUnits: SynchronizationUnit[] =
-            synchronizationUnitsQuery.map(s => ({
+    public async getSynchronizationUnitsRevision(
+        driveId: string,
+        syncUnitsQuery: SynchronizationUnitQuery[],
+        loadedDrive?: DocumentDriveDocument
+    ): Promise<SynchronizationUnit[]> {
+        const drive = loadedDrive || (await this.getDrive(driveId));
+
+        const revisions =
+            await this.storage.getSynchronizationUnitsRevision(syncUnitsQuery);
+
+        const synchronizationUnits: SynchronizationUnit[] = syncUnitsQuery.map(
+            s => ({
                 ...s,
                 lastUpdated: drive.created,
                 revision: -1
-            }));
+            })
+        );
         for (const revision of revisions) {
             const syncUnit = synchronizationUnits.find(
                 s =>

package/src/server/listener/manager.ts

@@ -160,6 +160,15 @@ export class ListenerManager extends BaseListenerManager {
             ) {
                 continue;
             }
+
+            const transmitter = await this.getTransmitter(
+                driveId,
+                listener.listener.listenerId
+            );
+            if (!transmitter?.transmit) {
+                continue;
+            }
+
             for (const syncUnit of syncUnits) {
                 if (!this._checkFilter(listener.listener.filter, syncUnit)) {
                     continue;
@@ -229,7 +238,7 @@ export class ListenerManager extends BaseListenerManager {
         for (const [driveId, drive] of this.listenerState) {
             for (const [id, listener] of drive) {
                 const transmitter = await this.getTransmitter(driveId, id);
-                if (!transmitter) {
+                if (!transmitter?.transmit) {
                     continue;
                 }
 
@@ -239,6 +248,8 @@ export class ListenerManager extends BaseListenerManager {
                 );
 
                 const strandUpdates: StrandUpdate[] = [];
+
+                // TODO change to push one after the other, reusing operation data
                 await Promise.all(
                     syncUnits.map(async syncUnit => {
                         const unitState = listener.syncUnits.get(
@@ -292,7 +303,7 @@ export class ListenerManager extends BaseListenerManager {
 
                 // TODO update listeners in parallel, blocking for listeners with block=true
                 try {
-                    const listenerRevisions = await transmitter?.transmit(
+                    const listenerRevisions = await transmitter.transmit(
                         strandUpdates,
                         source
                     );
@@ -321,22 +332,31 @@ export class ListenerManager extends BaseListenerManager {
                             );
                         }
                     }
-                    const revisionError = listenerRevisions.find(
-                        l => l.status !== 'SUCCESS'
-                    );
-                    if (revisionError) {
-                        throw new OperationError(
-                            revisionError.status as ErrorStatus,
-                            undefined,
-                            revisionError.error,
-                            revisionError.error
-                        );
+
+                    for (const revision of listenerRevisions) {
+                        const error = revision.status === 'ERROR';
+                        if (revision.error?.includes('Missing operations')) {
+                            const updates = await this._triggerUpdate(
+                                source,
+                                onError
+                            );
+                            listenerUpdates.push(...updates);
+                        } else {
+                            listenerUpdates.push({
+                                listenerId: listener.listener.listenerId,
+                                listenerRevisions
+                            });
+                            if (error) {
+                                throw new OperationError(
+                                    revision.status as ErrorStatus,
+                                    undefined,
+                                    revision.error,
+                                    revision.error
+                                );
+                            }
+                        }
                     }
                     listener.listenerStatus = 'SUCCESS';
-                    listenerUpdates.push({
-                        listenerId: listener.listener.listenerId,
-                        listenerRevisions
-                    });
                 } catch (e) {
                     // TODO: Handle error based on listener params (blocking, retry, etc)
                     onError?.(e as Error, driveId, listener);

package/src/server/listener/transmitter/pull-responder.ts

@@ -59,10 +59,6 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
         this.manager = manager;
     }
 
-    async transmit(): Promise<ListenerRevision[]> {
-        return [];
-    }
-
     getStrands(since?: string | undefined): Promise<StrandUpdate[]> {
         return this.manager.getStrands(
             this.listener.driveId,

package/src/server/listener/transmitter/types.ts

@@ -11,7 +11,7 @@ export type StrandUpdateSource =
     | { type: 'trigger'; trigger: Trigger };
 
 export interface ITransmitter {
-    transmit(
+    transmit?(
         strands: StrandUpdate[],
         source: StrandUpdateSource
     ): Promise<ListenerRevision[]>;

package/src/storage/prisma.ts

@@ -80,9 +80,13 @@ function getRetryTransactionsClient<T extends PrismaClient>(
                 // eslint-disable-next-line prefer-spread
                 return backOff(() => prisma.$transaction.apply(prisma, args), {
                     retry: e => {
+                        const code = (e as { code: string }).code;
                         // Retry the transaction only if the error was due to a write conflict or deadlock
                         // See: https://www.prisma.io/docs/reference/api-reference/error-reference#p2034
-                        return (e as { code: string }).code === 'P2034';
+                        if (code !== 'P2034') {
+                            logger.error('TRANSACTION ERROR', e);
+                        }
+                        return code === 'P2034';
                     },
                     ...backOffOptions
                 });
@@ -502,12 +506,13 @@ export class PrismaStorage implements IDriveStorage {
                     id: id
                 }
             });
-        } catch (e: any) {
+        } catch (e: unknown) {
+            const prismaError = e as { code?: string; message?: string };
             // Ignore Error: P2025: An operation failed because it depends on one or more records that were required but not found.
             if (
-                (e.code && e.code === 'P2025') ||
-                (e.message &&
-                    e.message.includes(
+                (prismaError.code && prismaError.code === 'P2025') ||
+                (prismaError.message &&
+                    prismaError.message.includes(
                         'An operation failed because it depends on one or more records that were required but not found.'
                     ))
             ) {