@powerhousedao/reactor 6.0.0-dev.6 → 6.0.0-dev.60

package/dist/src/sync/sync-builder.js

@@ -1,10 +1,13 @@
 import { KyselySyncCursorStorage } from "../storage/kysely/sync-cursor-storage.js";
+import { KyselySyncDeadLetterStorage } from "../storage/kysely/sync-dead-letter-storage.js";
 import { KyselySyncRemoteStorage } from "../storage/kysely/sync-remote-storage.js";
 import { SyncManager } from "./sync-manager.js";
 export class SyncBuilder {
     channelFactory;
     remoteStorage;
     cursorStorage;
+    deadLetterStorage;
+    maxDeadLettersPerRemote = 100;
     withChannelFactory(factory) {
         this.channelFactory = factory;
         return this;
@@ -17,6 +20,14 @@ export class SyncBuilder {
         this.cursorStorage = storage;
         return this;
     }
+    withDeadLetterStorage(storage) {
+        this.deadLetterStorage = storage;
+        return this;
+    }
+    withMaxDeadLettersPerRemote(limit) {
+        this.maxDeadLettersPerRemote = limit;
+        return this;
+    }
     build(reactor, logger, operationIndex, eventBus, db) {
         const module = this.buildModule(reactor, logger, operationIndex, eventBus, db);
         return module.syncManager;
@@ -27,10 +38,12 @@ export class SyncBuilder {
         }
         const remoteStorage = this.remoteStorage ?? new KyselySyncRemoteStorage(db);
         const cursorStorage = this.cursorStorage ?? new KyselySyncCursorStorage(db);
-        const syncManager = new SyncManager(logger, remoteStorage, cursorStorage, this.channelFactory, operationIndex, reactor, eventBus);
+        const deadLetterStorage = this.deadLetterStorage ?? new KyselySyncDeadLetterStorage(db);
+        const syncManager = new SyncManager(logger, remoteStorage, cursorStorage, deadLetterStorage, this.channelFactory, operationIndex, reactor, eventBus, this.maxDeadLettersPerRemote);
         return {
             remoteStorage,
             cursorStorage,
+            deadLetterStorage,
             channelFactory: this.channelFactory,
             syncManager,
         };

package/dist/src/sync/sync-builder.js.map

@@ -1 +1 @@
-{"version":3,"file":"sync-builder.js","sourceRoot":"","sources":["../../../src/sync/sync-builder.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,uBAAuB,EAAE,MAAM,0CAA0C,CAAC;AACnF,OAAO,EAAE,uBAAuB,EAAE,MAAM,0CAA0C,CAAC;AAGnF,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,OAAO,WAAW;IACd,cAAc,CAAmB;IACjC,aAAa,CAAsB;IACnC,aAAa,CAAsB;IAE3C,kBAAkB,CAAC,OAAwB;QACzC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,iBAAiB,CAAC,OAA2B;QAC3C,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,iBAAiB,CAAC,OAA2B;QAC3C,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CACH,OAAiB,EACjB,MAAe,EACf,cAA+B,EAC/B,QAAmB,EACnB,EAAoB;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAC7B,OAAO,EACP,MAAM,EACN,cAAc,EACd,QAAQ,EACR,EAAE,CACH,CAAC;QACF,OAAO,MAAM,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,WAAW,CACT,OAAiB,EACjB,MAAe,EACf,cAA+B,EAC/B,QAAmB,EACnB,EAAoB;QAEpB,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,uBAAuB,CAAC,EAAE,CAAC,CAAC;QAC5E,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,uBAAuB,CAAC,EAAE,CAAC,CAAC;QAE5E,MAAM,WAAW,GAAG,IAAI,WAAW,CACjC,MAAM,EACN,aAAa,EACb,aAAa,EACb,IAAI,CAAC,cAAc,EACnB,cAAc,EACd,OAAO,EACP,QAAQ,CACT,CAAC;QAEF,OAAO;YACL,aAAa;YACb,aAAa;YACb,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,WAAW;SACZ,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"sync-builder.js","sourceRoot":"","sources":["../../../src/sync/sync-builder.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,uBAAuB,EAAE,MAAM,0CAA0C,CAAC;AACnF,OAAO,EAAE,2BAA2B,EAAE,MAAM,+CAA+C,CAAC;AAC5F,OAAO,EAAE,uBAAuB,EAAE,MAAM,0CAA0C,CAAC;AAGnF,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,OAAO,WAAW;IACd,cAAc,CAAmB;IACjC,aAAa,CAAsB;IACnC,aAAa,CAAsB;IACnC,iBAAiB,CAA0B;IAC3C,uBAAuB,GAAW,GAAG,CAAC;IAE9C,kBAAkB,CAAC,OAAwB;QACzC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,iBAAiB,CAAC,OAA2B;QAC3C,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,iBAAiB,CAAC,OAA2B;QAC3C,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,qBAAqB,CAAC,OAA+B;QACnD,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC;QACjC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,2BAA2B,CAAC,KAAa;QACvC,IAAI,CAAC,uBAAuB,GAAG,KAAK,CAAC;QACrC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CACH,OAAiB,EACjB,MAAe,EACf,cAA+B,EAC/B,QAAmB,EACnB,EAAoB;QAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAC7B,OAAO,EACP,MAAM,EACN,cAAc,EACd,QAAQ,EACR,EAAE,CACH,CAAC;QACF,OAAO,MAAM,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,WAAW,CACT,OAAiB,EACjB,MAAe,EACf,cAA+B,EAC/B,QAAmB,EACnB,EAAoB;QAEpB,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,uBAAuB,CAAC,EAAE,CAAC,CAAC;QAC5E,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,uBAAuB,CAAC,EAAE,CAAC,CAAC;QAC5E,MAAM,iBAAiB,GACrB,IAAI,CAAC,iBAAiB,IAAI,IAAI,2BAA2B,CAAC,EAAE,CAAC,CAAC;QAEhE,MAAM,WAAW,GAAG,IAAI,WAAW,CACjC,MAAM,EACN,aAAa,EACb,aAAa,EACb,iBAAiB,EACjB,IAAI,CAAC,cAAc,EACnB,cAAc,EACd,OAAO,EACP,QAAQ,EACR,IAAI,CAAC,uBAAuB,CAC7B,CAAC;QAEF,OAAO;YACL,aAAa;YACb,aAAa;YACb,iBAAiB;YACjB,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,WAAW;SACZ,CAAC;IACJ,CAAC;CACF"}

package/dist/src/sync/sync-manager.d.ts

@@ -3,35 +3,48 @@ import type { IReactor } from "../core/types.js";
 import type { IEventBus } from "../events/interfaces.js";
 import type { ILogger } from "../logging/types.js";
 import { type JobInfo, type ShutdownStatus } from "../shared/types.js";
-import type { ISyncCursorStorage, ISyncRemoteStorage } from "../storage/interfaces.js";
+import type { ISyncCursorStorage, ISyncDeadLetterStorage, ISyncRemoteStorage } from "../storage/interfaces.js";
 import type { IChannelFactory, ISyncManager, Remote } from "./interfaces.js";
-import type { ChannelConfig, RemoteFilter, RemoteOptions } from "./types.js";
+import { type SyncStatus, type SyncStatusChangeCallback } from "./sync-status-tracker.js";
+import type { ChannelConfig, RemoteFilter, RemoteOptions, SyncResult } from "./types.js";
 export declare class SyncManager implements ISyncManager {
     private readonly logger;
     private readonly remoteStorage;
     private readonly cursorStorage;
+    private readonly deadLetterStorage;
     private readonly channelFactory;
-    private readonly _operationIndex;
+    private readonly operationIndex;
     private readonly reactor;
     private readonly eventBus;
     private readonly remotes;
     private readonly awaiter;
+    private readonly syncAwaiter;
     private isShutdown;
     private eventUnsubscribe?;
+    private failedEventUnsubscribe?;
+    private readonly batchAggregator;
+    private readonly syncStatusTracker;
+    private readonly maxDeadLettersPerRemote;
     loadJobs: Map<string, JobInfo>;
-    constructor(logger: ILogger, remoteStorage: ISyncRemoteStorage, cursorStorage: ISyncCursorStorage, channelFactory: IChannelFactory, operationIndex: IOperationIndex, reactor: IReactor, eventBus: IEventBus);
+    constructor(logger: ILogger, remoteStorage: ISyncRemoteStorage, cursorStorage: ISyncCursorStorage, deadLetterStorage: ISyncDeadLetterStorage, channelFactory: IChannelFactory, operationIndex: IOperationIndex, reactor: IReactor, eventBus: IEventBus, maxDeadLettersPerRemote?: number);
     startup(): Promise<void>;
     shutdown(): ShutdownStatus;
     getByName(name: string): Remote;
     getById(id: string): Remote;
     add(name: string, collectionId: string, channelConfig: ChannelConfig, filter?: RemoteFilter, options?: RemoteOptions, id?: string): Promise<Remote>;
-    private backfillOutbox;
     remove(name: string): Promise<void>;
     list(): Remote[];
+    waitForSync(jobId: string, signal?: AbortSignal): Promise<SyncResult>;
+    getSyncStatus(documentId: string): SyncStatus | undefined;
+    onSyncStatusChange(callback: SyncStatusChangeCallback): () => void;
     private wireChannelCallbacks;
-    private handleOperationWritten;
-    private handleInboxJob;
-    private handleOutboxJob;
+    private loadDeadLetters;
+    private getRemotesForCollection;
+    private processCompleteBatch;
+    private handleInboxAdded;
     private applyInboxJob;
+    private applyInboxBatch;
+    private updateOutbox;
+    private getOperationsForRemote;
 }
 //# sourceMappingURL=sync-manager.d.ts.map

package/dist/src/sync/sync-manager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sync-manager.d.ts","sourceRoot":"","sources":["../../../src/sync/sync-manager.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAKzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAEL,KAAK,OAAO,EACZ,KAAK,cAAc,EACpB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EACV,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,0BAA0B,CAAC;AAElC,OAAO,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAE7E,OAAO,KAAK,EACV,aAAa,EACb,YAAY,EACZ,aAAa,EAGd,MAAM,YAAY,CAAC;AAQpB,qBAAa,WAAY,YAAW,YAAY;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAkB;IACjD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAW;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAY;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsB;IAC9C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAa;IACrC,OAAO,CAAC,UAAU,CAAU;IAC5B,OAAO,CAAC,gBAAgB,CAAC,CAAa;IAE/B,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAa;gBAGhD,MAAM,EAAE,OAAO,EACf,aAAa,EAAE,kBAAkB,EACjC,aAAa,EAAE,kBAAkB,EACjC,cAAc,EAAE,eAAe,EAC/B,cAAc,EAAE,eAAe,EAC/B,OAAO,EAAE,QAAQ,EACjB,QAAQ,EAAE,SAAS;IAgBf,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IA6C9B,QAAQ,IAAI,cAAc;IA2B1B,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAQ/B,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IASrB,GAAG,CACP,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,aAAa,EAC5B,MAAM,GAAE,YAAwD,EAChE,OAAO,GAAE,aAAkB,EAC3B,EAAE,CAAC,EAAE,MAAM,GACV,OAAO,CAAC,MAAM,CAAC;YAkEJ,cAAc;IAsDtB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYzC,IAAI,IAAI,MAAM,EAAE;IAIhB,OAAO,CAAC,oBAAoB;IAU5B,OAAO,CAAC,sBAAsB;IAiC9B,OAAO,CAAC,cAAc;IAQtB,OAAO,CAAC,eAAe;YAUT,aAAa;CAsD5B"}
+{"version":3,"file":"sync-manager.d.ts","sourceRoot":"","sources":["../../../src/sync/sync-manager.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,KAAK,EAGV,QAAQ,EACT,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAMzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAEL,KAAK,OAAO,EACZ,KAAK,cAAc,EACpB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAEV,kBAAkB,EAClB,sBAAsB,EACtB,kBAAkB,EACnB,MAAM,0BAA0B,CAAC;AAGlC,OAAO,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAG7E,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,wBAAwB,EAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EACV,aAAa,EACb,YAAY,EACZ,aAAa,EAGb,UAAU,EACX,MAAM,YAAY,CAAC;AAUpB,qBAAa,WAAY,YAAW,YAAY;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAyB;IAC3D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAkB;IACjD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAkB;IACjD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAW;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAY;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsB;IAC9C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAa;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,UAAU,CAAU;IAC5B,OAAO,CAAC,gBAAgB,CAAC,CAAa;IACtC,OAAO,CAAC,sBAAsB,CAAC,CAAa;IAC5C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAoB;IACtD,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAS;IAE1C,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAa;gBAGhD,MAAM,EAAE,OAAO,EACf,aAAa,EAAE,kBAAkB,EACjC,aAAa,EAAE,kBAAkB,EACjC,iBAAiB,EAAE,sBAAsB,EACzC,cAAc,EAAE,eAAe,EAC/B,cAAc,EAAE,eAAe,EAC/B,OAAO,EAAE,QAAQ,EACjB,QAAQ,EAAE,SAAS,EACnB,uBAAuB,GAAE,MAAY;IAuBjC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IA6D9B,QAAQ,IAAI,cAAc;IA+B1B,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAQ/B,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IASrB,GAAG,CACP,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,aAAa,EAC5B,MAAM,GAAE,YAAwD,EAChE,OAAO,GAAE,aAA4C,EACrD,EAAE,CAAC,EAAE,MAAM,GACV,OAAO,CAAC,MAAM,CAAC;IA2EZ,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBzC,IAAI,IAAI,MAAM,EAAE;IAIhB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC;IAIrE,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAIzD,kBAAkB,CAAC,QAAQ,EAAE,wBAAwB,GAAG,MAAM,IAAI;IAIlE,OAAO,CAAC,oBAAoB;YAkDd,eAAe;IAqD7B,OAAO,CAAC,uBAAuB;YAMjB,oBAAoB;IAgClC,OAAO,CAAC,gBAAgB;YAyBV,aAAa;YA0Eb,eAAe;YAgFf,YAAY;YAiDZ,sBAAsB;CAyBrC"}

package/dist/src/sync/sync-manager.js

@@ -1,34 +1,48 @@
-import { OperationEventTypes, } from "../events/types.js";
+import { ReactorEventTypes, } from "../events/types.js";
 import { JobAwaiter } from "../shared/awaiter.js";
 import { JobStatus, } from "../shared/types.js";
+import { BatchAggregator } from "./batch-aggregator.js";
 import { ChannelError } from "./errors.js";
+import { SyncAwaiter } from "./sync-awaiter.js";
 import { SyncOperation } from "./sync-operation.js";
-import { ChannelErrorSource, SyncOperationStatus } from "./types.js";
-import { batchOperationsByDocument, createIdleHealth, filterOperations, } from "./utils.js";
+import { SyncStatusTracker, } from "./sync-status-tracker.js";
+import { ChannelErrorSource } from "./types.js";
+import { batchOperationsByDocument, createIdleHealth, filterOperations, toOperationWithContext, trimMailboxFromBatch, } from "./utils.js";
 export class SyncManager {
     logger;
     remoteStorage;
     cursorStorage;
+    deadLetterStorage;
     channelFactory;
-    _operationIndex;
+    operationIndex;
     reactor;
     eventBus;
     remotes;
     awaiter;
+    syncAwaiter;
     isShutdown;
     eventUnsubscribe;
+    failedEventUnsubscribe;
+    batchAggregator;
+    syncStatusTracker;
+    maxDeadLettersPerRemote;
     loadJobs = new Map();
-    constructor(logger, remoteStorage, cursorStorage, channelFactory, operationIndex, reactor, eventBus) {
+    constructor(logger, remoteStorage, cursorStorage, deadLetterStorage, channelFactory, operationIndex, reactor, eventBus, maxDeadLettersPerRemote = 100) {
         this.logger = logger;
         this.remoteStorage = remoteStorage;
         this.cursorStorage = cursorStorage;
+        this.deadLetterStorage = deadLetterStorage;
         this.channelFactory = channelFactory;
-        this._operationIndex = operationIndex;
+        this.operationIndex = operationIndex;
         this.reactor = reactor;
         this.eventBus = eventBus;
+        this.maxDeadLettersPerRemote = maxDeadLettersPerRemote;
         this.remotes = new Map();
         this.awaiter = new JobAwaiter(eventBus, (jobId, signal) => reactor.getJobStatus(jobId, signal));
+        this.syncAwaiter = new SyncAwaiter(eventBus);
         this.isShutdown = false;
+        this.batchAggregator = new BatchAggregator(logger, (batch) => this.processCompleteBatch(batch));
+        this.syncStatusTracker = new SyncStatusTracker();
     }
     async startup() {
         if (this.isShutdown) {
@@ -36,14 +50,7 @@ export class SyncManager {
         }
         const remoteRecords = await this.remoteStorage.list();
         for (const record of remoteRecords) {
-            const channel = this.channelFactory.instance(record.id, record.name, record.channelConfig, this.cursorStorage, record.collectionId, record.filter);
-            try {
-                await channel.init();
-            }
-            catch (error) {
-                console.error(`Error initializing channel for remote ${record.name}: ${error instanceof Error ? error.message : String(error)}`);
-                continue;
-            }
+            const channel = this.channelFactory.instance(record.id, record.name, record.channelConfig, this.cursorStorage, record.collectionId, record.filter, this.operationIndex);
             const remote = {
                 id: record.id,
                 name: record.name,
@@ -53,28 +60,47 @@ export class SyncManager {
                 channel,
             };
             this.remotes.set(record.name, remote);
+            await this.loadDeadLetters(remote);
             this.wireChannelCallbacks(remote);
+            try {
+                await channel.init();
+            }
+            catch (error) {
+                this.logger.error("Error initializing channel for remote (@name, @error)", record.name, error instanceof Error ? error.message : String(error));
+                this.remotes.delete(record.name);
+                continue;
+            }
+            // backfill channels
+            const outboxAckOrdinal = remote.channel.outbox.ackOrdinal;
+            if (outboxAckOrdinal > 0) {
+                await this.updateOutbox(remote, outboxAckOrdinal);
+            }
         }
-        this.eventUnsubscribe = this.eventBus.subscribe(OperationEventTypes.OPERATION_WRITTEN, (_type, event) => this.handleOperationWritten(event));
+        this.eventUnsubscribe = this.eventBus.subscribe(ReactorEventTypes.JOB_WRITE_READY, async (_type, event) => this.batchAggregator.enqueueWriteReady(event));
+        this.failedEventUnsubscribe = this.eventBus.subscribe(ReactorEventTypes.JOB_FAILED, async (_type, event) => this.batchAggregator.handleJobFailed(event));
     }
     shutdown() {
         this.isShutdown = true;
+        this.batchAggregator.clear();
         if (this.eventUnsubscribe) {
             this.eventUnsubscribe();
             this.eventUnsubscribe = undefined;
         }
+        if (this.failedEventUnsubscribe) {
+            this.failedEventUnsubscribe();
+            this.failedEventUnsubscribe = undefined;
+        }
         this.awaiter.shutdown();
+        this.syncAwaiter.shutdown();
+        this.syncStatusTracker.clear();
+        const promises = [];
         for (const remote of this.remotes.values()) {
-            try {
-                remote.channel.shutdown();
-            }
-            catch (error) {
-                console.error(`Error shutting down channel for remote ${remote.name}: ${error instanceof Error ? error.message : String(error)}`);
-            }
+            promises.push(remote.channel.shutdown());
         }
         this.remotes.clear();
         return {
             isShutdown: true,
+            completed: Promise.all(promises).then(() => undefined),
         };
     }
     getByName(name) {
@@ -92,7 +118,7 @@ export class SyncManager {
         }
         throw new Error(`Remote with id '${id}' does not exist`);
     }
-    async add(name, collectionId, channelConfig, filter = { documentId: [], scope: [], branch: "" }, options = {}, id) {
+    async add(name, collectionId, channelConfig, filter = { documentId: [], scope: [], branch: "" }, options = { sinceTimestampUtcMs: "0" }, id) {
         if (this.isShutdown) {
             throw new Error("SyncManager is shutdown and cannot add remotes");
         }
@@ -115,8 +141,7 @@ export class SyncManager {
             status,
         };
         await this.remoteStorage.upsert(remoteRecord);
-        const channel = this.channelFactory.instance(remoteId, name, channelConfig, this.cursorStorage, collectionId, filter);
-        await channel.init();
+        const channel = this.channelFactory.instance(remoteId, name, channelConfig, this.cursorStorage, collectionId, filter, this.operationIndex);
         const remote = {
             id: remoteId,
             name,
@@ -126,103 +151,152 @@ export class SyncManager {
             channel,
         };
         this.remotes.set(name, remote);
+        await this.loadDeadLetters(remote);
         this.wireChannelCallbacks(remote);
-        await this.backfillOutbox(remote, collectionId, filter);
-        return remote;
-    }
-    async backfillOutbox(remote, collectionId, filter) {
-        let historicalOps;
         try {
-            historicalOps = await this._operationIndex.find(collectionId);
-        }
-        catch {
-            return;
+            await channel.init();
         }
-        if (historicalOps.items.length === 0) {
-            return;
-        }
-        const opsWithContext = historicalOps.items.map((entry) => ({
-            operation: {
-                id: entry.id,
-                index: entry.index,
-                skip: entry.skip,
-                hash: entry.hash,
-                timestampUtcMs: entry.timestampUtcMs,
-                action: entry.action,
-            },
-            context: {
-                documentId: entry.documentId,
-                documentType: entry.documentType,
-                scope: entry.scope,
-                branch: entry.branch,
-                ordinal: entry.ordinal ?? 0,
-            },
-        }));
-        const filteredOps = filterOperations(opsWithContext, filter);
-        if (filteredOps.length === 0) {
-            return;
-        }
-        const batches = batchOperationsByDocument(filteredOps);
-        for (const batch of batches) {
-            const syncOp = new SyncOperation(crypto.randomUUID(), remote.name, batch.documentId, [batch.scope], batch.branch, batch.operations);
-            remote.channel.outbox.add(syncOp);
+        catch (error) {
+            this.remotes.delete(name);
+            await this.remoteStorage.remove(name);
+            throw error;
         }
+        // backfill
+        await this.updateOutbox(remote, 0);
+        return remote;
     }
     async remove(name) {
         const remote = this.remotes.get(name);
         if (!remote) {
             throw new Error(`Remote with name '${name}' does not exist`);
         }
+        // shutdown the channel
+        await remote.channel.shutdown();
+        // delete the remote's data
         await this.remoteStorage.remove(name);
-        remote.channel.shutdown();
+        await this.cursorStorage.remove(name);
+        this.syncStatusTracker.untrackRemote(name);
         this.remotes.delete(name);
     }
     list() {
         return Array.from(this.remotes.values());
     }
+    waitForSync(jobId, signal) {
+        return this.syncAwaiter.waitForSync(jobId, signal);
+    }
+    getSyncStatus(documentId) {
+        return this.syncStatusTracker.getStatus(documentId);
+    }
+    onSyncStatusChange(callback) {
+        return this.syncStatusTracker.onChange(callback);
+    }
     wireChannelCallbacks(remote) {
-        remote.channel.inbox.onAdded((syncOp) => {
-            this.handleInboxJob(remote, syncOp);
-        });
-        remote.channel.outbox.onAdded((syncOp) => {
-            this.handleOutboxJob(remote, syncOp);
+        remote.channel.inbox.onAdded((syncOps) => this.handleInboxAdded(remote, syncOps));
+        this.syncStatusTracker.trackRemote(remote.name, remote.channel);
+        remote.channel.deadLetter.onAdded((syncOps) => {
+            for (const syncOp of syncOps) {
+                this.logger.error("Dead letter (@remote, @documentId, @jobId, @error, @dependencies)", remote.name, syncOp.documentId, syncOp.jobId, syncOp.error?.message ?? "unknown", syncOp.jobDependencies);
+                const record = {
+                    id: syncOp.id,
+                    jobId: syncOp.jobId,
+                    jobDependencies: syncOp.jobDependencies,
+                    remoteName: syncOp.remoteName,
+                    documentId: syncOp.documentId,
+                    scopes: syncOp.scopes,
+                    branch: syncOp.branch,
+                    operations: syncOp.operations,
+                    errorSource: syncOp.error?.source ?? ChannelErrorSource.None,
+                    errorMessage: syncOp.error?.error.message ?? "unknown",
+                };
+                void this.deadLetterStorage.add(record).catch((err) => {
+                    this.logger.error("Failed to persist dead letter (@id, @error)", record.id, err instanceof Error ? err.message : String(err));
+                });
+            }
+            // Evict oldest dead letters from mailbox if over capacity
+            const items = remote.channel.deadLetter.items;
+            if (items.length > this.maxDeadLettersPerRemote) {
+                const excessCount = items.length - this.maxDeadLettersPerRemote;
+                const toEvict = items.slice(0, excessCount);
+                remote.channel.deadLetter.remove(...toEvict);
+            }
         });
     }
-    handleOperationWritten(event) {
-        if (this.isShutdown) {
+    async loadDeadLetters(remote) {
+        let records;
+        try {
+            const page = await this.deadLetterStorage.list(remote.name, {
+                cursor: "0",
+                limit: this.maxDeadLettersPerRemote,
+            });
+            records = page.results;
+        }
+        catch (error) {
+            this.logger.error("Failed to load dead letters for remote (@name, @error)", remote.name, error instanceof Error ? error.message : String(error));
             return;
         }
-        const sourceRemote = event.jobMeta?.sourceRemote;
-        for (const remote of this.remotes.values()) {
-            if (sourceRemote && remote.name === sourceRemote) {
-                continue;
-            }
-            const filteredOps = filterOperations(event.operations, remote.filter);
-            if (filteredOps.length === 0) {
-                continue;
-            }
-            const batches = batchOperationsByDocument(filteredOps);
-            for (const batch of batches) {
-                const syncOp = new SyncOperation(crypto.randomUUID(), remote.name, batch.documentId, [batch.scope], batch.branch, batch.operations);
-                remote.channel.outbox.add(syncOp);
+        if (records.length === 0) {
+            return;
+        }
+        // Records come in ordinal DESC order (newest first).
+        // Reverse so the Map maintains chronological insertion order (oldest first),
+        // which makes eviction (slice from the front) straightforward.
+        records.reverse();
+        const syncOps = [];
+        for (const record of records) {
+            const syncOp = new SyncOperation(record.id, record.jobId, record.jobDependencies, record.remoteName, record.documentId, record.scopes, record.branch, record.operations);
+            syncOp.failed(new ChannelError(record.errorSource, new Error(record.errorMessage)));
+            syncOps.push(syncOp);
+        }
+        remote.channel.deadLetter.add(...syncOps);
+        this.logger.debug("Loaded @count persisted dead letters for remote @name", records.length, remote.name);
+    }
+    getRemotesForCollection(collectionId) {
+        return Array.from(this.remotes.values()).filter((remote) => remote.collectionId === collectionId);
+    }
+    async processCompleteBatch(batch) {
+        // get the unique set of collection ids
+        const collectionIds = [
+            ...new Set(Object.values(batch.collectionMemberships).flatMap((collections) => collections)),
+        ];
+        // get the unique set of affected remotes
+        const affectedRemotes = [];
+        for (const collectionId of collectionIds) {
+            const remotes = this.getRemotesForCollection(collectionId);
+            for (const remote of remotes) {
+                if (!affectedRemotes.includes(remote)) {
+                    affectedRemotes.push(remote);
+                }
             }
         }
+        // ack matching inbox items
+        for (const remote of affectedRemotes) {
+            trimMailboxFromBatch(remote.channel.inbox, batch);
+        }
+        // finally, work through the affected remotes and backfill based on the last operation in the outbox
+        for (const remote of affectedRemotes) {
+            await this.updateOutbox(remote, remote.channel.outbox.latestOrdinal);
+        }
     }
-    handleInboxJob(remote, syncOp) {
+    handleInboxAdded(remote, syncOps) {
         if (this.isShutdown) {
             return;
         }
-        void this.applyInboxJob(remote, syncOp);
-    }
-    handleOutboxJob(remote, syncOp) {
-        syncOp.on((syncOp, _prev, next) => {
-            if (next === SyncOperationStatus.Applied) {
-                remote.channel.outbox.remove(syncOp);
+        const keyed = [];
+        const nonKeyed = [];
+        for (const syncOp of syncOps) {
+            if (syncOp.jobId) {
+                keyed.push(syncOp);
             }
-            else if (next === SyncOperationStatus.Error) {
-                remote.channel.outbox.remove(syncOp);
+            else {
+                nonKeyed.push(syncOp);
             }
-        });
+        }
+        for (const syncOp of nonKeyed) {
+            void this.applyInboxJob(remote, syncOp);
+        }
+        if (keyed.length > 0) {
+            void this.applyInboxBatch(keyed.map((syncOp) => ({ remote, syncOp })));
+        }
     }
     async applyInboxJob(remote, syncOp) {
         const operations = syncOp.operations.map((op) => op.operation);
@@ -232,6 +306,7 @@ export class SyncManager {
         }
         catch (error) {
             const err = error instanceof Error ? error : new Error(String(error));
+            this.logger.error("Failed to load operations from inbox (@remote, @documentId, @error)", remote.name, syncOp.documentId, err.message);
             const channelError = new ChannelError(ChannelErrorSource.Inbox, err);
             syncOp.failed(channelError);
             remote.channel.deadLetter.add(syncOp);
@@ -244,6 +319,7 @@ export class SyncManager {
         }
         catch (error) {
             const err = error instanceof Error ? error : new Error(String(error));
+            this.logger.error("Failed to wait for job completion (@remote, @documentId, @jobId, @error)", remote.name, syncOp.documentId, jobInfo.id, err.message);
             const channelError = new ChannelError(ChannelErrorSource.Inbox, err);
             syncOp.failed(channelError);
             remote.channel.deadLetter.add(syncOp);
@@ -253,7 +329,9 @@ export class SyncManager {
         const jobKey = `${syncOp.documentId}:${syncOp.branch}`;
         this.loadJobs.set(jobKey, completedJobInfo);
         if (completedJobInfo.status === JobStatus.FAILED) {
-            const error = new ChannelError(ChannelErrorSource.Inbox, new Error(`Failed to apply operations: ${completedJobInfo.error?.message || "Unknown error"}`));
+            const errorMessage = completedJobInfo.error?.message || "Unknown error";
+            this.logger.error("Failed to apply operations from inbox (@remote, @documentId, @jobId, @error)", remote.name, syncOp.documentId, completedJobInfo.id, errorMessage);
+            const error = new ChannelError(ChannelErrorSource.Inbox, new Error(`Failed to apply operations: ${errorMessage}`));
             syncOp.failed(error);
             remote.channel.deadLetter.add(syncOp);
         }
@@ -262,5 +340,108 @@ export class SyncManager {
         }
         remote.channel.inbox.remove(syncOp);
     }
+    async applyInboxBatch(items) {
+        const sourceRemote = items[0].remote.name;
+        const jobs = items.map(({ syncOp }) => ({
+            key: syncOp.jobId,
+            documentId: syncOp.documentId,
+            scope: syncOp.scopes[0],
+            branch: syncOp.branch,
+            operations: syncOp.operations.map((op) => op.operation),
+            dependsOn: syncOp.jobDependencies.filter(Boolean),
+        }));
+        const request = { jobs };
+        let result;
+        try {
+            result = await this.reactor.loadBatch(request, undefined, {
+                sourceRemote,
+            });
+        }
+        catch (error) {
+            for (const { remote, syncOp } of items) {
+                const err = error instanceof Error ? error : new Error(String(error));
+                syncOp.failed(new ChannelError(ChannelErrorSource.Inbox, err));
+                remote.channel.deadLetter.add(syncOp);
+                remote.channel.inbox.remove(syncOp);
+            }
+            return;
+        }
+        for (const { remote, syncOp } of items) {
+            if (!(syncOp.jobId in result.jobs)) {
+                this.logger.error("Job key missing from batch load result (@remote, @documentId, @jobId)", remote.name, syncOp.documentId, syncOp.jobId);
+                const error = new ChannelError(ChannelErrorSource.Inbox, new Error(`Job key '${syncOp.jobId}' missing from batch load result`));
+                syncOp.failed(error);
+                remote.channel.deadLetter.add(syncOp);
+                remote.channel.inbox.remove(syncOp);
+                continue;
+            }
+            const jobInfo = result.jobs[syncOp.jobId];
+            let completedJobInfo;
+            try {
+                completedJobInfo = await this.awaiter.waitForJob(jobInfo.id);
+            }
+            catch (error) {
+                const err = error instanceof Error ? error : new Error(String(error));
+                syncOp.failed(new ChannelError(ChannelErrorSource.Inbox, err));
+                remote.channel.deadLetter.add(syncOp);
+                remote.channel.inbox.remove(syncOp);
+                continue;
+            }
+            const jobKey = `${syncOp.documentId}:${syncOp.branch}`;
+            this.loadJobs.set(jobKey, completedJobInfo);
+            if (completedJobInfo.status === JobStatus.FAILED) {
+                const errorMessage = completedJobInfo.error?.message || "Unknown error";
+                const channelError = new ChannelError(ChannelErrorSource.Inbox, new Error(`Failed to apply operations: ${errorMessage}`));
+                syncOp.failed(channelError);
+                remote.channel.deadLetter.add(syncOp);
+            }
+            else {
+                syncOp.executed();
+            }
+            remote.channel.inbox.remove(syncOp);
+        }
+    }
+    async updateOutbox(remote, ackOrdinal) {
+        const operations = await this.getOperationsForRemote(remote, ackOrdinal);
+        if (operations.length === 0) {
+            return;
+        }
+        // sort by (documentId, scope, ordinal) so batchOperationsByDocument
+        // groups all operations for the same document together
+        operations.sort((a, b) => {
+            if (a.context.documentId !== b.context.documentId) {
+                return a.context.documentId < b.context.documentId ? -1 : 1;
+            }
+            if (a.context.scope !== b.context.scope) {
+                return a.context.scope < b.context.scope ? -1 : 1;
+            }
+            return a.context.ordinal - b.context.ordinal;
+        });
+        const batches = batchOperationsByDocument(operations);
+        // per-document dependency chain: each batch depends on the previous
+        // batch for the same documentId only, allowing independent documents
+        // to be processed in parallel
+        const lastJobByDoc = new Map();
+        const syncOps = [];
+        for (const batch of batches) {
+            const jobId = crypto.randomUUID();
+            const prevJobId = lastJobByDoc.get(batch.documentId);
+            const syncOp = new SyncOperation(crypto.randomUUID(), jobId, prevJobId ? [prevJobId] : [], remote.name, batch.documentId, [batch.scope], batch.branch, batch.operations);
+            syncOps.push(syncOp);
+            lastJobByDoc.set(batch.documentId, jobId);
+        }
+        remote.channel.outbox.add(...syncOps);
+    }
+    async getOperationsForRemote(remote, ackOrdinal) {
+        const results = await this.operationIndex.find(remote.collectionId, ackOrdinal, { excludeSourceRemote: remote.name });
+        let operations = results.results.map((entry) => toOperationWithContext(entry));
+        // apply the sinceTimestampUtcMs filter
+        const sinceTimestamp = remote.options.sinceTimestampUtcMs;
+        if (sinceTimestamp && sinceTimestamp !== "0") {
+            operations = operations.filter((op) => op.operation.timestampUtcMs >= sinceTimestamp);
+        }
+        // apply the remote filter
+        return filterOperations(operations, remote.filter);
+    }
 }
 //# sourceMappingURL=sync-manager.js.map