@luvio/environments 0.126.2 → 0.128.0

package/dist/es/es2018/DurableStore.d.ts

@@ -58,7 +58,7 @@ export interface DurableStoreChange {
      */
     isExternalChange?: boolean;
 }
-export declare type OnDurableStoreChangedListener = (changes: DurableStoreChange[]) => void;
+export declare type OnDurableStoreChangedListener = (changes: DurableStoreChange[]) => Promise<void>;
 /**
  * The DurableStore implementation must have the following behaviors:
  * - It must handle read/write synchronization (ie: reads should wait for any

package/dist/es/es2018/environments.js

@@ -452,7 +452,7 @@ function makeDurable(environment, { durableStore, instrumentation }) {
             throw new Error('This makeDurable instance has been disposed');
         }
     };
-    const unsubscribe = durableStore.registerOnChangedListener((changes) => {
+    const unsubscribe = durableStore.registerOnChangedListener(async (changes) => {
         const defaultSegmentKeys = [];
         const adapterContextSegmentKeys = [];
         for (let i = 0, len = changes.length; i < len; i++) {
@@ -481,11 +481,8 @@ function makeDurable(environment, { durableStore, instrumentation }) {
             }
         }
         if (adapterContextKeysFromDifferentInstance.length > 0) {
-            // change handlers are sync, so kick off L2 read of the changed
-            // segment keys but we can't await it
-            durableStore
-                .getEntries(adapterContextKeysFromDifferentInstance, AdapterContextSegment)
-                .then((entries) => {
+            try {
+                const entries = await durableStore.getEntries(adapterContextKeysFromDifferentInstance, AdapterContextSegment);
                 if (entries !== undefined) {
                     const entryKeys = keys(entries);
                     for (let i = 0, len = entryKeys.length; i < len; i++) {
@@ -494,8 +491,10 @@ function makeDurable(environment, { durableStore, instrumentation }) {
                         contextStores[entryKey] = entry.data;
                     }
                 }
-            })
-                .catch(durableStoreErrorHandler);
+            }
+            catch (error) {
+                durableStoreErrorHandler(error);
+            }
         }
         // process default segment changes
         const defaultSegmentKeysLength = defaultSegmentKeys.length;
@@ -513,7 +512,7 @@ function makeDurable(environment, { durableStore, instrumentation }) {
                 // call base environment storeEvict so this evict is not tracked for durable deletion
                 environment.storeEvict(key);
             }
-            environment.storeBroadcast(rebuildSnapshot, environment.snapshotAvailable);
+            await environment.storeBroadcast(rebuildSnapshot, environment.snapshotAvailable);
         }
     });
     const dispose = function () {
@@ -558,9 +557,10 @@ function makeDurable(environment, { durableStore, instrumentation }) {
     };
     const storeBroadcast = function (_rebuildSnapshot, _snapshotDataAvailable) {
         validateNotDisposed();
-        // no-op here and wait for the L2 flush to happen in handleSuccessResponse,
+        // return resolved promise here and wait for the L2 flush to happen in handleSuccessResponse,
         // that flush will cause the onChanged handler to fire which will revive
         // records to the main L1 store and call the base storeBroadcast
+        return Promise.resolve();
     };
     const publishChangesToDurableStore = function () {
         validateNotDisposed();
@@ -603,28 +603,28 @@ function makeDurable(environment, { durableStore, instrumentation }) {
         }
         return environment.wrapNormalizedGraphNode(normalized, ingestStagingStore);
     };
-    const rebuildSnapshot = function (snapshot, onAsyncRebuild) {
+    const rebuildSnapshot = function (snapshot, onRebuild) {
         validateNotDisposed();
         // try rebuilding from memory
-        const rebuilt = environment.rebuildSnapshot(snapshot, onAsyncRebuild);
-        // only try reviving from durable store if snapshot is unfulfilled
-        if (rebuilt.state !== 'Unfulfilled') {
-            return rebuilt;
-        }
-        // Do an L2 revive and emit to subscriber using the callback.
-        reviveSnapshot(environment, durableStore, rebuilt, durableStoreErrorHandler, () => 
-        // reviveSnapshot will revive into L1, and since "records" is a reference
-        // (and not a copy) to the L1 records we can use it for rebuild
-        environment.rebuildSnapshot(snapshot, () => { })).then((result) => {
-            onAsyncRebuild(result.snapshot);
+        environment.rebuildSnapshot(snapshot, (rebuilt) => {
+            // only try reviving from durable store if snapshot is unfulfilled
+            if (rebuilt.state !== 'Unfulfilled') {
+                onRebuild(rebuilt);
+                return;
+            }
+            // Do an L2 revive and emit to subscriber using the callback.
+            reviveSnapshot(environment, durableStore, rebuilt, durableStoreErrorHandler, () => {
+                // reviveSnapshot will revive into L1, and since "records" is a reference
+                // (and not a copy) to the L1 records we can use it for rebuild
+                let rebuiltSnap;
+                environment.rebuildSnapshot(snapshot, (rebuilt) => {
+                    rebuiltSnap = rebuilt;
+                });
+                return rebuiltSnap;
+            }).then((result) => {
+                onRebuild(result.snapshot);
+            });
         });
-        // synchronously return the base snapshot as Pending if not already
-        return snapshot.state === 'Pending'
-            ? snapshot
-            : {
-                ...snapshot,
-                state: 'Pending',
-            };
     };
     const withContext = function (adapter, options) {
         validateNotDisposed();
@@ -773,7 +773,7 @@ function makeDurable(environment, { durableStore, instrumentation }) {
                 }, 
                 // we don't need to prime metadata
                 () => { });
-                snapshotFromMemoryIngest = ingestAndBroadcastFunc();
+                snapshotFromMemoryIngest = await ingestAndBroadcastFunc();
                 await publishChangesToDurableStore();
             })();
             for (const key of keysAsArray) {
@@ -801,7 +801,7 @@ function makeDurable(environment, { durableStore, instrumentation }) {
             // underlying DurableStore implementation takes care of R/W sync
             // so all we have to do is ingest then write to L2
             ingestStagingStore = buildIngestStagingStore(environment);
-            snapshotFromMemoryIngest = ingestAndBroadcastFunc();
+            snapshotFromMemoryIngest = await ingestAndBroadcastFunc();
             await publishChangesToDurableStore();
         }
         if (snapshotFromMemoryIngest === undefined) {

package/dist/umd/es2018/DurableStore.d.ts

@@ -58,7 +58,7 @@ export interface DurableStoreChange {
      */
     isExternalChange?: boolean;
 }
-export declare type OnDurableStoreChangedListener = (changes: DurableStoreChange[]) => void;
+export declare type OnDurableStoreChangedListener = (changes: DurableStoreChange[]) => Promise<void>;
 /**
  * The DurableStore implementation must have the following behaviors:
  * - It must handle read/write synchronization (ie: reads should wait for any

package/dist/umd/es2018/environments.js

@@ -456,7 +456,7 @@
                 throw new Error('This makeDurable instance has been disposed');
             }
         };
-        const unsubscribe = durableStore.registerOnChangedListener((changes) => {
+        const unsubscribe = durableStore.registerOnChangedListener(async (changes) => {
             const defaultSegmentKeys = [];
             const adapterContextSegmentKeys = [];
             for (let i = 0, len = changes.length; i < len; i++) {
@@ -485,11 +485,8 @@
                 }
             }
             if (adapterContextKeysFromDifferentInstance.length > 0) {
-                // change handlers are sync, so kick off L2 read of the changed
-                // segment keys but we can't await it
-                durableStore
-                    .getEntries(adapterContextKeysFromDifferentInstance, AdapterContextSegment)
-                    .then((entries) => {
+                try {
+                    const entries = await durableStore.getEntries(adapterContextKeysFromDifferentInstance, AdapterContextSegment);
                     if (entries !== undefined) {
                         const entryKeys = keys(entries);
                         for (let i = 0, len = entryKeys.length; i < len; i++) {
@@ -498,8 +495,10 @@
                             contextStores[entryKey] = entry.data;
                         }
                     }
-                })
-                    .catch(durableStoreErrorHandler);
+                }
+                catch (error) {
+                    durableStoreErrorHandler(error);
+                }
             }
             // process default segment changes
             const defaultSegmentKeysLength = defaultSegmentKeys.length;
@@ -517,7 +516,7 @@
                     // call base environment storeEvict so this evict is not tracked for durable deletion
                     environment.storeEvict(key);
                 }
-                environment.storeBroadcast(rebuildSnapshot, environment.snapshotAvailable);
+                await environment.storeBroadcast(rebuildSnapshot, environment.snapshotAvailable);
             }
         });
         const dispose = function () {
@@ -562,9 +561,10 @@
         };
         const storeBroadcast = function (_rebuildSnapshot, _snapshotDataAvailable) {
             validateNotDisposed();
-            // no-op here and wait for the L2 flush to happen in handleSuccessResponse,
+            // return resolved promise here and wait for the L2 flush to happen in handleSuccessResponse,
             // that flush will cause the onChanged handler to fire which will revive
             // records to the main L1 store and call the base storeBroadcast
+            return Promise.resolve();
         };
         const publishChangesToDurableStore = function () {
             validateNotDisposed();
@@ -607,28 +607,28 @@
             }
             return environment.wrapNormalizedGraphNode(normalized, ingestStagingStore);
         };
-        const rebuildSnapshot = function (snapshot, onAsyncRebuild) {
+        const rebuildSnapshot = function (snapshot, onRebuild) {
             validateNotDisposed();
             // try rebuilding from memory
-            const rebuilt = environment.rebuildSnapshot(snapshot, onAsyncRebuild);
-            // only try reviving from durable store if snapshot is unfulfilled
-            if (rebuilt.state !== 'Unfulfilled') {
-                return rebuilt;
-            }
-            // Do an L2 revive and emit to subscriber using the callback.
-            reviveSnapshot(environment, durableStore, rebuilt, durableStoreErrorHandler, () => 
-            // reviveSnapshot will revive into L1, and since "records" is a reference
-            // (and not a copy) to the L1 records we can use it for rebuild
-            environment.rebuildSnapshot(snapshot, () => { })).then((result) => {
-                onAsyncRebuild(result.snapshot);
+            environment.rebuildSnapshot(snapshot, (rebuilt) => {
+                // only try reviving from durable store if snapshot is unfulfilled
+                if (rebuilt.state !== 'Unfulfilled') {
+                    onRebuild(rebuilt);
+                    return;
+                }
+                // Do an L2 revive and emit to subscriber using the callback.
+                reviveSnapshot(environment, durableStore, rebuilt, durableStoreErrorHandler, () => {
+                    // reviveSnapshot will revive into L1, and since "records" is a reference
+                    // (and not a copy) to the L1 records we can use it for rebuild
+                    let rebuiltSnap;
+                    environment.rebuildSnapshot(snapshot, (rebuilt) => {
+                        rebuiltSnap = rebuilt;
+                    });
+                    return rebuiltSnap;
+                }).then((result) => {
+                    onRebuild(result.snapshot);
+                });
             });
-            // synchronously return the base snapshot as Pending if not already
-            return snapshot.state === 'Pending'
-                ? snapshot
-                : {
-                    ...snapshot,
-                    state: 'Pending',
-                };
         };
         const withContext = function (adapter, options) {
             validateNotDisposed();
@@ -777,7 +777,7 @@
                     }, 
                     // we don't need to prime metadata
                     () => { });
-                    snapshotFromMemoryIngest = ingestAndBroadcastFunc();
+                    snapshotFromMemoryIngest = await ingestAndBroadcastFunc();
                     await publishChangesToDurableStore();
                 })();
                 for (const key of keysAsArray) {
@@ -805,7 +805,7 @@
                 // underlying DurableStore implementation takes care of R/W sync
                 // so all we have to do is ingest then write to L2
                 ingestStagingStore = buildIngestStagingStore(environment);
-                snapshotFromMemoryIngest = ingestAndBroadcastFunc();
+                snapshotFromMemoryIngest = await ingestAndBroadcastFunc();
                 await publishChangesToDurableStore();
             }
             if (snapshotFromMemoryIngest === undefined) {

package/dist/umd/es5/DurableStore.d.ts

@@ -58,7 +58,7 @@ export interface DurableStoreChange {
      */
     isExternalChange?: boolean;
 }
-export declare type OnDurableStoreChangedListener = (changes: DurableStoreChange[]) => void;
+export declare type OnDurableStoreChangedListener = (changes: DurableStoreChange[]) => Promise<void>;
 /**
  * The DurableStore implementation must have the following behaviors:
  * - It must handle read/write synchronization (ie: reads should wait for any

package/dist/umd/es5/environments.js

@@ -535,6 +535,7 @@
      * @param instrumentation An instrumentation function implementation
      */
     function makeDurable(environment, _a) {
+        var _this = this;
         var durableStore = _a.durableStore, instrumentation = _a.instrumentation;
         var ingestStagingStore = null;
         var durableTTLStore = new DurableTTLStore(durableStore);
@@ -559,71 +560,82 @@
                 throw new Error('This makeDurable instance has been disposed');
             }
         };
-        var unsubscribe = durableStore.registerOnChangedListener(function (changes) {
-            var defaultSegmentKeys = [];
-            var adapterContextSegmentKeys = [];
-            for (var i = 0, len = changes.length; i < len; i++) {
-                var change = changes[i];
-                // we only care about changes to the data which is stored in the default
-                // segment or the adapter context
-                if (change.segment === DefaultDurableSegment) {
-                    defaultSegmentKeys.push.apply(defaultSegmentKeys, change.ids);
-                }
-                else if (change.segment === AdapterContextSegment) {
-                    adapterContextSegmentKeys.push.apply(adapterContextSegmentKeys, change.ids);
-                }
-            }
-            // process adapter context changes
-            var adapterContextKeysFromDifferentInstance = [];
-            for (var _i = 0, adapterContextSegmentKeys_1 = adapterContextSegmentKeys; _i < adapterContextSegmentKeys_1.length; _i++) {
-                var key = adapterContextSegmentKeys_1[_i];
-                if (pendingContextStoreKeys.has(key)) {
-                    // if this instance caused the L2 write then remove from the
-                    // "pending" Set and move on
-                    pendingContextStoreKeys.delete(key);
-                }
-                else {
-                    // else it came from another luvio instance and we need to
-                    // read from L2
-                    adapterContextKeysFromDifferentInstance.push(key);
-                }
-            }
-            if (adapterContextKeysFromDifferentInstance.length > 0) {
-                // change handlers are sync, so kick off L2 read of the changed
-                // segment keys but we can't await it
-                durableStore
-                    .getEntries(adapterContextKeysFromDifferentInstance, AdapterContextSegment)
-                    .then(function (entries) {
-                    if (entries !== undefined) {
-                        var entryKeys = keys(entries);
-                        for (var i = 0, len = entryKeys.length; i < len; i++) {
-                            var entryKey = entryKeys[i];
-                            var entry = entries[entryKey];
-                            contextStores[entryKey] = entry.data;
+        var unsubscribe = durableStore.registerOnChangedListener(function (changes) { return __awaiter(_this, void 0, void 0, function () {
+            var defaultSegmentKeys, adapterContextSegmentKeys, i, len, change, adapterContextKeysFromDifferentInstance, _i, adapterContextSegmentKeys_1, key, entries, entryKeys, i, len, entryKey, entry, error_2, defaultSegmentKeysLength, i, key, canonical;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        defaultSegmentKeys = [];
+                        adapterContextSegmentKeys = [];
+                        for (i = 0, len = changes.length; i < len; i++) {
+                            change = changes[i];
+                            // we only care about changes to the data which is stored in the default
+                            // segment or the adapter context
+                            if (change.segment === DefaultDurableSegment) {
+                                defaultSegmentKeys.push.apply(defaultSegmentKeys, change.ids);
+                            }
+                            else if (change.segment === AdapterContextSegment) {
+                                adapterContextSegmentKeys.push.apply(adapterContextSegmentKeys, change.ids);
+                            }
                         }
-                    }
-                })
-                    .catch(durableStoreErrorHandler);
-            }
-            // process default segment changes
-            var defaultSegmentKeysLength = defaultSegmentKeys.length;
-            if (defaultSegmentKeysLength > 0) {
-                for (var i = 0; i < defaultSegmentKeysLength; i++) {
-                    var key = defaultSegmentKeys[i];
-                    var canonical = environment.storeGetCanonicalKey(key);
-                    if (canonical !== key) {
-                        continue;
-                    }
-                    // TODO: W-8909393 If expiration is the only thing that changed we should not evict the data... so
-                    // if we stored expiration and data at different keys (or same keys in different segments)
-                    // then we could know if only the expiration has changed and we wouldn't need to evict
-                    // and go through an entire broadcast/revive cycle for unchanged data
-                    // call base environment storeEvict so this evict is not tracked for durable deletion
-                    environment.storeEvict(key);
+                        adapterContextKeysFromDifferentInstance = [];
+                        for (_i = 0, adapterContextSegmentKeys_1 = adapterContextSegmentKeys; _i < adapterContextSegmentKeys_1.length; _i++) {
+                            key = adapterContextSegmentKeys_1[_i];
+                            if (pendingContextStoreKeys.has(key)) {
+                                // if this instance caused the L2 write then remove from the
+                                // "pending" Set and move on
+                                pendingContextStoreKeys.delete(key);
+                            }
+                            else {
+                                // else it came from another luvio instance and we need to
+                                // read from L2
+                                adapterContextKeysFromDifferentInstance.push(key);
+                            }
+                        }
+                        if (!(adapterContextKeysFromDifferentInstance.length > 0)) return [3 /*break*/, 4];
+                        _a.label = 1;
+                    case 1:
+                        _a.trys.push([1, 3, , 4]);
+                        return [4 /*yield*/, durableStore.getEntries(adapterContextKeysFromDifferentInstance, AdapterContextSegment)];
+                    case 2:
+                        entries = _a.sent();
+                        if (entries !== undefined) {
+                            entryKeys = keys(entries);
+                            for (i = 0, len = entryKeys.length; i < len; i++) {
+                                entryKey = entryKeys[i];
+                                entry = entries[entryKey];
+                                contextStores[entryKey] = entry.data;
+                            }
+                        }
+                        return [3 /*break*/, 4];
+                    case 3:
+                        error_2 = _a.sent();
+                        durableStoreErrorHandler(error_2);
+                        return [3 /*break*/, 4];
+                    case 4:
+                        defaultSegmentKeysLength = defaultSegmentKeys.length;
+                        if (!(defaultSegmentKeysLength > 0)) return [3 /*break*/, 6];
+                        for (i = 0; i < defaultSegmentKeysLength; i++) {
+                            key = defaultSegmentKeys[i];
+                            canonical = environment.storeGetCanonicalKey(key);
+                            if (canonical !== key) {
+                                continue;
+                            }
+                            // TODO: W-8909393 If expiration is the only thing that changed we should not evict the data... so
+                            // if we stored expiration and data at different keys (or same keys in different segments)
+                            // then we could know if only the expiration has changed and we wouldn't need to evict
+                            // and go through an entire broadcast/revive cycle for unchanged data
+                            // call base environment storeEvict so this evict is not tracked for durable deletion
+                            environment.storeEvict(key);
+                        }
+                        return [4 /*yield*/, environment.storeBroadcast(rebuildSnapshot, environment.snapshotAvailable)];
+                    case 5:
+                        _a.sent();
+                        _a.label = 6;
+                    case 6: return [2 /*return*/];
                 }
-                environment.storeBroadcast(rebuildSnapshot, environment.snapshotAvailable);
-            }
-        });
+            });
+        }); });
         var dispose = function () {
             validateNotDisposed();
             disposed = true;
@@ -666,9 +678,10 @@
         };
         var storeBroadcast = function (_rebuildSnapshot, _snapshotDataAvailable) {
             validateNotDisposed();
-            // no-op here and wait for the L2 flush to happen in handleSuccessResponse,
+            // return resolved promise here and wait for the L2 flush to happen in handleSuccessResponse,
             // that flush will cause the onChanged handler to fire which will revive
             // records to the main L1 store and call the base storeBroadcast
+            return Promise.resolve();
         };
         var publishChangesToDurableStore = function () {
             validateNotDisposed();
@@ -711,26 +724,28 @@
             }
             return environment.wrapNormalizedGraphNode(normalized, ingestStagingStore);
         };
-        var rebuildSnapshot = function (snapshot, onAsyncRebuild) {
+        var rebuildSnapshot = function (snapshot, onRebuild) {
             validateNotDisposed();
             // try rebuilding from memory
-            var rebuilt = environment.rebuildSnapshot(snapshot, onAsyncRebuild);
-            // only try reviving from durable store if snapshot is unfulfilled
-            if (rebuilt.state !== 'Unfulfilled') {
-                return rebuilt;
-            }
-            // Do an L2 revive and emit to subscriber using the callback.
-            reviveSnapshot(environment, durableStore, rebuilt, durableStoreErrorHandler, function () {
-                // reviveSnapshot will revive into L1, and since "records" is a reference
-                // (and not a copy) to the L1 records we can use it for rebuild
-                return environment.rebuildSnapshot(snapshot, function () { });
-            }).then(function (result) {
-                onAsyncRebuild(result.snapshot);
+            environment.rebuildSnapshot(snapshot, function (rebuilt) {
+                // only try reviving from durable store if snapshot is unfulfilled
+                if (rebuilt.state !== 'Unfulfilled') {
+                    onRebuild(rebuilt);
+                    return;
+                }
+                // Do an L2 revive and emit to subscriber using the callback.
+                reviveSnapshot(environment, durableStore, rebuilt, durableStoreErrorHandler, function () {
+                    // reviveSnapshot will revive into L1, and since "records" is a reference
+                    // (and not a copy) to the L1 records we can use it for rebuild
+                    var rebuiltSnap;
+                    environment.rebuildSnapshot(snapshot, function (rebuilt) {
+                        rebuiltSnap = rebuilt;
+                    });
+                    return rebuiltSnap;
+                }).then(function (result) {
+                    onRebuild(result.snapshot);
+                });
             });
-            // synchronously return the base snapshot as Pending if not already
-            return snapshot.state === 'Pending'
-                ? snapshot
-                : __assign(__assign({}, snapshot), { state: 'Pending' });
         };
         var withContext = function (adapter, options) {
             validateNotDisposed();
@@ -899,9 +914,11 @@
                                             }, 
                                             // we don't need to prime metadata
                                             function () { });
-                                            snapshotFromMemoryIngest = ingestAndBroadcastFunc();
-                                            return [4 /*yield*/, publishChangesToDurableStore()];
+                                            return [4 /*yield*/, ingestAndBroadcastFunc()];
                                         case 3:
+                                            snapshotFromMemoryIngest = _a.sent();
+                                            return [4 /*yield*/, publishChangesToDurableStore()];
+                                        case 4:
                                             _a.sent();
                                             return [2 /*return*/];
                                     }
@@ -932,18 +949,20 @@
                                 }
                             }
                             return [7 /*endfinally*/];
-                        case 4: return [3 /*break*/, 7];
+                        case 4: return [3 /*break*/, 8];
                         case 5:
                             // we aren't doing any merging so we don't have to synchronize, the
                             // underlying DurableStore implementation takes care of R/W sync
                             // so all we have to do is ingest then write to L2
                             ingestStagingStore = buildIngestStagingStore(environment);
-                            snapshotFromMemoryIngest = ingestAndBroadcastFunc();
-                            return [4 /*yield*/, publishChangesToDurableStore()];
+                            return [4 /*yield*/, ingestAndBroadcastFunc()];
                         case 6:
-                            _d.sent();
-                            _d.label = 7;
+                            snapshotFromMemoryIngest = _d.sent();
+                            return [4 /*yield*/, publishChangesToDurableStore()];
                         case 7:
+                            _d.sent();
+                            _d.label = 8;
+                        case 8:
                             if (snapshotFromMemoryIngest === undefined) {
                                 return [2 /*return*/, undefined];
                             }
@@ -952,7 +971,7 @@
                             }
                             _c = snapshotFromMemoryIngest, select = _c.select, refresh = _c.refresh;
                             return [4 /*yield*/, reviveSnapshot(environment, durableStore, snapshotFromMemoryIngest, durableStoreErrorHandler, function () { return environment.storeLookup(select, environment.createSnapshot, refresh); })];
-                        case 8:
+                        case 9:
                             result = _d.sent();
                             return [2 /*return*/, result.snapshot];
                     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@luvio/environments",
-    "version": "0.126.2",
+    "version": "0.128.0",
     "description": "Luvio Environments",
     "repository": {
         "type": "git",
@@ -23,7 +23,7 @@
         "watch": "yarn build --watch"
     },
     "dependencies": {
-        "@luvio/engine": "0.126.2"
+        "@luvio/engine": "0.128.0"
     },
     "bundlesize": [
         {