@luvio/environments 0.109.0 → 0.109.1

package/dist/es/es2018/environments.js

@@ -365,18 +365,19 @@ function buildIngestStagingStore(environment) {
 
 const AdapterContextSegment = 'ADAPTER-CONTEXT';
 const ADAPTER_CONTEXT_ID_SUFFIX = '__NAMED_CONTEXT';
-function reviveOrCreateContext(adapterId, durableStore, durableStoreErrorHandler, onContextLoaded) {
+async function reviveOrCreateContext(adapterId, durableStore, durableStoreErrorHandler, contextStores, pendingContextStoreKeys, onContextLoaded) {
     // initialize empty context store
-    let contextStore = create(null);
+    contextStores[adapterId] = create(null);
     const context = {
         set(key, value) {
-            contextStore[key] = value;
+            contextStores[adapterId][key] = value;
             durableStore.setEntries({
-                [adapterId]: { data: contextStore },
+                [adapterId]: { data: contextStores[adapterId] },
             }, AdapterContextSegment);
+            pendingContextStoreKeys.add(adapterId);
         },
         get(key) {
-            return contextStore[key];
+            return contextStores[adapterId][key];
         },
     };
     const contextReturn = () => {
@@ -387,17 +388,17 @@ function reviveOrCreateContext(adapterId, durableStore, durableStoreErrorHandler
         }
         return context;
     };
-    return durableStore.getEntries([adapterId], AdapterContextSegment).then((entries) => {
-        // if durable store has a saved context then load it in the store
+    try {
+        const entries = await durableStore.getEntries([adapterId], AdapterContextSegment);
         if (entries !== undefined && entries[adapterId] !== undefined) {
-            contextStore = entries[adapterId].data;
+            // if durable store has a saved context then load it in the store
+            contextStores[adapterId] = entries[adapterId].data;
         }
-        return contextReturn();
-    }, (error) => {
+    }
+    catch (error) {
         durableStoreErrorHandler(error);
-        //Return empty store when getEntries fails
-        return contextReturn();
-    });
+    }
+    return contextReturn();
 }
 function isUnfulfilledSnapshot(cachedSnapshotResult) {
     if (cachedSnapshotResult === undefined) {
@@ -420,6 +421,11 @@ function makeDurable(environment, { durableStore, instrumentation }) {
     let ingestStagingStore = null;
     const durableTTLStore = new DurableTTLStore(durableStore);
     const mergeKeysPromiseMap = new Map();
+    // When a context store is mutated we write it to L2, which causes DS on change
+    // event. If this instance of makeDurable caused that L2 write we can ignore that
+    // on change event. This Set helps us do that.
+    const pendingContextStoreKeys = new Set();
+    const contextStores = create(null);
     let initializationPromise = new Promise((resolve) => {
         const finish = () => {
             resolve();
@@ -436,38 +442,68 @@ function makeDurable(environment, { durableStore, instrumentation }) {
         }
     };
     const unsubscribe = durableStore.registerOnChangedListener((changes) => {
-        const applicableChanges = changes.filter((change) => {
+        const defaultSegmentKeys = [];
+        const adapterContextSegmentKeys = [];
+        for (let i = 0, len = changes.length; i < len; i++) {
+            const change = changes[i];
             // we only care about changes to the data which is stored in the default
-            // segment
-            return change.segment === DefaultDurableSegment;
-        });
-        if (applicableChanges.length === 0) {
-            return;
-        }
-        let allKeys = [];
-        for (let i = 0, len = applicableChanges.length; i < len; i++) {
-            const change = applicableChanges[i];
-            allKeys.push(...change.ids);
-        }
-        const keysLength = allKeys.length;
-        if (keysLength === 0) {
-            // if no keys changed then no need to broadcast
-            return;
-        }
-        for (let i = 0; i < keysLength; i++) {
-            const key = allKeys[i];
-            const canonical = environment.storeGetCanonicalKey(key);
-            if (canonical !== key) {
-                continue;
+            // segment or the adapter context
+            if (change.segment === DefaultDurableSegment) {
+                defaultSegmentKeys.push(...change.ids);
+            }
+            else if (change.segment === AdapterContextSegment) {
+                adapterContextSegmentKeys.push(...change.ids);
             }
-            // TODO: W-8909393 If expiration is the only thing that changed we should not evict the data... so
-            // if we stored expiration and data at different keys (or same keys in different segments)
-            // then we could know if only the expiration has changed and we wouldn't need to evict
-            // and go through an entire broadcast/revive cycle for unchanged data
-            // call base environment storeEvict so this evict is not tracked for durable deletion
-            environment.storeEvict(key);
         }
-        environment.storeBroadcast(rebuildSnapshot, environment.snapshotAvailable);
+        // process adapter context changes
+        const adapterContextKeysFromDifferentInstance = [];
+        for (const key of adapterContextSegmentKeys) {
+            if (pendingContextStoreKeys.has(key)) {
+                // if this instance caused the L2 write then remove from the
+                // "pending" Set and move on
+                pendingContextStoreKeys.delete(key);
+            }
+            else {
+                // else it came from another luvio instance and we need to
+                // read from L2
+                adapterContextKeysFromDifferentInstance.push(key);
+            }
+        }
+        if (adapterContextKeysFromDifferentInstance.length > 0) {
+            // change handlers are sync, so kick off L2 read of the changed
+            // segment keys but we can't await it
+            durableStore
+                .getEntries(adapterContextKeysFromDifferentInstance, AdapterContextSegment)
+                .then((entries) => {
+                if (entries !== undefined) {
+                    const entryKeys = keys(entries);
+                    for (let i = 0, len = entryKeys.length; i < len; i++) {
+                        const entryKey = entryKeys[i];
+                        const entry = entries[entryKey];
+                        contextStores[entryKey] = entry.data;
+                    }
+                }
+            })
+                .catch(durableStoreErrorHandler);
+        }
+        // process default segment changes
+        const defaultSegmentKeysLength = defaultSegmentKeys.length;
+        if (defaultSegmentKeysLength > 0) {
+            for (let i = 0; i < defaultSegmentKeysLength; i++) {
+                const key = defaultSegmentKeys[i];
+                const canonical = environment.storeGetCanonicalKey(key);
+                if (canonical !== key) {
+                    continue;
+                }
+                // TODO: W-8909393 If expiration is the only thing that changed we should not evict the data... so
+                // if we stored expiration and data at different keys (or same keys in different segments)
+                // then we could know if only the expiration has changed and we wouldn't need to evict
+                // and go through an entire broadcast/revive cycle for unchanged data
+                // call base environment storeEvict so this evict is not tracked for durable deletion
+                environment.storeEvict(key);
+            }
+            environment.storeBroadcast(rebuildSnapshot, environment.snapshotAvailable);
+        }
     });
     const dispose = function () {
         validateNotDisposed();
@@ -583,7 +619,7 @@ function makeDurable(environment, { durableStore, instrumentation }) {
         validateNotDisposed();
         const { contextId, onContextLoaded } = options;
         let context = undefined;
-        const contextAsPromise = reviveOrCreateContext(`${contextId}${ADAPTER_CONTEXT_ID_SUFFIX}`, durableStore, durableStoreErrorHandler, onContextLoaded);
+        const contextAsPromise = reviveOrCreateContext(`${contextId}${ADAPTER_CONTEXT_ID_SUFFIX}`, durableStore, durableStoreErrorHandler, contextStores, pendingContextStoreKeys, onContextLoaded);
         return (config, requestContext) => {
             if (context === undefined) {
                 return contextAsPromise.then((revivedContext) => {

package/dist/umd/es2018/environments.js

@@ -369,18 +369,19 @@
 
     const AdapterContextSegment = 'ADAPTER-CONTEXT';
     const ADAPTER_CONTEXT_ID_SUFFIX = '__NAMED_CONTEXT';
-    function reviveOrCreateContext(adapterId, durableStore, durableStoreErrorHandler, onContextLoaded) {
+    async function reviveOrCreateContext(adapterId, durableStore, durableStoreErrorHandler, contextStores, pendingContextStoreKeys, onContextLoaded) {
         // initialize empty context store
-        let contextStore = create(null);
+        contextStores[adapterId] = create(null);
         const context = {
             set(key, value) {
-                contextStore[key] = value;
+                contextStores[adapterId][key] = value;
                 durableStore.setEntries({
-                    [adapterId]: { data: contextStore },
+                    [adapterId]: { data: contextStores[adapterId] },
                 }, AdapterContextSegment);
+                pendingContextStoreKeys.add(adapterId);
             },
             get(key) {
-                return contextStore[key];
+                return contextStores[adapterId][key];
             },
         };
         const contextReturn = () => {
@@ -391,17 +392,17 @@
             }
             return context;
         };
-        return durableStore.getEntries([adapterId], AdapterContextSegment).then((entries) => {
-            // if durable store has a saved context then load it in the store
+        try {
+            const entries = await durableStore.getEntries([adapterId], AdapterContextSegment);
             if (entries !== undefined && entries[adapterId] !== undefined) {
-                contextStore = entries[adapterId].data;
+                // if durable store has a saved context then load it in the store
+                contextStores[adapterId] = entries[adapterId].data;
             }
-            return contextReturn();
-        }, (error) => {
+        }
+        catch (error) {
             durableStoreErrorHandler(error);
-            //Return empty store when getEntries fails
-            return contextReturn();
-        });
+        }
+        return contextReturn();
     }
     function isUnfulfilledSnapshot(cachedSnapshotResult) {
         if (cachedSnapshotResult === undefined) {
@@ -424,6 +425,11 @@
         let ingestStagingStore = null;
         const durableTTLStore = new DurableTTLStore(durableStore);
         const mergeKeysPromiseMap = new Map();
+        // When a context store is mutated we write it to L2, which causes DS on change
+        // event. If this instance of makeDurable caused that L2 write we can ignore that
+        // on change event. This Set helps us do that.
+        const pendingContextStoreKeys = new Set();
+        const contextStores = create(null);
         let initializationPromise = new Promise((resolve) => {
             const finish = () => {
                 resolve();
@@ -440,38 +446,68 @@
             }
         };
         const unsubscribe = durableStore.registerOnChangedListener((changes) => {
-            const applicableChanges = changes.filter((change) => {
+            const defaultSegmentKeys = [];
+            const adapterContextSegmentKeys = [];
+            for (let i = 0, len = changes.length; i < len; i++) {
+                const change = changes[i];
                 // we only care about changes to the data which is stored in the default
-                // segment
-                return change.segment === DefaultDurableSegment;
-            });
-            if (applicableChanges.length === 0) {
-                return;
-            }
-            let allKeys = [];
-            for (let i = 0, len = applicableChanges.length; i < len; i++) {
-                const change = applicableChanges[i];
-                allKeys.push(...change.ids);
-            }
-            const keysLength = allKeys.length;
-            if (keysLength === 0) {
-                // if no keys changed then no need to broadcast
-                return;
-            }
-            for (let i = 0; i < keysLength; i++) {
-                const key = allKeys[i];
-                const canonical = environment.storeGetCanonicalKey(key);
-                if (canonical !== key) {
-                    continue;
+                // segment or the adapter context
+                if (change.segment === DefaultDurableSegment) {
+                    defaultSegmentKeys.push(...change.ids);
+                }
+                else if (change.segment === AdapterContextSegment) {
+                    adapterContextSegmentKeys.push(...change.ids);
                 }
-                // TODO: W-8909393 If expiration is the only thing that changed we should not evict the data... so
-                // if we stored expiration and data at different keys (or same keys in different segments)
-                // then we could know if only the expiration has changed and we wouldn't need to evict
-                // and go through an entire broadcast/revive cycle for unchanged data
-                // call base environment storeEvict so this evict is not tracked for durable deletion
-                environment.storeEvict(key);
             }
-            environment.storeBroadcast(rebuildSnapshot, environment.snapshotAvailable);
+            // process adapter context changes
+            const adapterContextKeysFromDifferentInstance = [];
+            for (const key of adapterContextSegmentKeys) {
+                if (pendingContextStoreKeys.has(key)) {
+                    // if this instance caused the L2 write then remove from the
+                    // "pending" Set and move on
+                    pendingContextStoreKeys.delete(key);
+                }
+                else {
+                    // else it came from another luvio instance and we need to
+                    // read from L2
+                    adapterContextKeysFromDifferentInstance.push(key);
+                }
+            }
+            if (adapterContextKeysFromDifferentInstance.length > 0) {
+                // change handlers are sync, so kick off L2 read of the changed
+                // segment keys but we can't await it
+                durableStore
+                    .getEntries(adapterContextKeysFromDifferentInstance, AdapterContextSegment)
+                    .then((entries) => {
+                    if (entries !== undefined) {
+                        const entryKeys = keys(entries);
+                        for (let i = 0, len = entryKeys.length; i < len; i++) {
+                            const entryKey = entryKeys[i];
+                            const entry = entries[entryKey];
+                            contextStores[entryKey] = entry.data;
+                        }
+                    }
+                })
+                    .catch(durableStoreErrorHandler);
+            }
+            // process default segment changes
+            const defaultSegmentKeysLength = defaultSegmentKeys.length;
+            if (defaultSegmentKeysLength > 0) {
+                for (let i = 0; i < defaultSegmentKeysLength; i++) {
+                    const key = defaultSegmentKeys[i];
+                    const canonical = environment.storeGetCanonicalKey(key);
+                    if (canonical !== key) {
+                        continue;
+                    }
+                    // TODO: W-8909393 If expiration is the only thing that changed we should not evict the data... so
+                    // if we stored expiration and data at different keys (or same keys in different segments)
+                    // then we could know if only the expiration has changed and we wouldn't need to evict
+                    // and go through an entire broadcast/revive cycle for unchanged data
+                    // call base environment storeEvict so this evict is not tracked for durable deletion
+                    environment.storeEvict(key);
+                }
+                environment.storeBroadcast(rebuildSnapshot, environment.snapshotAvailable);
+            }
         });
         const dispose = function () {
             validateNotDisposed();
@@ -587,7 +623,7 @@
             validateNotDisposed();
             const { contextId, onContextLoaded } = options;
             let context = undefined;
-            const contextAsPromise = reviveOrCreateContext(`${contextId}${ADAPTER_CONTEXT_ID_SUFFIX}`, durableStore, durableStoreErrorHandler, onContextLoaded);
+            const contextAsPromise = reviveOrCreateContext(`${contextId}${ADAPTER_CONTEXT_ID_SUFFIX}`, durableStore, durableStoreErrorHandler, contextStores, pendingContextStoreKeys, onContextLoaded);
             return (config, requestContext) => {
                 if (context === undefined) {
                     return contextAsPromise.then((revivedContext) => {

package/dist/umd/es5/environments.js

@@ -460,39 +460,53 @@
 
     var AdapterContextSegment = 'ADAPTER-CONTEXT';
     var ADAPTER_CONTEXT_ID_SUFFIX = '__NAMED_CONTEXT';
-    function reviveOrCreateContext(adapterId, durableStore, durableStoreErrorHandler, onContextLoaded) {
-        // initialize empty context store
-        var contextStore = create(null);
-        var context = {
-            set: function (key, value) {
-                var _a;
-                contextStore[key] = value;
-                durableStore.setEntries((_a = {},
-                    _a[adapterId] = { data: contextStore },
-                    _a), AdapterContextSegment);
-            },
-            get: function (key) {
-                return contextStore[key];
-            },
-        };
-        var contextReturn = function () {
-            if (onContextLoaded !== undefined) {
-                return onContextLoaded(context).then(function () {
-                    return context;
-                });
-            }
-            return context;
-        };
-        return durableStore.getEntries([adapterId], AdapterContextSegment).then(function (entries) {
-            // if durable store has a saved context then load it in the store
-            if (entries !== undefined && entries[adapterId] !== undefined) {
-                contextStore = entries[adapterId].data;
-            }
-            return contextReturn();
-        }, function (error) {
-            durableStoreErrorHandler(error);
-            //Return empty store when getEntries fails
-            return contextReturn();
+    function reviveOrCreateContext(adapterId, durableStore, durableStoreErrorHandler, contextStores, pendingContextStoreKeys, onContextLoaded) {
+        return __awaiter(this, void 0, void 0, function () {
+            var context, contextReturn, entries, error_1;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        // initialize empty context store
+                        contextStores[adapterId] = create(null);
+                        context = {
+                            set: function (key, value) {
+                                var _a;
+                                contextStores[adapterId][key] = value;
+                                durableStore.setEntries((_a = {},
+                                    _a[adapterId] = { data: contextStores[adapterId] },
+                                    _a), AdapterContextSegment);
+                                pendingContextStoreKeys.add(adapterId);
+                            },
+                            get: function (key) {
+                                return contextStores[adapterId][key];
+                            },
+                        };
+                        contextReturn = function () {
+                            if (onContextLoaded !== undefined) {
+                                return onContextLoaded(context).then(function () {
+                                    return context;
+                                });
+                            }
+                            return context;
+                        };
+                        _a.label = 1;
+                    case 1:
+                        _a.trys.push([1, 3, , 4]);
+                        return [4 /*yield*/, durableStore.getEntries([adapterId], AdapterContextSegment)];
+                    case 2:
+                        entries = _a.sent();
+                        if (entries !== undefined && entries[adapterId] !== undefined) {
+                            // if durable store has a saved context then load it in the store
+                            contextStores[adapterId] = entries[adapterId].data;
+                        }
+                        return [3 /*break*/, 4];
+                    case 3:
+                        error_1 = _a.sent();
+                        durableStoreErrorHandler(error_1);
+                        return [3 /*break*/, 4];
+                    case 4: return [2 /*return*/, contextReturn()];
+                }
+            });
         });
     }
     function isUnfulfilledSnapshot(cachedSnapshotResult) {
@@ -517,6 +531,11 @@
         var ingestStagingStore = null;
         var durableTTLStore = new DurableTTLStore(durableStore);
         var mergeKeysPromiseMap = new Map();
+        // When a context store is mutated we write it to L2, which causes DS on change
+        // event. If this instance of makeDurable caused that L2 write we can ignore that
+        // on change event. This Set helps us do that.
+        var pendingContextStoreKeys = new Set();
+        var contextStores = create(null);
         var initializationPromise = new Promise(function (resolve) {
             var finish = function () {
                 resolve();
@@ -533,38 +552,69 @@
             }
         };
         var unsubscribe = durableStore.registerOnChangedListener(function (changes) {
-            var applicableChanges = changes.filter(function (change) {
+            var defaultSegmentKeys = [];
+            var adapterContextSegmentKeys = [];
+            for (var i = 0, len = changes.length; i < len; i++) {
+                var change = changes[i];
                 // we only care about changes to the data which is stored in the default
-                // segment
-                return change.segment === DefaultDurableSegment;
-            });
-            if (applicableChanges.length === 0) {
-                return;
-            }
-            var allKeys = [];
-            for (var i = 0, len = applicableChanges.length; i < len; i++) {
-                var change = applicableChanges[i];
-                allKeys.push.apply(allKeys, change.ids);
-            }
-            var keysLength = allKeys.length;
-            if (keysLength === 0) {
-                // if no keys changed then no need to broadcast
-                return;
-            }
-            for (var i = 0; i < keysLength; i++) {
-                var key = allKeys[i];
-                var canonical = environment.storeGetCanonicalKey(key);
-                if (canonical !== key) {
-                    continue;
+                // segment or the adapter context
+                if (change.segment === DefaultDurableSegment) {
+                    defaultSegmentKeys.push.apply(defaultSegmentKeys, change.ids);
+                }
+                else if (change.segment === AdapterContextSegment) {
+                    adapterContextSegmentKeys.push.apply(adapterContextSegmentKeys, change.ids);
                 }
-                // TODO: W-8909393 If expiration is the only thing that changed we should not evict the data... so
-                // if we stored expiration and data at different keys (or same keys in different segments)
-                // then we could know if only the expiration has changed and we wouldn't need to evict
-                // and go through an entire broadcast/revive cycle for unchanged data
-                // call base environment storeEvict so this evict is not tracked for durable deletion
-                environment.storeEvict(key);
             }
-            environment.storeBroadcast(rebuildSnapshot, environment.snapshotAvailable);
+            // process adapter context changes
+            var adapterContextKeysFromDifferentInstance = [];
+            for (var _i = 0, adapterContextSegmentKeys_1 = adapterContextSegmentKeys; _i < adapterContextSegmentKeys_1.length; _i++) {
+                var key = adapterContextSegmentKeys_1[_i];
+                if (pendingContextStoreKeys.has(key)) {
+                    // if this instance caused the L2 write then remove from the
+                    // "pending" Set and move on
+                    pendingContextStoreKeys.delete(key);
+                }
+                else {
+                    // else it came from another luvio instance and we need to
+                    // read from L2
+                    adapterContextKeysFromDifferentInstance.push(key);
+                }
+            }
+            if (adapterContextKeysFromDifferentInstance.length > 0) {
+                // change handlers are sync, so kick off L2 read of the changed
+                // segment keys but we can't await it
+                durableStore
+                    .getEntries(adapterContextKeysFromDifferentInstance, AdapterContextSegment)
+                    .then(function (entries) {
+                    if (entries !== undefined) {
+                        var entryKeys = keys(entries);
+                        for (var i = 0, len = entryKeys.length; i < len; i++) {
+                            var entryKey = entryKeys[i];
+                            var entry = entries[entryKey];
+                            contextStores[entryKey] = entry.data;
+                        }
+                    }
+                })
+                    .catch(durableStoreErrorHandler);
+            }
+            // process default segment changes
+            var defaultSegmentKeysLength = defaultSegmentKeys.length;
+            if (defaultSegmentKeysLength > 0) {
+                for (var i = 0; i < defaultSegmentKeysLength; i++) {
+                    var key = defaultSegmentKeys[i];
+                    var canonical = environment.storeGetCanonicalKey(key);
+                    if (canonical !== key) {
+                        continue;
+                    }
+                    // TODO: W-8909393 If expiration is the only thing that changed we should not evict the data... so
+                    // if we stored expiration and data at different keys (or same keys in different segments)
+                    // then we could know if only the expiration has changed and we wouldn't need to evict
+                    // and go through an entire broadcast/revive cycle for unchanged data
+                    // call base environment storeEvict so this evict is not tracked for durable deletion
+                    environment.storeEvict(key);
+                }
+                environment.storeBroadcast(rebuildSnapshot, environment.snapshotAvailable);
+            }
         });
         var dispose = function () {
             validateNotDisposed();
@@ -678,7 +728,7 @@
             validateNotDisposed();
             var contextId = options.contextId, onContextLoaded = options.onContextLoaded;
             var context = undefined;
-            var contextAsPromise = reviveOrCreateContext("".concat(contextId).concat(ADAPTER_CONTEXT_ID_SUFFIX), durableStore, durableStoreErrorHandler, onContextLoaded);
+            var contextAsPromise = reviveOrCreateContext("".concat(contextId).concat(ADAPTER_CONTEXT_ID_SUFFIX), durableStore, durableStoreErrorHandler, contextStores, pendingContextStoreKeys, onContextLoaded);
             return function (config, requestContext) {
                 if (context === undefined) {
                     return contextAsPromise.then(function (revivedContext) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@luvio/environments",
-    "version": "0.109.0",
+    "version": "0.109.1",
     "description": "Luvio Environments",
     "repository": {
         "type": "git",
@@ -23,7 +23,7 @@
         "watch": "yarn build --watch"
     },
     "dependencies": {
-        "@luvio/engine": "0.109.0"
+        "@luvio/engine": "0.109.1"
     },
     "bundlesize": [
         {