@luvio/environments 0.142.3 → 0.143.0

package/dist/environments.js

@@ -18,6 +18,7 @@ function isDeprecatedDurableStoreEntry(durableRecord) {
     return false;
 }
 const DefaultDurableSegment = 'DEFAULT';
+const RedirectDurableSegment = 'REDIRECT_KEYS';
 
 const { keys, create, assign, freeze } = Object;
 
@@ -261,7 +262,7 @@ class DurableTTLStore {
     }
 }
 
-function flushInMemoryStoreValuesToDurableStore(store, durableStore, durableStoreErrorHandler, additionalDurableStoreOperations = []) {
+function flushInMemoryStoreValuesToDurableStore(store, durableStore, durableStoreErrorHandler, redirects, additionalDurableStoreOperations = []) {
     const durableRecords = create(null);
     const evictedRecords = create(null);
     const { records, metadata: storeMetadata, visitedIds, refreshedIds, } = store.fallbackStringKeyInMemoryStore;
@@ -298,6 +299,18 @@ function flushInMemoryStoreValuesToDurableStore(store, durableStore, durableStor
             segment: DefaultDurableSegment,
         });
     }
+    // redirects
+    redirects.forEach((value, key) => {
+        durableStoreOperations.push({
+            type: 'setEntries',
+            entries: {
+                [key]: {
+                    data: { key, redirect: value },
+                },
+            },
+            segment: RedirectDurableSegment,
+        });
+    });
     // evicts
     const evictedKeys = keys(evictedRecords);
     if (evictedKeys.length > 0) {
@@ -346,6 +359,19 @@ function buildIngestStagingStore(environment) {
     return environment.storeBuildIngestionStagingStore();
 }
 
+async function reviveRedirects(durableStore, env) {
+    const entries = await durableStore.getAllEntries(RedirectDurableSegment);
+    if (entries) {
+        for (const durableEntry of Object.keys(entries)) {
+            const entry = entries[durableEntry];
+            const { data: { key, redirect }, } = entry;
+            if (entry) {
+                env.storeRedirect(key, redirect);
+            }
+        }
+    }
+}
+
 const AdapterContextSegment = 'ADAPTER-CONTEXT';
 const ADAPTER_CONTEXT_ID_SUFFIX = '__NAMED_CONTEXT';
 async function reviveOrCreateContext(adapterId, durableStore, durableStoreErrorHandler, contextStores, pendingContextStoreKeys, onContextLoaded) {
@@ -409,13 +435,18 @@ function makeDurable(environment, { durableStore, instrumentation }) {
     // event. If this instance of makeDurable caused that L2 write we can ignore that
     // on change event. This Set helps us do that.
     const pendingContextStoreKeys = new Set();
+    // redirects that need to be flushed to the durable store
+    const pendingStoreRedirects = new Map();
     const contextStores = create(null);
     let initializationPromise = new Promise((resolve) => {
         const finish = () => {
             resolve();
             initializationPromise = undefined;
         };
-        reviveTTLOverrides(durableTTLStore, environment).then(finish);
+        Promise.all([
+            reviveTTLOverrides(durableTTLStore, environment),
+            reviveRedirects(durableStore, environment),
+        ]).then(finish);
     });
     //instrumentation for durable store errors
     const durableStoreErrorHandler = handleDurableStoreRejection(instrumentation);
@@ -428,6 +459,8 @@ function makeDurable(environment, { durableStore, instrumentation }) {
     const unsubscribe = durableStore.registerOnChangedListener(async (changes) => {
         const defaultSegmentKeys = [];
         const adapterContextSegmentKeys = [];
+        const redirectSegmentKeys = [];
+        let shouldBroadcast = false;
         for (let i = 0, len = changes.length; i < len; i++) {
             const change = changes[i];
             // we only care about changes to the data which is stored in the default
@@ -438,6 +471,20 @@ function makeDurable(environment, { durableStore, instrumentation }) {
             else if (change.segment === AdapterContextSegment) {
                 adapterContextSegmentKeys.push(...change.ids);
             }
+            else if (change.segment === RedirectDurableSegment) {
+                redirectSegmentKeys.push(...change.ids);
+            }
+        }
+        if (redirectSegmentKeys.length > 0) {
+            const redirectEntries = await durableStore.getEntries(redirectSegmentKeys, RedirectDurableSegment);
+            if (redirectEntries !== undefined) {
+                const redirectKeys = Object.keys(redirectEntries);
+                for (const key of redirectKeys) {
+                    const redirectData = redirectEntries[key];
+                    environment.storeRedirect(redirectData.data.key, redirectData.data.redirect);
+                    shouldBroadcast = true;
+                }
+            }
         }
         // process adapter context changes
         const adapterContextKeysFromDifferentInstance = [];
@@ -474,10 +521,6 @@ function makeDurable(environment, { durableStore, instrumentation }) {
         if (defaultSegmentKeysLength > 0) {
             for (let i = 0; i < defaultSegmentKeysLength; i++) {
                 const key = defaultSegmentKeys[i];
-                const canonical = environment.storeGetCanonicalKey(key);
-                if (canonical !== key) {
-                    continue;
-                }
                 // TODO: W-8909393 If expiration is the only thing that changed we should not evict the data... so
                 // if we stored expiration and data at different keys (or same keys in different segments)
                 // then we could know if only the expiration has changed and we wouldn't need to evict
@@ -485,6 +528,9 @@ function makeDurable(environment, { durableStore, instrumentation }) {
                 // call base environment storeEvict so this evict is not tracked for durable deletion
                 environment.storeEvict(key);
             }
+            shouldBroadcast = true;
+        }
+        if (shouldBroadcast) {
             await environment.storeBroadcast(rebuildSnapshot, environment.snapshotAvailable);
         }
     });
@@ -540,7 +586,8 @@ function makeDurable(environment, { durableStore, instrumentation }) {
         if (ingestStagingStore === null) {
             return Promise.resolve();
         }
-        const promise = flushInMemoryStoreValuesToDurableStore(ingestStagingStore, durableStore, durableStoreErrorHandler, additionalDurableStoreOperations);
+        const promise = flushInMemoryStoreValuesToDurableStore(ingestStagingStore, durableStore, durableStoreErrorHandler, new Map(pendingStoreRedirects), additionalDurableStoreOperations);
+        pendingStoreRedirects.clear();
         ingestStagingStore = null;
         return promise;
     };
@@ -622,6 +669,7 @@ function makeDurable(environment, { durableStore, instrumentation }) {
     };
     const storeRedirect = function (existingKey, canonicalKey) {
         validateNotDisposed();
+        pendingStoreRedirects.set(existingKey, canonicalKey);
         // call redirect on staging store so "old" keys are removed from L2 on
         // the next publishChangesToDurableStore.  NOTE: we don't need to call
         // redirect on the base environment store because staging store and base

package/dist/types/DurableStore.d.ts

@@ -42,7 +42,9 @@ export interface DurableStoreEvictOperation extends BaseOperation {
 }
 export type DurableStoreOperation<T> = DurableStoreSetOperation<T> | DurableStoreEvictOperation;
 export type DefaultDurableSegmentName = 'DEFAULT';
+export type RedirectDurableSegmentName = 'REDIRECT_KEYS';
 export declare const DefaultDurableSegment: DefaultDurableSegmentName;
+export declare const RedirectDurableSegment: RedirectDurableSegmentName;
 export interface DurableStoreChange {
     /** The entry IDs of the entries that have changed */
     ids: string[];

package/dist/types/makeDurable/flush.d.ts

@@ -1,4 +1,4 @@
 import type { InMemoryStore } from '@luvio/engine';
 import type { DurableStore, DurableStoreOperation } from '../DurableStore';
 import type { DurableStoreRejectionHandler } from './error';
-export declare function flushInMemoryStoreValuesToDurableStore(store: InMemoryStore, durableStore: DurableStore, durableStoreErrorHandler: DurableStoreRejectionHandler, additionalDurableStoreOperations?: DurableStoreOperation<unknown>[]): Promise<void>;
+export declare function flushInMemoryStoreValuesToDurableStore(store: InMemoryStore, durableStore: DurableStore, durableStoreErrorHandler: DurableStoreRejectionHandler, redirects: Map<string, string>, additionalDurableStoreOperations?: DurableStoreOperation<unknown>[]): Promise<void>;

package/dist/types/makeDurable/redirects.d.ts

@@ -0,0 +1,7 @@
+import type { Environment } from '@luvio/engine';
+import type { DurableStore } from '../DurableStore';
+export interface RedirectData {
+    key: string;
+    redirect: string;
+}
+export declare function reviveRedirects(durableStore: DurableStore, env: Environment): Promise<void>;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@luvio/environments",
-    "version": "0.142.3",
+    "version": "0.143.0",
     "description": "Luvio Environments",
     "repository": {
         "type": "git",
@@ -27,7 +27,7 @@
         "watch": "yarn build --watch"
     },
     "dependencies": {
-        "@luvio/engine": "^0.142.3"
+        "@luvio/engine": "^0.143.0"
     },
     "volta": {
         "extends": "../../../package.json"