@luvio/environments 0.68.0 → 0.72.0

package/dist/es/es2018/DurableStore.d.ts

@@ -12,7 +12,6 @@ export interface DurableStoreEntry<T = unknown> {
     metadata?: {
         ingestionTimestamp: number;
         expirationTimestamp: number;
-        staleTimestamp?: number;
         namespace: string;
         representationName: string;
     };

package/dist/es/es2018/environments.js

@@ -18,7 +18,11 @@ const { keys, create, assign, freeze } = Object;
 const { isArray } = Array;
 
 function appendTTLStrategy(storeLookup, ttlStrategy) {
-    return (sel, refresh) => storeLookup(sel, refresh, ttlStrategy);
+    const returnStoreLookup = (sel, refresh) => storeLookup(sel, refresh, ttlStrategy);
+    // append ttlStrategy to storeLookup function (in cases where custom adapter
+    // wants to perform it's own lookup)
+    returnStoreLookup.ttlStrategy = ttlStrategy;
+    return returnStoreLookup;
 }
 function buildNetworkSnapshot(args) {
     const { buildNetworkSnapshot, buildSnapshotContext, coercedAdapterRequestContext } = args;
@@ -41,102 +45,112 @@ function buildTTLStrategy(staleDurationMilliseconds = 0) {
         }
         return StoreResolveResultState.Found;
     };
+}
+// TODO - update userland-facing APIs to return `AvailableSnapshot` instead of `Snapshot`
+// and then the signatures here can be updated as well
+function buildAvailableSnapshotFromCachedSnapshotResponse(cachedSnapshot, availableSnapshotFunc) {
+    if (isPromise(cachedSnapshot)) {
+        return cachedSnapshot.then(availableSnapshotFunc);
+    }
+    return availableSnapshotFunc(cachedSnapshot);
+}
+function isPromise(value) {
+    if (value === undefined) {
+        return false;
+    }
+    // check for Thenable due to test frameworks using custom Promise impls
+    return value.then !== undefined;
 }
 
-function buildCacheAndNetworkImplementation(funcs, staleDurationSeconds) {
+function buildCacheAndNetworkImplementation(funcs, staleDurationSeconds = 0) {
     return function (args) {
         funcs.validateNotDisposed();
-        const { buildInMemorySnapshot, buildNetworkSnapshot: buildNetworkSnapshot$1, buildSnapshotContext, storeLookup, coercedAdapterRequestContext, } = args;
-        const staleDurationMilliseconds = staleDurationSeconds === undefined ? undefined : staleDurationSeconds * 1000;
+        const { buildCachedSnapshot, buildNetworkSnapshot: buildNetworkSnapshot$1, buildSnapshotContext, storeLookup, coercedAdapterRequestContext, } = args;
+        const staleDurationMilliseconds = staleDurationSeconds * 1000;
         const cachePolicyStoreLookup = appendTTLStrategy(storeLookup, buildTTLStrategy(staleDurationMilliseconds));
-        const snapshot = buildInMemorySnapshot(buildSnapshotContext, cachePolicyStoreLookup);
-        if (snapshot !== undefined) {
-            // data found in L1 cache
-            if (snapshot.state === 'Fulfilled' || snapshot.state === 'Error') {
-                // kick off network request, do not await it
-                buildNetworkSnapshot$1(buildSnapshotContext, coercedAdapterRequestContext);
-                // return the cached snapshot to caller
-                return snapshot;
-            }
-            // network request outstanding
-            if (snapshot.state === 'Pending') {
-                // kick off another network request, do not await it
-                buildNetworkSnapshot$1(buildSnapshotContext, coercedAdapterRequestContext);
-                return args.resolvePendingSnapshot(snapshot);
-            }
-            // stale data found in L1 cache
-            if (snapshot.state === 'Stale') {
-                // kick off network request, do not await it
-                buildNetworkSnapshot$1(buildSnapshotContext, coercedAdapterRequestContext);
-                // return the cached snapshot to caller
-                return snapshot;
-            }
-            // if unfulfilled we have enough info to do an L2 lookup
-            if (snapshot.state === 'Unfulfilled') {
-                return funcs
-                    .reviveSnapshotWithCachePolicy(snapshot, cachePolicyStoreLookup)
-                    .then((revivedSnapshot) => {
-                    // data found in L2 cache
-                    if (revivedSnapshot.state === 'Fulfilled' ||
-                        revivedSnapshot.state === 'Error') {
-                        // kick off network request, do not await it
-                        buildNetworkSnapshot$1(buildSnapshotContext, coercedAdapterRequestContext);
-                        // return the L2 cached snapshot to caller
-                        return revivedSnapshot;
-                    }
-                    if (revivedSnapshot.state === 'Pending') {
-                        // kick off network request, do not await it
-                        buildNetworkSnapshot$1(buildSnapshotContext, coercedAdapterRequestContext);
-                        return args.resolvePendingSnapshot(revivedSnapshot);
-                    }
-                    // stale data found in L2 cache
-                    if (revivedSnapshot.state === 'Stale') {
-                        // kick off network request, do not await it
-                        buildNetworkSnapshot$1(buildSnapshotContext, coercedAdapterRequestContext);
-                        // return the L2 cached snapshot to caller
-                        return revivedSnapshot;
-                    }
-                    // data not found in L2 cache, go to the network
-                    return buildNetworkSnapshot(args);
-                });
+        const cachedSnapshot = buildCachedSnapshot(buildSnapshotContext, cachePolicyStoreLookup);
+        return buildAvailableSnapshotFromCachedSnapshotResponse(cachedSnapshot, (snapshot) => {
+            if (snapshot !== undefined) {
+                // data found in L1 cache
+                if (snapshot.state === 'Fulfilled' ||
+                    snapshot.state === 'Error' ||
+                    snapshot.state === 'Stale') {
+                    // kick off network request, do not await it
+                    buildNetworkSnapshot$1(buildSnapshotContext, coercedAdapterRequestContext);
+                    // return the cached snapshot to caller
+                    return snapshot;
+                }
+                // network request outstanding
+                if (snapshot.state === 'Pending') {
+                    // kick off another network request, do not await it
+                    buildNetworkSnapshot$1(buildSnapshotContext, coercedAdapterRequestContext);
+                    return args.resolvePendingSnapshot(snapshot);
+                }
+                // if unfulfilled we have enough info to do an L2 lookup
+                if (snapshot.state === 'Unfulfilled') {
+                    return funcs
+                        .reviveSnapshotWithCachePolicy(snapshot, cachePolicyStoreLookup)
+                        .then((revivedSnapshot) => {
+                        // data found in L2 cache
+                        if (revivedSnapshot.state === 'Fulfilled' ||
+                            revivedSnapshot.state === 'Error' ||
+                            revivedSnapshot.state === 'Stale') {
+                            // kick off network request, do not await it
+                            buildNetworkSnapshot$1(buildSnapshotContext, coercedAdapterRequestContext);
+                            // return the L2 cached snapshot to caller
+                            return revivedSnapshot;
+                        }
+                        if (revivedSnapshot.state === 'Pending') {
+                            // kick off network request, do not await it
+                            buildNetworkSnapshot$1(buildSnapshotContext, coercedAdapterRequestContext);
+                            return args.resolvePendingSnapshot(revivedSnapshot);
+                        }
+                        // data not found in L2 cache, go to the network
+                        return buildNetworkSnapshot(args);
+                    });
+                }
             }
-        }
-        return buildNetworkSnapshot(args);
+            return buildNetworkSnapshot(args);
+        });
     };
 }
 
 function buildCacheThenNetworkImplementation(funcs) {
     return function (args) {
         funcs.validateNotDisposed();
-        const { buildInMemorySnapshot, buildSnapshotContext, storeLookup } = args;
+        const { buildCachedSnapshot, buildSnapshotContext, storeLookup } = args;
         const cachePolicyStoreLookup = appendTTLStrategy(storeLookup, buildTTLStrategy());
-        const snapshot = buildInMemorySnapshot(buildSnapshotContext, cachePolicyStoreLookup);
-        if (snapshot !== undefined) {
-            // data found in L1 cache
-            if (snapshot.state === 'Fulfilled' || snapshot.state === 'Error') {
-                return snapshot;
-            }
-            if (snapshot.state === 'Pending') {
-                return args.resolvePendingSnapshot(snapshot);
-            }
-            // data not found in L1 cache, try L2 cache
-            return funcs
-                .reviveSnapshotWithCachePolicy(snapshot, cachePolicyStoreLookup)
-                .then((revivedSnapshot) => {
-                // data found in L2 cache
-                if (revivedSnapshot.state === 'Fulfilled' ||
-                    revivedSnapshot.state === 'Error') {
-                    return revivedSnapshot;
+        const cachedSnapshot = buildCachedSnapshot(buildSnapshotContext, cachePolicyStoreLookup);
+        return buildAvailableSnapshotFromCachedSnapshotResponse(cachedSnapshot, (snapshot) => {
+            if (snapshot !== undefined) {
+                // data found in L1 cache
+                if (snapshot.state === 'Fulfilled' || snapshot.state === 'Error') {
+                    return snapshot;
                 }
-                if (revivedSnapshot.state === 'Pending') {
-                    return args.resolvePendingSnapshot(revivedSnapshot);
+                if (snapshot.state === 'Pending') {
+                    return args.resolvePendingSnapshot(snapshot);
                 }
-                // data not found in L2 cache, go to the network
-                return buildNetworkSnapshot(args);
-            });
-        }
-        // L1 lookup could not find enough information to even construct a snapshot, go to the network
-        return buildNetworkSnapshot(args);
+                // if unfulfilled we have enough info to do an L2 lookup
+                if (snapshot.state === 'Unfulfilled') {
+                    return funcs
+                        .reviveSnapshotWithCachePolicy(snapshot, cachePolicyStoreLookup)
+                        .then((revivedSnapshot) => {
+                        // data found in L2 cache
+                        if (revivedSnapshot.state === 'Fulfilled' ||
+                            revivedSnapshot.state === 'Error') {
+                            return revivedSnapshot;
+                        }
+                        if (revivedSnapshot.state === 'Pending') {
+                            return args.resolvePendingSnapshot(revivedSnapshot);
+                        }
+                        // data not found in L2 cache, go to the network
+                        return buildNetworkSnapshot(args);
+                    });
+                }
+            }
+            // L1 lookup could not find enough information to even construct a snapshot, go to the network
+            return buildNetworkSnapshot(args);
+        });
     };
 }
 
@@ -180,84 +194,90 @@ function buildNotCachedErrorSnapshot() {
         error,
         state: 'Error',
         data: undefined,
-        // TODO[@W-10164067]: copy refresh data from the snapshot returned by buildInMemorySnapshot (if any)
+        // TODO[@W-10164067]: copy refresh data from the snapshot returned by buildCachedSnapshot (if any)
         // refresh: ...
     };
 }
 function buildOnlyIfCachedImplementation(funcs) {
     return function (args) {
         funcs.validateNotDisposed();
-        const { buildInMemorySnapshot, buildSnapshotContext, storeLookup } = args;
+        const { buildCachedSnapshot, buildSnapshotContext, storeLookup } = args;
         const cachePolicyStoreLookup = appendTTLStrategy(storeLookup, buildTTLStrategy());
-        const snapshot = buildInMemorySnapshot(buildSnapshotContext, cachePolicyStoreLookup);
-        if (snapshot !== undefined) {
-            // data found in L1 cache
-            if (snapshot.state === 'Fulfilled' || snapshot.state === 'Error') {
-                return snapshot;
-            }
-            // network request outstanding, data is not cached
-            if (snapshot.state === 'Pending') {
-                return buildNotCachedErrorSnapshot();
-            }
-            // data not found in L1 cache, try L2 cache
-            return funcs
-                .reviveSnapshotWithCachePolicy(snapshot, cachePolicyStoreLookup)
-                .then((revivedSnapshot) => {
-                // data found in L2 cache
-                if (revivedSnapshot.state === 'Fulfilled' ||
-                    revivedSnapshot.state === 'Error') {
-                    return revivedSnapshot;
+        const cachedSnapshot = buildCachedSnapshot(buildSnapshotContext, cachePolicyStoreLookup);
+        return buildAvailableSnapshotFromCachedSnapshotResponse(cachedSnapshot, (snapshot) => {
+            if (snapshot !== undefined) {
+                // data found in L1 cache
+                if (snapshot.state === 'Fulfilled' || snapshot.state === 'Error') {
+                    return snapshot;
                 }
-                // data is not cached
-                return buildNotCachedErrorSnapshot();
-            });
-        }
-        return buildNotCachedErrorSnapshot();
+                // network request outstanding, data is not cached
+                if (snapshot.state === 'Pending') {
+                    return buildNotCachedErrorSnapshot();
+                }
+                // if unfulfilled we have enough info to do an L2 lookup
+                if (snapshot.state === 'Unfulfilled') {
+                    return funcs
+                        .reviveSnapshotWithCachePolicy(snapshot, cachePolicyStoreLookup)
+                        .then((revivedSnapshot) => {
+                        // data found in L2 cache
+                        if (revivedSnapshot.state === 'Fulfilled' ||
+                            revivedSnapshot.state === 'Error') {
+                            return revivedSnapshot;
+                        }
+                        // data is not cached
+                        return buildNotCachedErrorSnapshot();
+                    });
+                }
+            }
+            return buildNotCachedErrorSnapshot();
+        });
     };
 }
 
 function buildStaleWhileRevalidateImplementation(funcs, staleDurationSeconds) {
     return function (args) {
         funcs.validateNotDisposed();
-        const { buildInMemorySnapshot, buildSnapshotContext, storeLookup } = args;
+        const { buildCachedSnapshot, buildSnapshotContext, storeLookup } = args;
         const cachePolicyStoreLookup = appendTTLStrategy(storeLookup, buildTTLStrategy(staleDurationSeconds * 1000));
-        const snapshot = buildInMemorySnapshot(buildSnapshotContext, cachePolicyStoreLookup);
-        if (snapshot !== undefined) {
-            // data found in L1 cache
-            if (snapshot.state === 'Fulfilled' || snapshot.state === 'Error') {
-                return snapshot;
-            }
-            if (snapshot.state === 'Pending') {
-                return args.resolvePendingSnapshot(snapshot);
-            }
-            // stale data found in L1 cache
-            if (snapshot.state === 'Stale') {
-                buildNetworkSnapshot(args);
-                return snapshot;
-            }
-            // data not found in L1 cache, try L2 cache
-            return funcs
-                .reviveSnapshotWithCachePolicy(snapshot, cachePolicyStoreLookup)
-                .then((revivedSnapshot) => {
-                // data found in L2 cache
-                if (revivedSnapshot.state === 'Fulfilled' ||
-                    revivedSnapshot.state === 'Error') {
-                    return revivedSnapshot;
+        const cachedSnapshot = buildCachedSnapshot(buildSnapshotContext, cachePolicyStoreLookup);
+        return buildAvailableSnapshotFromCachedSnapshotResponse(cachedSnapshot, (snapshot) => {
+            if (snapshot !== undefined) {
+                // data found in L1 cache
+                if (snapshot.state === 'Fulfilled' || snapshot.state === 'Error') {
+                    return snapshot;
                 }
-                if (revivedSnapshot.state === 'Pending') {
-                    return args.resolvePendingSnapshot(revivedSnapshot);
+                if (snapshot.state === 'Pending') {
+                    return args.resolvePendingSnapshot(snapshot);
                 }
-                // stale data found in L2 cache
-                if (revivedSnapshot.state === 'Stale') {
+                // stale data found in L1 cache
+                if (snapshot.state === 'Stale') {
                     buildNetworkSnapshot(args);
-                    return revivedSnapshot;
+                    return snapshot;
                 }
-                // data not found in L2 cache, go to the network
-                return buildNetworkSnapshot(args);
-            });
-        }
-        // L1 lookup could not find enough information to even construct a snapshot, go to the network
-        return buildNetworkSnapshot(args);
+                // data not found in L1 cache, try L2 cache
+                return funcs
+                    .reviveSnapshotWithCachePolicy(snapshot, cachePolicyStoreLookup)
+                    .then((revivedSnapshot) => {
+                    // data found in L2 cache
+                    if (revivedSnapshot.state === 'Fulfilled' ||
+                        revivedSnapshot.state === 'Error') {
+                        return revivedSnapshot;
+                    }
+                    if (revivedSnapshot.state === 'Pending') {
+                        return args.resolvePendingSnapshot(revivedSnapshot);
+                    }
+                    // stale data found in L2 cache
+                    if (revivedSnapshot.state === 'Stale') {
+                        buildNetworkSnapshot(args);
+                        return revivedSnapshot;
+                    }
+                    // data not found in L2 cache, go to the network
+                    return buildNetworkSnapshot(args);
+                });
+            }
+            // L1 lookup could not find enough information to even construct a snapshot, go to the network
+            return buildNetworkSnapshot(args);
+        });
     };
 }
 
@@ -268,7 +288,7 @@ function buildValidAtImplementation(funcs, basePolicyImplementation, timestamp)
         // TTLStrategy to use the the valid-at cache policy's timestamp. The flow goes:
         //
         // Environment.applyCachePolicy => validAtImplementation (this function) =>
-        //    basePolicyImplementation => adapter's buildInMemorySnapshot =>
+        //    basePolicyImplementation => adapter's buildCachedSnapshot =>
         //    basePolicyImplementation's storeLookup => validAtStoreLookup (below) =>
         //    Environment.applyCachePolicy's storeLookup => Store/Reader code =>
         //    valid-at TTLStrategy (below) =>
@@ -375,9 +395,6 @@ function isStoreEntryError(storeRecord) {
     return storeRecord.__type === 'error';
 }
 
-function isStoreEntryExpiredAndError(storeRecord, expirationTimestamp, now) {
-    return isStoreEntryError(storeRecord) && expirationTimestamp < now;
-}
 /**
  * Takes a set of entries from DurableStore and publishes them via the passed in funcs.
  * This respects expiration and checks for valid DurableStore data shapes.  This should
@@ -400,7 +417,6 @@ function publishDurableStoreEntries(durableRecords, publish, publishMetadata) {
         // no records to revive
         return { revivedKeys, hadUnexpectedShape };
     }
-    const now = Date.now();
     for (let i = 0, len = durableKeys.length; i < len; i += 1) {
         const key = durableKeys[i];
         const durableRecord = durableRecords[key];
@@ -416,27 +432,12 @@ function publishDurableStoreEntries(durableRecords, publish, publishMetadata) {
             continue;
         }
         if (metadata !== undefined) {
-            const { expirationTimestamp, staleTimestamp } = metadata;
+            const { expirationTimestamp } = metadata;
             if (expirationTimestamp === undefined) {
                 // if unexpected expiration data skip reviving
                 hadUnexpectedShape = true;
                 continue;
             }
-            // if past stale TTL then don't revive
-            if (staleTimestamp !== undefined && staleTimestamp < now) {
-                continue;
-            }
-            // We don't want to revive a cached value if it's an error and it's
-            // expirationTimestamp TTL is expired, otherwise we would never hit the network
-            // during resolveSnapshot or rebuildSnapshot because the way the Reader works.
-            // If a StoreEntry is an error, the Reader will return UnfulfilledSnapshot
-            // if stale TTL is expired.  But it will return ErrorSnapshot (instead of
-            // StaleSnapshot) if expirationTimestamp TTL is expired (but stale TTL isn't).
-            // In our environment the stale TTL is maxed out so Reader will always
-            // return ErrorSnapshot.
-            if (isStoreEntryExpiredAndError(data, expirationTimestamp, now)) {
-                continue;
-            }
             publishMetadata(key, metadata);
         }
         if (isStoreEntryError(data)) {
@@ -827,7 +828,7 @@ function makeDurable(environment, { durableStore, instrumentation }) {
         if (ingestStagingStore !== null) {
             return ingestStagingStore.lookup(sel, createSnapshot, refresh, ttlStrategy);
         }
-        // otherwise this is from buildInMemorySnapshot and we should use the luvio
+        // otherwise this is from buildCachedSnapshot and we should use the luvio
         // L1 store
         return environment.storeLookup(sel, createSnapshot, refresh, ttlStrategy);
     };
@@ -988,7 +989,7 @@ function makeDurable(environment, { durableStore, instrumentation }) {
             }
         }
     }
-    const applyCachePolicy = function (adapterRequestContext, buildSnapshotContext, buildInMemorySnapshot, buildNetworkSnapshot) {
+    const applyCachePolicy = function (adapterRequestContext, buildSnapshotContext, buildCachedSnapshot, buildNetworkSnapshot) {
         validateNotDisposed();
         const { cachePolicy } = adapterRequestContext;
         const cachePolicyImpl = resolveCachePolicy(cachePolicy);
@@ -996,7 +997,7 @@ function makeDurable(environment, { durableStore, instrumentation }) {
         const storeLookup = (sel, refresh, ttlStrategy) => environment.storeLookup(sel, environment.createSnapshot, refresh, ttlStrategy);
         const applyCachePolicy = () => {
             return cachePolicyImpl({
-                buildInMemorySnapshot,
+                buildCachedSnapshot,
                 buildNetworkSnapshot,
                 buildSnapshotContext,
                 resolvePendingSnapshot,

package/dist/es/es2018/makeDurable/cachepolicies/utils.d.ts

@@ -1,8 +1,9 @@
-import { BuildInMemorySnapshot, CachePolicyImplementationArgs, Snapshot, StoreLookup, TTLStrategy, UnAvailableSnapshot } from '@luvio/engine';
+import { BuildCachedSnapshot, CachePolicyImplementationArgs, Snapshot, StoreLookup, TTLStrategy, UnAvailableSnapshot } from '@luvio/engine';
 export declare type DurableCachePolicyFunctions = {
     validateNotDisposed: () => void;
     reviveSnapshotWithCachePolicy: <D, V>(unavailableSnapshot: UnAvailableSnapshot<D, V>, storeLookup: StoreLookup<D, V>) => Promise<Snapshot<D, V>>;
 };
-export declare function appendTTLStrategy<C, D>(storeLookup: CachePolicyImplementationArgs<C, D>['storeLookup'], ttlStrategy: TTLStrategy): Parameters<BuildInMemorySnapshot<C, D>>[1];
+export declare function appendTTLStrategy<C, D>(storeLookup: CachePolicyImplementationArgs<C, D>['storeLookup'], ttlStrategy: TTLStrategy): Parameters<BuildCachedSnapshot<C, D>>[1];
 export declare function buildNetworkSnapshot<C, D>(args: CachePolicyImplementationArgs<C, D>): Promise<Snapshot<D, unknown>>;
 export declare function buildTTLStrategy(staleDurationMilliseconds?: number): TTLStrategy;
+export declare function buildAvailableSnapshotFromCachedSnapshotResponse<C, D>(cachedSnapshot: ReturnType<BuildCachedSnapshot<C, D>>, availableSnapshotFunc: (snapshot: Snapshot<D> | undefined) => Snapshot<D> | Promise<Snapshot<D>>): Snapshot<D> | Promise<Snapshot<D>>;

package/dist/umd/es2018/DurableStore.d.ts

@@ -12,7 +12,6 @@ export interface DurableStoreEntry<T = unknown> {
     metadata?: {
         ingestionTimestamp: number;
         expirationTimestamp: number;
-        staleTimestamp?: number;
         namespace: string;
         representationName: string;
     };