@luvio/environments 0.96.0 → 0.98.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/es2018/environments.js +47 -13
- package/dist/es/es2018/events.d.ts +15 -0
- package/dist/es/es2018/main.d.ts +1 -0
- package/dist/es/es2018/makeDurable/revive.d.ts +9 -1
- package/dist/umd/es2018/environments.js +46 -11
- package/dist/umd/es2018/events.d.ts +15 -0
- package/dist/umd/es2018/main.d.ts +1 -0
- package/dist/umd/es2018/makeDurable/revive.d.ts +9 -1
- package/dist/umd/es5/environments.js +46 -10
- package/dist/umd/es5/events.d.ts +15 -0
- package/dist/umd/es5/main.d.ts +1 -0
- package/dist/umd/es5/makeDurable/revive.d.ts +9 -1
- package/package.json +2 -2
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { buildStaleWhileRevalidateImplementation, InMemoryStore } from '@luvio/engine';
|
|
1
|
+
import { emitAdapterEvent, buildStaleWhileRevalidateImplementation, InMemoryStore } from '@luvio/engine';
|
|
2
2
|
|
|
3
3
|
function isDeprecatedDurableStoreEntry(durableRecord) {
|
|
4
4
|
if (durableRecord.expiration !== undefined) {
|
|
@@ -126,19 +126,25 @@ function publishDurableStoreEntries(durableRecords, publish, publishMetadata) {
|
|
|
126
126
|
*/
|
|
127
127
|
function reviveSnapshot(baseEnvironment, durableStore,
|
|
128
128
|
// TODO [W-10165787]: We should only allow Unfulfilled snapshot be passed in
|
|
129
|
-
unavailableSnapshot, durableStoreErrorHandler, buildL1Snapshot) {
|
|
129
|
+
unavailableSnapshot, durableStoreErrorHandler, buildL1Snapshot, reviveMetrics = { l2ReadCount: 0, l2ReadDuration: 0 }) {
|
|
130
130
|
const { recordId, select, seenRecords, state } = unavailableSnapshot;
|
|
131
131
|
// L2 can only revive Unfulfilled snapshots that have a selector since they have the
|
|
132
132
|
// info needed to revive (like missingLinks) and rebuild. Otherwise return L1 snapshot.
|
|
133
133
|
if (state !== 'Unfulfilled' || select === undefined) {
|
|
134
|
-
return Promise.resolve(
|
|
134
|
+
return Promise.resolve({
|
|
135
|
+
snapshot: unavailableSnapshot,
|
|
136
|
+
metrics: reviveMetrics,
|
|
137
|
+
});
|
|
135
138
|
}
|
|
136
139
|
// in case L1 store changes/deallocs a record while we are doing the async read
|
|
137
140
|
// we attempt to read all keys from L2 - so combine recordId with any seenRecords
|
|
138
141
|
const keysToReviveSet = assign({ [recordId]: true }, seenRecords);
|
|
139
142
|
const keysToRevive = keys(keysToReviveSet);
|
|
140
143
|
const canonicalKeys = keysToRevive.map((x) => baseEnvironment.storeGetCanonicalKey(x));
|
|
144
|
+
const start = Date.now();
|
|
145
|
+
reviveMetrics.l2ReadCount++;
|
|
141
146
|
return durableStore.getEntries(canonicalKeys, DefaultDurableSegment).then((durableRecords) => {
|
|
147
|
+
reviveMetrics.l2ReadDuration += Date.now() - start;
|
|
142
148
|
const { revivedKeys, hadUnexpectedShape } = publishDurableStoreEntries(durableRecords,
|
|
143
149
|
// TODO [W-10072584]: instead of implicitly using L1 we should take in
|
|
144
150
|
// publish and publishMetadata funcs, so callers can decide where to
|
|
@@ -147,18 +153,18 @@ unavailableSnapshot, durableStoreErrorHandler, buildL1Snapshot) {
|
|
|
147
153
|
// if the data coming back from DS had an unexpected shape then just
|
|
148
154
|
// return the L1 snapshot
|
|
149
155
|
if (hadUnexpectedShape === true) {
|
|
150
|
-
return unavailableSnapshot;
|
|
156
|
+
return { snapshot: unavailableSnapshot, metrics: reviveMetrics };
|
|
151
157
|
}
|
|
152
158
|
if (keys(revivedKeys).length === 0) {
|
|
153
159
|
// durable store doesn't have what we asked for so return L1 snapshot
|
|
154
|
-
return unavailableSnapshot;
|
|
160
|
+
return { snapshot: unavailableSnapshot, metrics: reviveMetrics };
|
|
155
161
|
}
|
|
156
162
|
// try building the snapshot from L1 now that we have revived the missingLinks
|
|
157
163
|
const snapshot = buildL1Snapshot();
|
|
158
164
|
// if snapshot is pending then some other in-flight refresh will broadcast
|
|
159
165
|
// later
|
|
160
166
|
if (snapshot.state === 'Pending') {
|
|
161
|
-
return snapshot;
|
|
167
|
+
return { snapshot, metrics: reviveMetrics };
|
|
162
168
|
}
|
|
163
169
|
if (snapshot.state === 'Unfulfilled') {
|
|
164
170
|
// have to check if the new snapshot has any additional seenRecords
|
|
@@ -174,15 +180,15 @@ unavailableSnapshot, durableStoreErrorHandler, buildL1Snapshot) {
|
|
|
174
180
|
for (let i = 0, len = newKeys.length; i < len; i++) {
|
|
175
181
|
const newSnapshotSeenKey = newKeys[i];
|
|
176
182
|
if (alreadyRequestedOrRevivedSet[newSnapshotSeenKey] !== true) {
|
|
177
|
-
return reviveSnapshot(baseEnvironment, durableStore, snapshot, durableStoreErrorHandler, buildL1Snapshot);
|
|
183
|
+
return reviveSnapshot(baseEnvironment, durableStore, snapshot, durableStoreErrorHandler, buildL1Snapshot, reviveMetrics);
|
|
178
184
|
}
|
|
179
185
|
}
|
|
180
186
|
}
|
|
181
|
-
return snapshot;
|
|
187
|
+
return { snapshot, metrics: reviveMetrics };
|
|
182
188
|
}, (error) => {
|
|
183
189
|
durableStoreErrorHandler(error);
|
|
184
190
|
// getEntries failed, return the L1 snapshot
|
|
185
|
-
return unavailableSnapshot;
|
|
191
|
+
return { snapshot: unavailableSnapshot, metrics: reviveMetrics };
|
|
186
192
|
});
|
|
187
193
|
}
|
|
188
194
|
|
|
@@ -324,6 +330,19 @@ function flushInMemoryStoreValuesToDurableStore(store, durableStore, durableStor
|
|
|
324
330
|
return Promise.resolve();
|
|
325
331
|
}
|
|
326
332
|
|
|
333
|
+
const DurableEnvironmentEventDiscriminator = 'durable';
|
|
334
|
+
function isDurableEnvironmentEvent(event) {
|
|
335
|
+
return (event.type === 'environment' && event.environment === DurableEnvironmentEventDiscriminator);
|
|
336
|
+
}
|
|
337
|
+
function emitDurableEnvironmentAdapterEvent(eventData, observers) {
|
|
338
|
+
emitAdapterEvent({
|
|
339
|
+
type: 'environment',
|
|
340
|
+
timestamp: Date.now(),
|
|
341
|
+
environment: DurableEnvironmentEventDiscriminator,
|
|
342
|
+
data: eventData,
|
|
343
|
+
}, observers);
|
|
344
|
+
}
|
|
345
|
+
|
|
327
346
|
const AdapterContextSegment = 'ADAPTER-CONTEXT';
|
|
328
347
|
const ADAPTER_CONTEXT_ID_SUFFIX = '__NAMED_CONTEXT';
|
|
329
348
|
function reviveOrCreateContext(adapterId, durableStore, durableStoreErrorHandler, onContextLoaded) {
|
|
@@ -535,7 +554,9 @@ function makeDurable(environment, { durableStore, instrumentation }) {
|
|
|
535
554
|
reviveSnapshot(environment, durableStore, rebuilt, durableStoreErrorHandler, () =>
|
|
536
555
|
// reviveSnapshot will revive into L1, and since "records" is a reference
|
|
537
556
|
// (and not a copy) to the L1 records we can use it for rebuild
|
|
538
|
-
environment.rebuildSnapshot(snapshot, records, storeMetadataMap, redirects, () => { })).then(
|
|
557
|
+
environment.rebuildSnapshot(snapshot, records, storeMetadataMap, redirects, () => { })).then((result) => {
|
|
558
|
+
onAsyncRebuild(result.snapshot);
|
|
559
|
+
});
|
|
539
560
|
// synchronously return the base snapshot as Pending if not already
|
|
540
561
|
return snapshot.state === 'Pending'
|
|
541
562
|
? snapshot
|
|
@@ -592,7 +613,18 @@ function makeDurable(environment, { durableStore, instrumentation }) {
|
|
|
592
613
|
// if the adapter attempted to do an L1 lookup and it was unfulfilled
|
|
593
614
|
// then we can attempt an L2 lookup
|
|
594
615
|
if (isUnfulfilledSnapshot(snapshot)) {
|
|
595
|
-
|
|
616
|
+
const start = Date.now();
|
|
617
|
+
emitDurableEnvironmentAdapterEvent({ type: 'l2-revive-start' }, adapterRequestContext.eventObservers);
|
|
618
|
+
const revivedSnapshot = reviveSnapshot(environment, durableStore, snapshot, durableStoreErrorHandler, () => storeLookup(snapshot.select, snapshot.refresh)).then((result) => {
|
|
619
|
+
emitDurableEnvironmentAdapterEvent({
|
|
620
|
+
type: 'l2-revive-end',
|
|
621
|
+
duration: Date.now() - start,
|
|
622
|
+
l2Trips: result.metrics.l2ReadCount,
|
|
623
|
+
l2Duration: result.metrics.l2ReadDuration,
|
|
624
|
+
}, adapterRequestContext.eventObservers);
|
|
625
|
+
return result.snapshot;
|
|
626
|
+
});
|
|
627
|
+
return revivedSnapshot;
|
|
596
628
|
}
|
|
597
629
|
// otherwise just return what buildCachedSnapshot gave us
|
|
598
630
|
return snapshot;
|
|
@@ -645,7 +677,9 @@ function makeDurable(environment, { durableStore, instrumentation }) {
|
|
|
645
677
|
return snapshotFromMemoryIngest;
|
|
646
678
|
}
|
|
647
679
|
// if snapshot from staging store lookup is unfulfilled then do an L2 lookup
|
|
648
|
-
return reviveSnapshot(environment, durableStore, snapshotFromMemoryIngest, durableStoreErrorHandler, () => environment.storeLookup(snapshotFromMemoryIngest.select, environment.createSnapshot, snapshotFromMemoryIngest.refresh))
|
|
680
|
+
return reviveSnapshot(environment, durableStore, snapshotFromMemoryIngest, durableStoreErrorHandler, () => environment.storeLookup(snapshotFromMemoryIngest.select, environment.createSnapshot, snapshotFromMemoryIngest.refresh)).then((result) => {
|
|
681
|
+
return result.snapshot;
|
|
682
|
+
});
|
|
649
683
|
});
|
|
650
684
|
};
|
|
651
685
|
if (keys(keysToRevive).length === 0) {
|
|
@@ -711,4 +745,4 @@ function makeDurable(environment, { durableStore, instrumentation }) {
|
|
|
711
745
|
});
|
|
712
746
|
}
|
|
713
747
|
|
|
714
|
-
export { DefaultDurableSegment, DurableStoreOperationType, makeDurable, publishDurableStoreEntries };
|
|
748
|
+
export { DefaultDurableSegment, DurableStoreOperationType, isDurableEnvironmentEvent, makeDurable, publishDurableStoreEntries };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { EnvironmentAdapterEvent } from '@luvio/engine';
|
|
2
|
+
import type { LuvioAdapterEventObserver } from '@luvio/engine';
|
|
3
|
+
interface ReviveStartEvent {
|
|
4
|
+
type: 'l2-revive-start';
|
|
5
|
+
}
|
|
6
|
+
interface ReviveEndEvent {
|
|
7
|
+
type: 'l2-revive-end';
|
|
8
|
+
duration: number;
|
|
9
|
+
l2Trips: number;
|
|
10
|
+
l2Duration: number;
|
|
11
|
+
}
|
|
12
|
+
export declare type DurableEnvironmentAdapterEventData = ReviveStartEvent | ReviveEndEvent;
|
|
13
|
+
export declare function isDurableEnvironmentEvent(event: EnvironmentAdapterEvent<unknown>): event is EnvironmentAdapterEvent<DurableEnvironmentAdapterEventData>;
|
|
14
|
+
export declare function emitDurableEnvironmentAdapterEvent(eventData: DurableEnvironmentAdapterEventData, observers: LuvioAdapterEventObserver[] | undefined): void;
|
|
15
|
+
export {};
|
package/dist/es/es2018/main.d.ts
CHANGED
|
@@ -2,3 +2,4 @@ export { DurableStore, DurableStoreEntries, DurableStoreEntry, DurableStoreChang
|
|
|
2
2
|
export { DurableTTLOverride, DefaultDurableTTLOverride } from './DurableTTLStore';
|
|
3
3
|
export { makeDurable, DurableEnvironment } from './makeDurable';
|
|
4
4
|
export { publishDurableStoreEntries } from './makeDurable/revive';
|
|
5
|
+
export { isDurableEnvironmentEvent } from './events';
|
|
@@ -20,11 +20,19 @@ declare type ReviveResponse = {
|
|
|
20
20
|
* @returns
|
|
21
21
|
*/
|
|
22
22
|
export declare function publishDurableStoreEntries(durableRecords: DurableStoreEntries<unknown> | undefined, publish: (key: string, record: unknown) => void, publishMetadata: (key: string, metadata: StoreMetadata) => void): ReviveResponse;
|
|
23
|
+
interface ReviveMetrics {
|
|
24
|
+
l2ReadCount: number;
|
|
25
|
+
l2ReadDuration: number;
|
|
26
|
+
}
|
|
27
|
+
interface ReviveResult<D, V> {
|
|
28
|
+
snapshot: Snapshot<D, V>;
|
|
29
|
+
metrics: ReviveMetrics;
|
|
30
|
+
}
|
|
23
31
|
/**
|
|
24
32
|
* This method returns a Promise to a snapshot that is revived from L2 cache. If
|
|
25
33
|
* L2 does not have the entries necessary to fulfill the snapshot then this method
|
|
26
34
|
* will refresh the snapshot from network, and then run the results from network
|
|
27
35
|
* through L2 ingestion, returning the subsequent revived snapshot.
|
|
28
36
|
*/
|
|
29
|
-
export declare function reviveSnapshot<D, V = unknown>(baseEnvironment: Environment, durableStore: DurableStore, unavailableSnapshot: UnAvailableSnapshot<D, V>, durableStoreErrorHandler: DurableStoreRejectionHandler, buildL1Snapshot: () => Snapshot<D, V
|
|
37
|
+
export declare function reviveSnapshot<D, V = unknown>(baseEnvironment: Environment, durableStore: DurableStore, unavailableSnapshot: UnAvailableSnapshot<D, V>, durableStoreErrorHandler: DurableStoreRejectionHandler, buildL1Snapshot: () => Snapshot<D, V>, reviveMetrics?: ReviveMetrics): Promise<ReviveResult<D, V>>;
|
|
30
38
|
export {};
|
|
@@ -130,19 +130,25 @@
|
|
|
130
130
|
*/
|
|
131
131
|
function reviveSnapshot(baseEnvironment, durableStore,
|
|
132
132
|
// TODO [W-10165787]: We should only allow Unfulfilled snapshot be passed in
|
|
133
|
-
unavailableSnapshot, durableStoreErrorHandler, buildL1Snapshot) {
|
|
133
|
+
unavailableSnapshot, durableStoreErrorHandler, buildL1Snapshot, reviveMetrics = { l2ReadCount: 0, l2ReadDuration: 0 }) {
|
|
134
134
|
const { recordId, select, seenRecords, state } = unavailableSnapshot;
|
|
135
135
|
// L2 can only revive Unfulfilled snapshots that have a selector since they have the
|
|
136
136
|
// info needed to revive (like missingLinks) and rebuild. Otherwise return L1 snapshot.
|
|
137
137
|
if (state !== 'Unfulfilled' || select === undefined) {
|
|
138
|
-
return Promise.resolve(
|
|
138
|
+
return Promise.resolve({
|
|
139
|
+
snapshot: unavailableSnapshot,
|
|
140
|
+
metrics: reviveMetrics,
|
|
141
|
+
});
|
|
139
142
|
}
|
|
140
143
|
// in case L1 store changes/deallocs a record while we are doing the async read
|
|
141
144
|
// we attempt to read all keys from L2 - so combine recordId with any seenRecords
|
|
142
145
|
const keysToReviveSet = assign({ [recordId]: true }, seenRecords);
|
|
143
146
|
const keysToRevive = keys(keysToReviveSet);
|
|
144
147
|
const canonicalKeys = keysToRevive.map((x) => baseEnvironment.storeGetCanonicalKey(x));
|
|
148
|
+
const start = Date.now();
|
|
149
|
+
reviveMetrics.l2ReadCount++;
|
|
145
150
|
return durableStore.getEntries(canonicalKeys, DefaultDurableSegment).then((durableRecords) => {
|
|
151
|
+
reviveMetrics.l2ReadDuration += Date.now() - start;
|
|
146
152
|
const { revivedKeys, hadUnexpectedShape } = publishDurableStoreEntries(durableRecords,
|
|
147
153
|
// TODO [W-10072584]: instead of implicitly using L1 we should take in
|
|
148
154
|
// publish and publishMetadata funcs, so callers can decide where to
|
|
@@ -151,18 +157,18 @@
|
|
|
151
157
|
// if the data coming back from DS had an unexpected shape then just
|
|
152
158
|
// return the L1 snapshot
|
|
153
159
|
if (hadUnexpectedShape === true) {
|
|
154
|
-
return unavailableSnapshot;
|
|
160
|
+
return { snapshot: unavailableSnapshot, metrics: reviveMetrics };
|
|
155
161
|
}
|
|
156
162
|
if (keys(revivedKeys).length === 0) {
|
|
157
163
|
// durable store doesn't have what we asked for so return L1 snapshot
|
|
158
|
-
return unavailableSnapshot;
|
|
164
|
+
return { snapshot: unavailableSnapshot, metrics: reviveMetrics };
|
|
159
165
|
}
|
|
160
166
|
// try building the snapshot from L1 now that we have revived the missingLinks
|
|
161
167
|
const snapshot = buildL1Snapshot();
|
|
162
168
|
// if snapshot is pending then some other in-flight refresh will broadcast
|
|
163
169
|
// later
|
|
164
170
|
if (snapshot.state === 'Pending') {
|
|
165
|
-
return snapshot;
|
|
171
|
+
return { snapshot, metrics: reviveMetrics };
|
|
166
172
|
}
|
|
167
173
|
if (snapshot.state === 'Unfulfilled') {
|
|
168
174
|
// have to check if the new snapshot has any additional seenRecords
|
|
@@ -178,15 +184,15 @@
|
|
|
178
184
|
for (let i = 0, len = newKeys.length; i < len; i++) {
|
|
179
185
|
const newSnapshotSeenKey = newKeys[i];
|
|
180
186
|
if (alreadyRequestedOrRevivedSet[newSnapshotSeenKey] !== true) {
|
|
181
|
-
return reviveSnapshot(baseEnvironment, durableStore, snapshot, durableStoreErrorHandler, buildL1Snapshot);
|
|
187
|
+
return reviveSnapshot(baseEnvironment, durableStore, snapshot, durableStoreErrorHandler, buildL1Snapshot, reviveMetrics);
|
|
182
188
|
}
|
|
183
189
|
}
|
|
184
190
|
}
|
|
185
|
-
return snapshot;
|
|
191
|
+
return { snapshot, metrics: reviveMetrics };
|
|
186
192
|
}, (error) => {
|
|
187
193
|
durableStoreErrorHandler(error);
|
|
188
194
|
// getEntries failed, return the L1 snapshot
|
|
189
|
-
return unavailableSnapshot;
|
|
195
|
+
return { snapshot: unavailableSnapshot, metrics: reviveMetrics };
|
|
190
196
|
});
|
|
191
197
|
}
|
|
192
198
|
|
|
@@ -328,6 +334,19 @@
|
|
|
328
334
|
return Promise.resolve();
|
|
329
335
|
}
|
|
330
336
|
|
|
337
|
+
const DurableEnvironmentEventDiscriminator = 'durable';
|
|
338
|
+
function isDurableEnvironmentEvent(event) {
|
|
339
|
+
return (event.type === 'environment' && event.environment === DurableEnvironmentEventDiscriminator);
|
|
340
|
+
}
|
|
341
|
+
function emitDurableEnvironmentAdapterEvent(eventData, observers) {
|
|
342
|
+
engine.emitAdapterEvent({
|
|
343
|
+
type: 'environment',
|
|
344
|
+
timestamp: Date.now(),
|
|
345
|
+
environment: DurableEnvironmentEventDiscriminator,
|
|
346
|
+
data: eventData,
|
|
347
|
+
}, observers);
|
|
348
|
+
}
|
|
349
|
+
|
|
331
350
|
const AdapterContextSegment = 'ADAPTER-CONTEXT';
|
|
332
351
|
const ADAPTER_CONTEXT_ID_SUFFIX = '__NAMED_CONTEXT';
|
|
333
352
|
function reviveOrCreateContext(adapterId, durableStore, durableStoreErrorHandler, onContextLoaded) {
|
|
@@ -539,7 +558,9 @@
|
|
|
539
558
|
reviveSnapshot(environment, durableStore, rebuilt, durableStoreErrorHandler, () =>
|
|
540
559
|
// reviveSnapshot will revive into L1, and since "records" is a reference
|
|
541
560
|
// (and not a copy) to the L1 records we can use it for rebuild
|
|
542
|
-
environment.rebuildSnapshot(snapshot, records, storeMetadataMap, redirects, () => { })).then(
|
|
561
|
+
environment.rebuildSnapshot(snapshot, records, storeMetadataMap, redirects, () => { })).then((result) => {
|
|
562
|
+
onAsyncRebuild(result.snapshot);
|
|
563
|
+
});
|
|
543
564
|
// synchronously return the base snapshot as Pending if not already
|
|
544
565
|
return snapshot.state === 'Pending'
|
|
545
566
|
? snapshot
|
|
@@ -596,7 +617,18 @@
|
|
|
596
617
|
// if the adapter attempted to do an L1 lookup and it was unfulfilled
|
|
597
618
|
// then we can attempt an L2 lookup
|
|
598
619
|
if (isUnfulfilledSnapshot(snapshot)) {
|
|
599
|
-
|
|
620
|
+
const start = Date.now();
|
|
621
|
+
emitDurableEnvironmentAdapterEvent({ type: 'l2-revive-start' }, adapterRequestContext.eventObservers);
|
|
622
|
+
const revivedSnapshot = reviveSnapshot(environment, durableStore, snapshot, durableStoreErrorHandler, () => storeLookup(snapshot.select, snapshot.refresh)).then((result) => {
|
|
623
|
+
emitDurableEnvironmentAdapterEvent({
|
|
624
|
+
type: 'l2-revive-end',
|
|
625
|
+
duration: Date.now() - start,
|
|
626
|
+
l2Trips: result.metrics.l2ReadCount,
|
|
627
|
+
l2Duration: result.metrics.l2ReadDuration,
|
|
628
|
+
}, adapterRequestContext.eventObservers);
|
|
629
|
+
return result.snapshot;
|
|
630
|
+
});
|
|
631
|
+
return revivedSnapshot;
|
|
600
632
|
}
|
|
601
633
|
// otherwise just return what buildCachedSnapshot gave us
|
|
602
634
|
return snapshot;
|
|
@@ -649,7 +681,9 @@
|
|
|
649
681
|
return snapshotFromMemoryIngest;
|
|
650
682
|
}
|
|
651
683
|
// if snapshot from staging store lookup is unfulfilled then do an L2 lookup
|
|
652
|
-
return reviveSnapshot(environment, durableStore, snapshotFromMemoryIngest, durableStoreErrorHandler, () => environment.storeLookup(snapshotFromMemoryIngest.select, environment.createSnapshot, snapshotFromMemoryIngest.refresh))
|
|
684
|
+
return reviveSnapshot(environment, durableStore, snapshotFromMemoryIngest, durableStoreErrorHandler, () => environment.storeLookup(snapshotFromMemoryIngest.select, environment.createSnapshot, snapshotFromMemoryIngest.refresh)).then((result) => {
|
|
685
|
+
return result.snapshot;
|
|
686
|
+
});
|
|
653
687
|
});
|
|
654
688
|
};
|
|
655
689
|
if (keys(keysToRevive).length === 0) {
|
|
@@ -716,6 +750,7 @@
|
|
|
716
750
|
}
|
|
717
751
|
|
|
718
752
|
exports.DefaultDurableSegment = DefaultDurableSegment;
|
|
753
|
+
exports.isDurableEnvironmentEvent = isDurableEnvironmentEvent;
|
|
719
754
|
exports.makeDurable = makeDurable;
|
|
720
755
|
exports.publishDurableStoreEntries = publishDurableStoreEntries;
|
|
721
756
|
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { EnvironmentAdapterEvent } from '@luvio/engine';
|
|
2
|
+
import type { LuvioAdapterEventObserver } from '@luvio/engine';
|
|
3
|
+
interface ReviveStartEvent {
|
|
4
|
+
type: 'l2-revive-start';
|
|
5
|
+
}
|
|
6
|
+
interface ReviveEndEvent {
|
|
7
|
+
type: 'l2-revive-end';
|
|
8
|
+
duration: number;
|
|
9
|
+
l2Trips: number;
|
|
10
|
+
l2Duration: number;
|
|
11
|
+
}
|
|
12
|
+
export declare type DurableEnvironmentAdapterEventData = ReviveStartEvent | ReviveEndEvent;
|
|
13
|
+
export declare function isDurableEnvironmentEvent(event: EnvironmentAdapterEvent<unknown>): event is EnvironmentAdapterEvent<DurableEnvironmentAdapterEventData>;
|
|
14
|
+
export declare function emitDurableEnvironmentAdapterEvent(eventData: DurableEnvironmentAdapterEventData, observers: LuvioAdapterEventObserver[] | undefined): void;
|
|
15
|
+
export {};
|
|
@@ -2,3 +2,4 @@ export { DurableStore, DurableStoreEntries, DurableStoreEntry, DurableStoreChang
|
|
|
2
2
|
export { DurableTTLOverride, DefaultDurableTTLOverride } from './DurableTTLStore';
|
|
3
3
|
export { makeDurable, DurableEnvironment } from './makeDurable';
|
|
4
4
|
export { publishDurableStoreEntries } from './makeDurable/revive';
|
|
5
|
+
export { isDurableEnvironmentEvent } from './events';
|
|
@@ -20,11 +20,19 @@ declare type ReviveResponse = {
|
|
|
20
20
|
* @returns
|
|
21
21
|
*/
|
|
22
22
|
export declare function publishDurableStoreEntries(durableRecords: DurableStoreEntries<unknown> | undefined, publish: (key: string, record: unknown) => void, publishMetadata: (key: string, metadata: StoreMetadata) => void): ReviveResponse;
|
|
23
|
+
interface ReviveMetrics {
|
|
24
|
+
l2ReadCount: number;
|
|
25
|
+
l2ReadDuration: number;
|
|
26
|
+
}
|
|
27
|
+
interface ReviveResult<D, V> {
|
|
28
|
+
snapshot: Snapshot<D, V>;
|
|
29
|
+
metrics: ReviveMetrics;
|
|
30
|
+
}
|
|
23
31
|
/**
|
|
24
32
|
* This method returns a Promise to a snapshot that is revived from L2 cache. If
|
|
25
33
|
* L2 does not have the entries necessary to fulfill the snapshot then this method
|
|
26
34
|
* will refresh the snapshot from network, and then run the results from network
|
|
27
35
|
* through L2 ingestion, returning the subsequent revived snapshot.
|
|
28
36
|
*/
|
|
29
|
-
export declare function reviveSnapshot<D, V = unknown>(baseEnvironment: Environment, durableStore: DurableStore, unavailableSnapshot: UnAvailableSnapshot<D, V>, durableStoreErrorHandler: DurableStoreRejectionHandler, buildL1Snapshot: () => Snapshot<D, V
|
|
37
|
+
export declare function reviveSnapshot<D, V = unknown>(baseEnvironment: Environment, durableStore: DurableStore, unavailableSnapshot: UnAvailableSnapshot<D, V>, durableStoreErrorHandler: DurableStoreRejectionHandler, buildL1Snapshot: () => Snapshot<D, V>, reviveMetrics?: ReviveMetrics): Promise<ReviveResult<D, V>>;
|
|
30
38
|
export {};
|
|
@@ -167,21 +167,28 @@
|
|
|
167
167
|
*/
|
|
168
168
|
function reviveSnapshot(baseEnvironment, durableStore,
|
|
169
169
|
// TODO [W-10165787]: We should only allow Unfulfilled snapshot be passed in
|
|
170
|
-
unavailableSnapshot, durableStoreErrorHandler, buildL1Snapshot) {
|
|
170
|
+
unavailableSnapshot, durableStoreErrorHandler, buildL1Snapshot, reviveMetrics) {
|
|
171
171
|
var _a;
|
|
172
|
+
if (reviveMetrics === void 0) { reviveMetrics = { l2ReadCount: 0, l2ReadDuration: 0 }; }
|
|
172
173
|
var recordId = unavailableSnapshot.recordId, select = unavailableSnapshot.select, seenRecords = unavailableSnapshot.seenRecords, state = unavailableSnapshot.state;
|
|
173
174
|
// L2 can only revive Unfulfilled snapshots that have a selector since they have the
|
|
174
175
|
// info needed to revive (like missingLinks) and rebuild. Otherwise return L1 snapshot.
|
|
175
176
|
if (state !== 'Unfulfilled' || select === undefined) {
|
|
176
|
-
return Promise.resolve(
|
|
177
|
+
return Promise.resolve({
|
|
178
|
+
snapshot: unavailableSnapshot,
|
|
179
|
+
metrics: reviveMetrics,
|
|
180
|
+
});
|
|
177
181
|
}
|
|
178
182
|
// in case L1 store changes/deallocs a record while we are doing the async read
|
|
179
183
|
// we attempt to read all keys from L2 - so combine recordId with any seenRecords
|
|
180
184
|
var keysToReviveSet = assign((_a = {}, _a[recordId] = true, _a), seenRecords);
|
|
181
185
|
var keysToRevive = keys(keysToReviveSet);
|
|
182
186
|
var canonicalKeys = keysToRevive.map(function (x) { return baseEnvironment.storeGetCanonicalKey(x); });
|
|
187
|
+
var start = Date.now();
|
|
188
|
+
reviveMetrics.l2ReadCount++;
|
|
183
189
|
return durableStore.getEntries(canonicalKeys, DefaultDurableSegment).then(function (durableRecords) {
|
|
184
190
|
var _a;
|
|
191
|
+
reviveMetrics.l2ReadDuration += Date.now() - start;
|
|
185
192
|
var _b = publishDurableStoreEntries(durableRecords,
|
|
186
193
|
// TODO [W-10072584]: instead of implicitly using L1 we should take in
|
|
187
194
|
// publish and publishMetadata funcs, so callers can decide where to
|
|
@@ -190,18 +197,18 @@
|
|
|
190
197
|
// if the data coming back from DS had an unexpected shape then just
|
|
191
198
|
// return the L1 snapshot
|
|
192
199
|
if (hadUnexpectedShape === true) {
|
|
193
|
-
return unavailableSnapshot;
|
|
200
|
+
return { snapshot: unavailableSnapshot, metrics: reviveMetrics };
|
|
194
201
|
}
|
|
195
202
|
if (keys(revivedKeys).length === 0) {
|
|
196
203
|
// durable store doesn't have what we asked for so return L1 snapshot
|
|
197
|
-
return unavailableSnapshot;
|
|
204
|
+
return { snapshot: unavailableSnapshot, metrics: reviveMetrics };
|
|
198
205
|
}
|
|
199
206
|
// try building the snapshot from L1 now that we have revived the missingLinks
|
|
200
207
|
var snapshot = buildL1Snapshot();
|
|
201
208
|
// if snapshot is pending then some other in-flight refresh will broadcast
|
|
202
209
|
// later
|
|
203
210
|
if (snapshot.state === 'Pending') {
|
|
204
|
-
return snapshot;
|
|
211
|
+
return { snapshot: snapshot, metrics: reviveMetrics };
|
|
205
212
|
}
|
|
206
213
|
if (snapshot.state === 'Unfulfilled') {
|
|
207
214
|
// have to check if the new snapshot has any additional seenRecords
|
|
@@ -217,15 +224,15 @@
|
|
|
217
224
|
for (var i = 0, len = newKeys.length; i < len; i++) {
|
|
218
225
|
var newSnapshotSeenKey = newKeys[i];
|
|
219
226
|
if (alreadyRequestedOrRevivedSet[newSnapshotSeenKey] !== true) {
|
|
220
|
-
return reviveSnapshot(baseEnvironment, durableStore, snapshot, durableStoreErrorHandler, buildL1Snapshot);
|
|
227
|
+
return reviveSnapshot(baseEnvironment, durableStore, snapshot, durableStoreErrorHandler, buildL1Snapshot, reviveMetrics);
|
|
221
228
|
}
|
|
222
229
|
}
|
|
223
230
|
}
|
|
224
|
-
return snapshot;
|
|
231
|
+
return { snapshot: snapshot, metrics: reviveMetrics };
|
|
225
232
|
}, function (error) {
|
|
226
233
|
durableStoreErrorHandler(error);
|
|
227
234
|
// getEntries failed, return the L1 snapshot
|
|
228
|
-
return unavailableSnapshot;
|
|
235
|
+
return { snapshot: unavailableSnapshot, metrics: reviveMetrics };
|
|
229
236
|
});
|
|
230
237
|
}
|
|
231
238
|
|
|
@@ -370,6 +377,19 @@
|
|
|
370
377
|
return Promise.resolve();
|
|
371
378
|
}
|
|
372
379
|
|
|
380
|
+
var DurableEnvironmentEventDiscriminator = 'durable';
|
|
381
|
+
function isDurableEnvironmentEvent(event) {
|
|
382
|
+
return (event.type === 'environment' && event.environment === DurableEnvironmentEventDiscriminator);
|
|
383
|
+
}
|
|
384
|
+
function emitDurableEnvironmentAdapterEvent(eventData, observers) {
|
|
385
|
+
engine.emitAdapterEvent({
|
|
386
|
+
type: 'environment',
|
|
387
|
+
timestamp: Date.now(),
|
|
388
|
+
environment: DurableEnvironmentEventDiscriminator,
|
|
389
|
+
data: eventData,
|
|
390
|
+
}, observers);
|
|
391
|
+
}
|
|
392
|
+
|
|
373
393
|
var AdapterContextSegment = 'ADAPTER-CONTEXT';
|
|
374
394
|
var ADAPTER_CONTEXT_ID_SUFFIX = '__NAMED_CONTEXT';
|
|
375
395
|
function reviveOrCreateContext(adapterId, durableStore, durableStoreErrorHandler, onContextLoaded) {
|
|
@@ -584,7 +604,9 @@
|
|
|
584
604
|
// reviveSnapshot will revive into L1, and since "records" is a reference
|
|
585
605
|
// (and not a copy) to the L1 records we can use it for rebuild
|
|
586
606
|
return environment.rebuildSnapshot(snapshot, records, storeMetadataMap, redirects, function () { });
|
|
587
|
-
}).then(
|
|
607
|
+
}).then(function (result) {
|
|
608
|
+
onAsyncRebuild(result.snapshot);
|
|
609
|
+
});
|
|
588
610
|
// synchronously return the base snapshot as Pending if not already
|
|
589
611
|
return snapshot.state === 'Pending'
|
|
590
612
|
? snapshot
|
|
@@ -638,7 +660,18 @@
|
|
|
638
660
|
// if the adapter attempted to do an L1 lookup and it was unfulfilled
|
|
639
661
|
// then we can attempt an L2 lookup
|
|
640
662
|
if (isUnfulfilledSnapshot(snapshot)) {
|
|
641
|
-
|
|
663
|
+
var start_1 = Date.now();
|
|
664
|
+
emitDurableEnvironmentAdapterEvent({ type: 'l2-revive-start' }, adapterRequestContext.eventObservers);
|
|
665
|
+
var revivedSnapshot = reviveSnapshot(environment, durableStore, snapshot, durableStoreErrorHandler, function () { return storeLookup(snapshot.select, snapshot.refresh); }).then(function (result) {
|
|
666
|
+
emitDurableEnvironmentAdapterEvent({
|
|
667
|
+
type: 'l2-revive-end',
|
|
668
|
+
duration: Date.now() - start_1,
|
|
669
|
+
l2Trips: result.metrics.l2ReadCount,
|
|
670
|
+
l2Duration: result.metrics.l2ReadDuration,
|
|
671
|
+
}, adapterRequestContext.eventObservers);
|
|
672
|
+
return result.snapshot;
|
|
673
|
+
});
|
|
674
|
+
return revivedSnapshot;
|
|
642
675
|
}
|
|
643
676
|
// otherwise just return what buildCachedSnapshot gave us
|
|
644
677
|
return snapshot;
|
|
@@ -692,6 +725,8 @@
|
|
|
692
725
|
// if snapshot from staging store lookup is unfulfilled then do an L2 lookup
|
|
693
726
|
return reviveSnapshot(environment, durableStore, snapshotFromMemoryIngest, durableStoreErrorHandler, function () {
|
|
694
727
|
return environment.storeLookup(snapshotFromMemoryIngest.select, environment.createSnapshot, snapshotFromMemoryIngest.refresh);
|
|
728
|
+
}).then(function (result) {
|
|
729
|
+
return result.snapshot;
|
|
695
730
|
});
|
|
696
731
|
});
|
|
697
732
|
};
|
|
@@ -759,6 +794,7 @@
|
|
|
759
794
|
}
|
|
760
795
|
|
|
761
796
|
exports.DefaultDurableSegment = DefaultDurableSegment;
|
|
797
|
+
exports.isDurableEnvironmentEvent = isDurableEnvironmentEvent;
|
|
762
798
|
exports.makeDurable = makeDurable;
|
|
763
799
|
exports.publishDurableStoreEntries = publishDurableStoreEntries;
|
|
764
800
|
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { EnvironmentAdapterEvent } from '@luvio/engine';
|
|
2
|
+
import type { LuvioAdapterEventObserver } from '@luvio/engine';
|
|
3
|
+
interface ReviveStartEvent {
|
|
4
|
+
type: 'l2-revive-start';
|
|
5
|
+
}
|
|
6
|
+
interface ReviveEndEvent {
|
|
7
|
+
type: 'l2-revive-end';
|
|
8
|
+
duration: number;
|
|
9
|
+
l2Trips: number;
|
|
10
|
+
l2Duration: number;
|
|
11
|
+
}
|
|
12
|
+
export declare type DurableEnvironmentAdapterEventData = ReviveStartEvent | ReviveEndEvent;
|
|
13
|
+
export declare function isDurableEnvironmentEvent(event: EnvironmentAdapterEvent<unknown>): event is EnvironmentAdapterEvent<DurableEnvironmentAdapterEventData>;
|
|
14
|
+
export declare function emitDurableEnvironmentAdapterEvent(eventData: DurableEnvironmentAdapterEventData, observers: LuvioAdapterEventObserver[] | undefined): void;
|
|
15
|
+
export {};
|
package/dist/umd/es5/main.d.ts
CHANGED
|
@@ -2,3 +2,4 @@ export { DurableStore, DurableStoreEntries, DurableStoreEntry, DurableStoreChang
|
|
|
2
2
|
export { DurableTTLOverride, DefaultDurableTTLOverride } from './DurableTTLStore';
|
|
3
3
|
export { makeDurable, DurableEnvironment } from './makeDurable';
|
|
4
4
|
export { publishDurableStoreEntries } from './makeDurable/revive';
|
|
5
|
+
export { isDurableEnvironmentEvent } from './events';
|
|
@@ -20,11 +20,19 @@ declare type ReviveResponse = {
|
|
|
20
20
|
* @returns
|
|
21
21
|
*/
|
|
22
22
|
export declare function publishDurableStoreEntries(durableRecords: DurableStoreEntries<unknown> | undefined, publish: (key: string, record: unknown) => void, publishMetadata: (key: string, metadata: StoreMetadata) => void): ReviveResponse;
|
|
23
|
+
interface ReviveMetrics {
|
|
24
|
+
l2ReadCount: number;
|
|
25
|
+
l2ReadDuration: number;
|
|
26
|
+
}
|
|
27
|
+
interface ReviveResult<D, V> {
|
|
28
|
+
snapshot: Snapshot<D, V>;
|
|
29
|
+
metrics: ReviveMetrics;
|
|
30
|
+
}
|
|
23
31
|
/**
|
|
24
32
|
* This method returns a Promise to a snapshot that is revived from L2 cache. If
|
|
25
33
|
* L2 does not have the entries necessary to fulfill the snapshot then this method
|
|
26
34
|
* will refresh the snapshot from network, and then run the results from network
|
|
27
35
|
* through L2 ingestion, returning the subsequent revived snapshot.
|
|
28
36
|
*/
|
|
29
|
-
export declare function reviveSnapshot<D, V = unknown>(baseEnvironment: Environment, durableStore: DurableStore, unavailableSnapshot: UnAvailableSnapshot<D, V>, durableStoreErrorHandler: DurableStoreRejectionHandler, buildL1Snapshot: () => Snapshot<D, V
|
|
37
|
+
export declare function reviveSnapshot<D, V = unknown>(baseEnvironment: Environment, durableStore: DurableStore, unavailableSnapshot: UnAvailableSnapshot<D, V>, durableStoreErrorHandler: DurableStoreRejectionHandler, buildL1Snapshot: () => Snapshot<D, V>, reviveMetrics?: ReviveMetrics): Promise<ReviveResult<D, V>>;
|
|
30
38
|
export {};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@luvio/environments",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.98.0",
|
|
4
4
|
"description": "Luvio Environments",
|
|
5
5
|
"main": "dist/umd/es2018/environments.js",
|
|
6
6
|
"module": "dist/es/es2018/environments.js",
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
"dist/"
|
|
28
28
|
],
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"@luvio/engine": "0.
|
|
30
|
+
"@luvio/engine": "0.98.0"
|
|
31
31
|
},
|
|
32
32
|
"bundlesize": [
|
|
33
33
|
{
|