@luvio/environments 0.47.7 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/es2018/DurableStore.d.ts +16 -0
- package/dist/es/es2018/environments.js +50 -19
- package/dist/umd/es2018/DurableStore.d.ts +16 -0
- package/dist/umd/es2018/environments.js +50 -19
- package/dist/umd/es5/DurableStore.d.ts +16 -0
- package/dist/umd/es5/environments.js +47 -19
- package/package.json +2 -2
|
@@ -1,8 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Contains store entry data along with any metadata for that entry that needs
|
|
3
3
|
* to be persisted to durable storage
|
|
4
|
+
*
|
|
5
|
+
* IMPORTANT: if you update this interface, you must also create a
|
|
6
|
+
* deprecated interface, and check for durable store entries of this
|
|
7
|
+
* type in isDeprecatedDurableStoreEntry so that revive discards
|
|
8
|
+
* invalid entries.
|
|
4
9
|
*/
|
|
5
10
|
export interface DurableStoreEntry<T = unknown> {
|
|
11
|
+
data: T;
|
|
12
|
+
metadata?: {
|
|
13
|
+
ingestionTimestamp: number;
|
|
14
|
+
expirationTimestamp: number;
|
|
15
|
+
staleTimestamp?: number;
|
|
16
|
+
namespace: string;
|
|
17
|
+
representationName: string;
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
export declare function isDeprecatedDurableStoreEntry(durableRecord: unknown): boolean;
|
|
21
|
+
export interface DeprecatedDurableStoreEntry1<T = unknown> {
|
|
6
22
|
data: T;
|
|
7
23
|
expiration?: {
|
|
8
24
|
fresh: number;
|
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
function isDeprecatedDurableStoreEntry(durableRecord) {
|
|
2
|
+
if (durableRecord.expiration !== undefined) {
|
|
3
|
+
return true;
|
|
4
|
+
}
|
|
5
|
+
// Add more deprecated shape checks here
|
|
6
|
+
return false;
|
|
7
|
+
}
|
|
1
8
|
var DurableStoreOperationType;
|
|
2
9
|
(function (DurableStoreOperationType) {
|
|
3
10
|
DurableStoreOperationType["SetEntries"] = "setEntries";
|
|
@@ -62,7 +69,7 @@ function buildPendingWriter() {
|
|
|
62
69
|
flushPendingWritesToDurableStore: function (environment, durableStore, durableStoreErrorHandler) {
|
|
63
70
|
const durableRecords = create({});
|
|
64
71
|
const records = environment.getStoreRecords();
|
|
65
|
-
const
|
|
72
|
+
const storeMetadataMap = environment.getStoreMetadataMap();
|
|
66
73
|
const keys$1 = keys(pendingDurableWrites);
|
|
67
74
|
for (let i = 0, len = keys$1.length; i < len; i += 1) {
|
|
68
75
|
const key = keys$1[i];
|
|
@@ -71,12 +78,12 @@ function buildPendingWriter() {
|
|
|
71
78
|
if (record === undefined) {
|
|
72
79
|
continue;
|
|
73
80
|
}
|
|
74
|
-
const
|
|
81
|
+
const recordMetadata = storeMetadataMap[key];
|
|
75
82
|
durableRecords[key] = {
|
|
76
83
|
// make copy of record b/c evict is coming
|
|
77
84
|
// NOTE: only spread copy if an array or object
|
|
78
85
|
data: copy(record),
|
|
79
|
-
|
|
86
|
+
metadata: recordMetadata,
|
|
80
87
|
};
|
|
81
88
|
// remove record from L1 cache while we are on the synchronous path
|
|
82
89
|
// because we do not want some other code attempting to use the
|
|
@@ -250,8 +257,8 @@ function buildDurableStoreAwareRefresh(refresh, snapshot, environment, pendingWr
|
|
|
250
257
|
function isStoreEntryError(storeRecord) {
|
|
251
258
|
return storeRecord.__type === 'error';
|
|
252
259
|
}
|
|
253
|
-
function isStoreEntryExpiredAndError(storeRecord,
|
|
254
|
-
return isStoreEntryError(storeRecord) &&
|
|
260
|
+
function isStoreEntryExpiredAndError(storeRecord, expirationTimestamp, now) {
|
|
261
|
+
return isStoreEntryError(storeRecord) && expirationTimestamp < now;
|
|
255
262
|
}
|
|
256
263
|
function reviveDurableEntriesToStore(durableRecords, environment, pendingWriter) {
|
|
257
264
|
const revivedKeys = create(null);
|
|
@@ -268,37 +275,42 @@ function reviveDurableEntriesToStore(durableRecords, environment, pendingWriter)
|
|
|
268
275
|
for (let i = 0, len = durableKeys.length; i < len; i += 1) {
|
|
269
276
|
const key = durableKeys[i];
|
|
270
277
|
const durableRecord = durableRecords[key];
|
|
271
|
-
|
|
278
|
+
if (isDeprecatedDurableStoreEntry(durableRecord)) {
|
|
279
|
+
// had the old shape, skip reviving this entry.
|
|
280
|
+
hadUnexpectedShape = true;
|
|
281
|
+
continue;
|
|
282
|
+
}
|
|
283
|
+
const { metadata, data } = durableRecord;
|
|
272
284
|
if (data === undefined) {
|
|
273
285
|
// if unexpected data skip reviving
|
|
274
286
|
hadUnexpectedShape = true;
|
|
275
287
|
continue;
|
|
276
288
|
}
|
|
277
|
-
if (
|
|
278
|
-
const {
|
|
279
|
-
if (
|
|
289
|
+
if (metadata !== undefined) {
|
|
290
|
+
const { expirationTimestamp, staleTimestamp } = metadata;
|
|
291
|
+
if (expirationTimestamp === undefined || staleTimestamp === undefined) {
|
|
280
292
|
// if unexpected expiration data skip reviving
|
|
281
293
|
hadUnexpectedShape = true;
|
|
282
294
|
continue;
|
|
283
295
|
}
|
|
284
296
|
// if past stale TTL then don't revive
|
|
285
|
-
if (
|
|
297
|
+
if (staleTimestamp < now) {
|
|
286
298
|
continue;
|
|
287
299
|
}
|
|
288
300
|
// We don't want to revive a cached value if it's an error and it's
|
|
289
|
-
//
|
|
301
|
+
// expirationTimestamp TTL is expired, otherwise we would never hit the network
|
|
290
302
|
// during resolveSnapshot or rebuildSnapshot in the makeOffline
|
|
291
303
|
// environment because the way the Reader works.
|
|
292
304
|
// If a StoreEntry is an error, the Reader will return UnfulfilledSnapshot
|
|
293
305
|
// if stale TTL is expired. But it will return ErrorSnapshot (instead of
|
|
294
|
-
// StaleSnapshot) if
|
|
306
|
+
// StaleSnapshot) if expirationTimestamp TTL is expired (but stale TTL isn't).
|
|
295
307
|
// In makeOffline environment the stale TTL is maxed out so Reader
|
|
296
308
|
// will always return ErrorSnapshot.
|
|
297
|
-
if (isStoreEntryExpiredAndError(data,
|
|
309
|
+
if (isStoreEntryExpiredAndError(data, expirationTimestamp, now)) {
|
|
298
310
|
continue;
|
|
299
311
|
}
|
|
300
|
-
// call base
|
|
301
|
-
environment.
|
|
312
|
+
// call base publishStoreMetadata so we don't add the key to pendingWriters
|
|
313
|
+
environment.publishStoreMetadata(key, metadata);
|
|
302
314
|
}
|
|
303
315
|
if (isStoreEntryError(data)) {
|
|
304
316
|
// freeze errors on way into L1
|
|
@@ -371,8 +383,11 @@ function reviveSnapshot(environment, durableStore, pendingWriter, unavailableSna
|
|
|
371
383
|
// durable store doesn't have what we asked for so refresh snapshot
|
|
372
384
|
return refreshSnapshot();
|
|
373
385
|
}
|
|
374
|
-
// attempt to
|
|
375
|
-
|
|
386
|
+
// attempt to lookup (or rebuild if baseSnapshot is provided) the snapshot
|
|
387
|
+
// now that we have revived the missingLinks
|
|
388
|
+
const snapshot = baseSnapshot === undefined
|
|
389
|
+
? environment.storeLookup(unavailableSnapshot.select, environment.createSnapshot, unavailableSnapshot.refresh)
|
|
390
|
+
: environment.rebuildSnapshot(baseSnapshot, environment.getStoreRecords(), environment.getStoreMetadataMap(), environment.getStoreRedirectKeys(), () => { });
|
|
376
391
|
// if snapshot is pending then some other in-flight refresh will broadcast
|
|
377
392
|
// later
|
|
378
393
|
if (snapshot.state === 'Pending') {
|
|
@@ -515,6 +530,12 @@ function makeDurable(environment, { durableStore, instrumentation, pendingWriter
|
|
|
515
530
|
pendingWriter.addPendingWrite(recordId);
|
|
516
531
|
environment.storeSetExpiration(recordId, expiration, staleExpiration);
|
|
517
532
|
};
|
|
533
|
+
const publishStoreMetadata = function (recordId, storeMetadata) {
|
|
534
|
+
validateNotDisposed();
|
|
535
|
+
// mark this as pending write since we need to re-write with updated metadata values
|
|
536
|
+
pendingWriter.addPendingWrite(recordId);
|
|
537
|
+
environment.publishStoreMetadata(recordId, storeMetadata);
|
|
538
|
+
};
|
|
518
539
|
const storeBroadcast = function (_rebuildSnapshot, _snapshotDataAvailable) {
|
|
519
540
|
// don't await the DS write - DS implementation will take care of R/W
|
|
520
541
|
// synchronization
|
|
@@ -549,10 +570,10 @@ function makeDurable(environment, { durableStore, instrumentation, pendingWriter
|
|
|
549
570
|
}
|
|
550
571
|
return reviveSnapshot(environment, durableStore, pendingWriter, snapshot, refresh, durableStoreErrorHandler);
|
|
551
572
|
};
|
|
552
|
-
const rebuildSnapshot = function (snapshot, records,
|
|
573
|
+
const rebuildSnapshot = function (snapshot, records, storeMetadataMap, redirects, onAsyncRebuild) {
|
|
553
574
|
validateNotDisposed();
|
|
554
575
|
// try rebuilding from memory
|
|
555
|
-
const rebuilt = environment.rebuildSnapshot(snapshot, records,
|
|
576
|
+
const rebuilt = environment.rebuildSnapshot(snapshot, records, storeMetadataMap, redirects, onAsyncRebuild);
|
|
556
577
|
// only try reviving from durable store if snapshot is unfulfilled
|
|
557
578
|
if (rebuilt.state !== 'Unfulfilled') {
|
|
558
579
|
return rebuilt;
|
|
@@ -613,6 +634,7 @@ function makeDurable(environment, { durableStore, instrumentation, pendingWriter
|
|
|
613
634
|
};
|
|
614
635
|
return create(environment, {
|
|
615
636
|
storeSetExpiration: { value: storeSetExpiration },
|
|
637
|
+
publishStoreMetadata: { value: publishStoreMetadata },
|
|
616
638
|
storeBroadcast: { value: storeBroadcast },
|
|
617
639
|
storeReset: { value: storeReset },
|
|
618
640
|
resolveSnapshot: { value: resolveSnapshot },
|
|
@@ -659,6 +681,12 @@ function makeOffline(environment) {
|
|
|
659
681
|
const setExpiration = function (recordId, expiration, _staleExpiration) {
|
|
660
682
|
environment.storeSetExpiration(recordId, expiration, Number.MAX_SAFE_INTEGER);
|
|
661
683
|
};
|
|
684
|
+
const publishStoreMetadata = function (recordId, storeMetadata) {
|
|
685
|
+
environment.publishStoreMetadata(recordId, {
|
|
686
|
+
...storeMetadata,
|
|
687
|
+
staleTimestamp: Number.MAX_SAFE_INTEGER,
|
|
688
|
+
});
|
|
689
|
+
};
|
|
662
690
|
return create(environment, {
|
|
663
691
|
storeLookup: { value: storeLookup },
|
|
664
692
|
snapshotAvailable: {
|
|
@@ -667,6 +695,9 @@ function makeOffline(environment) {
|
|
|
667
695
|
storeSetExpiration: {
|
|
668
696
|
value: setExpiration,
|
|
669
697
|
},
|
|
698
|
+
publishStoreMetadata: {
|
|
699
|
+
value: publishStoreMetadata,
|
|
700
|
+
},
|
|
670
701
|
});
|
|
671
702
|
}
|
|
672
703
|
|
|
@@ -1,8 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Contains store entry data along with any metadata for that entry that needs
|
|
3
3
|
* to be persisted to durable storage
|
|
4
|
+
*
|
|
5
|
+
* IMPORTANT: if you update this interface, you must also create a
|
|
6
|
+
* deprecated interface, and check for durable store entries of this
|
|
7
|
+
* type in isDeprecatedDurableStoreEntry so that revive discards
|
|
8
|
+
* invalid entries.
|
|
4
9
|
*/
|
|
5
10
|
export interface DurableStoreEntry<T = unknown> {
|
|
11
|
+
data: T;
|
|
12
|
+
metadata?: {
|
|
13
|
+
ingestionTimestamp: number;
|
|
14
|
+
expirationTimestamp: number;
|
|
15
|
+
staleTimestamp?: number;
|
|
16
|
+
namespace: string;
|
|
17
|
+
representationName: string;
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
export declare function isDeprecatedDurableStoreEntry(durableRecord: unknown): boolean;
|
|
21
|
+
export interface DeprecatedDurableStoreEntry1<T = unknown> {
|
|
6
22
|
data: T;
|
|
7
23
|
expiration?: {
|
|
8
24
|
fresh: number;
|
|
@@ -4,6 +4,13 @@
|
|
|
4
4
|
(global = global || self, factory(global.luvioEnvironments = {}));
|
|
5
5
|
}(this, (function (exports) { 'use strict';
|
|
6
6
|
|
|
7
|
+
function isDeprecatedDurableStoreEntry(durableRecord) {
|
|
8
|
+
if (durableRecord.expiration !== undefined) {
|
|
9
|
+
return true;
|
|
10
|
+
}
|
|
11
|
+
// Add more deprecated shape checks here
|
|
12
|
+
return false;
|
|
13
|
+
}
|
|
7
14
|
(function (DurableStoreOperationType) {
|
|
8
15
|
DurableStoreOperationType["SetEntries"] = "setEntries";
|
|
9
16
|
DurableStoreOperationType["EvictEntries"] = "evictEntries";
|
|
@@ -67,7 +74,7 @@
|
|
|
67
74
|
flushPendingWritesToDurableStore: function (environment, durableStore, durableStoreErrorHandler) {
|
|
68
75
|
const durableRecords = create({});
|
|
69
76
|
const records = environment.getStoreRecords();
|
|
70
|
-
const
|
|
77
|
+
const storeMetadataMap = environment.getStoreMetadataMap();
|
|
71
78
|
const keys$1 = keys(pendingDurableWrites);
|
|
72
79
|
for (let i = 0, len = keys$1.length; i < len; i += 1) {
|
|
73
80
|
const key = keys$1[i];
|
|
@@ -76,12 +83,12 @@
|
|
|
76
83
|
if (record === undefined) {
|
|
77
84
|
continue;
|
|
78
85
|
}
|
|
79
|
-
const
|
|
86
|
+
const recordMetadata = storeMetadataMap[key];
|
|
80
87
|
durableRecords[key] = {
|
|
81
88
|
// make copy of record b/c evict is coming
|
|
82
89
|
// NOTE: only spread copy if an array or object
|
|
83
90
|
data: copy(record),
|
|
84
|
-
|
|
91
|
+
metadata: recordMetadata,
|
|
85
92
|
};
|
|
86
93
|
// remove record from L1 cache while we are on the synchronous path
|
|
87
94
|
// because we do not want some other code attempting to use the
|
|
@@ -255,8 +262,8 @@
|
|
|
255
262
|
function isStoreEntryError(storeRecord) {
|
|
256
263
|
return storeRecord.__type === 'error';
|
|
257
264
|
}
|
|
258
|
-
function isStoreEntryExpiredAndError(storeRecord,
|
|
259
|
-
return isStoreEntryError(storeRecord) &&
|
|
265
|
+
function isStoreEntryExpiredAndError(storeRecord, expirationTimestamp, now) {
|
|
266
|
+
return isStoreEntryError(storeRecord) && expirationTimestamp < now;
|
|
260
267
|
}
|
|
261
268
|
function reviveDurableEntriesToStore(durableRecords, environment, pendingWriter) {
|
|
262
269
|
const revivedKeys = create(null);
|
|
@@ -273,37 +280,42 @@
|
|
|
273
280
|
for (let i = 0, len = durableKeys.length; i < len; i += 1) {
|
|
274
281
|
const key = durableKeys[i];
|
|
275
282
|
const durableRecord = durableRecords[key];
|
|
276
|
-
|
|
283
|
+
if (isDeprecatedDurableStoreEntry(durableRecord)) {
|
|
284
|
+
// had the old shape, skip reviving this entry.
|
|
285
|
+
hadUnexpectedShape = true;
|
|
286
|
+
continue;
|
|
287
|
+
}
|
|
288
|
+
const { metadata, data } = durableRecord;
|
|
277
289
|
if (data === undefined) {
|
|
278
290
|
// if unexpected data skip reviving
|
|
279
291
|
hadUnexpectedShape = true;
|
|
280
292
|
continue;
|
|
281
293
|
}
|
|
282
|
-
if (
|
|
283
|
-
const {
|
|
284
|
-
if (
|
|
294
|
+
if (metadata !== undefined) {
|
|
295
|
+
const { expirationTimestamp, staleTimestamp } = metadata;
|
|
296
|
+
if (expirationTimestamp === undefined || staleTimestamp === undefined) {
|
|
285
297
|
// if unexpected expiration data skip reviving
|
|
286
298
|
hadUnexpectedShape = true;
|
|
287
299
|
continue;
|
|
288
300
|
}
|
|
289
301
|
// if past stale TTL then don't revive
|
|
290
|
-
if (
|
|
302
|
+
if (staleTimestamp < now) {
|
|
291
303
|
continue;
|
|
292
304
|
}
|
|
293
305
|
// We don't want to revive a cached value if it's an error and it's
|
|
294
|
-
//
|
|
306
|
+
// expirationTimestamp TTL is expired, otherwise we would never hit the network
|
|
295
307
|
// during resolveSnapshot or rebuildSnapshot in the makeOffline
|
|
296
308
|
// environment because the way the Reader works.
|
|
297
309
|
// If a StoreEntry is an error, the Reader will return UnfulfilledSnapshot
|
|
298
310
|
// if stale TTL is expired. But it will return ErrorSnapshot (instead of
|
|
299
|
-
// StaleSnapshot) if
|
|
311
|
+
// StaleSnapshot) if expirationTimestamp TTL is expired (but stale TTL isn't).
|
|
300
312
|
// In makeOffline environment the stale TTL is maxed out so Reader
|
|
301
313
|
// will always return ErrorSnapshot.
|
|
302
|
-
if (isStoreEntryExpiredAndError(data,
|
|
314
|
+
if (isStoreEntryExpiredAndError(data, expirationTimestamp, now)) {
|
|
303
315
|
continue;
|
|
304
316
|
}
|
|
305
|
-
// call base
|
|
306
|
-
environment.
|
|
317
|
+
// call base publishStoreMetadata so we don't add the key to pendingWriters
|
|
318
|
+
environment.publishStoreMetadata(key, metadata);
|
|
307
319
|
}
|
|
308
320
|
if (isStoreEntryError(data)) {
|
|
309
321
|
// freeze errors on way into L1
|
|
@@ -376,8 +388,11 @@
|
|
|
376
388
|
// durable store doesn't have what we asked for so refresh snapshot
|
|
377
389
|
return refreshSnapshot();
|
|
378
390
|
}
|
|
379
|
-
// attempt to
|
|
380
|
-
|
|
391
|
+
// attempt to lookup (or rebuild if baseSnapshot is provided) the snapshot
|
|
392
|
+
// now that we have revived the missingLinks
|
|
393
|
+
const snapshot = baseSnapshot === undefined
|
|
394
|
+
? environment.storeLookup(unavailableSnapshot.select, environment.createSnapshot, unavailableSnapshot.refresh)
|
|
395
|
+
: environment.rebuildSnapshot(baseSnapshot, environment.getStoreRecords(), environment.getStoreMetadataMap(), environment.getStoreRedirectKeys(), () => { });
|
|
381
396
|
// if snapshot is pending then some other in-flight refresh will broadcast
|
|
382
397
|
// later
|
|
383
398
|
if (snapshot.state === 'Pending') {
|
|
@@ -520,6 +535,12 @@
|
|
|
520
535
|
pendingWriter.addPendingWrite(recordId);
|
|
521
536
|
environment.storeSetExpiration(recordId, expiration, staleExpiration);
|
|
522
537
|
};
|
|
538
|
+
const publishStoreMetadata = function (recordId, storeMetadata) {
|
|
539
|
+
validateNotDisposed();
|
|
540
|
+
// mark this as pending write since we need to re-write with updated metadata values
|
|
541
|
+
pendingWriter.addPendingWrite(recordId);
|
|
542
|
+
environment.publishStoreMetadata(recordId, storeMetadata);
|
|
543
|
+
};
|
|
523
544
|
const storeBroadcast = function (_rebuildSnapshot, _snapshotDataAvailable) {
|
|
524
545
|
// don't await the DS write - DS implementation will take care of R/W
|
|
525
546
|
// synchronization
|
|
@@ -554,10 +575,10 @@
|
|
|
554
575
|
}
|
|
555
576
|
return reviveSnapshot(environment, durableStore, pendingWriter, snapshot, refresh, durableStoreErrorHandler);
|
|
556
577
|
};
|
|
557
|
-
const rebuildSnapshot = function (snapshot, records,
|
|
578
|
+
const rebuildSnapshot = function (snapshot, records, storeMetadataMap, redirects, onAsyncRebuild) {
|
|
558
579
|
validateNotDisposed();
|
|
559
580
|
// try rebuilding from memory
|
|
560
|
-
const rebuilt = environment.rebuildSnapshot(snapshot, records,
|
|
581
|
+
const rebuilt = environment.rebuildSnapshot(snapshot, records, storeMetadataMap, redirects, onAsyncRebuild);
|
|
561
582
|
// only try reviving from durable store if snapshot is unfulfilled
|
|
562
583
|
if (rebuilt.state !== 'Unfulfilled') {
|
|
563
584
|
return rebuilt;
|
|
@@ -618,6 +639,7 @@
|
|
|
618
639
|
};
|
|
619
640
|
return create(environment, {
|
|
620
641
|
storeSetExpiration: { value: storeSetExpiration },
|
|
642
|
+
publishStoreMetadata: { value: publishStoreMetadata },
|
|
621
643
|
storeBroadcast: { value: storeBroadcast },
|
|
622
644
|
storeReset: { value: storeReset },
|
|
623
645
|
resolveSnapshot: { value: resolveSnapshot },
|
|
@@ -664,6 +686,12 @@
|
|
|
664
686
|
const setExpiration = function (recordId, expiration, _staleExpiration) {
|
|
665
687
|
environment.storeSetExpiration(recordId, expiration, Number.MAX_SAFE_INTEGER);
|
|
666
688
|
};
|
|
689
|
+
const publishStoreMetadata = function (recordId, storeMetadata) {
|
|
690
|
+
environment.publishStoreMetadata(recordId, {
|
|
691
|
+
...storeMetadata,
|
|
692
|
+
staleTimestamp: Number.MAX_SAFE_INTEGER,
|
|
693
|
+
});
|
|
694
|
+
};
|
|
667
695
|
return create(environment, {
|
|
668
696
|
storeLookup: { value: storeLookup },
|
|
669
697
|
snapshotAvailable: {
|
|
@@ -672,6 +700,9 @@
|
|
|
672
700
|
storeSetExpiration: {
|
|
673
701
|
value: setExpiration,
|
|
674
702
|
},
|
|
703
|
+
publishStoreMetadata: {
|
|
704
|
+
value: publishStoreMetadata,
|
|
705
|
+
},
|
|
675
706
|
});
|
|
676
707
|
}
|
|
677
708
|
|
|
@@ -1,8 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Contains store entry data along with any metadata for that entry that needs
|
|
3
3
|
* to be persisted to durable storage
|
|
4
|
+
*
|
|
5
|
+
* IMPORTANT: if you update this interface, you must also create a
|
|
6
|
+
* deprecated interface, and check for durable store entries of this
|
|
7
|
+
* type in isDeprecatedDurableStoreEntry so that revive discards
|
|
8
|
+
* invalid entries.
|
|
4
9
|
*/
|
|
5
10
|
export interface DurableStoreEntry<T = unknown> {
|
|
11
|
+
data: T;
|
|
12
|
+
metadata?: {
|
|
13
|
+
ingestionTimestamp: number;
|
|
14
|
+
expirationTimestamp: number;
|
|
15
|
+
staleTimestamp?: number;
|
|
16
|
+
namespace: string;
|
|
17
|
+
representationName: string;
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
export declare function isDeprecatedDurableStoreEntry(durableRecord: unknown): boolean;
|
|
21
|
+
export interface DeprecatedDurableStoreEntry1<T = unknown> {
|
|
6
22
|
data: T;
|
|
7
23
|
expiration?: {
|
|
8
24
|
fresh: number;
|
|
@@ -4,6 +4,13 @@
|
|
|
4
4
|
(global = global || self, factory(global.luvioEnvironments = {}));
|
|
5
5
|
}(this, (function (exports) { 'use strict';
|
|
6
6
|
|
|
7
|
+
function isDeprecatedDurableStoreEntry(durableRecord) {
|
|
8
|
+
if (durableRecord.expiration !== undefined) {
|
|
9
|
+
return true;
|
|
10
|
+
}
|
|
11
|
+
// Add more deprecated shape checks here
|
|
12
|
+
return false;
|
|
13
|
+
}
|
|
7
14
|
(function (DurableStoreOperationType) {
|
|
8
15
|
DurableStoreOperationType["SetEntries"] = "setEntries";
|
|
9
16
|
DurableStoreOperationType["EvictEntries"] = "evictEntries";
|
|
@@ -102,7 +109,7 @@
|
|
|
102
109
|
flushPendingWritesToDurableStore: function (environment, durableStore, durableStoreErrorHandler) {
|
|
103
110
|
var durableRecords = create({});
|
|
104
111
|
var records = environment.getStoreRecords();
|
|
105
|
-
var
|
|
112
|
+
var storeMetadataMap = environment.getStoreMetadataMap();
|
|
106
113
|
var keys$1 = keys(pendingDurableWrites);
|
|
107
114
|
for (var i = 0, len = keys$1.length; i < len; i += 1) {
|
|
108
115
|
var key = keys$1[i];
|
|
@@ -111,12 +118,12 @@
|
|
|
111
118
|
if (record === undefined) {
|
|
112
119
|
continue;
|
|
113
120
|
}
|
|
114
|
-
var
|
|
121
|
+
var recordMetadata = storeMetadataMap[key];
|
|
115
122
|
durableRecords[key] = {
|
|
116
123
|
// make copy of record b/c evict is coming
|
|
117
124
|
// NOTE: only spread copy if an array or object
|
|
118
125
|
data: copy(record),
|
|
119
|
-
|
|
126
|
+
metadata: recordMetadata,
|
|
120
127
|
};
|
|
121
128
|
// remove record from L1 cache while we are on the synchronous path
|
|
122
129
|
// because we do not want some other code attempting to use the
|
|
@@ -281,8 +288,8 @@
|
|
|
281
288
|
function isStoreEntryError(storeRecord) {
|
|
282
289
|
return storeRecord.__type === 'error';
|
|
283
290
|
}
|
|
284
|
-
function isStoreEntryExpiredAndError(storeRecord,
|
|
285
|
-
return isStoreEntryError(storeRecord) &&
|
|
291
|
+
function isStoreEntryExpiredAndError(storeRecord, expirationTimestamp, now) {
|
|
292
|
+
return isStoreEntryError(storeRecord) && expirationTimestamp < now;
|
|
286
293
|
}
|
|
287
294
|
function reviveDurableEntriesToStore(durableRecords, environment, pendingWriter) {
|
|
288
295
|
var revivedKeys = create(null);
|
|
@@ -299,37 +306,42 @@
|
|
|
299
306
|
for (var i = 0, len = durableKeys.length; i < len; i += 1) {
|
|
300
307
|
var key = durableKeys[i];
|
|
301
308
|
var durableRecord = durableRecords[key];
|
|
302
|
-
|
|
309
|
+
if (isDeprecatedDurableStoreEntry(durableRecord)) {
|
|
310
|
+
// had the old shape, skip reviving this entry.
|
|
311
|
+
hadUnexpectedShape = true;
|
|
312
|
+
continue;
|
|
313
|
+
}
|
|
314
|
+
var metadata = durableRecord.metadata, data = durableRecord.data;
|
|
303
315
|
if (data === undefined) {
|
|
304
316
|
// if unexpected data skip reviving
|
|
305
317
|
hadUnexpectedShape = true;
|
|
306
318
|
continue;
|
|
307
319
|
}
|
|
308
|
-
if (
|
|
309
|
-
var
|
|
310
|
-
if (
|
|
320
|
+
if (metadata !== undefined) {
|
|
321
|
+
var expirationTimestamp = metadata.expirationTimestamp, staleTimestamp = metadata.staleTimestamp;
|
|
322
|
+
if (expirationTimestamp === undefined || staleTimestamp === undefined) {
|
|
311
323
|
// if unexpected expiration data skip reviving
|
|
312
324
|
hadUnexpectedShape = true;
|
|
313
325
|
continue;
|
|
314
326
|
}
|
|
315
327
|
// if past stale TTL then don't revive
|
|
316
|
-
if (
|
|
328
|
+
if (staleTimestamp < now) {
|
|
317
329
|
continue;
|
|
318
330
|
}
|
|
319
331
|
// We don't want to revive a cached value if it's an error and it's
|
|
320
|
-
//
|
|
332
|
+
// expirationTimestamp TTL is expired, otherwise we would never hit the network
|
|
321
333
|
// during resolveSnapshot or rebuildSnapshot in the makeOffline
|
|
322
334
|
// environment because the way the Reader works.
|
|
323
335
|
// If a StoreEntry is an error, the Reader will return UnfulfilledSnapshot
|
|
324
336
|
// if stale TTL is expired. But it will return ErrorSnapshot (instead of
|
|
325
|
-
// StaleSnapshot) if
|
|
337
|
+
// StaleSnapshot) if expirationTimestamp TTL is expired (but stale TTL isn't).
|
|
326
338
|
// In makeOffline environment the stale TTL is maxed out so Reader
|
|
327
339
|
// will always return ErrorSnapshot.
|
|
328
|
-
if (isStoreEntryExpiredAndError(data,
|
|
340
|
+
if (isStoreEntryExpiredAndError(data, expirationTimestamp, now)) {
|
|
329
341
|
continue;
|
|
330
342
|
}
|
|
331
|
-
// call base
|
|
332
|
-
environment.
|
|
343
|
+
// call base publishStoreMetadata so we don't add the key to pendingWriters
|
|
344
|
+
environment.publishStoreMetadata(key, metadata);
|
|
333
345
|
}
|
|
334
346
|
if (isStoreEntryError(data)) {
|
|
335
347
|
// freeze errors on way into L1
|
|
@@ -404,8 +416,11 @@
|
|
|
404
416
|
// durable store doesn't have what we asked for so refresh snapshot
|
|
405
417
|
return refreshSnapshot();
|
|
406
418
|
}
|
|
407
|
-
// attempt to
|
|
408
|
-
|
|
419
|
+
// attempt to lookup (or rebuild if baseSnapshot is provided) the snapshot
|
|
420
|
+
// now that we have revived the missingLinks
|
|
421
|
+
var snapshot = baseSnapshot === undefined
|
|
422
|
+
? environment.storeLookup(unavailableSnapshot.select, environment.createSnapshot, unavailableSnapshot.refresh)
|
|
423
|
+
: environment.rebuildSnapshot(baseSnapshot, environment.getStoreRecords(), environment.getStoreMetadataMap(), environment.getStoreRedirectKeys(), function () { });
|
|
409
424
|
// if snapshot is pending then some other in-flight refresh will broadcast
|
|
410
425
|
// later
|
|
411
426
|
if (snapshot.state === 'Pending') {
|
|
@@ -550,6 +565,12 @@
|
|
|
550
565
|
pendingWriter.addPendingWrite(recordId);
|
|
551
566
|
environment.storeSetExpiration(recordId, expiration, staleExpiration);
|
|
552
567
|
};
|
|
568
|
+
var publishStoreMetadata = function (recordId, storeMetadata) {
|
|
569
|
+
validateNotDisposed();
|
|
570
|
+
// mark this as pending write since we need to re-write with updated metadata values
|
|
571
|
+
pendingWriter.addPendingWrite(recordId);
|
|
572
|
+
environment.publishStoreMetadata(recordId, storeMetadata);
|
|
573
|
+
};
|
|
553
574
|
var storeBroadcast = function (_rebuildSnapshot, _snapshotDataAvailable) {
|
|
554
575
|
// don't await the DS write - DS implementation will take care of R/W
|
|
555
576
|
// synchronization
|
|
@@ -584,10 +605,10 @@
|
|
|
584
605
|
}
|
|
585
606
|
return reviveSnapshot(environment, durableStore, pendingWriter, snapshot, refresh, durableStoreErrorHandler);
|
|
586
607
|
};
|
|
587
|
-
var rebuildSnapshot = function (snapshot, records,
|
|
608
|
+
var rebuildSnapshot = function (snapshot, records, storeMetadataMap, redirects, onAsyncRebuild) {
|
|
588
609
|
validateNotDisposed();
|
|
589
610
|
// try rebuilding from memory
|
|
590
|
-
var rebuilt = environment.rebuildSnapshot(snapshot, records,
|
|
611
|
+
var rebuilt = environment.rebuildSnapshot(snapshot, records, storeMetadataMap, redirects, onAsyncRebuild);
|
|
591
612
|
// only try reviving from durable store if snapshot is unfulfilled
|
|
592
613
|
if (rebuilt.state !== 'Unfulfilled') {
|
|
593
614
|
return rebuilt;
|
|
@@ -646,6 +667,7 @@
|
|
|
646
667
|
};
|
|
647
668
|
return create(environment, {
|
|
648
669
|
storeSetExpiration: { value: storeSetExpiration },
|
|
670
|
+
publishStoreMetadata: { value: publishStoreMetadata },
|
|
649
671
|
storeBroadcast: { value: storeBroadcast },
|
|
650
672
|
storeReset: { value: storeReset },
|
|
651
673
|
resolveSnapshot: { value: resolveSnapshot },
|
|
@@ -692,6 +714,9 @@
|
|
|
692
714
|
var setExpiration = function (recordId, expiration, _staleExpiration) {
|
|
693
715
|
environment.storeSetExpiration(recordId, expiration, Number.MAX_SAFE_INTEGER);
|
|
694
716
|
};
|
|
717
|
+
var publishStoreMetadata = function (recordId, storeMetadata) {
|
|
718
|
+
environment.publishStoreMetadata(recordId, __assign(__assign({}, storeMetadata), { staleTimestamp: Number.MAX_SAFE_INTEGER }));
|
|
719
|
+
};
|
|
695
720
|
return create(environment, {
|
|
696
721
|
storeLookup: { value: storeLookup },
|
|
697
722
|
snapshotAvailable: {
|
|
@@ -700,6 +725,9 @@
|
|
|
700
725
|
storeSetExpiration: {
|
|
701
726
|
value: setExpiration,
|
|
702
727
|
},
|
|
728
|
+
publishStoreMetadata: {
|
|
729
|
+
value: publishStoreMetadata,
|
|
730
|
+
},
|
|
703
731
|
});
|
|
704
732
|
}
|
|
705
733
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@luvio/environments",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.49.0",
|
|
4
4
|
"description": "Luvio Environments",
|
|
5
5
|
"main": "dist/umd/es2018/environments.js",
|
|
6
6
|
"module": "dist/es/es2018/environments.js",
|
|
@@ -16,6 +16,6 @@
|
|
|
16
16
|
"dist/"
|
|
17
17
|
],
|
|
18
18
|
"dependencies": {
|
|
19
|
-
"@luvio/engine": "0.
|
|
19
|
+
"@luvio/engine": "0.49.0"
|
|
20
20
|
}
|
|
21
21
|
}
|