layercache 1.2.7 → 1.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/cli.cjs +26 -2
- package/dist/cli.js +26 -2
- package/dist/{edge-BMmPVqaD.d.cts → edge-BXWTKlI1.d.cts} +21 -10
- package/dist/{edge-BMmPVqaD.d.ts → edge-BXWTKlI1.d.ts} +21 -10
- package/dist/edge.d.cts +1 -1
- package/dist/edge.d.ts +1 -1
- package/dist/index.cjs +1016 -834
- package/dist/index.d.cts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +919 -737
- package/package.json +1 -1
- package/packages/nestjs/dist/index.cjs +942 -715
- package/packages/nestjs/dist/index.d.cts +21 -10
- package/packages/nestjs/dist/index.d.ts +21 -10
- package/packages/nestjs/dist/index.js +942 -715
package/dist/index.js
CHANGED
|
@@ -440,7 +440,7 @@ function normalizeForSerialization(value) {
|
|
|
440
440
|
}
|
|
441
441
|
function serializeKeyPart(value) {
|
|
442
442
|
if (typeof value === "string") {
|
|
443
|
-
return `s:${value}`;
|
|
443
|
+
return `s:${value.replace(/%/g, "%25").replace(/:/g, "%3A")}`;
|
|
444
444
|
}
|
|
445
445
|
if (typeof value === "number") {
|
|
446
446
|
return `n:${value}`;
|
|
@@ -468,102 +468,6 @@ function createInstanceId() {
|
|
|
468
468
|
return `layercache-${Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("")}`;
|
|
469
469
|
}
|
|
470
470
|
|
|
471
|
-
// src/internal/CacheSnapshotFile.ts
|
|
472
|
-
function isWithinSnapshotBase(realBaseDir, candidatePath, pathSeparator, path) {
|
|
473
|
-
const relative = path.relative(realBaseDir, candidatePath);
|
|
474
|
-
return !(relative === ".." || relative.startsWith(`..${pathSeparator}`) || path.isAbsolute(relative));
|
|
475
|
-
}
|
|
476
|
-
async function findExistingAncestor(directory, fs2, path) {
|
|
477
|
-
let current = directory;
|
|
478
|
-
while (true) {
|
|
479
|
-
try {
|
|
480
|
-
await fs2.lstat(current);
|
|
481
|
-
return current;
|
|
482
|
-
} catch (error) {
|
|
483
|
-
if (error.code !== "ENOENT") {
|
|
484
|
-
throw error;
|
|
485
|
-
}
|
|
486
|
-
}
|
|
487
|
-
const parent = path.dirname(current);
|
|
488
|
-
if (parent === current) {
|
|
489
|
-
return current;
|
|
490
|
-
}
|
|
491
|
-
current = parent;
|
|
492
|
-
}
|
|
493
|
-
}
|
|
494
|
-
async function validateSnapshotFilePath(filePath, mode, snapshotBaseDir, cwd = process.cwd()) {
|
|
495
|
-
if (filePath.length === 0) {
|
|
496
|
-
throw new Error("filePath must not be empty.");
|
|
497
|
-
}
|
|
498
|
-
if (filePath.includes("\0")) {
|
|
499
|
-
throw new Error("filePath must not contain null bytes.");
|
|
500
|
-
}
|
|
501
|
-
const { promises: fs2 } = await import("fs");
|
|
502
|
-
const path = await import("path");
|
|
503
|
-
const resolved = path.resolve(filePath);
|
|
504
|
-
const baseDir = snapshotBaseDir === false ? false : path.resolve(snapshotBaseDir ?? cwd);
|
|
505
|
-
if (baseDir === false) {
|
|
506
|
-
return resolved;
|
|
507
|
-
}
|
|
508
|
-
await fs2.mkdir(baseDir, { recursive: true });
|
|
509
|
-
const realBaseDir = await fs2.realpath(baseDir);
|
|
510
|
-
if (!isWithinSnapshotBase(realBaseDir, resolved, path.sep, path)) {
|
|
511
|
-
throw new Error(`filePath is outside the allowed snapshot directory: ${realBaseDir}`);
|
|
512
|
-
}
|
|
513
|
-
if (mode === "read") {
|
|
514
|
-
const realTarget = await fs2.realpath(resolved);
|
|
515
|
-
if (!isWithinSnapshotBase(realBaseDir, realTarget, path.sep, path)) {
|
|
516
|
-
throw new Error(`filePath is outside the allowed snapshot directory: ${realBaseDir}`);
|
|
517
|
-
}
|
|
518
|
-
return realTarget;
|
|
519
|
-
}
|
|
520
|
-
const parentDir = path.dirname(resolved);
|
|
521
|
-
const existingAncestor = await findExistingAncestor(parentDir, fs2, path);
|
|
522
|
-
const realExistingAncestor = await fs2.realpath(existingAncestor);
|
|
523
|
-
if (!isWithinSnapshotBase(realBaseDir, realExistingAncestor, path.sep, path)) {
|
|
524
|
-
throw new Error(`filePath is outside the allowed snapshot directory: ${realBaseDir}`);
|
|
525
|
-
}
|
|
526
|
-
await fs2.mkdir(parentDir, { recursive: true });
|
|
527
|
-
const realParentDir = await fs2.realpath(parentDir);
|
|
528
|
-
if (!isWithinSnapshotBase(realBaseDir, realParentDir, path.sep, path)) {
|
|
529
|
-
throw new Error(`filePath is outside the allowed snapshot directory: ${realBaseDir}`);
|
|
530
|
-
}
|
|
531
|
-
const targetPath = path.join(realParentDir, path.basename(resolved));
|
|
532
|
-
try {
|
|
533
|
-
const existing = await fs2.lstat(targetPath);
|
|
534
|
-
if (existing.isSymbolicLink()) {
|
|
535
|
-
throw new Error("filePath must not point to a symbolic link.");
|
|
536
|
-
}
|
|
537
|
-
} catch (error) {
|
|
538
|
-
if (error.code !== "ENOENT") {
|
|
539
|
-
throw error;
|
|
540
|
-
}
|
|
541
|
-
}
|
|
542
|
-
return targetPath;
|
|
543
|
-
}
|
|
544
|
-
async function readUtf8HandleWithLimit(handle, byteLimit) {
|
|
545
|
-
if (byteLimit === false) {
|
|
546
|
-
return handle.readFile({ encoding: "utf8" });
|
|
547
|
-
}
|
|
548
|
-
const chunks = [];
|
|
549
|
-
let totalBytes = 0;
|
|
550
|
-
let position = 0;
|
|
551
|
-
while (true) {
|
|
552
|
-
const buffer = Buffer.allocUnsafe(Math.min(64 * 1024, byteLimit - totalBytes + 1));
|
|
553
|
-
const { bytesRead } = await handle.read(buffer, 0, buffer.byteLength, position);
|
|
554
|
-
if (bytesRead === 0) {
|
|
555
|
-
break;
|
|
556
|
-
}
|
|
557
|
-
totalBytes += bytesRead;
|
|
558
|
-
if (totalBytes > byteLimit) {
|
|
559
|
-
throw new Error(`Snapshot file exceeds snapshotMaxBytes limit (${totalBytes} bytes > ${byteLimit} bytes).`);
|
|
560
|
-
}
|
|
561
|
-
chunks.push(buffer.subarray(0, bytesRead));
|
|
562
|
-
position += bytesRead;
|
|
563
|
-
}
|
|
564
|
-
return Buffer.concat(chunks).toString("utf8");
|
|
565
|
-
}
|
|
566
|
-
|
|
567
471
|
// src/internal/CacheStackGeneration.ts
|
|
568
472
|
var DEFAULT_GENERATION_CLEANUP_BATCH_SIZE = 500;
|
|
569
473
|
function generationPrefix(generation) {
|
|
@@ -611,6 +515,206 @@ function planGenerationCleanupBatches(keys, generationCleanup) {
|
|
|
611
515
|
return batches;
|
|
612
516
|
}
|
|
613
517
|
|
|
518
|
+
// src/internal/CacheStackInvalidationSupport.ts
|
|
519
|
+
var CacheStackInvalidationSupport = class {
|
|
520
|
+
constructor(options) {
|
|
521
|
+
this.options = options;
|
|
522
|
+
}
|
|
523
|
+
options;
|
|
524
|
+
async collectKeysForTag(tag, maxKeys) {
|
|
525
|
+
const keys = /* @__PURE__ */ new Set();
|
|
526
|
+
if (this.options.tagIndex.forEachKeyForTag) {
|
|
527
|
+
await this.options.tagIndex.forEachKeyForTag(tag, async (key) => {
|
|
528
|
+
keys.add(key);
|
|
529
|
+
this.assertWithinInvalidationKeyLimit(keys.size, maxKeys);
|
|
530
|
+
});
|
|
531
|
+
return [...keys];
|
|
532
|
+
}
|
|
533
|
+
for (const key of await this.options.tagIndex.keysForTag(tag)) {
|
|
534
|
+
keys.add(key);
|
|
535
|
+
this.assertWithinInvalidationKeyLimit(keys.size, maxKeys);
|
|
536
|
+
}
|
|
537
|
+
return [...keys];
|
|
538
|
+
}
|
|
539
|
+
intersectKeys(groups) {
|
|
540
|
+
if (groups.length === 0) {
|
|
541
|
+
return [];
|
|
542
|
+
}
|
|
543
|
+
const [firstGroup, ...rest] = groups;
|
|
544
|
+
const restSets = rest.map((group) => new Set(group));
|
|
545
|
+
return [...new Set(firstGroup)].filter((key) => restSets.every((group) => group.has(key)));
|
|
546
|
+
}
|
|
547
|
+
async deleteKeysFromLayers(layers, keys) {
|
|
548
|
+
await Promise.all(
|
|
549
|
+
layers.map(async (layer) => {
|
|
550
|
+
if (this.options.shouldSkipLayer(layer)) {
|
|
551
|
+
return;
|
|
552
|
+
}
|
|
553
|
+
if (layer.deleteMany) {
|
|
554
|
+
try {
|
|
555
|
+
await layer.deleteMany(keys);
|
|
556
|
+
} catch (error) {
|
|
557
|
+
await this.options.handleLayerFailure(layer, "delete", error);
|
|
558
|
+
}
|
|
559
|
+
return;
|
|
560
|
+
}
|
|
561
|
+
await Promise.all(
|
|
562
|
+
keys.map(async (key) => {
|
|
563
|
+
try {
|
|
564
|
+
await layer.delete(key);
|
|
565
|
+
} catch (error) {
|
|
566
|
+
await this.options.handleLayerFailure(layer, "delete", error);
|
|
567
|
+
}
|
|
568
|
+
})
|
|
569
|
+
);
|
|
570
|
+
})
|
|
571
|
+
);
|
|
572
|
+
}
|
|
573
|
+
assertWithinInvalidationKeyLimit(size, maxKeys) {
|
|
574
|
+
if (maxKeys !== false && size > maxKeys) {
|
|
575
|
+
throw new Error(`Invalidation matched too many keys (${size} > ${maxKeys}).`);
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
};
|
|
579
|
+
|
|
580
|
+
// src/internal/CacheStackLayerWriter.ts
|
|
581
|
+
var CacheStackLayerWriter = class {
|
|
582
|
+
constructor(options) {
|
|
583
|
+
this.options = options;
|
|
584
|
+
}
|
|
585
|
+
options;
|
|
586
|
+
async writeAcrossLayers(key, kind, value, writeOptions) {
|
|
587
|
+
const now = Date.now();
|
|
588
|
+
const clearEpoch = this.options.maintenance.currentClearEpoch();
|
|
589
|
+
const keyEpoch = this.options.maintenance.currentKeyEpoch(key);
|
|
590
|
+
const immediateOperations = [];
|
|
591
|
+
const deferredOperations = [];
|
|
592
|
+
for (const layer of this.options.layers) {
|
|
593
|
+
const operation = async () => {
|
|
594
|
+
if (this.options.maintenance.isWriteOutdated(key, clearEpoch, keyEpoch)) {
|
|
595
|
+
return;
|
|
596
|
+
}
|
|
597
|
+
if (this.options.shouldSkipLayer(layer)) {
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
const entry = this.buildLayerSetEntry(layer, key, kind, value, writeOptions, now);
|
|
601
|
+
try {
|
|
602
|
+
await layer.set(entry.key, entry.value, entry.ttl);
|
|
603
|
+
} catch (error) {
|
|
604
|
+
await this.options.handleLayerFailure(layer, "write", error);
|
|
605
|
+
}
|
|
606
|
+
};
|
|
607
|
+
if (this.options.shouldWriteBehind(layer)) {
|
|
608
|
+
deferredOperations.push(operation);
|
|
609
|
+
} else {
|
|
610
|
+
immediateOperations.push(operation);
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
await this.executeLayerOperations(immediateOperations, { key, action: kind === "empty" ? "negative-set" : "set" });
|
|
614
|
+
await Promise.all(deferredOperations.map((operation) => this.options.enqueueWriteBehind(operation)));
|
|
615
|
+
}
|
|
616
|
+
async writeBatch(entries) {
|
|
617
|
+
const now = Date.now();
|
|
618
|
+
const clearEpoch = this.options.maintenance.currentClearEpoch();
|
|
619
|
+
const entryEpochs = new Map(
|
|
620
|
+
entries.map((entry) => [entry.key, this.options.maintenance.currentKeyEpoch(entry.key)])
|
|
621
|
+
);
|
|
622
|
+
const entriesByLayer = /* @__PURE__ */ new Map();
|
|
623
|
+
const immediateOperations = [];
|
|
624
|
+
const deferredOperations = [];
|
|
625
|
+
for (const entry of entries) {
|
|
626
|
+
for (const layer of this.options.layers) {
|
|
627
|
+
if (this.options.shouldSkipLayer(layer)) {
|
|
628
|
+
continue;
|
|
629
|
+
}
|
|
630
|
+
const layerEntry = this.buildLayerSetEntry(layer, entry.key, "value", entry.value, entry.options, now);
|
|
631
|
+
const bucket = entriesByLayer.get(layer) ?? [];
|
|
632
|
+
bucket.push(layerEntry);
|
|
633
|
+
entriesByLayer.set(layer, bucket);
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
for (const [layer, layerEntries] of entriesByLayer.entries()) {
|
|
637
|
+
const operation = async () => {
|
|
638
|
+
if (clearEpoch !== this.options.maintenance.currentClearEpoch()) {
|
|
639
|
+
return;
|
|
640
|
+
}
|
|
641
|
+
const activeEntries = layerEntries.filter(
|
|
642
|
+
(entry) => (entryEpochs.get(entry.key) ?? 0) === this.options.maintenance.currentKeyEpoch(entry.key)
|
|
643
|
+
);
|
|
644
|
+
if (activeEntries.length === 0) {
|
|
645
|
+
return;
|
|
646
|
+
}
|
|
647
|
+
try {
|
|
648
|
+
if (layer.setMany) {
|
|
649
|
+
await layer.setMany(activeEntries);
|
|
650
|
+
return;
|
|
651
|
+
}
|
|
652
|
+
await Promise.all(activeEntries.map((entry) => layer.set(entry.key, entry.value, entry.ttl)));
|
|
653
|
+
} catch (error) {
|
|
654
|
+
await this.options.handleLayerFailure(layer, "write", error);
|
|
655
|
+
}
|
|
656
|
+
};
|
|
657
|
+
if (this.options.shouldWriteBehind(layer)) {
|
|
658
|
+
deferredOperations.push(operation);
|
|
659
|
+
} else {
|
|
660
|
+
immediateOperations.push(operation);
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
await this.executeLayerOperations(immediateOperations, { key: "batch", action: "mset" });
|
|
664
|
+
await Promise.all(deferredOperations.map((operation) => this.options.enqueueWriteBehind(operation)));
|
|
665
|
+
return { clearEpoch, entryEpochs };
|
|
666
|
+
}
|
|
667
|
+
async executeLayerOperations(operations, context) {
|
|
668
|
+
if (this.options.writePolicy !== "best-effort") {
|
|
669
|
+
await Promise.all(operations.map((operation) => operation()));
|
|
670
|
+
return;
|
|
671
|
+
}
|
|
672
|
+
const results = await Promise.allSettled(operations.map((operation) => operation()));
|
|
673
|
+
const failures = results.filter((result) => result.status === "rejected");
|
|
674
|
+
const degraded = results.filter((result) => result.status === "fulfilled");
|
|
675
|
+
if (failures.length === 0) {
|
|
676
|
+
return;
|
|
677
|
+
}
|
|
678
|
+
this.options.onWriteFailures(
|
|
679
|
+
context,
|
|
680
|
+
failures.map((failure) => failure.reason)
|
|
681
|
+
);
|
|
682
|
+
if (failures.length === operations.length) {
|
|
683
|
+
throw new AggregateError(
|
|
684
|
+
failures.map((failure) => failure.reason),
|
|
685
|
+
`${context.action} failed for every cache layer`
|
|
686
|
+
);
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
buildLayerSetEntry(layer, key, kind, value, writeOptions, now) {
|
|
690
|
+
const freshTtl = this.options.resolveFreshTtl(key, layer.name, kind, writeOptions, layer.defaultTtl, value);
|
|
691
|
+
const staleWhileRevalidate = this.options.resolveLayerSeconds(
|
|
692
|
+
layer.name,
|
|
693
|
+
writeOptions?.staleWhileRevalidate,
|
|
694
|
+
this.options.globalStaleWhileRevalidate
|
|
695
|
+
);
|
|
696
|
+
const staleIfError = this.options.resolveLayerSeconds(
|
|
697
|
+
layer.name,
|
|
698
|
+
writeOptions?.staleIfError,
|
|
699
|
+
this.options.globalStaleIfError
|
|
700
|
+
);
|
|
701
|
+
const payload = createStoredValueEnvelope({
|
|
702
|
+
kind,
|
|
703
|
+
value,
|
|
704
|
+
freshTtlSeconds: freshTtl,
|
|
705
|
+
staleWhileRevalidateSeconds: staleWhileRevalidate,
|
|
706
|
+
staleIfErrorSeconds: staleIfError,
|
|
707
|
+
now
|
|
708
|
+
});
|
|
709
|
+
const ttl = remainingStoredTtlSeconds(payload, now) ?? freshTtl;
|
|
710
|
+
return {
|
|
711
|
+
key,
|
|
712
|
+
value: payload,
|
|
713
|
+
ttl
|
|
714
|
+
};
|
|
715
|
+
}
|
|
716
|
+
};
|
|
717
|
+
|
|
614
718
|
// src/internal/CacheStackMaintenance.ts
|
|
615
719
|
var CacheStackMaintenance = class {
|
|
616
720
|
keyEpochs = /* @__PURE__ */ new Map();
|
|
@@ -694,57 +798,347 @@ var CacheStackMaintenance = class {
|
|
|
694
798
|
await this.flushWriteBehindQueue(options, flushBatch);
|
|
695
799
|
}
|
|
696
800
|
}
|
|
697
|
-
scheduleGenerationCleanup(generation, task, onError) {
|
|
698
|
-
const scheduledTask = (this.generationCleanupPromise ?? Promise.resolve()).then(() => task(generation)).catch((error) => {
|
|
699
|
-
onError(generation, error);
|
|
700
|
-
});
|
|
701
|
-
this.generationCleanupPromise = scheduledTask.finally(() => {
|
|
702
|
-
if (this.generationCleanupPromise === scheduledTask) {
|
|
703
|
-
this.generationCleanupPromise = void 0;
|
|
801
|
+
scheduleGenerationCleanup(generation, task, onError) {
|
|
802
|
+
const scheduledTask = (this.generationCleanupPromise ?? Promise.resolve()).then(() => task(generation)).catch((error) => {
|
|
803
|
+
onError(generation, error);
|
|
804
|
+
});
|
|
805
|
+
this.generationCleanupPromise = scheduledTask.finally(() => {
|
|
806
|
+
if (this.generationCleanupPromise === scheduledTask) {
|
|
807
|
+
this.generationCleanupPromise = void 0;
|
|
808
|
+
}
|
|
809
|
+
});
|
|
810
|
+
}
|
|
811
|
+
async waitForGenerationCleanup() {
|
|
812
|
+
await this.generationCleanupPromise;
|
|
813
|
+
}
|
|
814
|
+
};
|
|
815
|
+
|
|
816
|
+
// src/internal/CacheStackRuntimePolicy.ts
|
|
817
|
+
function shouldSkipLayer(degradedUntil, now = Date.now()) {
|
|
818
|
+
return degradedUntil !== void 0 && degradedUntil > now;
|
|
819
|
+
}
|
|
820
|
+
function shouldStartBackgroundRefresh({
|
|
821
|
+
isDisconnecting,
|
|
822
|
+
hasRefreshInFlight
|
|
823
|
+
}) {
|
|
824
|
+
return !isDisconnecting && !hasRefreshInFlight;
|
|
825
|
+
}
|
|
826
|
+
function resolveRecoverableLayerFailure(gracefulDegradation, now = Date.now()) {
|
|
827
|
+
if (!gracefulDegradation) {
|
|
828
|
+
return { degrade: false };
|
|
829
|
+
}
|
|
830
|
+
const retryAfterMs = typeof gracefulDegradation === "object" ? gracefulDegradation.retryAfterMs ?? 1e4 : 1e4;
|
|
831
|
+
return {
|
|
832
|
+
degrade: true,
|
|
833
|
+
degradedUntil: now + retryAfterMs
|
|
834
|
+
};
|
|
835
|
+
}
|
|
836
|
+
function planFreshReadPolicies({
|
|
837
|
+
stored,
|
|
838
|
+
hasFetcher,
|
|
839
|
+
slidingTtl,
|
|
840
|
+
refreshAheadSeconds
|
|
841
|
+
}) {
|
|
842
|
+
const refreshedStored = slidingTtl && isStoredValueEnvelope(stored) ? refreshStoredEnvelope(stored) : void 0;
|
|
843
|
+
const refreshedStoredTtl = refreshedStored ? remainingStoredTtlSeconds(refreshedStored) ?? void 0 : void 0;
|
|
844
|
+
const remainingFreshTtl = remainingFreshTtlSeconds(stored) ?? 0;
|
|
845
|
+
return {
|
|
846
|
+
refreshedStored,
|
|
847
|
+
refreshedStoredTtl,
|
|
848
|
+
shouldScheduleBackgroundRefresh: hasFetcher && refreshAheadSeconds > 0 && remainingFreshTtl > 0 && remainingFreshTtl <= refreshAheadSeconds
|
|
849
|
+
};
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
// src/internal/CacheStackSnapshotManager.ts
|
|
853
|
+
import { randomBytes } from "crypto";
|
|
854
|
+
import { constants, promises as fs } from "fs";
|
|
855
|
+
import path from "path";
|
|
856
|
+
|
|
857
|
+
// src/internal/CacheSnapshotFile.ts
|
|
858
|
+
function isWithinSnapshotBase(realBaseDir, candidatePath, pathSeparator, path2) {
|
|
859
|
+
const relative = path2.relative(realBaseDir, candidatePath);
|
|
860
|
+
return !(relative === ".." || relative.startsWith(`..${pathSeparator}`) || path2.isAbsolute(relative));
|
|
861
|
+
}
|
|
862
|
+
async function findExistingAncestor(directory, fs3, path2) {
|
|
863
|
+
let current = directory;
|
|
864
|
+
while (true) {
|
|
865
|
+
try {
|
|
866
|
+
await fs3.lstat(current);
|
|
867
|
+
return current;
|
|
868
|
+
} catch (error) {
|
|
869
|
+
if (error.code !== "ENOENT") {
|
|
870
|
+
throw error;
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
const parent = path2.dirname(current);
|
|
874
|
+
if (parent === current) {
|
|
875
|
+
return current;
|
|
876
|
+
}
|
|
877
|
+
current = parent;
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
async function validateSnapshotFilePath(filePath, mode, snapshotBaseDir, cwd = process.cwd()) {
|
|
881
|
+
if (filePath.length === 0) {
|
|
882
|
+
throw new Error("filePath must not be empty.");
|
|
883
|
+
}
|
|
884
|
+
if (filePath.includes("\0")) {
|
|
885
|
+
throw new Error("filePath must not contain null bytes.");
|
|
886
|
+
}
|
|
887
|
+
const { promises: fs3 } = await import("fs");
|
|
888
|
+
const path2 = await import("path");
|
|
889
|
+
const resolved = path2.resolve(filePath);
|
|
890
|
+
const baseDir = snapshotBaseDir === false ? false : path2.resolve(snapshotBaseDir ?? cwd);
|
|
891
|
+
if (baseDir === false) {
|
|
892
|
+
return resolved;
|
|
893
|
+
}
|
|
894
|
+
await fs3.mkdir(baseDir, { recursive: true });
|
|
895
|
+
const realBaseDir = await fs3.realpath(baseDir);
|
|
896
|
+
if (!isWithinSnapshotBase(realBaseDir, resolved, path2.sep, path2)) {
|
|
897
|
+
throw new Error(`filePath is outside the allowed snapshot directory: ${realBaseDir}`);
|
|
898
|
+
}
|
|
899
|
+
if (mode === "read") {
|
|
900
|
+
const realTarget = await fs3.realpath(resolved);
|
|
901
|
+
if (!isWithinSnapshotBase(realBaseDir, realTarget, path2.sep, path2)) {
|
|
902
|
+
throw new Error(`filePath is outside the allowed snapshot directory: ${realBaseDir}`);
|
|
903
|
+
}
|
|
904
|
+
return realTarget;
|
|
905
|
+
}
|
|
906
|
+
const parentDir = path2.dirname(resolved);
|
|
907
|
+
const existingAncestor = await findExistingAncestor(parentDir, fs3, path2);
|
|
908
|
+
const realExistingAncestor = await fs3.realpath(existingAncestor);
|
|
909
|
+
if (!isWithinSnapshotBase(realBaseDir, realExistingAncestor, path2.sep, path2)) {
|
|
910
|
+
throw new Error(`filePath is outside the allowed snapshot directory: ${realBaseDir}`);
|
|
911
|
+
}
|
|
912
|
+
await fs3.mkdir(parentDir, { recursive: true });
|
|
913
|
+
const realParentDir = await fs3.realpath(parentDir);
|
|
914
|
+
if (!isWithinSnapshotBase(realBaseDir, realParentDir, path2.sep, path2)) {
|
|
915
|
+
throw new Error(`filePath is outside the allowed snapshot directory: ${realBaseDir}`);
|
|
916
|
+
}
|
|
917
|
+
const targetPath = path2.join(realParentDir, path2.basename(resolved));
|
|
918
|
+
try {
|
|
919
|
+
const existing = await fs3.lstat(targetPath);
|
|
920
|
+
if (existing.isSymbolicLink()) {
|
|
921
|
+
throw new Error("filePath must not point to a symbolic link.");
|
|
922
|
+
}
|
|
923
|
+
} catch (error) {
|
|
924
|
+
if (error.code !== "ENOENT") {
|
|
925
|
+
throw error;
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
return targetPath;
|
|
929
|
+
}
|
|
930
|
+
async function readUtf8HandleWithLimit(handle, byteLimit) {
|
|
931
|
+
if (byteLimit === false) {
|
|
932
|
+
return handle.readFile({ encoding: "utf8" });
|
|
933
|
+
}
|
|
934
|
+
const chunks = [];
|
|
935
|
+
let totalBytes = 0;
|
|
936
|
+
let position = 0;
|
|
937
|
+
while (true) {
|
|
938
|
+
const buffer = Buffer.allocUnsafe(Math.min(64 * 1024, byteLimit - totalBytes + 1));
|
|
939
|
+
const { bytesRead } = await handle.read(buffer, 0, buffer.byteLength, position);
|
|
940
|
+
if (bytesRead === 0) {
|
|
941
|
+
break;
|
|
942
|
+
}
|
|
943
|
+
totalBytes += bytesRead;
|
|
944
|
+
if (totalBytes > byteLimit) {
|
|
945
|
+
throw new Error(`Snapshot file exceeds snapshotMaxBytes limit (${totalBytes} bytes > ${byteLimit} bytes).`);
|
|
946
|
+
}
|
|
947
|
+
chunks.push(buffer.subarray(0, bytesRead));
|
|
948
|
+
position += bytesRead;
|
|
949
|
+
}
|
|
950
|
+
return Buffer.concat(chunks).toString("utf8");
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
// src/internal/StructuredDataSanitizer.ts
|
|
954
|
+
var DANGEROUS_KEYS = /* @__PURE__ */ new Set(["__proto__", "prototype", "constructor"]);
|
|
955
|
+
function sanitizeStructuredData(value, options) {
|
|
956
|
+
return sanitizeValue(value, 0, { count: 0 }, options);
|
|
957
|
+
}
|
|
958
|
+
function sanitizeValue(value, depth, state, options) {
|
|
959
|
+
state.count += 1;
|
|
960
|
+
if (state.count > options.maxNodes) {
|
|
961
|
+
throw new Error(`${options.label} exceeds max node count of ${options.maxNodes}.`);
|
|
962
|
+
}
|
|
963
|
+
if (depth > options.maxDepth) {
|
|
964
|
+
throw new Error(`${options.label} exceeds max depth of ${options.maxDepth}.`);
|
|
965
|
+
}
|
|
966
|
+
if (Array.isArray(value)) {
|
|
967
|
+
const sanitized2 = [];
|
|
968
|
+
for (const entry of value) {
|
|
969
|
+
sanitized2.push(sanitizeValue(entry, depth + 1, state, options));
|
|
970
|
+
}
|
|
971
|
+
return sanitized2;
|
|
972
|
+
}
|
|
973
|
+
if (!isPlainObject(value)) {
|
|
974
|
+
return value;
|
|
975
|
+
}
|
|
976
|
+
const sanitized = options.createObject?.() ?? {};
|
|
977
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
978
|
+
if (DANGEROUS_KEYS.has(key)) {
|
|
979
|
+
continue;
|
|
980
|
+
}
|
|
981
|
+
sanitized[key] = sanitizeValue(entry, depth + 1, state, options);
|
|
982
|
+
}
|
|
983
|
+
return sanitized;
|
|
984
|
+
}
|
|
985
|
+
function isPlainObject(value) {
|
|
986
|
+
return Object.prototype.toString.call(value) === "[object Object]";
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
// src/internal/CacheStackSnapshotManager.ts
|
|
990
|
+
var DEFAULT_SNAPSHOT_IMPORT_BATCH_SIZE = 50;
|
|
991
|
+
var CacheStackSnapshotManager = class {
|
|
992
|
+
constructor(options) {
|
|
993
|
+
this.options = options;
|
|
994
|
+
}
|
|
995
|
+
options;
|
|
996
|
+
async exportState(maxEntries) {
|
|
997
|
+
const entries = [];
|
|
998
|
+
await this.visitExportEntries(maxEntries, async (entry) => {
|
|
999
|
+
entries.push(entry);
|
|
1000
|
+
});
|
|
1001
|
+
return entries;
|
|
1002
|
+
}
|
|
1003
|
+
async importState(entries) {
|
|
1004
|
+
const normalizedEntries = entries.map((entry) => ({
|
|
1005
|
+
key: this.options.qualifyKey(this.options.validateCacheKey(entry.key)),
|
|
1006
|
+
value: entry.value,
|
|
1007
|
+
ttl: entry.ttl
|
|
1008
|
+
}));
|
|
1009
|
+
for (let index = 0; index < normalizedEntries.length; index += DEFAULT_SNAPSHOT_IMPORT_BATCH_SIZE) {
|
|
1010
|
+
const batch = normalizedEntries.slice(index, index + DEFAULT_SNAPSHOT_IMPORT_BATCH_SIZE);
|
|
1011
|
+
await Promise.all(
|
|
1012
|
+
batch.map(async (entry) => {
|
|
1013
|
+
await Promise.all(
|
|
1014
|
+
this.options.layers.map(async (layer) => {
|
|
1015
|
+
if (this.options.shouldSkipLayer(layer)) return;
|
|
1016
|
+
try {
|
|
1017
|
+
await layer.set(entry.key, entry.value, entry.ttl);
|
|
1018
|
+
} catch (error) {
|
|
1019
|
+
await this.options.handleLayerFailure(layer, "write", error);
|
|
1020
|
+
}
|
|
1021
|
+
})
|
|
1022
|
+
);
|
|
1023
|
+
await this.options.tagIndex.touch(entry.key);
|
|
1024
|
+
})
|
|
1025
|
+
);
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
async persistToFile(filePath, snapshotBaseDir, maxEntries) {
|
|
1029
|
+
const targetPath = await validateSnapshotFilePath(filePath, "write", snapshotBaseDir);
|
|
1030
|
+
const tempPath = path.join(
|
|
1031
|
+
path.dirname(targetPath),
|
|
1032
|
+
`.layercache-${process.pid}-${Date.now()}-${randomBytes(8).toString("hex")}.tmp`
|
|
1033
|
+
);
|
|
1034
|
+
let handle;
|
|
1035
|
+
try {
|
|
1036
|
+
handle = await fs.open(tempPath, "wx");
|
|
1037
|
+
const openedHandle = handle;
|
|
1038
|
+
await openedHandle.writeFile("[", "utf8");
|
|
1039
|
+
let wroteAny = false;
|
|
1040
|
+
await this.visitExportEntries(maxEntries, async (entry) => {
|
|
1041
|
+
await openedHandle.writeFile(wroteAny ? ",\n" : "\n", "utf8");
|
|
1042
|
+
await openedHandle.writeFile(JSON.stringify(entry, null, 2), "utf8");
|
|
1043
|
+
wroteAny = true;
|
|
1044
|
+
});
|
|
1045
|
+
await openedHandle.writeFile(wroteAny ? "\n]" : "]", "utf8");
|
|
1046
|
+
await openedHandle.close();
|
|
1047
|
+
handle = void 0;
|
|
1048
|
+
await fs.rename(tempPath, targetPath);
|
|
1049
|
+
} catch (error) {
|
|
1050
|
+
await handle?.close().catch(() => void 0);
|
|
1051
|
+
await fs.unlink(tempPath).catch(() => void 0);
|
|
1052
|
+
throw error;
|
|
1053
|
+
}
|
|
1054
|
+
}
|
|
1055
|
+
async restoreFromFile(filePath, snapshotBaseDir, maxBytes) {
|
|
1056
|
+
const validatedPath = await validateSnapshotFilePath(filePath, "read", snapshotBaseDir);
|
|
1057
|
+
const handle = await fs.open(validatedPath, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
|
|
1058
|
+
let raw;
|
|
1059
|
+
try {
|
|
1060
|
+
if (maxBytes !== false) {
|
|
1061
|
+
const stat = await handle.stat();
|
|
1062
|
+
if (stat.size > maxBytes) {
|
|
1063
|
+
throw new Error(`Snapshot file exceeds snapshotMaxBytes limit (${stat.size} bytes > ${maxBytes} bytes).`);
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
raw = await readUtf8HandleWithLimit(handle, maxBytes);
|
|
1067
|
+
} finally {
|
|
1068
|
+
await handle.close();
|
|
1069
|
+
}
|
|
1070
|
+
let parsed;
|
|
1071
|
+
try {
|
|
1072
|
+
parsed = JSON.parse(raw);
|
|
1073
|
+
} catch (cause) {
|
|
1074
|
+
throw new Error(`Invalid snapshot file: could not parse JSON (${this.options.formatError(cause)})`);
|
|
1075
|
+
}
|
|
1076
|
+
if (!this.isCacheSnapshotEntries(parsed)) {
|
|
1077
|
+
throw new Error("Invalid snapshot file: expected an array of { key: string, value, ttl? } entries");
|
|
1078
|
+
}
|
|
1079
|
+
await this.importState(
|
|
1080
|
+
parsed.map((entry) => ({
|
|
1081
|
+
key: entry.key,
|
|
1082
|
+
value: this.sanitizeSnapshotValue(entry.value),
|
|
1083
|
+
ttl: entry.ttl
|
|
1084
|
+
}))
|
|
1085
|
+
);
|
|
1086
|
+
}
|
|
1087
|
+
async visitExportEntries(maxEntries, visitor) {
|
|
1088
|
+
const exported = /* @__PURE__ */ new Set();
|
|
1089
|
+
for (const layer of this.options.layers) {
|
|
1090
|
+
if (!layer.keys && !layer.forEachKey) {
|
|
1091
|
+
continue;
|
|
1092
|
+
}
|
|
1093
|
+
const visitKey = async (key) => {
|
|
1094
|
+
const exportedKey = this.options.stripQualifiedKey(key);
|
|
1095
|
+
if (exported.has(exportedKey)) {
|
|
1096
|
+
return;
|
|
1097
|
+
}
|
|
1098
|
+
const stored = await this.options.readLayerEntry(layer, key);
|
|
1099
|
+
if (stored === null) {
|
|
1100
|
+
return;
|
|
1101
|
+
}
|
|
1102
|
+
exported.add(exportedKey);
|
|
1103
|
+
if (maxEntries !== false && exported.size > maxEntries) {
|
|
1104
|
+
throw new Error(`Snapshot export exceeds snapshotMaxEntries limit (${exported.size} > ${maxEntries}).`);
|
|
1105
|
+
}
|
|
1106
|
+
await visitor({
|
|
1107
|
+
key: exportedKey,
|
|
1108
|
+
value: stored,
|
|
1109
|
+
ttl: remainingStoredTtlSeconds(stored)
|
|
1110
|
+
});
|
|
1111
|
+
};
|
|
1112
|
+
if (layer.forEachKey) {
|
|
1113
|
+
await layer.forEachKey(visitKey);
|
|
1114
|
+
continue;
|
|
1115
|
+
}
|
|
1116
|
+
const keys = await layer.keys?.();
|
|
1117
|
+
for (const key of keys ?? []) {
|
|
1118
|
+
await visitKey(key);
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
isCacheSnapshotEntries(value) {
|
|
1123
|
+
return Array.isArray(value) && value.every((entry) => {
|
|
1124
|
+
if (!entry || typeof entry !== "object") {
|
|
1125
|
+
return false;
|
|
704
1126
|
}
|
|
1127
|
+
const candidate = entry;
|
|
1128
|
+
return typeof candidate.key === "string" && (candidate.ttl === void 0 || typeof candidate.ttl === "number" && Number.isFinite(candidate.ttl) && candidate.ttl >= 0);
|
|
705
1129
|
});
|
|
706
1130
|
}
|
|
707
|
-
|
|
708
|
-
|
|
1131
|
+
sanitizeSnapshotValue(value) {
|
|
1132
|
+
const roundTripped = this.options.snapshotSerializer.deserialize(this.options.snapshotSerializer.serialize(value));
|
|
1133
|
+
return sanitizeStructuredData(roundTripped, {
|
|
1134
|
+
label: "Snapshot value",
|
|
1135
|
+
maxDepth: 64,
|
|
1136
|
+
maxNodes: 1e4,
|
|
1137
|
+
createObject: () => /* @__PURE__ */ Object.create(null)
|
|
1138
|
+
});
|
|
709
1139
|
}
|
|
710
1140
|
};
|
|
711
1141
|
|
|
712
|
-
// src/internal/CacheStackRuntimePolicy.ts
|
|
713
|
-
function shouldSkipLayer(degradedUntil, now = Date.now()) {
|
|
714
|
-
return degradedUntil !== void 0 && degradedUntil > now;
|
|
715
|
-
}
|
|
716
|
-
function shouldStartBackgroundRefresh({
|
|
717
|
-
isDisconnecting,
|
|
718
|
-
hasRefreshInFlight
|
|
719
|
-
}) {
|
|
720
|
-
return !isDisconnecting && !hasRefreshInFlight;
|
|
721
|
-
}
|
|
722
|
-
function resolveRecoverableLayerFailure(gracefulDegradation, now = Date.now()) {
|
|
723
|
-
if (!gracefulDegradation) {
|
|
724
|
-
return { degrade: false };
|
|
725
|
-
}
|
|
726
|
-
const retryAfterMs = typeof gracefulDegradation === "object" ? gracefulDegradation.retryAfterMs ?? 1e4 : 1e4;
|
|
727
|
-
return {
|
|
728
|
-
degrade: true,
|
|
729
|
-
degradedUntil: now + retryAfterMs
|
|
730
|
-
};
|
|
731
|
-
}
|
|
732
|
-
function planFreshReadPolicies({
|
|
733
|
-
stored,
|
|
734
|
-
hasFetcher,
|
|
735
|
-
slidingTtl,
|
|
736
|
-
refreshAheadSeconds
|
|
737
|
-
}) {
|
|
738
|
-
const refreshedStored = slidingTtl && isStoredValueEnvelope(stored) ? refreshStoredEnvelope(stored) : void 0;
|
|
739
|
-
const refreshedStoredTtl = refreshedStored ? remainingStoredTtlSeconds(refreshedStored) ?? void 0 : void 0;
|
|
740
|
-
const remainingFreshTtl = remainingFreshTtlSeconds(stored) ?? 0;
|
|
741
|
-
return {
|
|
742
|
-
refreshedStored,
|
|
743
|
-
refreshedStoredTtl,
|
|
744
|
-
shouldScheduleBackgroundRefresh: hasFetcher && refreshAheadSeconds > 0 && remainingFreshTtl > 0 && remainingFreshTtl <= refreshAheadSeconds
|
|
745
|
-
};
|
|
746
|
-
}
|
|
747
|
-
|
|
748
1142
|
// src/internal/CacheStackValidation.ts
|
|
749
1143
|
var MAX_CACHE_KEY_LENGTH = 1024;
|
|
750
1144
|
var MAX_PATTERN_LENGTH = 1024;
|
|
@@ -972,7 +1366,11 @@ var FetchRateLimiter = class {
|
|
|
972
1366
|
fetcherBuckets = /* @__PURE__ */ new WeakMap();
|
|
973
1367
|
nextFetcherBucketId = 0;
|
|
974
1368
|
drainTimer;
|
|
1369
|
+
isDisposed = false;
|
|
975
1370
|
async schedule(options, context, task) {
|
|
1371
|
+
if (this.isDisposed) {
|
|
1372
|
+
throw new Error("FetchRateLimiter has been disposed.");
|
|
1373
|
+
}
|
|
976
1374
|
if (!options) {
|
|
977
1375
|
return task();
|
|
978
1376
|
}
|
|
@@ -995,6 +1393,27 @@ var FetchRateLimiter = class {
|
|
|
995
1393
|
this.drain();
|
|
996
1394
|
});
|
|
997
1395
|
}
|
|
1396
|
+
dispose() {
|
|
1397
|
+
this.isDisposed = true;
|
|
1398
|
+
if (this.drainTimer) {
|
|
1399
|
+
clearTimeout(this.drainTimer);
|
|
1400
|
+
this.drainTimer = void 0;
|
|
1401
|
+
}
|
|
1402
|
+
for (const bucket of this.buckets.values()) {
|
|
1403
|
+
if (bucket.cleanupTimer) {
|
|
1404
|
+
clearTimeout(bucket.cleanupTimer);
|
|
1405
|
+
bucket.cleanupTimer = void 0;
|
|
1406
|
+
}
|
|
1407
|
+
}
|
|
1408
|
+
for (const queue of this.queuesByBucket.values()) {
|
|
1409
|
+
for (const item of queue) {
|
|
1410
|
+
item.reject(new Error("FetchRateLimiter has been disposed."));
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
this.queuesByBucket.clear();
|
|
1414
|
+
this.pendingBuckets.clear();
|
|
1415
|
+
this.buckets.clear();
|
|
1416
|
+
}
|
|
998
1417
|
normalize(options) {
|
|
999
1418
|
const maxConcurrent = options.maxConcurrent;
|
|
1000
1419
|
const intervalMs = options.intervalMs;
|
|
@@ -1030,6 +1449,9 @@ var FetchRateLimiter = class {
|
|
|
1030
1449
|
return "global";
|
|
1031
1450
|
}
|
|
1032
1451
|
drain() {
|
|
1452
|
+
if (this.isDisposed) {
|
|
1453
|
+
return;
|
|
1454
|
+
}
|
|
1033
1455
|
if (this.drainTimer) {
|
|
1034
1456
|
clearTimeout(this.drainTimer);
|
|
1035
1457
|
this.drainTimer = void 0;
|
|
@@ -1093,7 +1515,13 @@ var FetchRateLimiter = class {
|
|
|
1093
1515
|
this.pendingBuckets.add(next.bucketKey);
|
|
1094
1516
|
}
|
|
1095
1517
|
this.cleanupBucket(next.bucketKey, bucket, next.options.intervalMs);
|
|
1096
|
-
this.
|
|
1518
|
+
if (!this.drainTimer) {
|
|
1519
|
+
this.drainTimer = setTimeout(() => {
|
|
1520
|
+
this.drainTimer = void 0;
|
|
1521
|
+
this.drain();
|
|
1522
|
+
}, 0);
|
|
1523
|
+
this.drainTimer.unref?.();
|
|
1524
|
+
}
|
|
1097
1525
|
});
|
|
1098
1526
|
}
|
|
1099
1527
|
}
|
|
@@ -1126,12 +1554,18 @@ var FetchRateLimiter = class {
|
|
|
1126
1554
|
}
|
|
1127
1555
|
}
|
|
1128
1556
|
bucketState(bucketKey) {
|
|
1557
|
+
if (this.isDisposed) {
|
|
1558
|
+
throw new Error("FetchRateLimiter has been disposed.");
|
|
1559
|
+
}
|
|
1129
1560
|
const existing = this.buckets.get(bucketKey);
|
|
1130
1561
|
if (existing) {
|
|
1131
1562
|
return existing;
|
|
1132
1563
|
}
|
|
1133
1564
|
if (this.buckets.size >= MAX_BUCKETS) {
|
|
1134
1565
|
this.evictIdleBuckets();
|
|
1566
|
+
if (this.buckets.size >= MAX_BUCKETS) {
|
|
1567
|
+
throw new Error(`FetchRateLimiter bucket limit (${MAX_BUCKETS}) exceeded.`);
|
|
1568
|
+
}
|
|
1135
1569
|
}
|
|
1136
1570
|
const bucket = { active: 0, startedAt: [] };
|
|
1137
1571
|
this.buckets.set(bucketKey, bucket);
|
|
@@ -1362,44 +1796,19 @@ var TtlResolver = class {
|
|
|
1362
1796
|
};
|
|
1363
1797
|
|
|
1364
1798
|
// src/serialization/JsonSerializer.ts
|
|
1365
|
-
var DANGEROUS_JSON_KEYS = /* @__PURE__ */ new Set(["__proto__", "prototype", "constructor"]);
|
|
1366
|
-
var MAX_SANITIZE_NODES = 1e4;
|
|
1367
1799
|
var JsonSerializer = class {
|
|
1368
1800
|
serialize(value) {
|
|
1369
1801
|
return JSON.stringify(value);
|
|
1370
1802
|
}
|
|
1371
1803
|
deserialize(payload) {
|
|
1372
1804
|
const normalized = Buffer.isBuffer(payload) ? payload.toString("utf8") : payload;
|
|
1373
|
-
return
|
|
1805
|
+
return sanitizeStructuredData(JSON.parse(normalized), {
|
|
1806
|
+
label: "JSON payload",
|
|
1807
|
+
maxDepth: 200,
|
|
1808
|
+
maxNodes: 1e4
|
|
1809
|
+
});
|
|
1374
1810
|
}
|
|
1375
1811
|
};
|
|
1376
|
-
var MAX_SANITIZE_DEPTH = 200;
|
|
1377
|
-
function sanitizeJsonValue(value, depth, state) {
|
|
1378
|
-
state.count += 1;
|
|
1379
|
-
if (state.count > MAX_SANITIZE_NODES) {
|
|
1380
|
-
throw new Error(`JSON payload exceeds max node count of ${MAX_SANITIZE_NODES}.`);
|
|
1381
|
-
}
|
|
1382
|
-
if (depth > MAX_SANITIZE_DEPTH) {
|
|
1383
|
-
throw new Error(`JSON payload exceeds max depth of ${MAX_SANITIZE_DEPTH}.`);
|
|
1384
|
-
}
|
|
1385
|
-
if (Array.isArray(value)) {
|
|
1386
|
-
return value.map((entry) => sanitizeJsonValue(entry, depth + 1, state));
|
|
1387
|
-
}
|
|
1388
|
-
if (!isPlainObject(value)) {
|
|
1389
|
-
return value;
|
|
1390
|
-
}
|
|
1391
|
-
const sanitized = {};
|
|
1392
|
-
for (const [key, entry] of Object.entries(value)) {
|
|
1393
|
-
if (DANGEROUS_JSON_KEYS.has(key)) {
|
|
1394
|
-
continue;
|
|
1395
|
-
}
|
|
1396
|
-
sanitized[key] = sanitizeJsonValue(entry, depth + 1, state);
|
|
1397
|
-
}
|
|
1398
|
-
return sanitized;
|
|
1399
|
-
}
|
|
1400
|
-
function isPlainObject(value) {
|
|
1401
|
-
return Object.prototype.toString.call(value) === "[object Object]";
|
|
1402
|
-
}
|
|
1403
1812
|
|
|
1404
1813
|
// src/stampede/StampedeGuard.ts
|
|
1405
1814
|
import { Mutex as Mutex2 } from "async-mutex";
|
|
@@ -1445,7 +1854,6 @@ var DEFAULT_SINGLE_FLIGHT_POLL_MS = 50;
|
|
|
1445
1854
|
var DEFAULT_BACKGROUND_REFRESH_TIMEOUT_MS = 3e4;
|
|
1446
1855
|
var DEFAULT_SNAPSHOT_MAX_BYTES = 16 * 1024 * 1024;
|
|
1447
1856
|
var DEFAULT_SNAPSHOT_MAX_ENTRIES = 1e4;
|
|
1448
|
-
var DEFAULT_SNAPSHOT_IMPORT_BATCH_SIZE = 50;
|
|
1449
1857
|
var DEFAULT_INVALIDATION_MAX_KEYS = 1e4;
|
|
1450
1858
|
var DEFAULT_MAX_PROFILE_ENTRIES = 1e5;
|
|
1451
1859
|
var DebugLogger = class {
|
|
@@ -1502,6 +1910,35 @@ var CacheStack = class extends EventEmitter {
|
|
|
1502
1910
|
await this.handleLayerFailure(layer, operation, error);
|
|
1503
1911
|
}
|
|
1504
1912
|
});
|
|
1913
|
+
this.invalidation = new CacheStackInvalidationSupport({
|
|
1914
|
+
tagIndex: this.tagIndex,
|
|
1915
|
+
shouldSkipLayer: (layer) => this.shouldSkipLayer(layer),
|
|
1916
|
+
handleLayerFailure: async (layer, operation, error) => {
|
|
1917
|
+
await this.handleLayerFailure(layer, operation, error);
|
|
1918
|
+
}
|
|
1919
|
+
});
|
|
1920
|
+
this.layerWriter = new CacheStackLayerWriter({
|
|
1921
|
+
layers: this.layers,
|
|
1922
|
+
maintenance: this.maintenance,
|
|
1923
|
+
shouldSkipLayer: (layer) => this.shouldSkipLayer(layer),
|
|
1924
|
+
shouldWriteBehind: (layer) => this.shouldWriteBehind(layer),
|
|
1925
|
+
handleLayerFailure: async (layer, operation, error) => {
|
|
1926
|
+
await this.handleLayerFailure(layer, operation, error);
|
|
1927
|
+
},
|
|
1928
|
+
enqueueWriteBehind: this.enqueueWriteBehind.bind(this),
|
|
1929
|
+
resolveFreshTtl: this.resolveFreshTtl.bind(this),
|
|
1930
|
+
resolveLayerSeconds: this.resolveLayerSeconds.bind(this),
|
|
1931
|
+
globalStaleWhileRevalidate: this.options.staleWhileRevalidate,
|
|
1932
|
+
globalStaleIfError: this.options.staleIfError,
|
|
1933
|
+
writePolicy: this.options.writePolicy,
|
|
1934
|
+
onWriteFailures: (context, failures) => {
|
|
1935
|
+
this.metricsCollector.increment("writeFailures", failures.length);
|
|
1936
|
+
this.logger.debug?.("write-failure", {
|
|
1937
|
+
...context,
|
|
1938
|
+
failures: failures.map((failure) => this.formatError(failure))
|
|
1939
|
+
});
|
|
1940
|
+
}
|
|
1941
|
+
});
|
|
1505
1942
|
if (!options.tagIndex && layers.some((layer) => layer.isLocal === false)) {
|
|
1506
1943
|
this.logger.warn?.(
|
|
1507
1944
|
"Using the default in-memory TagIndex with a shared cache layer only tracks keys seen by this process. Use RedisTagIndex for cross-instance tag invalidation."
|
|
@@ -1517,6 +1954,18 @@ var CacheStack = class extends EventEmitter {
|
|
|
1517
1954
|
"broadcastL1Invalidation defaults to false when an invalidation bus is configured; opt in explicitly if write-triggered L1 invalidation is desired."
|
|
1518
1955
|
);
|
|
1519
1956
|
}
|
|
1957
|
+
this.snapshots = new CacheStackSnapshotManager({
|
|
1958
|
+
layers: this.layers,
|
|
1959
|
+
tagIndex: this.tagIndex,
|
|
1960
|
+
snapshotSerializer: this.snapshotSerializer,
|
|
1961
|
+
readLayerEntry: this.readLayerEntry.bind(this),
|
|
1962
|
+
shouldSkipLayer: (layer) => this.shouldSkipLayer(layer),
|
|
1963
|
+
handleLayerFailure: async (layer, operation, error) => this.handleLayerFailure(layer, operation, error),
|
|
1964
|
+
qualifyKey: this.qualifyKey.bind(this),
|
|
1965
|
+
stripQualifiedKey: this.stripQualifiedKey.bind(this),
|
|
1966
|
+
validateCacheKey,
|
|
1967
|
+
formatError: this.formatError.bind(this)
|
|
1968
|
+
});
|
|
1520
1969
|
this.initializeWriteBehind(options.writeBehind);
|
|
1521
1970
|
this.startup = this.initialize();
|
|
1522
1971
|
}
|
|
@@ -1532,11 +1981,16 @@ var CacheStack = class extends EventEmitter {
|
|
|
1532
1981
|
keyDiscovery;
|
|
1533
1982
|
fetchRateLimiter = new FetchRateLimiter();
|
|
1534
1983
|
snapshotSerializer = new JsonSerializer();
|
|
1984
|
+
invalidation;
|
|
1985
|
+
layerWriter;
|
|
1986
|
+
snapshots;
|
|
1535
1987
|
backgroundRefreshes = /* @__PURE__ */ new Map();
|
|
1988
|
+
backgroundRefreshAbort = /* @__PURE__ */ new Map();
|
|
1536
1989
|
layerDegradedUntil = /* @__PURE__ */ new Map();
|
|
1537
1990
|
maintenance = new CacheStackMaintenance();
|
|
1538
1991
|
ttlResolver;
|
|
1539
1992
|
circuitBreakerManager;
|
|
1993
|
+
nextOperationId = 0;
|
|
1540
1994
|
currentGeneration;
|
|
1541
1995
|
isDisconnecting = false;
|
|
1542
1996
|
disconnectPromise;
|
|
@@ -1547,10 +2001,12 @@ var CacheStack = class extends EventEmitter {
|
|
|
1547
2001
|
* and no `fetcher` is provided.
|
|
1548
2002
|
*/
|
|
1549
2003
|
async get(key, fetcher, options) {
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
2004
|
+
return this.observeOperation("layercache.get", { "layercache.key": String(key ?? "") }, async () => {
|
|
2005
|
+
const normalizedKey = this.qualifyKey(validateCacheKey(key));
|
|
2006
|
+
this.validateWriteOptions(options);
|
|
2007
|
+
await this.awaitStartup("get");
|
|
2008
|
+
return this.getPrepared(normalizedKey, fetcher, options);
|
|
2009
|
+
});
|
|
1554
2010
|
}
|
|
1555
2011
|
async getPrepared(normalizedKey, fetcher, options) {
|
|
1556
2012
|
const hit = await this.readFromLayers(normalizedKey, options, "allow-stale");
|
|
@@ -1672,23 +2128,27 @@ var CacheStack = class extends EventEmitter {
|
|
|
1672
2128
|
* Stores a value in all cache layers. Overwrites any existing value.
|
|
1673
2129
|
*/
|
|
1674
2130
|
async set(key, value, options) {
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
2131
|
+
await this.observeOperation("layercache.set", { "layercache.key": String(key ?? "") }, async () => {
|
|
2132
|
+
const normalizedKey = this.qualifyKey(validateCacheKey(key));
|
|
2133
|
+
this.validateWriteOptions(options);
|
|
2134
|
+
await this.awaitStartup("set");
|
|
2135
|
+
await this.storeEntry(normalizedKey, "value", value, options);
|
|
2136
|
+
});
|
|
1679
2137
|
}
|
|
1680
2138
|
/**
|
|
1681
2139
|
* Deletes the key from all layers and publishes an invalidation message.
|
|
1682
2140
|
*/
|
|
1683
2141
|
async delete(key) {
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
2142
|
+
await this.observeOperation("layercache.delete", { "layercache.key": String(key ?? "") }, async () => {
|
|
2143
|
+
const normalizedKey = this.qualifyKey(validateCacheKey(key));
|
|
2144
|
+
await this.awaitStartup("delete");
|
|
2145
|
+
await this.deleteKeys([normalizedKey]);
|
|
2146
|
+
await this.publishInvalidation({
|
|
2147
|
+
scope: "key",
|
|
2148
|
+
keys: [normalizedKey],
|
|
2149
|
+
sourceId: this.instanceId,
|
|
2150
|
+
operation: "delete"
|
|
2151
|
+
});
|
|
1692
2152
|
});
|
|
1693
2153
|
}
|
|
1694
2154
|
async clear() {
|
|
@@ -1721,95 +2181,102 @@ var CacheStack = class extends EventEmitter {
|
|
|
1721
2181
|
});
|
|
1722
2182
|
}
|
|
1723
2183
|
async mget(entries) {
|
|
1724
|
-
this.
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
2184
|
+
return this.observeOperation("layercache.mget", void 0, async () => {
|
|
2185
|
+
this.assertActive("mget");
|
|
2186
|
+
if (entries.length === 0) {
|
|
2187
|
+
return [];
|
|
2188
|
+
}
|
|
2189
|
+
const normalizedEntries = entries.map((entry) => ({
|
|
2190
|
+
...entry,
|
|
2191
|
+
key: this.qualifyKey(validateCacheKey(entry.key))
|
|
2192
|
+
}));
|
|
2193
|
+
normalizedEntries.forEach((entry) => this.validateWriteOptions(entry.options));
|
|
2194
|
+
const canFastPath = normalizedEntries.every((entry) => entry.fetch === void 0 && entry.options === void 0);
|
|
2195
|
+
if (!canFastPath) {
|
|
2196
|
+
await this.awaitStartup("mget");
|
|
2197
|
+
const pendingReads = /* @__PURE__ */ new Map();
|
|
2198
|
+
return Promise.all(
|
|
2199
|
+
normalizedEntries.map((entry) => {
|
|
2200
|
+
const optionsSignature = serializeOptions(entry.options);
|
|
2201
|
+
const existing = pendingReads.get(entry.key);
|
|
2202
|
+
if (!existing) {
|
|
2203
|
+
const promise = this.getPrepared(entry.key, entry.fetch, entry.options);
|
|
2204
|
+
pendingReads.set(entry.key, {
|
|
2205
|
+
promise,
|
|
2206
|
+
fetch: entry.fetch,
|
|
2207
|
+
optionsSignature
|
|
2208
|
+
});
|
|
2209
|
+
return promise;
|
|
2210
|
+
}
|
|
2211
|
+
if (existing.fetch !== entry.fetch || existing.optionsSignature !== optionsSignature) {
|
|
2212
|
+
throw new Error(`mget received conflicting entries for key "${entry.key}".`);
|
|
2213
|
+
}
|
|
2214
|
+
return existing.promise;
|
|
2215
|
+
})
|
|
2216
|
+
);
|
|
2217
|
+
}
|
|
1735
2218
|
await this.awaitStartup("mget");
|
|
1736
|
-
const
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
});
|
|
1748
|
-
return promise;
|
|
1749
|
-
}
|
|
1750
|
-
if (existing.fetch !== entry.fetch || existing.optionsSignature !== optionsSignature) {
|
|
1751
|
-
throw new Error(`mget received conflicting entries for key "${entry.key}".`);
|
|
1752
|
-
}
|
|
1753
|
-
return existing.promise;
|
|
1754
|
-
})
|
|
1755
|
-
);
|
|
1756
|
-
}
|
|
1757
|
-
await this.awaitStartup("mget");
|
|
1758
|
-
const pending = /* @__PURE__ */ new Set();
|
|
1759
|
-
const indexesByKey = /* @__PURE__ */ new Map();
|
|
1760
|
-
const resultsByKey = /* @__PURE__ */ new Map();
|
|
1761
|
-
for (let index = 0; index < normalizedEntries.length; index += 1) {
|
|
1762
|
-
const entry = normalizedEntries[index];
|
|
1763
|
-
if (!entry) continue;
|
|
1764
|
-
const key = entry.key;
|
|
1765
|
-
const indexes = indexesByKey.get(key) ?? [];
|
|
1766
|
-
indexes.push(index);
|
|
1767
|
-
indexesByKey.set(key, indexes);
|
|
1768
|
-
pending.add(key);
|
|
1769
|
-
}
|
|
1770
|
-
for (let layerIndex = 0; layerIndex < this.layers.length; layerIndex += 1) {
|
|
1771
|
-
const layer = this.layers[layerIndex];
|
|
1772
|
-
if (!layer) continue;
|
|
1773
|
-
const keys = [...pending];
|
|
1774
|
-
if (keys.length === 0) {
|
|
1775
|
-
break;
|
|
2219
|
+
const pending = /* @__PURE__ */ new Set();
|
|
2220
|
+
const indexesByKey = /* @__PURE__ */ new Map();
|
|
2221
|
+
const resultsByKey = /* @__PURE__ */ new Map();
|
|
2222
|
+
for (let index = 0; index < normalizedEntries.length; index += 1) {
|
|
2223
|
+
const entry = normalizedEntries[index];
|
|
2224
|
+
if (!entry) continue;
|
|
2225
|
+
const key = entry.key;
|
|
2226
|
+
const indexes = indexesByKey.get(key) ?? [];
|
|
2227
|
+
indexes.push(index);
|
|
2228
|
+
indexesByKey.set(key, indexes);
|
|
2229
|
+
pending.add(key);
|
|
1776
2230
|
}
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
const
|
|
1781
|
-
if (
|
|
1782
|
-
|
|
2231
|
+
for (let layerIndex = 0; layerIndex < this.layers.length; layerIndex += 1) {
|
|
2232
|
+
const layer = this.layers[layerIndex];
|
|
2233
|
+
if (!layer || this.shouldSkipLayer(layer)) continue;
|
|
2234
|
+
const keys = [...pending];
|
|
2235
|
+
if (keys.length === 0) {
|
|
2236
|
+
break;
|
|
1783
2237
|
}
|
|
1784
|
-
const
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
2238
|
+
const values = layer.getMany ? await layer.getMany(keys) : await Promise.all(keys.map((key) => this.readLayerEntry(layer, key)));
|
|
2239
|
+
for (let offset = 0; offset < values.length; offset += 1) {
|
|
2240
|
+
const key = keys[offset];
|
|
2241
|
+
const stored = values[offset];
|
|
2242
|
+
if (!key || stored === null) {
|
|
2243
|
+
continue;
|
|
2244
|
+
}
|
|
2245
|
+
const resolved = resolveStoredValue(stored);
|
|
2246
|
+
if (resolved.state === "expired") {
|
|
2247
|
+
await layer.delete(key);
|
|
2248
|
+
continue;
|
|
2249
|
+
}
|
|
2250
|
+
if (resolved.state === "stale-while-revalidate" || resolved.state === "stale-if-error") {
|
|
2251
|
+
this.metricsCollector.increment("staleHits", indexesByKey.get(key)?.length ?? 1);
|
|
2252
|
+
}
|
|
2253
|
+
await this.tagIndex.touch(key);
|
|
2254
|
+
await this.backfill(key, stored, layerIndex - 1);
|
|
2255
|
+
resultsByKey.set(key, resolved.value);
|
|
2256
|
+
pending.delete(key);
|
|
2257
|
+
this.metricsCollector.increment("hits", indexesByKey.get(key)?.length ?? 1);
|
|
1788
2258
|
}
|
|
1789
|
-
await this.tagIndex.touch(key);
|
|
1790
|
-
await this.backfill(key, stored, layerIndex - 1);
|
|
1791
|
-
resultsByKey.set(key, resolved.value);
|
|
1792
|
-
pending.delete(key);
|
|
1793
|
-
this.metricsCollector.increment("hits", indexesByKey.get(key)?.length ?? 1);
|
|
1794
2259
|
}
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
2260
|
+
if (pending.size > 0) {
|
|
2261
|
+
for (const key of pending) {
|
|
2262
|
+
await this.tagIndex.remove(key);
|
|
2263
|
+
this.metricsCollector.increment("misses", indexesByKey.get(key)?.length ?? 1);
|
|
2264
|
+
}
|
|
1800
2265
|
}
|
|
1801
|
-
|
|
1802
|
-
|
|
2266
|
+
return normalizedEntries.map((entry) => resultsByKey.get(entry.key) ?? null);
|
|
2267
|
+
});
|
|
1803
2268
|
}
|
|
1804
2269
|
async mset(entries) {
|
|
1805
|
-
this.
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
2270
|
+
await this.observeOperation("layercache.mset", void 0, async () => {
|
|
2271
|
+
this.assertActive("mset");
|
|
2272
|
+
const normalizedEntries = entries.map((entry) => ({
|
|
2273
|
+
...entry,
|
|
2274
|
+
key: this.qualifyKey(validateCacheKey(entry.key))
|
|
2275
|
+
}));
|
|
2276
|
+
normalizedEntries.forEach((entry) => this.validateWriteOptions(entry.options));
|
|
2277
|
+
await this.awaitStartup("mset");
|
|
2278
|
+
await this.writeBatch(normalizedEntries);
|
|
2279
|
+
});
|
|
1813
2280
|
}
|
|
1814
2281
|
async warm(entries, options = {}) {
|
|
1815
2282
|
this.assertActive("warm");
|
|
@@ -1862,40 +2329,50 @@ var CacheStack = class extends EventEmitter {
|
|
|
1862
2329
|
return new CacheNamespace(this, prefix);
|
|
1863
2330
|
}
|
|
1864
2331
|
async invalidateByTag(tag) {
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
2332
|
+
await this.observeOperation("layercache.invalidate_by_tag", void 0, async () => {
|
|
2333
|
+
validateTag(tag);
|
|
2334
|
+
await this.awaitStartup("invalidateByTag");
|
|
2335
|
+
const keys = await this.invalidation.collectKeysForTag(tag, this.invalidationMaxKeys());
|
|
2336
|
+
await this.deleteKeys(keys);
|
|
2337
|
+
await this.publishInvalidation({ scope: "keys", keys, sourceId: this.instanceId, operation: "invalidate" });
|
|
2338
|
+
});
|
|
1870
2339
|
}
|
|
1871
2340
|
async invalidateByTags(tags, mode = "any") {
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
2341
|
+
await this.observeOperation("layercache.invalidate_by_tags", void 0, async () => {
|
|
2342
|
+
if (tags.length === 0) {
|
|
2343
|
+
return;
|
|
2344
|
+
}
|
|
2345
|
+
validateTags(tags);
|
|
2346
|
+
await this.awaitStartup("invalidateByTags");
|
|
2347
|
+
const keysByTag = await Promise.all(
|
|
2348
|
+
tags.map((tag) => this.invalidation.collectKeysForTag(tag, this.invalidationMaxKeys()))
|
|
2349
|
+
);
|
|
2350
|
+
const keys = mode === "all" ? this.invalidation.intersectKeys(keysByTag) : [...new Set(keysByTag.flat())];
|
|
2351
|
+
this.invalidation.assertWithinInvalidationKeyLimit(keys.length, this.invalidationMaxKeys());
|
|
2352
|
+
await this.deleteKeys(keys);
|
|
2353
|
+
await this.publishInvalidation({ scope: "keys", keys, sourceId: this.instanceId, operation: "invalidate" });
|
|
2354
|
+
});
|
|
1882
2355
|
}
|
|
1883
2356
|
async invalidateByPattern(pattern) {
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
this.
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
2357
|
+
await this.observeOperation("layercache.invalidate_by_pattern", void 0, async () => {
|
|
2358
|
+
validatePattern(pattern);
|
|
2359
|
+
await this.awaitStartup("invalidateByPattern");
|
|
2360
|
+
const keys = await this.keyDiscovery.collectKeysMatchingPattern(
|
|
2361
|
+
this.qualifyPattern(pattern),
|
|
2362
|
+
this.invalidationMaxKeys()
|
|
2363
|
+
);
|
|
2364
|
+
await this.deleteKeys(keys);
|
|
2365
|
+
await this.publishInvalidation({ scope: "keys", keys, sourceId: this.instanceId, operation: "invalidate" });
|
|
2366
|
+
});
|
|
1892
2367
|
}
|
|
1893
2368
|
async invalidateByPrefix(prefix) {
|
|
1894
|
-
await this.
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
2369
|
+
await this.observeOperation("layercache.invalidate_by_prefix", void 0, async () => {
|
|
2370
|
+
await this.awaitStartup("invalidateByPrefix");
|
|
2371
|
+
const qualifiedPrefix = this.qualifyKey(validateCacheKey(prefix));
|
|
2372
|
+
const keys = await this.keyDiscovery.collectKeysWithPrefix(qualifiedPrefix, this.invalidationMaxKeys());
|
|
2373
|
+
await this.deleteKeys(keys);
|
|
2374
|
+
await this.publishInvalidation({ scope: "keys", keys, sourceId: this.instanceId, operation: "invalidate" });
|
|
2375
|
+
});
|
|
1899
2376
|
}
|
|
1900
2377
|
getMetrics() {
|
|
1901
2378
|
return this.metricsCollector.snapshot;
|
|
@@ -2006,95 +2483,19 @@ var CacheStack = class extends EventEmitter {
|
|
|
2006
2483
|
}
|
|
2007
2484
|
async exportState() {
|
|
2008
2485
|
await this.awaitStartup("exportState");
|
|
2009
|
-
|
|
2010
|
-
await this.visitExportEntries(this.snapshotMaxEntries(), async (entry) => {
|
|
2011
|
-
entries.push(entry);
|
|
2012
|
-
});
|
|
2013
|
-
return entries;
|
|
2486
|
+
return this.snapshots.exportState(this.snapshotMaxEntries());
|
|
2014
2487
|
}
|
|
2015
2488
|
async importState(entries) {
|
|
2016
2489
|
await this.awaitStartup("importState");
|
|
2017
|
-
|
|
2018
|
-
key: this.qualifyKey(validateCacheKey(entry.key)),
|
|
2019
|
-
value: entry.value,
|
|
2020
|
-
ttl: entry.ttl
|
|
2021
|
-
}));
|
|
2022
|
-
for (let index = 0; index < normalizedEntries.length; index += DEFAULT_SNAPSHOT_IMPORT_BATCH_SIZE) {
|
|
2023
|
-
const batch = normalizedEntries.slice(index, index + DEFAULT_SNAPSHOT_IMPORT_BATCH_SIZE);
|
|
2024
|
-
await Promise.all(
|
|
2025
|
-
batch.map(async (entry) => {
|
|
2026
|
-
await Promise.all(this.layers.map((layer) => layer.set(entry.key, entry.value, entry.ttl)));
|
|
2027
|
-
await this.tagIndex.touch(entry.key);
|
|
2028
|
-
})
|
|
2029
|
-
);
|
|
2030
|
-
}
|
|
2490
|
+
await this.snapshots.importState(entries);
|
|
2031
2491
|
}
|
|
2032
2492
|
async persistToFile(filePath) {
|
|
2033
2493
|
this.assertActive("persistToFile");
|
|
2034
|
-
|
|
2035
|
-
const path = await import("path");
|
|
2036
|
-
const targetPath = await validateSnapshotFilePath(filePath, "write", this.options.snapshotBaseDir);
|
|
2037
|
-
const tempPath = path.join(
|
|
2038
|
-
path.dirname(targetPath),
|
|
2039
|
-
`.layercache-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.tmp`
|
|
2040
|
-
);
|
|
2041
|
-
let handle;
|
|
2042
|
-
try {
|
|
2043
|
-
handle = await fs2.open(tempPath, "wx");
|
|
2044
|
-
const openedHandle = handle;
|
|
2045
|
-
await openedHandle.writeFile("[", "utf8");
|
|
2046
|
-
let wroteAny = false;
|
|
2047
|
-
await this.visitExportEntries(this.snapshotMaxEntries(), async (entry) => {
|
|
2048
|
-
await openedHandle.writeFile(wroteAny ? ",\n" : "\n", "utf8");
|
|
2049
|
-
await openedHandle.writeFile(JSON.stringify(entry, null, 2), "utf8");
|
|
2050
|
-
wroteAny = true;
|
|
2051
|
-
});
|
|
2052
|
-
await openedHandle.writeFile(wroteAny ? "\n]" : "]", "utf8");
|
|
2053
|
-
await openedHandle.close();
|
|
2054
|
-
handle = void 0;
|
|
2055
|
-
await fs2.rename(tempPath, targetPath);
|
|
2056
|
-
} catch (error) {
|
|
2057
|
-
await handle?.close().catch(() => void 0);
|
|
2058
|
-
await fs2.unlink(tempPath).catch(() => void 0);
|
|
2059
|
-
throw error;
|
|
2060
|
-
}
|
|
2494
|
+
await this.snapshots.persistToFile(filePath, this.options.snapshotBaseDir, this.snapshotMaxEntries());
|
|
2061
2495
|
}
|
|
2062
2496
|
async restoreFromFile(filePath) {
|
|
2063
2497
|
this.assertActive("restoreFromFile");
|
|
2064
|
-
|
|
2065
|
-
const validatedPath = await validateSnapshotFilePath(filePath, "read", this.options.snapshotBaseDir);
|
|
2066
|
-
const handle = await fs2.open(validatedPath, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
|
|
2067
|
-
const snapshotMaxBytes = this.snapshotMaxBytes();
|
|
2068
|
-
let raw;
|
|
2069
|
-
try {
|
|
2070
|
-
if (snapshotMaxBytes !== false) {
|
|
2071
|
-
const stat = await handle.stat();
|
|
2072
|
-
if (stat.size > snapshotMaxBytes) {
|
|
2073
|
-
throw new Error(
|
|
2074
|
-
`Snapshot file exceeds snapshotMaxBytes limit (${stat.size} bytes > ${snapshotMaxBytes} bytes).`
|
|
2075
|
-
);
|
|
2076
|
-
}
|
|
2077
|
-
}
|
|
2078
|
-
raw = await readUtf8HandleWithLimit(handle, snapshotMaxBytes);
|
|
2079
|
-
} finally {
|
|
2080
|
-
await handle.close();
|
|
2081
|
-
}
|
|
2082
|
-
let parsed;
|
|
2083
|
-
try {
|
|
2084
|
-
parsed = JSON.parse(raw);
|
|
2085
|
-
} catch (cause) {
|
|
2086
|
-
throw new Error(`Invalid snapshot file: could not parse JSON (${this.formatError(cause)})`);
|
|
2087
|
-
}
|
|
2088
|
-
if (!this.isCacheSnapshotEntries(parsed)) {
|
|
2089
|
-
throw new Error("Invalid snapshot file: expected an array of { key: string, value, ttl? } entries");
|
|
2090
|
-
}
|
|
2091
|
-
await this.importState(
|
|
2092
|
-
parsed.map((entry) => ({
|
|
2093
|
-
key: entry.key,
|
|
2094
|
-
value: this.sanitizeSnapshotValue(entry.value),
|
|
2095
|
-
ttl: entry.ttl
|
|
2096
|
-
}))
|
|
2097
|
-
);
|
|
2498
|
+
await this.snapshots.restoreFromFile(filePath, this.options.snapshotBaseDir, this.snapshotMaxBytes());
|
|
2098
2499
|
}
|
|
2099
2500
|
async disconnect() {
|
|
2100
2501
|
if (!this.disconnectPromise) {
|
|
@@ -2104,8 +2505,27 @@ var CacheStack = class extends EventEmitter {
|
|
|
2104
2505
|
await this.unsubscribeInvalidation?.();
|
|
2105
2506
|
await this.flushWriteBehindQueue();
|
|
2106
2507
|
await this.maintenance.waitForGenerationCleanup();
|
|
2107
|
-
|
|
2508
|
+
for (const key of this.backgroundRefreshAbort.keys()) {
|
|
2509
|
+
this.backgroundRefreshAbort.set(key, true);
|
|
2510
|
+
}
|
|
2511
|
+
await Promise.allSettled(
|
|
2512
|
+
[...this.backgroundRefreshes.values()].map((promise) => {
|
|
2513
|
+
let timer;
|
|
2514
|
+
return Promise.race([
|
|
2515
|
+
promise,
|
|
2516
|
+
new Promise((resolve2) => {
|
|
2517
|
+
timer = setTimeout(resolve2, 5e3);
|
|
2518
|
+
timer.unref?.();
|
|
2519
|
+
})
|
|
2520
|
+
]).finally(() => {
|
|
2521
|
+
if (timer) clearTimeout(timer);
|
|
2522
|
+
});
|
|
2523
|
+
})
|
|
2524
|
+
);
|
|
2525
|
+
this.backgroundRefreshes.clear();
|
|
2526
|
+
this.backgroundRefreshAbort.clear();
|
|
2108
2527
|
this.maintenance.disposeWriteBehindTimer();
|
|
2528
|
+
this.fetchRateLimiter.dispose();
|
|
2109
2529
|
await Promise.allSettled(this.layers.map((layer) => layer.dispose?.() ?? Promise.resolve()));
|
|
2110
2530
|
})();
|
|
2111
2531
|
}
|
|
@@ -2219,7 +2639,7 @@ var CacheStack = class extends EventEmitter {
|
|
|
2219
2639
|
async storeEntry(key, kind, value, options) {
|
|
2220
2640
|
const clearEpoch = this.maintenance.currentClearEpoch();
|
|
2221
2641
|
const keyEpoch = this.maintenance.currentKeyEpoch(key);
|
|
2222
|
-
await this.writeAcrossLayers(key, kind, value, options);
|
|
2642
|
+
await this.layerWriter.writeAcrossLayers(key, kind, value, options);
|
|
2223
2643
|
if (this.maintenance.isWriteOutdated(key, clearEpoch, keyEpoch)) {
|
|
2224
2644
|
return;
|
|
2225
2645
|
}
|
|
@@ -2236,52 +2656,7 @@ var CacheStack = class extends EventEmitter {
|
|
|
2236
2656
|
}
|
|
2237
2657
|
}
|
|
2238
2658
|
async writeBatch(entries) {
|
|
2239
|
-
const
|
|
2240
|
-
const clearEpoch = this.maintenance.currentClearEpoch();
|
|
2241
|
-
const entryEpochs = new Map(entries.map((entry) => [entry.key, this.maintenance.currentKeyEpoch(entry.key)]));
|
|
2242
|
-
const entriesByLayer = /* @__PURE__ */ new Map();
|
|
2243
|
-
const immediateOperations = [];
|
|
2244
|
-
const deferredOperations = [];
|
|
2245
|
-
for (const entry of entries) {
|
|
2246
|
-
for (const layer of this.layers) {
|
|
2247
|
-
if (this.shouldSkipLayer(layer)) {
|
|
2248
|
-
continue;
|
|
2249
|
-
}
|
|
2250
|
-
const layerEntry = this.buildLayerSetEntry(layer, entry.key, "value", entry.value, entry.options, now);
|
|
2251
|
-
const bucket = entriesByLayer.get(layer) ?? [];
|
|
2252
|
-
bucket.push(layerEntry);
|
|
2253
|
-
entriesByLayer.set(layer, bucket);
|
|
2254
|
-
}
|
|
2255
|
-
}
|
|
2256
|
-
for (const [layer, layerEntries] of entriesByLayer.entries()) {
|
|
2257
|
-
const operation = async () => {
|
|
2258
|
-
if (clearEpoch !== this.maintenance.currentClearEpoch()) {
|
|
2259
|
-
return;
|
|
2260
|
-
}
|
|
2261
|
-
const activeEntries = layerEntries.filter(
|
|
2262
|
-
(entry) => (entryEpochs.get(entry.key) ?? 0) === this.maintenance.currentKeyEpoch(entry.key)
|
|
2263
|
-
);
|
|
2264
|
-
if (activeEntries.length === 0) {
|
|
2265
|
-
return;
|
|
2266
|
-
}
|
|
2267
|
-
try {
|
|
2268
|
-
if (layer.setMany) {
|
|
2269
|
-
await layer.setMany(activeEntries);
|
|
2270
|
-
return;
|
|
2271
|
-
}
|
|
2272
|
-
await Promise.all(activeEntries.map((entry) => layer.set(entry.key, entry.value, entry.ttl)));
|
|
2273
|
-
} catch (error) {
|
|
2274
|
-
await this.handleLayerFailure(layer, "write", error);
|
|
2275
|
-
}
|
|
2276
|
-
};
|
|
2277
|
-
if (this.shouldWriteBehind(layer)) {
|
|
2278
|
-
deferredOperations.push(operation);
|
|
2279
|
-
} else {
|
|
2280
|
-
immediateOperations.push(operation);
|
|
2281
|
-
}
|
|
2282
|
-
}
|
|
2283
|
-
await this.executeLayerOperations(immediateOperations, { key: "batch", action: "mset" });
|
|
2284
|
-
await Promise.all(deferredOperations.map((operation) => this.enqueueWriteBehind(operation)));
|
|
2659
|
+
const { clearEpoch, entryEpochs } = await this.layerWriter.writeBatch(entries);
|
|
2285
2660
|
if (clearEpoch !== this.maintenance.currentClearEpoch()) {
|
|
2286
2661
|
return;
|
|
2287
2662
|
}
|
|
@@ -2388,58 +2763,6 @@ var CacheStack = class extends EventEmitter {
|
|
|
2388
2763
|
this.emit("backfill", { key, layer: layer.name });
|
|
2389
2764
|
}
|
|
2390
2765
|
}
|
|
2391
|
-
async writeAcrossLayers(key, kind, value, options) {
|
|
2392
|
-
const now = Date.now();
|
|
2393
|
-
const clearEpoch = this.maintenance.currentClearEpoch();
|
|
2394
|
-
const keyEpoch = this.maintenance.currentKeyEpoch(key);
|
|
2395
|
-
const immediateOperations = [];
|
|
2396
|
-
const deferredOperations = [];
|
|
2397
|
-
for (const layer of this.layers) {
|
|
2398
|
-
const operation = async () => {
|
|
2399
|
-
if (this.maintenance.isWriteOutdated(key, clearEpoch, keyEpoch)) {
|
|
2400
|
-
return;
|
|
2401
|
-
}
|
|
2402
|
-
if (this.shouldSkipLayer(layer)) {
|
|
2403
|
-
return;
|
|
2404
|
-
}
|
|
2405
|
-
const entry = this.buildLayerSetEntry(layer, key, kind, value, options, now);
|
|
2406
|
-
try {
|
|
2407
|
-
await layer.set(entry.key, entry.value, entry.ttl);
|
|
2408
|
-
} catch (error) {
|
|
2409
|
-
await this.handleLayerFailure(layer, "write", error);
|
|
2410
|
-
}
|
|
2411
|
-
};
|
|
2412
|
-
if (this.shouldWriteBehind(layer)) {
|
|
2413
|
-
deferredOperations.push(operation);
|
|
2414
|
-
} else {
|
|
2415
|
-
immediateOperations.push(operation);
|
|
2416
|
-
}
|
|
2417
|
-
}
|
|
2418
|
-
await this.executeLayerOperations(immediateOperations, { key, action: kind === "empty" ? "negative-set" : "set" });
|
|
2419
|
-
await Promise.all(deferredOperations.map((operation) => this.enqueueWriteBehind(operation)));
|
|
2420
|
-
}
|
|
2421
|
-
async executeLayerOperations(operations, context) {
|
|
2422
|
-
if (this.options.writePolicy !== "best-effort") {
|
|
2423
|
-
await Promise.all(operations.map((operation) => operation()));
|
|
2424
|
-
return;
|
|
2425
|
-
}
|
|
2426
|
-
const results = await Promise.allSettled(operations.map((operation) => operation()));
|
|
2427
|
-
const failures = results.filter((result) => result.status === "rejected");
|
|
2428
|
-
if (failures.length === 0) {
|
|
2429
|
-
return;
|
|
2430
|
-
}
|
|
2431
|
-
this.metricsCollector.increment("writeFailures", failures.length);
|
|
2432
|
-
this.logger.debug?.("write-failure", {
|
|
2433
|
-
...context,
|
|
2434
|
-
failures: failures.map((failure) => this.formatError(failure.reason))
|
|
2435
|
-
});
|
|
2436
|
-
if (failures.length === operations.length) {
|
|
2437
|
-
throw new AggregateError(
|
|
2438
|
-
failures.map((failure) => failure.reason),
|
|
2439
|
-
`${context.action} failed for every cache layer`
|
|
2440
|
-
);
|
|
2441
|
-
}
|
|
2442
|
-
}
|
|
2443
2766
|
resolveFreshTtl(key, layerName, kind, options, fallbackTtl, value) {
|
|
2444
2767
|
return this.ttlResolver.resolveFreshTtl(
|
|
2445
2768
|
key,
|
|
@@ -2467,15 +2790,19 @@ var CacheStack = class extends EventEmitter {
|
|
|
2467
2790
|
}
|
|
2468
2791
|
const clearEpoch = this.maintenance.currentClearEpoch();
|
|
2469
2792
|
const keyEpoch = this.maintenance.currentKeyEpoch(key);
|
|
2793
|
+
this.backgroundRefreshAbort.set(key, false);
|
|
2470
2794
|
const refresh = (async () => {
|
|
2471
2795
|
this.metricsCollector.increment("refreshes");
|
|
2472
2796
|
try {
|
|
2797
|
+
if (this.backgroundRefreshAbort.get(key)) return;
|
|
2473
2798
|
await this.runBackgroundRefresh(key, fetcher, options, clearEpoch, keyEpoch);
|
|
2474
2799
|
} catch (error) {
|
|
2800
|
+
if (this.backgroundRefreshAbort.get(key)) return;
|
|
2475
2801
|
this.metricsCollector.increment("refreshErrors");
|
|
2476
2802
|
this.logger.debug?.("refresh-error", { key, error: this.formatError(error) });
|
|
2477
2803
|
} finally {
|
|
2478
2804
|
this.backgroundRefreshes.delete(key);
|
|
2805
|
+
this.backgroundRefreshAbort.delete(key);
|
|
2479
2806
|
}
|
|
2480
2807
|
})();
|
|
2481
2808
|
this.backgroundRefreshes.set(key, refresh);
|
|
@@ -2505,7 +2832,7 @@ var CacheStack = class extends EventEmitter {
|
|
|
2505
2832
|
return;
|
|
2506
2833
|
}
|
|
2507
2834
|
this.maintenance.bumpKeyEpochs(keys);
|
|
2508
|
-
await this.deleteKeysFromLayers(this.layers, keys);
|
|
2835
|
+
await this.invalidation.deleteKeysFromLayers(this.layers, keys);
|
|
2509
2836
|
for (const key of keys) {
|
|
2510
2837
|
await this.tagIndex.remove(key);
|
|
2511
2838
|
this.ttlResolver.deleteProfile(key);
|
|
@@ -2537,7 +2864,7 @@ var CacheStack = class extends EventEmitter {
|
|
|
2537
2864
|
}
|
|
2538
2865
|
const keys = message.keys ?? [];
|
|
2539
2866
|
this.maintenance.bumpKeyEpochs(keys);
|
|
2540
|
-
await this.deleteKeysFromLayers(localLayers, keys);
|
|
2867
|
+
await this.invalidation.deleteKeysFromLayers(localLayers, keys);
|
|
2541
2868
|
if (message.operation !== "write") {
|
|
2542
2869
|
for (const key of keys) {
|
|
2543
2870
|
await this.tagIndex.remove(key);
|
|
@@ -2578,7 +2905,7 @@ var CacheStack = class extends EventEmitter {
|
|
|
2578
2905
|
timer.unref?.();
|
|
2579
2906
|
})
|
|
2580
2907
|
]);
|
|
2581
|
-
if (result && typeof result === "object" && "kind" in result) {
|
|
2908
|
+
if (result !== null && result !== void 0 && typeof result === "object" && "kind" in result) {
|
|
2582
2909
|
if (result.kind === "error") {
|
|
2583
2910
|
throw result.error;
|
|
2584
2911
|
}
|
|
@@ -2594,6 +2921,31 @@ var CacheStack = class extends EventEmitter {
|
|
|
2594
2921
|
shouldBroadcastL1Invalidation() {
|
|
2595
2922
|
return this.options.broadcastL1Invalidation ?? this.options.publishSetInvalidation ?? false;
|
|
2596
2923
|
}
|
|
2924
|
+
async observeOperation(name, attributes, execute) {
|
|
2925
|
+
const id = this.nextOperationId;
|
|
2926
|
+
this.nextOperationId = (this.nextOperationId + 1) % Number.MAX_SAFE_INTEGER;
|
|
2927
|
+
this.emit("operation-start", { id, name, attributes });
|
|
2928
|
+
try {
|
|
2929
|
+
const result = await execute();
|
|
2930
|
+
this.emit("operation-end", {
|
|
2931
|
+
id,
|
|
2932
|
+
name,
|
|
2933
|
+
attributes,
|
|
2934
|
+
success: true,
|
|
2935
|
+
result: result === null ? "null" : void 0
|
|
2936
|
+
});
|
|
2937
|
+
return result;
|
|
2938
|
+
} catch (error) {
|
|
2939
|
+
this.emit("operation-end", {
|
|
2940
|
+
id,
|
|
2941
|
+
name,
|
|
2942
|
+
attributes,
|
|
2943
|
+
success: false,
|
|
2944
|
+
error
|
|
2945
|
+
});
|
|
2946
|
+
throw error;
|
|
2947
|
+
}
|
|
2948
|
+
}
|
|
2597
2949
|
scheduleGenerationCleanup(generation) {
|
|
2598
2950
|
this.maintenance.scheduleGenerationCleanup(
|
|
2599
2951
|
generation,
|
|
@@ -2649,37 +3001,6 @@ var CacheStack = class extends EventEmitter {
|
|
|
2649
3001
|
});
|
|
2650
3002
|
this.emitError("write-behind", { failed: failures.length, total: batch.length });
|
|
2651
3003
|
}
|
|
2652
|
-
buildLayerSetEntry(layer, key, kind, value, options, now) {
|
|
2653
|
-
const freshTtl = this.resolveFreshTtl(key, layer.name, kind, options, layer.defaultTtl, value);
|
|
2654
|
-
const staleWhileRevalidate = this.resolveLayerSeconds(
|
|
2655
|
-
layer.name,
|
|
2656
|
-
options?.staleWhileRevalidate,
|
|
2657
|
-
this.options.staleWhileRevalidate
|
|
2658
|
-
);
|
|
2659
|
-
const staleIfError = this.resolveLayerSeconds(layer.name, options?.staleIfError, this.options.staleIfError);
|
|
2660
|
-
const payload = createStoredValueEnvelope({
|
|
2661
|
-
kind,
|
|
2662
|
-
value,
|
|
2663
|
-
freshTtlSeconds: freshTtl,
|
|
2664
|
-
staleWhileRevalidateSeconds: staleWhileRevalidate,
|
|
2665
|
-
staleIfErrorSeconds: staleIfError,
|
|
2666
|
-
now
|
|
2667
|
-
});
|
|
2668
|
-
const ttl = remainingStoredTtlSeconds(payload, now) ?? freshTtl;
|
|
2669
|
-
return {
|
|
2670
|
-
key,
|
|
2671
|
-
value: payload,
|
|
2672
|
-
ttl
|
|
2673
|
-
};
|
|
2674
|
-
}
|
|
2675
|
-
intersectKeys(groups) {
|
|
2676
|
-
if (groups.length === 0) {
|
|
2677
|
-
return [];
|
|
2678
|
-
}
|
|
2679
|
-
const [firstGroup, ...rest] = groups;
|
|
2680
|
-
const restSets = rest.map((group) => new Set(group));
|
|
2681
|
-
return [...new Set(firstGroup)].filter((key) => restSets.every((group) => group.has(key)));
|
|
2682
|
-
}
|
|
2683
3004
|
qualifyKey(key) {
|
|
2684
3005
|
return qualifyGenerationKey(key, this.currentGeneration);
|
|
2685
3006
|
}
|
|
@@ -2689,32 +3010,6 @@ var CacheStack = class extends EventEmitter {
|
|
|
2689
3010
|
stripQualifiedKey(key) {
|
|
2690
3011
|
return stripGenerationPrefix(key, this.currentGeneration);
|
|
2691
3012
|
}
|
|
2692
|
-
async deleteKeysFromLayers(layers, keys) {
|
|
2693
|
-
await Promise.all(
|
|
2694
|
-
layers.map(async (layer) => {
|
|
2695
|
-
if (this.shouldSkipLayer(layer)) {
|
|
2696
|
-
return;
|
|
2697
|
-
}
|
|
2698
|
-
if (layer.deleteMany) {
|
|
2699
|
-
try {
|
|
2700
|
-
await layer.deleteMany(keys);
|
|
2701
|
-
} catch (error) {
|
|
2702
|
-
await this.handleLayerFailure(layer, "delete", error);
|
|
2703
|
-
}
|
|
2704
|
-
return;
|
|
2705
|
-
}
|
|
2706
|
-
await Promise.all(
|
|
2707
|
-
keys.map(async (key) => {
|
|
2708
|
-
try {
|
|
2709
|
-
await layer.delete(key);
|
|
2710
|
-
} catch (error) {
|
|
2711
|
-
await this.handleLayerFailure(layer, "delete", error);
|
|
2712
|
-
}
|
|
2713
|
-
})
|
|
2714
|
-
);
|
|
2715
|
-
})
|
|
2716
|
-
);
|
|
2717
|
-
}
|
|
2718
3013
|
validateConfiguration() {
|
|
2719
3014
|
if (this.options.broadcastL1Invalidation !== void 0 && this.options.publishSetInvalidation !== void 0 && this.options.broadcastL1Invalidation !== this.options.publishSetInvalidation) {
|
|
2720
3015
|
throw new Error("broadcastL1Invalidation and publishSetInvalidation cannot conflict.");
|
|
@@ -2845,18 +3140,6 @@ var CacheStack = class extends EventEmitter {
|
|
|
2845
3140
|
this.emit("error", { operation, ...context });
|
|
2846
3141
|
}
|
|
2847
3142
|
}
|
|
2848
|
-
isCacheSnapshotEntries(value) {
|
|
2849
|
-
return Array.isArray(value) && value.every((entry) => {
|
|
2850
|
-
if (!entry || typeof entry !== "object") {
|
|
2851
|
-
return false;
|
|
2852
|
-
}
|
|
2853
|
-
const candidate = entry;
|
|
2854
|
-
return typeof candidate.key === "string" && (candidate.ttl === void 0 || typeof candidate.ttl === "number" && Number.isFinite(candidate.ttl) && candidate.ttl >= 0);
|
|
2855
|
-
});
|
|
2856
|
-
}
|
|
2857
|
-
sanitizeSnapshotValue(value) {
|
|
2858
|
-
return this.snapshotSerializer.deserialize(this.snapshotSerializer.serialize(value));
|
|
2859
|
-
}
|
|
2860
3143
|
snapshotMaxBytes() {
|
|
2861
3144
|
return this.options.snapshotMaxBytes === false ? false : this.options.snapshotMaxBytes ?? DEFAULT_SNAPSHOT_MAX_BYTES;
|
|
2862
3145
|
}
|
|
@@ -2866,62 +3149,6 @@ var CacheStack = class extends EventEmitter {
|
|
|
2866
3149
|
invalidationMaxKeys() {
|
|
2867
3150
|
return this.options.invalidationMaxKeys === false ? false : this.options.invalidationMaxKeys ?? DEFAULT_INVALIDATION_MAX_KEYS;
|
|
2868
3151
|
}
|
|
2869
|
-
async collectKeysForTag(tag) {
|
|
2870
|
-
const keys = /* @__PURE__ */ new Set();
|
|
2871
|
-
if (this.tagIndex.forEachKeyForTag) {
|
|
2872
|
-
await this.tagIndex.forEachKeyForTag(tag, async (key) => {
|
|
2873
|
-
keys.add(key);
|
|
2874
|
-
this.assertWithinInvalidationKeyLimit(keys.size);
|
|
2875
|
-
});
|
|
2876
|
-
return [...keys];
|
|
2877
|
-
}
|
|
2878
|
-
for (const key of await this.tagIndex.keysForTag(tag)) {
|
|
2879
|
-
keys.add(key);
|
|
2880
|
-
this.assertWithinInvalidationKeyLimit(keys.size);
|
|
2881
|
-
}
|
|
2882
|
-
return [...keys];
|
|
2883
|
-
}
|
|
2884
|
-
assertWithinInvalidationKeyLimit(size) {
|
|
2885
|
-
const maxKeys = this.invalidationMaxKeys();
|
|
2886
|
-
if (maxKeys !== false && size > maxKeys) {
|
|
2887
|
-
throw new Error(`Invalidation matched too many keys (${size} > ${maxKeys}).`);
|
|
2888
|
-
}
|
|
2889
|
-
}
|
|
2890
|
-
async visitExportEntries(maxEntries, visitor) {
|
|
2891
|
-
const exported = /* @__PURE__ */ new Set();
|
|
2892
|
-
for (const layer of this.layers) {
|
|
2893
|
-
if (!layer.keys && !layer.forEachKey) {
|
|
2894
|
-
continue;
|
|
2895
|
-
}
|
|
2896
|
-
const visitKey = async (key) => {
|
|
2897
|
-
const exportedKey = this.stripQualifiedKey(key);
|
|
2898
|
-
if (exported.has(exportedKey)) {
|
|
2899
|
-
return;
|
|
2900
|
-
}
|
|
2901
|
-
const stored = await this.readLayerEntry(layer, key);
|
|
2902
|
-
if (stored === null) {
|
|
2903
|
-
return;
|
|
2904
|
-
}
|
|
2905
|
-
exported.add(exportedKey);
|
|
2906
|
-
if (maxEntries !== false && exported.size > maxEntries) {
|
|
2907
|
-
throw new Error(`Snapshot export exceeds snapshotMaxEntries limit (${exported.size} > ${maxEntries}).`);
|
|
2908
|
-
}
|
|
2909
|
-
await visitor({
|
|
2910
|
-
key: exportedKey,
|
|
2911
|
-
value: stored,
|
|
2912
|
-
ttl: remainingStoredTtlSeconds(stored)
|
|
2913
|
-
});
|
|
2914
|
-
};
|
|
2915
|
-
if (layer.forEachKey) {
|
|
2916
|
-
await layer.forEachKey(visitKey);
|
|
2917
|
-
continue;
|
|
2918
|
-
}
|
|
2919
|
-
const keys = await layer.keys?.();
|
|
2920
|
-
for (const key of keys ?? []) {
|
|
2921
|
-
await visitKey(key);
|
|
2922
|
-
}
|
|
2923
|
-
}
|
|
2924
|
-
}
|
|
2925
3152
|
};
|
|
2926
3153
|
|
|
2927
3154
|
// src/invalidation/RedisInvalidationBus.ts
|
|
@@ -2932,6 +3159,7 @@ var RedisInvalidationBus = class {
|
|
|
2932
3159
|
logger;
|
|
2933
3160
|
handlers = /* @__PURE__ */ new Set();
|
|
2934
3161
|
sharedListener;
|
|
3162
|
+
subscribePromise;
|
|
2935
3163
|
constructor(options) {
|
|
2936
3164
|
this.publisher = options.publisher;
|
|
2937
3165
|
this.subscriber = options.subscriber ?? options.publisher.duplicate();
|
|
@@ -2939,15 +3167,27 @@ var RedisInvalidationBus = class {
|
|
|
2939
3167
|
this.logger = options.logger;
|
|
2940
3168
|
}
|
|
2941
3169
|
async subscribe(handler) {
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
await
|
|
3170
|
+
const previousPromise = this.subscribePromise;
|
|
3171
|
+
let resolveThis;
|
|
3172
|
+
this.subscribePromise = new Promise((resolve2) => {
|
|
3173
|
+
resolveThis = resolve2;
|
|
3174
|
+
});
|
|
3175
|
+
if (previousPromise) {
|
|
3176
|
+
await previousPromise;
|
|
3177
|
+
}
|
|
3178
|
+
try {
|
|
3179
|
+
if (this.handlers.size === 0) {
|
|
3180
|
+
const listener = (_channel, payload) => {
|
|
3181
|
+
void this.dispatchToHandlers(payload);
|
|
3182
|
+
};
|
|
3183
|
+
this.sharedListener = listener;
|
|
3184
|
+
this.subscriber.on("message", listener);
|
|
3185
|
+
await this.subscriber.subscribe(this.channel);
|
|
3186
|
+
}
|
|
3187
|
+
this.handlers.add(handler);
|
|
3188
|
+
} finally {
|
|
3189
|
+
resolveThis();
|
|
2949
3190
|
}
|
|
2950
|
-
this.handlers.add(handler);
|
|
2951
3191
|
return async () => {
|
|
2952
3192
|
this.handlers.delete(handler);
|
|
2953
3193
|
if (this.handlers.size === 0 && this.sharedListener) {
|
|
@@ -2963,7 +3203,12 @@ var RedisInvalidationBus = class {
|
|
|
2963
3203
|
async dispatchToHandlers(payload) {
|
|
2964
3204
|
let message;
|
|
2965
3205
|
try {
|
|
2966
|
-
const parsed =
|
|
3206
|
+
const parsed = sanitizeStructuredData(JSON.parse(payload), {
|
|
3207
|
+
label: "Invalidation payload",
|
|
3208
|
+
maxDepth: 64,
|
|
3209
|
+
maxNodes: 1e4,
|
|
3210
|
+
createObject: () => /* @__PURE__ */ Object.create(null)
|
|
3211
|
+
});
|
|
2967
3212
|
if (!this.isInvalidationMessage(parsed)) {
|
|
2968
3213
|
throw new Error("Invalid invalidation payload shape.");
|
|
2969
3214
|
}
|
|
@@ -3000,31 +3245,6 @@ var RedisInvalidationBus = class {
|
|
|
3000
3245
|
console.error(`[layercache] ${message}`, error);
|
|
3001
3246
|
}
|
|
3002
3247
|
};
|
|
3003
|
-
var DANGEROUS_KEYS = /* @__PURE__ */ new Set(["__proto__", "prototype", "constructor"]);
|
|
3004
|
-
var MAX_SANITIZE_DEPTH2 = 64;
|
|
3005
|
-
var MAX_SANITIZE_NODES2 = 1e4;
|
|
3006
|
-
function sanitizeJsonValue2(value, depth = 0, state = { count: 0 }) {
|
|
3007
|
-
state.count += 1;
|
|
3008
|
-
if (state.count > MAX_SANITIZE_NODES2) {
|
|
3009
|
-
throw new Error(`Invalidation payload exceeds max node count of ${MAX_SANITIZE_NODES2}.`);
|
|
3010
|
-
}
|
|
3011
|
-
if (depth > MAX_SANITIZE_DEPTH2) {
|
|
3012
|
-
throw new Error(`Invalidation payload exceeds max depth of ${MAX_SANITIZE_DEPTH2}.`);
|
|
3013
|
-
}
|
|
3014
|
-
if (Array.isArray(value)) {
|
|
3015
|
-
return value.map((entry) => sanitizeJsonValue2(entry, depth + 1, state));
|
|
3016
|
-
}
|
|
3017
|
-
if (value && typeof value === "object") {
|
|
3018
|
-
const result = /* @__PURE__ */ Object.create(null);
|
|
3019
|
-
for (const key of Object.keys(value)) {
|
|
3020
|
-
if (!DANGEROUS_KEYS.has(key)) {
|
|
3021
|
-
result[key] = sanitizeJsonValue2(value[key], depth + 1, state);
|
|
3022
|
-
}
|
|
3023
|
-
}
|
|
3024
|
-
return result;
|
|
3025
|
-
}
|
|
3026
|
-
return value;
|
|
3027
|
-
}
|
|
3028
3248
|
|
|
3029
3249
|
// src/http/createCacheStatsHandler.ts
|
|
3030
3250
|
function createCacheStatsHandler(cache, options = {}) {
|
|
@@ -3168,65 +3388,52 @@ function cacheGraphqlResolver(cache, prefix, resolver, options = {}) {
|
|
|
3168
3388
|
}
|
|
3169
3389
|
|
|
3170
3390
|
// src/integrations/opentelemetry.ts
|
|
3391
|
+
var MAX_SPANS = 1e4;
|
|
3171
3392
|
function createOpenTelemetryPlugin(cache, tracer) {
|
|
3172
|
-
const
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
|
|
3177
|
-
|
|
3178
|
-
|
|
3179
|
-
|
|
3180
|
-
|
|
3181
|
-
|
|
3182
|
-
|
|
3183
|
-
|
|
3184
|
-
"layercache.key": String(args[0] ?? "")
|
|
3185
|
-
}));
|
|
3186
|
-
cache.set = instrument("layercache.set", tracer, originals.set, (args) => ({
|
|
3187
|
-
"layercache.key": String(args[0] ?? "")
|
|
3188
|
-
}));
|
|
3189
|
-
cache.delete = instrument("layercache.delete", tracer, originals.delete, (args) => ({
|
|
3190
|
-
"layercache.key": String(args[0] ?? "")
|
|
3191
|
-
}));
|
|
3192
|
-
cache.mget = instrument("layercache.mget", tracer, originals.mget);
|
|
3193
|
-
cache.mset = instrument("layercache.mset", tracer, originals.mset);
|
|
3194
|
-
cache.invalidateByTag = instrument("layercache.invalidate_by_tag", tracer, originals.invalidateByTag);
|
|
3195
|
-
cache.invalidateByTags = instrument("layercache.invalidate_by_tags", tracer, originals.invalidateByTags);
|
|
3196
|
-
cache.invalidateByPattern = instrument("layercache.invalidate_by_pattern", tracer, originals.invalidateByPattern);
|
|
3197
|
-
cache.invalidateByPrefix = instrument("layercache.invalidate_by_prefix", tracer, originals.invalidateByPrefix);
|
|
3198
|
-
return {
|
|
3199
|
-
uninstall() {
|
|
3200
|
-
cache.get = originals.get;
|
|
3201
|
-
cache.set = originals.set;
|
|
3202
|
-
cache.delete = originals.delete;
|
|
3203
|
-
cache.mget = originals.mget;
|
|
3204
|
-
cache.mset = originals.mset;
|
|
3205
|
-
cache.invalidateByTag = originals.invalidateByTag;
|
|
3206
|
-
cache.invalidateByTags = originals.invalidateByTags;
|
|
3207
|
-
cache.invalidateByPattern = originals.invalidateByPattern;
|
|
3208
|
-
cache.invalidateByPrefix = originals.invalidateByPrefix;
|
|
3393
|
+
const spans = /* @__PURE__ */ new Map();
|
|
3394
|
+
const onStart = (event) => {
|
|
3395
|
+
try {
|
|
3396
|
+
if (spans.size >= MAX_SPANS) {
|
|
3397
|
+
const oldest = spans.keys().next().value;
|
|
3398
|
+
if (oldest !== void 0) {
|
|
3399
|
+
spans.get(oldest)?.end();
|
|
3400
|
+
spans.delete(oldest);
|
|
3401
|
+
}
|
|
3402
|
+
}
|
|
3403
|
+
spans.set(event.id, tracer.startSpan(event.name, { attributes: event.attributes }));
|
|
3404
|
+
} catch {
|
|
3209
3405
|
}
|
|
3210
3406
|
};
|
|
3211
|
-
|
|
3212
|
-
|
|
3213
|
-
|
|
3214
|
-
|
|
3407
|
+
const onEnd = (event) => {
|
|
3408
|
+
const span = spans.get(event.id);
|
|
3409
|
+
if (!span) {
|
|
3410
|
+
return;
|
|
3411
|
+
}
|
|
3412
|
+
spans.delete(event.id);
|
|
3215
3413
|
try {
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
span.setAttribute?.("layercache.result", "null");
|
|
3414
|
+
span.setAttribute?.("layercache.success", event.success);
|
|
3415
|
+
if (event.result) {
|
|
3416
|
+
span.setAttribute?.("layercache.result", event.result);
|
|
3220
3417
|
}
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
throw error;
|
|
3226
|
-
} finally {
|
|
3227
|
-
span.end();
|
|
3418
|
+
if (event.error !== void 0) {
|
|
3419
|
+
span.recordException?.(event.error);
|
|
3420
|
+
}
|
|
3421
|
+
} catch {
|
|
3228
3422
|
}
|
|
3229
|
-
|
|
3423
|
+
span.end();
|
|
3424
|
+
};
|
|
3425
|
+
cache.on("operation-start", onStart);
|
|
3426
|
+
cache.on("operation-end", onEnd);
|
|
3427
|
+
return {
|
|
3428
|
+
uninstall() {
|
|
3429
|
+
cache.off("operation-start", onStart);
|
|
3430
|
+
cache.off("operation-end", onEnd);
|
|
3431
|
+
for (const span of spans.values()) {
|
|
3432
|
+
span.end();
|
|
3433
|
+
}
|
|
3434
|
+
spans.clear();
|
|
3435
|
+
}
|
|
3436
|
+
};
|
|
3230
3437
|
}
|
|
3231
3438
|
|
|
3232
3439
|
// src/integrations/trpc.ts
|
|
@@ -3589,8 +3796,8 @@ var RedisLayer = class {
|
|
|
3589
3796
|
};
|
|
3590
3797
|
|
|
3591
3798
|
// src/layers/DiskLayer.ts
|
|
3592
|
-
import { createHash } from "crypto";
|
|
3593
|
-
import { promises as
|
|
3799
|
+
import { createHash, randomBytes as randomBytes2 } from "crypto";
|
|
3800
|
+
import { promises as fs2 } from "fs";
|
|
3594
3801
|
import { join, resolve } from "path";
|
|
3595
3802
|
var FILE_SCAN_CONCURRENCY = 32;
|
|
3596
3803
|
var DiskLayer = class {
|
|
@@ -3634,7 +3841,7 @@ var DiskLayer = class {
|
|
|
3634
3841
|
}
|
|
3635
3842
|
async set(key, value, ttl = this.defaultTtl) {
|
|
3636
3843
|
await this.enqueueWrite(async () => {
|
|
3637
|
-
await
|
|
3844
|
+
await fs2.mkdir(this.directory, { recursive: true });
|
|
3638
3845
|
const entry = {
|
|
3639
3846
|
key,
|
|
3640
3847
|
value,
|
|
@@ -3642,10 +3849,10 @@ var DiskLayer = class {
|
|
|
3642
3849
|
};
|
|
3643
3850
|
const payload = this.serializer.serialize(entry);
|
|
3644
3851
|
const targetPath = this.keyToPath(key);
|
|
3645
|
-
const tempPath = `${targetPath}.${process.pid}.${Date.now()}.${
|
|
3852
|
+
const tempPath = `${targetPath}.${process.pid}.${Date.now()}.${randomBytes2(8).toString("hex")}.tmp`;
|
|
3646
3853
|
try {
|
|
3647
|
-
await
|
|
3648
|
-
await
|
|
3854
|
+
await fs2.writeFile(tempPath, payload);
|
|
3855
|
+
await fs2.rename(tempPath, targetPath);
|
|
3649
3856
|
} catch (error) {
|
|
3650
3857
|
await this.safeDelete(tempPath);
|
|
3651
3858
|
throw error;
|
|
@@ -3699,7 +3906,7 @@ var DiskLayer = class {
|
|
|
3699
3906
|
await this.enqueueWrite(async () => {
|
|
3700
3907
|
let entries;
|
|
3701
3908
|
try {
|
|
3702
|
-
entries = await
|
|
3909
|
+
entries = await fs2.readdir(this.directory);
|
|
3703
3910
|
} catch {
|
|
3704
3911
|
return;
|
|
3705
3912
|
}
|
|
@@ -3733,7 +3940,7 @@ var DiskLayer = class {
|
|
|
3733
3940
|
}
|
|
3734
3941
|
async ping() {
|
|
3735
3942
|
try {
|
|
3736
|
-
await
|
|
3943
|
+
await fs2.mkdir(this.directory, { recursive: true });
|
|
3737
3944
|
return true;
|
|
3738
3945
|
} catch {
|
|
3739
3946
|
return false;
|
|
@@ -3776,7 +3983,7 @@ var DiskLayer = class {
|
|
|
3776
3983
|
async readEntryFile(filePath) {
|
|
3777
3984
|
let handle;
|
|
3778
3985
|
try {
|
|
3779
|
-
handle = await
|
|
3986
|
+
handle = await fs2.open(filePath, "r");
|
|
3780
3987
|
return await this.readHandleWithLimit(handle);
|
|
3781
3988
|
} catch {
|
|
3782
3989
|
await this.safeDelete(filePath);
|
|
@@ -3816,7 +4023,7 @@ var DiskLayer = class {
|
|
|
3816
4023
|
async scanEntries(visitor) {
|
|
3817
4024
|
let entries;
|
|
3818
4025
|
try {
|
|
3819
|
-
entries = await
|
|
4026
|
+
entries = await fs2.readdir(this.directory);
|
|
3820
4027
|
} catch {
|
|
3821
4028
|
return;
|
|
3822
4029
|
}
|
|
@@ -3879,7 +4086,7 @@ var DiskLayer = class {
|
|
|
3879
4086
|
}
|
|
3880
4087
|
async safeDelete(filePath) {
|
|
3881
4088
|
try {
|
|
3882
|
-
await
|
|
4089
|
+
await fs2.unlink(filePath);
|
|
3883
4090
|
} catch {
|
|
3884
4091
|
}
|
|
3885
4092
|
}
|
|
@@ -3897,7 +4104,7 @@ var DiskLayer = class {
|
|
|
3897
4104
|
}
|
|
3898
4105
|
let entries;
|
|
3899
4106
|
try {
|
|
3900
|
-
entries = await
|
|
4107
|
+
entries = await fs2.readdir(this.directory);
|
|
3901
4108
|
} catch {
|
|
3902
4109
|
return;
|
|
3903
4110
|
}
|
|
@@ -3909,7 +4116,7 @@ var DiskLayer = class {
|
|
|
3909
4116
|
lcFiles.map(async (name) => {
|
|
3910
4117
|
const filePath = join(this.directory, name);
|
|
3911
4118
|
try {
|
|
3912
|
-
const stat = await
|
|
4119
|
+
const stat = await fs2.stat(filePath);
|
|
3913
4120
|
return { filePath, mtimeMs: stat.mtimeMs };
|
|
3914
4121
|
} catch {
|
|
3915
4122
|
return { filePath, mtimeMs: 0 };
|
|
@@ -4005,44 +4212,19 @@ var MemcachedLayer = class {
|
|
|
4005
4212
|
|
|
4006
4213
|
// src/serialization/MsgpackSerializer.ts
|
|
4007
4214
|
import { decode, encode } from "@msgpack/msgpack";
|
|
4008
|
-
var DANGEROUS_KEYS2 = /* @__PURE__ */ new Set(["__proto__", "prototype", "constructor"]);
|
|
4009
|
-
var MAX_SANITIZE_DEPTH3 = 64;
|
|
4010
|
-
var MAX_SANITIZE_NODES3 = 1e4;
|
|
4011
4215
|
var MsgpackSerializer = class {
|
|
4012
4216
|
serialize(value) {
|
|
4013
4217
|
return Buffer.from(encode(value));
|
|
4014
4218
|
}
|
|
4015
4219
|
deserialize(payload) {
|
|
4016
4220
|
const normalized = Buffer.isBuffer(payload) ? payload : Buffer.from(payload, "latin1");
|
|
4017
|
-
return
|
|
4221
|
+
return sanitizeStructuredData(decode(normalized), {
|
|
4222
|
+
label: "MessagePack payload",
|
|
4223
|
+
maxDepth: 64,
|
|
4224
|
+
maxNodes: 1e4
|
|
4225
|
+
});
|
|
4018
4226
|
}
|
|
4019
4227
|
};
|
|
4020
|
-
function sanitizeMsgpackValue(value, depth, state) {
|
|
4021
|
-
state.count += 1;
|
|
4022
|
-
if (state.count > MAX_SANITIZE_NODES3) {
|
|
4023
|
-
throw new Error(`MessagePack payload exceeds max node count of ${MAX_SANITIZE_NODES3}.`);
|
|
4024
|
-
}
|
|
4025
|
-
if (depth > MAX_SANITIZE_DEPTH3) {
|
|
4026
|
-
throw new Error(`MessagePack payload exceeds max depth of ${MAX_SANITIZE_DEPTH3}.`);
|
|
4027
|
-
}
|
|
4028
|
-
if (Array.isArray(value)) {
|
|
4029
|
-
return value.map((entry) => sanitizeMsgpackValue(entry, depth + 1, state));
|
|
4030
|
-
}
|
|
4031
|
-
if (!isPlainObject2(value)) {
|
|
4032
|
-
return value;
|
|
4033
|
-
}
|
|
4034
|
-
const sanitized = {};
|
|
4035
|
-
for (const [key, entry] of Object.entries(value)) {
|
|
4036
|
-
if (DANGEROUS_KEYS2.has(key)) {
|
|
4037
|
-
continue;
|
|
4038
|
-
}
|
|
4039
|
-
sanitized[key] = sanitizeMsgpackValue(entry, depth + 1, state);
|
|
4040
|
-
}
|
|
4041
|
-
return sanitized;
|
|
4042
|
-
}
|
|
4043
|
-
function isPlainObject2(value) {
|
|
4044
|
-
return Object.prototype.toString.call(value) === "[object Object]";
|
|
4045
|
-
}
|
|
4046
4228
|
|
|
4047
4229
|
// src/singleflight/RedisSingleFlightCoordinator.ts
|
|
4048
4230
|
import { randomUUID } from "crypto";
|