@lage-run/cache 1.4.11 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/getCacheDirectory.d.ts +20 -0
- package/lib/getCacheDirectory.js +11 -2
- package/lib/getCacheDirectory.js.map +1 -1
- package/lib/providers/BackfillCacheProvider.js +6 -8
- package/lib/providers/BackfillCacheProvider.js.map +1 -1
- package/lib/providers/RemoteFallbackCacheProvider.js +10 -3
- package/lib/providers/RemoteFallbackCacheProvider.js.map +1 -1
- package/package.json +8 -6
|
@@ -1,3 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Get the root cache directory: `<root>/node_modules/.cache/lage`
|
|
3
|
+
*/
|
|
1
4
|
export declare function getCacheDirectoryRoot(root: string): string;
|
|
5
|
+
/**
|
|
6
|
+
* Get the cache subdirectory for a hash: `<root>/node_modules/.cache/lage/cache/ab12`
|
|
7
|
+
* for a hash that starts with `ab12`
|
|
8
|
+
*/
|
|
2
9
|
export declare function getCacheDirectory(root: string, hash: string): string;
|
|
10
|
+
/**
|
|
11
|
+
* Get the logs subdirectory for a hash: `<root>/node_modules/.cache/lage/logs/ab12`
|
|
12
|
+
* for a hash that starts with `ab12`
|
|
13
|
+
*/
|
|
3
14
|
export declare function getLogsCacheDirectory(root: string, hash: string): string;
|
|
15
|
+
/**
|
|
16
|
+
* Get the parent `cache` and `logs` directories that contain subfolders with hash prefixes.
|
|
17
|
+
*/
|
|
18
|
+
export declare function getHashCacheDirectories(root: string): {
|
|
19
|
+
/** `<root>/node_modules/.cache/lage/cache` */
|
|
20
|
+
cache: string;
|
|
21
|
+
/** `<root>/node_modules/.cache/lage/logs` */
|
|
22
|
+
logs: string;
|
|
23
|
+
};
|
package/lib/getCacheDirectory.js
CHANGED
|
@@ -15,6 +15,9 @@ _export(exports, {
|
|
|
15
15
|
get getCacheDirectoryRoot () {
|
|
16
16
|
return getCacheDirectoryRoot;
|
|
17
17
|
},
|
|
18
|
+
get getHashCacheDirectories () {
|
|
19
|
+
return getHashCacheDirectories;
|
|
20
|
+
},
|
|
18
21
|
get getLogsCacheDirectory () {
|
|
19
22
|
return getLogsCacheDirectory;
|
|
20
23
|
}
|
|
@@ -29,8 +32,14 @@ function getCacheDirectoryRoot(root) {
|
|
|
29
32
|
return _path.default.join(root, "node_modules", ".cache", "lage");
|
|
30
33
|
}
|
|
31
34
|
function getCacheDirectory(root, hash) {
|
|
32
|
-
return _path.default.join(
|
|
35
|
+
return _path.default.join(getHashCacheDirectories(root).cache, hash.substring(0, 4));
|
|
33
36
|
}
|
|
34
37
|
function getLogsCacheDirectory(root, hash) {
|
|
35
|
-
return _path.default.join(
|
|
38
|
+
return _path.default.join(getHashCacheDirectories(root).logs, hash.substring(0, 4));
|
|
39
|
+
}
|
|
40
|
+
function getHashCacheDirectories(root) {
|
|
41
|
+
return {
|
|
42
|
+
cache: _path.default.join(getCacheDirectoryRoot(root), "cache"),
|
|
43
|
+
logs: _path.default.join(getCacheDirectoryRoot(root), "logs")
|
|
44
|
+
};
|
|
36
45
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/getCacheDirectory.ts"],"sourcesContent":["import path from \"path\";\n\nexport function getCacheDirectoryRoot(root: string): string {\n return path.join(root, \"node_modules\", \".cache\", \"lage\");\n}\n\nexport function getCacheDirectory(root: string, hash: string): string {\n return path.join(
|
|
1
|
+
{"version":3,"sources":["../src/getCacheDirectory.ts"],"sourcesContent":["import path from \"path\";\n\n/**\n * Get the root cache directory: `<root>/node_modules/.cache/lage`\n */\nexport function getCacheDirectoryRoot(root: string): string {\n return path.join(root, \"node_modules\", \".cache\", \"lage\");\n}\n\n/**\n * Get the cache subdirectory for a hash: `<root>/node_modules/.cache/lage/cache/ab12`\n * for a hash that starts with `ab12`\n */\nexport function getCacheDirectory(root: string, hash: string): string {\n return path.join(getHashCacheDirectories(root).cache, hash.substring(0, 4));\n}\n\n/**\n * Get the logs subdirectory for a hash: `<root>/node_modules/.cache/lage/logs/ab12`\n * for a hash that starts with `ab12`\n */\nexport function getLogsCacheDirectory(root: string, hash: string): string {\n return path.join(getHashCacheDirectories(root).logs, hash.substring(0, 4));\n}\n\n/**\n * Get the parent `cache` and `logs` directories that contain subfolders with hash prefixes.\n */\nexport function getHashCacheDirectories(root: string): {\n /** `<root>/node_modules/.cache/lage/cache` */\n cache: string;\n /** `<root>/node_modules/.cache/lage/logs` */\n logs: string;\n} {\n return {\n cache: path.join(getCacheDirectoryRoot(root), \"cache\"),\n logs: path.join(getCacheDirectoryRoot(root), \"logs\"),\n };\n}\n"],"names":["getCacheDirectory","getCacheDirectoryRoot","getHashCacheDirectories","getLogsCacheDirectory","root","path","join","hash","cache","substring","logs"],"mappings":";;;;;;;;;;;QAagBA;eAAAA;;QARAC;eAAAA;;QAuBAC;eAAAA;;QAPAC;eAAAA;;;6DArBC;;;;;;AAKV,SAASF,sBAAsBG,IAAY;IAChD,OAAOC,aAAI,CAACC,IAAI,CAACF,MAAM,gBAAgB,UAAU;AACnD;AAMO,SAASJ,kBAAkBI,IAAY,EAAEG,IAAY;IAC1D,OAAOF,aAAI,CAACC,IAAI,CAACJ,wBAAwBE,MAAMI,KAAK,EAAED,KAAKE,SAAS,CAAC,GAAG;AAC1E;AAMO,SAASN,sBAAsBC,IAAY,EAAEG,IAAY;IAC9D,OAAOF,aAAI,CAACC,IAAI,CAACJ,wBAAwBE,MAAMM,IAAI,EAAEH,KAAKE,SAAS,CAAC,GAAG;AACzE;AAKO,SAASP,wBAAwBE,IAAY;IAMlD,OAAO;QACLI,OAAOH,aAAI,CAACC,IAAI,CAACL,sBAAsBG,OAAO;QAC9CM,MAAML,aAAI,CAACC,IAAI,CAACL,sBAAsBG,OAAO;IAC/C;AACF"}
|
|
@@ -128,18 +128,16 @@ class BackfillCacheProvider {
|
|
|
128
128
|
}
|
|
129
129
|
async purge(prunePeriod = 30, concurrency = 10) {
|
|
130
130
|
const now = new Date();
|
|
131
|
-
const
|
|
132
|
-
"cache",
|
|
133
|
-
"logs"
|
|
134
|
-
];
|
|
131
|
+
const directories = (0, _getCacheDirectory.getHashCacheDirectories)(this.options.root);
|
|
135
132
|
const entries = [];
|
|
136
|
-
for (const
|
|
137
|
-
|
|
133
|
+
for (const cacheTypeDirectory of [
|
|
134
|
+
directories.cache,
|
|
135
|
+
directories.logs
|
|
136
|
+
]){
|
|
138
137
|
if (_fs.existsSync(cacheTypeDirectory)) {
|
|
139
138
|
const hashPrefixes = await readdir(cacheTypeDirectory);
|
|
140
139
|
for (const prefix of hashPrefixes){
|
|
141
|
-
|
|
142
|
-
entries.push(cachePath);
|
|
140
|
+
entries.push(_path.join(cacheTypeDirectory, prefix));
|
|
143
141
|
}
|
|
144
142
|
}
|
|
145
143
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/providers/BackfillCacheProvider.ts"],"sourcesContent":["import { createBackfillCacheConfig, createBackfillLogger } from \"../backfillWrapper.js\";\nimport { getCacheStorageProvider } from \"backfill-cache\";\nimport { promisify } from \"util\";\nimport * as fs from \"fs\";\nimport * as path from \"path\";\nimport type { CacheProvider } from \"../types/CacheProvider.js\";\nimport type { CacheOptions } from \"@lage-run/config\";\nimport type { Logger as BackfillLogger } from \"backfill-logger\";\nimport type { Target } from \"@lage-run/target-graph\";\nimport type { Logger } from \"@lage-run/logger\";\nimport { getCacheDirectory,
|
|
1
|
+
{"version":3,"sources":["../../src/providers/BackfillCacheProvider.ts"],"sourcesContent":["import { createBackfillCacheConfig, createBackfillLogger } from \"../backfillWrapper.js\";\nimport { getCacheStorageProvider } from \"backfill-cache\";\nimport { promisify } from \"util\";\nimport * as fs from \"fs\";\nimport * as path from \"path\";\nimport type { CacheProvider } from \"../types/CacheProvider.js\";\nimport type { CacheOptions } from \"@lage-run/config\";\nimport type { Logger as BackfillLogger } from \"backfill-logger\";\nimport type { Target } from \"@lage-run/target-graph\";\nimport type { Logger } from \"@lage-run/logger\";\nimport { getCacheDirectory, getHashCacheDirectories } from \"../getCacheDirectory.js\";\nimport { chunkPromise } from \"../chunkPromise.js\";\n\nconst rm = promisify(fs.rm);\nconst readdir = promisify(fs.readdir);\nconst stat = promisify(fs.stat);\n\nconst MS_IN_A_DAY = 1000 * 60 * 60 * 24;\n\nexport interface BackfillCacheProviderOptions {\n root: string;\n logger: Logger;\n cacheOptions: Partial<CacheOptions>;\n}\n\nexport class BackfillCacheProvider implements CacheProvider {\n /**\n * logger for backfill\n */\n private backfillLogger: BackfillLogger;\n\n private getTargetCacheStorageProvider(cwd: string, hash: string) {\n const { cacheOptions } = this.options;\n const { cacheStorageConfig, incrementalCaching } = createBackfillCacheConfig(cwd, cacheOptions, this.backfillLogger);\n\n const cachePath = this.getCachePath(cwd, hash);\n\n return getCacheStorageProvider(cacheStorageConfig ?? { provider: \"local\" }, cachePath, this.backfillLogger, cwd, incrementalCaching);\n }\n\n constructor(private options: BackfillCacheProviderOptions) {\n this.backfillLogger = createBackfillLogger();\n }\n\n public async fetch(hash: string, target: Target): Promise<boolean> {\n const { logger } = this.options;\n\n if (!hash) {\n return false;\n }\n\n const cacheStorage = this.getTargetCacheStorageProvider(target.cwd, hash);\n\n try {\n return await cacheStorage.fetch(hash);\n } catch (error) {\n let message;\n\n if (error instanceof Error) {\n message = error.message;\n } else message = String(error);\n\n logger.silly(`Cache fetch failed: ${message}`, { target });\n\n // backfill fetch can error, but we should simply ignore and continue\n return false;\n }\n }\n\n public async put(hash: string, target: Target): Promise<void> {\n const { logger } = this.options;\n\n if (!hash) {\n return;\n }\n\n const cacheStorage = this.getTargetCacheStorageProvider(target.cwd, hash);\n\n try {\n await cacheStorage.put(hash, target.outputs ?? this.options.cacheOptions.outputGlob ?? [\"**/*\"]);\n } catch (error) {\n let message;\n\n if (error instanceof Error) {\n message = error.message;\n } else message = String(error);\n\n logger.silly(`Cache fetch failed: ${message}`, { target });\n\n // backfill throws an error if outputGlob doesn't match any files, we will skip this error\n }\n }\n\n public async clear(concurrency = 10): Promise<void> {\n return this.purge(0, concurrency);\n }\n\n public async purge(prunePeriod = 30, concurrency = 10): Promise<void> {\n const now = new Date();\n\n const directories = getHashCacheDirectories(this.options.root);\n const entries: string[] = [];\n\n for (const cacheTypeDirectory of [directories.cache, directories.logs]) {\n if (fs.existsSync(cacheTypeDirectory)) {\n const hashPrefixes = await readdir(cacheTypeDirectory);\n for (const prefix of hashPrefixes) {\n entries.push(path.join(cacheTypeDirectory, prefix));\n }\n }\n }\n\n await chunkPromise(\n entries.map((entry) => {\n return async () => {\n const entryPath = entry;\n const entryStat = await stat(entryPath);\n\n if (now.getTime() - entryStat.mtime.getTime() > prunePeriod * MS_IN_A_DAY) {\n await removeCache(entryPath, entryStat);\n }\n };\n }),\n concurrency\n );\n }\n\n private getCachePath(packagePath: string, hash: string): string {\n return path.relative(packagePath, getCacheDirectory(this.options.root, hash));\n }\n}\n\nasync function removeCache(cachePath: string, entryStat: fs.Stats) {\n if (entryStat.isDirectory()) {\n return rm(cachePath, { recursive: true });\n } else {\n return rm(cachePath);\n }\n}\n"],"names":["BackfillCacheProvider","rm","promisify","fs","readdir","stat","MS_IN_A_DAY","getTargetCacheStorageProvider","cwd","hash","cacheOptions","options","cacheStorageConfig","incrementalCaching","createBackfillCacheConfig","backfillLogger","cachePath","getCachePath","getCacheStorageProvider","provider","fetch","target","logger","cacheStorage","error","message","Error","String","silly","put","outputs","outputGlob","clear","concurrency","purge","prunePeriod","now","Date","directories","getHashCacheDirectories","root","entries","cacheTypeDirectory","cache","logs","existsSync","hashPrefixes","prefix","push","path","join","chunkPromise","map","entry","entryPath","entryStat","getTime","mtime","removeCache","packagePath","relative","getCacheDirectory","createBackfillLogger","isDirectory","recursive"],"mappings":";;;;+BAyBaA;;;eAAAA;;;iCAzBmD;+BACxB;sBACd;4DACN;8DACE;mCAMqC;8BAC9B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE7B,MAAMC,KAAKC,IAAAA,eAAS,EAACC,IAAGF,EAAE;AAC1B,MAAMG,UAAUF,IAAAA,eAAS,EAACC,IAAGC,OAAO;AACpC,MAAMC,OAAOH,IAAAA,eAAS,EAACC,IAAGE,IAAI;AAE9B,MAAMC,cAAc,OAAO,KAAK,KAAK;AAQ9B,MAAMN;IAMHO,8BAA8BC,GAAW,EAAEC,IAAY,EAAE;QAC/D,MAAM,EAAEC,YAAY,EAAE,GAAG,IAAI,CAACC,OAAO;QACrC,MAAM,EAAEC,kBAAkB,EAAEC,kBAAkB,EAAE,GAAGC,IAAAA,0CAAyB,EAACN,KAAKE,cAAc,IAAI,CAACK,cAAc;QAEnH,MAAMC,YAAY,IAAI,CAACC,YAAY,CAACT,KAAKC;QAEzC,OAAOS,IAAAA,sCAAuB,EAACN,sBAAsB;YAAEO,UAAU;QAAQ,GAAGH,WAAW,IAAI,CAACD,cAAc,EAAEP,KAAKK;IACnH;IAMA,MAAaO,MAAMX,IAAY,EAAEY,MAAc,EAAoB;QACjE,MAAM,EAAEC,MAAM,EAAE,GAAG,IAAI,CAACX,OAAO;QAE/B,IAAI,CAACF,MAAM;YACT,OAAO;QACT;QAEA,MAAMc,eAAe,IAAI,CAAChB,6BAA6B,CAACc,OAAOb,GAAG,EAAEC;QAEpE,IAAI;YACF,OAAO,MAAMc,aAAaH,KAAK,CAACX;QAClC,EAAE,OAAOe,OAAO;YACd,IAAIC;YAEJ,IAAID,iBAAiBE,OAAO;gBAC1BD,UAAUD,MAAMC,OAAO;YACzB,OAAOA,UAAUE,OAAOH;YAExBF,OAAOM,KAAK,CAAC,CAAC,oBAAoB,EAAEH,SAAS,EAAE;gBAAEJ;YAAO;YAExD,qEAAqE;YACrE,OAAO;QACT;IACF;IAEA,MAAaQ,IAAIpB,IAAY,EAAEY,MAAc,EAAiB;QAC5D,MAAM,EAAEC,MAAM,EAAE,GAAG,IAAI,CAACX,OAAO;QAE/B,IAAI,CAACF,MAAM;YACT;QACF;QAEA,MAAMc,eAAe,IAAI,CAAChB,6BAA6B,CAACc,OAAOb,GAAG,EAAEC;QAEpE,IAAI;YACF,MAAMc,aAAaM,GAAG,CAACpB,MAAMY,OAAOS,OAAO,IAAI,IAAI,CAACnB,OAAO,CAACD,YAAY,CAACqB,UAAU,IAAI;gBAAC;aAAO;QACjG,EAAE,OAAOP,OAAO;YACd,IAAIC;YAEJ,IAAID,iBAAiBE,OAAO;gBAC1BD,UAAUD,MAAMC,OAAO;YACzB,OAAOA,UAAUE,OAAOH;YAExBF,OAAOM,KAAK,CAAC,CAAC,oBAAoB,EAAEH,SAAS,EAAE;gBAAEJ;YAAO;QAExD,0FAA0F;QAC5F;IACF;IAEA,MAAaW,MAAMC,cAAc,EAAE,EAAiB;QAClD,OAAO,IAAI,CAACC,KAAK,CAAC,GAAGD;IACvB;IAEA,MAAaC,MAAMC,cAAc,EAAE,EAAEF,cAAc,EAAE,EAAiB;QACpE,MAAMG,MAAM,IAAIC;QAEhB,MAAMC,cAAcC,IAAAA,0CAAuB,EAAC,IAAI,CAAC5B,OAAO,CAAC6B,IAAI;QAC7D,MAAMC,UAAoB,EAAE;QAE5B,KAAK,MAAMC,sBAAsB;YAACJ,YAAYK,KAAK;YAAEL,YAAYM,IAAI;SAAC,CAAE;YACtE,IAAIzC,IAAG0C,UAAU,CAACH,qBAAqB;gBACrC,MAAMI,eAAe,MAAM1C,QAAQsC;gBACnC,KAAK,MAAMK,UAAUD,aAAc;oBACjCL,QAAQO,IAAI,CAACC,MAAKC,IAAI,CAACR,oBAAoBK;gBAC7C;YACF;QACF;QAEA,MAAMI,IAAAA,0BAAY,EAChBV,QAAQW,GAAG,CAAC,CAACC;YACX,OAAO;gBACL,MAAMC,YAAYD;gBAClB,MAAME,YAAY,MAAMlD,KAAKiD;gBAE7B,IAAIlB,IAAIoB,OAAO,KAAKD,UAAUE,KAAK,CAACD,OAAO,KAAKrB,cAAc7B,aAAa;oBACzE,MAAMoD,YAAYJ,WAAWC;gBAC/B;YACF;QACF,IACAtB;IAEJ;IAEQhB,aAAa0C,WAAmB,EAAElD,IAAY,EAAU;QAC9D,OAAOwC,MAAKW,QAAQ,CAACD,aAAaE,IAAAA,oCAAiB,EAAC,IAAI,CAAClD,OAAO,CAAC6B,IAAI,EAAE/B;IACzE;IAzFA,YAAY,AAAQE,OAAqC,CAAE;;QAd3D;;GAEC,GACD,uBAAQI,kBAAR,KAAA;aAWoBJ,UAAAA;QAClB,IAAI,CAACI,cAAc,GAAG+C,IAAAA,qCAAoB;IAC5C;AAwFF;AAEA,eAAeJ,YAAY1C,SAAiB,EAAEuC,SAAmB;IAC/D,IAAIA,UAAUQ,WAAW,IAAI;QAC3B,OAAO9D,GAAGe,WAAW;YAAEgD,WAAW;QAAK;IACzC,OAAO;QACL,OAAO/D,GAAGe;IACZ;AACF"}
|
|
@@ -24,11 +24,12 @@ function _define_property(obj, key, value) {
|
|
|
24
24
|
class RemoteFallbackCacheProvider {
|
|
25
25
|
async fetch(hash, target) {
|
|
26
26
|
const { logger, remoteCacheProvider, localCacheProvider } = this.options;
|
|
27
|
+
const skipRemoteCacheForTarget = target.skipRemoteCache === true;
|
|
27
28
|
if (localCacheProvider) {
|
|
28
29
|
RemoteFallbackCacheProvider.localHits[hash] = await localCacheProvider.fetch(hash, target);
|
|
29
30
|
logger.silly(`local cache fetch: ${hash} ${RemoteFallbackCacheProvider.localHits[hash]}`);
|
|
30
31
|
}
|
|
31
|
-
if (!RemoteFallbackCacheProvider.localHits[hash] && remoteCacheProvider) {
|
|
32
|
+
if (!skipRemoteCacheForTarget && !RemoteFallbackCacheProvider.localHits[hash] && remoteCacheProvider) {
|
|
32
33
|
RemoteFallbackCacheProvider.remoteHits[hash] = await remoteCacheProvider.fetch(hash, target);
|
|
33
34
|
logger.silly(`remote fallback fetch: ${hash} ${RemoteFallbackCacheProvider.remoteHits[hash]}`);
|
|
34
35
|
// now save this into the localCacheProvider, if available
|
|
@@ -49,8 +50,14 @@ class RemoteFallbackCacheProvider {
|
|
|
49
50
|
logger.silly(`local cache put: ${hash}`);
|
|
50
51
|
putPromises.push(localCacheProvider.put(hash, target));
|
|
51
52
|
}
|
|
52
|
-
|
|
53
|
-
|
|
53
|
+
/**
|
|
54
|
+
* Write to remote if:
|
|
55
|
+
* - the target has not disabled remote caching
|
|
56
|
+
* - there is no hit in the remote cache
|
|
57
|
+
* - a remote cache storage provider exists
|
|
58
|
+
* - the "writeRemoteCache" config flag is set to true
|
|
59
|
+
*/ const skipRemoteCacheForTarget = target.skipRemoteCache === true;
|
|
60
|
+
const shouldWriteRemoteCache = !skipRemoteCacheForTarget && !this.isRemoteHit(hash) && !!remoteCacheProvider && writeRemoteCache;
|
|
54
61
|
if (shouldWriteRemoteCache) {
|
|
55
62
|
logger.silly(`remote fallback put: ${hash}`);
|
|
56
63
|
const remotePut = remoteCacheProvider.put(hash, target);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/providers/RemoteFallbackCacheProvider.ts"],"sourcesContent":["import type { CacheProvider } from \"../types/CacheProvider.js\";\nimport type { Logger } from \"@lage-run/logger\";\nimport type { Target } from \"@lage-run/target-graph\";\n\nexport interface RemoteFallbackCacheProviderOptions {\n root: string;\n logger: Logger;\n\n localCacheProvider?: CacheProvider;\n remoteCacheProvider?: CacheProvider;\n\n writeRemoteCache?: boolean;\n}\n\n/**\n * Remote Fallback Cache Provider\n *\n * This backfill cache provider will fallback to a remote cache provider if the local cache does not contain the item.\n * It will also automatically populate the local cache with the remote cache.\n */\nexport class RemoteFallbackCacheProvider implements CacheProvider {\n private static localHits: { [hash: string]: boolean } = {};\n private static remoteHits: { [hash: string]: boolean } = {};\n\n constructor(private options: RemoteFallbackCacheProviderOptions) {}\n\n public async fetch(hash: string, target: Target): Promise<boolean> {\n const { logger, remoteCacheProvider, localCacheProvider } = this.options;\n\n if (localCacheProvider) {\n RemoteFallbackCacheProvider.localHits[hash] = await localCacheProvider.fetch(hash, target);\n logger.silly(`local cache fetch: ${hash} ${RemoteFallbackCacheProvider.localHits[hash]}`);\n }\n\n if (!RemoteFallbackCacheProvider.localHits[hash] && remoteCacheProvider) {\n RemoteFallbackCacheProvider.remoteHits[hash] = await remoteCacheProvider.fetch(hash, target);\n logger.silly(`remote fallback fetch: ${hash} ${RemoteFallbackCacheProvider.remoteHits[hash]}`);\n\n // now save this into the localCacheProvider, if available\n if (localCacheProvider && RemoteFallbackCacheProvider.remoteHits[hash]) {\n logger.silly(`local cache put, fetched cache from remote: ${hash}`);\n await localCacheProvider.put(hash, target);\n }\n\n return RemoteFallbackCacheProvider.remoteHits[hash];\n }\n\n return RemoteFallbackCacheProvider.localHits[hash];\n }\n\n public async put(hash: string, target: Target): Promise<void> {\n const { logger, remoteCacheProvider, localCacheProvider, writeRemoteCache } = this.options;\n const putPromises: Promise<void>[] = [];\n\n // Write local cache if it doesn't already exist, or if the the hash isn't in the localHits\n const shouldWriteLocalCache = !this.isLocalHit(hash) && !!localCacheProvider;\n\n if (shouldWriteLocalCache) {\n logger.silly(`local cache put: ${hash}`);\n putPromises.push(localCacheProvider.put(hash, target));\n }\n\n
|
|
1
|
+
{"version":3,"sources":["../../src/providers/RemoteFallbackCacheProvider.ts"],"sourcesContent":["import type { CacheProvider } from \"../types/CacheProvider.js\";\nimport type { Logger } from \"@lage-run/logger\";\nimport type { Target } from \"@lage-run/target-graph\";\n\nexport interface RemoteFallbackCacheProviderOptions {\n root: string;\n logger: Logger;\n\n localCacheProvider?: CacheProvider;\n remoteCacheProvider?: CacheProvider;\n\n writeRemoteCache?: boolean;\n}\n\n/**\n * Remote Fallback Cache Provider\n *\n * This backfill cache provider will fallback to a remote cache provider if the local cache does not contain the item.\n * It will also automatically populate the local cache with the remote cache.\n */\nexport class RemoteFallbackCacheProvider implements CacheProvider {\n private static localHits: { [hash: string]: boolean } = {};\n private static remoteHits: { [hash: string]: boolean } = {};\n\n constructor(private options: RemoteFallbackCacheProviderOptions) {}\n\n public async fetch(hash: string, target: Target): Promise<boolean> {\n const { logger, remoteCacheProvider, localCacheProvider } = this.options;\n const skipRemoteCacheForTarget = target.skipRemoteCache === true;\n\n if (localCacheProvider) {\n RemoteFallbackCacheProvider.localHits[hash] = await localCacheProvider.fetch(hash, target);\n logger.silly(`local cache fetch: ${hash} ${RemoteFallbackCacheProvider.localHits[hash]}`);\n }\n\n if (!skipRemoteCacheForTarget && !RemoteFallbackCacheProvider.localHits[hash] && remoteCacheProvider) {\n RemoteFallbackCacheProvider.remoteHits[hash] = await remoteCacheProvider.fetch(hash, target);\n logger.silly(`remote fallback fetch: ${hash} ${RemoteFallbackCacheProvider.remoteHits[hash]}`);\n\n // now save this into the localCacheProvider, if available\n if (localCacheProvider && RemoteFallbackCacheProvider.remoteHits[hash]) {\n logger.silly(`local cache put, fetched cache from remote: ${hash}`);\n await localCacheProvider.put(hash, target);\n }\n\n return RemoteFallbackCacheProvider.remoteHits[hash];\n }\n\n return RemoteFallbackCacheProvider.localHits[hash];\n }\n\n public async put(hash: string, target: Target): Promise<void> {\n const { logger, remoteCacheProvider, localCacheProvider, writeRemoteCache } = this.options;\n const putPromises: Promise<void>[] = [];\n\n // Write local cache if it doesn't already exist, or if the the hash isn't in the localHits\n const shouldWriteLocalCache = !this.isLocalHit(hash) && !!localCacheProvider;\n\n if (shouldWriteLocalCache) {\n logger.silly(`local cache put: ${hash}`);\n putPromises.push(localCacheProvider.put(hash, target));\n }\n\n /**\n * Write to remote if:\n * - the target has not disabled remote caching\n * - there is no hit in the remote cache\n * - a remote cache storage provider exists\n * - the \"writeRemoteCache\" config flag is set to true\n */\n const skipRemoteCacheForTarget = target.skipRemoteCache === true;\n const shouldWriteRemoteCache = !skipRemoteCacheForTarget && !this.isRemoteHit(hash) && !!remoteCacheProvider && writeRemoteCache;\n\n if (shouldWriteRemoteCache) {\n logger.silly(`remote fallback put: ${hash}`);\n const remotePut = remoteCacheProvider.put(hash, target);\n putPromises.push(remotePut);\n }\n\n await Promise.all(putPromises);\n }\n\n private isRemoteHit(hash: string) {\n return hash in RemoteFallbackCacheProvider.remoteHits && RemoteFallbackCacheProvider.remoteHits[hash];\n }\n\n private isLocalHit(hash: string) {\n return hash in RemoteFallbackCacheProvider.localHits && RemoteFallbackCacheProvider.localHits[hash];\n }\n\n public async clear(): Promise<void> {\n const { localCacheProvider } = this.options;\n if (localCacheProvider) {\n return localCacheProvider.clear();\n }\n }\n\n public async purge(sinceDays: number): Promise<void> {\n const { localCacheProvider } = this.options;\n if (localCacheProvider) {\n return localCacheProvider.purge(sinceDays);\n }\n }\n}\n"],"names":["RemoteFallbackCacheProvider","fetch","hash","target","logger","remoteCacheProvider","localCacheProvider","options","skipRemoteCacheForTarget","skipRemoteCache","localHits","silly","remoteHits","put","writeRemoteCache","putPromises","shouldWriteLocalCache","isLocalHit","push","shouldWriteRemoteCache","isRemoteHit","remotePut","Promise","all","clear","purge","sinceDays"],"mappings":";;;;+BAoBaA;;;eAAAA;;;;;;;;;;;;;;;;AAAN,MAAMA;IAMX,MAAaC,MAAMC,IAAY,EAAEC,MAAc,EAAoB;QACjE,MAAM,EAAEC,MAAM,EAAEC,mBAAmB,EAAEC,kBAAkB,EAAE,GAAG,IAAI,CAACC,OAAO;QACxE,MAAMC,2BAA2BL,OAAOM,eAAe,KAAK;QAE5D,IAAIH,oBAAoB;YACtBN,4BAA4BU,SAAS,CAACR,KAAK,GAAG,MAAMI,mBAAmBL,KAAK,CAACC,MAAMC;YACnFC,OAAOO,KAAK,CAAC,CAAC,mBAAmB,EAAET,KAAK,CAAC,EAAEF,4BAA4BU,SAAS,CAACR,KAAK,EAAE;QAC1F;QAEA,IAAI,CAACM,4BAA4B,CAACR,4BAA4BU,SAAS,CAACR,KAAK,IAAIG,qBAAqB;YACpGL,4BAA4BY,UAAU,CAACV,KAAK,GAAG,MAAMG,oBAAoBJ,KAAK,CAACC,MAAMC;YACrFC,OAAOO,KAAK,CAAC,CAAC,uBAAuB,EAAET,KAAK,CAAC,EAAEF,4BAA4BY,UAAU,CAACV,KAAK,EAAE;YAE7F,0DAA0D;YAC1D,IAAII,sBAAsBN,4BAA4BY,UAAU,CAACV,KAAK,EAAE;gBACtEE,OAAOO,KAAK,CAAC,CAAC,4CAA4C,EAAET,MAAM;gBAClE,MAAMI,mBAAmBO,GAAG,CAACX,MAAMC;YACrC;YAEA,OAAOH,4BAA4BY,UAAU,CAACV,KAAK;QACrD;QAEA,OAAOF,4BAA4BU,SAAS,CAACR,KAAK;IACpD;IAEA,MAAaW,IAAIX,IAAY,EAAEC,MAAc,EAAiB;QAC5D,MAAM,EAAEC,MAAM,EAAEC,mBAAmB,EAAEC,kBAAkB,EAAEQ,gBAAgB,EAAE,GAAG,IAAI,CAACP,OAAO;QAC1F,MAAMQ,cAA+B,EAAE;QAEvC,2FAA2F;QAC3F,MAAMC,wBAAwB,CAAC,IAAI,CAACC,UAAU,CAACf,SAAS,CAAC,CAACI;QAE1D,IAAIU,uBAAuB;YACzBZ,OAAOO,KAAK,CAAC,CAAC,iBAAiB,EAAET,MAAM;YACvCa,YAAYG,IAAI,CAACZ,mBAAmBO,GAAG,CAACX,MAAMC;QAChD;QAEA;;;;;;KAMC,GACD,MAAMK,2BAA2BL,OAAOM,eAAe,KAAK;QAC5D,MAAMU,yBAAyB,CAACX,4BAA4B,CAAC,IAAI,CAACY,WAAW,CAAClB,SAAS,CAAC,CAACG,uBAAuBS;QAEhH,IAAIK,wBAAwB;YAC1Bf,OAAOO,KAAK,CAAC,CAAC,qBAAqB,EAAET,MAAM;YAC3C,MAAMmB,YAAYhB,oBAAoBQ,GAAG,CAACX,MAAMC;YAChDY,YAAYG,IAAI,CAACG;QACnB;QAEA,MAAMC,QAAQC,GAAG,CAACR;IACpB;IAEQK,YAAYlB,IAAY,EAAE;QAChC,OAAOA,QAAQF,4BAA4BY,UAAU,IAAIZ,4BAA4BY,UAAU,CAACV,KAAK;IACvG;IAEQe,WAAWf,IAAY,EAAE;QAC/B,OAAOA,QAAQF,4BAA4BU,SAAS,IAAIV,4BAA4BU,SAAS,CAACR,KAAK;IACrG;IAEA,MAAasB,QAAuB;QAClC,MAAM,EAAElB,kBAAkB,EAAE,GAAG,IAAI,CAACC,OAAO;QAC3C,IAAID,oBAAoB;YACtB,OAAOA,mBAAmBkB,KAAK;QACjC;IACF;IAEA,MAAaC,MAAMC,SAAiB,EAAiB;QACnD,MAAM,EAAEpB,kBAAkB,EAAE,GAAG,IAAI,CAACC,OAAO;QAC3C,IAAID,oBAAoB;YACtB,OAAOA,mBAAmBmB,KAAK,CAACC;QAClC;IACF;IA9EA,YAAY,AAAQnB,OAA2C,CAAE;;aAA7CA,UAAAA;IAA8C;AA+EpE;AAlFE,iBADWP,6BACIU,aAAyC,CAAC;AACzD,iBAFWV,6BAEIY,cAA0C,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lage-run/cache",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.5.0",
|
|
4
4
|
"description": "Cache for Lage",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -21,17 +21,19 @@
|
|
|
21
21
|
"dependencies": {
|
|
22
22
|
"@azure/core-auth": "^1.9.0",
|
|
23
23
|
"@azure/identity": "^4.9.1",
|
|
24
|
-
"@lage-run/config": "^0.9.
|
|
25
|
-
"@lage-run/logger": "^1.
|
|
26
|
-
"@lage-run/target-graph": "^0.
|
|
27
|
-
"backfill-cache": "^5.12.
|
|
24
|
+
"@lage-run/config": "^0.9.4",
|
|
25
|
+
"@lage-run/logger": "^1.4.0",
|
|
26
|
+
"@lage-run/target-graph": "^0.15.0",
|
|
27
|
+
"backfill-cache": "^5.12.3",
|
|
28
28
|
"backfill-config": "^6.7.2",
|
|
29
29
|
"backfill-logger": "^5.4.1",
|
|
30
30
|
"glob-hasher": "^1.4.2"
|
|
31
31
|
},
|
|
32
32
|
"devDependencies": {
|
|
33
33
|
"@lage-run/monorepo-scripts": "^1.0.0",
|
|
34
|
-
"@lage-run/test-utilities": "^0.1.0"
|
|
34
|
+
"@lage-run/test-utilities": "^0.1.0",
|
|
35
|
+
"@types/fs-extra": "^8.0.0",
|
|
36
|
+
"fs-extra": "8.1.0"
|
|
35
37
|
},
|
|
36
38
|
"files": [
|
|
37
39
|
"lib/!(__*)",
|