@lage-run/cache 1.4.10 → 1.4.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,3 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Get the root cache directory: `<root>/node_modules/.cache/lage`
|
|
3
|
+
*/
|
|
1
4
|
export declare function getCacheDirectoryRoot(root: string): string;
|
|
5
|
+
/**
|
|
6
|
+
* Get the cache subdirectory for a hash: `<root>/node_modules/.cache/lage/cache/ab12`
|
|
7
|
+
* for a hash that starts with `ab12`
|
|
8
|
+
*/
|
|
2
9
|
export declare function getCacheDirectory(root: string, hash: string): string;
|
|
10
|
+
/**
|
|
11
|
+
* Get the logs subdirectory for a hash: `<root>/node_modules/.cache/lage/logs/ab12`
|
|
12
|
+
* for a hash that starts with `ab12`
|
|
13
|
+
*/
|
|
3
14
|
export declare function getLogsCacheDirectory(root: string, hash: string): string;
|
|
15
|
+
/**
|
|
16
|
+
* Get the parent `cache` and `logs` directories that contain subfolders with hash prefixes.
|
|
17
|
+
*/
|
|
18
|
+
export declare function getHashCacheDirectories(root: string): {
|
|
19
|
+
/** `<root>/node_modules/.cache/lage/cache` */
|
|
20
|
+
cache: string;
|
|
21
|
+
/** `<root>/node_modules/.cache/lage/logs` */
|
|
22
|
+
logs: string;
|
|
23
|
+
};
|
package/lib/getCacheDirectory.js
CHANGED
|
@@ -15,6 +15,9 @@ _export(exports, {
|
|
|
15
15
|
get getCacheDirectoryRoot () {
|
|
16
16
|
return getCacheDirectoryRoot;
|
|
17
17
|
},
|
|
18
|
+
get getHashCacheDirectories () {
|
|
19
|
+
return getHashCacheDirectories;
|
|
20
|
+
},
|
|
18
21
|
get getLogsCacheDirectory () {
|
|
19
22
|
return getLogsCacheDirectory;
|
|
20
23
|
}
|
|
@@ -29,8 +32,14 @@ function getCacheDirectoryRoot(root) {
|
|
|
29
32
|
return _path.default.join(root, "node_modules", ".cache", "lage");
|
|
30
33
|
}
|
|
31
34
|
function getCacheDirectory(root, hash) {
|
|
32
|
-
return _path.default.join(
|
|
35
|
+
return _path.default.join(getHashCacheDirectories(root).cache, hash.substring(0, 4));
|
|
33
36
|
}
|
|
34
37
|
function getLogsCacheDirectory(root, hash) {
|
|
35
|
-
return _path.default.join(
|
|
38
|
+
return _path.default.join(getHashCacheDirectories(root).logs, hash.substring(0, 4));
|
|
39
|
+
}
|
|
40
|
+
function getHashCacheDirectories(root) {
|
|
41
|
+
return {
|
|
42
|
+
cache: _path.default.join(getCacheDirectoryRoot(root), "cache"),
|
|
43
|
+
logs: _path.default.join(getCacheDirectoryRoot(root), "logs")
|
|
44
|
+
};
|
|
36
45
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/getCacheDirectory.ts"],"sourcesContent":["import path from \"path\";\n\nexport function getCacheDirectoryRoot(root: string): string {\n return path.join(root, \"node_modules\", \".cache\", \"lage\");\n}\n\nexport function getCacheDirectory(root: string, hash: string): string {\n return path.join(
|
|
1
|
+
{"version":3,"sources":["../src/getCacheDirectory.ts"],"sourcesContent":["import path from \"path\";\n\n/**\n * Get the root cache directory: `<root>/node_modules/.cache/lage`\n */\nexport function getCacheDirectoryRoot(root: string): string {\n return path.join(root, \"node_modules\", \".cache\", \"lage\");\n}\n\n/**\n * Get the cache subdirectory for a hash: `<root>/node_modules/.cache/lage/cache/ab12`\n * for a hash that starts with `ab12`\n */\nexport function getCacheDirectory(root: string, hash: string): string {\n return path.join(getHashCacheDirectories(root).cache, hash.substring(0, 4));\n}\n\n/**\n * Get the logs subdirectory for a hash: `<root>/node_modules/.cache/lage/logs/ab12`\n * for a hash that starts with `ab12`\n */\nexport function getLogsCacheDirectory(root: string, hash: string): string {\n return path.join(getHashCacheDirectories(root).logs, hash.substring(0, 4));\n}\n\n/**\n * Get the parent `cache` and `logs` directories that contain subfolders with hash prefixes.\n */\nexport function getHashCacheDirectories(root: string): {\n /** `<root>/node_modules/.cache/lage/cache` */\n cache: string;\n /** `<root>/node_modules/.cache/lage/logs` */\n logs: string;\n} {\n return {\n cache: path.join(getCacheDirectoryRoot(root), \"cache\"),\n logs: path.join(getCacheDirectoryRoot(root), \"logs\"),\n };\n}\n"],"names":["getCacheDirectory","getCacheDirectoryRoot","getHashCacheDirectories","getLogsCacheDirectory","root","path","join","hash","cache","substring","logs"],"mappings":";;;;;;;;;;;QAagBA;eAAAA;;QARAC;eAAAA;;QAuBAC;eAAAA;;QAPAC;eAAAA;;;6DArBC;;;;;;AAKV,SAASF,sBAAsBG,IAAY;IAChD,OAAOC,aAAI,CAACC,IAAI,CAACF,MAAM,gBAAgB,UAAU;AACnD;AAMO,SAASJ,kBAAkBI,IAAY,EAAEG,IAAY;IAC1D,OAAOF,aAAI,CAACC,IAAI,CAACJ,wBAAwBE,MAAMI,KAAK,EAAED,KAAKE,SAAS,CAAC,GAAG;AAC1E;AAMO,SAASN,sBAAsBC,IAAY,EAAEG,IAAY;IAC9D,OAAOF,aAAI,CAACC,IAAI,CAACJ,wBAAwBE,MAAMM,IAAI,EAAEH,KAAKE,SAAS,CAAC,GAAG;AACzE;AAKO,SAASP,wBAAwBE,IAAY;IAMlD,OAAO;QACLI,OAAOH,aAAI,CAACC,IAAI,CAACL,sBAAsBG,OAAO;QAC9CM,MAAML,aAAI,CAACC,IAAI,CAACL,sBAAsBG,OAAO;IAC/C;AACF"}
|
|
@@ -128,18 +128,16 @@ class BackfillCacheProvider {
|
|
|
128
128
|
}
|
|
129
129
|
async purge(prunePeriod = 30, concurrency = 10) {
|
|
130
130
|
const now = new Date();
|
|
131
|
-
const
|
|
132
|
-
"cache",
|
|
133
|
-
"logs"
|
|
134
|
-
];
|
|
131
|
+
const directories = (0, _getCacheDirectory.getHashCacheDirectories)(this.options.root);
|
|
135
132
|
const entries = [];
|
|
136
|
-
for (const
|
|
137
|
-
|
|
133
|
+
for (const cacheTypeDirectory of [
|
|
134
|
+
directories.cache,
|
|
135
|
+
directories.logs
|
|
136
|
+
]){
|
|
138
137
|
if (_fs.existsSync(cacheTypeDirectory)) {
|
|
139
138
|
const hashPrefixes = await readdir(cacheTypeDirectory);
|
|
140
139
|
for (const prefix of hashPrefixes){
|
|
141
|
-
|
|
142
|
-
entries.push(cachePath);
|
|
140
|
+
entries.push(_path.join(cacheTypeDirectory, prefix));
|
|
143
141
|
}
|
|
144
142
|
}
|
|
145
143
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/providers/BackfillCacheProvider.ts"],"sourcesContent":["import { createBackfillCacheConfig, createBackfillLogger } from \"../backfillWrapper.js\";\nimport { getCacheStorageProvider } from \"backfill-cache\";\nimport { promisify } from \"util\";\nimport * as fs from \"fs\";\nimport * as path from \"path\";\nimport type { CacheProvider } from \"../types/CacheProvider.js\";\nimport type { CacheOptions } from \"@lage-run/config\";\nimport type { Logger as BackfillLogger } from \"backfill-logger\";\nimport type { Target } from \"@lage-run/target-graph\";\nimport type { Logger } from \"@lage-run/logger\";\nimport { getCacheDirectory,
|
|
1
|
+
{"version":3,"sources":["../../src/providers/BackfillCacheProvider.ts"],"sourcesContent":["import { createBackfillCacheConfig, createBackfillLogger } from \"../backfillWrapper.js\";\nimport { getCacheStorageProvider } from \"backfill-cache\";\nimport { promisify } from \"util\";\nimport * as fs from \"fs\";\nimport * as path from \"path\";\nimport type { CacheProvider } from \"../types/CacheProvider.js\";\nimport type { CacheOptions } from \"@lage-run/config\";\nimport type { Logger as BackfillLogger } from \"backfill-logger\";\nimport type { Target } from \"@lage-run/target-graph\";\nimport type { Logger } from \"@lage-run/logger\";\nimport { getCacheDirectory, getHashCacheDirectories } from \"../getCacheDirectory.js\";\nimport { chunkPromise } from \"../chunkPromise.js\";\n\nconst rm = promisify(fs.rm);\nconst readdir = promisify(fs.readdir);\nconst stat = promisify(fs.stat);\n\nconst MS_IN_A_DAY = 1000 * 60 * 60 * 24;\n\nexport interface BackfillCacheProviderOptions {\n root: string;\n logger: Logger;\n cacheOptions: Partial<CacheOptions>;\n}\n\nexport class BackfillCacheProvider implements CacheProvider {\n /**\n * logger for backfill\n */\n private backfillLogger: BackfillLogger;\n\n private getTargetCacheStorageProvider(cwd: string, hash: string) {\n const { cacheOptions } = this.options;\n const { cacheStorageConfig, incrementalCaching } = createBackfillCacheConfig(cwd, cacheOptions, this.backfillLogger);\n\n const cachePath = this.getCachePath(cwd, hash);\n\n return getCacheStorageProvider(cacheStorageConfig ?? { provider: \"local\" }, cachePath, this.backfillLogger, cwd, incrementalCaching);\n }\n\n constructor(private options: BackfillCacheProviderOptions) {\n this.backfillLogger = createBackfillLogger();\n }\n\n public async fetch(hash: string, target: Target): Promise<boolean> {\n const { logger } = this.options;\n\n if (!hash) {\n return false;\n }\n\n const cacheStorage = this.getTargetCacheStorageProvider(target.cwd, hash);\n\n try {\n return await cacheStorage.fetch(hash);\n } catch (error) {\n let message;\n\n if (error instanceof Error) {\n message = error.message;\n } else message = String(error);\n\n logger.silly(`Cache fetch failed: ${message}`, { target });\n\n // backfill fetch can error, but we should simply ignore and continue\n return false;\n }\n }\n\n public async put(hash: string, target: Target): Promise<void> {\n const { logger } = this.options;\n\n if (!hash) {\n return;\n }\n\n const cacheStorage = this.getTargetCacheStorageProvider(target.cwd, hash);\n\n try {\n await cacheStorage.put(hash, target.outputs ?? this.options.cacheOptions.outputGlob ?? [\"**/*\"]);\n } catch (error) {\n let message;\n\n if (error instanceof Error) {\n message = error.message;\n } else message = String(error);\n\n logger.silly(`Cache fetch failed: ${message}`, { target });\n\n // backfill throws an error if outputGlob doesn't match any files, we will skip this error\n }\n }\n\n public async clear(concurrency = 10): Promise<void> {\n return this.purge(0, concurrency);\n }\n\n public async purge(prunePeriod = 30, concurrency = 10): Promise<void> {\n const now = new Date();\n\n const directories = getHashCacheDirectories(this.options.root);\n const entries: string[] = [];\n\n for (const cacheTypeDirectory of [directories.cache, directories.logs]) {\n if (fs.existsSync(cacheTypeDirectory)) {\n const hashPrefixes = await readdir(cacheTypeDirectory);\n for (const prefix of hashPrefixes) {\n entries.push(path.join(cacheTypeDirectory, prefix));\n }\n }\n }\n\n await chunkPromise(\n entries.map((entry) => {\n return async () => {\n const entryPath = entry;\n const entryStat = await stat(entryPath);\n\n if (now.getTime() - entryStat.mtime.getTime() > prunePeriod * MS_IN_A_DAY) {\n await removeCache(entryPath, entryStat);\n }\n };\n }),\n concurrency\n );\n }\n\n private getCachePath(packagePath: string, hash: string): string {\n return path.relative(packagePath, getCacheDirectory(this.options.root, hash));\n }\n}\n\nasync function removeCache(cachePath: string, entryStat: fs.Stats) {\n if (entryStat.isDirectory()) {\n return rm(cachePath, { recursive: true });\n } else {\n return rm(cachePath);\n }\n}\n"],"names":["BackfillCacheProvider","rm","promisify","fs","readdir","stat","MS_IN_A_DAY","getTargetCacheStorageProvider","cwd","hash","cacheOptions","options","cacheStorageConfig","incrementalCaching","createBackfillCacheConfig","backfillLogger","cachePath","getCachePath","getCacheStorageProvider","provider","fetch","target","logger","cacheStorage","error","message","Error","String","silly","put","outputs","outputGlob","clear","concurrency","purge","prunePeriod","now","Date","directories","getHashCacheDirectories","root","entries","cacheTypeDirectory","cache","logs","existsSync","hashPrefixes","prefix","push","path","join","chunkPromise","map","entry","entryPath","entryStat","getTime","mtime","removeCache","packagePath","relative","getCacheDirectory","createBackfillLogger","isDirectory","recursive"],"mappings":";;;;+BAyBaA;;;eAAAA;;;iCAzBmD;+BACxB;sBACd;4DACN;8DACE;mCAMqC;8BAC9B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE7B,MAAMC,KAAKC,IAAAA,eAAS,EAACC,IAAGF,EAAE;AAC1B,MAAMG,UAAUF,IAAAA,eAAS,EAACC,IAAGC,OAAO;AACpC,MAAMC,OAAOH,IAAAA,eAAS,EAACC,IAAGE,IAAI;AAE9B,MAAMC,cAAc,OAAO,KAAK,KAAK;AAQ9B,MAAMN;IAMHO,8BAA8BC,GAAW,EAAEC,IAAY,EAAE;QAC/D,MAAM,EAAEC,YAAY,EAAE,GAAG,IAAI,CAACC,OAAO;QACrC,MAAM,EAAEC,kBAAkB,EAAEC,kBAAkB,EAAE,GAAGC,IAAAA,0CAAyB,EAACN,KAAKE,cAAc,IAAI,CAACK,cAAc;QAEnH,MAAMC,YAAY,IAAI,CAACC,YAAY,CAACT,KAAKC;QAEzC,OAAOS,IAAAA,sCAAuB,EAACN,sBAAsB;YAAEO,UAAU;QAAQ,GAAGH,WAAW,IAAI,CAACD,cAAc,EAAEP,KAAKK;IACnH;IAMA,MAAaO,MAAMX,IAAY,EAAEY,MAAc,EAAoB;QACjE,MAAM,EAAEC,MAAM,EAAE,GAAG,IAAI,CAACX,OAAO;QAE/B,IAAI,CAACF,MAAM;YACT,OAAO;QACT;QAEA,MAAMc,eAAe,IAAI,CAAChB,6BAA6B,CAACc,OAAOb,GAAG,EAAEC;QAEpE,IAAI;YACF,OAAO,MAAMc,aAAaH,KAAK,CAACX;QAClC,EAAE,OAAOe,OAAO;YACd,IAAIC;YAEJ,IAAID,iBAAiBE,OAAO;gBAC1BD,UAAUD,MAAMC,OAAO;YACzB,OAAOA,UAAUE,OAAOH;YAExBF,OAAOM,KAAK,CAAC,CAAC,oBAAoB,EAAEH,SAAS,EAAE;gBAAEJ;YAAO;YAExD,qEAAqE;YACrE,OAAO;QACT;IACF;IAEA,MAAaQ,IAAIpB,IAAY,EAAEY,MAAc,EAAiB;QAC5D,MAAM,EAAEC,MAAM,EAAE,GAAG,IAAI,CAACX,OAAO;QAE/B,IAAI,CAACF,MAAM;YACT;QACF;QAEA,MAAMc,eAAe,IAAI,CAAChB,6BAA6B,CAACc,OAAOb,GAAG,EAAEC;QAEpE,IAAI;YACF,MAAMc,aAAaM,GAAG,CAACpB,MAAMY,OAAOS,OAAO,IAAI,IAAI,CAACnB,OAAO,CAACD,YAAY,CAACqB,UAAU,IAAI;gBAAC;aAAO;QACjG,EAAE,OAAOP,OAAO;YACd,IAAIC;YAEJ,IAAID,iBAAiBE,OAAO;gBAC1BD,UAAUD,MAAMC,OAAO;YACzB,OAAOA,UAAUE,OAAOH;YAExBF,OAAOM,KAAK,CAAC,CAAC,oBAAoB,EAAEH,SAAS,EAAE;gBAAEJ;YAAO;QAExD,0FAA0F;QAC5F;IACF;IAEA,MAAaW,MAAMC,cAAc,EAAE,EAAiB;QAClD,OAAO,IAAI,CAACC,KAAK,CAAC,GAAGD;IACvB;IAEA,MAAaC,MAAMC,cAAc,EAAE,EAAEF,cAAc,EAAE,EAAiB;QACpE,MAAMG,MAAM,IAAIC;QAEhB,MAAMC,cAAcC,IAAAA,0CAAuB,EAAC,IAAI,CAAC5B,OAAO,CAAC6B,IAAI;QAC7D,MAAMC,UAAoB,EAAE;QAE5B,KAAK,MAAMC,sBAAsB;YAACJ,YAAYK,KAAK;YAAEL,YAAYM,IAAI;SAAC,CAAE;YACtE,IAAIzC,IAAG0C,UAAU,CAACH,qBAAqB;gBACrC,MAAMI,eAAe,MAAM1C,QAAQsC;gBACnC,KAAK,MAAMK,UAAUD,aAAc;oBACjCL,QAAQO,IAAI,CAACC,MAAKC,IAAI,CAACR,oBAAoBK;gBAC7C;YACF;QACF;QAEA,MAAMI,IAAAA,0BAAY,EAChBV,QAAQW,GAAG,CAAC,CAACC;YACX,OAAO;gBACL,MAAMC,YAAYD;gBAClB,MAAME,YAAY,MAAMlD,KAAKiD;gBAE7B,IAAIlB,IAAIoB,OAAO,KAAKD,UAAUE,KAAK,CAACD,OAAO,KAAKrB,cAAc7B,aAAa;oBACzE,MAAMoD,YAAYJ,WAAWC;gBAC/B;YACF;QACF,IACAtB;IAEJ;IAEQhB,aAAa0C,WAAmB,EAAElD,IAAY,EAAU;QAC9D,OAAOwC,MAAKW,QAAQ,CAACD,aAAaE,IAAAA,oCAAiB,EAAC,IAAI,CAAClD,OAAO,CAAC6B,IAAI,EAAE/B;IACzE;IAzFA,YAAY,AAAQE,OAAqC,CAAE;;QAd3D;;GAEC,GACD,uBAAQI,kBAAR,KAAA;aAWoBJ,UAAAA;QAClB,IAAI,CAACI,cAAc,GAAG+C,IAAAA,qCAAoB;IAC5C;AAwFF;AAEA,eAAeJ,YAAY1C,SAAiB,EAAEuC,SAAmB;IAC/D,IAAIA,UAAUQ,WAAW,IAAI;QAC3B,OAAO9D,GAAGe,WAAW;YAAEgD,WAAW;QAAK;IACzC,OAAO;QACL,OAAO/D,GAAGe;IACZ;AACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lage-run/cache",
|
|
3
|
-
"version": "1.4.
|
|
3
|
+
"version": "1.4.12",
|
|
4
4
|
"description": "Cache for Lage",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -21,17 +21,19 @@
|
|
|
21
21
|
"dependencies": {
|
|
22
22
|
"@azure/core-auth": "^1.9.0",
|
|
23
23
|
"@azure/identity": "^4.9.1",
|
|
24
|
-
"@lage-run/config": "^0.9.
|
|
24
|
+
"@lage-run/config": "^0.9.3",
|
|
25
25
|
"@lage-run/logger": "^1.3.3",
|
|
26
|
-
"@lage-run/target-graph": "^0.14.
|
|
27
|
-
"backfill-cache": "^5.12.
|
|
26
|
+
"@lage-run/target-graph": "^0.14.2",
|
|
27
|
+
"backfill-cache": "^5.12.3",
|
|
28
28
|
"backfill-config": "^6.7.2",
|
|
29
29
|
"backfill-logger": "^5.4.1",
|
|
30
30
|
"glob-hasher": "^1.4.2"
|
|
31
31
|
},
|
|
32
32
|
"devDependencies": {
|
|
33
33
|
"@lage-run/monorepo-scripts": "^1.0.0",
|
|
34
|
-
"@lage-run/test-utilities": "^0.1.0"
|
|
34
|
+
"@lage-run/test-utilities": "^0.1.0",
|
|
35
|
+
"@types/fs-extra": "^8.0.0",
|
|
36
|
+
"fs-extra": "8.1.0"
|
|
35
37
|
},
|
|
36
38
|
"files": [
|
|
37
39
|
"lib/!(__*)",
|