file-entry-cache 7.0.2 → 9.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -2
- package/cache.js +244 -237
- package/package.json +21 -17
package/README.md
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
# file-entry-cache
|
|
2
|
-
> Super simple cache for file metadata, useful for process that work on a given series of files
|
|
3
|
-
> and that only need to repeat the job on the changed ones since the previous run of the process — Edit
|
|
2
|
+
> Super simple cache for file metadata, useful for process that work on a given series of files and that only need to repeat the job on the changed ones since the previous run of the process
|
|
4
3
|
|
|
5
4
|
[](https://npmjs.org/package/file-entry-cache)
|
|
6
5
|
[](https://github.com/jaredwray/file-entry-cache/actions/workflows/tests.yaml)
|
package/cache.js
CHANGED
|
@@ -1,64 +1,66 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
/* eslint-disable unicorn/no-this-assignment, func-names, no-multi-assign */
|
|
2
|
+
const path = require('node:path');
|
|
3
|
+
const process = require('node:process');
|
|
4
|
+
const crypto = require('node:crypto');
|
|
3
5
|
|
|
4
6
|
module.exports = {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
*
|
|
7
|
+
createFromFile(filePath, useChecksum) {
|
|
8
|
+
const fname = path.basename(filePath);
|
|
9
|
+
const dir = path.dirname(filePath);
|
|
10
|
+
return this.create(fname, dir, useChecksum);
|
|
11
|
+
},
|
|
12
|
+
|
|
13
|
+
create(cacheId, _path, useChecksum) {
|
|
14
|
+
const fs = require('node:fs');
|
|
15
|
+
const flatCache = require('flat-cache');
|
|
16
|
+
const cache = flatCache.load(cacheId, _path);
|
|
17
|
+
let normalizedEntries = {};
|
|
18
|
+
|
|
19
|
+
const removeNotFoundFiles = function removeNotFoundFiles() {
|
|
20
|
+
const cachedEntries = cache.keys();
|
|
21
|
+
// Remove not found entries
|
|
22
|
+
for (const fPath of cachedEntries) {
|
|
23
|
+
try {
|
|
24
|
+
fs.statSync(fPath);
|
|
25
|
+
} catch (error) {
|
|
26
|
+
if (error.code === 'ENOENT') {
|
|
27
|
+
cache.removeKey(fPath);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
removeNotFoundFiles();
|
|
34
|
+
|
|
35
|
+
return {
|
|
36
|
+
/**
|
|
37
|
+
* The flat cache storage used to persist the metadata of the `files
|
|
36
38
|
* @type {Object}
|
|
37
39
|
*/
|
|
38
|
-
|
|
40
|
+
cache,
|
|
39
41
|
|
|
40
|
-
|
|
42
|
+
/**
|
|
41
43
|
* Given a buffer, calculate md5 hash of its content.
|
|
42
44
|
* @method getHash
|
|
43
45
|
* @param {Buffer} buffer buffer to calculate hash on
|
|
44
46
|
* @return {String} content hash digest
|
|
45
47
|
*/
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
48
|
+
getHash(buffer) {
|
|
49
|
+
return crypto.createHash('md5').update(buffer).digest('hex');
|
|
50
|
+
},
|
|
49
51
|
|
|
50
|
-
|
|
52
|
+
/**
|
|
51
53
|
* Return whether or not a file has changed since last time reconcile was called.
|
|
52
54
|
* @method hasFileChanged
|
|
53
55
|
* @param {String} file the filepath to check
|
|
54
56
|
* @return {Boolean} wheter or not the file has changed
|
|
55
57
|
*/
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
58
|
+
hasFileChanged(file) {
|
|
59
|
+
return this.getFileDescriptor(file).changed;
|
|
60
|
+
},
|
|
59
61
|
|
|
60
|
-
|
|
61
|
-
*
|
|
62
|
+
/**
|
|
63
|
+
* Given an array of file paths it return and object with three arrays:
|
|
62
64
|
* - changedFiles: Files that changed since previous run
|
|
63
65
|
* - notChangedFiles: Files that haven't change
|
|
64
66
|
* - notFoundFiles: Files that were not found, probably deleted
|
|
@@ -66,103 +68,110 @@ module.exports = {
|
|
|
66
68
|
* @param {Array} files the files to analyze and compare to the previous seen files
|
|
67
69
|
* @return {[type]} [description]
|
|
68
70
|
*/
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
71
|
+
analyzeFiles(files) {
|
|
72
|
+
const me = this;
|
|
73
|
+
files ||= [];
|
|
74
|
+
|
|
75
|
+
const res = {
|
|
76
|
+
changedFiles: [],
|
|
77
|
+
notFoundFiles: [],
|
|
78
|
+
notChangedFiles: [],
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
for (const entry of me.normalizeEntries(files)) {
|
|
82
|
+
if (entry.changed) {
|
|
83
|
+
res.changedFiles.push(entry.key);
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
if (entry.notFound) {
|
|
88
|
+
res.notFoundFiles.push(entry.key);
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
res.notChangedFiles.push(entry.key);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
return res;
|
|
96
|
+
},
|
|
97
|
+
|
|
98
|
+
getFileDescriptor(file) {
|
|
99
|
+
let fstat;
|
|
100
|
+
|
|
101
|
+
try {
|
|
102
|
+
if (!path.isAbsolute(file)) {
|
|
103
|
+
file = path.resolve(process.cwd(), file);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
fstat = fs.statSync(file);
|
|
107
|
+
} catch (error) {
|
|
108
|
+
this.removeEntry(file);
|
|
109
|
+
return {key: file, notFound: true, err: error};
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
if (useChecksum) {
|
|
113
|
+
return this._getFileDescriptorUsingChecksum(file);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
return this._getFileDescriptorUsingMtimeAndSize(file, fstat);
|
|
117
|
+
},
|
|
118
|
+
|
|
119
|
+
_getFileDescriptorUsingMtimeAndSize(file, fstat) {
|
|
120
|
+
let meta = cache.getKey(file);
|
|
121
|
+
const cacheExists = Boolean(meta);
|
|
122
|
+
|
|
123
|
+
const cSize = fstat.size;
|
|
124
|
+
const cTime = fstat.mtime.getTime();
|
|
125
|
+
|
|
126
|
+
let isDifferentDate;
|
|
127
|
+
let isDifferentSize;
|
|
128
|
+
|
|
129
|
+
if (meta) {
|
|
130
|
+
isDifferentDate = cTime !== meta.mtime;
|
|
131
|
+
isDifferentSize = cSize !== meta.size;
|
|
132
|
+
} else {
|
|
133
|
+
meta = {size: cSize, mtime: cTime};
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const nEntry = (normalizedEntries[file] = {
|
|
137
|
+
key: file,
|
|
138
|
+
changed: !cacheExists || isDifferentDate || isDifferentSize,
|
|
139
|
+
meta,
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
return nEntry;
|
|
143
|
+
},
|
|
144
|
+
|
|
145
|
+
_getFileDescriptorUsingChecksum(file) {
|
|
146
|
+
let meta = cache.getKey(file);
|
|
147
|
+
const cacheExists = Boolean(meta);
|
|
148
|
+
|
|
149
|
+
let contentBuffer;
|
|
150
|
+
try {
|
|
151
|
+
contentBuffer = fs.readFileSync(file);
|
|
152
|
+
} catch {
|
|
153
|
+
contentBuffer = '';
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
let isDifferent = true;
|
|
157
|
+
const hash = this.getHash(contentBuffer);
|
|
158
|
+
|
|
159
|
+
if (meta) {
|
|
160
|
+
isDifferent = hash !== meta.hash;
|
|
161
|
+
} else {
|
|
162
|
+
meta = {hash};
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const nEntry = (normalizedEntries[file] = {
|
|
166
|
+
key: file,
|
|
167
|
+
changed: !cacheExists || isDifferent,
|
|
168
|
+
meta,
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
return nEntry;
|
|
172
|
+
},
|
|
173
|
+
|
|
174
|
+
/**
|
|
166
175
|
* Return the list o the files that changed compared
|
|
167
176
|
* against the ones stored in the cache
|
|
168
177
|
*
|
|
@@ -170,122 +179,120 @@ module.exports = {
|
|
|
170
179
|
* @param files {Array} the array of files to compare against the ones in the cache
|
|
171
180
|
* @returns {Array}
|
|
172
181
|
*/
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
},
|
|
186
|
-
|
|
187
|
-
/**
|
|
188
|
-
* return the list of files
|
|
182
|
+
getUpdatedFiles(files) {
|
|
183
|
+
const me = this;
|
|
184
|
+
files ||= [];
|
|
185
|
+
|
|
186
|
+
return me
|
|
187
|
+
.normalizeEntries(files)
|
|
188
|
+
.filter(entry => entry.changed)
|
|
189
|
+
.map(entry => entry.key);
|
|
190
|
+
},
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Return the list of files
|
|
189
194
|
* @method normalizeEntries
|
|
190
195
|
* @param files
|
|
191
196
|
* @returns {*}
|
|
192
197
|
*/
|
|
193
|
-
|
|
194
|
-
|
|
198
|
+
normalizeEntries(files) {
|
|
199
|
+
files ||= [];
|
|
195
200
|
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
return me.getFileDescriptor(file);
|
|
199
|
-
});
|
|
201
|
+
const me = this;
|
|
202
|
+
const nEntries = files.map(file => me.getFileDescriptor(file));
|
|
200
203
|
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
+
// NormalizeEntries = nEntries;
|
|
205
|
+
return nEntries;
|
|
206
|
+
},
|
|
204
207
|
|
|
205
|
-
|
|
208
|
+
/**
|
|
206
209
|
* Remove an entry from the file-entry-cache. Useful to force the file to still be considered
|
|
207
210
|
* modified the next time the process is run
|
|
208
211
|
*
|
|
209
212
|
* @method removeEntry
|
|
210
213
|
* @param entryName
|
|
211
214
|
*/
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
215
|
+
removeEntry(entryName) {
|
|
216
|
+
if (!path.isAbsolute(entryName)) {
|
|
217
|
+
entryName = path.resolve(process.cwd(), entryName);
|
|
218
|
+
}
|
|
216
219
|
|
|
217
|
-
|
|
220
|
+
delete normalizedEntries[entryName];
|
|
221
|
+
cache.removeKey(entryName);
|
|
222
|
+
},
|
|
223
|
+
|
|
224
|
+
/**
|
|
218
225
|
* Delete the cache file from the disk
|
|
219
226
|
* @method deleteCacheFile
|
|
220
227
|
*/
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
228
|
+
deleteCacheFile() {
|
|
229
|
+
cache.removeCacheFile();
|
|
230
|
+
},
|
|
224
231
|
|
|
225
|
-
|
|
226
|
-
*
|
|
232
|
+
/**
|
|
233
|
+
* Remove the cache from the file and clear the memory cache
|
|
227
234
|
*/
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
235
|
+
destroy() {
|
|
236
|
+
normalizedEntries = {};
|
|
237
|
+
cache.destroy();
|
|
238
|
+
},
|
|
239
|
+
|
|
240
|
+
_getMetaForFileUsingCheckSum(cacheEntry) {
|
|
241
|
+
const contentBuffer = fs.readFileSync(cacheEntry.key);
|
|
242
|
+
const hash = this.getHash(contentBuffer);
|
|
243
|
+
const meta = Object.assign(cacheEntry.meta, {hash});
|
|
244
|
+
delete meta.size;
|
|
245
|
+
delete meta.mtime;
|
|
246
|
+
return meta;
|
|
247
|
+
},
|
|
248
|
+
|
|
249
|
+
_getMetaForFileUsingMtimeAndSize(cacheEntry) {
|
|
250
|
+
const stat = fs.statSync(cacheEntry.key);
|
|
251
|
+
const meta = Object.assign(cacheEntry.meta, {
|
|
252
|
+
size: stat.size,
|
|
253
|
+
mtime: stat.mtime.getTime(),
|
|
254
|
+
});
|
|
255
|
+
delete meta.hash;
|
|
256
|
+
return meta;
|
|
257
|
+
},
|
|
258
|
+
|
|
259
|
+
/**
|
|
253
260
|
* Sync the files and persist them to the cache
|
|
254
261
|
* @method reconcile
|
|
255
262
|
*/
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
263
|
+
reconcile(noPrune) {
|
|
264
|
+
removeNotFoundFiles();
|
|
265
|
+
|
|
266
|
+
noPrune = noPrune === undefined ? true : noPrune;
|
|
267
|
+
|
|
268
|
+
const entries = normalizedEntries;
|
|
269
|
+
const keys = Object.keys(entries);
|
|
270
|
+
|
|
271
|
+
if (keys.length === 0) {
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
const me = this;
|
|
276
|
+
|
|
277
|
+
for (const entryName of keys) {
|
|
278
|
+
const cacheEntry = entries[entryName];
|
|
279
|
+
|
|
280
|
+
try {
|
|
281
|
+
const meta = useChecksum
|
|
282
|
+
? me._getMetaForFileUsingCheckSum(cacheEntry)
|
|
283
|
+
: me._getMetaForFileUsingMtimeAndSize(cacheEntry);
|
|
284
|
+
cache.setKey(entryName, meta);
|
|
285
|
+
} catch (error) {
|
|
286
|
+
// If the file does not exists we don't save it
|
|
287
|
+
// other errors are just thrown
|
|
288
|
+
if (error.code !== 'ENOENT') {
|
|
289
|
+
throw error;
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
cache.save(noPrune);
|
|
295
|
+
},
|
|
296
|
+
};
|
|
297
|
+
},
|
|
291
298
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "file-entry-cache",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "9.0.0",
|
|
4
4
|
"description": "Super simple cache for file metadata, useful for process that work o a given series of files and that only need to repeat the job on the changed ones since the previous run of the process",
|
|
5
5
|
"repository": "jaredwray/file-entry-cache",
|
|
6
6
|
"license": "MIT",
|
|
@@ -13,20 +13,19 @@
|
|
|
13
13
|
"cache.js"
|
|
14
14
|
],
|
|
15
15
|
"engines": {
|
|
16
|
-
"node": ">=
|
|
16
|
+
"node": ">=18"
|
|
17
17
|
},
|
|
18
18
|
"scripts": {
|
|
19
|
-
"
|
|
20
|
-
"
|
|
21
|
-
"test": "
|
|
22
|
-
"test:ci": "npm run eslint --silent && c8 --reporter=lcov mocha -R spec test/specs",
|
|
19
|
+
"clean": "rimraf ./coverage /node_modules ./package-lock.json ./yarn.lock ./pnpm-lock.yaml",
|
|
20
|
+
"test": "xo --fix && c8 mocha -R spec test/specs",
|
|
21
|
+
"test:ci": "xo && c8 --reporter=lcov mocha -R spec test/specs",
|
|
23
22
|
"perf": "node perf.js"
|
|
24
23
|
},
|
|
25
24
|
"prepush": [
|
|
26
|
-
"npm run
|
|
25
|
+
"npm run test"
|
|
27
26
|
],
|
|
28
27
|
"precommit": [
|
|
29
|
-
"npm run
|
|
28
|
+
"npm run test"
|
|
30
29
|
],
|
|
31
30
|
"keywords": [
|
|
32
31
|
"file cache",
|
|
@@ -37,18 +36,23 @@
|
|
|
37
36
|
"cache"
|
|
38
37
|
],
|
|
39
38
|
"devDependencies": {
|
|
40
|
-
"c8": "^
|
|
39
|
+
"c8": "^9.1.0",
|
|
41
40
|
"chai": "^4.3.10",
|
|
42
|
-
"eslint": "^8.50.0",
|
|
43
|
-
"eslint-config-prettier": "^9.0.0",
|
|
44
|
-
"eslint-plugin-mocha": "^10.2.0",
|
|
45
|
-
"eslint-plugin-prettier": "^3.1.4",
|
|
46
41
|
"glob-expand": "^0.2.1",
|
|
47
|
-
"mocha": "^10.
|
|
48
|
-
"
|
|
49
|
-
"
|
|
42
|
+
"mocha": "^10.4.0",
|
|
43
|
+
"rimraf": "^5.0.7",
|
|
44
|
+
"webpack": "^5.91.0",
|
|
45
|
+
"write": "^2.0.0",
|
|
46
|
+
"xo": "^0.58.0"
|
|
50
47
|
},
|
|
51
48
|
"dependencies": {
|
|
52
|
-
"flat-cache": "^
|
|
49
|
+
"flat-cache": "^5.0.0"
|
|
50
|
+
},
|
|
51
|
+
"xo": {
|
|
52
|
+
"rules": {
|
|
53
|
+
"unicorn/prefer-module": "off",
|
|
54
|
+
"n/prefer-global/process": "off",
|
|
55
|
+
"unicorn/prevent-abbreviations": "off"
|
|
56
|
+
}
|
|
53
57
|
}
|
|
54
58
|
}
|