file-entry-cache 1.3.1 → 5.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/README.md +11 -6
  2. package/cache.js +113 -19
  3. package/changelog.md +72 -0
  4. package/package.json +8 -8
package/README.md CHANGED
@@ -13,6 +13,17 @@ npm i --save file-entry-cache
13
13
 
14
14
  ## Usage
15
15
 
16
+ The module exposes two functions `create` and `createFromFile`.
17
+
18
+ ## `create(cacheName, [directory, useCheckSum])`
19
+ - **cacheName**: the name of the cache to be created
20
+ - **directory**: Optional the directory to load the cache from
21
+ - **usecheckSum**: Whether to use md5 checksum to verify if file changed. If false the default will be to use the mtime and size of the file.
22
+
23
+ ## `createFromFile(pathToCache, [useCheckSum])`
24
+ - **pathToCache**: the path to the cache file (this combines the cache name and directory)
25
+ - **useCheckSum**: Whether to use md5 checksum to verify if file changed. If false the default will be to use the mtime and size of the file.
26
+
16
27
  ```js
17
28
  // loads the cache, if one does not exists for the given
18
29
  // Id a new one will be prepared to be created
@@ -93,12 +104,6 @@ In the worst case scenario all the files will be processed. In the best case sce
93
104
  ## Important notes
94
105
  - The values set on the meta attribute of the entries should be `stringify-able` ones if possible, flat-cache uses `circular-json` to try to persist circular structures, but this should be considered experimental. The best results are always obtained with non circular values
95
106
  - All the changes to the cache state are done to memory first and only persisted after reconcile.
96
- - By default non visited entries are removed from the cache. This is done to prevent the file from growing too much. If this is not an issue and
97
- you prefer to do a manual pruning of the cache files, you can pass `true` to the `reconcile` call. Like this:
98
-
99
- ```javascript
100
- cache.reconcile( true /* noPrune */ );
101
- ```
102
107
 
103
108
  ## License
104
109
 
package/cache.js CHANGED
@@ -1,25 +1,55 @@
1
1
  var path = require( 'path' );
2
+ var crypto = require( 'crypto' );
2
3
 
3
4
  module.exports = {
4
- createFromFile: function ( filePath ) {
5
+ createFromFile: function ( filePath, useChecksum ) {
5
6
  var fname = path.basename( filePath );
6
7
  var dir = path.dirname( filePath );
7
- return this.create( fname, dir );
8
+ return this.create( fname, dir, useChecksum );
8
9
  },
9
10
 
10
- create: function ( cacheId, _path ) {
11
+ create: function ( cacheId, _path, useChecksum ) {
11
12
  var fs = require( 'fs' );
12
13
  var flatCache = require( 'flat-cache' );
13
14
  var cache = flatCache.load( cacheId, _path );
14
- var assign = require( 'object-assign' );
15
15
  var normalizedEntries = { };
16
16
 
17
+ var removeNotFoundFiles = function removeNotFoundFiles() {
18
+ const cachedEntries = cache.keys();
19
+ // remove not found entries
20
+ cachedEntries.forEach( function remover( fPath ) {
21
+ try {
22
+ fs.statSync( fPath );
23
+ } catch (err) {
24
+ if ( err.code === 'ENOENT' ) {
25
+ cache.removeKey( fPath );
26
+ }
27
+ }
28
+ } );
29
+ };
30
+
31
+ removeNotFoundFiles();
32
+
17
33
  return {
18
34
  /**
19
35
  * the flat cache storage used to persist the metadata of the `files
20
36
  * @type {Object}
21
37
  */
22
38
  cache: cache,
39
+
40
+ /**
41
+ * Given a buffer, calculate md5 hash of its content.
42
+ * @method getHash
43
+ * @param {Buffer} buffer buffer to calculate hash on
44
+ * @return {String} content hash digest
45
+ */
46
+ getHash: function ( buffer ) {
47
+ return crypto
48
+ .createHash( 'md5' )
49
+ .update( buffer )
50
+ .digest( 'hex' );
51
+ },
52
+
23
53
  /**
24
54
  * Return whether or not a file has changed since last time reconcile was called.
25
55
  * @method hasFileChanged
@@ -64,26 +94,37 @@ module.exports = {
64
94
  },
65
95
 
66
96
  getFileDescriptor: function ( file ) {
67
- var meta = cache.getKey( file );
68
- var cacheExists = !!meta;
69
97
  var fstat;
70
- var me = this;
71
98
 
72
99
  try {
73
100
  fstat = fs.statSync( file );
74
101
  } catch (ex) {
75
- me.removeEntry( file );
102
+ this.removeEntry( file );
76
103
  return { key: file, notFound: true, err: ex };
77
104
  }
78
105
 
106
+ if ( useChecksum ) {
107
+ return this._getFileDescriptorUsingChecksum( file );
108
+ }
109
+
110
+ return this._getFileDescriptorUsingMtimeAndSize( file, fstat );
111
+ },
112
+
113
+ _getFileDescriptorUsingMtimeAndSize: function ( file, fstat ) {
114
+ var meta = cache.getKey( file );
115
+ var cacheExists = !!meta;
116
+
79
117
  var cSize = fstat.size;
80
118
  var cTime = fstat.mtime.getTime();
81
119
 
120
+ var isDifferentDate;
121
+ var isDifferentSize;
122
+
82
123
  if ( !meta ) {
83
124
  meta = { size: cSize, mtime: cTime };
84
125
  } else {
85
- var isDifferentDate = cTime !== meta.mtime;
86
- var isDifferentSize = cSize !== meta.size;
126
+ isDifferentDate = cTime !== meta.mtime;
127
+ isDifferentSize = cSize !== meta.size;
87
128
  }
88
129
 
89
130
  var nEntry = normalizedEntries[ file ] = {
@@ -95,6 +136,35 @@ module.exports = {
95
136
  return nEntry;
96
137
  },
97
138
 
139
+ _getFileDescriptorUsingChecksum: function ( file ) {
140
+ var meta = cache.getKey( file );
141
+ var cacheExists = !!meta;
142
+
143
+ var contentBuffer;
144
+ try {
145
+ contentBuffer = fs.readFileSync( file );
146
+ } catch (ex) {
147
+ contentBuffer = '';
148
+ }
149
+
150
+ var isDifferent = true;
151
+ var hash = this.getHash( contentBuffer );
152
+
153
+ if ( !meta ) {
154
+ meta = { hash: hash };
155
+ } else {
156
+ isDifferent = hash !== meta.hash;
157
+ }
158
+
159
+ var nEntry = normalizedEntries[ file ] = {
160
+ key: file,
161
+ changed: !cacheExists || isDifferent,
162
+ meta: meta
163
+ };
164
+
165
+ return nEntry;
166
+ },
167
+
98
168
  /**
99
169
  * Return the list o the files that changed compared
100
170
  * against the ones stored in the cache
@@ -159,30 +229,54 @@ module.exports = {
159
229
  normalizedEntries = { };
160
230
  cache.destroy();
161
231
  },
232
+
233
+ _getMetaForFileUsingCheckSum: function ( cacheEntry ) {
234
+ var contentBuffer = fs.readFileSync( cacheEntry.key );
235
+ var hash = this.getHash( contentBuffer );
236
+ var meta = Object.assign( cacheEntry.meta, { hash: hash } );
237
+ return meta;
238
+ },
239
+
240
+ _getMetaForFileUsingMtimeAndSize: function ( cacheEntry ) {
241
+ var stat = fs.statSync( cacheEntry.key );
242
+ var meta = Object.assign( cacheEntry.meta, {
243
+ size: stat.size,
244
+ mtime: stat.mtime.getTime()
245
+ } );
246
+ return meta;
247
+ },
248
+
162
249
  /**
163
250
  * Sync the files and persist them to the cache
164
- * @param [noPrune=false] {Boolean} whether to remove non visited/saved entries
165
251
  * @method reconcile
166
252
  */
167
253
  reconcile: function ( noPrune ) {
168
- var entries = normalizedEntries;
254
+ removeNotFoundFiles();
169
255
 
256
+ noPrune = typeof noPrune === 'undefined' ? true : noPrune;
257
+
258
+ var entries = normalizedEntries;
170
259
  var keys = Object.keys( entries );
171
260
 
172
261
  if ( keys.length === 0 ) {
173
262
  return;
174
263
  }
175
264
 
265
+ var me = this;
266
+
176
267
  keys.forEach( function ( entryName ) {
177
268
  var cacheEntry = entries[ entryName ];
178
- var stat = fs.statSync( cacheEntry.key );
179
-
180
- var meta = assign( cacheEntry.meta, {
181
- size: stat.size,
182
- mtime: stat.mtime.getTime()
183
- } );
184
269
 
185
- cache.setKey( entryName, meta );
270
+ try {
271
+ var meta = useChecksum ? me._getMetaForFileUsingCheckSum( cacheEntry ) : me._getMetaForFileUsingMtimeAndSize( cacheEntry );
272
+ cache.setKey( entryName, meta );
273
+ } catch (err) {
274
+ // if the file does not exists we don't save it
275
+ // other errors are just thrown
276
+ if ( err.code !== 'ENOENT' ) {
277
+ throw err;
278
+ }
279
+ }
186
280
  } );
187
281
 
188
282
  cache.save( noPrune );
package/changelog.md CHANGED
@@ -1,5 +1,77 @@
1
1
 
2
2
  # file-entry-cache - Changelog
3
+ ## v5.0.1
4
+ - **Bug Fixes**
5
+ - Fix missing checksum comparison from reconcile since now we use mtime and size by default. - [e858aa9]( https://github.com/royriojas/file-entry-cache/commit/e858aa9 ), [Roy Riojas](https://github.com/Roy Riojas), 04/02/2019 12:30:22
6
+
7
+ Old mode using checkSum can still be used by passing the `useCheckSum` parameter to the `create` or `createFromFile` methods.
8
+
9
+ ## v5.0.0
10
+ - **Refactoring**
11
+ - Make checksum comparison optional - [b0f9ae0]( https://github.com/royriojas/file-entry-cache/commit/b0f9ae0 ), [Roy Riojas](https://github.com/Roy Riojas), 03/02/2019 21:17:39
12
+
13
+ To determine if a file has changed we were using the checksum in the newer versions, but eslint was relying on the old behavior where we use the mtime and file size to determine if a file changed. That's why we decided to make the checksum check optional.
14
+
15
+ To use it:
16
+
17
+ ```js
18
+ // to make the cache use the checkSum check do the following:
19
+ var fCache = fileEntryCache.create(cacheName, dir, useCheckSum); // pass the third parameter as true
20
+ var otherCache = fileEntryCache.createFromFile(cacheName, useCheckSum); // pass the second parameter as true
21
+ ```
22
+
23
+ ## v4.0.0
24
+ - **Build Scripts Changes**
25
+ - use the same node versions eslint use - [563cfee]( https://github.com/royriojas/file-entry-cache/commit/563cfee ), [Roy Riojas](https://github.com/Roy Riojas), 08/01/2019 23:29:34
26
+
27
+
28
+ - **Other changes**
29
+ - Remove object-assign dependency. - [d0f598e]( https://github.com/royriojas/file-entry-cache/commit/d0f598e ), [Corey Farrell](https://github.com/Corey Farrell), 08/01/2019 23:09:51
30
+
31
+ node.js >=4 is required so object-assign is no longer needed, the native
32
+ Object.assign can be used instead.
33
+
34
+ ## v3.0.0
35
+ - **Build Scripts Changes**
36
+ - Upgrade flat-cache dep to latest - [078b0df]( https://github.com/royriojas/file-entry-cache/commit/078b0df ), [Roy Riojas](https://github.com/Roy Riojas), 08/01/2019 21:54:40
37
+
38
+
39
+ - Commit new package-lock.json file - [245fe62]( https://github.com/royriojas/file-entry-cache/commit/245fe62 ), [Roy Riojas](https://github.com/Roy Riojas), 08/01/2019 20:56:21
40
+
41
+
42
+ - **Refactoring**
43
+ - add eslintrc file - [6dd32d8]( https://github.com/royriojas/file-entry-cache/commit/6dd32d8 ), [Roy Riojas](https://github.com/Roy Riojas), 22/08/2018 11:58:17
44
+
45
+
46
+ - **Other changes**
47
+ - Move variable definition out of else block - [ea05441]( https://github.com/royriojas/file-entry-cache/commit/ea05441 ), [Zakhar Shapurau](https://github.com/Zakhar Shapurau), 25/04/2017 13:19:00
48
+
49
+
50
+ - Add script and cmd to test hash/checksum performance - [7f60e0a]( https://github.com/royriojas/file-entry-cache/commit/7f60e0a ), [Zakhar Shapurau](https://github.com/Zakhar Shapurau), 24/04/2017 16:43:12
51
+
52
+
53
+ - Calculate md5 hexdigest instead of Adler-32 checksum - [f9e5c69]( https://github.com/royriojas/file-entry-cache/commit/f9e5c69 ), [Zakhar Shapurau](https://github.com/Zakhar Shapurau), 24/04/2017 16:43:12
54
+
55
+
56
+ - How to reproduce - [4edc2dc]( https://github.com/royriojas/file-entry-cache/commit/4edc2dc ), [Zakhar Shapurau](https://github.com/Zakhar Shapurau), 24/04/2017 15:49:32
57
+
58
+
59
+ - Test handling of removed files - [09d9ec5]( https://github.com/royriojas/file-entry-cache/commit/09d9ec5 ), [Zakhar Shapurau](https://github.com/Zakhar Shapurau), 19/04/2017 21:51:50
60
+
61
+
62
+ - Use content checksum instead of mtime and fsize - [343b340]( https://github.com/royriojas/file-entry-cache/commit/343b340 ), [Zakhar Shapurau](https://github.com/Zakhar Shapurau), 19/04/2017 21:51:47
63
+
64
+
65
+ - **Revert**
66
+ - Revert "How to reproduce" - [4b4e54a]( https://github.com/royriojas/file-entry-cache/commit/4b4e54a ), [Zakhar Shapurau](https://github.com/Zakhar Shapurau), 25/04/2017 13:15:36
67
+
68
+ This reverts commit 4edc2dcec01574247bfc2e0a2fe26527332b7df3.
69
+
70
+ ## v2.0.0
71
+ - **Features**
72
+ - do not persist and prune removed files from cache. Relates to [#2](https://github.com/royriojas/file-entry-cache/issues/2) - [408374d]( https://github.com/royriojas/file-entry-cache/commit/408374d ), [Roy Riojas](https://github.com/Roy Riojas), 16/08/2016 15:47:58
73
+
74
+
3
75
  ## v1.3.1
4
76
  - **Build Scripts Changes**
5
77
  - remove older node version - [0a26ac4]( https://github.com/royriojas/file-entry-cache/commit/0a26ac4 ), [Roy Riojas](https://github.com/Roy Riojas), 01/08/2016 06:09:17
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "file-entry-cache",
3
- "version": "1.3.1",
3
+ "version": "5.0.1",
4
4
  "description": "Super simple cache for file metadata, useful for process that work o a given series of files and that only need to repeat the job on the changed ones since the previous run of the process",
5
5
  "repository": "royriojas/file-entry-cache",
6
6
  "license": "MIT",
@@ -13,12 +13,12 @@
13
13
  "cache.js"
14
14
  ],
15
15
  "engines": {
16
- "node": ">=0.10.0"
16
+ "node": ">=4"
17
17
  },
18
18
  "scripts": {
19
- "beautify": "esbeautifier 'cache.js' 'test/**/*.js'",
19
+ "beautify": "esbeautifier 'cache.js' 'test/**/*.js' 'perf.js'",
20
20
  "beautify-check": "npm run beautify -- -k",
21
- "eslint": "eslinter 'cache.js' 'specs/**/*.js'",
21
+ "eslint": "eslinter 'cache.js' 'specs/**/*.js' 'perf.js'",
22
22
  "lint": "npm run beautify && npm run eslint",
23
23
  "verify": "npm run beautify-check && npm run eslint",
24
24
  "install-hooks": "prepush install && changelogx install-hook && precommit install",
@@ -29,7 +29,8 @@
29
29
  "bump-major": "npm run pre-v && npm version major -m 'BLD: Release v%s' && npm run post-v",
30
30
  "bump-minor": "npm run pre-v && npm version minor -m 'BLD: Release v%s' && npm run post-v",
31
31
  "bump-patch": "npm run pre-v && npm version patch -m 'BLD: Release v%s' && npm run post-v",
32
- "test": "npm run verify && mocha -R spec test/specs",
32
+ "test": "npm run verify --silent && mocha -R spec test/specs",
33
+ "perf": "node perf.js",
33
34
  "cover": "istanbul cover test/runner.js html text-summary",
34
35
  "watch": "watch-run -i -p 'test/specs/**/*.js' istanbul cover test/runner.js html text-summary"
35
36
  },
@@ -61,7 +62,7 @@
61
62
  },
62
63
  "devDependencies": {
63
64
  "chai": "^3.2.0",
64
- "changelogx": "^1.0.18",
65
+ "changelogx": "3.0.0",
65
66
  "commander": "^2.6.0",
66
67
  "del": "^2.0.2",
67
68
  "esbeautifier": "^4.2.11",
@@ -78,7 +79,6 @@
78
79
  "write": "^0.3.1"
79
80
  },
80
81
  "dependencies": {
81
- "flat-cache": "^1.2.1",
82
- "object-assign": "^4.0.1"
82
+ "flat-cache": "^2.0.1"
83
83
  }
84
84
  }