file-entry-cache 1.2.4 → 4.0.0

package/README.md

@@ -1,5 +1,5 @@
 # file-entry-cache
-> Super simple cache for file metadata, useful for process that work o a given series of files 
+> Super simple cache for file metadata, useful for process that work o a given series of files
 > and that only need to repeat the job on the changed ones since the previous run of the process — Edit
 
 [![NPM Version](http://img.shields.io/npm/v/file-entry-cache.svg?style=flat)](https://npmjs.org/package/file-entry-cache)
@@ -14,7 +14,7 @@ npm i --save file-entry-cache
 ## Usage
 
 ```js
-// loads the cache, if one does not exists for the given 
+// loads the cache, if one does not exists for the given
 // Id a new one will be prepared to be created
 var fileEntryCache = require('file-entry-cache');
 
@@ -25,18 +25,23 @@ var files = expand('../fixtures/*.txt');
 // the first time this method is called, will return all the files
 var oFiles = cache.getUpdatedFiles(files);
 
-// this will persist this to disk checking each file stats and 
+// this will persist this to disk checking each file stats and
 // updating the meta attributes `size` and `mtime`.
 // custom fields could also be added to the meta object and will be persisted
 // in order to retrieve them later
-cache.reconcile(); 
+cache.reconcile();
+
+// use this if you want the non visited file entries to be kept in the cache
+// for more than one execution
+//
+// cache.reconcile( true /* noPrune */)
 
 // on a second run
 var cache2 = fileEntryCache.create('testCache');
 
 // will return now only the files that were modified or none
 // if no files were modified previous to the execution of this function
-var oFiles = cache.getUpdatedFiles(files);  
+var oFiles = cache.getUpdatedFiles(files);
 
 // if you want to prevent a file from being considered non modified
 // something useful if a file failed some sort of validation
@@ -64,21 +69,21 @@ entry = {
 
 ## Motivation for this module
 
-I needed a super simple and dumb **in-memory cache** with optional disk persistence (write-back cache) in order to make 
+I needed a super simple and dumb **in-memory cache** with optional disk persistence (write-back cache) in order to make
 a script that will beautify files with `esformatter` to execute only on the files that were changed since the last run.
 
 In doing so the process of beautifying files was reduced from several seconds to a small fraction of a second.
 
-This module uses [flat-cache](https://www.npmjs.com/package/flat-cache) a super simple `key/value` cache storage with 
+This module uses [flat-cache](https://www.npmjs.com/package/flat-cache) a super simple `key/value` cache storage with
 optional file persistance.
 
-The main idea is to read the files when the task begins, apply the transforms required, and if the process succeed, 
-then store the new state of the files. The next time this module request for `getChangedFiles` will return only 
+The main idea is to read the files when the task begins, apply the transforms required, and if the process succeed,
+then store the new state of the files. The next time this module request for `getChangedFiles` will return only
 the files that were modified. Making the process to end faster.
 
 This module could also be used by processes that modify the files applying a transform, in that case the result of the
-transform could be stored in the `meta` field, of the entries. Anything added to the meta field will be persisted. 
-Those processes won't need to call `getChangedFiles` they will instead call `normalizeEntries` that will return the 
+transform could be stored in the `meta` field, of the entries. Anything added to the meta field will be persisted.
+Those processes won't need to call `getChangedFiles` they will instead call `normalizeEntries` that will return the
 entries with a `changed` field that can be used to determine if the file was changed or not. If it was not changed
 the transformed stored data could be used instead of actually applying the transformation, saving time in case of only
 a few files changed.
@@ -86,10 +91,16 @@ a few files changed.
 In the worst case scenario all the files will be processed. In the best case scenario only a few of them will be processed.
 
 ## Important notes
-- The values set on the meta attribute of the entries should be `stringify-able` ones, meaning no circular references
-- All the changes to the cache state are done to memory first and only persisted after reconcile
+- The values set on the meta attribute of the entries should be `stringify-able` ones if possible, flat-cache uses `circular-json` to try to persist circular structures, but this should be considered experimental. The best results are always obtained with non circular values
+- All the changes to the cache state are done to memory first and only persisted after reconcile.
+- By default non visited entries are removed from the cache. This is done to prevent the file from growing too much. If this is not an issue and
+  you prefer to do a manual pruning of the cache files, you can pass `true` to the `reconcile` call. Like this:
+
+  ```javascript
+  cache.reconcile( true /* noPrune */ );
+  ```
 
-## License 
+## License
 
 MIT
 

package/cache.js

@@ -1,4 +1,5 @@
 var path = require( 'path' );
+var crypto = require( 'crypto' );
 
 module.exports = {
   createFromFile: function ( filePath ) {
@@ -11,15 +12,44 @@ module.exports = {
     var fs = require( 'fs' );
     var flatCache = require( 'flat-cache' );
     var cache = flatCache.load( cacheId, _path );
-    var assign = require( 'object-assign' );
     var normalizedEntries = { };
 
+    var removeNotFoundFiles = function removeNotFoundFiles() {
+      const cachedEntries = cache.keys();
+      // remove not found entries
+      cachedEntries.forEach( function remover( fPath ) {
+        try {
+          fs.statSync( fPath );
+        } catch (err) {
+          if ( err.code === 'ENOENT' ) {
+            cache.removeKey( fPath );
+          }
+        }
+      } );
+    };
+
+    removeNotFoundFiles();
+
     return {
       /**
-       * the flat cache storage used to persist the metadata of the files
+       * the flat cache storage used to persist the metadata of the `files
        * @type {Object}
        */
       cache: cache,
+
+      /**
+       * Given a buffer, calculate md5 hash of its content.
+       * @method getHash
+       * @param  {Buffer} buffer   buffer to calculate hash on
+       * @return {String}          content hash digest
+       */
+      getHash: function ( buffer ) {
+        return crypto
+          .createHash( 'md5' )
+          .update( buffer )
+          .digest( 'hex' );
+      },
+
       /**
        * Return whether or not a file has changed since last time reconcile was called.
        * @method hasFileChanged
@@ -66,29 +96,28 @@ module.exports = {
       getFileDescriptor: function ( file ) {
         var meta = cache.getKey( file );
         var cacheExists = !!meta;
-        var fstat;
         var me = this;
+        var contentBuffer;
 
         try {
-          fstat = fs.statSync( file );
+          contentBuffer = fs.readFileSync( file );
         } catch (ex) {
           me.removeEntry( file );
           return { key: file, notFound: true, err: ex };
         }
 
-        var cSize = fstat.size;
-        var cTime = fstat.mtime.getTime();
+        var isDifferent = true;
+        var hash = this.getHash( contentBuffer );
 
         if ( !meta ) {
-          meta = { size: cSize, mtime: cTime };
+          meta = { hash: hash };
         } else {
-          var isDifferentDate = cTime !== meta.mtime;
-          var isDifferentSize = cSize !== meta.size;
+          isDifferent = hash !== meta.hash;
         }
 
         var nEntry = normalizedEntries[ file ] = {
           key: file,
-          changed: !cacheExists || isDifferentDate || isDifferentSize,
+          changed: !cacheExists || isDifferent,
           meta: meta
         };
 
@@ -161,29 +190,38 @@ module.exports = {
       },
       /**
        * Sync the files and persist them to the cache
-       *
        * @method reconcile
        */
       reconcile: function () {
-        var entries = normalizedEntries;
+        removeNotFoundFiles();
 
+        var entries = normalizedEntries;
         var keys = Object.keys( entries );
+        var me = this;
+
         if ( keys.length === 0 ) {
           return;
         }
+
         keys.forEach( function ( entryName ) {
           var cacheEntry = entries[ entryName ];
-          var stat = fs.statSync( cacheEntry.key );
 
-          var meta = assign( cacheEntry.meta, {
-            size: stat.size,
-            mtime: stat.mtime.getTime()
-          } );
-
-          cache.setKey( entryName, meta );
+          try {
+            var contentBuffer = fs.readFileSync( cacheEntry.key );
+            var hash = me.getHash( contentBuffer );
+            var meta = Object.assign( cacheEntry.meta, { hash: hash } );
+
+            cache.setKey( entryName, meta );
+          } catch (err) {
+            // if the file does not exists we don't save it
+            // other errors are just thrown
+            if ( err.code !== 'ENOENT' ) {
+              throw err;
+            }
+          }
         } );
 
-        cache.save();
+        cache.save( true );
       }
     };
   }

package/changelog.md

@@ -1,5 +1,67 @@
 
 # file-entry-cache - Changelog
+## v4.0.0
+- **Build Scripts Changes**
+  - use the same node versions eslint use - [563cfee]( https://github.com/royriojas/file-entry-cache/commit/563cfee ), [Roy Riojas](https://github.com/Roy Riojas), 08/01/2019 23:29:34
+
+    
+- **Other changes**
+  - Remove object-assign dependency. - [d0f598e]( https://github.com/royriojas/file-entry-cache/commit/d0f598e ), [Corey Farrell](https://github.com/Corey Farrell), 08/01/2019 23:09:51
+
+    node.js >=4 is required so object-assign is no longer needed, the native
+    Object.assign can be used instead.
+    
+## v3.0.0
+- **Build Scripts Changes**
+  - Upgrade flat-cache dep to latest - [078b0df]( https://github.com/royriojas/file-entry-cache/commit/078b0df ), [Roy Riojas](https://github.com/Roy Riojas), 08/01/2019 21:54:40
+
+    
+  - Commit new package-lock.json file - [245fe62]( https://github.com/royriojas/file-entry-cache/commit/245fe62 ), [Roy Riojas](https://github.com/Roy Riojas), 08/01/2019 20:56:21
+
+    
+- **Refactoring**
+  - add eslintrc file - [6dd32d8]( https://github.com/royriojas/file-entry-cache/commit/6dd32d8 ), [Roy Riojas](https://github.com/Roy Riojas), 22/08/2018 11:58:17
+
+    
+- **Other changes**
+  - Move variable definition out of else block - [ea05441]( https://github.com/royriojas/file-entry-cache/commit/ea05441 ), [Zakhar Shapurau](https://github.com/Zakhar Shapurau), 25/04/2017 13:19:00
+
+    
+  - Add script and cmd to test hash/checksum performance - [7f60e0a]( https://github.com/royriojas/file-entry-cache/commit/7f60e0a ), [Zakhar Shapurau](https://github.com/Zakhar Shapurau), 24/04/2017 16:43:12
+
+    
+  - Calculate md5 hexdigest instead of Adler-32 checksum - [f9e5c69]( https://github.com/royriojas/file-entry-cache/commit/f9e5c69 ), [Zakhar Shapurau](https://github.com/Zakhar Shapurau), 24/04/2017 16:43:12
+
+    
+  - How to reproduce - [4edc2dc]( https://github.com/royriojas/file-entry-cache/commit/4edc2dc ), [Zakhar Shapurau](https://github.com/Zakhar Shapurau), 24/04/2017 15:49:32
+
+    
+  - Test handling of removed files - [09d9ec5]( https://github.com/royriojas/file-entry-cache/commit/09d9ec5 ), [Zakhar Shapurau](https://github.com/Zakhar Shapurau), 19/04/2017 21:51:50
+
+    
+  - Use content checksum instead of mtime and fsize - [343b340]( https://github.com/royriojas/file-entry-cache/commit/343b340 ), [Zakhar Shapurau](https://github.com/Zakhar Shapurau), 19/04/2017 21:51:47
+
+    
+- **Revert**
+  - Revert "How to reproduce" - [4b4e54a]( https://github.com/royriojas/file-entry-cache/commit/4b4e54a ), [Zakhar Shapurau](https://github.com/Zakhar Shapurau), 25/04/2017 13:15:36
+
+    This reverts commit 4edc2dcec01574247bfc2e0a2fe26527332b7df3.
+    
+## v2.0.0
+- **Features**
+  - do not persist and prune removed files from cache. Relates to [#2](https://github.com/royriojas/file-entry-cache/issues/2) - [408374d]( https://github.com/royriojas/file-entry-cache/commit/408374d ), [Roy Riojas](https://github.com/Roy Riojas), 16/08/2016 15:47:58
+
+    
+## v1.3.1
+- **Build Scripts Changes**
+  - remove older node version - [0a26ac4]( https://github.com/royriojas/file-entry-cache/commit/0a26ac4 ), [Roy Riojas](https://github.com/Roy Riojas), 01/08/2016 06:09:17
+
+    
+## v1.3.0
+- **Features**
+  - Add an option to not prune non visited keys. Closes [#2](https://github.com/royriojas/file-entry-cache/issues/2) - [b1a64db]( https://github.com/royriojas/file-entry-cache/commit/b1a64db ), [Roy Riojas](https://github.com/Roy Riojas), 01/08/2016 05:52:12
+
+    
 ## v1.2.4
 - **Enhancements**
   - Expose the flat-cache instance - [f34c557]( https://github.com/royriojas/file-entry-cache/commit/f34c557 ), [royriojas](https://github.com/royriojas), 23/09/2015 20:26:33

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "file-entry-cache",
-  "version": "1.2.4",
+  "version": "4.0.0",
   "description": "Super simple cache for file metadata, useful for process that work o a given series of files and that only need to repeat the job on the changed ones since the previous run of the process",
   "repository": "royriojas/file-entry-cache",
   "license": "MIT",
@@ -13,23 +13,24 @@
     "cache.js"
   ],
   "engines": {
-    "node": ">=0.10.0"
+    "node": ">=4"
   },
   "scripts": {
-    "beautify": "esbeautifier 'cache.js' 'specs/**/*.js'",
-    "beautify-check": "esbeautifier -k 'cache.js' 'specs/**/*.js'",
-    "eslint": "eslinter 'cache.js' 'specs/**/*.js'",
+    "beautify": "esbeautifier 'cache.js' 'test/**/*.js' 'perf.js'",
+    "beautify-check": "npm run beautify -- -k",
+    "eslint": "eslinter 'cache.js' 'specs/**/*.js' 'perf.js'",
     "lint": "npm run beautify && npm run eslint",
     "verify": "npm run beautify-check && npm run eslint",
     "install-hooks": "prepush install && changelogx install-hook && precommit install",
     "changelog": "changelogx -f markdown -o ./changelog.md",
     "do-changelog": "npm run changelog && git add ./changelog.md && git commit -m 'DOC: Generate changelog' --no-verify",
-    "pre-v": "npm run verify",
+    "pre-v": "npm run test",
     "post-v": "npm run do-changelog && git push --no-verify && git push --tags --no-verify",
     "bump-major": "npm run pre-v && npm version major -m 'BLD: Release v%s' && npm run post-v",
     "bump-minor": "npm run pre-v && npm version minor -m 'BLD: Release v%s' && npm run post-v",
     "bump-patch": "npm run pre-v && npm version patch -m 'BLD: Release v%s' && npm run post-v",
-    "test": "mocha -R spec test/specs",
+    "test": "npm run verify --silent && mocha -R spec test/specs",
+    "perf": "node perf.js",
     "cover": "istanbul cover test/runner.js html text-summary",
     "watch": "watch-run -i -p 'test/specs/**/*.js' istanbul cover test/runner.js html text-summary"
   },
@@ -61,7 +62,7 @@
   },
   "devDependencies": {
     "chai": "^3.2.0",
-    "changelogx": "^1.0.18",
+    "changelogx": "3.0.0",
     "commander": "^2.6.0",
     "del": "^2.0.2",
     "esbeautifier": "^4.2.11",
@@ -72,14 +73,12 @@
     "precommit": "^1.1.5",
     "prepush": "^3.1.4",
     "proxyquire": "^1.3.1",
-    "read-file": "^0.2.0",
     "sinon": "^1.12.2",
     "sinon-chai": "^2.7.0",
     "watch-run": "^1.2.1",
-    "write": "^0.2.1"
+    "write": "^0.3.1"
   },
   "dependencies": {
-    "flat-cache": "^1.0.9",
-    "object-assign": "^4.0.1"
+    "flat-cache": "^2.0.1"
   }
 }