atomically 1.3.2 → 1.7.0

package/dist/consts.d.ts

@@ -8,5 +8,6 @@ declare const DEFAULT_TIMEOUT_SYNC = 100;
 declare const IS_POSIX = true;
 declare const IS_USER_ROOT: boolean;
 declare const LIMIT_BASENAME_LENGTH = 128;
+declare const LIMIT_FILES_DESCRIPTORS = 10000;
 declare const NOOP: () => void;
-export { DEFAULT_ENCODING, DEFAULT_FILE_MODE, DEFAULT_FOLDER_MODE, DEFAULT_READ_OPTIONS, DEFAULT_WRITE_OPTIONS, DEFAULT_TIMEOUT_ASYNC, DEFAULT_TIMEOUT_SYNC, IS_POSIX, IS_USER_ROOT, LIMIT_BASENAME_LENGTH, NOOP };
+export { DEFAULT_ENCODING, DEFAULT_FILE_MODE, DEFAULT_FOLDER_MODE, DEFAULT_READ_OPTIONS, DEFAULT_WRITE_OPTIONS, DEFAULT_TIMEOUT_ASYNC, DEFAULT_TIMEOUT_SYNC, IS_POSIX, IS_USER_ROOT, LIMIT_BASENAME_LENGTH, LIMIT_FILES_DESCRIPTORS, NOOP };

package/dist/consts.js

@@ -1,7 +1,7 @@
 "use strict";
 /* CONSTS */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.NOOP = exports.LIMIT_BASENAME_LENGTH = exports.IS_USER_ROOT = exports.IS_POSIX = exports.DEFAULT_TIMEOUT_SYNC = exports.DEFAULT_TIMEOUT_ASYNC = exports.DEFAULT_WRITE_OPTIONS = exports.DEFAULT_READ_OPTIONS = exports.DEFAULT_FOLDER_MODE = exports.DEFAULT_FILE_MODE = exports.DEFAULT_ENCODING = void 0;
+exports.NOOP = exports.LIMIT_FILES_DESCRIPTORS = exports.LIMIT_BASENAME_LENGTH = exports.IS_USER_ROOT = exports.IS_POSIX = exports.DEFAULT_TIMEOUT_SYNC = exports.DEFAULT_TIMEOUT_ASYNC = exports.DEFAULT_WRITE_OPTIONS = exports.DEFAULT_READ_OPTIONS = exports.DEFAULT_FOLDER_MODE = exports.DEFAULT_FILE_MODE = exports.DEFAULT_ENCODING = void 0;
 const DEFAULT_ENCODING = 'utf8';
 exports.DEFAULT_ENCODING = DEFAULT_ENCODING;
 const DEFAULT_FILE_MODE = 0o666;
@@ -22,5 +22,7 @@ const IS_USER_ROOT = process.getuid ? !process.getuid() : false;
 exports.IS_USER_ROOT = IS_USER_ROOT;
 const LIMIT_BASENAME_LENGTH = 128; //TODO: fetch the real limit from the filesystem //TODO: fetch the whole-path length limit too
 exports.LIMIT_BASENAME_LENGTH = LIMIT_BASENAME_LENGTH;
+const LIMIT_FILES_DESCRIPTORS = 10000; //TODO: fetch the real limit from the filesystem
+exports.LIMIT_FILES_DESCRIPTORS = LIMIT_FILES_DESCRIPTORS;
 const NOOP = () => { };
 exports.NOOP = NOOP;

package/dist/utils/fs.d.ts

@@ -14,6 +14,7 @@ declare const FS: {
     openRetry: import("../types").FN<[number], typeof fs.open.__promisify__>;
     readFileRetry: import("../types").FN<[number], typeof fs.readFile.__promisify__>;
     renameRetry: import("../types").FN<[number], typeof fs.rename.__promisify__>;
+    statRetry: import("../types").FN<[number], typeof fs.stat.__promisify__>;
     writeRetry: import("../types").FN<[number], typeof fs.write.__promisify__>;
     chmodSyncAttempt: typeof fs.chmodSync;
     chownSyncAttempt: typeof fs.chownSync;
@@ -27,6 +28,7 @@ declare const FS: {
     openSyncRetry: import("../types").FN<[number], typeof fs.openSync>;
     readFileSyncRetry: import("../types").FN<[number], typeof fs.readFileSync>;
     renameSyncRetry: import("../types").FN<[number], typeof fs.renameSync>;
+    statSyncRetry: import("../types").FN<[number], typeof fs.statSync>;
     writeSyncRetry: import("../types").FN<[number], typeof fs.writeSync>;
 };
 export default FS;

package/dist/utils/fs.js

@@ -21,6 +21,7 @@ const FS = {
     openRetry: retryify_1.retryifyAsync(util_1.promisify(fs.open), fs_handlers_1.default.isRetriableError),
     readFileRetry: retryify_1.retryifyAsync(util_1.promisify(fs.readFile), fs_handlers_1.default.isRetriableError),
     renameRetry: retryify_1.retryifyAsync(util_1.promisify(fs.rename), fs_handlers_1.default.isRetriableError),
+    statRetry: retryify_1.retryifyAsync(util_1.promisify(fs.stat), fs_handlers_1.default.isRetriableError),
     writeRetry: retryify_1.retryifyAsync(util_1.promisify(fs.write), fs_handlers_1.default.isRetriableError),
     chmodSyncAttempt: attemptify_1.attemptifySync(fs.chmodSync, fs_handlers_1.default.onChangeError),
     chownSyncAttempt: attemptify_1.attemptifySync(fs.chownSync, fs_handlers_1.default.onChangeError),
@@ -34,6 +35,7 @@ const FS = {
     openSyncRetry: retryify_1.retryifySync(fs.openSync, fs_handlers_1.default.isRetriableError),
     readFileSyncRetry: retryify_1.retryifySync(fs.readFileSync, fs_handlers_1.default.isRetriableError),
     renameSyncRetry: retryify_1.retryifySync(fs.renameSync, fs_handlers_1.default.isRetriableError),
+    statSyncRetry: retryify_1.retryifySync(fs.statSync, fs_handlers_1.default.isRetriableError),
     writeSyncRetry: retryify_1.retryifySync(fs.writeSync, fs_handlers_1.default.isRetriableError)
 };
 /* EXPORT */

package/dist/utils/retryify.js

@@ -2,18 +2,25 @@
 /* IMPORT */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.retryifySync = exports.retryifyAsync = void 0;
+const retryify_queue_1 = require("./retryify_queue");
 /* RETRYIFY */
-//TODO: Maybe publish this as a standalone package
-//TODO: Implement a more sophisticaed, less intensive, retry strategy
 const retryifyAsync = (fn, isRetriableError) => {
     return function (timestamp) {
         return function attempt() {
-            return fn.apply(undefined, arguments).catch(error => {
-                if (Date.now() > timestamp)
+            return retryify_queue_1.default.schedule().then(cleanup => {
+                return fn.apply(undefined, arguments).then(result => {
+                    cleanup();
+                    return result;
+                }, error => {
+                    cleanup();
+                    if (Date.now() >= timestamp)
+                        throw error;
+                    if (isRetriableError(error)) {
+                        const delay = Math.round(100 + (400 * Math.random())), delayPromise = new Promise(resolve => setTimeout(resolve, delay));
+                        return delayPromise.then(() => attempt.apply(undefined, arguments));
+                    }
                     throw error;
-                if (isRetriableError(error))
-                    return attempt.apply(undefined, arguments);
-                throw error;
+                });
             });
         };
     };

package/dist/utils/retryify_queue.d.ts

@@ -0,0 +1,15 @@
+/// <reference types="node" />
+declare const RetryfyQueue: {
+    interval: number;
+    intervalId: NodeJS.Timeout | undefined;
+    limit: number;
+    queueActive: Set<Function>;
+    queueWaiting: Set<Function>;
+    init: () => void;
+    reset: () => void;
+    add: (fn: Function) => void;
+    remove: (fn: Function) => void;
+    schedule: () => Promise<Function>;
+    tick: () => void;
+};
+export default RetryfyQueue;

package/dist/utils/retryify_queue.js

@@ -0,0 +1,58 @@
+"use strict";
+/* IMPORT */
+Object.defineProperty(exports, "__esModule", { value: true });
+const consts_1 = require("../consts");
+/* RETRYIFY QUEUE */
+const RetryfyQueue = {
+    interval: 25,
+    intervalId: undefined,
+    limit: consts_1.LIMIT_FILES_DESCRIPTORS,
+    queueActive: new Set(),
+    queueWaiting: new Set(),
+    init: () => {
+        if (RetryfyQueue.intervalId)
+            return;
+        RetryfyQueue.intervalId = setInterval(RetryfyQueue.tick, RetryfyQueue.interval);
+    },
+    reset: () => {
+        if (!RetryfyQueue.intervalId)
+            return;
+        clearInterval(RetryfyQueue.intervalId);
+        delete RetryfyQueue.intervalId;
+    },
+    add: (fn) => {
+        RetryfyQueue.queueWaiting.add(fn);
+        if (RetryfyQueue.queueActive.size < (RetryfyQueue.limit / 2)) { // Active queue not under preassure, executing immediately
+            RetryfyQueue.tick();
+        }
+        else {
+            RetryfyQueue.init();
+        }
+    },
+    remove: (fn) => {
+        RetryfyQueue.queueWaiting.delete(fn);
+        RetryfyQueue.queueActive.delete(fn);
+    },
+    schedule: () => {
+        return new Promise(resolve => {
+            const cleanup = () => RetryfyQueue.remove(resolver);
+            const resolver = () => resolve(cleanup);
+            RetryfyQueue.add(resolver);
+        });
+    },
+    tick: () => {
+        if (RetryfyQueue.queueActive.size >= RetryfyQueue.limit)
+            return;
+        if (!RetryfyQueue.queueWaiting.size)
+            return RetryfyQueue.reset();
+        for (const fn of RetryfyQueue.queueWaiting) {
+            if (RetryfyQueue.queueActive.size >= RetryfyQueue.limit)
+                break;
+            RetryfyQueue.queueWaiting.delete(fn);
+            RetryfyQueue.queueActive.add(fn);
+            fn();
+        }
+    }
+};
+/* EXPORT */
+exports.default = RetryfyQueue;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "atomically",
   "description": "Read and write files atomically and reliably.",
-  "version": "1.3.2",
+  "version": "1.7.0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {

package/src/consts.ts

@@ -21,8 +21,10 @@ const IS_USER_ROOT = process.getuid ? !process.getuid () : false;
 
 const LIMIT_BASENAME_LENGTH = 128; //TODO: fetch the real limit from the filesystem //TODO: fetch the whole-path length limit too
 
+const LIMIT_FILES_DESCRIPTORS = 10000; //TODO: fetch the real limit from the filesystem
+
 const NOOP = () => {};
 
 /* EXPORT */
 
-export {DEFAULT_ENCODING, DEFAULT_FILE_MODE, DEFAULT_FOLDER_MODE, DEFAULT_READ_OPTIONS, DEFAULT_WRITE_OPTIONS, DEFAULT_TIMEOUT_ASYNC, DEFAULT_TIMEOUT_SYNC, IS_POSIX, IS_USER_ROOT, LIMIT_BASENAME_LENGTH, NOOP};
+export {DEFAULT_ENCODING, DEFAULT_FILE_MODE, DEFAULT_FOLDER_MODE, DEFAULT_READ_OPTIONS, DEFAULT_WRITE_OPTIONS, DEFAULT_TIMEOUT_ASYNC, DEFAULT_TIMEOUT_SYNC, IS_POSIX, IS_USER_ROOT, LIMIT_BASENAME_LENGTH, LIMIT_FILES_DESCRIPTORS, NOOP};

package/src/utils/fs.ts

@@ -25,6 +25,7 @@ const FS = {
   openRetry: retryifyAsync ( promisify ( fs.open ), Handlers.isRetriableError ),
   readFileRetry: retryifyAsync ( promisify ( fs.readFile ), Handlers.isRetriableError ),
   renameRetry: retryifyAsync ( promisify ( fs.rename ), Handlers.isRetriableError ),
+  statRetry: retryifyAsync ( promisify ( fs.stat ), Handlers.isRetriableError ),
   writeRetry: retryifyAsync ( promisify ( fs.write ), Handlers.isRetriableError ),
 
   chmodSyncAttempt: attemptifySync ( fs.chmodSync, Handlers.onChangeError ),
@@ -40,6 +41,7 @@ const FS = {
   openSyncRetry: retryifySync ( fs.openSync, Handlers.isRetriableError ),
   readFileSyncRetry: retryifySync ( fs.readFileSync, Handlers.isRetriableError ),
   renameSyncRetry: retryifySync ( fs.renameSync, Handlers.isRetriableError ),
+  statSyncRetry: retryifySync ( fs.statSync, Handlers.isRetriableError ),
   writeSyncRetry: retryifySync ( fs.writeSync, Handlers.isRetriableError )
 
 };

package/src/utils/retryify.ts

@@ -2,25 +2,42 @@
 /* IMPORT */
 
 import {Exception, FN} from '../types';
+import RetryfyQueue from './retryify_queue';
 
 /* RETRYIFY */
 
-//TODO: Maybe publish this as a standalone package
-//TODO: Implement a more sophisticaed, less intensive, retry strategy
-
 const retryifyAsync = <T extends FN> ( fn: T, isRetriableError: FN<[Exception], boolean | void> ): FN<[number], T> => {
 
   return function ( timestamp: number ) {
 
     return function attempt () {
 
-      return fn.apply ( undefined, arguments ).catch ( error => {
+      return RetryfyQueue.schedule ().then ( cleanup => {
 
-        if ( Date.now () > timestamp ) throw error;
+        return fn.apply ( undefined, arguments ).then ( result => {
 
-        if ( isRetriableError ( error ) ) return attempt.apply ( undefined, arguments );
+          cleanup ();
 
-        throw error;
+          return result;
+
+        }, error => {
+
+          cleanup ();
+
+          if ( Date.now () >= timestamp ) throw error;
+
+          if ( isRetriableError ( error ) ) {
+
+            const delay = Math.round ( 100 + ( 400 * Math.random () ) ),
+                  delayPromise = new Promise ( resolve => setTimeout ( resolve, delay ) );
+
+            return delayPromise.then ( () => attempt.apply ( undefined, arguments ) );
+
+          }
+
+          throw error;
+
+        });
 
       });
 

package/src/utils/retryify_queue.ts

@@ -0,0 +1,95 @@
+
+/* IMPORT */
+
+import {LIMIT_FILES_DESCRIPTORS} from '../consts';
+
+/* RETRYIFY QUEUE */
+
+const RetryfyQueue = {
+
+  interval: 25,
+  intervalId: <NodeJS.Timeout | undefined> undefined,
+  limit: LIMIT_FILES_DESCRIPTORS,
+  queueActive: new Set<Function> (),
+  queueWaiting: new Set<Function> (),
+
+  init: (): void => {
+
+    if ( RetryfyQueue.intervalId ) return;
+
+    RetryfyQueue.intervalId = setInterval ( RetryfyQueue.tick, RetryfyQueue.interval );
+
+  },
+
+  reset: (): void => {
+
+    if ( !RetryfyQueue.intervalId ) return;
+
+    clearInterval ( RetryfyQueue.intervalId );
+
+    delete RetryfyQueue.intervalId;
+
+  },
+
+  add: ( fn: Function ): void => {
+
+    RetryfyQueue.queueWaiting.add ( fn );
+
+    if ( RetryfyQueue.queueActive.size < ( RetryfyQueue.limit / 2 ) ) { // Active queue not under preassure, executing immediately
+
+      RetryfyQueue.tick ();
+
+    } else {
+
+      RetryfyQueue.init ();
+
+    }
+
+  },
+
+  remove: ( fn: Function ): void => {
+
+    RetryfyQueue.queueWaiting.delete ( fn );
+
+    RetryfyQueue.queueActive.delete ( fn );
+
+  },
+
+  schedule: (): Promise<Function> => {
+
+    return new Promise ( resolve => {
+
+      const cleanup = () => RetryfyQueue.remove ( resolver );
+
+      const resolver = () => resolve ( cleanup );
+
+      RetryfyQueue.add ( resolver );
+
+    });
+
+  },
+
+  tick: (): void => {
+
+    if ( RetryfyQueue.queueActive.size >= RetryfyQueue.limit ) return;
+
+    if ( !RetryfyQueue.queueWaiting.size ) return RetryfyQueue.reset ();
+
+    for ( const fn of RetryfyQueue.queueWaiting ) {
+
+      if ( RetryfyQueue.queueActive.size >= RetryfyQueue.limit ) break;
+
+      RetryfyQueue.queueWaiting.delete ( fn );
+      RetryfyQueue.queueActive.add ( fn );
+
+      fn ();
+
+    }
+
+  }
+
+};
+
+/* EXPORT */
+
+export default RetryfyQueue;

package/test/basic.js

@@ -116,7 +116,7 @@ const fsMock = Object.assign ( {}, fs, {
 
 const makeUnstableAsyncFn = function () {
   return function () {
-    if ( Math.random () <= .9 ) {
+    if ( Math.random () <= .75 ) {
       const code = _.shuffle ([ 'EMFILE', 'ENFILE', 'EAGAIN', 'EBUSY', 'EACCESS', 'EPERM' ])[0];
       throw createErr ( code );
     }
@@ -126,7 +126,7 @@ const makeUnstableAsyncFn = function () {
 
 const makeUnstableSyncFn = function ( fn ) {
   return function () {
-    if ( Math.random () <= .9 ) {
+    if ( Math.random () <= .75 ) {
       const code = _.shuffle ([ 'EMFILE', 'ENFILE', 'EAGAIN', 'EBUSY', 'EACCESS', 'EPERM' ])[0];
       throw createErr ( code );
     }