@withjoy/limiter 0.1.2 → 0.1.4-test

package/limitd-redis/lib/db.js

@@ -0,0 +1,501 @@
+const ms = require('ms');
+const fs = require('fs');
+const _ = require('lodash');
+const async = require('async');
+const LRU = require('lru-cache');
+const utils = require('./utils');
+const Redis = require('ioredis');
+const { validateParams } = require('./validation');
+const DBPing = require("./db_ping");
+const EventEmitter = require('events').EventEmitter;
+
+const TAKE_LUA = fs.readFileSync(`${__dirname}/take.lua`, "utf8");
+const PUT_LUA = fs.readFileSync(`${__dirname}/put.lua`, "utf8");
+
+const PING_SUCCESS = "successful";
+const PING_ERROR = "error";
+const PING_RECONNECT = "reconnect";
+const PING_RECONNECT_DRY_RUN = "reconnect-dry-run";
+
+const DEFAULT_COMMAND_TIMEOUT = 125; // Milliseconds
+
+class LimitDBRedis extends EventEmitter {
+  static get PING_SUCCESS() {
+    return PING_SUCCESS;
+  }
+  static get PING_ERROR() {
+    return PING_ERROR;
+  }
+  static get PING_RECONNECT() {
+    return PING_RECONNECT;
+  }
+  static get PING_RECONNECT_DRY_RUN() {
+    return PING_RECONNECT_DRY_RUN;
+  }
+
+  /**
+   * Creates an instance of LimitDB client for Redis.
+   * @param {params} params - The configuration for the database and client.
+   */
+  constructor(config) {
+    super();
+    config = config || {};
+
+    if (!config.nodes && !config.uri) {
+      throw new Error('Redis connection information must be specified');
+    }
+
+    if (!config.buckets) {
+      throw new Error('Buckets must be specified for Limitd');
+    }
+
+    this.configurateBuckets(config.buckets);
+    this.prefix = config.prefix;
+    this.globalTTL = (config.globalTTL || ms('7d')) / 1000;
+    this.callCounts = new LRU({ max: 50 });
+
+    const redisOptions = {
+      // a low commandTimeout value would likely cause sharded clusters to fail `enableReadyCheck` due to it running `CLUSTER INFO`
+      // which is a slow command. timeouts are being handled by the client#dispatch method.
+      enableOfflineQueue: false,
+      keyPrefix: config.prefix,
+      password: config.password,
+      tls: config.tls,
+      reconnectOnError: (err) => {
+        // will force a reconnect when error starts with `READONLY`
+        // this code is only triggered when auto-failover is disabled
+        // more: https://github.com/luin/ioredis#reconnect-on-error
+        return err.message.includes('READONLY');
+      },
+    };
+
+    const clusterOptions = {
+      slotsRefreshTimeout: config.slotsRefreshTimeout || 3000,
+      slotsRefreshInterval: config.slotsRefreshInterval || ms('5m'),
+      keyPrefix: config.prefix,
+      dnsLookup: config.dnsLookup,
+      enableReadyCheck: true,
+      redisOptions
+    };
+
+    this.redis = null;
+    if (config.nodes) {
+      this.redis = new Redis.Cluster(config.nodes, clusterOptions);
+    } else {
+      this.redis = new Redis(config.uri, redisOptions);
+      this.setupPing(config);
+    }
+
+    this.redis.defineCommand('take', {
+      numberOfKeys: 1,
+      lua: TAKE_LUA
+    });
+
+    this.redis.defineCommand('put', {
+      numberOfKeys: 1,
+      lua: PUT_LUA
+    });
+
+    this.redis.on('ready', () => {
+      this.emit('ready');
+    });
+
+    this.redis.on('error', (err) => {
+      this.emit('error', err);
+    });
+
+    this.redis.on('node error', (err, node) => {
+      this.emit('node error', err, node);
+    });
+
+  }
+
+  setupPing(config) {
+    this.redis.on("ready", () => this.startPing(config));
+    this.redis.on("close", () => this.stopPing());
+  }
+
+  startPing(config) {
+    this.stopPing();
+    this.ping = new DBPing(config.ping, this.redis);
+    this.ping.on("ping", (data) => this.emit("ping", data));
+  }
+
+  stopPing() {
+    if(this.ping) {
+      this.ping.stop();
+      this.ping.removeAllListeners();
+    }
+  }
+
+  close(callback) {
+    this.stopPing();
+    this.redis.quit(callback);
+  }
+
+  configurateBuckets(buckets) {
+    if (buckets) {
+      this.buckets = utils.buildBuckets(buckets);
+    }
+  }
+
+  configurateBucket(key, bucket) {
+    this.buckets[key] = utils.buildBucket(bucket);
+  }
+
+  /**
+   * @param {string} type
+   * @param {object} params
+   * @returns
+   */
+  bucketKeyConfig(type, params) {
+    if (typeof params.configOverride === 'object') {
+      return utils.normalizeTemporals(params.configOverride);
+    }
+
+    const fromOverride = type.overrides[params.key];
+    if (fromOverride) {
+      return fromOverride;
+    }
+
+    const fromCache = type.overridesCache && type.overridesCache.get(params.key);
+    if (fromCache) {
+      return fromCache;
+    }
+
+    const fromMatch = _.find(type.overridesMatch, (o) => {
+      return o.match.exec(params.key);
+    });
+    if (fromMatch) {
+      type.overridesCache.set(params.key, fromMatch);
+      return fromMatch;
+    }
+
+    return type;
+  }
+
+  // not super accurate given clock drift across redis and host
+  calculateReset(bucketKeyConfig, remaining, now) {
+    if (!bucketKeyConfig.per_interval) {
+      return 0;
+    }
+
+    now = now || Date.now();
+    const missing = bucketKeyConfig.size - remaining;
+    const msToCompletion = Math.ceil(missing * bucketKeyConfig.drip_interval);
+    return Math.ceil((now + msToCompletion) / 1000);
+  }
+
+  /**
+   * Take N elements from a bucket if available.
+   *
+   * @param {takeParams} params - The params for take.
+   * @param {function(Error, takeResult)} callback.
+   */
+  take(params, callback) {
+    const valError = validateParams(params, this.buckets);
+    if (valError) {
+      return process.nextTick(callback, valError);
+    }
+
+    const bucket = this.buckets[params.type];
+    const bucketKeyConfig = this.bucketKeyConfig(bucket, params);
+
+    const key = `${params.type}:${params.key}`;
+
+    let count = this._determineCount({
+      paramsCount: params.count,
+      defaultCount: 1,
+      bucketKeyConfigSize: bucketKeyConfig.size,
+    });
+
+    if (bucketKeyConfig.unlimited) {
+      return process.nextTick(callback, null, {
+        conformant: true,
+        remaining: bucketKeyConfig.size,
+        reset: Math.ceil(Date.now() / 1000),
+        limit: bucketKeyConfig.size,
+        delayed: false,
+      });
+    }
+
+    if (bucketKeyConfig.skip_n_calls > 0) {
+      const prevCall = this.callCounts.get(key);
+
+      if (prevCall) {
+        const shouldGoToRedis = prevCall.count >= bucketKeyConfig.skip_n_calls
+
+
+        if (!shouldGoToRedis) {
+          prevCall.count ++;
+          return process.nextTick(callback, null, prevCall.res);
+        }
+
+        // if lastCall not exists it's the first time that we go to redis.
+        // so we don't change count; subsequently calls take count should be
+        // proportional to the number of call that we skip.
+        // if count=3, and we go every 5 times, take should 15
+        // This parameter is most likely 1, and doing times is an overkill but better safe than sorry.
+        if (shouldGoToRedis) {
+          // we need to account for the skipped calls + the current call
+          count *= (bucketKeyConfig.skip_n_calls + 1);
+        }
+      }
+
+
+    }
+
+    this.redis.take(key,
+      bucketKeyConfig.ms_per_interval || 0,
+      bucketKeyConfig.size,
+      count,
+      Math.ceil(bucketKeyConfig.ttl || this.globalTTL),
+      bucketKeyConfig.drip_interval || 0,
+      (err, results) => {
+        if (err) {
+          return callback(err);
+        }
+
+        const remaining = parseInt(results[0], 10);
+        const conformant = parseInt(results[1], 10) ? true : false;
+        const currentMS = parseInt(results[2], 10);
+        const reset = parseInt(results[3], 10);
+        const res = {
+          conformant,
+          remaining,
+          reset: Math.ceil(reset / 1000),
+          limit: bucketKeyConfig.size,
+          delayed: false,
+        };
+
+        if (bucketKeyConfig.skip_n_calls > 0) {
+          this.callCounts.set(key, { res, count: 0 });
+        }
+
+        return callback(null, res);
+      });
+  }
+
+  /**
+   * Take N elements from a bucket if available otherwise wait for them.
+   * The callback is called when the number of request tokens is available.
+   *
+   * @param {waitParams} params - The params for take.
+   * @param {function(Error, waitResult)} callback.
+   */
+  wait(params, callback) {
+    this.take(params, (err, result) => {
+      if (err || result.conformant) {
+        return callback(err, result);
+      }
+
+      const bucket = this.buckets[params.type];
+      const bucketKeyConfig = this.bucketKeyConfig(bucket, params);
+      const count = params.count || 1;
+      const required = count - result.remaining;
+      const minWait = Math.ceil(required * bucketKeyConfig.interval / bucketKeyConfig.per_interval);
+
+      return setTimeout(() => {
+        this.wait(params, (err, result) => {
+          if (err) {
+            return callback(err);
+          }
+          result.delayed = true;
+          callback(null, result);
+        });
+      }, minWait);
+    });
+  }
+
+  /**
+   * Put N elements in the bucket.
+   *
+   * @param {putParams} params - The params for take.
+   * @param {function(Error, putResult)} [callback].
+   */
+  put(params, callback) {
+    callback = callback || _.noop;
+
+    const valError = validateParams(params, this.buckets);
+    if (valError) {
+      return process.nextTick(callback, valError);
+    }
+
+    const bucket = this.buckets[params.type];
+    const bucketKeyConfig = this.bucketKeyConfig(bucket, params);
+
+    const count = Math.min(
+      this._determineCount({
+        paramsCount: params.count,
+        defaultCount: bucketKeyConfig.size,
+        bucketKeyConfigSize: bucketKeyConfig.size,
+      }),
+      bucketKeyConfig.size);
+
+    if (bucketKeyConfig.unlimited) {
+      return process.nextTick(callback, null, {
+        remaining: bucketKeyConfig.size,
+        reset: Math.ceil(Date.now() / 1000),
+        limit: bucketKeyConfig.size
+      });
+    }
+
+    const key = `${params.type}:${params.key}`;
+    this.redis.put(key,
+      count,
+      bucketKeyConfig.size,
+      Math.ceil(bucketKeyConfig.ttl || this.globalTTL),
+      bucketKeyConfig.drip_interval || 0,
+      (err, results) => {
+        if (err) {
+          return callback(err);
+        }
+
+        const remaining = parseInt(results[0], 10);
+        return callback(null, {
+          remaining: remaining,
+          reset: Math.ceil(parseInt(results[3], 10) / 1000),
+          limit: bucketKeyConfig.size
+        });
+      });
+  }
+
+  /**
+   * Get elements in the bucket.
+   *
+   * @param {getParams} params - The params for take.
+   * @param {function(Error, getResult)} [callback].
+   */
+  get(params, callback) {
+    callback = callback || _.noop;
+
+    const valError = validateParams(params, this.buckets);
+    if (valError) {
+      return process.nextTick(callback, valError);
+    }
+
+    const bucket = this.buckets[params.type];
+    const bucketKeyConfig = this.bucketKeyConfig(bucket, params);
+
+    if (bucketKeyConfig.unlimited) {
+      return process.nextTick(callback, null, {
+        remaining: bucketKeyConfig.size,
+        reset: Math.ceil(Date.now() / 1000),
+        limit: bucketKeyConfig.size
+      });
+    }
+
+    const key = `${params.type}:${params.key}`;
+    this.redis.hmget(key, 'r', 'd',
+      (err, results) => {
+        if (err) {
+          return callback(err);
+        }
+
+        let remaining = parseInt(results[0], 10);
+        remaining = Number.isInteger(remaining) ? remaining : bucketKeyConfig.size;
+        return callback(null, {
+          remaining,
+          reset: this.calculateReset(bucketKeyConfig, remaining, parseInt(results[1], 10)),
+          limit: bucketKeyConfig.size
+        });
+      });
+  }
+
+  /**
+   * Resets/re-fills all keys in all buckets.
+   * @param {function(Error)} [callback].
+   */
+  resetAll(callback) {
+    callback = callback || _.noop;
+
+    const dbs = this.redis.nodes ? this.redis.nodes('master') : [this.redis];
+    async.each(dbs, (db, cb) => {
+      db.flushdb(cb);
+    }, callback);
+  }
+
+  _determineCount({ paramsCount, defaultCount, bucketKeyConfigSize }) {
+    if (paramsCount === 'all') {
+      return bucketKeyConfigSize;
+    }
+
+    if (Number.isInteger(paramsCount)) {
+      return paramsCount;
+    }
+
+    if (!paramsCount) {
+      return defaultCount;
+    }
+
+    throw new Error('if provided, count must be \'all\' or an integer value');
+  }
+}
+
+
+module.exports = LimitDBRedis;
+
+/**
+ * And now some typedefs for you:
+ *
+ * @typedef {Object} type
+ * @property {integer} [per_interval] The number of tokens to add per interval.
+ * @property {integer} [interval] The length of the interval in milliseconds.
+ * @property {integer} [size] The maximum number of tokens in the bucket.
+ * @property {integer} [per_second] The number of tokens to add per second. Equivalent to "interval: 1000, per_interval: x".
+ * @property {integer} [per_minute] The number of tokens to add per minute. Equivalent to "interval: 60000, per_interval: x".
+ * @property {integer} [per_hour] The number of tokens to add per hour. Equivalent to "interval: 3600000, per_interval: x".
+ * @property {integer} [per_day] The number of tokens to add per day. Equivalent to "interval: 86400000, per_interval: x".
+ *
+ * @typedef {Object} params
+ * uri nodes buckets prefix
+ * @property {string} [params.uri] Address of Redis.
+ * @property {Object.<string, object>} [params.nodes] Redis Cluster Configuration https://github.com/luin/ioredis#cluster".
+ * @property {Object.<string, type>} [params.types] The buckets configuration.
+ * @property {string} [params.prefix] Prefix keys in Redis.
+ * @property {type} [params.configOverride] Bucket configuration override
+ *
+ * @typedef takeParams
+ * @property {string} type The name of the bucket type.
+ * @property {string} key The key of the bucket instance.
+ * @property {integer} [count=1] The number of tokens to take from the bucket.
+ * @property {type} configOverride Externally provided bucket configruation
+ *
+ * @typedef takeResult
+ * @property {boolean} conformant Returns true if there is enough capacity in the bucket and the tokens has been removed.
+ * @property {integer} remaining The number of tokens remaining in the bucket.
+ * @property {integer} reset A unix timestamp indicating when the bucket is going to be full.
+ * @property {integer} limit The size of the bucket.
+ *
+ * @typedef waitParams
+ * @property {string} type The name of the bucket type.
+ * @property {string} key The key of the bucket instance.
+ * @property {integer} [count=1] The number of tokens to wait for.
+ * @property {type} configOverride Externally provided bucket configruation
+ *
+ * @typedef waitResult
+ * @property {integer} remaining The number of tokens remaining in the bucket.
+ * @property {integer} reset A unix timestamp indicating when the bucket is going to be full.
+ * @property {integer} limit The size of the bucket.
+ *
+ * @typedef putParams
+ * @property {string} type The name of the bucket type.
+ * @property {string} key The key of the bucket instance.
+ * @property {integer} [count=SIZE] The number of tokens to put in the bucket. Defaults to the size of the bucket.
+ * @property {type} configOverride Externally provided bucket configruation
+ *
+ * @typedef putResult
+ * @property {integer} remaining The number of tokens remaining in the bucket.
+ * @property {integer} reset A unix timestamp indicating when the bucket is going to be full.
+ * @property {integer} limit The size of the bucket.
+ *
+ * @typedef getParams
+ * @property {string} type The name of the bucket type.
+ * @property {string} key The key of the bucket instance.
+ * @property {type} configOverride Externally provided bucket configruation
+ *
+ * @typedef getResult
+ * @property {integer} remaining The number of tokens remaining in the bucket.
+ * @property {integer} reset A unix timestamp indicating when the bucket is going to be full.
+ * @property {integer} limit The size of the bucket.
+*/

package/limitd-redis/lib/db_ping.js

@@ -0,0 +1,106 @@
+const cbControl = require('./cb');
+const utils = require("./utils");
+const EventEmitter = require("events").EventEmitter;
+
+const PING_SUCCESS = "successful";
+const PING_ERROR = "error";
+const PING_RECONNECT = "reconnect";
+const PING_RECONNECT_DRY_RUN = "reconnect-dry-run";
+
+const DEFAULT_PING_INTERVAL = 3000; // Milliseconds
+
+class DBPing extends EventEmitter {
+  constructor(config, redis) {
+    super();
+
+    this.redis = redis;
+    this.config = {
+      commandTimeout: 125,
+      enabled: config ? true : false,
+      interval: config && config.interval ?  config.interval : DEFAULT_PING_INTERVAL,
+      maxFailedAttempts: config && config.maxFailedAttempts ? config.maxFailedAttempts : 5,
+      reconnectIfFailed:
+        utils.functionOrFalse(config ? config.reconnectIfFailed : undefined) || (() => false),
+    };
+
+    this.failedPings = 0;
+
+    this.start();
+  }
+
+  start() {
+    const doPing = () => {
+      if (!this.config.enabled) {
+        return;
+      }
+
+      let start = Date.now();
+      this.redis.ping(cbControl((err) => {
+        let duration = Date.now() - start;
+        err
+          ? this.pingKO(triggerLoop, err, duration)
+          : this.pingOK(triggerLoop, duration);
+      }).timeout(this.config.commandTimeout));
+    };
+
+    const triggerLoop = () => setTimeout(doPing, this.config.interval);
+
+    doPing();
+  }
+
+  stop() {
+    this.enabled = false;
+  }
+
+  pingOK(callback, duration) {
+    this.reconnecting = false;
+    this.failedPings = 0;
+    this.emitPingResult(PING_SUCCESS, undefined, duration, 0);
+    callback();
+  }
+
+  pingKO(callback, err, duration) {
+    this.failedPings++;
+    this.emitPingResult(PING_ERROR, err, duration, this.failedPings);
+
+    if (this.failedPings < this.config.maxFailedAttempts) {
+      return callback();
+    }
+
+    if (!this.config.reconnectIfFailed()) {
+      return this.emitPingResult(
+        PING_RECONNECT_DRY_RUN,
+        undefined,
+        0,
+        this.failedPings
+      );
+    }
+
+    this.retryStrategy(() => {
+      this.emitPingResult(PING_RECONNECT, undefined, 0, this.failedPings);
+      this.redis.disconnect(true);
+    });
+  }
+
+  emitPingResult(status, err, duration, failedPings) {
+    const result = {
+      status: status,
+      duration: duration,
+      error: err,
+      failedPings: failedPings,
+    };
+    this.emit("ping", result);
+  }
+
+  retryStrategy(callback) {
+    //jitter between 0% and 10% of the total wait time needed to reconnect
+    //i.e. if interval = 100 and maxFailedAttempts = 3 => it'll randomly jitter between 0 and 30 ms
+    const deviation =
+      utils.randomBetween(0, 0.1) *
+      this.config.interval *
+      this.config.maxFailedAttempts;
+    setTimeout(callback, deviation);
+  }
+}
+
+module.exports = DBPing;

package/limitd-redis/lib/put.lua

@@ -0,0 +1,31 @@
+local tokens_to_add        = tonumber(ARGV[1])
+local bucket_size          = tonumber(ARGV[2])
+local ttl                  = tonumber(ARGV[3])
+local drip_interval        = tonumber(ARGV[4])
+
+local current_time = redis.call('TIME')
+local current_timestamp_ms = current_time[1] * 1000 + current_time[2] / 1000
+
+local current_remaining = redis.call('HMGET', KEYS[1], 'r')[1]
+if current_remaining == false then
+  current_remaining = bucket_size
+end
+
+local new_content = math.min(current_remaining + tokens_to_add, bucket_size)
+
+redis.replicate_commands()
+if new_content < bucket_size then
+  redis.call('HMSET', KEYS[1],
+            'd', current_timestamp_ms,
+            'r', new_content)
+  redis.call('EXPIRE', KEYS[1], ttl)
+else
+  redis.call('DEL', KEYS[1])
+end
+
+local reset_ms = 0
+if drip_interval > 0 then
+    reset_ms = math.ceil(current_timestamp_ms + (bucket_size - new_content) * drip_interval)
+end
+
+return { new_content, current_timestamp_ms, reset_ms }

package/limitd-redis/lib/take.lua

@@ -0,0 +1,48 @@
+local tokens_per_ms        = tonumber(ARGV[1])
+local bucket_size          = tonumber(ARGV[2])
+local new_content          = tonumber(ARGV[2])
+local tokens_to_take       = tonumber(ARGV[3])
+local ttl                  = tonumber(ARGV[4])
+local drip_interval        = tonumber(ARGV[5])
+
+local current_time = redis.call('TIME')
+local current_timestamp_ms = current_time[1] * 1000 + current_time[2] / 1000
+
+local current = redis.pcall('HMGET', KEYS[1], 'd', 'r')
+
+if current.err ~= nil then
+    current = {}
+end
+
+if current[1] and tokens_per_ms then
+    -- drip bucket
+    local last_drip = current[1]
+    local content = current[2]
+    local delta_ms = math.max(current_timestamp_ms - last_drip, 0)
+    local drip_amount = delta_ms * tokens_per_ms
+    new_content = math.min(content + drip_amount, bucket_size)
+elseif current[1] and tokens_per_ms == 0 then
+    -- fixed bucket
+    new_content = current[2]
+end
+
+local enough_tokens = new_content >= tokens_to_take
+
+if enough_tokens then
+    new_content = math.min(new_content - tokens_to_take, bucket_size)
+end
+
+-- https://redis.io/commands/EVAL#replicating-commands-instead-of-scripts
+redis.replicate_commands()
+
+redis.call('HMSET', KEYS[1],
+            'd', current_timestamp_ms,
+            'r', new_content)
+redis.call('EXPIRE', KEYS[1], ttl)
+
+local reset_ms = 0
+if drip_interval > 0 then
+    reset_ms = math.ceil(current_timestamp_ms + (bucket_size - new_content) * drip_interval)
+end
+
+return { new_content, enough_tokens, current_timestamp_ms, reset_ms }