@withjoy/limiter 0.0.9 → 0.1.0

package/README.md

@@ -1,4 +1,4 @@
-### A wrapper for limitd and limitdctl
+### A wrapper for limitd-redis
 
 ##### Why ?
 
@@ -7,16 +7,10 @@ Provides utilities and rollback functionality for our clients.
 ### Testing:
 
 `npm run test` will run offline tests using a mock.
-`npm run sanity-check` will run against a limitd service configured to run with the config at `/tests/limitd.config`.
+`npm run sanity-check` will run against a limitd service configured to run with the test buckets.
 
-##### How can I run that limitd instance quickly ?
-
-check [the Wiki](https://withjoy.atlassian.net/wiki/spaces/KNOW/pages/1905557728/Running+withjoy+limitd+locally)
-
-run this in the root of this repo:
+## Quick Test
 ```bash
-    docker run -p 9231:9231 \
-    --mount \
-        type=bind,source="$(pwd)"/tests/limitd.config,target=/etc/limitd.config \
-    -it joylife/limitd:release-6-d5e4805
+node tests/performTestWithTestPerMinute.js
+
 ```

package/limiter.js

@@ -1,10 +1,24 @@
 "use strict";
 
-
-var LimitdClient = require("limitd-client");
+var LimitdRedis = require("limitd-redis"); // name to redis matter
+
+const defaultBuckets = {
+	emailsByEventId: { size: 3000 },
+	emailsByEventIdRate: { size: 500, per_hour: 500 },
+	emailsByEventId_Warn: { size: 1000 },
+	emailsByEventIdRate_Warn: { size: 200, per_hour: 200 },
+	eventsByUserId: { size: 5, per_hour: 5 },
+	eventsByIpRate: { size: 240, per_hour: 240 },
+	paymentAttempts: { size: 9000 },
+	paymentAttemptsByReceiverId: { size: 900 },
+	// for development testing
+	testPerMinute: { size: 3, per_minute: 1 },
+};
+
+const defaultRedisKeyPrefix = "limitdRedis:";
 
 class Operation {
-	constructor({opname,type,id,amount}){
+	constructor({ opname, type, id, amount }) {
 		this.opname = opname;
 		this.type = type;
 		this.id = id;
@@ -24,12 +38,11 @@ class Operation {
 let connection = null;
 
 class Limiter {
-
 	constructor(config) {
 		this._console = config.console;
+		this._config = config;
 		this._operationList = [];
-		this._limitd = undefined;
-		this.connect(config.limitdUrl);
+		this._limitdRedis = this.connect();
 	}
 
 	/*
@@ -37,11 +50,21 @@ class Limiter {
 			and we invoke this method from the Constructor.
 			stub it to prevent a real connect from happening under Test Suite Conditions
 	*/
-	connect(url) {
-		if(!connection) {
-			connection = new LimitdClient(url);
+	connect() {
+		let buckets = this._config.buckets || {};
+		buckets = {
+			...defaultBuckets,
+			...buckets,
+		};
+		const keyPrefix = this._config.keyPrefix || defaultRedisKeyPrefix;
+		if (!connection) {
+			connection = new LimitdRedis({
+				uri: this._config.limitdUrl,
+				buckets,
+				keyPrefix,
+			});
 		}
-		this._limitd = connection;
+		this._limitdRedis = connection;
 		return connection;
 	}
 
@@ -49,16 +72,16 @@ class Limiter {
 		Go through the log of things we did and do the opposite
 	*/
 	revert() {
-		let ops = this._operationList.map(op => op.clone());
+		let ops = this._operationList.map((op) => op.clone());
 		this._operationList = [];
 
 		let p = Promise.resolve();
 		for (let op of ops) {
 			switch (op.opname) {
-				case 'put':
+				case "put":
 					p = p.then(() => this._take(op.type, op.id, op.amount));
 					break;
-				case 'take':
+				case "take":
 					p = p.then(() => this._put(op.type, op.id, op.amount));
 					break;
 			}
@@ -71,24 +94,36 @@ class Limiter {
 		this._operationList.push(op);
 	}
 
+	close() {
+		return new Promise((resolve, reject) => {
+			this._limitdRedis.close((err, resp) => {
+				if (err) {
+					this._console.error(err);
+					return reject(err);
+				}
+				console.log("Connection closed");
+				resolve(resp);
+			});
+		});
+	}
 	// promise wrapper for limitd lib put
-	_put (type, id, amount) {
+	_put(type, id, amount) {
 		return new Promise((resolve, reject) =>
-			this._limitd.put(type, id, amount, (err, resp) => {
+			this._limitdRedis.put(type, id, amount, (err, resp) => {
 				if (err) {
 					this._console.error(err);
 					return reject(err);
 				}
-				this.record(new Operation({opname: "put",type,id,amount}));
-				resolve(null);
+				this.record(new Operation({ opname: "put", type, id, amount }));
+				resolve(resp);
 			})
 		);
 	}
 
 	// promise wrapper for limitd lib take
-	_take (type, id, amount) {
+	_take(type, id, amount) {
 		return new Promise((resolve, reject) =>
-			this._limitd.take(type, id, amount, (err, resp) => {
+			this._limitdRedis.take(type, id, amount, (err, resp) => {
 				if (err) {
 					this._console.error(err);
 					return reject(err);
@@ -99,10 +134,10 @@ class Limiter {
 					return reject(new Error("Non Conformant"));
 				}
 
-				this.record(new Operation({opname: "take",type,id,amount}));
-				return resolve(null);
+				this.record(new Operation({ opname: "take", type, id, amount }));
+				return resolve(resp);
 			})
-		)
+		);
 	}
 
 	transfer(type, id, amount) {
@@ -112,17 +147,20 @@ class Limiter {
 
 		this._console.info(`Transfering ${amount} from ${type}-${id}`);
 		// If amount == 0, there is nothing to do
-		if (amount == 0) { return Promise.resolve(null); }
-		if (amount < 0) { return this._put(type, id, -amount); }
+		if (amount == 0) {
+			return Promise.resolve(null);
+		}
+		if (amount < 0) {
+			return this._put(type, id, -amount);
+		}
 		return this._take(type, id, amount);
 	}
 
 	transferAll(arr) {
 		return Promise.all(
-			arr.map(spec => this.transfer(spec[0], spec[1], spec[2]))
-		).then(() => null);
+			arr.map((spec) => this.transfer(spec[0], spec[1], spec[2]))
+		);
 	}
 }
 
-
 module.exports = Limiter;

package/package.json

@@ -1,21 +1,19 @@
 {
   "name": "@withjoy/limiter",
-  "version": "0.0.9",
+  "version": "0.1.0",
   "description": "Api Rate limiter",
   "main": "limiter.js",
   "scripts": {
-    "test": "jest",
+    "test": "NODE_ENV=test node --expose-gc --max-old-space-size=6144  node_modules/.bin/jest  --logHeapUsage  -i --forceExit",
     "sanity-check": "node tests/sanityCheck.js k 10"
   },
   "author": "services@withjoy.com",
   "license": "MIT",
   "dependencies": {
-    "limitd-client": "^2.9.1"
+    "limitd-redis": "^5.1.1"
   },
   "devDependencies": {
-    "jest": "^24.9.0",
-    "limitd": "^5.3.0",
-    "limitdctl": "^1.1.1"
+    "jest": "^24.9.0"
   },
   "directories": {
     "test": "tests"

package/tests/limiter.test.js

@@ -1,212 +1,270 @@
-const Limiter = require('../limiter');
+const Limiter = require("../limiter");
 
 const PUT_FAILED = "Failed to put!";
 const TAKE_FAILED = "Failed to take!";
+
+const TEST_BUCKETS = {
+	emails: {
+		size: 10,
+	},
+	emailRate: {
+		size: 10,
+		per_hour: 10,
+	},
+};
 /*
     The limitd service is essentially a token bank
     that we can withdraw from (take) or deposit to (put)
 */
 const limiterWithMockService = (succeeds = true, conforms = true) => {
-    let token_pile = 0;
-    const limiter = new Limiter({ console: console });
-
-    limiter._limitd = (new class MockLimitd {
-        constructor(should_succeed, should_conform) {
-            this.should_succeed = should_succeed;
-            this.should_conform = should_conform;
-        }
-        put(type, id, amount, callback) {
-            console.log("called put", type, id, amount);
-            console.log(this.should_succeed)
-            if(!this.should_succeed){
-                return callback(new Error(PUT_FAILED), "This is not falsey");
-            }
-            token_pile += amount;
-            callback(null, {})
-        }
-        take(type, id, amount, callback) {
-            console.log("called take", type, id, amount);
-            if(!this.should_succeed){
-                return callback(new Error(TAKE_FAILED), null);
-            }else if (!this.should_conform) {
-                return callback(null, {}); // non coformant
-            }
-            token_pile -= amount;
-            callback(null, { conformant: true })
-        }
-        _num_tokens(){ return token_pile }
-    }(succeeds, conforms))
-    return limiter;
-}
-
-test("Resolves to null if we aren't doing a real transfer", done => {
-    let limiter = limiterWithMockService();
-    limiter.transfer("a", "b", 0, null)
-        .then(result => {
-            expect(result).toBe(null);
-            expect(limiter._limitd._num_tokens()).toBe(0);
-            done();
-        })
+	let token_pile = 0;
+	const limiter = new Limiter({
+		console: console,
+		limitdUrl: "localhost:6379",
+		buckets: TEST_BUCKETS,
+	});
+
+	limiter._limitdRedis = new (class MockLimitd {
+		constructor(should_succeed, should_conform) {
+			this.should_succeed = should_succeed;
+			this.should_conform = should_conform;
+		}
+		put(type, id, amount, callback) {
+			console.log("called put", type, id, amount);
+			console.log(this.should_succeed);
+			if (!this.should_succeed) {
+				return callback(new Error(PUT_FAILED), "This is not falsey");
+			}
+			token_pile += amount;
+			callback(null, {});
+		}
+		take(type, id, amount, callback) {
+			console.log("called take", type, id, amount);
+			if (!this.should_succeed) {
+				return callback(new Error(TAKE_FAILED), null);
+			} else if (!this.should_conform) {
+				return callback(null, {}); // non coformant
+			}
+			token_pile -= amount;
+			callback(null, { conformant: true });
+		}
+		_num_tokens() {
+			return token_pile;
+		}
+	})(succeeds, conforms);
+	return limiter;
+};
+
+test("perfrom limitd-redis testing with test bucket", async (done) => {
+	const limiter = new Limiter({
+		limitdUrl: "localhost:6379",
+		console: console,
+		buckets: {
+			test: { size: 3, per_second: 2 },
+		},
+	});
+	// take 2  # Conformant - Remaining: 1
+	const result1 = await limiter.transfer("test", "test-key", 2);
+	expect(result1).toMatchObject({
+		conformant: true,
+		remaining: 1,
+		limit: 3,
+		delayed: false,
+	});
+	await expect(limiter.transfer("test", "test-key", 2)).rejects.toThrow(
+		"Non Conformant"
+	);
+
+	// Put 1
+	const result3 = await limiter.transfer("test", "test-key", -1);
+	expect(result3).toMatchObject({
+		remaining: 2,
+		limit: 3,
+	});
+
+    //Now Conformant - Remaining: 2
+	const result4 = await limiter.transfer("test", "test-key", 2);
+	expect(result4).toMatchObject({
+		conformant: true,
+		remaining: 0,
+		limit: 3,
+		delayed: false,
+	});
+	await limiter.close();
+	done();
+});
+
+test("Resolves to null if we aren't doing a real transfer", (done) => {
+	let limiter = limiterWithMockService();
+	limiter.transfer("a", "b", 0, null).then((result) => {
+		expect(result).toBe(null);
+		expect(limiter._limitdRedis._num_tokens()).toBe(0);
+		done();
+	});
 });
 
-test("Adds tokens if provided a negative token amount", done => {
-    let limiter = limiterWithMockService();
-    limiter.transfer("a", "b", -5)
-        .then(result => {
-            expect(result).toBe(null)
-            expect(limiter._limitd._num_tokens()).toBe(5);
-            done();
-        })
+test("Adds tokens if provided a negative token amount", (done) => {
+	let limiter = limiterWithMockService();
+	limiter.transfer("a", "b", -5).then((result) => {
+		expect(result).toMatchObject({});
+		expect(limiter._limitdRedis._num_tokens()).toBe(5);
+		done();
+	});
 });
 
-test("Removes tokens if provided a positive token amount", done => {
-    let limiter = limiterWithMockService();
-    limiter.transfer("a", "b", 5)
-        .then(result => {
-            expect(result).toBe(null)
-            expect(limiter._limitd._num_tokens()).toBe(-5);
-            done();
-        })
+test("Removes tokens if provided a positive token amount", (done) => {
+	let limiter = limiterWithMockService();
+	limiter.transfer("a", "b", 5).then((result) => {
+		expect(result).toMatchObject({ conformant: true });
+		expect(limiter._limitdRedis._num_tokens()).toBe(-5);
+		done();
+	});
 });
 
-test("Reverts all taken tokens when revert is called", done => {
-    let limiter = limiterWithMockService();
-    limiter.transfer("a", "b", 5)
-        .then(result => {
-            expect(result).toBe(null)
-            expect(limiter._limitd._num_tokens()).toBe(-5);
-        })
-        .then(() => limiter.revert())
-        .then(() => {
-            expect(limiter._limitd._num_tokens()).toBe(0);
-            done();
-        })
+test("Reverts all taken tokens when revert is called", (done) => {
+	let limiter = limiterWithMockService();
+	limiter
+		.transfer("a", "b", 5)
+		.then((result) => {
+			expect(result).toMatchObject({ conformant: true });
+			expect(limiter._limitdRedis._num_tokens()).toBe(-5);
+		})
+		.then(() => limiter.revert())
+		.then(() => {
+			expect(limiter._limitdRedis._num_tokens()).toBe(0);
+			done();
+		});
 });
 
-test("Reverts all given tokens when revert is called", done => {
-    let limiter = limiterWithMockService();
-    limiter.transfer("a", "b", -5)
-        .then(result => {
-            expect(result).toBe(null)
-            expect(limiter._limitd._num_tokens()).toBe(5);
-        })
-        .then(() => limiter.revert())
-        .then(() => {
-            expect(limiter._limitd._num_tokens()).toBe(0);
-            done();
-        })
+test("Reverts all given tokens when revert is called", (done) => {
+	let limiter = limiterWithMockService();
+	limiter
+		.transfer("a", "b", -5)
+		.then((result) => {
+			expect(result).toMatchObject({});
+			expect(limiter._limitdRedis._num_tokens()).toBe(5);
+		})
+		.then(() => limiter.revert())
+		.then(() => {
+			expect(limiter._limitdRedis._num_tokens()).toBe(0);
+			done();
+		});
 });
 
-test("Reverts a series of actions when revert is called", done => {
-    let limiter = limiterWithMockService();
-    limiter.transfer("a", "b", -5)
-        .then(result => {
-            expect(result).toBe(null)
-            expect(limiter._limitd._num_tokens()).toBe(5);
-        })
-        .then(() => limiter.transfer("a", "b", 2))
-        .then(result => {
-            expect(result).toBe(null)
-            expect(limiter._limitd._num_tokens()).toBe(3);
-        })
-        .then(() => limiter.revert())
-        .then(() => {
-            expect(limiter._limitd._num_tokens()).toBe(0);
-            done();
-        })
+test("Reverts a series of actions when revert is called", (done) => {
+	let limiter = limiterWithMockService();
+	limiter
+		.transfer("a", "b", -5)
+		.then((result) => {
+			expect(result).toMatchObject({});
+			expect(limiter._limitdRedis._num_tokens()).toBe(5);
+		})
+		.then(() => limiter.transfer("a", "b", 2))
+		.then((result) => {
+			expect(result).toMatchObject({ conformant: true });
+			expect(limiter._limitdRedis._num_tokens()).toBe(3);
+		})
+		.then(() => limiter.revert())
+		.then(() => {
+			expect(limiter._limitdRedis._num_tokens()).toBe(0);
+			done();
+		});
 });
 
-test("Properly executes a transferAll()", done => {
-    let limiter = limiterWithMockService();
-    limiter.transferAll([
-        ["a", "b", -5],
-        ["a", "b",  2]
-    ])
-        .then(result => {
-            expect(result).toBe(null)
-            expect(limiter._limitd._num_tokens()).toBe(3);
-        })
-        .then(() => limiter.revert())
-        .then(() => {
-            expect(limiter._limitd._num_tokens()).toBe(0);
-            done();
-        })
+test("Properly executes a transferAll()", (done) => {
+	let limiter = limiterWithMockService();
+	limiter
+		.transferAll([
+			["a", "b", -5],
+			["a", "b", 2],
+		])
+		.then((result) => {
+			expect(result).toMatchObject([{}, { conformant: true }]);
+			expect(limiter._limitdRedis._num_tokens()).toBe(3);
+		})
+		.then(() => limiter.revert())
+		.then(() => {
+			expect(limiter._limitdRedis._num_tokens()).toBe(0);
+			done();
+		});
 });
 
 test("Errors from the underlying service are propagated (put)", () => {
-    let limiter = limiterWithMockService(succeeds=false);
-    let endsInFailure = limiter.transfer("a", "b", -5);
-    return expect(endsInFailure).rejects.toThrow(PUT_FAILED);
+	let limiter = limiterWithMockService((succeeds = false));
+	let endsInFailure = limiter.transfer("a", "b", -5);
+	return expect(endsInFailure).rejects.toThrow(PUT_FAILED);
 });
 
 test("Errors from the underlying service are propagated (take)", () => {
-    let limiter = limiterWithMockService(succeeds=false);
-    let endsInFailure = limiter.transfer("a", "b", 5);
-    return expect(endsInFailure).rejects.toThrow(TAKE_FAILED);
+	let limiter = limiterWithMockService((succeeds = false));
+	let endsInFailure = limiter.transfer("a", "b", 5);
+	return expect(endsInFailure).rejects.toThrow(TAKE_FAILED);
 });
 
 test("Non-conforming response results in an error", () => {
-    let limiter = limiterWithMockService(succeeds=true, conforms=false);
-    let endsInFailure = limiter.transfer("a", "b", 5);
-    return expect(endsInFailure).rejects.toThrow("Non Conformant");
+	let limiter = limiterWithMockService((succeeds = true), (conforms = false));
+	let endsInFailure = limiter.transfer("a", "b", 5);
+	return expect(endsInFailure).rejects.toThrow("Non Conformant");
 });
 
-test("An operation that fails is not logged", () => {
-    let limiter = limiterWithMockService();
-    limiter.transfer("a", "b", -5)
-        .then(result => {
-            expect(result).toBe(null)
-            expect(limiter._limitd._num_tokens()).toBe(5);
-            limiter.succeeds = false; // somebody is playing games
-        })
-        .then(() => limiter.transfer("a", "b", 2))
-        .catch(() => "failed uniquely!") // do nothing!
-        .then(result => {
-            expect(result).toBe("failed uniquely!")
-            expect(limiter._limitd._num_tokens()).toBe(5);
-            limiter.succeeds = true; // for real this time
-        })
-        .then(() => limiter.revert())
-        .then(() => {
-            expect(limiter._limitd._num_tokens()).toBe(0);
-            done();
-        })
+test("An operation that fails is not logged", (done) => {
+	let limiter = limiterWithMockService();
+	limiter
+		.transfer("emailRate", "b", -5)
+		.then((result) => {
+			expect(result).toBe(null);
+			expect(limiter._limitdRedis._num_tokens()).toBe(5);
+			limiter.succeeds = false; // somebody is playing games
+		})
+		.then(() => limiter.transfer("emailRate", "b", 2))
+		.catch(() => "failed uniquely!") // do nothing!
+		.then((result) => {
+			expect(result).toBe("failed uniquely!");
+			expect(limiter._limitdRedis._num_tokens()).toBe(5);
+			limiter.succeeds = true; // for real this time
+		})
+		.then(() => limiter.revert())
+		.then(() => {
+			expect(limiter._limitdRedis._num_tokens()).toBe(0);
+			done();
+		});
 });
 
-test("Properly handles spamming revert() calls", done => {
-    let limiter = limiterWithMockService();
-    limiter.transfer("a", "b", -5)
-        .then(result => {
-            expect(result).toBe(null)
-            expect(limiter._limitd._num_tokens()).toBe(5);
-        })
-        .then(() => limiter.transfer("a", "b", 2))
-        .then(result => {
-            expect(result).toBe(null)
-            expect(limiter._limitd._num_tokens()).toBe(3);
-        })
-        .then(() =>
-            Promise.all([
-                limiter.revert(),
-                limiter.revert(),
-                limiter.revert(),
-                limiter.revert(),
-                limiter.revert(),
-                limiter.revert(),
-                limiter.revert(),
-                limiter.revert(),
-                limiter.revert(),
-                limiter.revert(),
-                limiter.revert(),
-                limiter.revert(),
-                limiter.revert(),
-                limiter.revert(),
-                limiter.revert(),
-                limiter.revert(),
-            ])
-        )
-        .then(() => {
-            expect(limiter._limitd._num_tokens()).toBe(0);
-            done();
-        })
-});
+test("Properly handles spamming revert() calls", (done) => {
+	let limiter = limiterWithMockService();
+	limiter
+		.transfer("a", "b", -5)
+		.then((result) => {
+			expect(result).toMatchObject({});
+			expect(limiter._limitdRedis._num_tokens()).toBe(5);
+		})
+		.then(() => limiter.transfer("a", "b", 2))
+		.then((result) => {
+			expect(result).toMatchObject({ conformant: true });
+			expect(limiter._limitdRedis._num_tokens()).toBe(3);
+		})
+		.then(() =>
+			Promise.all([
+				limiter.revert(),
+				limiter.revert(),
+				limiter.revert(),
+				limiter.revert(),
+				limiter.revert(),
+				limiter.revert(),
+				limiter.revert(),
+				limiter.revert(),
+				limiter.revert(),
+				limiter.revert(),
+				limiter.revert(),
+				limiter.revert(),
+				limiter.revert(),
+				limiter.revert(),
+				limiter.revert(),
+				limiter.revert(),
+			])
+		)
+		.then(() => {
+			expect(limiter._limitdRedis._num_tokens()).toBe(0);
+			done();
+		});
+});

package/tests/performTestWithTestPerMinute.js

@@ -0,0 +1,31 @@
+var Limiter = require("../limiter");
+
+async function performTestWithTestPerMinute() {
+	console.log("Testing limitd-redius with bucket testPerMinute");
+	const limiter = new Limiter({
+		limitdUrl: "localhost:6379",
+		console: console,
+	});
+	// take 2  # Conformant - Remaining: 1
+	const result1 = await limiter.transfer("testPerMinute", "key", 2);
+	console.log("First Turn Take Result", result1);
+
+	//Now Conformant - Remaining: 1
+	const result2 = await limiter
+		.transfer("testPerMinute", "key", 2)
+		.catch((err) => {
+			console.log("Second Turn Take Error", err.message);
+		});
+	console.log("Second Turn Take Result", result2);
+
+	// Put 1
+	const result3 = await limiter.transfer("testPerMinute", "key", -1);
+	console.log("Third Turn PUT Result", result3);
+
+	//Now Conformant - Remaining: 2
+	const result4 = await limiter.transfer("testPerMinute", "key", 2);
+	console.log("Fourth Turn Take Result", result4);
+	process.exit(1);
+}
+
+performTestWithTestPerMinute();

package/tests/sanityCheck.js

@@ -1,12 +1,17 @@
 "use strict";
 
-
 var Limiter = require("../limiter");
 
+const buckets = {
+	emails: { size: 10 },
+	emailRate: { size: 10, per_hour: 10 },
+};
+
 var limiter = new Limiter({
-	limitdUrl: "limitd://localhost:9231",
-	console: console
-})
+	limitdUrl: "localhost:6379",
+	console: console,
+	buckets,
+});
 
 var callback = function (err) {
 	if (err) {
@@ -18,10 +23,10 @@ var callback = function (err) {
 };
 
 var withdrawls = [
+	["emails", process.argv[2], -parseInt(process.argv[3])],
+	["emailRate", process.argv[2], -parseInt(process.argv[3])],
 	["emails", process.argv[2], parseInt(process.argv[3])],
-	["emailRate", process.argv[2], parseInt(process.argv[3])]
+	["emailRate", process.argv[2], parseInt(process.argv[3])],
 ];
 
-var withdrawlsComplete = [];
 limiter.transferAll(withdrawls).then(() => callback(), callback);
-

package/tests/limitd.config

@@ -1,14 +0,0 @@
-#port to listen on
-port: 9231
-
-#db path
-db: ./db
-
-#define the bucket types
-buckets:
-  emails:
-    size: 10
-  emailRate:
-    size: 10
-    per_hour: 10
-