@smpx/koa-request 1.0.4 → 1.2.0

package/GeoIP.js

@@ -1,4 +1,4 @@
-const maxmind = require('maxmind');
+const mmdb = require('mmdb-lib');
 const https = require('https');
 const zlib = require('zlib');
 const path = require('path');
@@ -9,17 +9,89 @@ const fileName = 'GeoLite2-City';
 const fileUrl = `https://raw.githubusercontent.com/GitSquared/node-geolite2-redist/master/redist/${fileName}.tar.gz`;
 const fileDir = __dirname;
 const filePath = `${fileDir}/${fileName}.mmdb`;
+const ONE_DAY = 24 * 3600 * 1000;
+// eslint-disable-next-line max-len
+const processMs = ((process.env.pm_id || process.pid || Math.floor(Math.random() * 60)) % 60) * 60 * 1000;
 let geoInitPromise;
+let geoInitSynced;
 let geoip;
+let updateTimer;
 
-function save(url, outDir) {
+class LRU {
+	constructor() {
+		this.maxItems = 10000;
+		this.cache = new Map();
+		this.oldCache = new Map();
+		this._size = 0;
+	}
+
+	_set(key, value) {
+		this.cache.set(key, value);
+		if (this.cache.size >= this.maxItems) {
+			this._size = this.cache.size;
+			this.oldCache = this.cache;
+			this.cache = new Map();
+		}
+	}
+
+	get(key) {
+		const value = this.cache.get(key);
+		if (value !== undefined) return value;
+		const oldValue = this.oldCache.get(key);
+		if (oldValue) {
+			this._set(key, oldValue);
+			return oldValue;
+		}
+		return undefined;
+	}
+
+	set(key, value) {
+		if (this.cache.has(key)) {
+			this.cache.set(key, value);
+		}
+		else {
+			this._size++;
+			this._set(key, value);
+		}
+		return this;
+	}
+}
+
+const readFile = fs.promises.readFile;
+async function mtime(file) {
+	try {
+		const stats = await fs.promises.stat(file);
+		return stats.mtimeMs || stats.ctimeMs;
+	}
+	catch (e) {
+		return 0;
+	}
+}
+
+function getMmdbReader(buffer) {
+	const cache = new LRU();
+	return new mmdb.Reader(buffer, {cache});
+}
+
+function validateReader(geoIPReader) {
+	try {
+		const country = geoIPReader.get('1.2.3.4')?.country?.iso_code;
+		return !!country;
+	}
+	catch (e) {
+		console.error(e);
+		return false;
+	}
+}
+
+function downloadNewDatabase() {
 	return new Promise((resolve, reject) => {
-		https.get(url, (res) => {
+		https.get(fileUrl, (res) => {
 			try {
 				const untar = res.pipe(zlib.createGunzip({})).pipe(tar.t());
 				untar.on('entry', (entry) => {
 					if (entry.path.endsWith('.mmdb')) {
-						const dstFilename = path.join(outDir, path.basename(entry.path) + '-tmp');
+						const dstFilename = path.join(fileDir, path.basename(entry.path) + '-tmp');
 						try {
 							entry.pipe(fs.createWriteStream(dstFilename));
 						}
@@ -30,98 +102,151 @@ function save(url, outDir) {
 				});
 				untar.on('error', e => reject(e));
 				untar.on('finish', () => {
-					fs.rename(filePath + '-tmp', filePath, (err) => {
-						if (err) {
-							reject(err);
+					setTimeout(async () => {
+						try {
+							const newGeoIp = getMmdbReader(await readFile(filePath + '-tmp'));
+							if (validateReader(newGeoIp)) {
+								resolve(newGeoIp);
+								fs.promises.rename(filePath + '-tmp', filePath).catch(console.error);
+							}
+							else {
+								reject(new Error('Invalid GeoIP Database!'));
+							}
 						}
-						else {
-							resolve();
+						catch (e) {
+							reject(e);
 						}
-					});
+					}, 100);
 				});
 			}
 			catch (error) {
-				throw new Error(`Could not fetch ${url}\n\nError:\n${error}`);
+				reject(error);
 			}
 		});
 	});
 }
 
-function mtime(file) {
-	return new Promise((resolve) => {
-		try {
-			fs.stat(file, (err, stats) => {
-				if (err) {
-					resolve(false);
-				}
-				else {
-					resolve(stats.mtimeMs || stats.ctimeMs);
-				}
-			});
-		}
-		catch (e) {
-			resolve(false);
-		}
-	});
-}
-
 async function download() {
-	const modifyTime = await mtime(filePath);
-	if (!modifyTime) {
-		console.log(`[Request::GeoIP] Downloading ${fileName} ...`);
-		await save(fileUrl, fileDir);
+	console.log(`[Request::GeoIP] Downloading ${fileName} ...`);
+	try {
+		const newGeoIp = await downloadNewDatabase();
 		console.log(`[Request::GeoIP] Downloaded ${fileName} !`);
+		return newGeoIp;
+	}
+	catch (e) {
+		console.error(e);
+		return null;
 	}
 }
 
 async function updateDb() {
+	clearTimeout(updateTimer);
 	const modifyTime = await mtime(filePath);
-	if (modifyTime < Date.now() - 2 * 24 * 3600 * 1000) {
-		console.log(`[Request::GeoIP] Downloading ${fileName} ...`);
-		await save(fileUrl, fileDir);
-		console.log(`[Request::GeoIP] Downloaded ${fileName} !`);
-		geoip = await maxmind.open(filePath);
+	if (modifyTime < Date.now() - 3 * ONE_DAY) {
+		const newGeoIp = await download();
+		if (newGeoIp) {
+			console.log('[Request::GeoIP] Updated GeoIP Database!');
+			geoip = newGeoIp;
+		}
 	}
-	setTimeout(updateDb, 3 * 24 * 3600 * 1000);
+	// add some random time in timeout to avoid race condition between multiple processes
+	const randomMs = Math.floor(Math.random() * 10000000);
+	updateTimer = setTimeout(updateDb, 3 * ONE_DAY + randomMs + processMs);
+	updateTimer.unref();
+}
+
+async function updateDbOnInit() {
+	if (process.env.NODE_ENV !== 'production') return;
+	clearTimeout(updateTimer);
+	// add some random time in timeout to avoid race condition between multiple processes
+	const randomMs = Math.floor(Math.random() * 1000000);
+	updateTimer = setTimeout(updateDb, randomMs + processMs);
+	updateTimer.unref();
+}
+
+async function _geoIpInit() {
+	try {
+		const newGeoIp = getMmdbReader(await readFile(filePath));
+		if (!validateReader(newGeoIp)) {
+			throw new Error('Invalid GeoIP file');
+		}
+		geoip = newGeoIp;
+	}
+	catch (e) {
+		console.error(e);
+		const newGeoIp = await download();
+		if (newGeoIp) {
+			geoip = newGeoIp;
+		}
+	}
+	updateDbOnInit();
 }
 
 async function geoIpInit() {
-	await download();
-	geoip = await maxmind.open(filePath);
-	if (process.env.NODE_ENV === 'production') {
-		updateDb();
+	if (!geoInitPromise) {
+		geoInitPromise = _geoIpInit();
+	}
+	return geoInitPromise;
+}
+
+function _geoIpInitSync() {
+	try {
+		const newGeoIp = getMmdbReader(fs.readFileSync(filePath));
+		if (!validateReader(newGeoIp)) {
+			throw new Error('Invalid GeoIP file');
+		}
+		geoip = newGeoIp;
+		updateDbOnInit();
+	}
+	catch (e) {
+		console.error(e);
+		geoIpInit().catch((err) => {
+			console.error(err);
+		});
+	}
+}
+
+function geoIpInitSync() {
+	if (!geoInitSynced) {
+		geoInitSynced = true;
+		_geoIpInitSync();
 	}
+	return geoInitSynced;
 }
 
 async function getGeoIp() {
 	if (!geoip) {
-		if (!geoInitPromise) {
-			geoInitPromise = geoIpInit();
-		}
-		await geoInitPromise;
-		geoInitPromise = null;
+		await geoIpInit();
 	}
 	return geoip;
 }
 
 function getGeoIpSync() {
 	if (!geoip) {
-		const buffer = fs.readFileSync(filePath);
-		geoip = new maxmind.Reader(buffer);
-		if (process.env.NODE_ENV === 'production') {
-			updateDb();
-		}
+		geoIpInitSync();
 	}
 	return geoip;
 }
 
 class GeoIP {
 	static async get(ip) {
-		return (await getGeoIp()).get(ip);
+		try {
+			return (await getGeoIp()).get(ip);
+		}
+		catch (e) {
+			console.error('Error getting geoip location', e);
+			return null;
+		}
 	}
 
 	static getSync(ip) {
-		return getGeoIpSync().get(ip);
+		try {
+			return getGeoIpSync().get(ip);
+		}
+		catch (e) {
+			console.error('Error getting geoip location', e);
+			return null;
+		}
 	}
 
 	static async init() {
@@ -141,7 +266,7 @@ async function main() {
 		ips.push(`${random(1, 255)}.${random(1, 255)}.${random(1, 255)}.${random(1, 255)}`);
 	}
 
-	console.log((await GeoIP.get('1.2.3.4')).subdivisions);
+	console.log((await GeoIP.get('1.2.3.4')).location);
 
 	console.time('ipLookup');
 	for (let i = 0; i < 100000; i++) {
@@ -150,6 +275,15 @@ async function main() {
 	}
 	console.timeEnd('ipLookup');
 	// takes approx ~ 900ms for 100,000 lookups
+
+	console.log((GeoIP.getSync('1.2.3.4')).location);
+
+	console.time('ipLookupSync');
+	for (let i = 0; i < 100000; i++) {
+		GeoIP.getSync(ips[i]);
+	}
+	console.timeEnd('ipLookupSync');
+	// takes approx ~ 900ms for 100,000 lookups
 }
 
 if (require.main === module) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@smpx/koa-request",
-  "version": "1.0.4",
+  "version": "1.2.0",
   "description": "Handle basic tasks for koajs",
   "main": "index.js",
   "types": "index.d.ts",
@@ -35,9 +35,9 @@
     "koa-body": "^5.0.0",
     "koa-send": "^5.0.1",
     "koa2-ratelimit": "^1.1.3",
-    "maxmind": "^4.3.20",
-    "tar": "^6.2.1",
-    "ua-parser-js": "^1.0.38"
+    "mmdb-lib": "^3.0.1",
+    "tar": "^7.4.3",
+    "ua-parser-js": "^1.0.40"
   },
   "devDependencies": {
     "eslint": "^5.7.0",