ac-geoip 1.3.0 → 2.0.0

package/.acsemver.js

@@ -1,6 +1,6 @@
 module.exports = {
   repository: {
-    url: 'https://github.com/mmpro/ac-geoip'
+    url: 'https://github.com/admiralcloud/ac-geoip'
   },
   changelogFile: __dirname + '/CHANGELOG.md',
   sections: [

package/CHANGELOG.md

@@ -1,3 +1,68 @@
+<a name="2.0.0"></a>
+ 
+# [2.0.0](https://github.com/admiralcloud/ac-geoip/compare/v1.4.2..v2.0.0) (2022-01-29 11:56:17)
+
+
+### Bug Fix
+
+* **App:** Open geolite DB during initialisation | VD | [37b6981913f08798c1b764021d528497f8cc238f](https://github.com/admiralcloud/ac-geoip/commit/37b6981913f08798c1b764021d528497f8cc238f)    
+Open geolite DB during initialisation  
+Related issues: [undefined/undefined#AC-2500](undefined/browse/AC-2500)
+* **App:** Cache geoip2 Reader object | VD | [54a67bb2d186ccf10ec72e8b370ce9a51c2ab89c](https://github.com/admiralcloud/ac-geoip/commit/54a67bb2d186ccf10ec72e8b370ce9a51c2ab89c)    
+Store object to re-use instead of opening DB file each time it's needed.  
+Related issues: [undefined/undefined#AC-2500](undefined/browse/AC-2500)
+* **App:** Remove callbacks from async functions | VD | [6c4a6ecd0ce8f8350eb206b41be0d90ddaf09ee5](https://github.com/admiralcloud/ac-geoip/commit/6c4a6ecd0ce8f8350eb206b41be0d90ddaf09ee5)    
+Mixing async and callback functions makes code read hard, misunderstand the results and make function behave unpredictably.  
+Related issues: [undefined/undefined#AC-2500](undefined/browse/AC-2500)
+### Style
+
+* **App:** Get rid of semicollons, code re-aligning | VD | [e27005f2001a0da06d12fe272a3e007937f1604b](https://github.com/admiralcloud/ac-geoip/commit/e27005f2001a0da06d12fe272a3e007937f1604b)    
+Get rid of semicollons, code re-aligning
+### Chores
+
+* **App:** Updated packages | MP | [49f68ab1575163f6b9e9a845a23cd779195d9a41](https://github.com/admiralcloud/ac-geoip/commit/49f68ab1575163f6b9e9a845a23cd779195d9a41)    
+Updated packages
+## BREAKING CHANGES
+* **App:** * lookupLocal, lookup functions fully async and doesn't have callback parameters
+<a name="1.4.2"></a>
+
+## [1.4.2](https://github.com/admiralcloud/ac-geoip/compare/v1.4.1..v1.4.2) (2021-10-09 10:25:40)
+
+
+### Bug Fix
+
+* **App:** Package updates | MP | [08dd237ecc7274cfb083f3f6279036fa94d17398](https://github.com/admiralcloud/ac-geoip/commit/08dd237ecc7274cfb083f3f6279036fa94d17398)    
+Package updates
+<a name="1.4.1"></a>
+
+## [1.4.1](https://github.com/mmpro/ac-geoip/compare/v1.4.0..v1.4.1) (2021-05-02 09:42:55)
+
+
+### Bug Fix
+
+* **App:** Improved debug log | MP | [e5985e0a10d08979ab2f1ddadcc51217f43e0342](https://github.com/mmpro/ac-geoip/commit/e5985e0a10d08979ab2f1ddadcc51217f43e0342)    
+Improved debug log
+### Tests
+
+* **App:** Minor fix | MP | [42f5ffec32d0fa9d54306ddc1281feef7ef6e3df](https://github.com/mmpro/ac-geoip/commit/42f5ffec32d0fa9d54306ddc1281feef7ef6e3df)    
+Minor fix
+### Chores
+
+* **App:** Updated packages | MP | [39b1e855d0d8b7419868f84038163f142f5a4b26](https://github.com/mmpro/ac-geoip/commit/39b1e855d0d8b7419868f84038163f142f5a4b26)    
+Updated packages
+<a name="1.4.0"></a>
+ 
+# [1.4.0](https://github.com/mmpro/ac-geoip/compare/v1.3.0..v1.4.0) (2021-04-12 09:36:41)
+
+
+### Feature
+
+* **App:** Use node-cache per default | MP | [1a5481a30245fc53ccb6333a60f4ae38a8245eb5](https://github.com/mmpro/ac-geoip/commit/1a5481a30245fc53ccb6333a60f4ae38a8245eb5)    
+If Redis is not defined, use node-cache (memory) to improve performance.
+### Bug Fix
+
+* **App:** Allow Redis usage when using local database | MP | [943ab0cf914245cceab0c23da1e03078783a1b66](https://github.com/mmpro/ac-geoip/commit/943ab0cf914245cceab0c23da1e03078783a1b66)    
+To improve performance it makes sense to use Redis even for local database. Lookup is a little slow and consumes a lot of CPU
 <a name="1.3.0"></a>
  
 # [1.3.0](https://github.com/mmpro/ac-geoip/compare/v1.2.1..v1.3.0) (2021-04-02 09:52:27)

package/index.js

@@ -1,17 +1,23 @@
 const _ = require('lodash')
 const ipPackage = require('ip')
+const fs = require('fs')
 
-const WebServiceClient = require('@maxmind/geoip2-node').WebServiceClient;
-const Reader = require('@maxmind/geoip2-node').Reader;
+const WebServiceClient = require('@maxmind/geoip2-node').WebServiceClient
+const Reader = require('@maxmind/geoip2-node').Reader
+
+const NodeCache = require("node-cache")
+const geoCache = new NodeCache({ stdTTL: 7 * 86400, checkperiod: 3600 })
 
 const acgeoip = () => {
-  
+
   let geoip = {
     userId: 'userId',
     licenseKey: 'licenseKey',
     environment: 'development',
     // redis, // instance of redis
+    // reader // initiated if useBuffer with local database
     geolite: {
+      useBuffer: false,
       enabled: false,
       path: '/path/to/GeoLite2-City.mmdb'
     },
@@ -23,6 +29,8 @@ const acgeoip = () => {
       { response: 'isp', geoIP: 'traits.isp' },
       { response: 'organization', geoIP: 'traits.organization' },
       { response: 'domain', geoIP: 'traits.domain' },
+      { response: 'latitude', geoIP: 'location.latitude' },
+      { response: 'longitude', geoIP: 'location.longitude' }
     ]
   }
 
@@ -32,46 +40,90 @@ const acgeoip = () => {
     if (_.has(params, 'env')) _.set(geoip, 'environment', _.get(params, 'env'))
     if (_.has(params, 'redis')) _.set(geoip, 'redis', _.get(params, 'redis'))
     if (_.has(params, 'geolite')) _.set(geoip, 'geolite', _.get(params, 'geolite'))
-  }
-
 
+    if (_.get(params, 'geolite.enabled') && _.get(params, 'geolite.useBuffer')) {
+      const dbBuffer = fs.readFileSync(_.get(geoip, 'geolite.path'))
+      geoip.reader = Reader.openBuffer(dbBuffer)
+    }
+    else {
+      Reader.open(_.get(geoip, 'geolite.path')).then(geoipReader => {
+        geoip.geolite.reader = geoipReader
+      })
+    }
+  }
 
-  const lookupLocal = async(params, cb) => {
+  const lookupLocal = async (params) => {
+    const functionName = 'ac-geoip | lookupLocal'
     if (!_.get(geoip, 'geolite.enabled')) {
-      let message = 'acgeoip_geolite_notEnabled'
-      if (_.isFunction(cb)) return cb({ message })
+      const message = 'acgeoip_geolite_notEnabled'
       throw Error(message)
     }
+
     const ip = _.get(params, 'ip')
-    if (ipPackage.isPrivate(ip)) {
-      if (_.isFunction(cb)) return cb()
-      return
-    }
+    if (ipPackage.isPrivate(ip)) return
 
     const mapping = _.get(params, 'mapping', geoip.mapping)
     const debug = _.get(params, 'debug')
+    const debugPerformance = _.get(params, 'debugPerforance')
+    const start = process.hrtime()
 
     let response = {
       ip
     }
     let geoipResponse
 
-    try {
-      if (_.get(geoip, 'geolite.enabled')) {
-        geoipResponse = await new Promise((resolve, reject) => {
-            Reader.open(_.get(geoip, 'geolite.path')).then(reader => {
-            const response = reader.city(ip)
-            resolve(response)
-          }).catch(reject)
-        })
+    if (geoip.redis) {
+      try {
+        geoipResponse = await checkRedis(params)
       }
-
-      if (debug) {
-        console.log('AC-GEOIP | From Geolite | %s', JSON.stringify(geoipResponse, null, 2))
+      catch (err) {
+        console.error('AC-GEOIP | From Geolite | Failed | %j', err)
       }
     }
-    catch(e) {
-      console.error('AC-GEOIP | From Geolite | Failed | %j', e)
+    else {
+      geoipResponse = getFromMemory({ ip })
+    }
+
+    if (debugPerformance) console.log('%s | getFromCache %d', functionName, performanceHelper(start, process.hrtime()))
+
+    if (!geoipResponse) {
+      if (_.get(geoip, 'geolite.useBuffer') && geoip.reader) {
+        geoipResponse = geoip.reader.city(ip)
+        if (debugPerformance) console.log('%s | readFromBuffer %d', functionName, performanceHelper(start, process.hrtime()))
+      }
+      else {
+        try {
+          if (_.get(geoip, 'geolite.enabled')) {
+            let geoipReader = _.get(geoip, 'geolite.reader')
+            if (!geoipReader) {
+              geoipReader = await Reader.open(_.get(geoip, 'geolite.path'))
+              _.set(geoip, 'geolite.reader', geoipReader)
+            }
+            if (geoipReader) geoipResponse = geoipReader.city(ip)
+          }
+
+          if (debugPerformance) console.log('%s | readFromDB %d', functionName, performanceHelper(start, process.hrtime()))
+
+          if (debug) {
+            console.log('AC-GEOIP | From Geolite | %j', geoipResponse)
+          }
+        }
+        catch (e) {
+          console.error('AC-GEOIP | From Geolite | Failed | %j', e)
+        }
+      }
+
+      if (geoipResponse) {
+        _.set(geoipResponse, 'origin', 'db')
+        if (geoip.redis) {
+          await storeRedis({ ip, geoipResponse })
+        }
+        else {
+          storeInMemory({ ip, geoipResponse })
+        }
+
+        if (debugPerformance) console.log('%s | storeInCache %d', functionName, performanceHelper(start, process.hrtime()))
+      }
     }
 
     // prepare response
@@ -83,72 +135,73 @@ const acgeoip = () => {
     else {
       response = geoipResponse
     }
-    if (_.isFunction(cb)) return cb(null, response)
+
+    _.set(response, 'origin', _.get(geoipResponse, 'origin'))
+    if (_.get(geoipResponse, 'fromCache')) _.set(response, 'fromCache', true)
+
+    if (debugPerformance) console.log('%s | Finished %d', functionName, performanceHelper(start, process.hrtime()))
+
     return response
   }
 
-
-  const lookup = async(params, cb) => {
+  const lookup = async (params) => {
+    const functionName = 'ac-geoip | lookup'
     if (!_.get(geoip, 'licenseKey') || _.get(geoip, 'licenseKey') === 'licenseKey') {
-      let message = 'acgeoip_licenseKey_missing'
-      if (_.isFunction(cb)) return cb({ message })
+      const message = 'acgeoip_licenseKey_missing'
       throw Error(message)
     }
     const ip = _.get(params, 'ip')
     if (ipPackage.isPrivate(ip)) {
-      if (_.isFunction(cb)) return cb()
       return
     }
 
-    const refresh = _.get(params, 'refresh')
-    const redisKey = _.get(geoip, 'environment') + ':geoip:' + ip
     const mapping = _.get(params, 'mapping', geoip.mapping)
     const debug = _.get(params, 'debug')
+    const debugPerformance = _.get(params, 'debugPerforance')
+    const start = process.hrtime()
 
     let response = {
       ip
     }
     let geoipResponse
 
-    // try redis
-    if (geoip.redis && !refresh) {
-      try {
-        geoipResponse = await geoip.redis.get(redisKey)
-        geoipResponse = JSON.parse(geoipResponse)
-        if (_.isPlainObject(geoipResponse)) {
-          geoipResponse.fromCache = true
-        }
-        if (debug) {
-          console.log('AC-GEOIP | From Cache | %j', JSON.stringify(geoipResponse, null, 2))
-        }
-      }
-      catch(e) {
-        console.error('AC-GEOIP | From Cache | Failed | %j', e)
-      }
+    if (geoip.redis) {
+      geoipResponse = await checkRedis(params)
     }
+    else {
+      geoipResponse = getFromMemory({ ip })
+    }
+    if (debugPerformance) console.log('%s | getFromCache %d', functionName, performanceHelper(start, process.hrtime()))
 
     // fetch fresh
-    if (refresh || !_.get(geoipResponse, 'country')) {
+    if (!_.get(geoipResponse, 'country')) {
       try {
         const client = new WebServiceClient(geoip.userId, geoip.licenseKey)
-        geoipResponse = await new Promise((resolve, reject) => {
-            client.city(ip).then(result => {
-              return resolve(result)
-          }).catch(reject)
-        })
+        geoipResponse = await client.city(ip)
+        if (debugPerformance) console.log('%s | readFromWebservice %d', functionName, performanceHelper(start, process.hrtime()))
+        if (geoipResponse) {
+          _.set(geoipResponse, 'origin', 'webservice')
+        }
 
         if (debug) {
-          console.log('AC-GEOIP | From Maxmind | %s', JSON.stringify(geoipResponse, null, 2))
+          console.log('AC-GEOIP | From Maxmind | %j', geoipResponse)
         }
-        if (geoip.redis) {
-          await geoip.redis.setex(redisKey, geoip.cacheTime, JSON.stringify(geoipResponse))
-        }      
       }
-      catch(e) {
+      catch (e) {
         console.error('AC-GEOIP | From Maxmind | Failed | %j', e)
       }
     }
 
+    if (geoipResponse) {
+      if (geoip.redis) {
+        await storeRedis({ ip, geoipResponse })
+      }
+      else {
+        storeInMemory({ ip, geoipResponse })
+      }
+      if (debugPerformance) console.log('%s | storeInCache %d', functionName, performanceHelper(start, process.hrtime()))
+    }
+
     // prepare response
     if (!_.isEmpty(mapping)) {
       _.forEach(mapping, item => {
@@ -158,12 +211,70 @@ const acgeoip = () => {
     else {
       response = geoipResponse
     }
+
+    _.set(response, 'origin', _.get(geoipResponse, 'origin'))
     if (_.get(geoipResponse, 'fromCache')) _.set(response, 'fromCache', true)
 
-    if (_.isFunction(cb)) return cb(null, response)
     return response
   }
 
+  const storeInMemory = (params) => {
+    const geoipResponse = _.get(params, 'geoipResponse')
+    const ip = _.get(params, 'ip')
+    const storageKey = _.get(geoip, 'environment') + ':geoip:' + ip
+    geoCache.set(storageKey, geoipResponse)
+  }
+
+  const getFromMemory = (params) => {
+    const ip = _.get(params, 'ip')
+    const storageKey = _.get(geoip, 'environment') + ':geoip:' + ip
+    return geoCache.get(storageKey)
+  }
+
+  const checkRedis = async (params) => {
+    const refresh = _.get(params, 'refresh')
+    if (!geoip.redis || refresh) return
+
+    const ip = _.get(params, 'ip')
+    const redisKey = _.get(geoip, 'environment') + ':geoip:' + ip
+    const debug = _.get(params, 'debug')
+
+    let geoipResponse
+    try {
+      geoipResponse = await geoip.redis.get(redisKey)
+      geoipResponse = JSON.parse(geoipResponse)
+      if (_.isPlainObject(geoipResponse)) {
+        geoipResponse.fromCache = true
+      }
+      if (debug) {
+        console.log('AC-GEOIP | From Cache | %j', geoipResponse)
+      }
+    }
+    catch (e) {
+      console.log(e)
+      console.error('AC-GEOIP | From Cache | Failed | %j', e)
+    }
+    return geoipResponse
+  }
+
+  const storeRedis = async (params) => {
+    const refresh = _.get(params, 'refresh')
+    if (!geoip.redis || refresh) {
+      return
+    }
+    const ip = _.get(params, 'ip')
+    const geoipResponse = _.get(params, 'geoipResponse')
+    const redisKey = _.get(geoip, 'environment') + ':geoip:' + ip
+
+    await geoip.redis.setex(redisKey, geoip.cacheTime, JSON.stringify(geoipResponse))
+  }
+
+  const performanceHelper = (t1, t2, options) => {
+    const accuracy = _.get(options, 'accuracy', 1e6)
+    const s = t2[0] - t1[0]
+    const mms = t2[1] - t1[1]
+    return (s * 1e9 + mms) / accuracy
+  }
 
   return {
     init,

package/package.json

@@ -1,20 +1,21 @@
 {
   "name": "ac-geoip",
-  "author": "Mark Poepping (https://www.mmpro.de)",
+  "author": "Mark Poepping (https://www.admiralcloud.com)",
   "license": "MIT",
-  "repository": "mmpro/ac-geoip",
-  "version": "1.3.0",
+  "repository": "admiralcloud/ac-geoip",
+  "version": "2.0.0",
   "dependencies": {
-    "@maxmind/geoip2-node": "^2.3.1",
+    "@maxmind/geoip2-node": "^3.4.0",
     "ip": "^1.1.5",
-    "lodash": "^4.17.21"
+    "lodash": "^4.17.21",
+    "node-cache": "^5.1.2"
   },
   "devDependencies": {
-    "ac-semantic-release": "^0.2.5",
-    "chai": "^4.3.4",
-    "eslint": "^7.22.0",
-    "ioredis": "^4.24.2",
-    "mocha": "^8.3.2"
+    "ac-semantic-release": "^0.2.7",
+    "chai": "^4.3.6",
+    "eslint": "^8.8.0",
+    "ioredis": "^4.28.3",
+    "mocha": "^9.2.0"
   },
   "scripts": {
     "test": "./node_modules/.bin/mocha --bail --exit --slow 1000 ./test/test.js || :"

package/test/test.js

@@ -23,6 +23,7 @@ const expectedValue = {
 }
 
 
+
 describe('Test Webservice', () => {
 
   it('Webservice is not yet enabled - should fail', async function() {
@@ -30,7 +31,7 @@ describe('Test Webservice', () => {
     try {
       await acgeoip.lookup({ ip, debug: false })    
     }
- catch (e) {
+    catch (e) {
       expect(e).to.be.instanceOf(Error)
       expect(e.message).to.eql('acgeoip_licenseKey_missing')
     }
@@ -46,24 +47,9 @@ describe('Test Webservice', () => {
     return done()
   })
 
-  it('Shoud be tested with callback', function(done) {
-    this.timeout(5000)
-    acgeoip.lookup({
-      ip,
-      debug: false
-    }, (err, result) => {
-      if (err) return done(err)
-      //console.log(51, result)
-      _.forOwn(expectedValue, (val, key) => {
-        expect(result).to.have.property(key, val)
-      })
-      return done()
-    })
-  })
-
   it('Shoud be tested with async/await', async function() {
     this.timeout(5000)
-    const result = await acgeoip.lookup({ ip, debug: false })     
+    const result = await acgeoip.lookup({ ip, debug: false, refresh: true })     
     _.forOwn(expectedValue, (val, key) => {
       expect(result).to.have.property(key, val)
     })
@@ -101,7 +87,7 @@ describe('Test Geolite2 local database', () => {
 
   it('Init geolite', done => {
     const geoip = {
-      redis: undefined,
+      redis,
       userId: undefined,
       licenseKey: undefined,
       geolite: {
@@ -121,13 +107,27 @@ describe('Test Geolite2 local database', () => {
 
   it('Shoud be tested with async/await', async function() {
       this.timeout(5000)
-      const result = await acgeoip.lookupLocal({ ip, debug: false })    
+      const result = await acgeoip.lookupLocal({ ip, debug: false, refresh: true })    
       let fields = ['iso2', 'city', 'region']
       _.forEach(fields, key => {
         let val = _.get(expectedValue, key)
         expect(result).to.have.property(key, val)
       })
+      expect(result).to.have.property('origin', 'db')
       return 
   })
+
+  it('Test again - should be from cache', async function() {
+    this.timeout(5000)
+    const result = await acgeoip.lookupLocal({ ip, debug: false })    
+    let fields = ['iso2', 'city', 'region']
+    _.forEach(fields, key => {
+      let val = _.get(expectedValue, key)
+      expect(result).to.have.property(key, val)
+    })
+    expect(result).to.have.property('origin', 'db')
+    expect(result).to.have.property('fromCache', true)
+    return 
+  })
 })