@streamr/geoip-location 100.2.4

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@streamr/geoip-location",
+  "version": "100.2.4",
+  "description": "Library for getting location information from IP addresses based on MaxMind GeoLite2 databases",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/streamr-dev/network.git",
+    "directory": "packages/geoip-location"
+  },
+  "main": "dist/src/exports.js",
+  "browser": {
+    "dist/src/exports.js": false,
+    "dist/src/GeoIpLocator.js": false,
+    "dist/src/downloadGeoIpDatabase.js": false
+  },
+  "types": "dist/src/exports.d.ts",
+  "license": "Apache-2.0",
+  "author": "Streamr Network AG <contact@streamr.network>",
+  "scripts": {
+    "build": "tsc -b tsconfig.node.json",
+    "check": "tsc -p ./tsconfig.jest.json --noEmit",
+    "clean": "jest --clearCache || true; rm -rf dist *.tsbuildinfo node_modules/.cache || true",
+    "eslint": "eslint --cache --cache-location=node_modules/.cache/.eslintcache/ '*/**/*.{js,ts}'",
+    "test": "jest test/unit",
+    "test-unit": "jest test/unit"
+  },
+  "dependencies": {
+    "@streamr/utils": "100.2.4",
+    "long-timeout": "^0.1.1",
+    "mmdb-lib": "^2.1.0",
+    "tar": "^6.2.1",
+    "uuid": "^9.0.1"
+  },
+  "devDependencies": {
+    "@types/long-timeout": "^0.1.2",
+    "@types/tar": "^6.1.11",
+    "express": "^4.19.2"
+  }
+}

package/src/GeoIpLocator.ts

@@ -0,0 +1,109 @@
+import { Logger, filePathToNodeFormat } from '@streamr/utils'
+import { CityResponse, Reader } from 'mmdb-lib'
+import LongTimeout from 'long-timeout'
+import { downloadGeoIpDatabase } from './downloadGeoIpDatabase'
+
+const logger = new Logger(module)
+
+interface GeoIpLookupResult {
+    latitude: number
+    longitude: number
+}
+
+// 30 days in milliseconds
+const DEFAULT_DB_CHECK_INTERVAL = 30 * 24 * 60 * 60 * 1000
+// 24 hours in milliseconds
+const DEFAULT_DB_CHECK_ERROR_INTERVAL = 24 * 60 * 60 * 1000
+
+export class GeoIpLocator { 
+    private abortController: AbortController
+    private readonly geoIpDatabaseFolder: string
+    private readonly dbCheckInterval: number
+    private readonly dbCheckErrorInterval: number
+    private readonly mirrorUrl?: string
+    private reader?: Reader<CityResponse>
+    private dbCheckTimeout?: LongTimeout.Timeout
+
+    constructor(
+        geoIpDatabaseFolder: string,
+        dbCheckInterval = DEFAULT_DB_CHECK_INTERVAL,
+        dbCheckErrorInterval = DEFAULT_DB_CHECK_ERROR_INTERVAL,
+        mirrorUrl?: string
+    ) {
+        this.abortController = new AbortController()
+        this.dbCheckInterval = dbCheckInterval
+        this.dbCheckErrorInterval = dbCheckErrorInterval
+        if (!geoIpDatabaseFolder.endsWith('/')) {
+            geoIpDatabaseFolder += '/'
+        }
+        this.geoIpDatabaseFolder = filePathToNodeFormat(geoIpDatabaseFolder)
+        this.mirrorUrl = mirrorUrl
+    }
+
+    private checkDatabase: () => Promise<void> = async () => {
+        if (this.reader === undefined) {
+            // if we do not have a reader, create a new one in any case
+            this.reader = await downloadGeoIpDatabase(this.geoIpDatabaseFolder, true, this.abortController.signal, this.mirrorUrl)
+        } else {
+            // if we already have a reader, create a new one only if db has changed
+            const newReader = await downloadGeoIpDatabase(this.geoIpDatabaseFolder, false, this.abortController.signal, this.mirrorUrl)
+            if (newReader !== undefined) {
+                this.reader = newReader
+            }
+        }
+    }
+
+    private scheduleCheck: (timeout: number) => void = async (timeout: number) => {
+        if (this.abortController.signal.aborted) {
+            return
+        }
+        this.dbCheckTimeout = LongTimeout.setTimeout(async () => {
+            try {
+                await this.checkDatabase()
+                this.scheduleCheck(this.dbCheckInterval)
+            } catch (err) {
+                logger.warn('GeoIpLocator: GeoIP database check failed', { err })
+                this.scheduleCheck(this.dbCheckErrorInterval)
+            } 
+        }, timeout)
+    }
+
+    async start(): Promise<void> {
+        if (this.dbCheckTimeout !== undefined) {
+            return
+        }
+        await this.checkDatabase()
+        this.scheduleCheck(this.dbCheckInterval)
+    }
+
+    stop(): void {
+        if (this.dbCheckTimeout !== undefined) {
+            LongTimeout.clearTimeout(this.dbCheckTimeout)
+        }
+        this.abortController.abort()
+    }
+
+    lookup(ip: string): GeoIpLookupResult | undefined {
+        if (this.reader === undefined) {
+            logger.warn('GeoIpLocator: lookup called before database is ready (maybe start() was not called?')
+            return undefined
+        }
+
+        // If ip is falsy, the library will crash
+        // this might happen despite the ts typings because the ip address 
+        // comes from the ws server socket and is not under our control
+        if (!ip) {
+            return undefined
+        }
+
+        const result = this.reader.get(ip)
+        if (!result || !result.location || !result.location.latitude || !result.location.longitude) {
+            return undefined
+        } else {
+            return {
+                latitude: result.location.latitude,
+                longitude: result.location.longitude
+            }
+        }
+    }
+}

package/src/downloadGeoIpDatabase.ts

@@ -0,0 +1,185 @@
+import crypto from 'crypto'
+import fs from 'fs'
+import { CityResponse, Reader } from 'mmdb-lib'
+import { extractFileFromTarStream } from './tarHelper'
+import { v4 } from 'uuid'
+import { Logger } from '@streamr/utils'
+
+const GEOIP_MIRROR_URL = 'https://raw.githubusercontent.com/GitSquared/node-geolite2-redist/master/redist/'
+const DB_NAME = 'GeoLite2-City'
+const TAR_SUFFFIX = '.tar.gz'
+const DB_SUFFIX = '.mmdb'
+const HASH_SUFFIX = '.mmdb.sha384'
+
+const logger = new Logger(module)
+
+const downloadNewDb = async (
+    url: string,
+    dbFolder: string,
+    remoteHash: string,
+    abortSignal: AbortSignal
+): Promise<void> => {
+    // make a unique name for the temporary download folder
+    // in case there are multiple downloads happening at the same time
+
+    const uniqueName = v4()
+    const downloadFolder = dbFolder + '.download' + uniqueName
+    const dbFileName = DB_NAME + DB_SUFFIX
+    const dbFileInDownloadFolder = downloadFolder + '/' + dbFileName
+    const dbFileInDbFolder = dbFolder + dbFileName
+
+    let response: Response
+
+    try {
+        logger.debug('Downloading GeoIP database from: ' + url)
+        response = await fetch(url, { keepalive: false, signal: abortSignal })
+    } catch (e) {
+        // Catching and re-throwing as async exception 
+        // here is necessary, synch exceptions cannot be caught by the caller
+        throw new Error('Fetch error when downloading ' + url + ', error: ' + e)
+    }
+
+    if (!response.ok) {
+        throw new Error('HTTP error when downloading ' + url + ', status: ' + response.status)
+    }
+
+    // extract the tarball to a temporary folder
+
+    try {
+        fs.mkdirSync(downloadFolder, { recursive: true })
+    } catch (e) {
+        throw new Error('Error creating temporary folder ' + downloadFolder + ', error: ' + e)
+    }
+
+    try {
+        await extractFileFromTarStream(dbFileName, response.body!, downloadFolder)
+    } catch (e) {
+        try {
+            fs.rmSync(downloadFolder, { recursive: true })
+        } catch (e2) {
+            // ignore error when removing the temporary folder
+        }
+        throw e
+    }
+
+    // check the hash of the extracted file
+
+    if (!isDbFileValid(dbFileInDownloadFolder, remoteHash)) {
+        try {
+            fs.rmSync(downloadFolder, { recursive: true })
+        } catch (e2) {
+            // ignore error when removing the temporary folder
+        }
+        throw new Error('Downloaded database hash does not match the expected hash')
+    }
+
+    try {
+        // move the extracted file to the correct location
+        fs.renameSync(dbFileInDownloadFolder, dbFileInDbFolder)
+    } catch (e) {
+        throw new Error('Error moving ' + dbFileInDownloadFolder + ' to ' + dbFileInDbFolder + ', error: ' + e)
+    } finally {
+        try {
+            fs.rmSync(downloadFolder, { recursive: true })
+        } catch (e2) {
+            // ignore error when removing the temporary folder
+        }
+    }
+
+    // set the db file permissions to rw only for the owner
+
+    try {
+        fs.chmodSync(dbFileInDbFolder, 0o600)
+    } catch (err) {
+        throw new Error('Error setting permissions on ' + dbFileInDbFolder + ', error: ' + err)
+    }
+
+    logger.debug('Downloaded GeoIP database to: ' + dbFileInDbFolder)
+
+}
+
+const downloadRemoteHash = async (remoteHashUrl: string, abortSignal: AbortSignal): Promise<string> => {
+    // download the hash of the latest GeoIP database using fetch as text and trim it
+    let response: Response
+
+    try {
+        logger.debug('Downloading GeoIP database hash from: ' + remoteHashUrl)
+        response = await fetch(remoteHashUrl, { signal: abortSignal })
+    } catch (e) {
+        // Catching and re-throwing as async exception 
+        // here is necessary, synch exceptions cannot be caught by the caller
+        throw new Error('Fetch error when downloading ' + remoteHashUrl + ', error: ' + e)
+    }
+
+    if (!response.ok) {
+        throw new Error('HTTP error when downloading ' + remoteHashUrl + ', status: ' + response.status)
+    }
+
+    return (await response.text()).trim()
+}
+
+const isDbFileValid = async (dbFile: string, remoteHash: string): Promise<boolean> => {
+    // check if the local db exists and calculate its hash
+
+    try {
+        const db = fs.readFileSync(dbFile)
+        const localHash = crypto.createHash('sha384').update(db).digest('hex')
+
+        // if the hashes are different, download the latest database
+        if (localHash !== remoteHash) {
+            return false
+        } else {
+            return true
+        }
+    } catch (e) {
+        // if the local db does not exist, or some other exception occurres db is not considered valid
+        return false
+    }
+}
+
+// returns a Reader if a new db was downloaded, or if the caller wants to force return a reader
+// also if there was no need to download a new db
+
+export const downloadGeoIpDatabase = async (
+    dbFolder: string,
+    forceReturnReader: boolean,
+    abortSignal: AbortSignal,
+    mirrorUrl?: string
+): Promise<Reader<CityResponse> | undefined> => {
+    // This will throw if the download folder is not readable
+    if (!fs.existsSync(dbFolder)) {
+        // This will throw if the download folder is not writable
+        fs.mkdirSync(dbFolder, { recursive: true })
+    }
+    if (!dbFolder.endsWith('/')) {
+        dbFolder += '/'
+    }
+    let geoIpMirrorUrl = GEOIP_MIRROR_URL
+    if (mirrorUrl !== undefined) {
+        if (!mirrorUrl.endsWith('/')) {
+            mirrorUrl += '/'
+        }
+        geoIpMirrorUrl = mirrorUrl
+    }
+    const remoteHashUrl = geoIpMirrorUrl + DB_NAME + HASH_SUFFIX
+    const dbDownloadUrl = geoIpMirrorUrl + DB_NAME + TAR_SUFFFIX
+    const dbFileInDbFolder = dbFolder + DB_NAME + DB_SUFFIX
+
+    const remoteHash = await downloadRemoteHash(remoteHashUrl, abortSignal)
+    const dbValid = await isDbFileValid(dbFileInDbFolder, remoteHash)
+    if (dbValid === false) {
+        await downloadNewDb(dbDownloadUrl, dbFolder, remoteHash, abortSignal)
+        // return new reader if db was downloaded
+        return new Reader<CityResponse>(fs.readFileSync(dbFileInDbFolder))
+    } else {
+        logger.debug('The hash of the local GeoIP database matches the remote hash, no need to download a new database')
+    }
+    if (forceReturnReader) {
+        // return reader also for old db the caller wants it
+        return new Reader<CityResponse>(fs.readFileSync(dbFileInDbFolder))
+    } else {
+        // return undefined if the db is already up to date
+        return undefined
+    }
+}
+

package/src/exports.ts

@@ -0,0 +1 @@
+export { GeoIpLocator } from './GeoIpLocator'

package/src/tarHelper.ts

@@ -0,0 +1,35 @@
+import { Readable, pipeline } from 'stream'
+import tar from 'tar'
+import { ReadableStream } from 'stream/web'
+import NodePath from 'path'     // use NodePath to avoid conflict with other 'path' symbols
+import fs from 'fs'
+
+const doExtractFileFromTarStream = (fileName: string, stream: ReadableStream<any>, downloadFolder: string): Promise<void> => {
+    return new Promise((resolve, reject) => {
+        try {
+            const nodeStream = Readable.fromWeb(stream)
+            pipeline(nodeStream,
+                tar.x({
+                    cwd: downloadFolder,
+                    filter: (entryPath: string): boolean => NodePath.basename(entryPath) === fileName,
+                    strip: 1
+                }), (err) => {
+                    if (err) {
+                        reject(new Error('Error extracting tarball to ' + downloadFolder + ', error: ' + err))
+                    } else {
+                        resolve()
+                    }
+                })
+        } catch (e) {
+            reject(new Error('Failed to create nodejs Readable from web stream: ' + e))
+        }
+    })
+}
+
+export const extractFileFromTarStream = async (fileName: string, stream: ReadableStream<any>, downloadFolder: string): Promise<void> => {
+    await doExtractFileFromTarStream(fileName, stream, downloadFolder)
+    if (!fs.existsSync(NodePath.join(downloadFolder, fileName))) {
+        throw new Error('File not found in tarball: ' + fileName)
+    }
+}
+

package/test/helpers/TestServer.ts

@@ -0,0 +1,176 @@
+import express from 'express'
+import http from 'http'
+import { Logger, wait } from '@streamr/utils'
+import { fetchFileToMemory } from './fetchFileToMemory'
+import fs from 'fs'
+import { v4 } from 'uuid'
+import EventEmitter from 'eventemitter3'
+import { Duplex, pipeline } from 'stream'
+
+const logger = new Logger(module)
+
+type ExpressType = ReturnType<typeof express>
+type ServerType = ReturnType<ExpressType['listen']>
+
+const dbUrl = 'https://raw.githubusercontent.com/GitSquared/node-geolite2-redist/master/redist/GeoLite2-City.tar.gz'
+const hashUrl = 'https://raw.githubusercontent.com/GitSquared/node-geolite2-redist/master/redist/GeoLite2-City.mmdb.sha384'
+
+const dbFileName = '/GeoLite2-City.tar.gz'
+const hashFileName = '/GeoLite2-City.mmdb.sha384'
+
+const CACHE_PATH = '/tmp/geoip-location-test-cache'
+
+export interface TestServerEvents {
+    closed: () => void
+}
+
+function bufferToStream(buf: Buffer) {
+    const tmp = new Duplex()
+    tmp.push(buf)
+    tmp.push(null)
+    return tmp
+}
+
+export class TestServer extends EventEmitter<TestServerEvents> {
+    private server?: ServerType
+    private abortController?: AbortController
+
+    private static hashData?: Uint8Array
+    private static dbData?: Uint8Array
+
+    private static async prefetchData(): Promise<void> {
+
+        TestServer.hashData = await fetchFileToMemory(hashUrl)
+
+        // check if db data is already prefetched to CACHE_PATH
+
+        if (fs.existsSync(CACHE_PATH + hashFileName) && fs.existsSync(CACHE_PATH + dbFileName)) {
+            // read hash data from CACHE_PATH
+            const cachedHash = fs.readFileSync(CACHE_PATH + hashFileName)
+
+            if (cachedHash.equals(TestServer.hashData)) {
+                TestServer.dbData = fs.readFileSync(CACHE_PATH + dbFileName)
+                return
+            }
+        }
+
+        // eslint-disable-next-line require-atomic-updates
+        TestServer.dbData = await fetchFileToMemory(dbUrl)
+
+        // save db and hash data to CACHE_PATH
+        try {
+            fs.mkdirSync(CACHE_PATH, { recursive: true })
+        } catch (e) {
+            // ignore error when creating the cache folder
+        }
+        // ensure there is never an incomplete file in the fs
+        const uniqueName = v4()
+
+        fs.writeFileSync(CACHE_PATH + hashFileName + uniqueName, TestServer.hashData)
+        fs.renameSync(CACHE_PATH + hashFileName + uniqueName, CACHE_PATH + hashFileName)
+
+        fs.writeFileSync(CACHE_PATH + dbFileName + uniqueName, TestServer.dbData)
+        fs.renameSync(CACHE_PATH + dbFileName + uniqueName, CACHE_PATH + dbFileName)
+    }
+
+    private async writeDataKilobytesPerSecond(res: http.ServerResponse, data: Uint8Array, kilobytesPerSecond?: number): Promise<void> {
+        let delayMilliseconds = 1
+
+        if (kilobytesPerSecond) {
+            delayMilliseconds = 1000 / kilobytesPerSecond
+        }
+        const chuckSize = 1024
+        for (let i = 0; i < data.length && !this.abortController?.signal.aborted; i += chuckSize) {
+            let end = i + chuckSize
+            if (end > data.length) {
+                end = data.length
+            }
+
+            res.write(data.slice(i, end))
+
+            if (delayMilliseconds !== undefined) {
+                await wait(delayMilliseconds, this.abortController?.signal)
+            } else {
+                await wait(0, this.abortController?.signal)
+            }
+        }
+    }
+
+    startServer(port: number, kiloBytesPerSecond?: number): Promise<void> {
+        return new Promise((resolve, _reject) => {
+            const app = express()
+
+            app.get(dbFileName, (_req, res) => {
+                if (kiloBytesPerSecond !== undefined) {
+                    res.setHeader('Content-Type', 'application/gzip')
+                    this.writeDataKilobytesPerSecond(res, TestServer.dbData!,
+                        kiloBytesPerSecond).then(() => {
+                        res.end()
+                    }).catch((_err) => {
+                        res.end()
+                    })
+                } else {
+                    // send data without throttling from file
+                    const readable = bufferToStream(Buffer.from(TestServer.dbData!))
+                    pipeline(readable, res, (err) => {
+                        if (err) {
+                            logger.error('Error sending db file: ', { err })
+                        }
+                    })
+                }
+            })
+
+            app.get(hashFileName, (_req, res) => {
+                // always send hash data without throttling
+                const readable = bufferToStream(Buffer.from(TestServer.hashData!))
+                pipeline(readable, res, (err) => {
+                    if (err) {
+                        logger.error('Error sending hash file: ', { err })
+                    }
+                })
+            })
+
+            this.server = app.listen(port, '127.0.0.1', () => {
+                logger.info('Test server is running on port ' + port)
+                
+                // The server is not really ready after listen callback, possible bug in express
+                setTimeout(() => {
+                    resolve()
+                }, 1000)
+            })
+        })
+    }
+
+    async start(port: number, kiloBytesPerSecond?: number): Promise<void> {
+        if (!TestServer.hashData || !TestServer.dbData) {
+            await TestServer.prefetchData()
+        }
+
+        if (this.server) {
+            throw new Error('Test server already running')
+        }
+
+        this.abortController = new AbortController()
+        await this.startServer(port, kiloBytesPerSecond)
+    }
+
+    stop(): Promise<void> {
+        return new Promise((resolve, _reject) => {
+            if (this.server) {
+                this.abortController!.abort()
+
+                this.server.close((err) => {
+                    if (err) {
+                        logger.warn('Error closing server: ', { err })
+                    }
+                    this.server = undefined
+                    this.emit('closed')
+                    resolve()
+                })
+                this.server.closeAllConnections()
+            } else {
+                resolve()
+            }
+        })
+    }
+}

package/test/helpers/fetchFileToMemory.ts

@@ -0,0 +1,7 @@
+export const fetchFileToMemory = async (url: string): Promise<Uint8Array> => {
+    const response = await fetch(url)
+    if (!response.ok) {
+        throw new Error('HTTP error when downloading ' + url + ', status: ' + response.status)
+    }
+    return new Uint8Array(await response.arrayBuffer())
+}

package/test/unit/GeoIpLocator-intervals.test.ts

@@ -0,0 +1,107 @@
+import { GeoIpLocator } from '../../src/GeoIpLocator'
+import { wait, waitForCondition } from '@streamr/utils'
+import fs from 'fs'
+import { TestServer } from '../helpers/TestServer'
+
+describe('GeoIpLocator', () => {
+  
+    const serverPort = 31991
+    const mirrorUrl = 'http://127.0.0.1:' + serverPort + '/'
+
+    const DB_FILENAME = 'GeoLite2-City.mmdb'
+    let dirCounter = 0
+    const dbPath = '/tmp'
+    let dbDir: string
+    let locator: GeoIpLocator
+    let testServer: TestServer
+
+    const getDbDir = () => {
+        dirCounter++
+        return dbPath + '/geolitelocator2-intervals' + dirCounter
+    }
+
+    beforeAll(async () => {
+        testServer = new TestServer()
+        await testServer.start(serverPort)
+    }, 120000)
+    
+    afterEach(async () => {
+        locator!.stop()
+        testServer!.stop()
+        fs.rmSync(dbDir!, { recursive: true })
+    })
+
+    it('schedules a new check with a diffrent interval if monthly database check fails', async () => {
+        
+        dbDir = getDbDir()
+
+        try {
+            fs.unlinkSync(dbDir + '/' + DB_FILENAME)
+        } catch (e) {
+            // ignore
+        }
+
+        locator = new GeoIpLocator(dbDir, 3000, 1000, mirrorUrl)
+       
+        // start locator normally
+        await locator.start()
+
+        // delete the db
+        try {
+            fs.unlinkSync(dbDir + '/' + DB_FILENAME)
+        } catch (e) {
+            // ignore
+        }
+
+        // mock fetch to fail
+        const fetchMock = jest
+            .spyOn(globalThis, 'fetch')
+            .mockImplementation(async () => {
+                throw new Error('API is down')
+            })
+
+        // wait for the first check to happen
+        await wait(3500)
+
+        // normal check interval should have been run 
+        // after 3000ms, this should have tried
+        // downloading the hash, but failed
+
+        expect(fetchMock).toHaveBeenCalledTimes(1)
+
+        // wait for the failure interval to happen
+        await wait(1200)
+
+        // failure interval should have been run after 1500ms from the failure
+        // it should have tried downloading the hash again and
+        // failed
+
+        expect(fetchMock).toHaveBeenCalledTimes(2)
+
+        // restore fetch 
+        fetchMock.mockRestore()
+        
+        // mock fetch again to just count the calls
+        const fetchMock2 = jest
+            .spyOn(globalThis, 'fetch')
+            
+        // wait for failure interval to happen
+        await wait(1200)
+        
+        // failure interval should have downloaded
+        // both the hash and the db
+
+        expect(fetchMock2).toHaveBeenCalledTimes(2)
+        
+        // expect the db to be there
+        await waitForCondition(() => fs.existsSync(dbDir + '/' + DB_FILENAME), 10000)
+
+        // suomi.fi
+        const location = locator.lookup('62.241.198.245')
+        expect(location).toBeDefined()
+
+        // Helsinki, Finland
+        expect(location!.latitude).toBe(60.1797)
+        expect(location!.longitude).toBe(24.9344)
+    }, 60000)
+})