geoip-lite2 2.2.8-beta.1 → 2.2.8-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/city.checksum +1 -1
- package/data/country.checksum +1 -1
- package/data/geoip-city-names.dat +0 -0
- package/data/geoip-city.dat +0 -0
- package/data/geoip-city6.dat +0 -0
- package/data/geoip-country.dat +0 -0
- package/data/geoip-country6.dat +0 -0
- package/dist/main.js +1 -1
- package/dist/utils.js +1 -1
- package/index.d.ts +3 -3
- package/package.json +3 -3
- package/tools/updatedb.js +116 -59
package/data/city.checksum
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
f4a4c624512ef085feded9b74f5a797c1fedfae6c9a31d1a2c65462e23338d24 GeoLite2-City-CSV_20251107.zip
|
package/data/country.checksum
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
f6acd69d1aae734a4ccf1130d6133c99042c716ff814a4cc045b2109406c543b GeoLite2-Country-CSV_20251107.zip
|
|
Binary file
|
package/data/geoip-city.dat
CHANGED
|
Binary file
|
package/data/geoip-city6.dat
CHANGED
|
Binary file
|
package/data/geoip-country.dat
CHANGED
|
Binary file
|
package/data/geoip-country6.dat
CHANGED
|
Binary file
|
package/dist/main.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
const{open:e,fstat:
|
|
1
|
+
const{open:e,fstat:r,read:t,close:n,openSync:i,fstatSync:o,readSync:a,closeSync:l}=require('node:fs'),{join:f,resolve:u}=require('node:path'),{isIP:c}=require('node:net'),s=require('async'),{aton4:d,aton6:B,cmp6:y,ntoa4:I,ntoa6:m,cmp:E}=require('./utils.js'),S=require('./fsWatcher.js'),{version:g}=require('../package.json'),z='dataWatcher',p=u(__dirname,global.geoDataDir||process.env.GEODATADIR||'../data/'),P={city:f(p,'geoip-city.dat'),city6:f(p,'geoip-city6.dat'),cityNames:f(p,'geoip-city-names.dat'),country:f(p,'geoip-country.dat'),country6:f(p,'geoip-country6.dat')},h=[[d('10.0.0.0'),d('10.255.255.255')],[d('172.16.0.0'),d('172.31.255.255')],[d('192.168.0.0'),d('192.168.255.255')]],F={firstIP:null,lastIP:null,lastLine:0,locationBuffer:null,locationRecordSize:88,mainBuffer:null,recordSize:24},L={firstIP:null,lastIP:null,lastLine:0,mainBuffer:null,recordSize:48};let U={...F},D={...L};const N=e=>{const r=e.indexOf('\0');return-1===r?e:e.substring(0,r)},T=(e,r,t,n)=>{const i=[];for(let o=0;o<2;o++)i.push(e.readUInt32BE(r*t+16*n+4*o));return i},w=e=>{let r,t,n=0,i=U.lastLine,o=U.lastIP,a=U.firstIP;const l=U.mainBuffer,f=U.locationBuffer,u=h,c=U.recordSize,s=U.locationRecordSize,d={range:[null,null],country:'',region:'',eu:'',timezone:'',city:'',ll:[null,null],metro:null,area:null};if(e>U.lastIP||e<U.firstIP)return null;for(let r=0;r<u.length;r++)if(e>=u[r][0]&&e<=u[r][1])return null;for(;;){r=Math.round((i-n)/2)+n;const u=r*c;if(o=l.readUInt32BE(u),a=l.readUInt32BE(u+4),o<=e&&a>=e){if(d.range=[o,a],10===c)d.country=l.toString('utf8',u+8,u+10);else if(t=l.readUInt32BE(u+8),-1>>>0>t){const e=t*s;d.country=N(f.toString('utf8',e,e+2)),d.region=N(f.toString('utf8',e+2,e+5)),d.metro=f.readInt32BE(e+5),d.ll[0]=l.readInt32BE(u+12)/1e4,d.ll[1]=l.readInt32BE(u+16)/1e4,d.area=l.readUInt32BE(u+20),d.eu=N(f.toString('utf8',e+9,e+10)),d.timezone=N(f.toString('utf8',e+10,e+42)),d.city=N(f.toString('utf8',e+42,e+s))}return d}if(n===i)return null;n===i-1?r===n?n=i:i=n:o>e?i=r:a<e&&(n=r)}};function A(f){let u,c;const d={...F};if('function'==typeof arguments[0])s.series([i=>{s.series([r=>{e(P.cityNames,'r',(e,t)=>{u=t,r(e)})},e=>{r(u,(r,t)=>{c=t.size,d.locationBuffer=Buffer.alloc(c),e(r)})},e=>{t(u,d.locationBuffer,0,c,0,e)},e=>{n(u,e)},r=>{e(P.city,'r',(e,t)=>{u=t,r(e)})},e=>{r(u,(r,t)=>{c=t.size,e(r)})}],t=>{if(t){if('ENOENT'!==t.code&&'EBADF'!==t.code)throw t;e(P.country,'r',(e,t)=>{e?i(e):(u=t,r(u,(e,r)=>{c=r.size,d.recordSize=10,i()}))})}else i()})},()=>{d.mainBuffer=Buffer.alloc(c),s.series([e=>{t(u,d.mainBuffer,0,c,0,e)},e=>{n(u,e)}],e=>{e||(d.lastLine=c/d.recordSize-1,d.lastIP=d.mainBuffer.readUInt32BE(d.lastLine*d.recordSize+4),d.firstIP=d.mainBuffer.readUInt32BE(0),U=d),f(e)})}]);else{try{if(u=i(P.cityNames,'r'),c=o(u).size,0===c){const e=new Error('Empty file');throw e.code='EMPTY_FILE',e}U.locationBuffer=Buffer.alloc(c),a(u,U.locationBuffer,0,c,0),l(u),u=i(P.city,'r'),c=o(u).size}catch(e){if('ENOENT'!==e.code&&'EBADF'!==e.code&&'EMPTY_FILE'!==e.code)throw e;u=i(P.country,'r'),c=o(u).size,U.recordSize=10}U.mainBuffer=Buffer.alloc(c),a(u,U.mainBuffer,0,c,0),l(u),U.lastLine=c/U.recordSize-1,U.lastIP=U.mainBuffer.readUInt32BE(U.lastLine*U.recordSize+4),U.firstIP=U.mainBuffer.readUInt32BE(0)}}function W(f){let u,c;const d={...L};if('function'==typeof arguments[0])s.series([t=>{s.series([r=>{e(P.city6,'r',(e,t)=>{u=t,r(e)})},e=>{r(u,(r,t)=>{c=t.size,e(r)})}],n=>{if(n){if('ENOENT'!==n.code&&'EBADF'!==n.code)throw n;e(P.country6,'r',(e,n)=>{e?t(e):(u=n,r(u,(e,r)=>{c=r.size,d.recordSize=34,t()}))})}else t()})},()=>{d.mainBuffer=Buffer.alloc(c),s.series([e=>{t(u,d.mainBuffer,0,c,0,e)},e=>{n(u,e)}],e=>{e||(d.lastLine=c/d.recordSize-1,d.lastIP=T(d.mainBuffer,d.lastLine,d.recordSize,1),d.firstIP=T(d.mainBuffer,0,d.recordSize,0),D=d),f(e)})}]);else{try{if(u=i(P.city6,'r'),c=o(u).size,0===c){const e=new Error('Empty file');throw e.code='EMPTY_FILE',e}}catch(e){if('ENOENT'!==e.code&&'EBADF'!==e.code&&'EMPTY_FILE'!==e.code)throw e;u=i(P.country6,'r'),c=o(u).size,D.recordSize=34}D.mainBuffer=Buffer.alloc(c),a(u,D.mainBuffer,0,c,0),l(u),D.lastLine=c/D.recordSize-1,D.lastIP=T(D.mainBuffer,D.lastLine,D.recordSize,1),D.firstIP=T(D.mainBuffer,0,D.recordSize,0)}}module.exports={cmp:E,lookup:e=>{if(!e)return null;if('number'==typeof e)return w(e);if(4===c(e))return w(d(e));if(6===c(e)){const r=(e=>{const r=e.toUpperCase();return r.startsWith("0:0:0:0:0:FFFF:")?r.substring(15):r.startsWith("::FFFF:")?r.substring(7):null})(e);return r?w(d(r)):(e=>{const r=D.mainBuffer,t=D.recordSize,n=U.locationBuffer,i=U.locationRecordSize,o={range:[null,null],country:'',region:'',city:'',ll:[0,0],metro:null,area:null,eu:'',timezone:''};let a,l,f=0,u=D.lastLine,c=D.lastIP,s=D.firstIP;if(y(e,D.lastIP)>0||y(e,D.firstIP)<0)return null;for(;;){if(a=Math.round((u-f)/2)+f,c=T(r,a,t,0),s=T(r,a,t,1),y(c,e)<=0&&y(s,e)>=0){const e=a*t;if(34===t)o.country=N(r.toString('utf8',e+32,e+34));else if(l=r.readUInt32BE(e+32),-1>>>0>l){const t=l*i;o.country=N(n.toString('utf8',t,t+2)),o.region=N(n.toString('utf8',t+2,t+5)),o.metro=n.readInt32BE(t+5),o.ll[0]=r.readInt32BE(e+36)/1e4,o.ll[1]=r.readInt32BE(e+40)/1e4,o.area=r.readUInt32BE(e+44),o.eu=N(n.toString('utf8',t+9,t+10)),o.timezone=N(n.toString('utf8',t+10,t+42)),o.city=N(n.toString('utf8',t+42,t+i))}return o}if(f===u)return null;f===u-1?a===f?f=u:u=f:y(c,e)>0?u=a:y(s,e)<0&&(f=a)}})(B(e))}return null},pretty:e=>'string'==typeof e?e:'number'==typeof e?I(e):Array.isArray(e)?m(e):e,startWatchingDataUpdate:e=>{S.makeFsWatchFilter(z,p,6e4,()=>{s.series([e=>{A(e)},e=>{W(e)}],e)})},stopWatchingDataUpdate:()=>S.stopWatching(z),clear:()=>{U={...F},D={...L}},reloadDataSync:()=>{A(),W()},reloadData:e=>{s.series([e=>{A(e)},e=>{W(e)}],e)},version:g},A(),W();
|
package/dist/utils.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
const t=module.exports={};t.aton4=t=>{const r=t.split('.');return(parseInt(r[0],10)<<24>>>0)+(parseInt(r[1],10)<<16>>>0)+(parseInt(r[2],10)<<8>>>0)+(parseInt(r[3],10)>>>0)},t.aton6=t=>{const r=(t=t.replace(/"/g,'').split(':')).length-1;let e;if(''===t[r]&&(t[r]=0),r<7)for(t.length=8,e=r;e>=0&&''!==t[e];e--)t[7-r+e]=t[e];for(e=0;e<8;e++)t[e]?t[e]=parseInt(t[e],16):t[e]=0;const n=[];for(e=0;e<4;e++)n.push((t[2*e]<<16)+t[2*e+1]>>>0);return n},t.
|
|
1
|
+
const t=module.exports={};t.aton4=t=>{const r=t.split('.');return(parseInt(r[0],10)<<24>>>0)+(parseInt(r[1],10)<<16>>>0)+(parseInt(r[2],10)<<8>>>0)+(parseInt(r[3],10)>>>0)},t.ntoa4=t=>(t>>>24&255)+'.'+(t>>>16&255)+'.'+(t>>>8&255)+'.'+(255&t),t.aton6=t=>{const r=(t=t.replace(/"/g,'').split(':')).length-1;let e;if(''===t[r]&&(t[r]=0),r<7)for(t.length=8,e=r;e>=0&&''!==t[e];e--)t[7-r+e]=t[e];for(e=0;e<8;e++)t[e]?t[e]=parseInt(t[e],16):t[e]=0;const n=[];for(e=0;e<4;e++)n.push((t[2*e]<<16)+t[2*e+1]>>>0);return n},t.ntoa6=t=>{let r='[';for(let e=0;e<t.length;e++)r+=(t[e]>>>16).toString(16)+':',r+=(65535&t[e]).toString(16)+':';return r=r.replace(/:$/,']').replace(/:0+/g,':').replace(/::+/,'::'),r},t.cmp=(r,e)=>'number'==typeof r&&'number'==typeof e?r<e?-1:r>e?1:0:r instanceof Array&&e instanceof Array?t.cmp6(r,e):null,t.cmp6=(t,r)=>{for(let e=0;e<2;e++){if(t[e]<r[e])return-1;if(t[e]>r[e])return 1}return 0},t.isPrivateIP=t=>{const r=t.toString();return r.startsWith('10.')||r.startsWith('192.168.')||r.startsWith('172.16.')||r.startsWith('127.')||r.startsWith('169.254.')||r.startsWith('fc00:')||r.startsWith('fe80:')};
|
package/index.d.ts
CHANGED
|
@@ -2,12 +2,12 @@ interface GeoIp2Location {
|
|
|
2
2
|
range: [number | null, number | null];
|
|
3
3
|
country: string;
|
|
4
4
|
region: string;
|
|
5
|
-
eu: '0' | '1';
|
|
5
|
+
eu: '0' | '1' | '';
|
|
6
6
|
timezone: string;
|
|
7
7
|
city: string;
|
|
8
8
|
ll: [number | null, number | null];
|
|
9
|
-
metro: number;
|
|
10
|
-
area: number;
|
|
9
|
+
metro: number | null;
|
|
10
|
+
area: number | null;
|
|
11
11
|
}
|
|
12
12
|
|
|
13
13
|
export function lookup(ip: string | number): GeoIp2Location | null;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "geoip-lite2",
|
|
3
|
-
"version": "2.2.8-beta.
|
|
3
|
+
"version": "2.2.8-beta.3",
|
|
4
4
|
"description": "A light weight native JavaScript implementation of GeoIP API from MaxMind. Improved and faster version by Sefinek.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"city",
|
|
@@ -79,14 +79,14 @@
|
|
|
79
79
|
"adm-zip": "^0.5.16",
|
|
80
80
|
"async": "^3.2.6",
|
|
81
81
|
"iconv-lite": "0.7.0",
|
|
82
|
-
"ip-address": "^10.0
|
|
82
|
+
"ip-address": "^10.1.0",
|
|
83
83
|
"rimraf": "^6.1.0"
|
|
84
84
|
},
|
|
85
85
|
"devDependencies": {
|
|
86
86
|
"@eslint/js": "^9.39.1",
|
|
87
87
|
"globals": "^16.5.0",
|
|
88
88
|
"jest": "^30.2.0",
|
|
89
|
-
"terser": "^5.44.
|
|
89
|
+
"terser": "^5.44.1"
|
|
90
90
|
},
|
|
91
91
|
"engines": {
|
|
92
92
|
"node": ">=10.3.0"
|
package/tools/updatedb.js
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
|
-
//
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// GeoIP Database Updater
|
|
3
|
+
// Fetches and converts MaxMind GeoLite2 databases
|
|
4
|
+
// ============================================================================
|
|
2
5
|
|
|
3
6
|
'use strict';
|
|
4
7
|
|
|
8
|
+
// ============================================================================
|
|
9
|
+
// Dependencies
|
|
10
|
+
// ============================================================================
|
|
11
|
+
|
|
5
12
|
const { name, version } = require('../package.json');
|
|
6
13
|
const UserAgent = `Mozilla/5.0 (compatible; ${name}/${version}; +https://github.com/sefinek/geoip-lite2)`;
|
|
7
14
|
|
|
@@ -19,6 +26,24 @@ const AdmZip = require('adm-zip');
|
|
|
19
26
|
const utils = require('../lib/utils.js');
|
|
20
27
|
const { Address6, Address4 } = require('ip-address');
|
|
21
28
|
|
|
29
|
+
// ============================================================================
|
|
30
|
+
// Utility Functions - Logging
|
|
31
|
+
// ============================================================================
|
|
32
|
+
|
|
33
|
+
// Logging utility for consistent and readable output
|
|
34
|
+
const log = {
|
|
35
|
+
info: (msg, ...logArgs) => console.log('[INFO]', msg, ...logArgs),
|
|
36
|
+
success: (msg, ...logArgs) => console.log('[SUCCESS]', msg, ...logArgs),
|
|
37
|
+
warn: (msg, ...logArgs) => console.warn('[WARN]', msg, ...logArgs),
|
|
38
|
+
error: (msg, ...logArgs) => console.error('[ERROR]', msg, ...logArgs),
|
|
39
|
+
progress: (msg) => process.stdout.write(`[INFO] ${msg}... `),
|
|
40
|
+
done: () => console.log('Done'),
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
// ============================================================================
|
|
44
|
+
// Configuration
|
|
45
|
+
// ============================================================================
|
|
46
|
+
|
|
22
47
|
const args = process.argv.slice(2);
|
|
23
48
|
let license_key = args.find(arg => arg.match(/^license_key=[a-zA-Z0-9]+/) !== null);
|
|
24
49
|
if (typeof license_key === 'undefined' && typeof process.env.LICENSE_KEY !== 'undefined') {
|
|
@@ -34,7 +59,7 @@ let dataPath = path.resolve(__dirname, '..', 'data');
|
|
|
34
59
|
if (typeof geoDataDir !== 'undefined') {
|
|
35
60
|
dataPath = path.resolve(process.cwd(), geoDataDir.split('=')[1]);
|
|
36
61
|
if (!fs.existsSync(dataPath)) {
|
|
37
|
-
|
|
62
|
+
log.error('Directory does not exist:', dataPath);
|
|
38
63
|
process.exit(1);
|
|
39
64
|
}
|
|
40
65
|
}
|
|
@@ -67,6 +92,10 @@ const databases = [{
|
|
|
67
92
|
dest: ['geoip-city-names.dat', 'geoip-city.dat', 'geoip-city6.dat'],
|
|
68
93
|
}];
|
|
69
94
|
|
|
95
|
+
// ============================================================================
|
|
96
|
+
// Utility Functions
|
|
97
|
+
// ============================================================================
|
|
98
|
+
|
|
70
99
|
function mkdir(dirName) {
|
|
71
100
|
const dir = path.dirname(dirName);
|
|
72
101
|
if (!fs.existsSync(dir)) fs.mkdirSync(dir);
|
|
@@ -118,6 +147,10 @@ function CSVtoArray(text) {
|
|
|
118
147
|
return a;
|
|
119
148
|
}
|
|
120
149
|
|
|
150
|
+
// ============================================================================
|
|
151
|
+
// HTTP Configuration
|
|
152
|
+
// ============================================================================
|
|
153
|
+
|
|
121
154
|
function getHTTPOptions(downloadUrl) {
|
|
122
155
|
const parsedUrl = new URL(downloadUrl);
|
|
123
156
|
const options = {
|
|
@@ -132,7 +165,7 @@ function getHTTPOptions(downloadUrl) {
|
|
|
132
165
|
const HttpsProxyAgent = require('https-proxy-agent');
|
|
133
166
|
options.agent = new HttpsProxyAgent(process.env.http_proxy || process.env.https_proxy);
|
|
134
167
|
} catch (err) {
|
|
135
|
-
|
|
168
|
+
log.error(`Install https-proxy-agent to use an HTTP/HTTPS proxy. ${err.message}`);
|
|
136
169
|
process.exit(-1);
|
|
137
170
|
}
|
|
138
171
|
}
|
|
@@ -140,6 +173,10 @@ function getHTTPOptions(downloadUrl) {
|
|
|
140
173
|
return options;
|
|
141
174
|
}
|
|
142
175
|
|
|
176
|
+
// ============================================================================
|
|
177
|
+
// Database Download Functions
|
|
178
|
+
// ============================================================================
|
|
179
|
+
|
|
143
180
|
function check(database, cb) {
|
|
144
181
|
if (args.indexOf('force') !== -1) {
|
|
145
182
|
// We are forcing database upgrade,
|
|
@@ -154,14 +191,16 @@ function check(database, cb) {
|
|
|
154
191
|
fs.readFile(path.join(dataPath, `${database.type}.checksum`), { encoding: 'utf8' }, (err, data) => {
|
|
155
192
|
if (!err && data && data.length) database.checkValue = data;
|
|
156
193
|
|
|
157
|
-
|
|
194
|
+
log.info('Checking database:', database.fileName);
|
|
195
|
+
|
|
196
|
+
const client = https.get(getHTTPOptions(checksumUrl), onResponse);
|
|
158
197
|
|
|
159
198
|
function onResponse(response) {
|
|
160
199
|
const status = response.statusCode;
|
|
161
200
|
if ([301, 302, 303, 307, 308].includes(status)) {
|
|
162
201
|
return https.get(getHTTPOptions(response.headers.location), onResponse);
|
|
163
202
|
} else if (status !== 200) {
|
|
164
|
-
|
|
203
|
+
log.error('HTTP Request Failed [%d %s]', status, http.STATUS_CODES[status]);
|
|
165
204
|
client.end();
|
|
166
205
|
process.exit(1);
|
|
167
206
|
}
|
|
@@ -174,24 +213,22 @@ function check(database, cb) {
|
|
|
174
213
|
response.on('end', () => {
|
|
175
214
|
if (str && str.length) {
|
|
176
215
|
if (str === database.checkValue) {
|
|
177
|
-
|
|
216
|
+
log.info(`Database "${database.type}" is up to date`);
|
|
178
217
|
database.skip = true;
|
|
179
218
|
} else {
|
|
180
|
-
|
|
219
|
+
log.info(`Database "${database.type}" has new data available`);
|
|
181
220
|
database.checkValue = str;
|
|
182
221
|
}
|
|
183
222
|
}
|
|
184
223
|
else {
|
|
185
|
-
|
|
186
|
-
|
|
224
|
+
log.error(`Could not retrieve checksum for ${database.type}. Aborting.`);
|
|
225
|
+
log.error('Run with "force" to update without checksum');
|
|
187
226
|
client.end();
|
|
188
227
|
process.exit(1);
|
|
189
228
|
}
|
|
190
229
|
cb(null, database);
|
|
191
230
|
});
|
|
192
231
|
}
|
|
193
|
-
|
|
194
|
-
var client = https.get(getHTTPOptions(checksumUrl), onResponse);
|
|
195
232
|
});
|
|
196
233
|
}
|
|
197
234
|
|
|
@@ -206,14 +243,16 @@ function fetch(database, cb) {
|
|
|
206
243
|
const tmpFile = path.join(tmpPath, fileName);
|
|
207
244
|
if (fs.existsSync(tmpFile)) return cb(null, tmpFile, fileName, database);
|
|
208
245
|
|
|
209
|
-
|
|
246
|
+
log.info('Downloading:', fileName);
|
|
247
|
+
|
|
248
|
+
const client = https.get(getHTTPOptions(downloadUrl), onResponse);
|
|
210
249
|
|
|
211
250
|
function onResponse(response) {
|
|
212
251
|
const status = response.statusCode;
|
|
213
252
|
if ([301, 302, 303, 307, 308].includes(status)) {
|
|
214
253
|
return https.get(getHTTPOptions(response.headers.location), onResponse);
|
|
215
254
|
} else if (status !== 200) {
|
|
216
|
-
|
|
255
|
+
log.error('HTTP Request Failed [%d %s]', status, http.STATUS_CODES[status]);
|
|
217
256
|
client.end();
|
|
218
257
|
process.exit(1);
|
|
219
258
|
}
|
|
@@ -228,16 +267,14 @@ function fetch(database, cb) {
|
|
|
228
267
|
}
|
|
229
268
|
|
|
230
269
|
tmpFilePipe.on('close', () => {
|
|
231
|
-
|
|
270
|
+
log.done();
|
|
232
271
|
cb(null, tmpFile, fileName, database);
|
|
233
272
|
});
|
|
234
273
|
}
|
|
235
274
|
|
|
236
275
|
mkdir(tmpFile);
|
|
237
276
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
process.stdout.write(`Retrieving ${fileName}...`);
|
|
277
|
+
log.progress(`Retrieving ${fileName}`);
|
|
241
278
|
}
|
|
242
279
|
|
|
243
280
|
function extract(tmpFile, tmpFileName, database, cb) {
|
|
@@ -246,7 +283,7 @@ function extract(tmpFile, tmpFileName, database, cb) {
|
|
|
246
283
|
if (path.extname(tmpFileName) !== '.zip') {
|
|
247
284
|
cb(null, database);
|
|
248
285
|
} else {
|
|
249
|
-
|
|
286
|
+
log.progress('Extracting ' + tmpFileName);
|
|
250
287
|
const zip = new AdmZip(tmpFile);
|
|
251
288
|
const zipEntries = zip.getEntries();
|
|
252
289
|
|
|
@@ -260,7 +297,7 @@ function extract(tmpFile, tmpFileName, database, cb) {
|
|
|
260
297
|
fs.writeFileSync(destinationPath, entry.getData());
|
|
261
298
|
});
|
|
262
299
|
|
|
263
|
-
|
|
300
|
+
log.done();
|
|
264
301
|
cb(null, database);
|
|
265
302
|
}
|
|
266
303
|
}
|
|
@@ -269,14 +306,14 @@ function processLookupCountry(src, cb) {
|
|
|
269
306
|
function processLine(line) {
|
|
270
307
|
const fields = CSVtoArray(line);
|
|
271
308
|
if (!fields || fields.length < 6) {
|
|
272
|
-
|
|
309
|
+
log.warn('Malformed line detected:', line);
|
|
273
310
|
return;
|
|
274
311
|
}
|
|
275
312
|
countryLookup[fields[0]] = fields[4];
|
|
276
313
|
}
|
|
277
314
|
const tmpDataFile = path.join(tmpPath, src);
|
|
278
315
|
|
|
279
|
-
|
|
316
|
+
log.progress('Processing lookup data (this may take a moment)');
|
|
280
317
|
|
|
281
318
|
const rl = readline.createInterface({ input: fs.createReadStream(tmpDataFile).pipe(decodeStream('latin1')), output: process.stdout, terminal: false });
|
|
282
319
|
|
|
@@ -287,16 +324,31 @@ function processLookupCountry(src, cb) {
|
|
|
287
324
|
});
|
|
288
325
|
|
|
289
326
|
rl.on('close', () => {
|
|
290
|
-
|
|
327
|
+
log.done();
|
|
291
328
|
cb();
|
|
292
329
|
});
|
|
293
330
|
}
|
|
294
331
|
|
|
332
|
+
// ============================================================================
|
|
333
|
+
// Data Processing Functions
|
|
334
|
+
// ============================================================================
|
|
335
|
+
|
|
295
336
|
async function processCountryData(src, dest) {
|
|
296
337
|
let lines = 0;
|
|
338
|
+
const dataFile = path.join(dataPath, dest);
|
|
339
|
+
const tmpDataFile = path.join(tmpPath, src);
|
|
340
|
+
|
|
341
|
+
rimraf(dataFile);
|
|
342
|
+
mkdir(dataFile);
|
|
343
|
+
|
|
344
|
+
process.stdout.write('\n');
|
|
345
|
+
log.progress('Processing country data (this may take a moment)');
|
|
346
|
+
let tstart = Date.now();
|
|
347
|
+
const datFile = fs.createWriteStream(dataFile);
|
|
348
|
+
|
|
297
349
|
function processLine(line) {
|
|
298
350
|
const fields = CSVtoArray(line);
|
|
299
|
-
if (!fields || fields.length < 6) return
|
|
351
|
+
if (!fields || fields.length < 6) return log.warn('Malformed line detected:', line);
|
|
300
352
|
|
|
301
353
|
lines++;
|
|
302
354
|
|
|
@@ -353,16 +405,6 @@ async function processCountryData(src, dest) {
|
|
|
353
405
|
}
|
|
354
406
|
}
|
|
355
407
|
|
|
356
|
-
const dataFile = path.join(dataPath, dest);
|
|
357
|
-
const tmpDataFile = path.join(tmpPath, src);
|
|
358
|
-
|
|
359
|
-
rimraf(dataFile);
|
|
360
|
-
mkdir(dataFile);
|
|
361
|
-
|
|
362
|
-
process.stdout.write('\nProcessing data (may take a moment)...');
|
|
363
|
-
var tstart = Date.now();
|
|
364
|
-
var datFile = fs.createWriteStream(dataFile);
|
|
365
|
-
|
|
366
408
|
const rl = readline.createInterface({ input: fs.createReadStream(tmpDataFile), crlfDelay: Infinity });
|
|
367
409
|
let i = 0;
|
|
368
410
|
for await (const line of rl) {
|
|
@@ -371,16 +413,26 @@ async function processCountryData(src, dest) {
|
|
|
371
413
|
await processLine(line);
|
|
372
414
|
}
|
|
373
415
|
datFile.close();
|
|
374
|
-
|
|
416
|
+
log.done();
|
|
375
417
|
}
|
|
376
418
|
|
|
377
419
|
async function processCityData(src, dest) {
|
|
378
420
|
let lines = 0;
|
|
421
|
+
const dataFile = path.join(dataPath, dest);
|
|
422
|
+
const tmpDataFile = path.join(tmpPath, src);
|
|
423
|
+
|
|
424
|
+
rimraf(dataFile);
|
|
425
|
+
|
|
426
|
+
process.stdout.write('\n');
|
|
427
|
+
log.progress('Processing city data (this may take a moment)');
|
|
428
|
+
let tstart = Date.now();
|
|
429
|
+
const datFile = fs.createWriteStream(dataFile);
|
|
430
|
+
|
|
379
431
|
async function processLine(line) {
|
|
380
432
|
if (line.match(/^Copyright/) || !line.match(/\d/)) return;
|
|
381
433
|
|
|
382
434
|
const fields = CSVtoArray(line);
|
|
383
|
-
if (!fields) return
|
|
435
|
+
if (!fields) return log.warn('Malformed line detected:', line);
|
|
384
436
|
let sip;
|
|
385
437
|
let eip;
|
|
386
438
|
let rngip;
|
|
@@ -450,7 +502,7 @@ async function processCityData(src, dest) {
|
|
|
450
502
|
|
|
451
503
|
if (Date.now() - tstart > 5000) {
|
|
452
504
|
tstart = Date.now();
|
|
453
|
-
process.stdout.write('\
|
|
505
|
+
process.stdout.write('\n[INFO] Processing... (' + lines + ' entries) ');
|
|
454
506
|
}
|
|
455
507
|
|
|
456
508
|
if (datFile._writableState.needDrain) {
|
|
@@ -462,15 +514,6 @@ async function processCityData(src, dest) {
|
|
|
462
514
|
}
|
|
463
515
|
}
|
|
464
516
|
|
|
465
|
-
const dataFile = path.join(dataPath, dest);
|
|
466
|
-
const tmpDataFile = path.join(tmpPath, src);
|
|
467
|
-
|
|
468
|
-
rimraf(dataFile);
|
|
469
|
-
|
|
470
|
-
process.stdout.write('\nProcessing data (may take a moment)...');
|
|
471
|
-
var tstart = Date.now();
|
|
472
|
-
var datFile = fs.createWriteStream(dataFile);
|
|
473
|
-
|
|
474
517
|
const rl = readline.createInterface({ input: fs.createReadStream(tmpDataFile), crlfDelay: Infinity });
|
|
475
518
|
let i = 0;
|
|
476
519
|
for await (const line of rl) {
|
|
@@ -484,6 +527,13 @@ async function processCityData(src, dest) {
|
|
|
484
527
|
function processCityDataNames(src, dest, cb) {
|
|
485
528
|
let locId = null;
|
|
486
529
|
let linesCount = 0;
|
|
530
|
+
const dataFile = path.join(dataPath, dest);
|
|
531
|
+
const tmpDataFile = path.join(tmpPath, src);
|
|
532
|
+
|
|
533
|
+
rimraf(dataFile);
|
|
534
|
+
|
|
535
|
+
const datFile = fs.openSync(dataFile, 'w');
|
|
536
|
+
|
|
487
537
|
function processLine(line) {
|
|
488
538
|
if (line.match(/^Copyright/) || !line.match(/\d/)) return;
|
|
489
539
|
|
|
@@ -491,7 +541,7 @@ function processCityDataNames(src, dest, cb) {
|
|
|
491
541
|
const fields = CSVtoArray(line);
|
|
492
542
|
if (!fields) {
|
|
493
543
|
// Lots of cities contain ` or ' in the name and can't be parsed correctly with current method
|
|
494
|
-
|
|
544
|
+
log.warn('Malformed line detected:', line);
|
|
495
545
|
return;
|
|
496
546
|
}
|
|
497
547
|
|
|
@@ -519,13 +569,6 @@ function processCityDataNames(src, dest, cb) {
|
|
|
519
569
|
linesCount++;
|
|
520
570
|
}
|
|
521
571
|
|
|
522
|
-
const dataFile = path.join(dataPath, dest);
|
|
523
|
-
const tmpDataFile = path.join(tmpPath, src);
|
|
524
|
-
|
|
525
|
-
rimraf(dataFile);
|
|
526
|
-
|
|
527
|
-
var datFile = fs.openSync(dataFile, 'w');
|
|
528
|
-
|
|
529
572
|
const rl = readline.createInterface({ input: fs.createReadStream(tmpDataFile).pipe(decodeStream('utf-8')), output: process.stdout, terminal: false });
|
|
530
573
|
|
|
531
574
|
let lineCount = 0;
|
|
@@ -538,6 +581,10 @@ function processCityDataNames(src, dest, cb) {
|
|
|
538
581
|
rl.on('close', cb);
|
|
539
582
|
}
|
|
540
583
|
|
|
584
|
+
// ============================================================================
|
|
585
|
+
// Main Processing Dispatcher
|
|
586
|
+
// ============================================================================
|
|
587
|
+
|
|
541
588
|
function processData(database, cb) {
|
|
542
589
|
if (database.skip) return cb(null, database);
|
|
543
590
|
|
|
@@ -563,27 +610,36 @@ function processData(database, cb) {
|
|
|
563
610
|
} else if (type === 'city') {
|
|
564
611
|
processCityDataNames(src[0], dest[0], () => {
|
|
565
612
|
processCityData(src[1], dest[1]).then(() => {
|
|
566
|
-
|
|
613
|
+
process.stdout.write('\n');
|
|
614
|
+
log.info('City IPv4 data processed');
|
|
567
615
|
return processCityData(src[2], dest[2]);
|
|
568
616
|
}).then(() => {
|
|
569
|
-
|
|
617
|
+
log.info('City IPv6 data processed');
|
|
570
618
|
cb(null, database);
|
|
571
619
|
});
|
|
572
620
|
});
|
|
573
621
|
}
|
|
574
622
|
}
|
|
575
623
|
|
|
624
|
+
// ============================================================================
|
|
625
|
+
// Checksum Management
|
|
626
|
+
// ============================================================================
|
|
627
|
+
|
|
576
628
|
function updateChecksum(database, cb) {
|
|
577
629
|
if (database.skip || !database.checkValue) return cb(); // Don't need to update checksums because it was not fetched or did not change
|
|
578
630
|
|
|
579
631
|
fs.writeFile(path.join(dataPath, database.type + '.checksum'), database.checkValue, 'utf8', err => {
|
|
580
|
-
if (err)
|
|
632
|
+
if (err) log.error('Failed to update checksum for database:', database.type);
|
|
581
633
|
cb();
|
|
582
634
|
});
|
|
583
635
|
}
|
|
584
636
|
|
|
637
|
+
// ============================================================================
|
|
638
|
+
// Main Execution Flow
|
|
639
|
+
// ============================================================================
|
|
640
|
+
|
|
585
641
|
if (!license_key) {
|
|
586
|
-
|
|
642
|
+
log.error('Missing license_key');
|
|
587
643
|
process.exit(1);
|
|
588
644
|
}
|
|
589
645
|
|
|
@@ -594,10 +650,11 @@ async.eachSeries(databases, (database, nextDatabase) => {
|
|
|
594
650
|
async.seq(check, fetch, extract, processData, updateChecksum)(database, nextDatabase);
|
|
595
651
|
}, err => {
|
|
596
652
|
if (err) {
|
|
597
|
-
|
|
653
|
+
log.error('Failed to update databases from MaxMind!', err);
|
|
598
654
|
process.exit(1);
|
|
599
655
|
} else {
|
|
600
|
-
|
|
656
|
+
process.stdout.write('\n');
|
|
657
|
+
log.success('All databases have been successfully updated from MaxMind');
|
|
601
658
|
if (args.indexOf('debug') !== -1) {
|
|
602
659
|
console.debug('Notice: temporary files are not deleted for debug purposes');
|
|
603
660
|
} else {
|