@hebcal/geo-sqlite 5.10.4 → 5.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -3
- package/bin/build-geonames-sqlite +9 -7
- package/bin/make-test-dbs +10 -8
- package/dist/build-geonames-sqlite.d.ts +28 -0
- package/dist/build-geonames-sqlite.js +309 -0
- package/dist/build-geonames-sqlite.js.map +1 -0
- package/dist/city2geonameid.json.d.ts +490 -0
- package/dist/city2geonameid.json.js +490 -0
- package/dist/city2geonameid.json.js.map +1 -0
- package/dist/geodb.d.ts +87 -0
- package/dist/geodb.js +451 -0
- package/dist/geodb.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +4 -1483
- package/dist/index.js.map +1 -0
- package/dist/make-zips-sqlite.d.ts +6 -0
- package/dist/make-zips-sqlite.js +15 -0
- package/dist/make-zips-sqlite.js.map +1 -0
- package/dist/munge.d.ts +6 -0
- package/dist/munge.js +13 -0
- package/dist/munge.js.map +1 -0
- package/dist/pkgVersion.d.ts +1 -0
- package/dist/pkgVersion.js +3 -0
- package/dist/pkgVersion.js.map +1 -0
- package/package.json +26 -30
- package/geo-sqlite.d.ts +0 -161
package/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# hebcal
|
|
1
|
+
# @hebcal/geo-sqlite
|
|
2
2
|
Hebcal ES6 interface to GeoNames and USA ZIP code SQLite databases
|
|
3
3
|
|
|
4
4
|
This package is designed for the hebcal.com and may not be generally reusable.
|
|
@@ -52,6 +52,9 @@ under the
|
|
|
52
52
|
<dd></dd>
|
|
53
53
|
<dt><a href="#doFile">doFile(logger, db, infile, tableName, expectedFields, callback)</a></dt>
|
|
54
54
|
<dd></dd>
|
|
55
|
+
<dt><a href="#makeZipsSqlite">makeZipsSqlite(dbFilename, sqlFile)</a></dt>
|
|
56
|
+
<dd><p>Builds <code>zips.sqlite3</code> from the bundled zips-dummy.sql schema file</p>
|
|
57
|
+
</dd>
|
|
55
58
|
</dl>
|
|
56
59
|
|
|
57
60
|
<a name="GeoDb"></a>
|
|
@@ -62,7 +65,7 @@ Wrapper around sqlite databases
|
|
|
62
65
|
**Kind**: global class
|
|
63
66
|
|
|
64
67
|
* [GeoDb](#GeoDb)
|
|
65
|
-
* [new GeoDb(logger, zipsFilename, geonamesFilename)](#new_GeoDb_new)
|
|
68
|
+
* [new GeoDb(logger, zipsFilename, geonamesFilename, options)](#new_GeoDb_new)
|
|
66
69
|
* _instance_
|
|
67
70
|
* [.zipCache](#GeoDb+zipCache) : <code>Map.<string, Location></code>
|
|
68
71
|
* [.geonamesCache](#GeoDb+geonamesCache) : <code>Map.<number, Location></code>
|
|
@@ -79,16 +82,18 @@ Wrapper around sqlite databases
|
|
|
79
82
|
* [.transliterate(source, [options])](#GeoDb.transliterate) ⇒ <code>string</code>
|
|
80
83
|
* [.geonameCityDescr(cityName, admin1, countryName)](#GeoDb.geonameCityDescr) ⇒ <code>string</code>
|
|
81
84
|
* [.version()](#GeoDb.version)
|
|
85
|
+
* [.is5DigitZip(str)](#GeoDb.is5DigitZip) ⇒ <code>boolean</code>
|
|
82
86
|
|
|
83
87
|
<a name="new_GeoDb_new"></a>
|
|
84
88
|
|
|
85
|
-
### new GeoDb(logger, zipsFilename, geonamesFilename)
|
|
89
|
+
### new GeoDb(logger, zipsFilename, geonamesFilename, options)
|
|
86
90
|
|
|
87
91
|
| Param | Type |
|
|
88
92
|
| --- | --- |
|
|
89
93
|
| logger | <code>any</code> |
|
|
90
94
|
| zipsFilename | <code>string</code> |
|
|
91
95
|
| geonamesFilename | <code>string</code> |
|
|
96
|
+
| options | <code>any</code> |
|
|
92
97
|
|
|
93
98
|
<a name="GeoDb+zipCache"></a>
|
|
94
99
|
|
|
@@ -195,6 +200,15 @@ Builds a city description from geonameid string components
|
|
|
195
200
|
Returns the version of the GeoDb package
|
|
196
201
|
|
|
197
202
|
**Kind**: static method of [<code>GeoDb</code>](#GeoDb)
|
|
203
|
+
<a name="GeoDb.is5DigitZip"></a>
|
|
204
|
+
|
|
205
|
+
### GeoDb.is5DigitZip(str) ⇒ <code>boolean</code>
|
|
206
|
+
**Kind**: static method of [<code>GeoDb</code>](#GeoDb)
|
|
207
|
+
|
|
208
|
+
| Param | Type |
|
|
209
|
+
| --- | --- |
|
|
210
|
+
| str | <code>string</code> |
|
|
211
|
+
|
|
198
212
|
<a name="buildGeonamesSqlite"></a>
|
|
199
213
|
|
|
200
214
|
## buildGeonamesSqlite(opts)
|
|
@@ -230,3 +244,15 @@ Builds `geonames.sqlite3` from files downloaded from geonames.org
|
|
|
230
244
|
| tableName | <code>string</code> |
|
|
231
245
|
| expectedFields | <code>number</code> |
|
|
232
246
|
| callback | <code>function</code> |
|
|
247
|
+
|
|
248
|
+
<a name="makeZipsSqlite"></a>
|
|
249
|
+
|
|
250
|
+
## makeZipsSqlite(dbFilename, sqlFile)
|
|
251
|
+
Builds `zips.sqlite3` from the bundled zips-dummy.sql schema file
|
|
252
|
+
|
|
253
|
+
**Kind**: global function
|
|
254
|
+
|
|
255
|
+
| Param | Type | Description |
|
|
256
|
+
| --- | --- | --- |
|
|
257
|
+
| dbFilename | <code>string</code> | path to the output SQLite database file |
|
|
258
|
+
| sqlFile | <code>string</code> | path to the SQL schema file |
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import {buildGeonamesSqlite} from '@hebcal/geo-sqlite';
|
|
4
4
|
import {pino} from 'pino';
|
|
5
|
+
import pretty from 'pino-pretty';
|
|
5
6
|
import minimist from 'minimist';
|
|
6
7
|
|
|
7
8
|
const argv = minimist(process.argv.slice(2), {
|
|
@@ -15,13 +16,14 @@ if (argv.help) {
|
|
|
15
16
|
process.exit(0);
|
|
16
17
|
}
|
|
17
18
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
19
|
+
// Run pino-pretty synchronously as the destination stream rather than via a
|
|
20
|
+
// worker-thread transport. The worker thread (thread-stream) can keep the
|
|
21
|
+
// process alive indefinitely on some platforms (e.g. GitHub Codespaces); a
|
|
22
|
+
// synchronous stream lets the process exit naturally once the work is done.
|
|
23
|
+
const logger = pino({level: argv.quiet ? 'warn' : 'info'}, pretty({
|
|
24
|
+
translateTime: 'SYS:standard',
|
|
25
|
+
ignore: 'pid,hostname',
|
|
26
|
+
}));
|
|
25
27
|
|
|
26
28
|
const args = argv._;
|
|
27
29
|
if (args.length !== 7) {
|
package/bin/make-test-dbs
CHANGED
|
@@ -7,7 +7,8 @@
|
|
|
7
7
|
|
|
8
8
|
import {buildGeonamesSqlite, makeZipsSqlite} from '@hebcal/geo-sqlite';
|
|
9
9
|
import {pino} from 'pino';
|
|
10
|
-
import
|
|
10
|
+
import pretty from 'pino-pretty';
|
|
11
|
+
import {statSync} from 'node:fs';
|
|
11
12
|
import path from 'path';
|
|
12
13
|
|
|
13
14
|
const outDir = process.cwd();
|
|
@@ -15,13 +16,14 @@ const repoDir = path.join(outDir, 'node_modules/@hebcal/geo-sqlite');
|
|
|
15
16
|
|
|
16
17
|
const population = process.env.POPULATION ? +process.env.POPULATION : 0;
|
|
17
18
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
19
|
+
// Run pino-pretty synchronously as the destination stream rather than via a
|
|
20
|
+
// worker-thread transport. The worker thread (thread-stream) can keep the
|
|
21
|
+
// process alive indefinitely on some platforms (e.g. GitHub Codespaces); a
|
|
22
|
+
// synchronous stream lets the process exit naturally once the work is done.
|
|
23
|
+
const logger = pino({level: 'info'}, pretty({
|
|
24
|
+
translateTime: 'SYS:standard',
|
|
25
|
+
ignore: 'pid,hostname',
|
|
26
|
+
}));
|
|
25
27
|
|
|
26
28
|
const geonamesDb = path.join(outDir, 'geonames.sqlite3');
|
|
27
29
|
const zipsDb = path.join(outDir, 'zips.sqlite3');
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Logger } from 'pino';
|
|
2
|
+
/**
|
|
3
|
+
* Options for {@link buildGeonamesSqlite}.
|
|
4
|
+
*/
|
|
5
|
+
export type BuildGeonamesSqliteOptions = {
|
|
6
|
+
/** Path to the output SQLite database file. */
|
|
7
|
+
dbFilename: string;
|
|
8
|
+
/** Path to countryInfo.txt from geonames.org. */
|
|
9
|
+
countryInfotxt: string;
|
|
10
|
+
/** Path to cities5000.txt (or similar) from geonames.org. */
|
|
11
|
+
cities5000txt: string;
|
|
12
|
+
/** Path to a TSV patch file with additional city rows. */
|
|
13
|
+
citiesPatch: string;
|
|
14
|
+
/** Path to admin1CodesASCII.txt from geonames.org. */
|
|
15
|
+
admin1CodesASCIItxt: string;
|
|
16
|
+
/** Path to IL.txt (Israel geonames) from geonames.org. */
|
|
17
|
+
ILtxt: string;
|
|
18
|
+
/** Path to IL alternate names file from geonames.org. */
|
|
19
|
+
ILalternate: string;
|
|
20
|
+
/** Logger instance (e.g. pino). */
|
|
21
|
+
logger: Logger;
|
|
22
|
+
/** Minimum population filter for PPL feature codes. */
|
|
23
|
+
population?: number;
|
|
24
|
+
};
|
|
25
|
+
/**
|
|
26
|
+
* Builds `geonames.sqlite3` from files downloaded from geonames.org
|
|
27
|
+
*/
|
|
28
|
+
export declare function buildGeonamesSqlite(opts: BuildGeonamesSqliteOptions): Promise<boolean>;
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
import events from 'node:events';
|
|
3
|
+
import fs from 'node:fs';
|
|
4
|
+
import readline from 'node:readline';
|
|
5
|
+
import { Locale } from '@hebcal/core';
|
|
6
|
+
const fcodeKeep = {
|
|
7
|
+
PPL: true, // populated place: a city, town, village, or other agglomeration of
|
|
8
|
+
PPLA: true, // seat of a first-order administrative division (PPLC takes precedence over PPLA)
|
|
9
|
+
PPLA2: true, // seat of a second-order administrative division
|
|
10
|
+
PPLA3: true, // seat of a third-order administrative division
|
|
11
|
+
PPLC: true, // capital of a political entity
|
|
12
|
+
// an area similar to a locality but with a small group of dwellings or other buildings
|
|
13
|
+
PPLG: true, // seat of government of a political entity
|
|
14
|
+
PPLL: true, // populated locality
|
|
15
|
+
// cities, towns, villages, or other agglomerations of buildings where people live and work
|
|
16
|
+
PPLS: true, // populated places
|
|
17
|
+
PPLX: true, // section of populated place
|
|
18
|
+
STLMT: true, // israeli settlement
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* Builds `geonames.sqlite3` from files downloaded from geonames.org
|
|
22
|
+
*/
|
|
23
|
+
export async function buildGeonamesSqlite(opts) {
|
|
24
|
+
const dbFilename = opts.dbFilename;
|
|
25
|
+
const countryInfotxt = opts.countryInfotxt;
|
|
26
|
+
const cities5000txt = opts.cities5000txt;
|
|
27
|
+
const citiesPatch = opts.citiesPatch;
|
|
28
|
+
const admin1CodesASCIItxt = opts.admin1CodesASCIItxt;
|
|
29
|
+
const ILtxt = opts.ILtxt;
|
|
30
|
+
const ILalternate = opts.ILalternate;
|
|
31
|
+
const logger = opts.logger;
|
|
32
|
+
logger.info(`Opening ${dbFilename}`);
|
|
33
|
+
const db = new DatabaseSync(dbFilename);
|
|
34
|
+
db.exec('PRAGMA journal_mode = MEMORY');
|
|
35
|
+
doSql(logger, db, 'DROP TABLE IF EXISTS country', `CREATE TABLE country (
|
|
36
|
+
ISO TEXT PRIMARY KEY,
|
|
37
|
+
ISO3 TEXT NOT NULL,
|
|
38
|
+
IsoNumeric TEXT NOT NULL,
|
|
39
|
+
fips TEXT NOT NULL,
|
|
40
|
+
Country TEXT NOT NULL,
|
|
41
|
+
Capital TEXT NOT NULL,
|
|
42
|
+
Area INT NOT NULL,
|
|
43
|
+
Population INT NOT NULL,
|
|
44
|
+
Continent TEXT NOT NULL,
|
|
45
|
+
tld TEXT NOT NULL,
|
|
46
|
+
CurrencyCode TEXT NOT NULL,
|
|
47
|
+
CurrencyName TEXT NOT NULL,
|
|
48
|
+
Phone TEXT NOT NULL,
|
|
49
|
+
PostalCodeFormat TEXT,
|
|
50
|
+
PostalCodeRegex TEXT,
|
|
51
|
+
Languages TEXT NOT NULL,
|
|
52
|
+
geonameid INT NOT NULL,
|
|
53
|
+
neighbours TEXT NOT NULL,
|
|
54
|
+
EquivalentFipsCode TEXT NOT NULL
|
|
55
|
+
);`);
|
|
56
|
+
await doFile(logger, db, countryInfotxt, 'country', 19);
|
|
57
|
+
doSql(logger, db, 'DROP TABLE IF EXISTS geoname', `CREATE TABLE geoname (
|
|
58
|
+
geonameid int PRIMARY KEY,
|
|
59
|
+
name nvarchar(200),
|
|
60
|
+
asciiname nvarchar(200),
|
|
61
|
+
alternatenames nvarchar(4000),
|
|
62
|
+
latitude decimal(18,15),
|
|
63
|
+
longitude decimal(18,15),
|
|
64
|
+
fclass nchar(1),
|
|
65
|
+
fcode nvarchar(10),
|
|
66
|
+
country nvarchar(2),
|
|
67
|
+
cc2 nvarchar(60),
|
|
68
|
+
admin1 nvarchar(20),
|
|
69
|
+
admin2 nvarchar(80),
|
|
70
|
+
admin3 nvarchar(20),
|
|
71
|
+
admin4 nvarchar(20),
|
|
72
|
+
population int,
|
|
73
|
+
elevation int,
|
|
74
|
+
gtopo30 int,
|
|
75
|
+
timezone nvarchar(40),
|
|
76
|
+
moddate date);`);
|
|
77
|
+
const truncateAlternateNames = (a) => {
|
|
78
|
+
a[3] = '';
|
|
79
|
+
return true;
|
|
80
|
+
};
|
|
81
|
+
const minPopulation = opts.population;
|
|
82
|
+
const citiesFilter = (a) => {
|
|
83
|
+
if (!a[17]) {
|
|
84
|
+
return false; // require a non-empty iana timezone id
|
|
85
|
+
}
|
|
86
|
+
const fcode = a[7];
|
|
87
|
+
logger.debug(`${a[0]} ${a[1]} ${fcode}`);
|
|
88
|
+
if (!fcodeKeep[fcode]) {
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
91
|
+
if (minPopulation) {
|
|
92
|
+
const population = Number(a[14]);
|
|
93
|
+
if (fcode === 'PPL' && population && population < minPopulation) {
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
a[3] = '';
|
|
98
|
+
return true;
|
|
99
|
+
};
|
|
100
|
+
await doFile(logger, db, cities5000txt, 'geoname', 19, citiesFilter);
|
|
101
|
+
await doFile(logger, db, citiesPatch, 'geoname', 19, truncateAlternateNames);
|
|
102
|
+
await doFile(logger, db, ILtxt, 'geoname', 19, a => {
|
|
103
|
+
a[3] = '';
|
|
104
|
+
return a[6] === 'P' && Boolean(fcodeKeep[a[7]]);
|
|
105
|
+
});
|
|
106
|
+
doSql(logger, db, 'DROP TABLE IF EXISTS admin1', `CREATE TABLE admin1 (
|
|
107
|
+
key TEXT PRIMARY KEY,
|
|
108
|
+
name nvarchar(200) NOT NULL,
|
|
109
|
+
asciiname nvarchar(200) NOT NULL,
|
|
110
|
+
geonameid int NOT NULL
|
|
111
|
+
);`);
|
|
112
|
+
await doFile(logger, db, admin1CodesASCIItxt, 'admin1', 4);
|
|
113
|
+
// fix inconsistencies with the USA capitol
|
|
114
|
+
doSql(logger, db, `UPDATE geoname
|
|
115
|
+
SET name = 'Washington, D.C.', asciiname = 'Washington, D.C.'
|
|
116
|
+
WHERE geonameid = 4140963;`, `UPDATE admin1
|
|
117
|
+
SET name = 'Washington, D.C.', asciiname = 'Washington, D.C.'
|
|
118
|
+
WHERE key = 'US.DC';`);
|
|
119
|
+
doSql(logger, db, 'DROP TABLE IF EXISTS alternatenames', `CREATE TABLE alternatenames (
|
|
120
|
+
id int PRIMARY KEY,
|
|
121
|
+
geonameid int NOT NULL,
|
|
122
|
+
isolanguage varchar(7),
|
|
123
|
+
name varchar(400),
|
|
124
|
+
isPreferredName tinyint,
|
|
125
|
+
isShortName tinyint,
|
|
126
|
+
isColloquial tinyint,
|
|
127
|
+
isHistoric tinyint,
|
|
128
|
+
periodFrom NULL,
|
|
129
|
+
periodTo NULL
|
|
130
|
+
);`);
|
|
131
|
+
await doFile(logger, db, ILalternate, 'alternatenames', 10, a => {
|
|
132
|
+
const firstchar = a[3][0];
|
|
133
|
+
if (a[2] === 'he' && (firstchar < 'א' || firstchar > 'ת')) {
|
|
134
|
+
a[2] = 'en';
|
|
135
|
+
}
|
|
136
|
+
if (a[2] === 'he' || a[2] === 'en') {
|
|
137
|
+
if (a[2] === 'he') {
|
|
138
|
+
a[3] = a[3].replaceAll('‘', '׳');
|
|
139
|
+
a[3] = a[3].replaceAll('’', '׳');
|
|
140
|
+
a[3] = a[3].replaceAll("'", '׳');
|
|
141
|
+
a[3] = Locale.hebrewStripNikkud(a[3]);
|
|
142
|
+
}
|
|
143
|
+
else {
|
|
144
|
+
a[3] = a[3].replaceAll('‘', "'");
|
|
145
|
+
a[3] = a[3].replaceAll('’', "'");
|
|
146
|
+
a[3] = a[3].replaceAll('Ḥ', 'Ch');
|
|
147
|
+
a[3] = a[3].replaceAll('H̱', 'Ch');
|
|
148
|
+
a[3] = a[3].replaceAll('ẖ', 'ch');
|
|
149
|
+
a[3] = a[3].replaceAll('Ẕ', 'Tz');
|
|
150
|
+
a[3] = a[3].replaceAll('ẕ', 'tz');
|
|
151
|
+
a[3] = a[3].replaceAll('ā', 'a');
|
|
152
|
+
a[3] = a[3].replaceAll('é', 'e');
|
|
153
|
+
}
|
|
154
|
+
return true;
|
|
155
|
+
}
|
|
156
|
+
return false;
|
|
157
|
+
});
|
|
158
|
+
// remove duplicates from alternatenames
|
|
159
|
+
doSql(logger, db, 'DROP TABLE IF EXISTS altnames', `CREATE TABLE altnames
|
|
160
|
+
AS SELECT geonameid, isolanguage, name
|
|
161
|
+
FROM alternatenames
|
|
162
|
+
GROUP BY 1, 2, 3
|
|
163
|
+
`);
|
|
164
|
+
doSql(logger, db,
|
|
165
|
+
// `update admin1 set name='',asciiname='' where key like 'PS.%';`,
|
|
166
|
+
// `update country set country = '' where iso = 'PS';`,
|
|
167
|
+
"delete from geoname where country = 'PS' and admin1 = 'GZ';", 'delete from geoname where geonameid = 7303419;', "update geoname set country = 'IL' where country = 'PS' and admin1 = 'WE';");
|
|
168
|
+
doSql(logger, db, 'DROP TABLE IF EXISTS geoname_fulltext', `CREATE VIRTUAL TABLE geoname_fulltext
|
|
169
|
+
USING fts5(geonameid UNINDEXED, longname, population, city, admin1, country);
|
|
170
|
+
`, 'DROP TABLE IF EXISTS geoname_non_ascii', `CREATE TABLE geoname_non_ascii AS
|
|
171
|
+
SELECT geonameid FROM geoname WHERE asciiname <> name`, `INSERT INTO geoname_fulltext
|
|
172
|
+
SELECT g.geonameid,
|
|
173
|
+
g.asciiname||', '||a.asciiname||', '||c.Country,
|
|
174
|
+
g.population,
|
|
175
|
+
g.asciiname,a.asciiname,c.Country
|
|
176
|
+
FROM geoname g, admin1 a, country c
|
|
177
|
+
WHERE g.country = c.ISO
|
|
178
|
+
AND g.country <> 'US'
|
|
179
|
+
AND g.country <> 'IL'
|
|
180
|
+
AND g.country <> 'GB'
|
|
181
|
+
AND g.country||'.'||g.admin1 = a.key
|
|
182
|
+
`, `INSERT INTO geoname_fulltext
|
|
183
|
+
SELECT g.geonameid,
|
|
184
|
+
g.asciiname||', '||a.asciiname||', USA',
|
|
185
|
+
g.population,
|
|
186
|
+
g.asciiname,a.asciiname,'United States'
|
|
187
|
+
FROM geoname g, admin1 a
|
|
188
|
+
WHERE g.country = 'US'
|
|
189
|
+
AND g.country||'.'||g.admin1 = a.key
|
|
190
|
+
`, `INSERT INTO geoname_fulltext
|
|
191
|
+
SELECT g.geonameid,
|
|
192
|
+
g.asciiname||', '||a.asciiname||', UK',
|
|
193
|
+
g.population,
|
|
194
|
+
g.asciiname,a.asciiname,'UK'
|
|
195
|
+
FROM geoname g, admin1 a
|
|
196
|
+
WHERE g.country = 'GB'
|
|
197
|
+
AND g.country||'.'||g.admin1 = a.key
|
|
198
|
+
`, `INSERT INTO geoname_fulltext
|
|
199
|
+
SELECT g.geonameid,
|
|
200
|
+
g.asciiname||', Israel',
|
|
201
|
+
g.population,
|
|
202
|
+
g.asciiname,NULL,'Israel'
|
|
203
|
+
FROM geoname g
|
|
204
|
+
WHERE g.country = 'IL'
|
|
205
|
+
`, `INSERT INTO geoname_fulltext
|
|
206
|
+
SELECT g.geonameid,
|
|
207
|
+
g.asciiname||', '||c.Country,
|
|
208
|
+
g.population,
|
|
209
|
+
g.asciiname,NULL,c.Country
|
|
210
|
+
FROM geoname g, country c
|
|
211
|
+
WHERE g.country = c.ISO
|
|
212
|
+
AND (g.admin1 = '' OR g.admin1 = '00')
|
|
213
|
+
`, `INSERT INTO geoname_fulltext
|
|
214
|
+
SELECT g.geonameid,
|
|
215
|
+
g.name||', '||a.name||', '||c.Country,
|
|
216
|
+
g.population,
|
|
217
|
+
g.name,a.name,c.Country
|
|
218
|
+
FROM geoname_non_ascii gna, geoname g, admin1 a, country c
|
|
219
|
+
WHERE gna.geonameid = g.geonameid
|
|
220
|
+
AND g.country = c.ISO
|
|
221
|
+
AND g.country||'.'||g.admin1 = a.key
|
|
222
|
+
`, `INSERT INTO geoname_fulltext
|
|
223
|
+
SELECT g.geonameid,
|
|
224
|
+
alt.name||', ישראל',
|
|
225
|
+
g.population,
|
|
226
|
+
alt.name,NULL,'ישראל'
|
|
227
|
+
FROM geoname g, altnames alt
|
|
228
|
+
WHERE g.country = 'IL'
|
|
229
|
+
AND alt.isolanguage = 'he'
|
|
230
|
+
AND g.geonameid = alt.geonameid
|
|
231
|
+
`, `INSERT INTO geoname_fulltext
|
|
232
|
+
SELECT g.geonameid,
|
|
233
|
+
alt.name||', Israel',
|
|
234
|
+
g.population,
|
|
235
|
+
alt.name,NULL,'Israel'
|
|
236
|
+
FROM geoname g, altnames alt
|
|
237
|
+
WHERE g.country = 'IL'
|
|
238
|
+
AND alt.isolanguage = 'en'
|
|
239
|
+
AND g.geonameid = alt.geonameid
|
|
240
|
+
`, 'VACUUM');
|
|
241
|
+
return new Promise((resolve, reject) => {
|
|
242
|
+
try {
|
|
243
|
+
logger.info(`Closing ${dbFilename}`);
|
|
244
|
+
db.close();
|
|
245
|
+
logger.info('buildGeonamesSqlite finished');
|
|
246
|
+
resolve(true);
|
|
247
|
+
}
|
|
248
|
+
catch (err) {
|
|
249
|
+
logger.error(err);
|
|
250
|
+
reject(err);
|
|
251
|
+
}
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
function doSql(logger, db, ...sqls) {
|
|
255
|
+
for (const sql of sqls) {
|
|
256
|
+
logger.info(sql);
|
|
257
|
+
db.exec(sql);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
async function doFile(logger, db, infile, tableName, expectedFields, callback) {
|
|
261
|
+
return new Promise((resolve, reject) => {
|
|
262
|
+
logger.info(`${infile} => ${tableName}`);
|
|
263
|
+
db.exec('BEGIN');
|
|
264
|
+
let sql = `INSERT OR IGNORE INTO ${tableName} VALUES (?`;
|
|
265
|
+
for (let i = 0; i < expectedFields - 1; i++) {
|
|
266
|
+
sql += ',?';
|
|
267
|
+
}
|
|
268
|
+
sql += ')';
|
|
269
|
+
logger.info(sql);
|
|
270
|
+
const stmt = db.prepare(sql);
|
|
271
|
+
try {
|
|
272
|
+
const rl = readline.createInterface({
|
|
273
|
+
input: fs.createReadStream(infile),
|
|
274
|
+
crlfDelay: Infinity,
|
|
275
|
+
});
|
|
276
|
+
let num = 0;
|
|
277
|
+
let accepted = 0;
|
|
278
|
+
rl.on('line', line => {
|
|
279
|
+
num++;
|
|
280
|
+
if (line.startsWith('#')) {
|
|
281
|
+
return;
|
|
282
|
+
}
|
|
283
|
+
const a = line.split('\t');
|
|
284
|
+
if (a.length !== expectedFields) {
|
|
285
|
+
logger.warn(`${infile}:${num}: got ${a.length} fields (expected ${expectedFields})`);
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
if (callback) {
|
|
289
|
+
const accept = callback(a);
|
|
290
|
+
if (!accept) {
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
stmt.run(...a);
|
|
295
|
+
accepted++;
|
|
296
|
+
});
|
|
297
|
+
rl.on('close', () => {
|
|
298
|
+
logger.info(`Inserted ${accepted} / ${num} into ${tableName} from ${infile}`);
|
|
299
|
+
db.exec('COMMIT');
|
|
300
|
+
});
|
|
301
|
+
return resolve(events.once(rl, 'close'));
|
|
302
|
+
}
|
|
303
|
+
catch (err) {
|
|
304
|
+
logger.error(err);
|
|
305
|
+
return reject(err);
|
|
306
|
+
}
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
//# sourceMappingURL=build-geonames-sqlite.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"build-geonames-sqlite.js","sourceRoot":"","sources":["../src/build-geonames-sqlite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,aAAa,CAAC;AACzC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,QAAQ,MAAM,eAAe,CAAC;AACrC,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAC;AA2BpC,MAAM,SAAS,GAA4B;IACzC,GAAG,EAAE,IAAI,EAAE,oEAAoE;IAC/E,IAAI,EAAE,IAAI,EAAE,kFAAkF;IAC9F,KAAK,EAAE,IAAI,EAAE,iDAAiD;IAC9D,KAAK,EAAE,IAAI,EAAE,gDAAgD;IAC7D,IAAI,EAAE,IAAI,EAAE,gCAAgC;IAC5C,uFAAuF;IACvF,IAAI,EAAE,IAAI,EAAE,2CAA2C;IACvD,IAAI,EAAE,IAAI,EAAE,qBAAqB;IACjC,2FAA2F;IAC3F,IAAI,EAAE,IAAI,EAAE,mBAAmB;IAC/B,IAAI,EAAE,IAAI,EAAE,6BAA6B;IACzC,KAAK,EAAE,IAAI,EAAE,qBAAqB;CACnC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAgC;IAEhC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;IACnC,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;IAC3C,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC;IACzC,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;IACrC,MAAM,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC;IACrD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACzB,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;IACrC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC3B,MAAM,CAAC,IAAI,CAAC,WAAW,UAAU,EAAE,CAAC,CAAC;IACrC,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,UAAU,CAAC,CAAC;IACxC,EAAE,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAExC,KAAK,CACH,MAAM,EACN,EAAE,EACF,8BAA8B,EAE9B;;;;;;;;;;;;;;;;;;;;OAoBG,CACJ,CAAC;IACF,MAAM,MAAM,CAAC,MAAM,EAAE,EAAE,EAAE,cAAc,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;IAExD,KAAK,CACH,MAAM,EACN,EAAE,EACF,8BAA8B,EAE9B;;;;;;;;;;;;;;;;;;;qBAmBiB,CAClB,CAAC;IAEF,MAAM,sBAAsB,GAAG,CAAC,CAAW,EAAW,EAAE;QACtD,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACV,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC;IACtC,MAAM,YAAY,GAAG,CAAC,CAAW,EAAW,EAAE;QAC5C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;YACX,OAAO,KAAK,CAAC,CAAC,uCAAuC;QACvD,CAAC;QACD,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACjC,IAAI,KAAK,KAAK,KAAK,IAAI,UAAU,IAAI,UAAU,GAAG,aAAa,EAAE,CAAC;gBAChE,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QACD,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACV,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,MAAM,MAAM,CAAC,MAAM,EAAE,EAAE,EAAE,aAAa,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,CAAC,CAAC;IACrE,MAAM,MAAM,CAAC,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,EAAE,sBAAsB,CAAC,CAAC;IAC7E,MAAM,MAAM,CAAC,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE;QACjD,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACV,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,KAAK,CACH,MAAM,EACN,EAAE,EACF,6BAA6B,EAE7B;;;;;SAKK,CACN,CAAC;IAEF,MAAM,MAAM,CAAC,MAAM,EAAE,EAAE,EAAE,mBAAmB,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;IAE3D,2CAA2C;IAC3C,KAAK,CACH,MAAM,EACN,EAAE,EACF;;iCAE6B,EAE7B;;2BAEuB,CACxB,CAAC;IAEF,KAAK,CACH,MAAM,EACN,EAAE,EACF,qCAAqC,EAErC;;;;;;;;;;;OAWG,CACJ,CAAC;IAEF,MAAM,MAAM,CAAC,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE,gBAAgB,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE;QAC9D,MAAM,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS,GAAG,GAAG,IAAI,SAAS,GAAG,GAAG,CAAC,EAAE,CAAC;YAC1D,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACnC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAClB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBACjC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBACjC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBACjC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBACjC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBACjC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBAClC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBACnC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBAClC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBAClC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBAClC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBACjC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACnC,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;IAEH,wCAAwC;IACxC,KAAK,CACH,MAAM,EACN,EAAE,EACF,+BAA+B,EAE/B;;;;KAIC,CACF,CAAC;IAEF,KAAK,CACH,MAAM,EACN,EAAE;IACF,mEAAmE;IACnE,uDAAuD;IACvD,6DAA6D,EAC7D,gDAAgD,EAChD,2EAA2E,CAC5E,CAAC;IAEF,KAAK,CACH,MAAM,EACN,EAAE,EACF,uCAAuC,EAEvC;;KAEC,EAED,wCAAwC,EAExC;4DACwD,EAExD;;;;;;;;;;;OAWG,EAEH;;;;;;;;OAQG,EAEH;;;;;;;;OAQG,EAEH;;;;;;;OAOG,EAEH;;;;;;;;OAQG,EAEH;;;;;;;;;OASG,EAEH;;;;;;;;;OASG,EAEH;;;;;;;;;OASG,EAEH,QAAQ,CACT,CAAC;IAEF,OAAO,IAAI,OAAO,CAAU,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC9C,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,WAAW,UAAU,EAAE,CAAC,CAAC;YACrC,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;YAC5C,OAAO,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClB,MAAM,CAAC,GAAG,CAAC,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,KAAK,CAAC,MAAc,EAAE,EAAgB,EAAE,GAAG,IAAc;IAChE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,MAAM,CACnB,MAAc,EACd,EAAgB,EAChB,MAAc,EACd,SAAiB,EACjB,cAAsB,EACtB,QAAmC;IAEnC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;QACzC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjB,IAAI,GAAG,GAAG,yBAAyB,SAAS,YAAY,CAAC;QACzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,GAAG,IAAI,IAAI,CAAC;QACd,CAAC;QACD,GAAG,IAAI,GAAG,CAAC;QACX,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;gBAClC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC;gBAClC,SAAS,EAAE,QAAQ;aACpB,CAAC,CAAC;YACH,IAAI,GAAG,GAAG,CAAC,CAAC;YACZ,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE;gBACnB,GAAG,EAAE,CAAC;gBACN,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzB,OAAO;gBACT,CAAC;gBACD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC3B,IAAI,CAAC,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;oBAChC,MAAM,CAAC,IAAI,CACT,GAAG,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,MAAM,qBAAqB,cAAc,GAAG,CACxE,CAAC;oBACF,OAAO;gBACT,CAAC;gBACD,IAAI,QAAQ,EAAE,CAAC;oBACb,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;oBAC3B,IAAI,CAAC,MAAM,EAAE,CAAC;wBACZ,OAAO;oBACT,CAAC;gBACH,CAAC;gBACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBACf,QAAQ,EAAE,CAAC;YACb,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBAClB,MAAM,CAAC,IAAI,CACT,YAAY,QAAQ,MAAM,GAAG,SAAS,SAAS,SAAS,MAAM,EAAE,CACjE,CAAC;gBACF,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACpB,CAAC,CAAC,CAAC;YAEH,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
|