@hebcal/geo-sqlite 3.4.2 → 3.6.0

package/dist/index.js

@@ -1,4 +1,4 @@
-/*! @hebcal/geo-sqlite v3.4.2 */
+/*! @hebcal/geo-sqlite v3.6.0 */
 'use strict';
 
 Object.defineProperty(exports, '__esModule', { value: true });
@@ -55,6 +55,7 @@ FROM ZIPCodes_Primary`;
 const ZIP_COMPLETE_SQL = `SELECT ZipCode,CityMixedCase,State,Latitude,Longitude,TimeZone,DayLightSaving
 FROM ZIPCodes_Primary
 WHERE ZipCode LIKE ?
+ORDER BY Population DESC
 LIMIT 10`;
 const GEONAME_COMPLETE_SQL = `SELECT geonameid, asciiname, admin1, country,
 population, latitude, longitude, timezone
@@ -74,11 +75,11 @@ class GeoDb {
   constructor(logger, zipsFilename, geonamesFilename) {
     this.logger = logger;
     if (logger) logger.info(`GeoDb: opening ${zipsFilename}...`);
-    this.zipsDb = new Database__default['default'](zipsFilename, {
+    this.zipsDb = new Database__default["default"](zipsFilename, {
       fileMustExist: true
     });
     if (logger) logger.info(`GeoDb: opening ${geonamesFilename}...`);
-    this.geonamesDb = new Database__default['default'](geonamesFilename, {
+    this.geonamesDb = new Database__default["default"](geonamesFilename, {
       fileMustExist: true
     });
     this.zipStmt = this.zipsDb.prepare(ZIPCODE_SQL);
@@ -244,6 +245,7 @@ class GeoDb {
           latitude: res.Latitude,
           longitude: res.Longitude,
           timezone: core.Location.getUsaTzid(res.State, res.TimeZone, res.DayLightSaving),
+          population: res.Population,
           geo: 'zip'
         };
         return obj;
@@ -316,13 +318,6 @@ class GeoDb {
 
 }
 
-const logger = pino__default['default']({
-  // level: argv.quiet ? 'warn' : 'info',
-  prettyPrint: {
-    translateTime: true,
-    ignore: 'pid,hostname'
-  }
-});
 /**
  * Builds `geonames.sqlite3` from files downloaded from geonames.org
  * @param {string} dbFilename
@@ -334,9 +329,16 @@ const logger = pino__default['default']({
  */
 
 async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, citiesPatch, admin1CodesASCIItxt, ILtxt) {
-  const db = new Database__default['default'](dbFilename);
+  const logger = pino__default["default"]({
+    // level: argv.quiet ? 'warn' : 'info',
+    prettyPrint: {
+      translateTime: true,
+      ignore: 'pid,hostname'
+    }
+  });
+  const db = new Database__default["default"](dbFilename);
   db.pragma('journal_mode = MEMORY');
-  doSql(db, `DROP TABLE IF EXISTS country`, `CREATE TABLE country (
+  doSql(logger, db, `DROP TABLE IF EXISTS country`, `CREATE TABLE country (
       ISO TEXT PRIMARY KEY,
       ISO3 TEXT NOT NULL,
       IsoNumeric TEXT NOT NULL,
@@ -357,8 +359,8 @@ async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, ci
       neighbours TEXT NOT NULL,
       EquivalentFipsCode TEXT NOT NULL
     );`);
-  await doFile(db, countryInfotxt, 'country', 19);
-  doSql(db, `DROP TABLE IF EXISTS geoname`, `CREATE TABLE geoname (
+  await doFile(logger, db, countryInfotxt, 'country', 19);
+  doSql(logger, db, `DROP TABLE IF EXISTS geoname`, `CREATE TABLE geoname (
       geonameid int PRIMARY KEY,
       name nvarchar(200),
       asciiname nvarchar(200),
@@ -378,28 +380,28 @@ async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, ci
       gtopo30 int,
       timezone nvarchar(40),
       moddate date);`);
-  await doFile(db, cities5000txt, 'geoname', 19);
-  await doFile(db, citiesPatch, 'geoname', 19);
-  await doFile(db, ILtxt, 'geoname', 19, a => {
+  await doFile(logger, db, cities5000txt, 'geoname', 19);
+  await doFile(logger, db, citiesPatch, 'geoname', 19);
+  await doFile(logger, db, ILtxt, 'geoname', 19, a => {
     return a[6] == 'P' && (a[7] == 'PPL' || a[7] == 'STLMT');
   });
-  doSql(db, `DROP TABLE IF EXISTS admin1`, `CREATE TABLE admin1 (
+  doSql(logger, db, `DROP TABLE IF EXISTS admin1`, `CREATE TABLE admin1 (
       key TEXT PRIMARY KEY,
       name nvarchar(200) NOT NULL,
       asciiname nvarchar(200) NOT NULL,
       geonameid int NOT NULL
       );`);
-  await doFile(db, admin1CodesASCIItxt, 'admin1', 4); // fix inconsistencies with the USA capitol
+  await doFile(logger, db, admin1CodesASCIItxt, 'admin1', 4); // fix inconsistencies with the USA capitol
 
-  doSql(db, `UPDATE geoname
+  doSql(logger, db, `UPDATE geoname
       SET name = 'Washington, D.C.', asciiname = 'Washington, D.C.'
       WHERE geonameid = 4140963;`, `UPDATE admin1
       SET name = 'Washington, D.C.', asciiname = 'Washington, D.C.'
       WHERE key = 'US.DC';`);
-  doSql(db, `DROP TABLE IF EXISTS geoname_he`, `CREATE TABLE geoname_he AS SELECT * FROM geoname LIMIT 0`);
-  await doFile(db, ILtxt, 'geoname_he', 19, filterPlacesHebrew);
-  doSql(db, `update admin1 set name='',asciiname='' where key like 'PS.%';`, `update country set country = '' where iso = 'PS';`, `delete from geoname where geonameid = 7303419;`);
-  doSql(db, `DROP TABLE IF EXISTS geoname_fulltext`, `CREATE VIRTUAL TABLE geoname_fulltext
+  doSql(logger, db, `DROP TABLE IF EXISTS geoname_he`, `CREATE TABLE geoname_he AS SELECT * FROM geoname LIMIT 0`);
+  await doFile(logger, db, ILtxt, 'geoname_he', 19, filterPlacesHebrew);
+  doSql(logger, db, `update admin1 set name='',asciiname='' where key like 'PS.%';`, `update country set country = '' where iso = 'PS';`, `delete from geoname where geonameid = 7303419;`);
+  doSql(logger, db, `DROP TABLE IF EXISTS geoname_fulltext`, `CREATE VIRTUAL TABLE geoname_fulltext
       USING fts3(geonameid int, longname text,
       asciiname text, admin1 text, country text,
       population int, latitude real, longitude real, timezone text
@@ -454,7 +456,7 @@ function filterPlacesHebrew(a) {
     const firstchar = name[0];
 
     if (firstchar >= '\u05D0' && firstchar <= '\u05EA') {
-      a[1] = name; // replace 'name' field with Hebrew
+      a[1] = core.Locale.hebrewStripNikkud(name); // replace 'name' field with Hebrew
 
       return true;
     }
@@ -463,18 +465,20 @@ function filterPlacesHebrew(a) {
   return false;
 }
 /**
+ * @param {pino.Logger} logger
  * @param {Database} db
  * @param  {...string} sqls
  */
 
 
-function doSql(db, ...sqls) {
+function doSql(logger, db, ...sqls) {
   for (let i = 0; i < sqls.length; i++) {
     logger.info(sqls[i]);
     db.exec(sqls[i]);
   }
 }
 /**
+ * @param {pino.Logger} logger
  * @param {Database} db
  * @param {string} infile
  * @param {string} tableName
@@ -483,7 +487,7 @@ function doSql(db, ...sqls) {
  */
 
 
-async function doFile(db, infile, tableName, expectedFields, callback) {
+async function doFile(logger, db, infile, tableName, expectedFields, callback) {
   logger.info(`${infile} => ${tableName}`);
   db.exec('BEGIN');
   let sql = `INSERT OR IGNORE INTO ${tableName} VALUES (?`;
@@ -497,8 +501,8 @@ async function doFile(db, infile, tableName, expectedFields, callback) {
   const stmt = db.prepare(sql);
   return new Promise((resolve, reject) => {
     try {
-      const rl = readline__default['default'].createInterface({
-        input: fs__default['default'].createReadStream(infile),
+      const rl = readline__default["default"].createInterface({
+        input: fs__default["default"].createReadStream(infile),
         crlfDelay: Infinity
       });
       let num = 0;
@@ -532,7 +536,7 @@ async function doFile(db, infile, tableName, expectedFields, callback) {
         logger.info(`Inserted ${accepted} / ${num} into ${tableName} from ${infile}`);
         db.exec('COMMIT');
       });
-      return resolve(events__default['default'].once(rl, 'close'));
+      return resolve(events__default["default"].once(rl, 'close'));
     } catch (err) {
       logger.error(err);
       return reject(err);

package/dist/index.mjs

@@ -1,6 +1,6 @@
-/*! @hebcal/geo-sqlite v3.4.2 */
+/*! @hebcal/geo-sqlite v3.6.0 */
 import Database from 'better-sqlite3';
-import { Location } from '@hebcal/core';
+import { Location, Locale } from '@hebcal/core';
 import pino from 'pino';
 import events from 'events';
 import fs from 'fs';
@@ -43,6 +43,7 @@ FROM ZIPCodes_Primary`;
 const ZIP_COMPLETE_SQL = `SELECT ZipCode,CityMixedCase,State,Latitude,Longitude,TimeZone,DayLightSaving
 FROM ZIPCodes_Primary
 WHERE ZipCode LIKE ?
+ORDER BY Population DESC
 LIMIT 10`;
 const GEONAME_COMPLETE_SQL = `SELECT geonameid, asciiname, admin1, country,
 population, latitude, longitude, timezone
@@ -232,6 +233,7 @@ class GeoDb {
           latitude: res.Latitude,
           longitude: res.Longitude,
           timezone: Location.getUsaTzid(res.State, res.TimeZone, res.DayLightSaving),
+          population: res.Population,
           geo: 'zip'
         };
         return obj;
@@ -304,13 +306,6 @@ class GeoDb {
 
 }
 
-const logger = pino({
-  // level: argv.quiet ? 'warn' : 'info',
-  prettyPrint: {
-    translateTime: true,
-    ignore: 'pid,hostname'
-  }
-});
 /**
  * Builds `geonames.sqlite3` from files downloaded from geonames.org
  * @param {string} dbFilename
@@ -322,9 +317,16 @@ const logger = pino({
  */
 
 async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, citiesPatch, admin1CodesASCIItxt, ILtxt) {
+  const logger = pino({
+    // level: argv.quiet ? 'warn' : 'info',
+    prettyPrint: {
+      translateTime: true,
+      ignore: 'pid,hostname'
+    }
+  });
   const db = new Database(dbFilename);
   db.pragma('journal_mode = MEMORY');
-  doSql(db, `DROP TABLE IF EXISTS country`, `CREATE TABLE country (
+  doSql(logger, db, `DROP TABLE IF EXISTS country`, `CREATE TABLE country (
       ISO TEXT PRIMARY KEY,
       ISO3 TEXT NOT NULL,
       IsoNumeric TEXT NOT NULL,
@@ -345,8 +347,8 @@ async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, ci
       neighbours TEXT NOT NULL,
       EquivalentFipsCode TEXT NOT NULL
     );`);
-  await doFile(db, countryInfotxt, 'country', 19);
-  doSql(db, `DROP TABLE IF EXISTS geoname`, `CREATE TABLE geoname (
+  await doFile(logger, db, countryInfotxt, 'country', 19);
+  doSql(logger, db, `DROP TABLE IF EXISTS geoname`, `CREATE TABLE geoname (
       geonameid int PRIMARY KEY,
       name nvarchar(200),
       asciiname nvarchar(200),
@@ -366,28 +368,28 @@ async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, ci
       gtopo30 int,
       timezone nvarchar(40),
       moddate date);`);
-  await doFile(db, cities5000txt, 'geoname', 19);
-  await doFile(db, citiesPatch, 'geoname', 19);
-  await doFile(db, ILtxt, 'geoname', 19, a => {
+  await doFile(logger, db, cities5000txt, 'geoname', 19);
+  await doFile(logger, db, citiesPatch, 'geoname', 19);
+  await doFile(logger, db, ILtxt, 'geoname', 19, a => {
     return a[6] == 'P' && (a[7] == 'PPL' || a[7] == 'STLMT');
   });
-  doSql(db, `DROP TABLE IF EXISTS admin1`, `CREATE TABLE admin1 (
+  doSql(logger, db, `DROP TABLE IF EXISTS admin1`, `CREATE TABLE admin1 (
       key TEXT PRIMARY KEY,
       name nvarchar(200) NOT NULL,
       asciiname nvarchar(200) NOT NULL,
       geonameid int NOT NULL
       );`);
-  await doFile(db, admin1CodesASCIItxt, 'admin1', 4); // fix inconsistencies with the USA capitol
+  await doFile(logger, db, admin1CodesASCIItxt, 'admin1', 4); // fix inconsistencies with the USA capitol
 
-  doSql(db, `UPDATE geoname
+  doSql(logger, db, `UPDATE geoname
       SET name = 'Washington, D.C.', asciiname = 'Washington, D.C.'
       WHERE geonameid = 4140963;`, `UPDATE admin1
       SET name = 'Washington, D.C.', asciiname = 'Washington, D.C.'
       WHERE key = 'US.DC';`);
-  doSql(db, `DROP TABLE IF EXISTS geoname_he`, `CREATE TABLE geoname_he AS SELECT * FROM geoname LIMIT 0`);
-  await doFile(db, ILtxt, 'geoname_he', 19, filterPlacesHebrew);
-  doSql(db, `update admin1 set name='',asciiname='' where key like 'PS.%';`, `update country set country = '' where iso = 'PS';`, `delete from geoname where geonameid = 7303419;`);
-  doSql(db, `DROP TABLE IF EXISTS geoname_fulltext`, `CREATE VIRTUAL TABLE geoname_fulltext
+  doSql(logger, db, `DROP TABLE IF EXISTS geoname_he`, `CREATE TABLE geoname_he AS SELECT * FROM geoname LIMIT 0`);
+  await doFile(logger, db, ILtxt, 'geoname_he', 19, filterPlacesHebrew);
+  doSql(logger, db, `update admin1 set name='',asciiname='' where key like 'PS.%';`, `update country set country = '' where iso = 'PS';`, `delete from geoname where geonameid = 7303419;`);
+  doSql(logger, db, `DROP TABLE IF EXISTS geoname_fulltext`, `CREATE VIRTUAL TABLE geoname_fulltext
       USING fts3(geonameid int, longname text,
       asciiname text, admin1 text, country text,
       population int, latitude real, longitude real, timezone text
@@ -442,7 +444,7 @@ function filterPlacesHebrew(a) {
     const firstchar = name[0];
 
     if (firstchar >= '\u05D0' && firstchar <= '\u05EA') {
-      a[1] = name; // replace 'name' field with Hebrew
+      a[1] = Locale.hebrewStripNikkud(name); // replace 'name' field with Hebrew
 
       return true;
     }
@@ -451,18 +453,20 @@ function filterPlacesHebrew(a) {
   return false;
 }
 /**
+ * @param {pino.Logger} logger
  * @param {Database} db
  * @param  {...string} sqls
  */
 
 
-function doSql(db, ...sqls) {
+function doSql(logger, db, ...sqls) {
   for (let i = 0; i < sqls.length; i++) {
     logger.info(sqls[i]);
     db.exec(sqls[i]);
   }
 }
 /**
+ * @param {pino.Logger} logger
  * @param {Database} db
  * @param {string} infile
  * @param {string} tableName
@@ -471,7 +475,7 @@ function doSql(db, ...sqls) {
  */
 
 
-async function doFile(db, infile, tableName, expectedFields, callback) {
+async function doFile(logger, db, infile, tableName, expectedFields, callback) {
   logger.info(`${infile} => ${tableName}`);
   db.exec('BEGIN');
   let sql = `INSERT OR IGNORE INTO ${tableName} VALUES (?`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hebcal/geo-sqlite",
-  "version": "3.4.2",
+  "version": "3.6.0",
   "author": "Michael J. Radwin (https://github.com/mjradwin)",
   "keywords": [
     "hebcal"
@@ -28,10 +28,10 @@
     "geo-sqlite.d.ts"
   ],
   "dependencies": {
-    "@hebcal/core": "^3.21.1",
-    "better-sqlite3": "^7.4.3",
-    "pino": "^7.0.0-rc.1",
-    "pino-pretty": "^5.1.2"
+    "@hebcal/core": "^3.32.0",
+    "better-sqlite3": "^7.4.6",
+    "pino": "^7.6.2",
+    "pino-pretty": "^7.3.0"
   },
   "scripts": {
     "build": "rollup -c",
@@ -60,19 +60,19 @@
   "license": "BSD-2-Clause",
   "devDependencies": {
     "@ava/babel": "^2.0.0",
-    "@babel/core": "^7.14.8",
+    "@babel/core": "^7.16.7",
     "@babel/polyfill": "^7.12.1",
-    "@babel/preset-env": "^7.14.8",
-    "@babel/register": "^7.14.5",
+    "@babel/preset-env": "^7.16.7",
+    "@babel/register": "^7.16.7",
     "@rollup/plugin-babel": "^5.3.0",
-    "@rollup/plugin-commonjs": "^19.0.1",
+    "@rollup/plugin-commonjs": "^21.0.1",
     "@rollup/plugin-json": "^4.1.0",
-    "@rollup/plugin-node-resolve": "^13.0.2",
+    "@rollup/plugin-node-resolve": "^13.1.3",
     "ava": "^3.15.0",
-    "eslint": "^7.31.0",
+    "eslint": "^8.6.0",
     "eslint-config-google": "^0.14.0",
     "jsdoc": "^3.6.7",
-    "jsdoc-to-markdown": "^7.0.1",
-    "rollup": "^2.53.3"
+    "jsdoc-to-markdown": "^7.1.0",
+    "rollup": "^2.63.0"
   }
 }

package/zips-dummy.sql

@@ -6,5 +6,5 @@ CREATE TABLE ZIPCodes_Primary (
   Longitude decimal(12, 6),
   TimeZone char(2) NULL,
   DayLightSaving char(1) NULL,
-  Elevation int NULL
+  Population int NULL
 );