@hebcal/geo-sqlite 3.4.1 → 3.5.0

package/dist/index.js

@@ -1,4 +1,4 @@
-/*! @hebcal/geo-sqlite v3.4.1 */
+/*! @hebcal/geo-sqlite v3.5.0 */
 'use strict';
 
 Object.defineProperty(exports, '__esModule', { value: true });
@@ -74,11 +74,11 @@ class GeoDb {
   constructor(logger, zipsFilename, geonamesFilename) {
     this.logger = logger;
     if (logger) logger.info(`GeoDb: opening ${zipsFilename}...`);
-    this.zipsDb = new Database__default['default'](zipsFilename, {
+    this.zipsDb = new Database__default["default"](zipsFilename, {
       fileMustExist: true
     });
     if (logger) logger.info(`GeoDb: opening ${geonamesFilename}...`);
-    this.geonamesDb = new Database__default['default'](geonamesFilename, {
+    this.geonamesDb = new Database__default["default"](geonamesFilename, {
       fileMustExist: true
     });
     this.zipStmt = this.zipsDb.prepare(ZIPCODE_SQL);
@@ -253,6 +253,7 @@ class GeoDb {
         this.geonamesCompStmt = this.geonamesDb.prepare(GEONAME_COMPLETE_SQL);
       }
 
+      qraw = qraw.replace(/\"/g, '""');
       return this.geonamesCompStmt.all(`"${qraw}*"`).map(res => {
         const country = res.country || '';
         const admin1 = res.admin1 || '';
@@ -315,13 +316,6 @@ class GeoDb {
 
 }
 
-const logger = pino__default['default']({
-  // level: argv.quiet ? 'warn' : 'info',
-  prettyPrint: {
-    translateTime: true,
-    ignore: 'pid,hostname'
-  }
-});
 /**
  * Builds `geonames.sqlite3` from files downloaded from geonames.org
  * @param {string} dbFilename
@@ -333,9 +327,16 @@ const logger = pino__default['default']({
  */
 
 async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, citiesPatch, admin1CodesASCIItxt, ILtxt) {
-  const db = new Database__default['default'](dbFilename);
+  const logger = pino__default["default"]({
+    // level: argv.quiet ? 'warn' : 'info',
+    prettyPrint: {
+      translateTime: true,
+      ignore: 'pid,hostname'
+    }
+  });
+  const db = new Database__default["default"](dbFilename);
   db.pragma('journal_mode = MEMORY');
-  doSql(db, `DROP TABLE IF EXISTS country`, `CREATE TABLE country (
+  doSql(logger, db, `DROP TABLE IF EXISTS country`, `CREATE TABLE country (
       ISO TEXT PRIMARY KEY,
       ISO3 TEXT NOT NULL,
       IsoNumeric TEXT NOT NULL,
@@ -356,8 +357,8 @@ async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, ci
       neighbours TEXT NOT NULL,
       EquivalentFipsCode TEXT NOT NULL
     );`);
-  await doFile(db, countryInfotxt, 'country', 19);
-  doSql(db, `DROP TABLE IF EXISTS geoname`, `CREATE TABLE geoname (
+  await doFile(logger, db, countryInfotxt, 'country', 19);
+  doSql(logger, db, `DROP TABLE IF EXISTS geoname`, `CREATE TABLE geoname (
       geonameid int PRIMARY KEY,
       name nvarchar(200),
       asciiname nvarchar(200),
@@ -377,28 +378,28 @@ async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, ci
       gtopo30 int,
       timezone nvarchar(40),
       moddate date);`);
-  await doFile(db, cities5000txt, 'geoname', 19);
-  await doFile(db, citiesPatch, 'geoname', 19);
-  await doFile(db, ILtxt, 'geoname', 19, a => {
+  await doFile(logger, db, cities5000txt, 'geoname', 19);
+  await doFile(logger, db, citiesPatch, 'geoname', 19);
+  await doFile(logger, db, ILtxt, 'geoname', 19, a => {
     return a[6] == 'P' && (a[7] == 'PPL' || a[7] == 'STLMT');
   });
-  doSql(db, `DROP TABLE IF EXISTS admin1`, `CREATE TABLE admin1 (
+  doSql(logger, db, `DROP TABLE IF EXISTS admin1`, `CREATE TABLE admin1 (
       key TEXT PRIMARY KEY,
       name nvarchar(200) NOT NULL,
       asciiname nvarchar(200) NOT NULL,
       geonameid int NOT NULL
       );`);
-  await doFile(db, admin1CodesASCIItxt, 'admin1', 4); // fix inconsistencies with the USA capitol
+  await doFile(logger, db, admin1CodesASCIItxt, 'admin1', 4); // fix inconsistencies with the USA capitol
 
-  doSql(db, `UPDATE geoname
+  doSql(logger, db, `UPDATE geoname
       SET name = 'Washington, D.C.', asciiname = 'Washington, D.C.'
       WHERE geonameid = 4140963;`, `UPDATE admin1
       SET name = 'Washington, D.C.', asciiname = 'Washington, D.C.'
       WHERE key = 'US.DC';`);
-  doSql(db, `DROP TABLE IF EXISTS geoname_he`, `CREATE TABLE geoname_he AS SELECT * FROM geoname LIMIT 0`);
-  await doFile(db, ILtxt, 'geoname_he', 19, filterPlacesHebrew);
-  doSql(db, `update admin1 set name='',asciiname='' where key like 'PS.%';`, `update country set country = '' where iso = 'PS';`, `delete from geoname where geonameid = 7303419;`);
-  doSql(db, `DROP TABLE IF EXISTS geoname_fulltext`, `CREATE VIRTUAL TABLE geoname_fulltext
+  doSql(logger, db, `DROP TABLE IF EXISTS geoname_he`, `CREATE TABLE geoname_he AS SELECT * FROM geoname LIMIT 0`);
+  await doFile(logger, db, ILtxt, 'geoname_he', 19, filterPlacesHebrew);
+  doSql(logger, db, `update admin1 set name='',asciiname='' where key like 'PS.%';`, `update country set country = '' where iso = 'PS';`, `delete from geoname where geonameid = 7303419;`);
+  doSql(logger, db, `DROP TABLE IF EXISTS geoname_fulltext`, `CREATE VIRTUAL TABLE geoname_fulltext
       USING fts3(geonameid int, longname text,
       asciiname text, admin1 text, country text,
       population int, latitude real, longitude real, timezone text
@@ -453,7 +454,7 @@ function filterPlacesHebrew(a) {
     const firstchar = name[0];
 
     if (firstchar >= '\u05D0' && firstchar <= '\u05EA') {
-      a[1] = name; // replace 'name' field with Hebrew
+      a[1] = core.Locale.hebrewStripNikkud(name); // replace 'name' field with Hebrew
 
       return true;
     }
@@ -462,18 +463,20 @@ function filterPlacesHebrew(a) {
   return false;
 }
 /**
+ * @param {pino.Logger} logger
  * @param {Database} db
  * @param  {...string} sqls
  */
 
 
-function doSql(db, ...sqls) {
+function doSql(logger, db, ...sqls) {
   for (let i = 0; i < sqls.length; i++) {
     logger.info(sqls[i]);
     db.exec(sqls[i]);
   }
 }
 /**
+ * @param {pino.Logger} logger
  * @param {Database} db
  * @param {string} infile
  * @param {string} tableName
@@ -482,7 +485,7 @@ function doSql(db, ...sqls) {
  */
 
 
-async function doFile(db, infile, tableName, expectedFields, callback) {
+async function doFile(logger, db, infile, tableName, expectedFields, callback) {
   logger.info(`${infile} => ${tableName}`);
   db.exec('BEGIN');
   let sql = `INSERT OR IGNORE INTO ${tableName} VALUES (?`;
@@ -496,8 +499,8 @@ async function doFile(db, infile, tableName, expectedFields, callback) {
   const stmt = db.prepare(sql);
   return new Promise((resolve, reject) => {
     try {
-      const rl = readline__default['default'].createInterface({
-        input: fs__default['default'].createReadStream(infile),
+      const rl = readline__default["default"].createInterface({
+        input: fs__default["default"].createReadStream(infile),
         crlfDelay: Infinity
       });
       let num = 0;
@@ -531,7 +534,7 @@ async function doFile(db, infile, tableName, expectedFields, callback) {
         logger.info(`Inserted ${accepted} / ${num} into ${tableName} from ${infile}`);
         db.exec('COMMIT');
       });
-      return resolve(events__default['default'].once(rl, 'close'));
+      return resolve(events__default["default"].once(rl, 'close'));
     } catch (err) {
       logger.error(err);
       return reject(err);

package/dist/index.mjs

@@ -1,6 +1,6 @@
-/*! @hebcal/geo-sqlite v3.4.1 */
+/*! @hebcal/geo-sqlite v3.5.0 */
 import Database from 'better-sqlite3';
-import { Location } from '@hebcal/core';
+import { Location, Locale } from '@hebcal/core';
 import pino from 'pino';
 import events from 'events';
 import fs from 'fs';
@@ -241,6 +241,7 @@ class GeoDb {
         this.geonamesCompStmt = this.geonamesDb.prepare(GEONAME_COMPLETE_SQL);
       }
 
+      qraw = qraw.replace(/\"/g, '""');
       return this.geonamesCompStmt.all(`"${qraw}*"`).map(res => {
         const country = res.country || '';
         const admin1 = res.admin1 || '';
@@ -303,13 +304,6 @@ class GeoDb {
 
 }
 
-const logger = pino({
-  // level: argv.quiet ? 'warn' : 'info',
-  prettyPrint: {
-    translateTime: true,
-    ignore: 'pid,hostname'
-  }
-});
 /**
  * Builds `geonames.sqlite3` from files downloaded from geonames.org
  * @param {string} dbFilename
@@ -321,9 +315,16 @@ const logger = pino({
  */
 
 async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, citiesPatch, admin1CodesASCIItxt, ILtxt) {
+  const logger = pino({
+    // level: argv.quiet ? 'warn' : 'info',
+    prettyPrint: {
+      translateTime: true,
+      ignore: 'pid,hostname'
+    }
+  });
   const db = new Database(dbFilename);
   db.pragma('journal_mode = MEMORY');
-  doSql(db, `DROP TABLE IF EXISTS country`, `CREATE TABLE country (
+  doSql(logger, db, `DROP TABLE IF EXISTS country`, `CREATE TABLE country (
       ISO TEXT PRIMARY KEY,
       ISO3 TEXT NOT NULL,
       IsoNumeric TEXT NOT NULL,
@@ -344,8 +345,8 @@ async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, ci
       neighbours TEXT NOT NULL,
       EquivalentFipsCode TEXT NOT NULL
     );`);
-  await doFile(db, countryInfotxt, 'country', 19);
-  doSql(db, `DROP TABLE IF EXISTS geoname`, `CREATE TABLE geoname (
+  await doFile(logger, db, countryInfotxt, 'country', 19);
+  doSql(logger, db, `DROP TABLE IF EXISTS geoname`, `CREATE TABLE geoname (
       geonameid int PRIMARY KEY,
       name nvarchar(200),
       asciiname nvarchar(200),
@@ -365,28 +366,28 @@ async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, ci
       gtopo30 int,
       timezone nvarchar(40),
       moddate date);`);
-  await doFile(db, cities5000txt, 'geoname', 19);
-  await doFile(db, citiesPatch, 'geoname', 19);
-  await doFile(db, ILtxt, 'geoname', 19, a => {
+  await doFile(logger, db, cities5000txt, 'geoname', 19);
+  await doFile(logger, db, citiesPatch, 'geoname', 19);
+  await doFile(logger, db, ILtxt, 'geoname', 19, a => {
     return a[6] == 'P' && (a[7] == 'PPL' || a[7] == 'STLMT');
   });
-  doSql(db, `DROP TABLE IF EXISTS admin1`, `CREATE TABLE admin1 (
+  doSql(logger, db, `DROP TABLE IF EXISTS admin1`, `CREATE TABLE admin1 (
       key TEXT PRIMARY KEY,
       name nvarchar(200) NOT NULL,
       asciiname nvarchar(200) NOT NULL,
       geonameid int NOT NULL
       );`);
-  await doFile(db, admin1CodesASCIItxt, 'admin1', 4); // fix inconsistencies with the USA capitol
+  await doFile(logger, db, admin1CodesASCIItxt, 'admin1', 4); // fix inconsistencies with the USA capitol
 
-  doSql(db, `UPDATE geoname
+  doSql(logger, db, `UPDATE geoname
       SET name = 'Washington, D.C.', asciiname = 'Washington, D.C.'
       WHERE geonameid = 4140963;`, `UPDATE admin1
       SET name = 'Washington, D.C.', asciiname = 'Washington, D.C.'
       WHERE key = 'US.DC';`);
-  doSql(db, `DROP TABLE IF EXISTS geoname_he`, `CREATE TABLE geoname_he AS SELECT * FROM geoname LIMIT 0`);
-  await doFile(db, ILtxt, 'geoname_he', 19, filterPlacesHebrew);
-  doSql(db, `update admin1 set name='',asciiname='' where key like 'PS.%';`, `update country set country = '' where iso = 'PS';`, `delete from geoname where geonameid = 7303419;`);
-  doSql(db, `DROP TABLE IF EXISTS geoname_fulltext`, `CREATE VIRTUAL TABLE geoname_fulltext
+  doSql(logger, db, `DROP TABLE IF EXISTS geoname_he`, `CREATE TABLE geoname_he AS SELECT * FROM geoname LIMIT 0`);
+  await doFile(logger, db, ILtxt, 'geoname_he', 19, filterPlacesHebrew);
+  doSql(logger, db, `update admin1 set name='',asciiname='' where key like 'PS.%';`, `update country set country = '' where iso = 'PS';`, `delete from geoname where geonameid = 7303419;`);
+  doSql(logger, db, `DROP TABLE IF EXISTS geoname_fulltext`, `CREATE VIRTUAL TABLE geoname_fulltext
       USING fts3(geonameid int, longname text,
       asciiname text, admin1 text, country text,
       population int, latitude real, longitude real, timezone text
@@ -441,7 +442,7 @@ function filterPlacesHebrew(a) {
     const firstchar = name[0];
 
     if (firstchar >= '\u05D0' && firstchar <= '\u05EA') {
-      a[1] = name; // replace 'name' field with Hebrew
+      a[1] = Locale.hebrewStripNikkud(name); // replace 'name' field with Hebrew
 
       return true;
     }
@@ -450,18 +451,20 @@ function filterPlacesHebrew(a) {
   return false;
 }
 /**
+ * @param {pino.Logger} logger
  * @param {Database} db
  * @param  {...string} sqls
  */
 
 
-function doSql(db, ...sqls) {
+function doSql(logger, db, ...sqls) {
   for (let i = 0; i < sqls.length; i++) {
     logger.info(sqls[i]);
     db.exec(sqls[i]);
   }
 }
 /**
+ * @param {pino.Logger} logger
  * @param {Database} db
  * @param {string} infile
  * @param {string} tableName
@@ -470,7 +473,7 @@ function doSql(db, ...sqls) {
  */
 
 
-async function doFile(db, infile, tableName, expectedFields, callback) {
+async function doFile(logger, db, infile, tableName, expectedFields, callback) {
   logger.info(`${infile} => ${tableName}`);
   db.exec('BEGIN');
   let sql = `INSERT OR IGNORE INTO ${tableName} VALUES (?`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hebcal/geo-sqlite",
-  "version": "3.4.1",
+  "version": "3.5.0",
   "author": "Michael J. Radwin (https://github.com/mjradwin)",
   "keywords": [
     "hebcal"
@@ -28,10 +28,10 @@
     "geo-sqlite.d.ts"
   ],
   "dependencies": {
-    "@hebcal/core": "^3.13.0",
-    "better-sqlite3": "^7.1.2",
-    "pino": "^6.11.2",
-    "pino-pretty": "^4.7.1"
+    "@hebcal/core": "^3.32.0",
+    "better-sqlite3": "^7.4.6",
+    "pino": "^7.6.2",
+    "pino-pretty": "^7.3.0"
   },
   "scripts": {
     "build": "rollup -c",
@@ -59,20 +59,20 @@
   },
   "license": "BSD-2-Clause",
   "devDependencies": {
-    "@ava/babel": "^1.0.1",
-    "@babel/core": "^7.13.14",
+    "@ava/babel": "^2.0.0",
+    "@babel/core": "^7.16.7",
     "@babel/polyfill": "^7.12.1",
-    "@babel/preset-env": "^7.13.12",
-    "@babel/register": "^7.13.14",
+    "@babel/preset-env": "^7.16.7",
+    "@babel/register": "^7.16.7",
     "@rollup/plugin-babel": "^5.3.0",
-    "@rollup/plugin-commonjs": "^18.0.0",
+    "@rollup/plugin-commonjs": "^21.0.1",
     "@rollup/plugin-json": "^4.1.0",
-    "@rollup/plugin-node-resolve": "^11.2.1",
+    "@rollup/plugin-node-resolve": "^13.1.3",
     "ava": "^3.15.0",
-    "eslint": "^7.23.0",
+    "eslint": "^8.6.0",
     "eslint-config-google": "^0.14.0",
-    "jsdoc": "^3.6.6",
-    "jsdoc-to-markdown": "^6.0.1",
-    "rollup": "^2.44.0"
+    "jsdoc": "^3.6.7",
+    "jsdoc-to-markdown": "^7.1.0",
+    "rollup": "^2.63.0"
   }
 }