@hebcal/geo-sqlite 3.7.1 → 4.0.0

package/bin/build-geonames-sqlite

@@ -2,21 +2,22 @@
 
 const {buildGeonamesSqlite} = require('@hebcal/geo-sqlite');
 
-if (process.argv.length != 8) {
-  const infiles = 'countryInfo.txt cities5000.txt cities-patch.txt admin1CodesASCII.txt IL.txt';
+const argv = process.argv.slice(2);
+if (argv.length !== 7) {
+  const infiles = 'countryInfo.txt cities5000.txt cities-patch.txt admin1CodesASCII.txt IL.txt IL-alternatenames.txt';
   console.error(`Usage: build-geonames-sqlite geonames.sqlite3 ${infiles}`);
   process.exit(1);
 }
 
-const [dbFilename, countryInfo, cities, citiesPatch, admin1CodesASCII, il] = process.argv.slice(2);
-
-buildGeonamesSqlite(
-    dbFilename,
-    countryInfo,
-    cities,
-    citiesPatch,
-    admin1CodesASCII,
-    il,
-).then(() => {
+const filenames = {
+  dbFilename: argv[0],
+  countryInfotxt: argv[1],
+  cities5000txt: argv[2],
+  citiesPatch: argv[3],
+  admin1CodesASCIItxt: argv[4],
+  ILtxt: argv[5],
+  ILalternate: argv[6],
+};
+buildGeonamesSqlite(filenames).then(() => {
   console.log('Done!');
 });

package/dist/index.js

@@ -1,10 +1,11 @@
-/*! @hebcal/geo-sqlite v3.7.1 */
+/*! @hebcal/geo-sqlite v4.0.0 */
 'use strict';
 
 Object.defineProperty(exports, '__esModule', { value: true });
 
 var Database = require('better-sqlite3');
 var core = require('@hebcal/core');
+require('@hebcal/cities');
 var pino = require('pino');
 var events = require('events');
 var fs = require('fs');
@@ -141,14 +142,32 @@ class GeoDb {
       fileMustExist: true
     });
     this.zipStmt = this.zipsDb.prepare(ZIPCODE_SQL);
+    /** @type {Map<string, Location>} */
+
     this.zipCache = new Map();
     this.geonamesStmt = this.geonamesDb.prepare(GEONAME_SQL);
+    /** @type {Map<number, Location>} */
+
     this.geonamesCache = new Map();
+    /** @type {Map<string, Location>} */
+
     this.legacyCities = new Map();
 
     for (const [name, id] of Object.entries(city2geonameid)) {
       this.legacyCities.set(GeoDb.munge(name), id);
     }
+
+    const stmt = this.geonamesDb.prepare(`SELECT ISO, Country FROM country WHERE Country <> ''`);
+    const rows = stmt.all();
+    const map = new Map();
+
+    for (const row of rows) {
+      map.set(row.ISO, row.Country);
+    }
+    /** @type {Map<string, string>} */
+
+
+    this.countryNames = map;
   }
   /** Closes database handles */
 
@@ -174,19 +193,20 @@ class GeoDb {
 
 
   lookupZip(zip) {
-    const found = this.zipCache.get(zip);
+    const zip5 = zip.trim().substring(0, 5);
+    const found = this.zipCache.get(zip5);
     if (typeof found !== 'undefined') return found;
-    const result = this.zipStmt.get(zip);
+    const result = this.zipStmt.get(zip5);
 
     if (!result) {
-      if (this.logger) this.logger.warn(`GeoDb: unknown zipcode=${zip}`);
-      this.zipCache.set(zip, null);
+      if (this.logger) this.logger.warn(`GeoDb: unknown zipcode=${zip5}`);
+      this.zipCache.set(zip5, null);
       return null;
     }
 
-    result.ZipCode = String(zip);
+    result.ZipCode = String(zip5);
     const location = this.makeZipLocation(result);
-    this.zipCache.set(zip, location);
+    this.zipCache.set(zip5, location);
     return location;
   }
   /**
@@ -272,6 +292,12 @@ class GeoDb {
     if (geonameid) {
       return this.lookupGeoname(geonameid);
     } else {
+      const location = core.Location.lookup(cityName);
+
+      if (location) {
+        return location;
+      }
+
       if (this.logger) this.logger.warn(`GeoDb: unknown city=${cityName}`);
       return null;
     }
@@ -420,20 +446,25 @@ class GeoDb {
 
 /**
  * Builds `geonames.sqlite3` from files downloaded from geonames.org
- * @param {string} dbFilename
- * @param {string} countryInfotxt
- * @param {string} cities5000txt
- * @param {string} citiesPatch
- * @param {string} admin1CodesASCIItxt
- * @param {string} ILtxt
+ * @param {any} opts
  */
 
-async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, citiesPatch, admin1CodesASCIItxt, ILtxt) {
+async function buildGeonamesSqlite(opts) {
+  const dbFilename = opts.dbFilename;
+  const countryInfotxt = opts.countryInfotxt;
+  const cities5000txt = opts.cities5000txt;
+  const citiesPatch = opts.citiesPatch;
+  const admin1CodesASCIItxt = opts.admin1CodesASCIItxt;
+  const ILtxt = opts.ILtxt;
+  const ILalternate = opts.ILalternate;
   const logger = pino__default["default"]({
     // level: argv.quiet ? 'warn' : 'info',
-    prettyPrint: {
-      translateTime: true,
-      ignore: 'pid,hostname'
+    transport: {
+      target: 'pino-pretty',
+      options: {
+        translateTime: 'SYS:standard',
+        ignore: 'pid,hostname'
+      }
     }
   });
   const db = new Database__default["default"](dbFilename);
@@ -498,8 +529,41 @@ async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, ci
       WHERE geonameid = 4140963;`, `UPDATE admin1
       SET name = 'Washington, D.C.', asciiname = 'Washington, D.C.'
       WHERE key = 'US.DC';`);
-  doSql(logger, db, `DROP TABLE IF EXISTS geoname_he`, `CREATE TABLE geoname_he AS SELECT * FROM geoname LIMIT 0`);
-  await doFile(logger, db, ILtxt, 'geoname_he', 19, filterPlacesHebrew);
+  doSql(logger, db, `DROP TABLE IF EXISTS alternatenames`, `CREATE TABLE alternatenames (
+    id int PRIMARY KEY,
+    geonameid int NOT NULL,
+    isolanguage varchar(7),
+    name varchar(400),
+    isPreferredName tinyint,
+    isShortName tinyint,
+    isColloquial tinyint,
+    isHistoric tinyint,
+    periodFrom NULL,
+    periodTo NULL
+    );`);
+  await doFile(logger, db, ILalternate, 'alternatenames', 10, a => {
+    if (a[2] === 'he' || a[2] === 'en') {
+      if (a[2] === 'he') {
+        a[3] = a[3].replace(/‘/g, '׳');
+        a[3] = a[3].replace(/’/g, '׳');
+        a[3] = a[3].replace(/\'/g, '׳');
+        a[3] = core.Locale.hebrewStripNikkud(a[3]);
+      } else {
+        a[3] = a[3].replace(/‘/g, '\'');
+        a[3] = a[3].replace(/’/g, '\'');
+      }
+
+      return true;
+    }
+
+    return false;
+  }); // remove duplicates from alternatenames
+
+  doSql(logger, db, `DROP TABLE IF EXISTS altnames`, `CREATE TABLE altnames
+    AS SELECT geonameid, isolanguage, name
+    FROM alternatenames
+    GROUP BY 1, 2, 3
+    `);
   doSql(logger, db, `update admin1 set name='',asciiname='' where key like 'PS.%';`, `update country set country = '' where iso = 'PS';`, `delete from geoname where geonameid = 7303419;`);
   doSql(logger, db, `DROP TABLE IF EXISTS geoname_fulltext`, `CREATE VIRTUAL TABLE geoname_fulltext
       USING fts3(geonameid int, longname text,
@@ -530,47 +594,23 @@ async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, ci
       AND g.country = c.ISO
       AND g.country||'.'||g.admin1 = a.key
       `, `INSERT INTO geoname_fulltext
-      SELECT g.geonameid, g.name||', ישראל',
-      g.name, '', 'ישראל',
+      SELECT g.geonameid, alt.name||', ישראל',
+      alt.name, '', 'ישראל',
       g.population, g.latitude, g.longitude, g.timezone
-      FROM geoname_he g, admin1 a, country c
-      WHERE g.country = c.ISO
-      AND g.country||'.'||g.admin1 = a.key
+      FROM geoname g, country c, altnames alt
+      WHERE g.country = 'IL'
+      AND alt.isolanguage = 'he'
+      AND g.geonameid = alt.geonameid
       `);
   db.close();
   return Promise.resolve(true);
 }
-/**
- * @param {string[]} a
- * @return {boolean}
- */
-
-function filterPlacesHebrew(a) {
-  if (a[6] != 'P' || a[7] != 'PPL' && a[7] != 'STLMT') {
-    return false;
-  }
-
-  const alternatenames = a[3].split(',');
-
-  for (const name of alternatenames) {
-    const firstchar = name[0];
-
-    if (firstchar >= '\u05D0' && firstchar <= '\u05EA') {
-      a[1] = core.Locale.hebrewStripNikkud(name); // replace 'name' field with Hebrew
-
-      return true;
-    }
-  }
-
-  return false;
-}
 /**
  * @param {pino.Logger} logger
  * @param {Database} db
  * @param  {...string} sqls
  */
 
-
 function doSql(logger, db, ...sqls) {
   for (let i = 0; i < sqls.length; i++) {
     logger.info(sqls[i]);

package/dist/index.mjs

@@ -1,6 +1,7 @@
-/*! @hebcal/geo-sqlite v3.7.1 */
+/*! @hebcal/geo-sqlite v4.0.0 */
 import Database from 'better-sqlite3';
 import { Location, Locale } from '@hebcal/core';
+import '@hebcal/cities';
 import pino from 'pino';
 import events from 'events';
 import fs from 'fs';
@@ -129,14 +130,32 @@ class GeoDb {
       fileMustExist: true
     });
     this.zipStmt = this.zipsDb.prepare(ZIPCODE_SQL);
+    /** @type {Map<string, Location>} */
+
     this.zipCache = new Map();
     this.geonamesStmt = this.geonamesDb.prepare(GEONAME_SQL);
+    /** @type {Map<number, Location>} */
+
     this.geonamesCache = new Map();
+    /** @type {Map<string, Location>} */
+
     this.legacyCities = new Map();
 
     for (const [name, id] of Object.entries(city2geonameid)) {
       this.legacyCities.set(GeoDb.munge(name), id);
     }
+
+    const stmt = this.geonamesDb.prepare(`SELECT ISO, Country FROM country WHERE Country <> ''`);
+    const rows = stmt.all();
+    const map = new Map();
+
+    for (const row of rows) {
+      map.set(row.ISO, row.Country);
+    }
+    /** @type {Map<string, string>} */
+
+
+    this.countryNames = map;
   }
   /** Closes database handles */
 
@@ -162,19 +181,20 @@ class GeoDb {
 
 
   lookupZip(zip) {
-    const found = this.zipCache.get(zip);
+    const zip5 = zip.trim().substring(0, 5);
+    const found = this.zipCache.get(zip5);
     if (typeof found !== 'undefined') return found;
-    const result = this.zipStmt.get(zip);
+    const result = this.zipStmt.get(zip5);
 
     if (!result) {
-      if (this.logger) this.logger.warn(`GeoDb: unknown zipcode=${zip}`);
-      this.zipCache.set(zip, null);
+      if (this.logger) this.logger.warn(`GeoDb: unknown zipcode=${zip5}`);
+      this.zipCache.set(zip5, null);
       return null;
     }
 
-    result.ZipCode = String(zip);
+    result.ZipCode = String(zip5);
     const location = this.makeZipLocation(result);
-    this.zipCache.set(zip, location);
+    this.zipCache.set(zip5, location);
     return location;
   }
   /**
@@ -260,6 +280,12 @@ class GeoDb {
     if (geonameid) {
       return this.lookupGeoname(geonameid);
     } else {
+      const location = Location.lookup(cityName);
+
+      if (location) {
+        return location;
+      }
+
       if (this.logger) this.logger.warn(`GeoDb: unknown city=${cityName}`);
       return null;
     }
@@ -408,20 +434,25 @@ class GeoDb {
 
 /**
  * Builds `geonames.sqlite3` from files downloaded from geonames.org
- * @param {string} dbFilename
- * @param {string} countryInfotxt
- * @param {string} cities5000txt
- * @param {string} citiesPatch
- * @param {string} admin1CodesASCIItxt
- * @param {string} ILtxt
+ * @param {any} opts
  */
 
-async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, citiesPatch, admin1CodesASCIItxt, ILtxt) {
+async function buildGeonamesSqlite(opts) {
+  const dbFilename = opts.dbFilename;
+  const countryInfotxt = opts.countryInfotxt;
+  const cities5000txt = opts.cities5000txt;
+  const citiesPatch = opts.citiesPatch;
+  const admin1CodesASCIItxt = opts.admin1CodesASCIItxt;
+  const ILtxt = opts.ILtxt;
+  const ILalternate = opts.ILalternate;
   const logger = pino({
     // level: argv.quiet ? 'warn' : 'info',
-    prettyPrint: {
-      translateTime: true,
-      ignore: 'pid,hostname'
+    transport: {
+      target: 'pino-pretty',
+      options: {
+        translateTime: 'SYS:standard',
+        ignore: 'pid,hostname'
+      }
     }
   });
   const db = new Database(dbFilename);
@@ -486,8 +517,41 @@ async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, ci
       WHERE geonameid = 4140963;`, `UPDATE admin1
       SET name = 'Washington, D.C.', asciiname = 'Washington, D.C.'
       WHERE key = 'US.DC';`);
-  doSql(logger, db, `DROP TABLE IF EXISTS geoname_he`, `CREATE TABLE geoname_he AS SELECT * FROM geoname LIMIT 0`);
-  await doFile(logger, db, ILtxt, 'geoname_he', 19, filterPlacesHebrew);
+  doSql(logger, db, `DROP TABLE IF EXISTS alternatenames`, `CREATE TABLE alternatenames (
+    id int PRIMARY KEY,
+    geonameid int NOT NULL,
+    isolanguage varchar(7),
+    name varchar(400),
+    isPreferredName tinyint,
+    isShortName tinyint,
+    isColloquial tinyint,
+    isHistoric tinyint,
+    periodFrom NULL,
+    periodTo NULL
+    );`);
+  await doFile(logger, db, ILalternate, 'alternatenames', 10, a => {
+    if (a[2] === 'he' || a[2] === 'en') {
+      if (a[2] === 'he') {
+        a[3] = a[3].replace(/‘/g, '׳');
+        a[3] = a[3].replace(/’/g, '׳');
+        a[3] = a[3].replace(/\'/g, '׳');
+        a[3] = Locale.hebrewStripNikkud(a[3]);
+      } else {
+        a[3] = a[3].replace(/‘/g, '\'');
+        a[3] = a[3].replace(/’/g, '\'');
+      }
+
+      return true;
+    }
+
+    return false;
+  }); // remove duplicates from alternatenames
+
+  doSql(logger, db, `DROP TABLE IF EXISTS altnames`, `CREATE TABLE altnames
+    AS SELECT geonameid, isolanguage, name
+    FROM alternatenames
+    GROUP BY 1, 2, 3
+    `);
   doSql(logger, db, `update admin1 set name='',asciiname='' where key like 'PS.%';`, `update country set country = '' where iso = 'PS';`, `delete from geoname where geonameid = 7303419;`);
   doSql(logger, db, `DROP TABLE IF EXISTS geoname_fulltext`, `CREATE VIRTUAL TABLE geoname_fulltext
       USING fts3(geonameid int, longname text,
@@ -518,47 +582,23 @@ async function buildGeonamesSqlite(dbFilename, countryInfotxt, cities5000txt, ci
       AND g.country = c.ISO
       AND g.country||'.'||g.admin1 = a.key
       `, `INSERT INTO geoname_fulltext
-      SELECT g.geonameid, g.name||', ישראל',
-      g.name, '', 'ישראל',
+      SELECT g.geonameid, alt.name||', ישראל',
+      alt.name, '', 'ישראל',
       g.population, g.latitude, g.longitude, g.timezone
-      FROM geoname_he g, admin1 a, country c
-      WHERE g.country = c.ISO
-      AND g.country||'.'||g.admin1 = a.key
+      FROM geoname g, country c, altnames alt
+      WHERE g.country = 'IL'
+      AND alt.isolanguage = 'he'
+      AND g.geonameid = alt.geonameid
       `);
   db.close();
   return Promise.resolve(true);
 }
-/**
- * @param {string[]} a
- * @return {boolean}
- */
-
-function filterPlacesHebrew(a) {
-  if (a[6] != 'P' || a[7] != 'PPL' && a[7] != 'STLMT') {
-    return false;
-  }
-
-  const alternatenames = a[3].split(',');
-
-  for (const name of alternatenames) {
-    const firstchar = name[0];
-
-    if (firstchar >= '\u05D0' && firstchar <= '\u05EA') {
-      a[1] = Locale.hebrewStripNikkud(name); // replace 'name' field with Hebrew
-
-      return true;
-    }
-  }
-
-  return false;
-}
 /**
  * @param {pino.Logger} logger
  * @param {Database} db
  * @param  {...string} sqls
  */
 
-
 function doSql(logger, db, ...sqls) {
   for (let i = 0; i < sqls.length; i++) {
     logger.info(sqls[i]);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hebcal/geo-sqlite",
-  "version": "3.7.1",
+  "version": "4.0.0",
   "author": "Michael J. Radwin (https://github.com/mjradwin)",
   "keywords": [
     "hebcal"
@@ -28,10 +28,11 @@
     "geo-sqlite.d.ts"
   ],
   "dependencies": {
-    "@hebcal/core": "^3.32.1",
-    "better-sqlite3": "^7.4.6",
-    "pino": "^7.6.3",
-    "pino-pretty": "^7.3.0"
+    "@hebcal/cities": "^3.1.1",
+    "@hebcal/core": "^3.33.3",
+    "better-sqlite3": "^7.5.0",
+    "pino": "^7.8.0",
+    "pino-pretty": "^7.5.3"
   },
   "scripts": {
     "build": "rollup -c",
@@ -45,34 +46,25 @@
     ],
     "require": [
       "@babel/register",
-      "@babel/polyfill"
+      "regenerator-runtime/runtime"
     ],
-    "babel": {
-      "testOptions": {
-        "presets": [
-          "@babel/env"
-        ]
-      }
-    },
     "inherit": true,
     "verbose": true
   },
   "license": "BSD-2-Clause",
   "devDependencies": {
-    "@ava/babel": "^2.0.0",
-    "@babel/core": "^7.16.7",
-    "@babel/polyfill": "^7.12.1",
-    "@babel/preset-env": "^7.16.8",
-    "@babel/register": "^7.16.8",
-    "@rollup/plugin-babel": "^5.3.0",
-    "@rollup/plugin-commonjs": "^21.0.1",
+    "@babel/core": "^7.17.5",
+    "@babel/preset-env": "^7.16.11",
+    "@babel/register": "^7.17.0",
+    "@rollup/plugin-babel": "^5.3.1",
+    "@rollup/plugin-commonjs": "^21.0.2",
     "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-node-resolve": "^13.1.3",
-    "ava": "^3.15.0",
-    "eslint": "^8.6.0",
+    "ava": "^4.0.1",
+    "eslint": "^8.10.0",
     "eslint-config-google": "^0.14.0",
-    "jsdoc": "^3.6.7",
-    "jsdoc-to-markdown": "^7.1.0",
-    "rollup": "^2.63.0"
+    "jsdoc": "^3.6.10",
+    "jsdoc-to-markdown": "^7.1.1",
+    "rollup": "^2.69.0"
   }
 }