@hebcal/geo-sqlite 4.4.0 → 4.4.3

package/dist/index.js

@@ -1,4 +1,4 @@
-/*! @hebcal/geo-sqlite v4.4.0 */
+/*! @hebcal/geo-sqlite v4.4.3 */
 'use strict';
 
 Object.defineProperty(exports, '__esModule', { value: true });
@@ -29,6 +29,7 @@ const GEONAME_SQL = `SELECT
   a.asciiname as admin1,
   g.latitude as latitude,
   g.longitude as longitude,
+  g.population as population,
   g.timezone as timezone
 FROM geoname g
 LEFT JOIN country c on g.country = c.iso
@@ -44,33 +45,29 @@ const GEONAME_ALL_SQL = `SELECT
   a.asciiname as admin1,
   g.latitude as latitude,
   g.longitude as longitude,
+  g.population as population,
   g.timezone as timezone
 FROM geoname g
 LEFT JOIN country c on g.country = c.iso
 LEFT JOIN admin1 a on g.country||'.'||g.admin1 = a.key
 `;
-const ZIPCODE_SQL = `SELECT CityMixedCase,State,Latitude,Longitude,TimeZone,DayLightSaving
+const ZIPCODE_SQL = `SELECT ZipCode,CityMixedCase,State,Latitude,Longitude,TimeZone,DayLightSaving,Population
 FROM ZIPCodes_Primary WHERE ZipCode = ?`;
-const ZIPCODE_ALL_SQL = `SELECT ZipCode,CityMixedCase,State,Latitude,Longitude,TimeZone,DayLightSaving
+const ZIPCODE_ALL_SQL = `SELECT ZipCode,CityMixedCase,State,Latitude,Longitude,TimeZone,DayLightSaving,Population
 FROM ZIPCodes_Primary`;
 const ZIP_COMPLETE_SQL = `SELECT ZipCode,CityMixedCase,State,Latitude,Longitude,TimeZone,DayLightSaving,Population
 FROM ZIPCodes_Primary
 WHERE ZipCode LIKE ?
 ORDER BY Population DESC
 LIMIT 10`;
-const ZIP_FULLTEXT_COMPLETE_SQL = `SELECT ZipCode,CityMixedCase,State,Latitude,Longitude,TimeZone,DayLightSaving,Population
-FROM ZIPCodes_CityFullText
-WHERE CityMixedCase MATCH ?
-ORDER BY Population DESC
-LIMIT 15`;
-const GEONAME_COMPLETE_SQL = `SELECT geonameid, asciiname, admin1, country,
-population, latitude, longitude, timezone,
-((sqrt(population)/40) + (-30 * rank)) as myrank
+const ZIP_FULLTEXT_COMPLETE_SQL = `SELECT ZipCode, rank
+FROM ZIPCodes_CityFullText5
+WHERE ZIPCodes_CityFullText5 MATCH ?
+LIMIT 20`;
+const GEONAME_COMPLETE_SQL = `SELECT geonameid, rank
 FROM geoname_fulltext
 WHERE geoname_fulltext MATCH ?
-GROUP BY geonameid
-ORDER BY myrank DESC
-LIMIT 15`;
+LIMIT 20`;
 const stateNames = {
   'AK': 'Alaska',
   'AL': 'Alabama',
@@ -226,6 +223,7 @@ class GeoDb {
     location.stateName = stateNames[location.state];
     location.geo = 'zip';
     location.zip = zip;
+    location.population = result.Population;
     return location;
   }
   /**
@@ -263,13 +261,9 @@ class GeoDb {
     const admin1 = result.admin1 || '';
     const cityDescr = core.Location.geonameCityDescr(result.name, admin1, country);
     const location = new core.Location(result.latitude, result.longitude, result.cc == 'IL', result.timezone, cityDescr, result.cc, geonameid);
-
-    if (result.asciiname !== result.name) {
-      location.asciiname = result.asciiname;
-    }
-
     location.geo = 'geoname';
     location.geonameid = geonameid;
+    location.asciiname = result.asciiname;
 
     if (admin1) {
       location.admin1 = admin1;
@@ -279,6 +273,10 @@ class GeoDb {
       location.jersualem = true;
     }
 
+    if (result.population) {
+      location.population = result.population;
+    }
+
     return location;
   }
   /**
@@ -356,25 +354,44 @@ class GeoDb {
       }
 
       qraw = qraw.replace(/\"/g, '""');
-      const geoRows = this.geonamesCompStmt.all(`{asciiname country admin1} : "${qraw}" *`);
+      const geoRows0 = this.geonamesCompStmt.all(`{longname} : "${qraw}" *`);
+      const ids = new Set();
+      const geoRows = [];
+
+      for (const row of geoRows0) {
+        const id = row.geonameid;
+
+        if (!ids.has(id)) {
+          ids.add(id);
+          geoRows.push(row);
+        }
+      }
+
       const geoMatches = geoRows.map(res => {
-        const country = res.country || '';
-        const admin1 = res.admin1 || '';
-        const rank = Math.trunc(res.myrank * 100.0) / 100.0;
+        const loc = this.lookupGeoname(res.geonameid);
+        const country = this.countryNames.get(loc.getCountryCode()) || '';
+        const admin1 = loc.admin1 || '';
+        const rankPop = Math.sqrt(loc.population) / 40;
         const obj = {
           id: res.geonameid,
-          value: core.Location.geonameCityDescr(res.asciiname, admin1, country),
-          asciiname: res.asciiname,
+          value: loc.name,
           admin1,
           country,
-          latitude: res.latitude,
-          longitude: res.longitude,
-          timezone: res.timezone,
-          population: res.population,
+          latitude: loc.latitude,
+          longitude: loc.longitude,
+          timezone: loc.getTzid(),
           geo: 'geoname',
-          rank: rank
+          rank: rankPop + -30 * res.rank
         };
 
+        if (loc.population) {
+          obj.population = loc.population;
+        }
+
+        if (loc.asciiname) {
+          obj.asciiname = loc.asciiname;
+        }
+
         if (country) {
           obj.country = country;
         }
@@ -390,8 +407,26 @@ class GeoDb {
         this.zipFulltextCompStmt = this.zipsDb.prepare(ZIP_FULLTEXT_COMPLETE_SQL);
       }
 
-      const zipRows = this.zipFulltextCompStmt.all(`"${qraw}*"`);
-      const zipMatches = zipRows.map(GeoDb.zipResultToObj);
+      const zipRows = this.zipFulltextCompStmt.all(`{longname} : "${qraw}" *`);
+      const zipMatches = zipRows.map(res => {
+        const zipCode = res.ZipCode;
+        const loc = this.lookupZip(zipCode);
+        const rankPop = Math.sqrt(loc.population) / 40;
+        const obj = {
+          id: zipCode,
+          value: loc.getName(),
+          admin1: loc.admin1,
+          asciiname: loc.getShortName(),
+          country: 'United States',
+          latitude: loc.latitude,
+          longitude: loc.longitude,
+          timezone: loc.getTzid(),
+          population: loc.population,
+          geo: 'zip',
+          rank: rankPop + -30 * res.rank
+        };
+        return obj;
+      });
       const map = new Map();
 
       for (const obj of zipMatches) {
@@ -409,7 +444,7 @@ class GeoDb {
       }
 
       const values = Array.from(map.values());
-      values.sort((a, b) => b.population - a.population);
+      values.sort((a, b) => b.rank - a.rank);
       const topN = values.slice(0, 10);
 
       if (!latlong) {
@@ -601,45 +636,56 @@ async function buildGeonamesSqlite(opts) {
     `);
   doSql(logger, db, `update admin1 set name='',asciiname='' where key like 'PS.%';`, `update country set country = '' where iso = 'PS';`, `delete from geoname where geonameid = 7303419;`);
   doSql(logger, db, `DROP TABLE IF EXISTS geoname_fulltext`, `CREATE VIRTUAL TABLE geoname_fulltext
-      USING fts5(geonameid,
-      asciiname, admin1, country,
-      population, latitude, longitude, timezone
-      );
+      USING fts5(geonameid UNINDEXED, longname, population, city, admin1, country);
     `, `DROP TABLE IF EXISTS geoname_non_ascii`, `CREATE TABLE geoname_non_ascii AS
       SELECT geonameid FROM geoname WHERE asciiname <> name`, `INSERT INTO geoname_fulltext
       SELECT g.geonameid,
-      g.asciiname, a.asciiname, c.Country,
-      g.population, g.latitude, g.longitude, g.timezone
+      g.asciiname||', '||a.asciiname||', '||c.Country,
+      g.population,
+      g.asciiname,a.asciiname,c.Country
       FROM geoname g, admin1 a, country c
       WHERE g.country = c.ISO
+      AND g.country <> 'US'
+      AND g.country||'.'||g.admin1 = a.key
+      `, `INSERT INTO geoname_fulltext
+      SELECT g.geonameid,
+      g.asciiname||', '||a.asciiname||', USA',
+      g.population,
+      g.asciiname,a.asciiname,'USA'
+      FROM geoname g, admin1 a
+      WHERE g.country = 'US'
       AND g.country||'.'||g.admin1 = a.key
       `, `INSERT INTO geoname_fulltext
       SELECT g.geonameid,
-      g.asciiname, '', c.Country,
-      g.population, g.latitude, g.longitude, g.timezone
+      g.asciiname||', '||c.Country,
+      g.population,
+      g.asciiname,NULL,c.Country
       FROM geoname g, country c
       WHERE g.country = c.ISO
       AND (g.admin1 = '' OR g.admin1 = '00')
       `, `INSERT INTO geoname_fulltext
       SELECT g.geonameid,
-      g.name, a.name, c.Country,
-      g.population, g.latitude, g.longitude, g.timezone
+      g.name||', '||a.name||', '||c.Country,
+      g.population,
+      g.name,a.name,c.Country
       FROM geoname_non_ascii gna, geoname g, admin1 a, country c
       WHERE gna.geonameid = g.geonameid
       AND g.country = c.ISO
       AND g.country||'.'||g.admin1 = a.key
       `, `INSERT INTO geoname_fulltext
       SELECT g.geonameid,
-      alt.name, '', 'ישראל',
-      g.population, g.latitude, g.longitude, g.timezone
+      alt.name||', ישראל',
+      g.population,
+      alt.name,NULL,'ישראל'
       FROM geoname g, altnames alt
       WHERE g.country = 'IL'
       AND alt.isolanguage = 'he'
       AND g.geonameid = alt.geonameid
       `, `INSERT INTO geoname_fulltext
       SELECT g.geonameid,
-      alt.name, a1.asciiname, 'Israel',
-      g.population, g.latitude, g.longitude, g.timezone
+      alt.name||', '||a1.asciiname||', Israel',
+      g.population,
+      alt.name,a1.asciiname,'Israel'
       FROM geoname g, admin1 a1, altnames alt
       WHERE g.country = 'IL'
       AND alt.isolanguage = 'en'

package/dist/index.mjs

@@ -1,4 +1,4 @@
-/*! @hebcal/geo-sqlite v4.4.0 */
+/*! @hebcal/geo-sqlite v4.4.3 */
 import Database from 'better-sqlite3';
 import { Location, Locale } from '@hebcal/core';
 import '@hebcal/cities';
@@ -17,6 +17,7 @@ const GEONAME_SQL = `SELECT
   a.asciiname as admin1,
   g.latitude as latitude,
   g.longitude as longitude,
+  g.population as population,
   g.timezone as timezone
 FROM geoname g
 LEFT JOIN country c on g.country = c.iso
@@ -32,33 +33,29 @@ const GEONAME_ALL_SQL = `SELECT
   a.asciiname as admin1,
   g.latitude as latitude,
   g.longitude as longitude,
+  g.population as population,
   g.timezone as timezone
 FROM geoname g
 LEFT JOIN country c on g.country = c.iso
 LEFT JOIN admin1 a on g.country||'.'||g.admin1 = a.key
 `;
-const ZIPCODE_SQL = `SELECT CityMixedCase,State,Latitude,Longitude,TimeZone,DayLightSaving
+const ZIPCODE_SQL = `SELECT ZipCode,CityMixedCase,State,Latitude,Longitude,TimeZone,DayLightSaving,Population
 FROM ZIPCodes_Primary WHERE ZipCode = ?`;
-const ZIPCODE_ALL_SQL = `SELECT ZipCode,CityMixedCase,State,Latitude,Longitude,TimeZone,DayLightSaving
+const ZIPCODE_ALL_SQL = `SELECT ZipCode,CityMixedCase,State,Latitude,Longitude,TimeZone,DayLightSaving,Population
 FROM ZIPCodes_Primary`;
 const ZIP_COMPLETE_SQL = `SELECT ZipCode,CityMixedCase,State,Latitude,Longitude,TimeZone,DayLightSaving,Population
 FROM ZIPCodes_Primary
 WHERE ZipCode LIKE ?
 ORDER BY Population DESC
 LIMIT 10`;
-const ZIP_FULLTEXT_COMPLETE_SQL = `SELECT ZipCode,CityMixedCase,State,Latitude,Longitude,TimeZone,DayLightSaving,Population
-FROM ZIPCodes_CityFullText
-WHERE CityMixedCase MATCH ?
-ORDER BY Population DESC
-LIMIT 15`;
-const GEONAME_COMPLETE_SQL = `SELECT geonameid, asciiname, admin1, country,
-population, latitude, longitude, timezone,
-((sqrt(population)/40) + (-30 * rank)) as myrank
+const ZIP_FULLTEXT_COMPLETE_SQL = `SELECT ZipCode, rank
+FROM ZIPCodes_CityFullText5
+WHERE ZIPCodes_CityFullText5 MATCH ?
+LIMIT 20`;
+const GEONAME_COMPLETE_SQL = `SELECT geonameid, rank
 FROM geoname_fulltext
 WHERE geoname_fulltext MATCH ?
-GROUP BY geonameid
-ORDER BY myrank DESC
-LIMIT 15`;
+LIMIT 20`;
 const stateNames = {
   'AK': 'Alaska',
   'AL': 'Alabama',
@@ -214,6 +211,7 @@ class GeoDb {
     location.stateName = stateNames[location.state];
     location.geo = 'zip';
     location.zip = zip;
+    location.population = result.Population;
     return location;
   }
   /**
@@ -251,13 +249,9 @@ class GeoDb {
     const admin1 = result.admin1 || '';
     const cityDescr = Location.geonameCityDescr(result.name, admin1, country);
     const location = new Location(result.latitude, result.longitude, result.cc == 'IL', result.timezone, cityDescr, result.cc, geonameid);
-
-    if (result.asciiname !== result.name) {
-      location.asciiname = result.asciiname;
-    }
-
     location.geo = 'geoname';
     location.geonameid = geonameid;
+    location.asciiname = result.asciiname;
 
     if (admin1) {
       location.admin1 = admin1;
@@ -267,6 +261,10 @@ class GeoDb {
       location.jersualem = true;
     }
 
+    if (result.population) {
+      location.population = result.population;
+    }
+
     return location;
   }
   /**
@@ -344,25 +342,44 @@ class GeoDb {
       }
 
       qraw = qraw.replace(/\"/g, '""');
-      const geoRows = this.geonamesCompStmt.all(`{asciiname country admin1} : "${qraw}" *`);
+      const geoRows0 = this.geonamesCompStmt.all(`{longname} : "${qraw}" *`);
+      const ids = new Set();
+      const geoRows = [];
+
+      for (const row of geoRows0) {
+        const id = row.geonameid;
+
+        if (!ids.has(id)) {
+          ids.add(id);
+          geoRows.push(row);
+        }
+      }
+
       const geoMatches = geoRows.map(res => {
-        const country = res.country || '';
-        const admin1 = res.admin1 || '';
-        const rank = Math.trunc(res.myrank * 100.0) / 100.0;
+        const loc = this.lookupGeoname(res.geonameid);
+        const country = this.countryNames.get(loc.getCountryCode()) || '';
+        const admin1 = loc.admin1 || '';
+        const rankPop = Math.sqrt(loc.population) / 40;
         const obj = {
           id: res.geonameid,
-          value: Location.geonameCityDescr(res.asciiname, admin1, country),
-          asciiname: res.asciiname,
+          value: loc.name,
           admin1,
           country,
-          latitude: res.latitude,
-          longitude: res.longitude,
-          timezone: res.timezone,
-          population: res.population,
+          latitude: loc.latitude,
+          longitude: loc.longitude,
+          timezone: loc.getTzid(),
           geo: 'geoname',
-          rank: rank
+          rank: rankPop + -30 * res.rank
         };
 
+        if (loc.population) {
+          obj.population = loc.population;
+        }
+
+        if (loc.asciiname) {
+          obj.asciiname = loc.asciiname;
+        }
+
         if (country) {
           obj.country = country;
         }
@@ -378,8 +395,26 @@ class GeoDb {
         this.zipFulltextCompStmt = this.zipsDb.prepare(ZIP_FULLTEXT_COMPLETE_SQL);
       }
 
-      const zipRows = this.zipFulltextCompStmt.all(`"${qraw}*"`);
-      const zipMatches = zipRows.map(GeoDb.zipResultToObj);
+      const zipRows = this.zipFulltextCompStmt.all(`{longname} : "${qraw}" *`);
+      const zipMatches = zipRows.map(res => {
+        const zipCode = res.ZipCode;
+        const loc = this.lookupZip(zipCode);
+        const rankPop = Math.sqrt(loc.population) / 40;
+        const obj = {
+          id: zipCode,
+          value: loc.getName(),
+          admin1: loc.admin1,
+          asciiname: loc.getShortName(),
+          country: 'United States',
+          latitude: loc.latitude,
+          longitude: loc.longitude,
+          timezone: loc.getTzid(),
+          population: loc.population,
+          geo: 'zip',
+          rank: rankPop + -30 * res.rank
+        };
+        return obj;
+      });
       const map = new Map();
 
       for (const obj of zipMatches) {
@@ -397,7 +432,7 @@ class GeoDb {
       }
 
       const values = Array.from(map.values());
-      values.sort((a, b) => b.population - a.population);
+      values.sort((a, b) => b.rank - a.rank);
       const topN = values.slice(0, 10);
 
       if (!latlong) {
@@ -589,45 +624,56 @@ async function buildGeonamesSqlite(opts) {
     `);
   doSql(logger, db, `update admin1 set name='',asciiname='' where key like 'PS.%';`, `update country set country = '' where iso = 'PS';`, `delete from geoname where geonameid = 7303419;`);
   doSql(logger, db, `DROP TABLE IF EXISTS geoname_fulltext`, `CREATE VIRTUAL TABLE geoname_fulltext
-      USING fts5(geonameid,
-      asciiname, admin1, country,
-      population, latitude, longitude, timezone
-      );
+      USING fts5(geonameid UNINDEXED, longname, population, city, admin1, country);
     `, `DROP TABLE IF EXISTS geoname_non_ascii`, `CREATE TABLE geoname_non_ascii AS
       SELECT geonameid FROM geoname WHERE asciiname <> name`, `INSERT INTO geoname_fulltext
       SELECT g.geonameid,
-      g.asciiname, a.asciiname, c.Country,
-      g.population, g.latitude, g.longitude, g.timezone
+      g.asciiname||', '||a.asciiname||', '||c.Country,
+      g.population,
+      g.asciiname,a.asciiname,c.Country
       FROM geoname g, admin1 a, country c
       WHERE g.country = c.ISO
+      AND g.country <> 'US'
+      AND g.country||'.'||g.admin1 = a.key
+      `, `INSERT INTO geoname_fulltext
+      SELECT g.geonameid,
+      g.asciiname||', '||a.asciiname||', USA',
+      g.population,
+      g.asciiname,a.asciiname,'USA'
+      FROM geoname g, admin1 a
+      WHERE g.country = 'US'
       AND g.country||'.'||g.admin1 = a.key
       `, `INSERT INTO geoname_fulltext
       SELECT g.geonameid,
-      g.asciiname, '', c.Country,
-      g.population, g.latitude, g.longitude, g.timezone
+      g.asciiname||', '||c.Country,
+      g.population,
+      g.asciiname,NULL,c.Country
       FROM geoname g, country c
       WHERE g.country = c.ISO
       AND (g.admin1 = '' OR g.admin1 = '00')
       `, `INSERT INTO geoname_fulltext
       SELECT g.geonameid,
-      g.name, a.name, c.Country,
-      g.population, g.latitude, g.longitude, g.timezone
+      g.name||', '||a.name||', '||c.Country,
+      g.population,
+      g.name,a.name,c.Country
       FROM geoname_non_ascii gna, geoname g, admin1 a, country c
       WHERE gna.geonameid = g.geonameid
       AND g.country = c.ISO
       AND g.country||'.'||g.admin1 = a.key
       `, `INSERT INTO geoname_fulltext
       SELECT g.geonameid,
-      alt.name, '', 'ישראל',
-      g.population, g.latitude, g.longitude, g.timezone
+      alt.name||', ישראל',
+      g.population,
+      alt.name,NULL,'ישראל'
       FROM geoname g, altnames alt
       WHERE g.country = 'IL'
       AND alt.isolanguage = 'he'
       AND g.geonameid = alt.geonameid
       `, `INSERT INTO geoname_fulltext
       SELECT g.geonameid,
-      alt.name, a1.asciiname, 'Israel',
-      g.population, g.latitude, g.longitude, g.timezone
+      alt.name||', '||a1.asciiname||', Israel',
+      g.population,
+      alt.name,a1.asciiname,'Israel'
       FROM geoname g, admin1 a1, altnames alt
       WHERE g.country = 'IL'
       AND alt.isolanguage = 'en'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hebcal/geo-sqlite",
-  "version": "4.4.0",
+  "version": "4.4.3",
   "author": "Michael J. Radwin (https://github.com/mjradwin)",
   "keywords": [
     "hebcal"
@@ -29,8 +29,8 @@
   ],
   "dependencies": {
     "@hebcal/cities": "^3.1.2",
-    "@hebcal/core": "^3.36.2",
-    "better-sqlite3": "^7.5.1",
+    "@hebcal/core": "^3.38.0",
+    "better-sqlite3": "^7.5.3",
     "pino": "^7.11.0",
     "pino-pretty": "^7.6.1"
   },
@@ -53,18 +53,18 @@
   },
   "license": "BSD-2-Clause",
   "devDependencies": {
-    "@babel/core": "^7.17.10",
-    "@babel/preset-env": "^7.17.10",
+    "@babel/core": "^7.18.2",
+    "@babel/preset-env": "^7.18.2",
     "@babel/register": "^7.17.7",
     "@rollup/plugin-babel": "^5.3.1",
     "@rollup/plugin-commonjs": "^22.0.0",
     "@rollup/plugin-json": "^4.1.0",
-    "@rollup/plugin-node-resolve": "^13.2.1",
+    "@rollup/plugin-node-resolve": "^13.3.0",
     "ava": "^4.2.0",
-    "eslint": "^8.14.0",
+    "eslint": "^8.16.0",
     "eslint-config-google": "^0.14.0",
     "jsdoc": "^3.6.10",
     "jsdoc-to-markdown": "^7.1.1",
-    "rollup": "^2.71.1"
+    "rollup": "^2.75.4"
   }
 }

package/zips-dummy.sql

@@ -10,4 +10,7 @@ CREATE TABLE ZIPCodes_Primary (
 );
 
 CREATE VIRTUAL TABLE ZIPCodes_CityFullText
-USING fts3(ZipCode,CityMixedCase,State,Latitude,Longitude,TimeZone,DayLightSaving,Population);
+USING fts4(ZipCode,CityMixedCase,State,Latitude,Longitude,TimeZone,DayLightSaving,Population);
+
+CREATE VIRTUAL TABLE ZIPCodes_CityFullText5
+USING fts5(ZipCode UNINDEXED,CityMixedCase,Population UNINDEXED,longname);