@hebcal/geo-sqlite 5.6.0 → 5.7.1

package/bin/build-geonames-sqlite

@@ -2,32 +2,51 @@
 
 import {buildGeonamesSqlite} from '@hebcal/geo-sqlite';
 import {pino} from 'pino';
+import minimist from 'minimist';
+
+const argv = minimist(process.argv.slice(2), {
+  boolean: ['help', 'quiet'],
+  string: ['population'],
+  alias: {h: 'help', q: 'quiet'},
+});
+
+if (argv.help) {
+  usage();
+  process.exit(0);
+}
 
 const logger = pino({
-  // level: argv.quiet ? 'warn' : 'info',
+  level: argv.quiet ? 'warn' : 'info',
   transport: {
     target: 'pino-pretty',
     options: {translateTime: 'SYS:standard', ignore: 'pid,hostname'},
   },
 });
 
-const argv = process.argv.slice(2);
-if (argv.length !== 7) {
-  const infiles = 'countryInfo.txt cities5000.txt cities-patch.txt admin1CodesASCII.txt IL.txt IL-alternatenames.txt';
-  console.error(`Usage: build-geonames-sqlite geonames.sqlite3 ${infiles}`);
+const args = argv._;
+if (args.length !== 7) {
+  usage();
   process.exit(1);
 }
 
-const filenames = {
-  dbFilename: argv[0],
-  countryInfotxt: argv[1],
-  cities5000txt: argv[2],
-  citiesPatch: argv[3],
-  admin1CodesASCIItxt: argv[4],
-  ILtxt: argv[5],
-  ILalternate: argv[6],
+const options = {
+  dbFilename: args[0],
+  countryInfotxt: args[1],
+  cities5000txt: args[2],
+  citiesPatch: args[3],
+  admin1CodesASCIItxt: args[4],
+  ILtxt: args[5],
+  ILalternate: args[6],
   logger: logger,
 };
-buildGeonamesSqlite(filenames).then(() => {
+if (argv.population) {
+  options.population = +argv.population;
+}
+buildGeonamesSqlite(options).then(() => {
   console.log('Done!');
 });
+
+function usage() {
+  const infiles = 'countryInfo.txt cities5000.txt cities-patch.txt admin1CodesASCII.txt IL.txt IL-alternatenames.txt';
+  console.log(`Usage: build-geonames-sqlite geonames.sqlite3 ${infiles}`);
+}

package/bin/download-and-make-dbs

@@ -5,14 +5,20 @@ set -x
 TMPDIR=`mktemp -d /tmp/geonames.XXXXXX`
 CURDIR=`pwd`
 
+# optionally specify minimum city population size with POPULATION=750
+POPULATION=${POPULATION:-"0"}
+
+# optionally specify a geonames export file such as cities500 or cities5000
+CITIES=${CITIES:-"cities1000"}
+
 curl -o $TMPDIR/countryInfo.txt http://download.geonames.org/export/dump/countryInfo.txt
 curl -o $TMPDIR/admin1CodesASCII.txt http://download.geonames.org/export/dump/admin1CodesASCII.txt
-curl -o $TMPDIR/cities5000.zip http://download.geonames.org/export/dump/cities5000.zip
+curl -o $TMPDIR/$CITIES.zip http://download.geonames.org/export/dump/$CITIES.zip
 curl -o $TMPDIR/IL.zip http://download.geonames.org/export/dump/IL.zip
 curl -o $TMPDIR/alt-IL.zip http://download.geonames.org/export/dump/alternatenames/IL.zip
 
 cd $TMPDIR
-unzip cities5000.zip
+unzip $CITIES.zip
 unzip alt-IL.zip
 mv IL.txt alt-IL.txt
 rm readme.txt
@@ -22,9 +28,10 @@ cd $CURDIR
 
 #npx --package '@hebcal/geo-sqlite' build-geonames-sqlite \
 $CURDIR/node_modules/.bin/build-geonames-sqlite \
+  --population=$POPULATION \
   $CURDIR/geonames.sqlite3 \
   $TMPDIR/countryInfo.txt \
-  $TMPDIR/cities5000.txt \
+  "$TMPDIR/$CITIES.txt" \
   "$CURDIR/node_modules/@hebcal/geo-sqlite/cities-patch.txt" \
   $TMPDIR/admin1CodesASCII.txt \
   $TMPDIR/IL.txt \

package/dist/index.js

@@ -1,4 +1,4 @@
-/*! @hebcal/geo-sqlite v5.6.0 */
+/*! @hebcal/geo-sqlite v5.7.1 */
 import Database from 'better-sqlite3';
 import QuickLRU from 'quick-lru';
 import { Location, Locale } from '@hebcal/core';
@@ -511,7 +511,7 @@ function munge(s) {
 }
 
 // DO NOT EDIT THIS AUTO-GENERATED FILE!
-const version = '5.6.0';
+const version = '5.7.1';
 
 const GEONAME_SQL = `SELECT
   g.name as name,
@@ -1044,6 +1044,22 @@ class GeoDb {
   }
 }
 
+/* eslint-disable no-multi-spaces */
+
+const fcodeKeep = {
+  PPL: true,   // populated place: a city, town, village, or other agglomeration of
+  PPLA: true,  // seat of a first-order administrative division (PPLC takes precedence over PPLA)
+  PPLA2: true, // seat of a second-order administrative division
+  PPLA3: true, // seat of a third-order administrative division
+  PPLC: true,  // capital of a political entity
+  // an area similar to a locality but with a small group of dwellings or other buildings
+  PPLL: true,  // populated locality
+  // cities, towns, villages, or other agglomerations of buildings where people live and work
+  PPLS: true,  // populated places
+  PPLX: true,  // section of populated place
+  STLMT: true, // israeli settlement
+};
+
 /**
  * Builds `geonames.sqlite3` from files downloaded from geonames.org
  * @param {any} opts
@@ -1117,11 +1133,30 @@ async function buildGeonamesSqlite(opts) {
     a[3] = '';
     return true;
   };
-  await doFile(logger, db, cities5000txt, 'geoname', 19, truncateAlternateNames);
+  const minPopulation = opts.population;
+  const citiesFilter = (a) => {
+    if (!a[17]) {
+      return false; // require a non-empty iana timezone id
+    }
+    const fcode = a[7];
+    logger.debug(a[0], a[1], fcode);
+    if (!fcodeKeep[fcode]) {
+      return false;
+    }
+    if (minPopulation) {
+      const population = a[14];
+      if (fcode === 'PPL' && population && population < minPopulation) {
+        return false;
+      }
+    }
+    a[3] = '';
+    return true;
+  };
+  await doFile(logger, db, cities5000txt, 'geoname', 19, citiesFilter);
   await doFile(logger, db, citiesPatch, 'geoname', 19, truncateAlternateNames);
   await doFile(logger, db, ILtxt, 'geoname', 19, (a) => {
     a[3] = '';
-    return a[6] == 'P' && (a[7] == 'PPL' || a[7] == 'STLMT');
+    return a[6] == 'P' && fcodeKeep[a[7]];
   });
 
   doSql(logger, db,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hebcal/geo-sqlite",
-  "version": "5.6.0",
+  "version": "5.7.1",
   "author": "Michael J. Radwin (https://github.com/mjradwin)",
   "keywords": [
     "hebcal"
@@ -38,10 +38,11 @@
     "@hebcal/cities": "^6.1.0",
     "@hebcal/core": "^6.0.6",
     "better-sqlite3": "^12.5.0",
+    "minimist": "^1.2.8",
     "pino": "^10.1.0",
     "pino-pretty": "^13.1.3",
     "quick-lru": "^7.3.0",
-    "transliteration": "^2.3.5"
+    "transliteration": "^2.6.0"
   },
   "scripts": {
     "build:rollup": "rollup -c",
@@ -59,16 +60,16 @@
   },
   "license": "BSD-2-Clause",
   "devDependencies": {
-    "@eslint/js": "^9.39.1",
+    "@eslint/js": "^9.39.2",
     "@rollup/plugin-json": "^6.1.0",
     "@rollup/plugin-node-resolve": "^16.0.3",
     "ava": "^6.4.1",
-    "eslint": "^9.39.1",
+    "eslint": "^9.39.2",
     "eslint-config-google": "^0.14.0",
     "eslint-plugin-n": "^17.23.1",
     "globals": "^16.5.0",
     "jsdoc": "^4.0.5",
     "jsdoc-to-markdown": "^9.1.3",
-    "rollup": "^4.53.3"
+    "rollup": "^4.54.0"
   }
 }