@hebcal/geo-sqlite 5.4.0 → 5.5.0

package/README.md

@@ -33,20 +33,43 @@ db.close();
 </dd>
 </dl>
 
+## Functions
+
+<dl>
+<dt><a href="#buildGeonamesSqlite">buildGeonamesSqlite(opts)</a></dt>
+<dd><p>Builds <code>geonames.sqlite3</code> from files downloaded from geonames.org</p>
+</dd>
+<dt><a href="#doSql">doSql(logger, db, ...sqls)</a></dt>
+<dd></dd>
+<dt><a href="#doFile">doFile(logger, db, infile, tableName, expectedFields, callback)</a></dt>
+<dd></dd>
+</dl>
 
 <a name="GeoDb"></a>
 
 ## GeoDb
 Wrapper around sqlite databases
 
-**Kind**: global class  
+**Kind**: global class
 
 * [GeoDb](#GeoDb)
     * [new GeoDb(logger, zipsFilename, geonamesFilename)](#new_GeoDb_new)
-    * [.close()](#GeoDb+close)
-    * [.lookupZip(zip)](#GeoDb+lookupZip) ⇒ <code>Location</code>
-    * [.lookupGeoname(geonameid)](#GeoDb+lookupGeoname) ⇒ <code>Location</code>
-    * [.lookupLegacyCity(cityName)](#GeoDb+lookupLegacyCity) ⇒ <code>Location</code>
+    * _instance_
+        * [.zipCache](#GeoDb+zipCache) : <code>Map.&lt;string, Location&gt;</code>
+        * [.geonamesCache](#GeoDb+geonamesCache) : <code>Map.&lt;number, Location&gt;</code>
+        * [.legacyCities](#GeoDb+legacyCities) : <code>Map.&lt;string, number&gt;</code>
+        * [.countryNames](#GeoDb+countryNames) : <code>Map.&lt;string, string&gt;</code>
+        * [.close()](#GeoDb+close)
+        * [.lookupZip(zip)](#GeoDb+lookupZip) ⇒ <code>Location</code>
+        * [.lookupGeoname(geonameid)](#GeoDb+lookupGeoname) ⇒ <code>Location</code>
+        * [.lookupLegacyCity(cityName)](#GeoDb+lookupLegacyCity) ⇒ <code>Location</code>
+        * [.autoComplete(qraw, latlong)](#GeoDb+autoComplete) ⇒ <code>Array.&lt;Object&gt;</code>
+        * [.cacheZips()](#GeoDb+cacheZips)
+        * [.cacheGeonames()](#GeoDb+cacheGeonames)
+    * _static_
+        * [.transliterate(source, [options])](#GeoDb.transliterate) ⇒ <code>string</code>
+        * [.geonameCityDescr(cityName, admin1, countryName)](#GeoDb.geonameCityDescr) ⇒ <code>string</code>
+        * [.version()](#GeoDb.version)
 
 <a name="new_GeoDb_new"></a>
 
@@ -54,39 +77,147 @@ Wrapper around sqlite databases
 
 | Param | Type |
 | --- | --- |
-| logger | <code>any</code> | 
-| zipsFilename | <code>string</code> | 
-| geonamesFilename | <code>string</code> | 
+| logger | <code>any</code> |
+| zipsFilename | <code>string</code> |
+| geonamesFilename | <code>string</code> |
+
+<a name="GeoDb+zipCache"></a>
+
+### geoDb.zipCache : <code>Map.&lt;string, Location&gt;</code>
+**Kind**: instance property of [<code>GeoDb</code>](#GeoDb)
+<a name="GeoDb+geonamesCache"></a>
 
+### geoDb.geonamesCache : <code>Map.&lt;number, Location&gt;</code>
+**Kind**: instance property of [<code>GeoDb</code>](#GeoDb)
+<a name="GeoDb+legacyCities"></a>
+
+### geoDb.legacyCities : <code>Map.&lt;string, number&gt;</code>
+**Kind**: instance property of [<code>GeoDb</code>](#GeoDb)
+<a name="GeoDb+countryNames"></a>
+
+### geoDb.countryNames : <code>Map.&lt;string, string&gt;</code>
+**Kind**: instance property of [<code>GeoDb</code>](#GeoDb)
 <a name="GeoDb+close"></a>
 
 ### geoDb.close()
 Closes database handles
 
-**Kind**: instance method of [<code>GeoDb</code>](#GeoDb)  
+**Kind**: instance method of [<code>GeoDb</code>](#GeoDb)
 <a name="GeoDb+lookupZip"></a>
 
 ### geoDb.lookupZip(zip) ⇒ <code>Location</code>
-**Kind**: instance method of [<code>GeoDb</code>](#GeoDb)  
+**Kind**: instance method of [<code>GeoDb</code>](#GeoDb)
 
 | Param | Type |
 | --- | --- |
-| zip | <code>string</code> | 
+| zip | <code>string</code> |
 
 <a name="GeoDb+lookupGeoname"></a>
 
 ### geoDb.lookupGeoname(geonameid) ⇒ <code>Location</code>
-**Kind**: instance method of [<code>GeoDb</code>](#GeoDb)  
+**Kind**: instance method of [<code>GeoDb</code>](#GeoDb)
 
 | Param | Type |
 | --- | --- |
-| geonameid | <code>number</code> | 
+| geonameid | <code>number</code> |
 
 <a name="GeoDb+lookupLegacyCity"></a>
 
 ### geoDb.lookupLegacyCity(cityName) ⇒ <code>Location</code>
-**Kind**: instance method of [<code>GeoDb</code>](#GeoDb)  
+**Kind**: instance method of [<code>GeoDb</code>](#GeoDb)
+
+| Param | Type |
+| --- | --- |
+| cityName | <code>string</code> |
+
+<a name="GeoDb+autoComplete"></a>
+
+### geoDb.autoComplete(qraw, latlong) ⇒ <code>Array.&lt;Object&gt;</code>
+Generates autocomplete results based on a query string
+
+**Kind**: instance method of [<code>GeoDb</code>](#GeoDb)
+
+| Param | Type | Default |
+| --- | --- | --- |
+| qraw | <code>string</code> |  |
+| latlong | <code>boolean</code> | <code>false</code> |
+
+<a name="GeoDb+cacheZips"></a>
+
+### geoDb.cacheZips()
+Reads entire ZIP database and caches in-memory
+
+**Kind**: instance method of [<code>GeoDb</code>](#GeoDb)
+<a name="GeoDb+cacheGeonames"></a>
+
+### geoDb.cacheGeonames()
+Reads entire geonames database and caches in-memory
+
+**Kind**: instance method of [<code>GeoDb</code>](#GeoDb)
+<a name="GeoDb.transliterate"></a>
+
+### GeoDb.transliterate(source, [options]) ⇒ <code>string</code>
+Convenience wrapper of the `transliterate` function from `transliteration` npm package.
+Transliterate the string `source` and return the result.
+
+**Kind**: static method of [<code>GeoDb</code>](#GeoDb)
+
+| Param | Type |
+| --- | --- |
+| source | <code>string</code> |
+| [options] | <code>any</code> |
+
+<a name="GeoDb.geonameCityDescr"></a>
+
+### GeoDb.geonameCityDescr(cityName, admin1, countryName) ⇒ <code>string</code>
+Builds a city description from geonameid string components
+
+**Kind**: static method of [<code>GeoDb</code>](#GeoDb)
+
+| Param | Type | Description |
+| --- | --- | --- |
+| cityName | <code>string</code> | e.g. 'Tel Aviv' or 'Chicago' |
+| admin1 | <code>string</code> | e.g. 'England' or 'Massachusetts' |
+| countryName | <code>string</code> | full country name, e.g. 'Israel' or 'United States' |
+
+<a name="GeoDb.version"></a>
+
+### GeoDb.version()
+Returns the version of the GeoDb package
+
+**Kind**: static method of [<code>GeoDb</code>](#GeoDb)
+<a name="buildGeonamesSqlite"></a>
+
+## buildGeonamesSqlite(opts)
+Builds `geonames.sqlite3` from files downloaded from geonames.org
+
+**Kind**: global function
+
+| Param | Type |
+| --- | --- |
+| opts | <code>any</code> |
+
+<a name="doSql"></a>
+
+## doSql(logger, db, ...sqls)
+**Kind**: global function
+
+| Param | Type |
+| --- | --- |
+| logger | <code>pino.Logger</code> |
+| db | <code>Database</code> |
+| ...sqls | <code>string</code> |
+
+<a name="doFile"></a>
+
+## doFile(logger, db, infile, tableName, expectedFields, callback)
+**Kind**: global function
 
 | Param | Type |
 | --- | --- |
-| cityName | <code>string</code> | 
+| logger | <code>pino.Logger</code> |
+| db | <code>Database</code> |
+| infile | <code>string</code> |
+| tableName | <code>string</code> |
+| expectedFields | <code>number</code> |
+| callback | <code>function</code> |

package/dist/index.js

@@ -1,4 +1,4 @@
-/*! @hebcal/geo-sqlite v5.4.0 */
+/*! @hebcal/geo-sqlite v5.5.0 */
 import Database from 'better-sqlite3';
 import { Location, Locale } from '@hebcal/core';
 import '@hebcal/cities';
@@ -6,6 +6,336 @@ import events from 'events';
 import fs from 'fs';
 import readline from 'readline';
 
+class QuickLRU extends Map {
+	#size = 0;
+	#cache = new Map();
+	#oldCache = new Map();
+	#maxSize;
+	#maxAge;
+	#onEviction;
+
+	constructor(options = {}) {
+		super();
+
+		if (!(options.maxSize && options.maxSize > 0)) {
+			throw new TypeError('`maxSize` must be a number greater than 0');
+		}
+
+		if (typeof options.maxAge === 'number' && options.maxAge === 0) {
+			throw new TypeError('`maxAge` must be a number greater than 0');
+		}
+
+		this.#maxSize = options.maxSize;
+		this.#maxAge = options.maxAge || Number.POSITIVE_INFINITY;
+		this.#onEviction = options.onEviction;
+	}
+
+	// For tests.
+	get __oldCache() {
+		return this.#oldCache;
+	}
+
+	#emitEvictions(cache) {
+		if (typeof this.#onEviction !== 'function') {
+			return;
+		}
+
+		for (const [key, item] of cache) {
+			this.#onEviction(key, item.value);
+		}
+	}
+
+	#deleteIfExpired(key, item) {
+		if (typeof item.expiry === 'number' && item.expiry <= Date.now()) {
+			if (typeof this.#onEviction === 'function') {
+				this.#onEviction(key, item.value);
+			}
+
+			return this.delete(key);
+		}
+
+		return false;
+	}
+
+	#getOrDeleteIfExpired(key, item) {
+		const deleted = this.#deleteIfExpired(key, item);
+		if (deleted === false) {
+			return item.value;
+		}
+	}
+
+	#getItemValue(key, item) {
+		return item.expiry ? this.#getOrDeleteIfExpired(key, item) : item.value;
+	}
+
+	#peek(key, cache) {
+		const item = cache.get(key);
+		return this.#getItemValue(key, item);
+	}
+
+	#set(key, value) {
+		this.#cache.set(key, value);
+		this.#size++;
+
+		if (this.#size >= this.#maxSize) {
+			this.#size = 0;
+			this.#emitEvictions(this.#oldCache);
+			this.#oldCache = this.#cache;
+			this.#cache = new Map();
+		}
+	}
+
+	#moveToRecent(key, item) {
+		this.#oldCache.delete(key);
+		this.#set(key, item);
+	}
+
+	* #entriesAscending() {
+		for (const item of this.#oldCache) {
+			const [key, value] = item;
+			if (!this.#cache.has(key)) {
+				const deleted = this.#deleteIfExpired(key, value);
+				if (deleted === false) {
+					yield item;
+				}
+			}
+		}
+
+		for (const item of this.#cache) {
+			const [key, value] = item;
+			const deleted = this.#deleteIfExpired(key, value);
+			if (deleted === false) {
+				yield item;
+			}
+		}
+	}
+
+	get(key) {
+		if (this.#cache.has(key)) {
+			const item = this.#cache.get(key);
+			return this.#getItemValue(key, item);
+		}
+
+		if (this.#oldCache.has(key)) {
+			const item = this.#oldCache.get(key);
+			if (this.#deleteIfExpired(key, item) === false) {
+				this.#moveToRecent(key, item);
+				return item.value;
+			}
+		}
+	}
+
+	set(key, value, {maxAge = this.#maxAge} = {}) {
+		const expiry = typeof maxAge === 'number' && maxAge !== Number.POSITIVE_INFINITY
+			? (Date.now() + maxAge)
+			: undefined;
+
+		if (this.#cache.has(key)) {
+			this.#cache.set(key, {
+				value,
+				expiry,
+			});
+		} else {
+			this.#set(key, {value, expiry});
+		}
+
+		return this;
+	}
+
+	has(key) {
+		if (this.#cache.has(key)) {
+			return !this.#deleteIfExpired(key, this.#cache.get(key));
+		}
+
+		if (this.#oldCache.has(key)) {
+			return !this.#deleteIfExpired(key, this.#oldCache.get(key));
+		}
+
+		return false;
+	}
+
+	peek(key) {
+		if (this.#cache.has(key)) {
+			return this.#peek(key, this.#cache);
+		}
+
+		if (this.#oldCache.has(key)) {
+			return this.#peek(key, this.#oldCache);
+		}
+	}
+
+	expiresIn(key) {
+		const item = this.#cache.get(key) ?? this.#oldCache.get(key);
+		if (item) {
+			return item.expiry ? item.expiry - Date.now() : Number.POSITIVE_INFINITY;
+		}
+	}
+
+	delete(key) {
+		const deleted = this.#cache.delete(key);
+		if (deleted) {
+			this.#size--;
+		}
+
+		return this.#oldCache.delete(key) || deleted;
+	}
+
+	clear() {
+		this.#cache.clear();
+		this.#oldCache.clear();
+		this.#size = 0;
+	}
+
+	resize(newSize) {
+		if (!(newSize && newSize > 0)) {
+			throw new TypeError('`maxSize` must be a number greater than 0');
+		}
+
+		const items = [...this.#entriesAscending()];
+		const removeCount = items.length - newSize;
+		if (removeCount < 0) {
+			this.#cache = new Map(items);
+			this.#oldCache = new Map();
+			this.#size = items.length;
+		} else {
+			if (removeCount > 0) {
+				this.#emitEvictions(items.slice(0, removeCount));
+			}
+
+			this.#oldCache = new Map(items.slice(removeCount));
+			this.#cache = new Map();
+			this.#size = 0;
+		}
+
+		this.#maxSize = newSize;
+	}
+
+	evict(count = 1) {
+		const requested = Number(count);
+		if (!requested || requested <= 0) {
+			return;
+		}
+
+		const items = [...this.#entriesAscending()];
+		const evictCount = Math.trunc(Math.min(requested, Math.max(items.length - 1, 0)));
+		if (evictCount <= 0) {
+			return;
+		}
+
+		this.#emitEvictions(items.slice(0, evictCount));
+		this.#oldCache = new Map(items.slice(evictCount));
+		this.#cache = new Map();
+		this.#size = 0;
+	}
+
+	* keys() {
+		for (const [key] of this) {
+			yield key;
+		}
+	}
+
+	* values() {
+		for (const [, value] of this) {
+			yield value;
+		}
+	}
+
+	* [Symbol.iterator]() {
+		for (const item of this.#cache) {
+			const [key, value] = item;
+			const deleted = this.#deleteIfExpired(key, value);
+			if (deleted === false) {
+				yield [key, value.value];
+			}
+		}
+
+		for (const item of this.#oldCache) {
+			const [key, value] = item;
+			if (!this.#cache.has(key)) {
+				const deleted = this.#deleteIfExpired(key, value);
+				if (deleted === false) {
+					yield [key, value.value];
+				}
+			}
+		}
+	}
+
+	* entriesDescending() {
+		let items = [...this.#cache];
+		for (let i = items.length - 1; i >= 0; --i) {
+			const item = items[i];
+			const [key, value] = item;
+			const deleted = this.#deleteIfExpired(key, value);
+			if (deleted === false) {
+				yield [key, value.value];
+			}
+		}
+
+		items = [...this.#oldCache];
+		for (let i = items.length - 1; i >= 0; --i) {
+			const item = items[i];
+			const [key, value] = item;
+			if (!this.#cache.has(key)) {
+				const deleted = this.#deleteIfExpired(key, value);
+				if (deleted === false) {
+					yield [key, value.value];
+				}
+			}
+		}
+	}
+
+	* entriesAscending() {
+		for (const [key, value] of this.#entriesAscending()) {
+			yield [key, value.value];
+		}
+	}
+
+	get size() {
+		if (!this.#size) {
+			return this.#oldCache.size;
+		}
+
+		let oldCacheSize = 0;
+		for (const key of this.#oldCache.keys()) {
+			if (!this.#cache.has(key)) {
+				oldCacheSize++;
+			}
+		}
+
+		return Math.min(this.#size + oldCacheSize, this.#maxSize);
+	}
+
+	get maxSize() {
+		return this.#maxSize;
+	}
+
+	get maxAge() {
+		return this.#maxAge;
+	}
+
+	entries() {
+		return this.entriesAscending();
+	}
+
+	forEach(callbackFunction, thisArgument = this) {
+		for (const [key, value] of this.entriesAscending()) {
+			callbackFunction.call(thisArgument, value, key, this);
+		}
+	}
+
+	get [Symbol.toStringTag]() {
+		return 'QuickLRU';
+	}
+
+	toString() {
+		return `QuickLRU(${this.size}/${this.maxSize})`;
+	}
+
+	[Symbol.for('nodejs.util.inspect.custom')]() {
+		return this.toString();
+	}
+}
+
 var city2geonameid = {
 "AD-Andorra La Vella":3041563,
 "AE-Abu Dhabi":292968,
@@ -511,7 +841,7 @@ function munge(s) {
 }
 
 // DO NOT EDIT THIS AUTO-GENERATED FILE!
-const version = '5.4.0';
+const version = '5.5.0';
 
 const GEONAME_SQL = `SELECT
   g.name as name,
@@ -644,10 +974,10 @@ class GeoDb {
     this.geonamesDb = new Database(geonamesFilename, {fileMustExist: true});
     this.zipStmt = this.zipsDb.prepare(ZIPCODE_SQL);
     /** @type {Map<string, Location>} */
-    this.zipCache = new Map();
+    this.zipCache = new QuickLRU({maxSize: 1000});
     this.geonamesStmt = this.geonamesDb.prepare(GEONAME_SQL);
     /** @type {Map<number, Location>} */
-    this.geonamesCache = new Map();
+    this.geonamesCache = new QuickLRU({maxSize: 1000});
     /** @type {Map<string, number>} */
     this.legacyCities = new Map();
     for (const [name, id] of Object.entries(city2geonameid)) {
@@ -1015,6 +1345,7 @@ class GeoDb {
     const start = Date.now();
     const stmt = this.zipsDb.prepare(ZIPCODE_ALL_SQL);
     const rows = stmt.all();
+    this.zipCache = new Map(); // replace QuickLRU
     for (const row of rows) {
       const location = this.makeZipLocation(row);
       this.zipCache.set(row.ZipCode, location);
@@ -1028,6 +1359,7 @@ class GeoDb {
     const start = Date.now();
     const stmt = this.geonamesDb.prepare(GEONAME_ALL_SQL);
     const rows = stmt.all();
+    this.geonamesCache = new Map(); // replace QuickLRU
     for (const row of rows) {
       const location = this.makeGeonameLocation(row.geonameid, row);
       this.geonamesCache.set(row.geonameid, location);
@@ -1036,6 +1368,7 @@ class GeoDb {
     if (this.logger) this.logger.info(`GeoDb: cached ${rows.length} geonames in ${end - start}ms`);
   }
 
+  /** Returns the version of the GeoDb package */
   static version() {
     return version;
   }