@ferrybig/cgeodata-reader 0.0.0-stage → 0.1.0

package/dist/reader.d.ts

@@ -0,0 +1,534 @@
+import { type BoundingBox } from "./cellBounds.ts";
+import { type ChunkRecords } from "./chunk.ts";
+import type { CGeoDataDirectory } from "./directory.ts";
+import type { CGeoDataHeader } from "./header.ts";
+import { type Inflater } from "./inflate.ts";
+import { LocateIndex } from "./locateIndex.ts";
+import { SearchIndex } from "./searchIndex.ts";
+import { type SourceInput } from "./source.ts";
+/** `lookup()`'s result: where a point resolves, without names. */
+export interface PlaceIdResult {
+    /** The global id (Section 7.5) of the finest place resolved -- the city,
+     * else the region, else the country -- or null where nothing resolved.
+     * Two results with the same id have the same names, so a caller doing
+     * many lookups can key per-place work (a color, a label) on it instead
+     * of on the names. */
+    placeId: number | null;
+    /**
+     * The lat/lon rectangle this result is valid for -- every coordinate in
+     * this box resolves to the exact same result as the one just queried.
+     * Callers doing many nearby lookups (e.g. a tile renderer sampling a
+     * grid of pixels) can cache on this to skip repeat queries that would
+     * land in the same underlying cell. It's as tight as the cell that
+     * produced the result when one was matched; for "unknown" results it's
+     * the whole empty chunk when possible, otherwise the single finest-grain
+     * cell containing the query point (always correct, just not maximal --
+     * the actual empty region in a gap between records isn't generally a
+     * plain rectangle, so this doesn't try to find its full extent).
+     */
+    boundingBox: BoundingBox;
+    /** Index of the chunk the query point falls in (Section 6.1's
+     * `r * W + c`), whether or not that chunk holds any records. Useful for
+     * narrowing a rebuild to one chunk with the compressor's `--only-chunks`
+     * when a query resolves unexpectedly. */
+    chunkIndex: number;
+}
+/** `query()`'s result: `lookup()`'s, with the place and its ancestors
+ * named. A level that doesn't apply is null: `city` for a point resolved
+ * only to a region, all three where nothing resolved. */
+export interface QueryResult extends PlaceIdResult {
+    /** The country the point lies in. */
+    country: PlaceRef | null;
+    /** The region (province, state, ...) the point lies in. */
+    region: PlaceRef | null;
+    /** The city -- or whatever finest division the country has; see the
+     * README -- the point lies in. */
+    city: PlaceRef | null;
+}
+/** The three tiers of the place hierarchy (Section 7.1). */
+export type PlaceLevel = "country" | "region" | "city";
+/** A place's global id (Section 7.5) and its name entry (Section 7.4). */
+export interface PlaceRef {
+    /** The place's global id (Section 7.5): what `place()`, `placeIds()`
+     * and `locate()` take. Stable within one file, not between builds. */
+    id: number;
+    /**
+     * The place's name in its own language and script, as people there
+     * write it: "東京都", "Den Haag". Field 0 of its name entry, never
+     * empty.
+     */
+    localName: string;
+    /**
+     * The place's international (English) name -- "Tokyo", "The Hague" --
+     * or `localName` where it has none, which is also the case for a place
+     * whose names are the same. Field 1 of its name entry. Pick this one
+     * when a name is shown to everyone regardless of their language, or
+     * stored once for all of them.
+     */
+    internationalName: string;
+    /**
+     * Every name, in name-entry order (Section 7.4): `localName`, then the
+     * international name (`""` where it has none but alternatives follow),
+     * then alternative names and spellings ("'s-Gravenhage"). For search
+     * and for showing alternatives; to display one name, use `localName`
+     * or `internationalName` rather than indexing into this.
+     */
+    names: string[];
+}
+/** A place's level and the global ids of it and its ancestors, shaped
+ * like `Place` without the names. */
+export interface PlaceIds {
+    /** The place's own level. */
+    level: PlaceLevel;
+    /** Id of its country (the place itself, for a country). */
+    country: number;
+    /** Id of its region (the place itself, for a region); null for a
+     * country. */
+    region: number | null;
+    /** The place's own id for a city; null otherwise. */
+    city: number | null;
+}
+/**
+ * One entry of the place hierarchy with its ancestors resolved. Shaped
+ * like QueryResult's country/region/city trio so the two are easy to
+ * relate: a country has `region` and `city` null, a region has `city`
+ * null and `region` pointing at itself, a city has all three set.
+ */
+export interface Place extends PlaceRef {
+    /** The place's own level. */
+    level: PlaceLevel;
+    /** Its country (itself, for a country). */
+    country: PlaceRef;
+    /** Its region (itself, for a region); null for a country. */
+    region: PlaceRef | null;
+    /** Itself, for a city; null otherwise. */
+    city: PlaceRef | null;
+}
+/** Narrows `places()` and `search()`. Filters combine: `{ level: "city",
+ * country }` is that country's cities. */
+export interface PlaceFilter {
+    /** Only places of this level. */
+    level?: PlaceLevel;
+    /** Only places inside this country (by id); the country itself included. */
+    country?: number;
+    /** Only places inside this region (by id); the region itself included. */
+    region?: number;
+}
+/** Options for `search()`. */
+export interface SearchOptions extends PlaceFilter {
+    /** Stop after this many matches. */
+    limit?: number;
+    /**
+     * Rank matches by relevance instead of id order (Appendix B.5): an
+     * exact name match outranks a match on a whole word of the name, which
+     * outranks a plain substring match, each tier further split by whether
+     * the query's own spelling (not just its folded form) matched; among
+     * equal matches, a country outranks a city, which outranks a region. `limit` still applies,
+     * but to the top of the ranked list rather than the first N found in
+     * id order -- so with `sort` the whole match set has to be scored
+     * before anything is yielded, since the best match isn't necessarily
+     * the first one found.
+     */
+    sort?: boolean;
+    /**
+     * With `sort`, whether to break ties by area (Appendix B.5.1): among
+     * equally good matches, larger places first, so "berlin" puts Berlin,
+     * Germany ahead of the many small Berlins in the United States, while a
+     * province still follows its own same-named city. Areas come from a
+     * locate index (`useLocateIndex()`):
+     *
+     * - `"auto"` (the default): by area when a locate index is attached,
+     *   otherwise by Appendix B.5's plain order. Unlike everything else an
+     *   index does, attaching one therefore changes the order of results
+     *   (never which places match).
+     * - `true`: by area; rejects without a locate index.
+     * - `false`: Appendix B.5's plain order, index or not.
+     */
+    areaTieBreak?: "auto" | boolean;
+}
+/** One record of a chunk (Section 8.2) with its cell and place resolved. */
+export interface ChunkRecord {
+    /** The record's stored Morton code within its chunk (Section 8.1). */
+    code: number;
+    /** The record's depth in the chunk's quadtree (0 = the whole chunk). */
+    depth: number;
+    /** The cell the record covers. */
+    boundingBox: BoundingBox;
+    /** null for a "no place" record (palette index 0). */
+    place: Place | null;
+}
+/** How many places of each level a file has. */
+export interface PlaceCounts {
+    /** Number of countries; their ids are `0 .. countries - 1`. */
+    countries: number;
+    /** Number of regions; their ids follow the countries'. */
+    regions: number;
+    /** Number of cities; their ids follow the regions'. */
+    cities: number;
+}
+/** `locate()`'s result: a point inside a place, and the cell it's the
+ * centre of. */
+export interface Location {
+    /** Latitude of the centre of `boundingBox`. */
+    lat: number;
+    /** Longitude of the centre of `boundingBox`. */
+    lon: number;
+    /** The place's representative cell (Section C.3). */
+    boundingBox: BoundingBox;
+    /**
+     * Roughly how large the place is, everything under it included, in
+     * square degrees -- from its area byte (Section C.3), so within about
+     * 9% either way. Useful for zooming a map to it: a square of this area
+     * is `sqrt(area)` degrees on a side, which a Web Mercator map shows
+     * across one 256-pixel tile at zoom `log2(360 / sqrt(area))`.
+     */
+    area: number;
+}
+/**
+ * A sidecar index for `OpenOptions`: the index itself (an open one stays
+ * the caller's to close), its file as a source, or a function that finds
+ * it on first use and resolves to undefined when there is none.
+ */
+export type IndexOption<I> = I | SourceInput | (() => Promise<I | SourceInput | undefined>);
+/** Options for `CGeoDataReader.open()`. */
+export interface OpenOptions {
+    /**
+     * Memory budget, in bytes, for decoded chunks and name blocks; least
+     * recently used ones are dropped past it. Default 64 MiB. A worldwide
+     * file decodes to about 1 GB in all, its largest chunk to about 3 MB,
+     * so a budget much below that makes lookups scattered across the
+     * world re-decode often. 0 disables caching, except for the one most
+     * recently used chunk.
+     */
+    cacheBytes?: number;
+    /** How compressed blocks are inflated. Default `streamInflater`, which
+     * works everywhere; on Node.js, `nodeInflater` from `./node` is
+     * several times faster. */
+    inflater?: Inflater;
+    /** A search index (Appendix B) for `search()`; see `useSearchIndex()`.
+     * Not read until the first search that can use it. */
+    searchIndex?: IndexOption<SearchIndex>;
+    /** A locate index (Appendix C) for `locate()`; see `useLocateIndex()`.
+     * Not read until the first `locate()`. */
+    locateIndex?: IndexOption<LocateIndex>;
+}
+/** `OpenOptions.cacheBytes` when not given. */
+export declare const DEFAULT_CACHE_BYTES: number;
+/**
+ * A CGEODATA file opened for reading: answers which country, region and
+ * city a point lies in (`query`, `lookup`), what a place is called and
+ * where it sits in the hierarchy (`place`, `placeIds`), which places
+ * match a name (`search`), and where a place is (`locate`).
+ *
+ * Opening reads only the header, directory and place counts; each lookup
+ * reads and decodes the one chunk its point lies in, which then stays
+ * cached (`OpenOptions.cacheBytes`).
+ *
+ * @example
+ * ```ts
+ * // Node.js
+ * import { openFile } from "@ferrybig/cgeodata-reader/node";
+ * const reader = await openFile("world.bin");
+ *
+ * // Anywhere else
+ * import { CGeoDataReader } from "@ferrybig/cgeodata-reader";
+ * const reader = await CGeoDataReader.open(await (await fetch(url)).arrayBuffer());
+ *
+ * const { city, region, country } = await reader.query(52.0799, 4.3113);
+ * city?.internationalName; // "The Hague"
+ * ```
+ */
+export declare class CGeoDataReader {
+    /** Decoded chunks (keyed by chunk index) and name blocks (keyed by
+     * `-1 - blockIndex`) under one byte budget. */
+    private readonly cache;
+    /** Loads in flight, under the same keys as `cache`, so concurrent
+     * lookups in one chunk decode it once. */
+    private readonly loading;
+    /** The chunk the last lookup used, held outside `cache`: lookups close
+     * together (a tile's samples) mostly land in the same chunk, and this
+     * answers those without touching the LRU's bookkeeping. */
+    private lastChunkIndex;
+    private lastChunk;
+    /** Decodes take turns, one per macrotask; see `TurnQueue`. */
+    private readonly decodes;
+    /** The full-precision cell of the point `gridCell()` last placed; read
+     * right after, in the same synchronous run. */
+    private gx;
+    private gy;
+    private readonly search_;
+    private readonly locate_;
+    /** Rank areas of the regions searches have matched so far (Appendix
+     * B.5.1), -1 where not yet known, for the area bytes they came from. */
+    private rankAreas;
+    private placeTableChecksumPromise;
+    private prefixChecksumPromise;
+    private readonly source;
+    private readonly inflater;
+    /** The file's header (Section 5), including its `attribution`. */
+    readonly header: CGeoDataHeader;
+    /** The file's directory (Section 6): where each chunk is. */
+    readonly directory: CGeoDataDirectory;
+    private readonly placeTable;
+    private constructor();
+    /**
+     * Opens a CGEODATA file: reads and validates its header, directory and
+     * the uncompressed part of its place table (Sections 5-7.2), and
+     * nothing else -- chunks and names are read on first use. The reader
+     * takes ownership of `input`, if it's a `ByteSource`: `close()` closes
+     * it, as does a failed open.
+     */
+    static open(input: SourceInput, options?: OpenOptions): Promise<CGeoDataReader>;
+    /** Closes the source, and any index this reader opened itself. */
+    close(): Promise<void>;
+    /** How many countries, regions and cities the file has. */
+    get placeCounts(): PlaceCounts;
+    /**
+     * The country, region and city a point lies in: Section 9's lookup
+     * procedure (`lookup()`) with the names resolved (`place()`). Rejects
+     * with a `RangeError` for a latitude outside [-90, 90) or a longitude
+     * outside [-180, 180).
+     *
+     * @example
+     * ```ts
+     * const result = await reader.query(35.6762, 139.6503);
+     * result.city?.localName;         // "杉並区"
+     * result.city?.internationalName; // "Suginami"
+     * result.country?.internationalName; // "Japan"
+     * ```
+     */
+    query(lat: number, lon: number): Promise<QueryResult>;
+    /**
+     * Section 9's lookup procedure: the place a point lies in, as an id, and
+     * the cell the answer holds for. Reads and decodes the point's chunk if
+     * it isn't cached; resolve the id with `place()` or `placeIds()`.
+     */
+    lookup(lat: number, lon: number): Promise<PlaceIdResult>;
+    /**
+     * `lookup()` answered from memory only: the same result when the
+     * point's chunk is already decoded, undefined when answering would
+     * need a read. For callers doing many lookups close together, which
+     * mostly hit the same few chunks:
+     *
+     * ```ts
+     * const result = reader.lookupCached(lat, lon) ?? (await reader.lookup(lat, lon));
+     * ```
+     *
+     * waits only for the lookups that actually read something.
+     */
+    lookupCached(lat: number, lon: number): PlaceIdResult | undefined;
+    /** The index of the chunk a point lies in (Section 6.1). */
+    chunkIndexAt(lat: number, lon: number): number;
+    /** Validates a point and places it on the full-precision grid
+     * (`this.gx`, `this.gy`), returning its chunk's index. */
+    private gridCell;
+    /** Section 9 steps 5-8 for the point `gridCell()` just placed, in its
+     * (decoded, or null for empty) chunk. */
+    private resolve;
+    /** The names of one place by global id, without resolving its ancestors.
+     * Cheaper than `place()` when only the names are needed. */
+    placeNames(id: number): Promise<string[]>;
+    /** FNV-1a 64-bit hash of the place table bytes, as defined by
+     * binary-format.md Section B.6. Used to bind a search index to its source. */
+    placeTableChecksum(): Promise<bigint>;
+    /** FNV-1a 64-bit hash of everything before the first chunk -- header,
+     * directory and place table -- as defined by binary-format.md Section
+     * C.4. Binds a locate index to its source: the chunk lengths it covers
+     * change whenever the chunks do. */
+    prefixChecksum(): Promise<bigint>;
+    /**
+     * Attaches a text search index (Appendix B, built by `buildSearchIndex`
+     * / `cgeodata build-search-index`) so `search()` only checks the places
+     * the index says could match instead of every place. Results are
+     * identical with or without it -- only the speed differs.
+     *
+     * Takes the index's file (as for `open()`; read on demand) or an open
+     * `SearchIndex` (which the caller keeps ownership of), or undefined to
+     * search without one. Replaces `OpenOptions.searchIndex`. Rejects with
+     * a `CGeoDataIndexMismatchError` if it was built from a different file.
+     */
+    useSearchIndex(index: SearchIndex | SourceInput | undefined): Promise<void>;
+    /** Whether `search()` will use an index. Reads (and validates) the one
+     * from `OpenOptions.searchIndex` if that hasn't happened yet. */
+    hasSearchIndex(): Promise<boolean>;
+    /** The attached search index, opening `OpenOptions.searchIndex` on
+     * first use. One that's there but unusable (stale, corrupt) rejects on
+     * every call rather than silently falling back to a linear scan. */
+    private activeSearchIndex;
+    private checkSearchIndex;
+    /**
+     * Attaches a locate index (Appendix C, built by `buildLocateIndex` /
+     * `cgeodata build-locate-index`) so `locate()` reads one entry instead
+     * of scanning every chunk. Results are identical with or without it --
+     * only the speed differs -- except that its areas also let a sorted
+     * `search()` rank equal matches by size (`SearchOptions.areaTieBreak`).
+     *
+     * Takes the index's file (as for `open()`; read on demand) or an open
+     * `LocateIndex` (which the caller keeps ownership of), or undefined to
+     * locate without one. Replaces `OpenOptions.locateIndex`. Rejects with
+     * a `CGeoDataIndexMismatchError` if it was built from a different file.
+     */
+    useLocateIndex(index: LocateIndex | SourceInput | undefined): Promise<void>;
+    /** Whether `locate()` will use an index. Reads (and validates) the one
+     * from `OpenOptions.locateIndex` if that hasn't happened yet. */
+    hasLocateIndex(): Promise<boolean>;
+    /** The attached locate index, opening `OpenOptions.locateIndex` on
+     * first use. One that's there but unusable (stale, corrupt) rejects on
+     * every call rather than silently falling back to a chunk scan. */
+    private activeLocateIndex;
+    private checkLocateIndex;
+    /** `useSearchIndex`/`useLocateIndex`, for either slot. */
+    private attach;
+    /** The slot's index, trying its pending one first. */
+    private activate;
+    private tryPending;
+    /** Resolves one place id (Section 7.5) to its names and ancestors. */
+    place(id: number): Promise<Place>;
+    /** `place()` when every name it needs is cached, else undefined. */
+    private loadedPlace;
+    /** `place()` loading each name block in turn, for when the cache is
+     * too small to hold them all at once. */
+    private placeFrom;
+    /** `place()` without the names: the place's level and the global ids
+     * of it and its ancestors, from the place table's counts alone, so no
+     * name block is read -- and so synchronous. */
+    placeIds(id: number): PlaceIds;
+    /** How many places sit directly under `id`: a country's regions, a
+     * region's cities, always 0 for a city. From the place table's counts
+     * alone, like `placeIds()`, so no name block is read and it is
+     * synchronous: `childCount(region) === 0` finds a region with no city
+     * mapped to it. */
+    childCount(id: number): number;
+    /**
+     * Walks the place hierarchy in id order (countries, then regions, then
+     * cities), optionally restricted to one level and/or one ancestor. Each
+     * level's ids for a given parent are contiguous (Section 7.5), so a
+     * filtered walk only touches the ids it yields.
+     */
+    places(filter?: PlaceFilter): AsyncIterableIterator<Place>;
+    /**
+     * Diacritic- and case-insensitive substring match against every name of
+     * every place (all levels unless filtered) -- Section B.5's "matches":
+     * the folded form (`foldDiacritics`) of a name contains the folded form
+     * of the query. Without a search index this is a plain linear scan over
+     * the name blocks; with one (`useSearchIndex()`), only the places the
+     * index can't rule out are compared. Same results either way.
+     *
+     * A text with commas also reads as a place and the places it lies in
+     * (Appendix B.13): "Paris, Texas" finds a place named "Paris, Texas"
+     * and every "Paris" whose region or country matches "Texas";
+     * "Springfield, Illinois, United States" narrows by both. Each part
+     * after a comma has to match a different ancestor, nearest first.
+     *
+     * @example
+     * ```ts
+     * for await (const place of reader.search("munchen", { sort: true, limit: 20 }))
+     * 	console.log(place.internationalName, place.country.internationalName);
+     * // Munich Germany
+     * // Grafing bei München Germany
+     * // ...
+     * ```
+     */
+    search(text: string, options?: SearchOptions): AsyncIterableIterator<Place>;
+    /** The `sort: true` path of `search()`: scores every match in relevance
+     * order (Appendix B.5, or B.5.1 when ranking by area; `relevance.ts`),
+     * then yields the best first. `limit` is applied after sorting, not
+     * during the scan -- the top match can turn up anywhere in id order. */
+    private searchRanked;
+    /** `text`'s readings (Appendix B.13) as needles, and the places a search
+     * index says could match any of them (ascending), or undefined when
+     * there's no index or it can't narrow one of the readings down. */
+    private prepareQuery;
+    /** Whether place `id` matches any reading of `query`. With `compare`,
+     * instead the best-ranked way it does (by that order), or undefined;
+     * `rankArea` is left at 0 for the caller to fill in. */
+    private matchQuery;
+    /** The names of place `id`'s ancestors, nearest first: a city's region
+     * and country, a region's country, nothing for a country. */
+    private ancestorNames;
+    /** `areaTieBreak` resolved to the area bytes to rank by, or undefined
+     * for Appendix B.5's plain order. */
+    private searchAreas;
+    /**
+     * The area byte (binary-format.md Section C.3) of every place, by id --
+     * the size of the place and everything under it, on a log scale, a
+     * quarter of a doubling per step -- read from the attached locate index
+     * (`useLocateIndex()`), or undefined without one. What
+     * `search({ sort: true })` breaks ties with (`SearchOptions.areaTieBreak`).
+     * Read once and then kept; the array is shared, so don't modify it.
+     */
+    placeAreas(): Promise<Uint8Array | undefined>;
+    /** Place `id`'s rank area (Appendix B.5.1, `regionRankArea`): its area
+     * byte, except for a region with a city of the same local name. That
+     * takes reading the region's city names, so it's worked out only for
+     * the regions a search actually matched, once each. */
+    private rankArea;
+    /** The ids (ascending) of `level` under the filter that `search()` has
+     * to check: the whole `idRange`, or, when a search index narrowed the
+     * query to `candidates` (ascending), just those within it. */
+    private searchScope;
+    /** The parsed records of one chunk (Section 8), or undefined when the
+     * chunk is empty. Cached like lookups, unless `cache` is false: a
+     * one-pass walk over every chunk should skip the cache, so it doesn't
+     * evict everything lookups had cached for chunks it only visits once. */
+    chunk(index: number, { cache }?: {
+        cache?: boolean;
+    }): Promise<ChunkRecords | undefined>;
+    /** The lat/lon extent of a chunk (Section 6.1). */
+    chunkBoundingBox(index: number): BoundingBox;
+    /**
+     * A representative lat/lon for a place id: the centre of the place's
+     * representative cell (binary-format.md Section C.3), a cell near the
+     * middle of the place's area -- its own cells plus those of every place
+     * under it -- that is guaranteed to lie in that area, so `query()` at
+     * the returned point resolves to the place or one of its descendants.
+     *
+     * @example
+     * ```ts
+     * const position = await reader.locate(place.id);
+     * if (position) map.setView([position.lat, position.lon]);
+     * ```
+     *
+     * Null if neither the place nor anything under it has any cells -- and
+     * that happens to real places, not only to broken files: a place
+     * smaller than the file's finest cell (a tiny island, an enclave, a
+     * boundary sliver) loses every cell to its neighbours. On a worldwide
+     * build that's about 0.15% of places, every one of them still found
+     * by `search()`; drop them from results that need a position rather
+     * than defaulting to a coordinate.
+     *
+     * With a locate index (`useLocateIndex()`) this reads one entry;
+     * without one it scans every chunk. Same result either way.
+     */
+    locate(id: number): Promise<Location | null>;
+    /**
+     * The ids of place `id` and of every place under it, as half-open
+     * ranges: `[id, id + 1]` first, then for a country its regions and
+     * their cities, for a region its cities. Each level's children are
+     * contiguous (Section 7.1), so this is at most three ranges.
+     */
+    subtreeIdRanges(id: number): Array<[number, number]>;
+    /** One chunk's records with their cells and places resolved -- what a
+     * lookup in that chunk can answer with, laid out for inspection. */
+    chunkRecords(index: number): AsyncIterableIterator<ChunkRecord>;
+    private recordBoundingBox;
+    /** The half-open global id range of `level` under the filter's
+     * narrowest ancestor, via the Section 7.5 prefix arrays. */
+    private idRange;
+    private checkPlaceId;
+    /** A chunk without reading anything: its records when decoded and
+     * cached, null when it's empty, undefined when it would need a read. */
+    private loadedChunk;
+    /** Reads and decodes a chunk, sharing one decode among concurrent
+     * callers. */
+    private loadChunk;
+    private decodeChunk;
+    private checkPalette;
+    /** A place's names when its name block is cached, else undefined. */
+    private loadedNames;
+    /** Reads and decodes a place's name block (sharing one read among
+     * concurrent callers), and returns its names. */
+    private loadNames;
+    private get nameBlockSize();
+    private entryOf;
+}