@ferrybig/cgeodata-reader 0.0.0-stage → 0.1.0

package/dist/node.d.ts

@@ -0,0 +1,65 @@
+import type { Inflater } from "./inflate.ts";
+import { CGeoDataReader, type OpenOptions } from "./reader.ts";
+import type { ByteSource } from "./source.ts";
+/**
+ * A file on disk as a `ByteSource`: each read reads just that range, on
+ * libuv's thread pool.
+ *
+ * Holds a plain file descriptor rather than a `fs.promises` `FileHandle`:
+ * Node.js treats a `FileHandle` that is garbage collected without being
+ * closed as an uncaught error, which would take a whole server down over
+ * a forgotten `close()`. A forgotten `FileSource` is closed when it's
+ * collected instead.
+ */
+export declare class FileSource implements ByteSource {
+    private fd;
+    readonly size: number;
+    readonly name: string;
+    private constructor();
+    /** Opens the file at `path` for reading. */
+    static open(path: string): Promise<FileSource>;
+    read(offset: number, length: number): Promise<Uint8Array>;
+    close(): Promise<void>;
+}
+/**
+ * Node's zlib, on the calling thread. Deliberately not zlib's async API:
+ * measured on a worldwide file, handing blocks to libuv's thread pool made
+ * a burst of zoomed-out map tiles 15-40% slower and cold lookups ~30%
+ * slower, while barely shortening the longest event-loop stall -- parsing
+ * a decoded chunk costs more than inflating it, and that stays on this
+ * thread either way. What keeps the event loop responsive is that every
+ * chunk read (`FileSource.read`) is asynchronous, so the loop gets a turn
+ * between chunks: a stall is one chunk's decode (~25 ms for the largest
+ * chunk of a worldwide file, most far less), not a whole tile's or batch's.
+ */
+export declare const nodeInflater: Inflater;
+/** Where a search index for `binPath` conventionally lives, and where
+ * `openFile()` looks for one: `data/03-final/world.bin` →
+ * `data/03-final/world.search-index.bin`. */
+export declare function searchIndexPathFor(binPath: string): string;
+/** Where a locate index for `binPath` conventionally lives, and where
+ * `openFile()` looks for one: `data/03-final/world.bin` →
+ * `data/03-final/world.locate-index.bin`. */
+export declare function locateIndexPathFor(binPath: string): string;
+/**
+ * Opens a `.bin` file from disk, reading only what each lookup needs, with
+ * Node's zlib as its inflater.
+ *
+ * The search and locate indexes next to it (`searchIndexPathFor`,
+ * `locateIndexPathFor`) are used automatically when they exist: each is
+ * opened on the first `search()` / `locate()` that can use it. One that
+ * exists but doesn't belong to this file (built from an older build of
+ * it, say) makes those calls reject with a `CGeoDataIndexMismatchError`
+ * naming it, rather than silently falling back to a slow scan; rebuild or
+ * delete it. `options.searchIndex` / `options.locateIndex` override the
+ * lookup next to the file.
+ *
+ * @example
+ * ```ts
+ * const reader = await openFile("data/world.bin");
+ * const { city } = await reader.query(52.37, 4.9);
+ * city?.internationalName; // "Amsterdam"
+ * await reader.close();
+ * ```
+ */
+export declare function openFile(path: string, options?: OpenOptions): Promise<CGeoDataReader>;

package/dist/node.js

@@ -0,0 +1,137 @@
+/**
+ * The Node.js side of the reader: files on disk, and zlib. Everything
+ * else (`CGeoDataReader` and friends) is platform-neutral and lives in the
+ * package's main entry.
+ *
+ * @module @ferrybig/cgeodata-reader/node
+ */
+import { close, fstat, open, read } from "node:fs";
+import { promisify } from "node:util";
+import { inflateSync } from "node:zlib";
+import { CGeoDataError } from "./errors.js";
+import { CGeoDataReader } from "./reader.js";
+const openAsync = promisify(open);
+const fstatAsync = promisify(fstat);
+const closeAsync = promisify(close);
+/** Closes the descriptor of a `FileSource` that was garbage collected
+ * without `close()`, instead of leaking it. */
+const unclosed = new FinalizationRegistry((fd) => close(fd, () => { }));
+/**
+ * A file on disk as a `ByteSource`: each read reads just that range, on
+ * libuv's thread pool.
+ *
+ * Holds a plain file descriptor rather than a `fs.promises` `FileHandle`:
+ * Node.js treats a `FileHandle` that is garbage collected without being
+ * closed as an uncaught error, which would take a whole server down over
+ * a forgotten `close()`. A forgotten `FileSource` is closed when it's
+ * collected instead.
+ */
+export class FileSource {
+    fd;
+    size;
+    name;
+    constructor(fd, size, path) {
+        this.fd = fd;
+        this.size = size;
+        this.name = path;
+        unclosed.register(this, fd, this);
+    }
+    /** Opens the file at `path` for reading. */
+    static async open(path) {
+        const fd = await openAsync(path, "r");
+        try {
+            return new FileSource(fd, (await fstatAsync(fd)).size, path);
+        }
+        catch (error) {
+            await closeAsync(fd);
+            throw error;
+        }
+    }
+    async read(offset, length) {
+        const wanted = Math.max(0, Math.min(length, this.size - offset));
+        const buffer = new Uint8Array(wanted);
+        if (wanted > 0) {
+            // The byte count has to be checked: a short read (the file being
+            // truncated after it was opened) would otherwise leave the tail of
+            // `buffer` zeroed, and zeros parse as perfectly good varints -- a
+            // silently wrong answer instead of an error.
+            const bytesRead = await new Promise((resolve, reject) => read(this.fd, buffer, 0, wanted, offset, (error, bytesRead) => error ? reject(error) : resolve(bytesRead)));
+            if (bytesRead !== wanted) {
+                throw new CGeoDataError(`short read: wanted ${wanted} bytes at offset ${offset} of ${this.name}, got ${bytesRead}`);
+            }
+        }
+        return buffer;
+    }
+    async close() {
+        if (this.fd < 0)
+            return;
+        const fd = this.fd;
+        this.fd = -1;
+        unclosed.unregister(this);
+        await closeAsync(fd);
+    }
+}
+/**
+ * Node's zlib, on the calling thread. Deliberately not zlib's async API:
+ * measured on a worldwide file, handing blocks to libuv's thread pool made
+ * a burst of zoomed-out map tiles 15-40% slower and cold lookups ~30%
+ * slower, while barely shortening the longest event-loop stall -- parsing
+ * a decoded chunk costs more than inflating it, and that stays on this
+ * thread either way. What keeps the event loop responsive is that every
+ * chunk read (`FileSource.read`) is asynchronous, so the loop gets a turn
+ * between chunks: a stall is one chunk's decode (~25 ms for the largest
+ * chunk of a worldwide file, most far less), not a whole tile's or batch's.
+ */
+export const nodeInflater = (compressed, maxOutputLength) => inflateSync(compressed, { maxOutputLength });
+/** Where a search index for `binPath` conventionally lives, and where
+ * `openFile()` looks for one: `data/03-final/world.bin` →
+ * `data/03-final/world.search-index.bin`. */
+export function searchIndexPathFor(binPath) {
+    return `${binPath.replace(/\.bin$/, "")}.search-index.bin`;
+}
+/** Where a locate index for `binPath` conventionally lives, and where
+ * `openFile()` looks for one: `data/03-final/world.bin` →
+ * `data/03-final/world.locate-index.bin`. */
+export function locateIndexPathFor(binPath) {
+    return `${binPath.replace(/\.bin$/, "")}.locate-index.bin`;
+}
+/**
+ * Opens a `.bin` file from disk, reading only what each lookup needs, with
+ * Node's zlib as its inflater.
+ *
+ * The search and locate indexes next to it (`searchIndexPathFor`,
+ * `locateIndexPathFor`) are used automatically when they exist: each is
+ * opened on the first `search()` / `locate()` that can use it. One that
+ * exists but doesn't belong to this file (built from an older build of
+ * it, say) makes those calls reject with a `CGeoDataIndexMismatchError`
+ * naming it, rather than silently falling back to a slow scan; rebuild or
+ * delete it. `options.searchIndex` / `options.locateIndex` override the
+ * lookup next to the file.
+ *
+ * @example
+ * ```ts
+ * const reader = await openFile("data/world.bin");
+ * const { city } = await reader.query(52.37, 4.9);
+ * city?.internationalName; // "Amsterdam"
+ * await reader.close();
+ * ```
+ */
+export async function openFile(path, options = {}) {
+    return CGeoDataReader.open(await FileSource.open(path), {
+        inflater: nodeInflater,
+        searchIndex: () => openIfExists(searchIndexPathFor(path)),
+        locateIndex: () => openIfExists(locateIndexPathFor(path)),
+        ...options,
+    });
+}
+async function openIfExists(path) {
+    try {
+        return await FileSource.open(path);
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return undefined;
+        throw error;
+    }
+}
+//# sourceMappingURL=node.js.map

package/dist/placeTable.d.ts

@@ -0,0 +1,69 @@
+import { type Inflater } from "./inflate.ts";
+import type { ByteSource } from "./source.ts";
+/** The place table's uncompressed part (Section 7.2), parsed: the place
+ * counts and where each name block is. */
+export interface PlaceTableInfo {
+    /** C, the number of countries. */
+    countryCount: number;
+    /** R, the number of regions. */
+    totalRegions: number;
+    /** The number of cities. */
+    totalCities: number;
+    /** N, every place: countries, regions and cities. */
+    totalPlaces: number;
+    /** Length countryCount + 1; prefix[c] = number of regions before country c. */
+    regionCountryPrefix: number[];
+    /** Length totalRegions + 1; prefix[r] = number of cities before region r. */
+    cityRegionPrefix: number[];
+    /** The header's keyframe_exp (Section 7.3). */
+    keyframeExp: number;
+    /** Absolute file offset of each name block's compressed bytes. */
+    blockOffsets: number[];
+    /** Compressed byte length of each name block. */
+    blockLengths: number[];
+}
+/** Parses Section 7.2's uncompressed prefix (counts + block_lengths),
+ * growing the read like readHeaderAndDirectory does, since its size depends
+ * on country_count/region_counts which aren't known up front. */
+export declare function parsePlaceTable(source: ByteSource, placeTableStart: number, placeTableLength: number, keyframeExp: number): Promise<PlaceTableInfo>;
+/** Largest index c such that prefix[c] <= value, per Section 7.5. `prefix`
+ * is non-decreasing with prefix[0] = 0. */
+export declare function largestIndexWithPrefixAtMost(prefix: number[], value: number): number;
+/** A place id classified (Section 7.5): its level, and the level-local
+ * index of it and of its ancestors -- a region index, not a global id. */
+export type ResolvedPlace = {
+    /** A country. */
+    level: "country";
+    /** The country's index, which is also its global id. */
+    countryId: number;
+} | {
+    /** A region. */
+    level: "region";
+    /** Its country's index. */
+    countryId: number;
+    /** The region's index among all regions. */
+    regionId: number;
+} | {
+    /** A city. */
+    level: "city";
+    /** Its country's index. */
+    countryId: number;
+    /** Its region's index among all regions. */
+    regionId: number;
+    /** The city's index among all cities. */
+    cityId: number;
+};
+/** Classifies a place id per Section 7.5. `regionId`/`cityId` are the
+ * level-local indices the spec calls "region index"/"city index" (not
+ * global ids) -- add countryCount / countryCount+totalRegions to get the
+ * global id for a name lookup. */
+export declare function resolveIdentifier(info: PlaceTableInfo, id: number): ResolvedPlace;
+/** Inflates one whole name block (Section 7.3) and splits it into its
+ * entries, each split on `|` (Section 7.4). For walking many consecutive
+ * ids this is one inflate per block instead of one per id. */
+export declare function readNameBlock(source: ByteSource, info: PlaceTableInfo, blockIndex: number, inflater: Inflater, 
+/** Where the CPU-bound part runs; see `TurnQueue`. */
+decode?: <T>(work: () => Promise<T>) => Promise<T>): Promise<string[][]>;
+/** Recovers a place's name entry (Section 7.4, split on `|`) by inflating
+ * only the one name block (Section 7.3) that contains it. */
+export declare function readNameEntry(source: ByteSource, info: PlaceTableInfo, globalId: number, inflater: Inflater): Promise<string[]>;

package/dist/placeTable.js

@@ -0,0 +1,177 @@
+import { CGeoDataFormatError } from "./errors.js";
+import { inflateBlock } from "./inflate.js";
+import { IncompleteBufferError, readVarint, readVarintList } from "./varint.js";
+const utf8 = new TextDecoder();
+function sum(values) {
+    return values.reduce((a, b) => a + b, 0);
+}
+function prefixSum(values) {
+    const prefix = new Array(values.length + 1);
+    prefix[0] = 0;
+    for (let i = 0; i < values.length; i++) {
+        prefix[i + 1] = prefix[i] + values[i];
+    }
+    return prefix;
+}
+/** Parses Section 7.2's uncompressed prefix (counts + block_lengths),
+ * growing the read like readHeaderAndDirectory does, since its size depends
+ * on country_count/region_counts which aren't known up front. */
+export async function parsePlaceTable(source, placeTableStart, placeTableLength, keyframeExp) {
+    let size = Math.min(1024, placeTableLength);
+    for (;;) {
+        const buf = await source.read(placeTableStart, size);
+        try {
+            return parsePrefix(buf, placeTableStart, placeTableLength, keyframeExp);
+        }
+        catch (err) {
+            if (err instanceof IncompleteBufferError && size < placeTableLength) {
+                size = Math.min(size * 4, placeTableLength);
+                continue;
+            }
+            throw err;
+        }
+    }
+}
+function parsePrefix(buf, placeTableStart, placeTableLength, keyframeExp) {
+    let offset = 0;
+    const { value: countryCount, bytesRead: b1 } = readVarint(buf, offset);
+    offset += b1;
+    const { values: regionCounts, bytesRead: b2 } = readVarintList(buf, offset, countryCount);
+    offset += b2;
+    const totalRegions = sum(regionCounts);
+    const { values: cityCounts, bytesRead: b3 } = readVarintList(buf, offset, totalRegions);
+    offset += b3;
+    const totalCities = sum(cityCounts);
+    const totalPlaces = countryCount + totalRegions + totalCities;
+    // Section 7.3.
+    const blockSize = keyframeExp === 0 ? totalPlaces : 2 ** keyframeExp;
+    const blockCount = totalPlaces === 0
+        ? 0
+        : keyframeExp === 0
+            ? 1
+            : Math.ceil(totalPlaces / blockSize);
+    const storedBlockLengthCount = Math.max(blockCount - 1, 0);
+    const { values: storedBlockLengths, bytesRead: b4 } = readVarintList(buf, offset, storedBlockLengthCount);
+    offset += b4;
+    const namesStart = placeTableStart + offset;
+    const blockOffsets = new Array(blockCount);
+    const blockLengths = new Array(blockCount);
+    let running = namesStart;
+    for (let b = 0; b < blockCount - 1; b++) {
+        blockOffsets[b] = running;
+        blockLengths[b] = storedBlockLengths[b];
+        running += storedBlockLengths[b];
+    }
+    if (blockCount > 0) {
+        // The last block's length isn't stored -- it extends to the end of the
+        // place table (Section 7.3).
+        blockOffsets[blockCount - 1] = running;
+        blockLengths[blockCount - 1] = placeTableStart + placeTableLength - running;
+    }
+    return {
+        countryCount,
+        totalRegions,
+        totalCities,
+        totalPlaces,
+        regionCountryPrefix: prefixSum(regionCounts),
+        cityRegionPrefix: prefixSum(cityCounts),
+        keyframeExp,
+        blockOffsets,
+        blockLengths,
+    };
+}
+/** Largest index c such that prefix[c] <= value, per Section 7.5. `prefix`
+ * is non-decreasing with prefix[0] = 0. */
+export function largestIndexWithPrefixAtMost(prefix, value) {
+    let lo = 0;
+    let hi = prefix.length - 2;
+    let answer = 0;
+    while (lo <= hi) {
+        const mid = (lo + hi) >>> 1;
+        if (prefix[mid] <= value) {
+            answer = mid;
+            lo = mid + 1;
+        }
+        else {
+            hi = mid - 1;
+        }
+    }
+    return answer;
+}
+/** Classifies a place id per Section 7.5. `regionId`/`cityId` are the
+ * level-local indices the spec calls "region index"/"city index" (not
+ * global ids) -- add countryCount / countryCount+totalRegions to get the
+ * global id for a name lookup. */
+export function resolveIdentifier(info, id) {
+    const { countryCount, totalRegions, totalPlaces, regionCountryPrefix, cityRegionPrefix, } = info;
+    if (!Number.isInteger(id) || id < 0 || id >= totalPlaces) {
+        throw new RangeError(`place id ${id} is out of range for this file (0 <= id < ${totalPlaces})`);
+    }
+    if (id < countryCount) {
+        return { level: "country", countryId: id };
+    }
+    if (id < countryCount + totalRegions) {
+        const regionId = id - countryCount;
+        const countryId = largestIndexWithPrefixAtMost(regionCountryPrefix, regionId);
+        return { level: "region", countryId, regionId };
+    }
+    const cityId = id - countryCount - totalRegions;
+    const regionId = largestIndexWithPrefixAtMost(cityRegionPrefix, cityId);
+    const countryId = largestIndexWithPrefixAtMost(regionCountryPrefix, regionId);
+    return { level: "city", countryId, regionId, cityId };
+}
+/** Inflates one whole name block (Section 7.3) and splits it into its
+ * entries, each split on `|` (Section 7.4). For walking many consecutive
+ * ids this is one inflate per block instead of one per id. */
+export async function readNameBlock(source, info, blockIndex, inflater, 
+/** Where the CPU-bound part runs; see `TurnQueue`. */
+decode = (work) => work()) {
+    const { blockOffsets, blockLengths } = info;
+    if (blockIndex < 0 || blockIndex >= blockOffsets.length) {
+        throw new RangeError(`name block ${blockIndex} is out of range for this file (0 <= block < ${blockOffsets.length})`);
+    }
+    const compressed = await source.read(blockOffsets[blockIndex], blockLengths[blockIndex]);
+    return decode(async () => splitNameBlock(await inflateBlock(inflater, compressed), blockIndex));
+}
+/** An inflated name block (Section 7.3) split into its entries, each
+ * split on `|` (Section 7.4). */
+function splitNameBlock(inflated, blockIndex) {
+    const end = inflated.lastIndexOf(0) + 1;
+    if (end !== inflated.length) {
+        throw new CGeoDataFormatError(`name block ${blockIndex} has ${inflated.length - end} trailing bytes after its last NUL terminator`);
+    }
+    if (end === 0)
+        return [];
+    // One decode for the whole block: NUL and `|` are single bytes that
+    // never occur inside a multi-byte UTF-8 sequence, so splitting the
+    // decoded text finds the same boundaries splitting the bytes would.
+    const entries = utf8.decode(inflated.subarray(0, end - 1)).split("\0");
+    return entries.map((entry) => entry.split("|"));
+}
+/** Recovers a place's name entry (Section 7.4, split on `|`) by inflating
+ * only the one name block (Section 7.3) that contains it. */
+export async function readNameEntry(source, info, globalId, inflater) {
+    const { keyframeExp, totalPlaces, blockOffsets, blockLengths } = info;
+    if (globalId < 0 || globalId >= totalPlaces) {
+        throw new RangeError(`place id ${globalId} is out of range for this file (0 <= id < ${totalPlaces})`);
+    }
+    const blockSize = keyframeExp === 0 ? totalPlaces : 2 ** keyframeExp;
+    const blockIndex = Math.floor(globalId / blockSize);
+    const withinBlock = globalId - blockIndex * blockSize;
+    const compressed = await source.read(blockOffsets[blockIndex], blockLengths[blockIndex]);
+    const inflated = await inflateBlock(inflater, compressed);
+    let start = 0;
+    for (let i = 0; i < withinBlock; i++) {
+        const nul = inflated.indexOf(0, start);
+        if (nul === -1) {
+            throw new CGeoDataFormatError(`name block ${blockIndex} ended before reaching entry ${withinBlock}`);
+        }
+        start = nul + 1;
+    }
+    const end = inflated.indexOf(0, start);
+    if (end === -1) {
+        throw new CGeoDataFormatError(`name entry for place ${globalId} is missing its NUL terminator`);
+    }
+    return utf8.decode(inflated.subarray(start, end)).split("|");
+}
+//# sourceMappingURL=placeTable.js.map