@ferrybig/cgeodata-reader 0.0.0-stage → 0.1.1

package/dist/cli/buildLocateIndex.js

@@ -0,0 +1,46 @@
+import { writeFileSync } from "node:fs";
+import { buildLocateIndex } from "../locateIndex.js";
+import { EXIT_OK, openReader, parse, UsageError, } from "./shared.js";
+export const buildLocateIndexCmd = {
+    name: "build-locate-index",
+    summary: "build a place location index for a .bin file",
+    usage: `usage: cgeodata build-locate-index <file.bin> [--output <index.bin>]
+
+Computes a representative cell for every place in <file.bin> and writes
+the result as a locate index to <index.bin> (or stdout when --output is
+omitted). Written next to <file.bin> as <file>.locate-index.bin,
+\`cgeodata locate\` and \`search --locate\` (and any CGeoDataReader opened
+from that path) use it automatically; elsewhere, pass it with
+\`locate --index\`. Turns a locate from a scan of every chunk into a
+read of one entry.
+
+  --output F  write to F instead of stdout`,
+    async run(argv) {
+        const { values, positionals } = parse(argv, {
+            output: { type: "string" },
+        });
+        const [file, ...extra] = positionals;
+        if (extra.length > 0)
+            throw new UsageError(`unexpected argument ${JSON.stringify(extra[0])}`);
+        const reader = await openReader(file);
+        const { countries, regions, cities } = reader.placeCounts;
+        process.stderr.write(`building locate index over ${countries + regions + cities} places...\n`);
+        const t0 = performance.now();
+        let bytes;
+        try {
+            bytes = await buildLocateIndex(reader);
+        }
+        finally {
+            await reader.close();
+        }
+        process.stderr.write(`done in ${((performance.now() - t0) / 1000).toFixed(1)} s, ${(bytes.length / 1024 / 1024).toFixed(1)} MiB\n`);
+        if (values.output !== undefined) {
+            writeFileSync(values.output, bytes);
+        }
+        else {
+            process.stdout.write(bytes);
+        }
+        return EXIT_OK;
+    },
+};
+//# sourceMappingURL=buildLocateIndex.js.map

package/dist/cli/buildSearchIndex.d.ts

@@ -0,0 +1,2 @@
+import { type Command } from "./shared.ts";
+export declare const buildSearchIndexCmd: Command;

package/dist/cli/buildSearchIndex.js

@@ -0,0 +1,90 @@
+import { writeFileSync } from "node:fs";
+import { buildSearchIndex, DEFAULT_SEARCH_INDEX_OPTIONS, } from "../searchIndex.js";
+import { EXIT_OK, openReader, parse, parseAreaTieBreak, UsageError, useIndexFile, } from "./shared.js";
+const { hashExp, topCount, topBlockExp } = DEFAULT_SEARCH_INDEX_OPTIONS;
+export const buildSearchIndexCmd = {
+    name: "build-search-index",
+    summary: "build a trigram search index for a .bin file",
+    usage: `usage: cgeodata build-search-index <file.bin> [--output <index.bin>] [--hash-exp N] [--top-count N] [--top-block-exp N] [--area-tie-break auto|on|off] [--locate-index <index.bin>]
+
+Builds a trigram search index (binary-format.md Appendix B) over every
+place in <file.bin> and writes it to <index.bin> (or stdout when
+--output is omitted). Written next to <file.bin> as
+<file>.search-index.bin, \`cgeodata search\` (and any CGeoDataReader
+opened from that path) uses it automatically; elsewhere, pass it with
+\`search --index\`. Queries of 3+ characters then only compare the
+places whose names contain all of their trigrams, and sorted searches of
+1-2 characters with --limit up to the top count are answered from a
+precomputed table.
+
+That table is ranked like \`search --sort --area-tie-break\`, and only
+answers searches ranked the same way: by area when a locate index is
+next to <file.bin> (build it first, with build-locate-index), otherwise
+plainly.
+
+  --output F          write to F instead of stdout
+  --hash-exp N        2^N shared lists for non-Latin trigrams (default ${hashExp})
+  --top-count N       places per 1-2 character query; 0 leaves the
+                      table out (default ${topCount})
+  --top-block-exp N   2^N table entries per block (default ${topBlockExp})
+  --area-tie-break M  rank the table by area (on; needs a locate index),
+                      plainly (off), or by area if there is a locate
+                      index (auto, the default)
+  --locate-index F    take areas from this locate index instead of
+                      <file>'s default <file>.locate-index.bin`,
+    async run(argv) {
+        const { values, positionals } = parse(argv, {
+            output: { type: "string" },
+            "hash-exp": { type: "string" },
+            "top-count": { type: "string" },
+            "top-block-exp": { type: "string" },
+            "area-tie-break": { type: "string" },
+            "locate-index": { type: "string" },
+        });
+        const [file, ...extra] = positionals;
+        if (extra.length > 0)
+            throw new UsageError(`unexpected argument ${JSON.stringify(extra[0])}`);
+        const options = {};
+        for (const [flag, key] of [
+            ["hash-exp", "hashExp"],
+            ["top-count", "topCount"],
+            ["top-block-exp", "topBlockExp"],
+        ]) {
+            const text = values[flag];
+            if (text === undefined)
+                continue;
+            const value = Number(text);
+            if (!(Number.isInteger(value) && value >= 0)) {
+                throw new UsageError(`--${flag} must be a non-negative integer, not ${JSON.stringify(text)}`);
+            }
+            options[key] = value;
+        }
+        options.areaTieBreak = parseAreaTieBreak(values["area-tie-break"]);
+        const reader = await openReader(file);
+        const { countries, regions, cities } = reader.placeCounts;
+        const t0 = performance.now();
+        let bytes;
+        try {
+            if (values["locate-index"] !== undefined)
+                await useIndexFile((index) => reader.useLocateIndex(index), values["locate-index"]);
+            const byArea = options.areaTieBreak !== false && (await reader.hasLocateIndex());
+            process.stderr.write(`building search index over ${countries + regions + cities} places, ranked ${byArea ? "by area" : "plainly"}...\n`);
+            bytes = await buildSearchIndex(reader, options);
+        }
+        catch (error) {
+            throw error instanceof RangeError ? new UsageError(error.message) : error;
+        }
+        finally {
+            await reader.close();
+        }
+        process.stderr.write(`done in ${((performance.now() - t0) / 1000).toFixed(1)} s, ${(bytes.length / 1024 / 1024).toFixed(1)} MiB\n`);
+        if (values.output !== undefined) {
+            writeFileSync(values.output, bytes);
+        }
+        else {
+            process.stdout.write(bytes);
+        }
+        return EXIT_OK;
+    },
+};
+//# sourceMappingURL=buildSearchIndex.js.map

package/dist/cli/chunk.d.ts

@@ -0,0 +1,2 @@
+import { type Command } from "./shared.ts";
+export declare const chunk: Command;

package/dist/cli/chunk.js

@@ -0,0 +1,67 @@
+import { names, openReader, parse, parseId, UsageError, writeLine, } from "./shared.js";
+export const chunk = {
+    name: "chunk",
+    summary: "dump one chunk's records with their cells and places",
+    usage: `usage: cgeodata chunk <file.bin> <index> [--json]
+
+Dumps the records of chunk <index> (the number \`lookup\` prints): first
+a header with the chunk's extent, compressed size, palette size and
+record count, then one record per line as tab-separated columns: code,
+depth, cell (lat min..max, lon min..max), place id, level, names,
+region, country. A record that resolves to no place has "-" columns.
+An empty chunk prints just the header.
+
+  --json   the header as one JSON object, then one object per record`,
+    async run(argv) {
+        const { values, positionals } = parse(argv, {
+            json: { type: "boolean", default: false },
+        });
+        const [file, indexArg, ...extra] = positionals;
+        if (indexArg === undefined)
+            throw new UsageError("missing <index>");
+        if (extra.length > 0)
+            throw new UsageError(`unexpected argument ${JSON.stringify(extra[0])}`);
+        const index = parseId(indexArg, "index");
+        const reader = await openReader(file);
+        try {
+            const records = await reader.chunk(index); // also range-checks the index
+            const box = reader.chunkBoundingBox(index);
+            const { width } = reader.header;
+            const header = {
+                index,
+                col: index % width,
+                row: Math.floor(index / width),
+                boundingBox: box,
+                compressedBytes: reader.directory.chunkLengths[index],
+                paletteSize: records?.palette.length ?? 0,
+                recordCount: records?.codes.length ?? 0,
+            };
+            if (values.json) {
+                writeLine(JSON.stringify(header));
+                for await (const record of reader.chunkRecords(index))
+                    writeLine(JSON.stringify(record));
+                return 0;
+            }
+            writeLine(`chunk ${index}  (col ${header.col}, row ${header.row})  lat ${box.minLat}..${box.maxLat}  lon ${box.minLon}..${box.maxLon}  ` +
+                `${header.compressedBytes} bytes, ${header.paletteSize} palette entries, ${header.recordCount} records`);
+            for await (const { code, depth, boundingBox: cell, place, } of reader.chunkRecords(index)) {
+                const where = `lat ${cell.minLat}..${cell.maxLat}, lon ${cell.minLon}..${cell.maxLon}`;
+                const who = place
+                    ? [
+                        place.id,
+                        place.level,
+                        names(place.names),
+                        names(place.level === "city" ? place.region?.names : null),
+                        names(place.level === "country" ? null : place.country.names),
+                    ]
+                    : ["-", "-", "-", "-", "-"];
+                writeLine([code, depth, where, ...who].join("\t"));
+            }
+            return 0;
+        }
+        finally {
+            await reader.close();
+        }
+    },
+};
+//# sourceMappingURL=chunk.js.map

package/dist/cli/info.d.ts

@@ -0,0 +1,2 @@
+import { type Command } from "./shared.ts";
+export declare const info: Command;

package/dist/cli/info.js

@@ -0,0 +1,87 @@
+import { openReader, parse, writeLine } from "./shared.js";
+export const info = {
+    name: "info",
+    summary: "describe a file: header, place counts, chunk statistics",
+    usage: `usage: cgeodata info <file.bin> [--top N] [--json]
+
+Prints the header fields, how many countries/regions/cities the place
+table holds, how the bytes split between place table and chunks, and the
+N (default 10) largest chunks by compressed size -- as indices that paste
+straight into the compressor's --only-chunks.
+
+  --top N  how many of the largest chunks to list (0 for none)
+  --json   one JSON object instead of the readable form`,
+    async run(argv) {
+        const { values, positionals } = parse(argv, {
+            top: { type: "string", default: "10" },
+            json: { type: "boolean", default: false },
+        });
+        const top = Number(values.top);
+        if (!Number.isInteger(top) || top < 0)
+            throw new Error(`--top must be a non-negative integer, not ${JSON.stringify(values.top)}`);
+        const reader = await openReader(positionals[0]);
+        try {
+            const { header, directory } = reader;
+            const chunkLengths = directory.chunkLengths;
+            const nonEmpty = chunkLengths.filter((len) => len > 0).length;
+            const chunkBytes = chunkLengths.reduce((a, b) => a + b, 0);
+            const fileSize = header.headerLength +
+                directory.directoryLength +
+                directory.placeTableLength +
+                chunkBytes;
+            const largest = chunkLengths
+                .map((bytes, index) => ({ index, bytes }))
+                .filter((c) => c.bytes > 0)
+                .sort((a, b) => b.bytes - a.bytes || a.index - b.index)
+                .slice(0, top);
+            const report = {
+                version: header.version,
+                attribution: header.attribution,
+                grid: {
+                    width: header.width,
+                    height: header.height,
+                    widthExp: header.widthExp,
+                    heightExp: header.heightExp,
+                },
+                maxDepth: header.maxDepth,
+                keyframeExp: header.keyframeExp,
+                places: reader.placeCounts,
+                bytes: {
+                    total: fileSize,
+                    header: header.headerLength,
+                    directory: directory.directoryLength,
+                    placeTable: directory.placeTableLength,
+                    chunks: chunkBytes,
+                },
+                chunks: { total: chunkLengths.length, nonEmpty, largest },
+            };
+            if (values.json) {
+                writeLine(JSON.stringify(report));
+                return 0;
+            }
+            const { places, bytes, grid } = report;
+            const pct = (n) => `${((100 * n) / bytes.total).toFixed(1)}%`;
+            const lines = [
+                `version      ${report.version}`,
+                `attribution  ${report.attribution}`,
+                `grid         ${grid.width}x${grid.height} chunks (width_exp ${grid.widthExp}, height_exp ${grid.heightExp})`,
+                `max depth    ${report.maxDepth}  (${grid.width * 2 ** report.maxDepth}x${grid.height * 2 ** report.maxDepth} finest cells worldwide)`,
+                `keyframe     ${report.keyframeExp}  (${2 ** report.keyframeExp} names per block)`,
+                `places       ${places.countries} countries, ${places.regions} regions, ${places.cities} cities`,
+                `bytes        ${bytes.total} total: header ${bytes.header}, directory ${bytes.directory}, place table ${bytes.placeTable} (${pct(bytes.placeTable)}), chunks ${bytes.chunks} (${pct(bytes.chunks)})`,
+                `chunks       ${nonEmpty} of ${chunkLengths.length} non-empty`,
+            ];
+            if (largest.length > 0) {
+                lines.push(`largest      ${largest.map((c) => `${c.index} (${c.bytes})`).join(", ")}`);
+                lines.push(`             --only-chunks ${largest.map((c) => c.index).join(",")}`);
+            }
+            for (const line of lines)
+                writeLine(line);
+            return 0;
+        }
+        finally {
+            await reader.close();
+        }
+    },
+};
+//# sourceMappingURL=info.js.map

package/dist/cli/list.d.ts

@@ -0,0 +1,2 @@
+import { type Command } from "./shared.ts";
+export declare const list: Command;

package/dist/cli/list.js

@@ -0,0 +1,71 @@
+import { formatPlaceLine, names, openReader, parse, parseLevel, resolveCountry, UsageError, writeLine, } from "./shared.js";
+const DEPTH = { country: 0, region: 1, city: 2 };
+async function printTree(reader, place, deepest, json) {
+    const depth = DEPTH[place.level];
+    writeLine(json
+        ? JSON.stringify(place)
+        : `${"  ".repeat(depth)}${place.id}\t${names(place.names)}`);
+    if (depth >= DEPTH[deepest])
+        return;
+    const children = place.level === "country"
+        ? reader.places({ level: "region", country: place.id })
+        : reader.places({ level: "city", region: place.id });
+    for await (const child of children)
+        await printTree(reader, child, deepest, json);
+}
+export const list = {
+    name: "list",
+    summary: "list places: countries by default, or a level, scoped to a country",
+    usage: `usage: cgeodata list <file.bin> [--level country|region|city] [--country <id|name>] [--childless] [--tree] [--json]
+
+Lists places one per line as tab-separated columns: id, level, names,
+region, country. Alternative names within a column are joined by " | ";
+a column that doesn't apply is "-".
+
+  --level L    which level to list (default: country)
+  --country C  only places inside this country, given as an id or as one
+               of its names (case-insensitive, must be unambiguous)
+  --childless  only places with nothing directly under them: regions with
+               no city mapped to them (with --level region), countries
+               with no region. Read from the place table's counts
+  --tree       print the hierarchy indented instead of flat, down to
+               --level (default with --tree: city); lines are "id<TAB>names"
+  --json       one JSON object per place instead of the text form`,
+    async run(argv) {
+        const { values, positionals } = parse(argv, {
+            level: { type: "string" },
+            country: { type: "string" },
+            childless: { type: "boolean", default: false },
+            tree: { type: "boolean", default: false },
+            json: { type: "boolean", default: false },
+        });
+        const level = parseLevel(values.level) ?? (values.tree ? "city" : "country");
+        if (values.childless && values.tree)
+            throw new UsageError("--childless cannot be combined with --tree");
+        const reader = await openReader(positionals[0]);
+        try {
+            const country = values.country === undefined
+                ? undefined
+                : await resolveCountry(reader, values.country);
+            if (values.tree) {
+                const roots = country === undefined
+                    ? reader.places({ level: "country" })
+                    : [await reader.place(country)];
+                for await (const root of roots)
+                    await printTree(reader, root, level, values.json);
+                return 0;
+            }
+            const filter = country === undefined ? { level } : { level, country };
+            for await (const place of reader.places(filter)) {
+                if (values.childless && reader.childCount(place.id) > 0)
+                    continue;
+                writeLine(values.json ? JSON.stringify(place) : formatPlaceLine(place));
+            }
+            return 0;
+        }
+        finally {
+            await reader.close();
+        }
+    },
+};
+//# sourceMappingURL=list.js.map

package/dist/cli/locate.d.ts

@@ -0,0 +1,2 @@
+import { type Command } from "./shared.ts";
+export declare const locate: Command;

package/dist/cli/locate.js

@@ -0,0 +1,64 @@
+import { coordPrecision, openReader, parse, parseId, UsageError, useIndexFile, writeLine, } from "./shared.js";
+export const locate = {
+    name: "locate",
+    summary: "find a representative lat/lon for a place id",
+    usage: `usage: cgeodata locate <file.bin> <id> [<id> ...] [--index <index.bin> | --no-index] [--json]
+
+Prints a representative lat, lon for each place id: the centre of a cell
+near the middle of the place's area (its own cells plus those of every
+place under it), and always inside that area, so looking the point up
+gives the place back. Printed with as many decimals as that cell's size
+warrants.
+
+With a locate index (build-locate-index) this reads one entry per id;
+without one it scans every chunk, which takes seconds on a worldwide
+file. Same result either way.
+
+  --index F   use this locate index instead of <file>'s default
+              <file>.locate-index.bin (used automatically when it exists)
+  --no-index  scan the chunks even if a locate index exists -- the
+              baseline an index is measured against
+  --json      one JSON object per id instead: {"id", "lat", "lon",
+              "area", "boundingBox"}, area being the place's size in
+              square degrees (null when it has no geometry)`,
+    async run(argv) {
+        const { values, positionals } = parse(argv, {
+            index: { type: "string" },
+            "no-index": { type: "boolean", default: false },
+            json: { type: "boolean", default: false },
+        });
+        const [file, ...idArgs] = positionals;
+        if (idArgs.length === 0)
+            throw new UsageError("missing <id>");
+        if (values.index !== undefined && values["no-index"])
+            throw new UsageError("--index and --no-index can't be combined");
+        const ids = idArgs.map((arg) => parseId(arg, "id"));
+        const reader = await openReader(file);
+        try {
+            if (values["no-index"])
+                await reader.useLocateIndex(undefined);
+            if (values.index !== undefined)
+                await useIndexFile((index) => reader.useLocateIndex(index), values.index);
+            let anyMissing = false;
+            for (const id of ids) {
+                const pos = await reader.locate(id);
+                if (values.json)
+                    writeLine(JSON.stringify(pos ? { id, ...pos } : null));
+                if (!pos) {
+                    process.stderr.write(`${id}: no geometry found\n`);
+                    anyMissing = true;
+                    continue;
+                }
+                if (values.json)
+                    continue;
+                const p = coordPrecision(pos.boundingBox);
+                writeLine(`${pos.lat.toFixed(p)}, ${pos.lon.toFixed(p)}`);
+            }
+            return anyMissing ? 1 : 0;
+        }
+        finally {
+            await reader.close();
+        }
+    },
+};
+//# sourceMappingURL=locate.js.map

package/dist/cli/lookup.d.ts

@@ -0,0 +1,2 @@
+import { type Command } from "./shared.ts";
+export declare const lookup: Command;

package/dist/cli/lookup.js

@@ -0,0 +1,128 @@
+import { createInterface } from "node:readline";
+import { names, openReader, parse, UsageError, writeLine, } from "./shared.js";
+function parseCoordinate(text, what) {
+    const trimmed = text.trim();
+    // `Number("")` is 0, not NaN, so an emptiness check has to come first:
+    // otherwise a truncated "52.3," parses as (52.3, 0) and reports a
+    // confident, wrong place off the coast of England instead of an error.
+    const value = trimmed === "" ? Number.NaN : Number(trimmed);
+    if (!Number.isFinite(value))
+        throw new Error(`${what} ${JSON.stringify(text)} is not a number`);
+    return value;
+}
+/** Accepts "lat,lon" as one field or lat and lon as two; anything after
+ * the coordinates is the tag. */
+function parsePoint(line) {
+    const fields = line.split("\t");
+    const first = fields[0] ?? "";
+    if (first.includes(",")) {
+        const [lat = "", lon = ""] = first.split(",", 2);
+        return {
+            lat: parseCoordinate(lat, "latitude"),
+            lon: parseCoordinate(lon, "longitude"),
+            tag: fields.slice(1).join("\t"),
+        };
+    }
+    if (fields.length < 2)
+        throw new Error(`expected "lat,lon" or "lat<TAB>lon", got ${JSON.stringify(line)}`);
+    return {
+        lat: parseCoordinate(first, "latitude"),
+        lon: parseCoordinate(fields[1] ?? "", "longitude"),
+        tag: fields.slice(2).join("\t"),
+    };
+}
+function formatReadable(point, result, reader) {
+    const { boundingBox: box, chunkIndex } = result;
+    const col = chunkIndex % reader.header.width;
+    const row = Math.floor(chunkIndex / reader.header.width);
+    const head = point.tag
+        ? `${point.lat},${point.lon}\t${point.tag}`
+        : `${point.lat},${point.lon}`;
+    return [
+        head,
+        `  country  ${names(result.country?.names)}`,
+        `  region   ${names(result.region?.names)}`,
+        `  city     ${names(result.city?.names)}`,
+        `  cell     lat ${box.minLat}..${box.maxLat}  lon ${box.minLon}..${box.maxLon}`,
+        `  chunk    ${chunkIndex}  (col ${col}, row ${row} of ${reader.header.width}x${reader.header.height})`,
+    ].join("\n");
+}
+function formatJson(point, result) {
+    return JSON.stringify({
+        lat: point.lat,
+        lon: point.lon,
+        ...(point.tag ? { tag: point.tag } : {}),
+        country: result.country,
+        region: result.region,
+        city: result.city,
+        boundingBox: result.boundingBox,
+        chunkIndex: result.chunkIndex,
+    });
+}
+export const lookup = {
+    name: "lookup",
+    summary: "resolve points to country/region/city",
+    usage: `usage: cgeodata lookup <file.bin> <lat,lon> [<lat,lon> ...] [--json]
+       cgeodata lookup <file.bin> [--json] < points.txt
+
+Resolves each point to its country/region/city, the lat/lon cell the
+answer is valid for, and the chunk it was read from. With no points on
+the command line they are read from stdin, one per line: "lat,lon" or
+"lat<TAB>lon"; anything after that (tab-separated) is echoed back on the
+output unchanged, so a batch can carry an id or name through. Blank lines
+and lines starting with # are skipped; a malformed line is reported on
+stderr and skipped rather than aborting the batch.
+
+  --json   one JSON object per point (the query result plus the input
+           lat/lon and any echoed text) instead of the readable form.`,
+    async run(argv) {
+        const { values, positionals } = parse(argv, {
+            json: { type: "boolean", default: false },
+        });
+        const [file, ...pointArgs] = positionals;
+        const fromStdin = pointArgs.length === 0;
+        if (fromStdin && file === "-")
+            throw new UsageError("stdin can't be both the file and the list of points");
+        const reader = await openReader(file);
+        const emit = async (point) => {
+            const result = await reader.query(point.lat, point.lon);
+            writeLine(values.json
+                ? formatJson(point, result)
+                : formatReadable(point, result, reader));
+        };
+        try {
+            if (!fromStdin) {
+                for (const arg of pointArgs)
+                    await emit(parsePoint(arg));
+                return 0;
+            }
+            // Batch mode: a bad line is somebody's data problem, not a reason to
+            // throw away the rest of a long run, so report it and keep going.
+            let lineNumber = 0;
+            let skipped = 0;
+            for await (const raw of createInterface({
+                input: process.stdin,
+                crlfDelay: Infinity,
+            })) {
+                lineNumber++;
+                const line = raw.trim();
+                if (line === "" || line.startsWith("#"))
+                    continue;
+                try {
+                    await emit(parsePoint(line));
+                }
+                catch (err) {
+                    skipped++;
+                    process.stderr.write(`line ${lineNumber}: ${err instanceof Error ? err.message : String(err)}\n`);
+                }
+            }
+            if (skipped > 0)
+                process.stderr.write(`skipped ${skipped} malformed line${skipped === 1 ? "" : "s"}\n`);
+            return 0;
+        }
+        finally {
+            await reader.close();
+        }
+    },
+};
+//# sourceMappingURL=lookup.js.map

package/dist/cli/search.d.ts

@@ -0,0 +1,2 @@
+import { type Command } from "./shared.ts";
+export declare const search: Command;