@ferrybig/cgeodata-reader 0.0.0-stage → 0.1.0

package/dist/source.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Where a reader gets its bytes from. This is what lets a reader honour
+ * Section 1.2's goal 2 ("a reader MUST be able to answer a lookup by
+ * reading and decompressing only a small, bounded part of the file"): a
+ * source is only ever asked for the bytes a lookup needs.
+ *
+ * Reads are asynchronous so that a source can be anything that hands out
+ * byte ranges: a file (`FileSource` in `./node.ts`), a `Blob` or `File`
+ * in a browser, or bytes already in memory. Nothing here depends on
+ * Node.js.
+ */
+export interface ByteSource {
+    /** Total size in bytes. */
+    readonly size: number;
+    /** Shown in error messages about this source, e.g. a file path. */
+    readonly name?: string;
+    /**
+     * `length` bytes from `offset`, fewer only where the source ends. The
+     * returned bytes may be a view of the source's own memory, so a caller
+     * must not modify them.
+     */
+    read(offset: number, length: number): Promise<Uint8Array>;
+    /** Releases whatever the source holds open, if anything. */
+    close?(): Promise<void> | void;
+}
+/** A source over bytes already in memory: a file read up front, or the
+ * body of a `fetch()`. */
+export declare class BytesSource implements ByteSource {
+    private readonly bytes;
+    readonly name?: string;
+    constructor(bytes: Uint8Array | ArrayBuffer, name?: string);
+    get size(): number;
+    read(offset: number, length: number): Promise<Uint8Array>;
+    /** `read()` without the promise: bytes in memory need no waiting. */
+    readNow(offset: number, length: number): Uint8Array;
+}
+/** A source over a `Blob` -- in a browser, a `File` the user picked --
+ * reading each range as it's asked for, without loading the whole file. */
+export declare class BlobSource implements ByteSource {
+    private readonly blob;
+    readonly name?: string;
+    constructor(blob: Blob, name?: string);
+    get size(): number;
+    read(offset: number, length: number): Promise<Uint8Array>;
+}
+/** What `CGeoDataReader.open()` and the index classes accept as a file. */
+export type SourceInput = ByteSource | Uint8Array | ArrayBuffer | Blob;
+/** `input` as a `ByteSource`. */
+export declare function toSource(input: SourceInput): ByteSource;

package/dist/source.js

@@ -0,0 +1,50 @@
+/** A source over bytes already in memory: a file read up front, or the
+ * body of a `fetch()`. */
+export class BytesSource {
+    bytes;
+    name;
+    constructor(bytes, name) {
+        this.bytes = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
+        if (name !== undefined)
+            this.name = name;
+    }
+    get size() {
+        return this.bytes.length;
+    }
+    read(offset, length) {
+        return Promise.resolve(this.readNow(offset, length));
+    }
+    /** `read()` without the promise: bytes in memory need no waiting. */
+    readNow(offset, length) {
+        const start = Math.min(offset, this.bytes.length);
+        return this.bytes.subarray(start, Math.min(start + length, this.bytes.length));
+    }
+}
+/** A source over a `Blob` -- in a browser, a `File` the user picked --
+ * reading each range as it's asked for, without loading the whole file. */
+export class BlobSource {
+    blob;
+    name;
+    constructor(blob, name) {
+        this.blob = blob;
+        const fileName = blob.name;
+        const resolved = name ?? (typeof fileName === "string" ? fileName : undefined);
+        if (resolved !== undefined)
+            this.name = resolved;
+    }
+    get size() {
+        return this.blob.size;
+    }
+    async read(offset, length) {
+        return new Uint8Array(await this.blob.slice(offset, offset + length).arrayBuffer());
+    }
+}
+/** `input` as a `ByteSource`. */
+export function toSource(input) {
+    if (input instanceof Uint8Array || input instanceof ArrayBuffer)
+        return new BytesSource(input);
+    if (typeof Blob !== "undefined" && input instanceof Blob)
+        return new BlobSource(input);
+    return input;
+}
+//# sourceMappingURL=source.js.map

package/dist/turns.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Decoding a chunk is CPU work on the calling thread, a few milliseconds
+ * for a large one. Reads run concurrently, but if every read that's done
+ * went straight on to decode, many lookups started at once would decode
+ * back to back -- Node.js delivers all finished reads in one go -- and
+ * hold the event loop for the whole batch (330 ms for 300 lookups across
+ * Europe, measured). `TurnQueue` runs decodes one at a time instead, each
+ * in a macrotask of its own, so other work gets a turn between any two.
+ */
+export declare class TurnQueue {
+    private tail;
+    /** Runs `work` after everything queued before it, in a fresh
+     * macrotask. */
+    run<T>(work: () => T | Promise<T>): Promise<T>;
+}
+/** Resolves in a new macrotask: Node's `setImmediate` (after pending I/O,
+ * before timers come round again), else a `MessageChannel` message (not
+ * clamped like a nested `setTimeout(0)` in browsers), else `setTimeout`. */
+export declare const yieldToEventLoop: () => Promise<void>;

package/dist/turns.js

@@ -0,0 +1,40 @@
+/**
+ * Decoding a chunk is CPU work on the calling thread, a few milliseconds
+ * for a large one. Reads run concurrently, but if every read that's done
+ * went straight on to decode, many lookups started at once would decode
+ * back to back -- Node.js delivers all finished reads in one go -- and
+ * hold the event loop for the whole batch (330 ms for 300 lookups across
+ * Europe, measured). `TurnQueue` runs decodes one at a time instead, each
+ * in a macrotask of its own, so other work gets a turn between any two.
+ */
+export class TurnQueue {
+    tail = Promise.resolve();
+    /** Runs `work` after everything queued before it, in a fresh
+     * macrotask. */
+    run(work) {
+        const result = this.tail.then(yieldToEventLoop).then(work);
+        this.tail = result.catch(() => { });
+        return result;
+    }
+}
+/** Resolves in a new macrotask: Node's `setImmediate` (after pending I/O,
+ * before timers come round again), else a `MessageChannel` message (not
+ * clamped like a nested `setTimeout(0)` in browsers), else `setTimeout`. */
+export const yieldToEventLoop = (() => {
+    const setImmediate = globalThis.setImmediate;
+    if (setImmediate)
+        return () => new Promise((r) => setImmediate(r));
+    if (typeof MessageChannel !== "undefined") {
+        const channel = new MessageChannel();
+        const waiting = [];
+        // A browser's MessagePort; Node's typings (which never reach here)
+        // lack `onmessage`.
+        channel.port1.onmessage = () => waiting.shift()?.();
+        return () => new Promise((resolve) => {
+            waiting.push(resolve);
+            channel.port2.postMessage(null);
+        });
+    }
+    return () => new Promise((resolve) => setTimeout(resolve, 0));
+})();
+//# sourceMappingURL=turns.js.map

package/dist/varint.d.ts

@@ -0,0 +1,52 @@
+import { CGeoDataFormatError } from "./errors.ts";
+/**
+ * varint (LEB128) codec, per binary-format.md Section 3.2, plus the
+ * delta-coded list helper from Section 3.3.
+ */
+/** Thrown when a parser ran out of bytes because the given buffer was too
+ * short to hold the full structure -- distinct from a genuine format error,
+ * so callers can retry with a larger read. */
+export declare class IncompleteBufferError extends CGeoDataFormatError {
+    name: string;
+}
+/** One decoded varint. */
+export interface VarintResult {
+    /** The value, below 2^53. */
+    value: number;
+    /** How many bytes it took (1-8). */
+    bytesRead: number;
+}
+/** The varint (Section 3.2) at `offset`; throws a `CGeoDataFormatError`
+ * for a malformed one, an `IncompleteBufferError` for a cut-off one. */
+export declare function readVarint(buf: Uint8Array, offset: number): VarintResult;
+/** `value`'s minimal varint encoding (Section 3.2). */
+export declare function writeVarint(value: number): Uint8Array;
+/** A decoded list of varints. */
+export interface VarintListResult {
+    /** The values, in order. */
+    values: number[];
+    /** How many bytes the whole list took. */
+    bytesRead: number;
+}
+/** A plain list of `count` varints (not delta-coded) -- used for
+ * region_counts, city_counts, chunk_lengths, and block_lengths. */
+export declare function readVarintList(buf: Uint8Array, offset: number, count: number): VarintListResult;
+/** A delta-coded list per Section 3.3 -- used for a chunk's palette and
+ * codes. Each stored varint is the difference from the previous element,
+ * with an implicit previous element of 0 before the first. */
+export declare function readDeltaVarintList(buf: Uint8Array, offset: number, count: number): VarintListResult;
+/**
+ * `out.length` delta-coded varints (Section 3.3) from `buf` at `offset`
+ * into `out`, returning the offset just past them. Decodes straight into
+ * a typed array, without an object per varint (see `varintEnd`).
+ */
+export declare function readDeltaVarintsInto(buf: Uint8Array, offset: number, out: Float64Array): number;
+/**
+ * `readDeltaVarintsInto` for values known to fit in 32 bits, such as a
+ * chunk's codes at `max_depth` 16 and under: a running total that
+ * outgrows the `Uint32Array` is a format error, not a silent wraparound.
+ * A separate function rather than one taking either array, because V8
+ * optimizes a loop for the array types its function has seen, and a
+ * loop that has seen both decoded chunks ~10% slower.
+ */
+export declare function readDeltaVarints32Into(buf: Uint8Array, offset: number, out: Uint32Array): number;

package/dist/varint.js

@@ -0,0 +1,137 @@
+import { CGeoDataFormatError } from "./errors.js";
+/**
+ * varint (LEB128) codec, per binary-format.md Section 3.2, plus the
+ * delta-coded list helper from Section 3.3.
+ */
+/** Thrown when a parser ran out of bytes because the given buffer was too
+ * short to hold the full structure -- distinct from a genuine format error,
+ * so callers can retry with a larger read. */
+export class IncompleteBufferError extends CGeoDataFormatError {
+    name = "IncompleteBufferError";
+}
+const CONTINUATION_BIT = 0x80;
+const PAYLOAD_MASK = 0x7f;
+const MAX_VARINT_BYTES = 8;
+// Section 3.2: every varint MUST be less than 2^53, so it round-trips
+// exactly through an IEEE 754 double.
+const MAX_VARINT_VALUE = 2 ** 53;
+/** The varint (Section 3.2) at `offset`; throws a `CGeoDataFormatError`
+ * for a malformed one, an `IncompleteBufferError` for a cut-off one. */
+export function readVarint(buf, offset) {
+    const value = varintAt(buf, offset);
+    return { value, bytesRead: varintEnd - offset };
+}
+/** `value`'s minimal varint encoding (Section 3.2). */
+export function writeVarint(value) {
+    if (!Number.isInteger(value) || value < 0) {
+        throw new RangeError("varint value must be a non-negative integer");
+    }
+    if (value >= MAX_VARINT_VALUE) {
+        throw new RangeError("varint value must be less than 2^53");
+    }
+    const bytes = [];
+    let remaining = value;
+    do {
+        let byte = remaining % 128;
+        remaining = Math.floor(remaining / 128);
+        if (remaining > 0)
+            byte |= CONTINUATION_BIT;
+        bytes.push(byte);
+    } while (remaining > 0);
+    return Uint8Array.from(bytes);
+}
+/** A plain list of `count` varints (not delta-coded) -- used for
+ * region_counts, city_counts, chunk_lengths, and block_lengths. */
+export function readVarintList(buf, offset, count) {
+    const values = new Array(count);
+    let cursor = offset;
+    for (let i = 0; i < count; i++) {
+        const { value, bytesRead } = readVarint(buf, cursor);
+        values[i] = value;
+        cursor += bytesRead;
+    }
+    return { values, bytesRead: cursor - offset };
+}
+/** A delta-coded list per Section 3.3 -- used for a chunk's palette and
+ * codes. Each stored varint is the difference from the previous element,
+ * with an implicit previous element of 0 before the first. */
+export function readDeltaVarintList(buf, offset, count) {
+    const values = new Float64Array(count);
+    const end = readDeltaVarintsInto(buf, offset, values);
+    return { values: Array.from(values), bytesRead: end - offset };
+}
+/** Where the varint `varintAt` last read ended: a second return value,
+ * kept here rather than in a returned object because a chunk's codes are
+ * millions of varints, and an object per varint showed up as most of a
+ * chunk's parse time. */
+let varintEnd = 0;
+/**
+ * The varint at `cursor`, leaving the offset just past it in
+ * `varintEnd`. The one varint decoder here, so every varint the reader
+ * reads gets the same checks.
+ */
+function varintAt(buf, cursor) {
+    let byte = buf[cursor++];
+    if (byte === undefined)
+        throw new IncompleteBufferError("varint runs past end of buffer");
+    let value = byte & PAYLOAD_MASK;
+    let scale = 128;
+    for (let n = 1; byte & CONTINUATION_BIT; n++) {
+        if (n === MAX_VARINT_BYTES)
+            throw new CGeoDataFormatError(`varint longer than ${MAX_VARINT_BYTES} bytes`);
+        byte = buf[cursor++];
+        if (byte === undefined)
+            throw new IncompleteBufferError("varint runs past end of buffer");
+        value += (byte & PAYLOAD_MASK) * scale;
+        if ((byte & CONTINUATION_BIT) === 0) {
+            if (value >= MAX_VARINT_VALUE) {
+                throw new CGeoDataFormatError("varint value must be less than 2^53");
+            }
+            if (byte === 0) {
+                // A minimal encoder never emits a trailing zero group (see
+                // writeVarint above), so this can only be non-minimal padding.
+                throw new CGeoDataFormatError("non-minimal varint encoding (trailing zero byte)");
+            }
+        }
+        scale *= 128;
+    }
+    varintEnd = cursor;
+    return value;
+}
+/**
+ * `out.length` delta-coded varints (Section 3.3) from `buf` at `offset`
+ * into `out`, returning the offset just past them. Decodes straight into
+ * a typed array, without an object per varint (see `varintEnd`).
+ */
+export function readDeltaVarintsInto(buf, offset, out) {
+    let cursor = offset;
+    let previous = 0;
+    for (let i = 0; i < out.length; i++) {
+        previous += varintAt(buf, cursor);
+        cursor = varintEnd;
+        out[i] = previous;
+    }
+    return cursor;
+}
+/**
+ * `readDeltaVarintsInto` for values known to fit in 32 bits, such as a
+ * chunk's codes at `max_depth` 16 and under: a running total that
+ * outgrows the `Uint32Array` is a format error, not a silent wraparound.
+ * A separate function rather than one taking either array, because V8
+ * optimizes a loop for the array types its function has seen, and a
+ * loop that has seen both decoded chunks ~10% slower.
+ */
+export function readDeltaVarints32Into(buf, offset, out) {
+    let cursor = offset;
+    let previous = 0;
+    for (let i = 0; i < out.length; i++) {
+        previous += varintAt(buf, cursor);
+        cursor = varintEnd;
+        if (previous > 0xffffffff) {
+            throw new CGeoDataFormatError(`delta-coded value ${previous} does not fit in 32 bits`);
+        }
+        out[i] = previous;
+    }
+    return cursor;
+}
+//# sourceMappingURL=varint.js.map

package/package.json

@@ -1,6 +1,77 @@
 {
-  "name": "@ferrybig/cgeodata-reader",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+	"name": "@ferrybig/cgeodata-reader",
+	"version": "0.1.0",
+	"description": "Reverse geocoding without a database: resolves a latitude/longitude to its country, region and city from OpenStreetMap boundaries, reading a compact CGEODATA file. No dependencies; Node.js and browsers.",
+	"keywords": [
+		"reverse-geocoding",
+		"geocoding",
+		"geodata",
+		"openstreetmap",
+		"administrative-boundaries",
+		"offline"
+	],
+	"license": "MIT",
+	"author": "Fernando van Loenhout",
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/ferrybig/loc-to-city.git",
+		"directory": "reader"
+	},
+	"homepage": "https://github.com/ferrybig/loc-to-city/tree/master/reader#readme",
+	"bugs": "https://github.com/ferrybig/loc-to-city/issues",
+	"type": "module",
+	"engines": {
+		"node": ">=24"
+	},
+	"sideEffects": false,
+	"main": "./dist/index.js",
+	"types": "./dist/index.d.ts",
+	"exports": {
+		".": {
+			"types": "./dist/index.d.ts",
+			"default": "./dist/index.js"
+		},
+		"./node": {
+			"types": "./dist/node.d.ts",
+			"default": "./dist/node.js"
+		},
+		"./indexes": {
+			"types": "./dist/indexes.d.ts",
+			"default": "./dist/indexes.js"
+		},
+		"./format": {
+			"types": "./dist/format.d.ts",
+			"default": "./dist/format.js"
+		},
+		"./package.json": "./package.json"
+	},
+	"bin": {
+		"cgeodata": "./dist/cli.js",
+		"cgeodata-reader": "./dist/cli.js"
+	},
+	"publishConfig": {
+		"access": "public"
+	},
+	"files": [
+		"dist",
+		"!dist/**/*.map",
+		"binary-format.md"
+	],
+	"scripts": {
+		"build": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\" && tsc -p tsconfig.json",
+		"test": "node --test test/*.test.ts",
+		"typecheck": "tsc -p tsconfig.test.json",
+		"bench": "node --expose-gc bench/bench.ts",
+		"docs": "typedoc",
+		"build:wasm": "node scripts/build-wasm.ts",
+		"prepack": "npm run build"
+	},
+	"devDependencies": {
+		"@biomejs/biome": "2.5.13",
+		"@playwright/test": "^1.48.0",
+		"@types/node": "^24.0.0",
+		"typedoc": "0.28.20",
+		"typescript": "^5.7.0",
+		"wabt": "1.0.36"
+	}
+}