@osmix/core 0.2.0 → 0.3.0

package/src/ids.ts

@@ -30,10 +30,10 @@ const BLOCK_SIZE = 256;
 export interface IdsTransferables<T extends BufferType = BufferType> {
   /** Entity IDs in insertion order. */
   ids: T;
-  /** Sorted IDs for binary search (aliases `ids` if already sorted). */
-  sortedIds: T;
-  /** Maps sorted position → original insertion index. */
-  sortedIdPositionToIndex: T;
+  /** Sorted IDs for binary search. Omitted when `idsAreSorted` is true. */
+  sortedIds?: T;
+  /** Maps sorted position → original insertion index. Omitted for sorted IDs. */
+  sortedIdPositionToIndex?: T;
   /** Anchor array: every BLOCK_SIZE-th sorted ID. */
   anchors: T;
   /** True if IDs were inserted in ascending order. */
@@ -67,8 +67,16 @@ export class Ids {
   constructor(transferables?: IdsTransferables) {
     if (transferables) {
       this.ids = RTA.from(Float64Array, transferables.ids);
-      this.idsSorted = new Float64Array(transferables.sortedIds);
-      this.sortedIdPositionToIndex = new Uint32Array(transferables.sortedIdPositionToIndex);
+      if (transferables.idsAreSorted) {
+        this.idsSorted = this.ids.array;
+        this.sortedIdPositionToIndex = new Uint32Array(new BufferConstructor(0));
+      } else {
+        if (!transferables.sortedIds || !transferables.sortedIdPositionToIndex) {
+          throw Error("Unsorted IDs require sorted lookup buffers.");
+        }
+        this.idsSorted = new Float64Array(transferables.sortedIds);
+        this.sortedIdPositionToIndex = new Uint32Array(transferables.sortedIdPositionToIndex);
+      }
       this.anchors = new Float64Array(transferables.anchors);
       this.idsAreSorted = transferables.idsAreSorted;
       this.indexBuilt = true;
@@ -157,12 +165,8 @@ export class Ids {
     } else {
       // Point to the same array
       this.idsSorted = this.ids.array;
-      // Create the sortedIdPositionToIndex array
-      const posBuffer = new BufferConstructor(this.size * Uint32Array.BYTES_PER_ELEMENT);
-      this.sortedIdPositionToIndex = new Uint32Array(posBuffer);
-      for (let i = 0; i < this.size; i++) {
-        this.sortedIdPositionToIndex[i] = i;
-      }
+      // Sorted position is already the insertion index; no identity map is needed.
+      this.sortedIdPositionToIndex = new Uint32Array(new BufferConstructor(0));
     }
 
     // Build anchors (every blockSize-th key)
@@ -238,37 +242,83 @@ export class Ids {
   /** @internal Iterate sorted IDs with their original storage positions. */
   *sortedEntries(): Generator<readonly [id: number, index: number]> {
     for (let i = 0; i < this.idsSorted.length; i++) {
-      const id = this.idsSorted[i];
-      const index = this.sortedIdPositionToIndex[i];
+      yield this.sortedEntry(i);
+    }
+  }
+
+  /**
+   * Iterate in canonical OSM file order: negative IDs first by increasing
+   * absolute value, followed by non-negative IDs in ascending order.
+   *
+   * @internal
+   */
+  *osmSortedEntries(): Generator<readonly [id: number, index: number]> {
+    let firstNonNegative = 0;
+    while (firstNonNegative < this.idsSorted.length && this.idsSorted[firstNonNegative]! < 0) {
+      firstNonNegative++;
+    }
+
+    // Numeric sorting places negative IDs in the opposite of canonical OSM
+    // order. Reverse ID groups while preserving duplicate insertion order.
+    let groupEnd = firstNonNegative;
+    while (groupEnd > 0) {
+      const id = this.idsSorted[groupEnd - 1];
       assertValue(id, "Sorted ID is undefined");
-      assertValue(index, "Sorted position is undefined");
-      yield [id, index];
+      let groupStart = groupEnd - 1;
+      while (groupStart > 0 && this.idsSorted[groupStart - 1] === id) groupStart--;
+      for (let position = groupStart; position < groupEnd; position++) {
+        yield this.sortedEntry(position);
+      }
+      groupEnd = groupStart;
+    }
+
+    for (let position = firstNonNegative; position < this.idsSorted.length; position++) {
+      yield this.sortedEntry(position);
     }
   }
 
+  private sortedEntry(position: number): readonly [id: number, index: number] {
+    const id = this.idsSorted[position];
+    assertValue(id, "Sorted ID is undefined");
+    if (this.idsAreSorted) return [id, position];
+    const index = this.sortedIdPositionToIndex[position];
+    assertValue(index, "Sorted position is undefined");
+    return [id, index];
+  }
+
   /**
    * Get transferable buffers for passing to another thread.
    * @returns Serializable representation of this index.
    */
   transferables(): IdsTransferables {
-    return {
+    const base: IdsTransferables = {
       ids: this.ids.array.buffer,
-      sortedIds: this.idsSorted.buffer,
-      sortedIdPositionToIndex: this.sortedIdPositionToIndex.buffer,
       anchors: this.anchors.buffer,
       idsAreSorted: this.idsAreSorted,
     };
+    if (this.idsAreSorted) return base;
+    return {
+      ...base,
+      sortedIds: this.idsSorted.buffer,
+      sortedIdPositionToIndex: this.sortedIdPositionToIndex.buffer,
+    };
   }
 
   /**
    * Get the approximate memory requirements for a given number of IDs in bytes.
+   *
+   * `idsAreSorted` defaults to true because PBF entity collections arrive in
+   * ascending ID order and retain no separate sort buffers. Pass false when
+   * estimating a programmatically built (unsorted) collection, which also
+   * retains sorted IDs plus a position map.
    */
-  static getBytesRequired(count: number) {
+  static getBytesRequired(count: number, idsAreSorted = true) {
     if (count === 0) return 0;
     return (
       count * Float64Array.BYTES_PER_ELEMENT + // ids
-      count * Float64Array.BYTES_PER_ELEMENT + // sortedIds
-      count * Uint32Array.BYTES_PER_ELEMENT + // sortedIdPositionToIndex
+      (idsAreSorted
+        ? 0
+        : count * (Float64Array.BYTES_PER_ELEMENT + Uint32Array.BYTES_PER_ELEMENT)) + // sort index
       Math.ceil(count / BLOCK_SIZE) * Float64Array.BYTES_PER_ELEMENT // anchors
     );
   }

package/src/index.ts

@@ -4,8 +4,8 @@
  * Efficiently stores and queries OpenStreetMap entities (nodes, ways, relations).
  *
  * Features:
- * - **Memory-efficient**: Uses typed arrays and Int32Array microdegrees.
- * - **Spatial indexing**: KDBush (points) and Flatbush (bboxes) for fast geographic queries.
+ * - **Memory-efficient**: Uses typed arrays; coordinates are Int32 at 1e-7 degrees.
+ * - **Spatial indexing**: Compact indirect KD trees (points) and Flatbush (bboxes).
  * - **Worker-ready**: Zero-copy transfer via `transferables()`.
  * - **Tag indexing**: Fast reverse lookup by tag key.
  *
@@ -22,19 +22,25 @@
  * @module @osmix/core
  */
 
-// oxlint-disable-next-line typescript/triple-slash-reference -- ambient module declarations for untyped packages
-/// <reference path="./types/kdbush.d.ts" />
-// oxlint-disable-next-line typescript/triple-slash-reference -- ambient module declarations for untyped packages
-/// <reference path="./types/geokdbush.d.ts" />
 // oxlint-disable-next-line typescript/triple-slash-reference -- ambient module declarations for untyped packages
 /// <reference path="./types/geoflatbush.d.ts" />
 
 export type { IdOrIndex } from "./ids.ts";
 export type { OsmReader, OsmWriter } from "./contracts.ts";
+export { OsmEntityIndexBuildError, type OsmEntityIndexComponent } from "./entities.ts";
+export * from "./limits.ts";
 export * from "./nodes.ts";
 export * from "./osm.ts";
 export * from "./relations.ts";
+export { negativeIdMap, renumberNegativeIds, type OsmIdMap } from "./renumber.ts";
 export * from "./stringtable.ts";
 export * from "./tags.ts";
-export { BufferConstructor, type BufferType } from "./typed-arrays.ts";
+export {
+  BufferConstructor,
+  ResizeableTypedArray,
+  TypedBufferAllocationError,
+  type BufferType,
+  type TypedBufferAllocationOperation,
+  type TypedBufferType,
+} from "./typed-arrays.ts";
 export * from "./ways.ts";

package/src/indirect-kd-index.ts

@@ -0,0 +1,212 @@
+/**
+ * A compact static KD index over coordinate columns owned by another object.
+ *
+ * The selection and traversal algorithms are adapted from KDBush 4.1.0,
+ * Copyright (c) 2018 Vladimir Agafonkin, distributed under the ISC License.
+ * Unlike KDBush, this index stores only a Uint32 permutation. Coordinates are
+ * dereferenced from the existing Int32 microdegree columns.
+ */
+
+import type { BufferType } from "./typed-arrays.ts";
+import { BufferConstructor } from "./typed-arrays.ts";
+
+const DEFAULT_NODE_SIZE = 64;
+
+export class IndirectKdIndex {
+  readonly indexes: Uint32Array;
+  private readonly lons: Int32Array;
+  private readonly lats: Int32Array;
+  private readonly nodeSize: number;
+
+  constructor(
+    lons: Int32Array,
+    lats: Int32Array,
+    indexes: Uint32Array,
+    nodeSize = DEFAULT_NODE_SIZE,
+  ) {
+    this.lons = lons;
+    this.lats = lats;
+    this.indexes = indexes;
+    this.nodeSize = nodeSize;
+  }
+
+  static build(
+    lons: Int32Array,
+    lats: Int32Array,
+    count: number,
+    fill: (indexes: Uint32Array) => void,
+  ): IndirectKdIndex {
+    const buffer = new BufferConstructor(count * Uint32Array.BYTES_PER_ELEMENT);
+    const indexes = new Uint32Array(buffer);
+    fill(indexes);
+    sortKd(indexes, lons, lats, DEFAULT_NODE_SIZE, 0, indexes.length - 1, 0);
+    return new IndirectKdIndex(lons, lats, indexes);
+  }
+
+  static from(lons: Int32Array, lats: Int32Array, buffer: BufferType): IndirectKdIndex {
+    if (buffer.byteLength % Uint32Array.BYTES_PER_ELEMENT !== 0) {
+      throw new Error("Indirect KD index buffer length must be divisible by four");
+    }
+    return new IndirectKdIndex(lons, lats, new Uint32Array(buffer));
+  }
+
+  get buffer(): BufferType {
+    return this.indexes.buffer as BufferType;
+  }
+
+  /**
+   * Find entity indexes within an inclusive bounding box.
+   *
+   * Bounds are **integer microdegrees** (the units of the underlying Int32
+   * coordinate columns), not degrees. `Nodes.findIndexesWithinBbox` performs
+   * the degree → microdegree conversion.
+   */
+  range(minLon: number, minLat: number, maxLon: number, maxLat: number): number[] {
+    const stack = [0, this.indexes.length - 1, 0];
+    const result: number[] = [];
+
+    while (stack.length > 0) {
+      const axis = stack.pop();
+      const right = stack.pop();
+      const left = stack.pop();
+      if (axis === undefined || right === undefined || left === undefined) {
+        throw new Error("Indirect KD index traversal stack is corrupt");
+      }
+
+      if (right - left <= this.nodeSize) {
+        for (let i = left; i <= right; i++) {
+          const entityIndex = entityAt(this.indexes, i);
+          const lon = coordinate(this.lons, this.lats, entityIndex, 0);
+          const lat = coordinate(this.lons, this.lats, entityIndex, 1);
+          if (lon >= minLon && lon <= maxLon && lat >= minLat && lat <= maxLat) {
+            result.push(entityIndex);
+          }
+        }
+        continue;
+      }
+
+      const middle = (left + right) >> 1;
+      const entityIndex = entityAt(this.indexes, middle);
+      const lon = coordinate(this.lons, this.lats, entityIndex, 0);
+      const lat = coordinate(this.lons, this.lats, entityIndex, 1);
+
+      if (lon >= minLon && lon <= maxLon && lat >= minLat && lat <= maxLat) {
+        result.push(entityIndex);
+      }
+
+      const splitCoordinate = axis === 0 ? lon : lat;
+      const minimum = axis === 0 ? minLon : minLat;
+      const maximum = axis === 0 ? maxLon : maxLat;
+      if (minimum <= splitCoordinate) stack.push(left, middle - 1, 1 - axis);
+      if (maximum >= splitCoordinate) stack.push(middle + 1, right, 1 - axis);
+    }
+
+    return result;
+  }
+}
+
+function sortKd(
+  indexes: Uint32Array,
+  lons: Int32Array,
+  lats: Int32Array,
+  nodeSize: number,
+  left: number,
+  right: number,
+  axis: number,
+) {
+  if (right - left <= nodeSize) return;
+  const middle = (left + right) >> 1;
+  select(indexes, lons, lats, middle, left, right, axis);
+  sortKd(indexes, lons, lats, nodeSize, left, middle - 1, 1 - axis);
+  sortKd(indexes, lons, lats, nodeSize, middle + 1, right, 1 - axis);
+}
+
+/** Floyd-Rivest selection adapted to dereference coordinates indirectly. */
+function select(
+  indexes: Uint32Array,
+  lons: Int32Array,
+  lats: Int32Array,
+  k: number,
+  initialLeft: number,
+  initialRight: number,
+  axis: number,
+) {
+  let left = initialLeft;
+  let right = initialRight;
+
+  while (right > left) {
+    if (right - left > 600) {
+      const n = right - left + 1;
+      const m = k - left + 1;
+      const z = Math.log(n);
+      const s = 0.5 * Math.exp((2 * z) / 3);
+      const sd = 0.5 * Math.sqrt((z * s * (n - s)) / n) * (m - n / 2 < 0 ? -1 : 1);
+      const newLeft = Math.max(left, Math.floor(k - (m * s) / n + sd));
+      const newRight = Math.min(right, Math.floor(k + ((n - m) * s) / n + sd));
+      select(indexes, lons, lats, k, newLeft, newRight, axis);
+    }
+
+    const pivotIndex = entityAt(indexes, k);
+    const pivot = coordinate(lons, lats, pivotIndex, axis);
+    let i = left;
+    let j = right;
+
+    swap(indexes, left, k);
+    if (compareCoordinate(lons, lats, entityAt(indexes, right), pivotIndex, axis) > 0) {
+      swap(indexes, left, right);
+    }
+
+    while (i < j) {
+      swap(indexes, i, j);
+      i++;
+      j--;
+      while (i <= right && coordinate(lons, lats, entityAt(indexes, i), axis) < pivot) i++;
+      while (j >= left && coordinate(lons, lats, entityAt(indexes, j), axis) > pivot) j--;
+    }
+
+    if (coordinate(lons, lats, entityAt(indexes, left), axis) === pivot) {
+      swap(indexes, left, j);
+    } else {
+      j++;
+      swap(indexes, j, right);
+    }
+
+    if (j <= k) left = j + 1;
+    if (k <= j) right = j - 1;
+  }
+}
+
+/**
+ * Read the permutation entry at `position`, throwing on an out-of-bounds read.
+ * A silent skip here would leave the permutation subtly unsorted and produce
+ * wrong query results, so fail loudly instead.
+ */
+function entityAt(indexes: Uint32Array, position: number): number {
+  const value = indexes[position];
+  if (value === undefined) {
+    throw new Error(`Indirect KD index read out of bounds at position ${position}`);
+  }
+  return value;
+}
+
+function coordinate(lons: Int32Array, lats: Int32Array, entityIndex: number, axis: number): number {
+  const value = axis === 0 ? lons[entityIndex] : lats[entityIndex];
+  if (value === undefined) throw new Error(`Missing coordinate for node index ${entityIndex}`);
+  return value;
+}
+
+function compareCoordinate(
+  lons: Int32Array,
+  lats: Int32Array,
+  a: number,
+  b: number,
+  axis: number,
+): number {
+  return coordinate(lons, lats, a, axis) - coordinate(lons, lats, b, axis);
+}
+
+function swap(indexes: Uint32Array, a: number, b: number) {
+  const valueA = entityAt(indexes, a);
+  indexes[a] = entityAt(indexes, b);
+  indexes[b] = valueA;
+}

package/src/limits.ts

@@ -0,0 +1,73 @@
+/**
+ * Hard capacity limits set by the typed-array layout of `Osm` storage.
+ *
+ * See `docs/limits.md` for the full list, including limits that come from the
+ * JavaScript engine rather than from this layout.
+ */
+
+/** Largest number of node refs one way can hold (`Ways.refCount` is a `Uint16Array`). */
+export const MAX_WAY_REFS = 0xffff;
+
+/** Largest number of members one relation can hold (`Relations.memberCount` is a `Uint16Array`). */
+export const MAX_RELATION_MEMBERS = 0xffff;
+
+/** Largest UTF-8 byte length of one string (`StringTable.count` is a `Uint16Array`). */
+export const MAX_STRING_BYTES = 0xffff;
+
+/**
+ * Largest element offset a `Uint32Array` offset column can store. Bounds the total
+ * way refs, relation members and string-table bytes in one dataset.
+ */
+export const MAX_UINT32_OFFSET = 0xffffffff;
+
+/** The storage limit an {@link OsmCapacityError} reports. */
+export type OsmCapacityLimit =
+  | "way-refs"
+  | "relation-members"
+  | "string-bytes"
+  | "total-way-refs"
+  | "total-relation-members"
+  | "total-string-bytes";
+
+/** Thrown when an entity or dataset is larger than the storage layout can hold. */
+export class OsmCapacityError extends Error {
+  readonly code = "OSM_CAPACITY_EXCEEDED";
+  readonly limit: OsmCapacityLimit;
+  readonly value: number;
+  readonly max: number;
+
+  constructor(limit: OsmCapacityLimit, value: number, max: number, subject: string) {
+    super(
+      `${subject} has ${value.toLocaleString("en-US")} ${capacityUnit(limit)}; ` +
+        `Osmix stores at most ${max.toLocaleString("en-US")}.`,
+    );
+    this.name = "OsmCapacityError";
+    this.limit = limit;
+    this.value = value;
+    this.max = max;
+  }
+}
+
+function capacityUnit(limit: OsmCapacityLimit): string {
+  switch (limit) {
+    case "way-refs":
+    case "total-way-refs":
+      return "node refs";
+    case "relation-members":
+    case "total-relation-members":
+      return "members";
+    case "string-bytes":
+    case "total-string-bytes":
+      return "UTF-8 bytes";
+  }
+}
+
+/** Throw {@link OsmCapacityError} when `value` is above `max`. */
+export function assertCapacity(
+  limit: OsmCapacityLimit,
+  value: number,
+  max: number,
+  subject: () => string,
+): void {
+  if (value > max) throw new OsmCapacityError(limit, value, max, subject());
+}