@osmix/core 0.0.2 → 0.1.1

package/src/nodes.ts

@@ -1,7 +1,8 @@
-import type { OsmNode, OsmTags } from "@osmix/json"
 import type { OsmPbfBlock, OsmPbfDenseNodes } from "@osmix/pbf"
 import { assertValue } from "@osmix/shared/assert"
-import type { GeoBbox2D } from "@osmix/shared/types"
+import { microToDegrees, toMicroDegrees } from "@osmix/shared/coordinates"
+import type { GeoBbox2D, OsmNode, OsmTags } from "@osmix/shared/types"
+import { around as geoAround } from "geokdbush"
 import KDBush from "kdbush"
 import { Entities, type EntitiesTransferables } from "./entities"
 import { type IdOrIndex, Ids } from "./ids"
@@ -10,15 +11,17 @@ import { Tags } from "./tags"
 import {
 	BufferConstructor,
 	type BufferType,
-	CoordinateArrayType,
-	ResizeableTypedArray,
+	ResizeableTypedArray as RTA,
 } from "./typed-arrays"
+import type { ContentHasher } from "./utils"
 
-export interface NodesTransferables extends EntitiesTransferables {
-	lons: BufferType
-	lats: BufferType
+export interface NodesTransferables<T extends BufferType = BufferType>
+	extends EntitiesTransferables<T> {
+	lons: T
+	lats: T
 	bbox: GeoBbox2D
-	spatialIndex: BufferType
+	/** Optional - can be rebuilt via buildSpatialIndex() */
+	spatialIndex?: T
 }
 
 export interface AddNodeOptions {
@@ -26,66 +29,96 @@ export interface AddNodeOptions {
 }
 
 export class Nodes extends Entities<OsmNode> {
-	lons: ResizeableTypedArray<Float64Array>
-	lats: ResizeableTypedArray<Float64Array>
-	bbox: GeoBbox2D = [
-		Number.POSITIVE_INFINITY,
-		Number.POSITIVE_INFINITY,
-		Number.NEGATIVE_INFINITY,
-		Number.NEGATIVE_INFINITY,
+	/**
+	 * Coordinates are stored as integer microdegrees (Int32Array).
+	 * Use OSM_COORD_SCALE (1e7) to convert between degrees and microdegrees.
+	 */
+	private lons: RTA<Int32Array>
+	private lats: RTA<Int32Array>
+	// Bbox stored in microdegrees internally
+	private bbox: [
+		minLon: number,
+		minLat: number,
+		maxLon: number,
+		maxLat: number,
+	] = [
+		Number.MAX_SAFE_INTEGER,
+		Number.MAX_SAFE_INTEGER,
+		Number.MIN_SAFE_INTEGER,
+		Number.MIN_SAFE_INTEGER,
 	]
-	spatialIndex: KDBush = new KDBush(0, 128, Float64Array, BufferConstructor)
-
-	static from(stringTable: StringTable, nits: NodesTransferables) {
-		const idIndex = Ids.from(nits)
-		const tagIndex = Tags.from(stringTable, nits)
-		const nodeIndex = new Nodes(stringTable, idIndex, tagIndex)
-		nodeIndex.lons = ResizeableTypedArray.from(CoordinateArrayType, nits.lons)
-		nodeIndex.lats = ResizeableTypedArray.from(CoordinateArrayType, nits.lats)
-		nodeIndex.bbox = nits.bbox
-		nodeIndex.spatialIndex = KDBush.from(nits.spatialIndex)
-		return nodeIndex
-	}
-
-	constructor(stringTable: StringTable, idIndex?: Ids, tagIndex?: Tags) {
-		super("node", stringTable, idIndex, tagIndex)
-		this.lons = new ResizeableTypedArray(CoordinateArrayType)
-		this.lats = new ResizeableTypedArray(CoordinateArrayType)
-	}
+	// Spatial index stores coordinates in degrees (Float64Array) for geokdbush compatibility
+	private spatialIndex: KDBush = new KDBush(
+		0,
+		128,
+		Float64Array,
+		BufferConstructor,
+	)
+	// Track if spatial index was properly built (vs default empty)
+	private spatialIndexBuilt = false
 
-	transferables(): NodesTransferables {
-		return {
-			lons: this.lons.array.buffer,
-			lats: this.lats.array.buffer,
-			bbox: this.bbox,
-			spatialIndex: this.spatialIndex.data,
-			...this.ids.transferables(),
-			...this.tags.transferables(),
+	/**
+	 * Create a new Nodes index.
+	 */
+	constructor(stringTable: StringTable, transferables?: NodesTransferables) {
+		if (transferables) {
+			super(
+				"node",
+				new Ids(transferables),
+				new Tags(stringTable, transferables),
+			)
+			this.lons = RTA.from(Int32Array, transferables.lons)
+			this.lats = RTA.from(Int32Array, transferables.lats)
+			// Only load spatial index if provided (not stored in IndexedDB)
+			if (transferables.spatialIndex?.byteLength) {
+				this.spatialIndex = KDBush.from(transferables.spatialIndex)
+				this.spatialIndexBuilt = true
+			}
+			this.bbox = transferables.bbox
+			this.indexBuilt = true
+		} else {
+			super("node", new Ids(), new Tags(stringTable))
+			this.lons = new RTA(Int32Array)
+			this.lats = new RTA(Int32Array)
+			// Spatial index uses Float64Array with degrees for geokdbush compatibility
+			this.spatialIndex = new KDBush(0, 128, Float64Array, BufferConstructor)
 		}
 	}
 
-	addNode(node: OsmNode) {
-		this.ids.add(node.id)
-		this.tags.addTags(node.tags)
+	/**
+	 * Add a single node to the index.
+	 */
+	addNode(node: OsmNode): number {
+		const nodeIndex = this.addEntity(node.id, node.tags ?? {})
 
-		this.lons.push(node.lon)
-		this.lats.push(node.lat)
+		const lonMicro = toMicroDegrees(node.lon)
+		const latMicro = toMicroDegrees(node.lat)
+
+		this.lons.push(lonMicro)
+		this.lats.push(latMicro)
 
 		if (node.lon < this.bbox[0]) this.bbox[0] = node.lon
 		if (node.lat < this.bbox[1]) this.bbox[1] = node.lat
 		if (node.lon > this.bbox[2]) this.bbox[2] = node.lon
 		if (node.lat > this.bbox[3]) this.bbox[3] = node.lat
+
+		return nodeIndex
 	}
 
+	/**
+	 * Add dense nodes from a PBF block.
+	 */
 	addDenseNodes(
 		dense: OsmPbfDenseNodes,
 		block: OsmPbfBlock,
-		blockStringIndexMap: Map<number, number>,
+		blockStringIndexMap: Uint32Array,
 		filter?: (node: OsmNode) => boolean,
 	): number {
+		// PBF block already has offsets and granularity converted to degrees
 		const lon_offset = block.lon_offset ?? 0
 		const lat_offset = block.lat_offset ?? 0
 		const granularity = block.granularity ?? 1e7
+
 		const delta = {
 			id: 0,
 			lat: 0,
@@ -99,7 +132,7 @@ export class Nodes extends Entities<OsmNode> {
 		const getStringTableIndex = (keyIndex: number) => {
 			const key = dense.keys_vals[keyIndex]
 			assertValue(key, "Block string key is undefined")
-			const index = blockStringIndexMap.get(key)
+			const index = blockStringIndexMap[key]
 			assertValue(index, "Block string not found")
 			return index
 		}
@@ -118,9 +151,14 @@ export class Nodes extends Entities<OsmNode> {
 			delta.lat += latSid
 			delta.lon += lonSid
 
+			// Calculate degrees from PBF delta encoding
 			const lon = lon_offset + delta.lon / granularity
 			const lat = lat_offset + delta.lat / granularity
 
+			// Convert to microdegrees for storage
+			const lonMicro = toMicroDegrees(lon)
+			const latMicro = toMicroDegrees(lat)
+
 			const tagKeys: number[] = []
 			const tagValues: number[] = []
 			if (dense.keys_vals.length > 0) {
@@ -146,46 +184,90 @@ export class Nodes extends Entities<OsmNode> {
 				: true
 			if (!shouldInclude) continue
 
-			this.ids.add(delta.id)
-			this.lons.push(lon)
-			this.lats.push(lat)
+			this.addEntity(delta.id, tagKeys, tagValues)
+			this.lons.push(lonMicro)
+			this.lats.push(latMicro)
 
 			if (lon < this.bbox[0]) this.bbox[0] = lon
 			if (lat < this.bbox[1]) this.bbox[1] = lat
 			if (lon > this.bbox[2]) this.bbox[2] = lon
 			if (lat > this.bbox[3]) this.bbox[3] = lat
-			this.tags.addTagKeysAndValues(tagKeys, tagValues)
 			added++
 		}
 
 		return added
 	}
 
+	/**
+	 * Compact the internal arrays to free up memory.
+	 */
 	buildEntityIndex() {
 		this.lons.compact()
 		this.lats.compact()
 	}
 
+	/**
+	 * Build the spatial index for nodes.
+	 * Spatial index stores degrees (Float64Array) for geokdbush compatibility.
+	 */
 	buildSpatialIndex() {
 		console.time("NodeIndex.buildSpatialIndex")
+		// Use Float64Array with degrees for geokdbush geographic queries
 		this.spatialIndex = new KDBush(
 			this.size,
-			128,
+			64,
 			Float64Array,
 			BufferConstructor,
 		)
 		for (let i = 0; i < this.size; i++) {
-			this.spatialIndex.add(this.lons.at(i), this.lats.at(i))
+			// Convert microdegrees to degrees for spatial index
+			const lon = microToDegrees(this.lons.at(i))
+			const lat = microToDegrees(this.lats.at(i))
+			this.spatialIndex.add(lon, lat)
 		}
 		this.spatialIndex.finish()
+		this.spatialIndexBuilt = true
 		console.timeEnd("NodeIndex.buildSpatialIndex")
 	}
 
+	/**
+	 * Check if the spatial index has been built.
+	 */
+	hasSpatialIndex(): boolean {
+		return this.spatialIndexBuilt
+	}
+
+	/**
+	 * Get the bounding box of all nodes.
+	 */
+	getBbox(): GeoBbox2D {
+		return this.bbox
+	}
+
+	/**
+	 * Get the bounding box of a specific node.
+	 */
+	getEntityBbox(i: IdOrIndex): GeoBbox2D {
+		const index = "index" in i ? i.index : this.ids.idOrIndex(i)[0]
+		const lon = microToDegrees(this.lons.at(index))
+		const lat = microToDegrees(this.lats.at(index))
+		return [lon, lat, lon, lat] as GeoBbox2D
+	}
+
+	/**
+	 * Get the longitude and latitude of a specific node.
+	 */
 	getNodeLonLat(i: IdOrIndex): [number, number] {
 		const index = "index" in i ? i.index : this.ids.idOrIndex(i)[0]
-		return [this.lons.at(index), this.lats.at(index)]
+		return [
+			microToDegrees(this.lons.at(index)),
+			microToDegrees(this.lats.at(index)),
+		]
 	}
 
+	/**
+	 * Get the full node entity.
+	 */
 	getFullEntity(index: number, id: number, tags?: OsmTags): OsmNode {
 		const [lon, lat] = this.getNodeLonLat({ index })
 		if (tags) {
@@ -204,11 +286,119 @@ export class Nodes extends Entities<OsmNode> {
 	}
 
 	// Spatial operations
-	withinBbox(bbox: GeoBbox2D): number[] {
+	/**
+	 * Find node indexes within a bounding box.
+	 */
+	findIndexesWithinBbox(bbox: GeoBbox2D): number[] {
+		// Spatial index stores degrees, so query with degrees directly
 		return this.spatialIndex.range(bbox[0], bbox[1], bbox[2], bbox[3])
 	}
 
-	withinRadius(x: number, y: number, radius: number): number[] {
-		return this.spatialIndex.within(x, y, radius)
+	/**
+	 * Find node indexes within a radius of a point.
+	 * Uses geokdbush for proper great-circle distance calculations.
+	 * @param lon - Longitude in degrees.
+	 * @param lat - Latitude in degrees.
+	 * @param radiusKm - Radius in kilometers.
+	 * @returns Array of node indexes within the radius.
+	 */
+	findIndexesWithinRadius(
+		lon: number,
+		lat: number,
+		radiusKm: number,
+	): number[] {
+		// Use geokdbush for proper geographic distance calculations
+		return geoAround(
+			this.spatialIndex,
+			lon,
+			lat,
+			Number.POSITIVE_INFINITY,
+			radiusKm,
+		)
+	}
+
+	/**
+	 * Get nodes within a bounding box.
+	 * @param bbox - The bounding box to search within.
+	 * @param include - A function to filter nodes. If provided, only nodes for which the function returns true will be included.
+	 * @returns An object containing the IDs and positions of the nodes within the bounding box.
+	 */
+	withinBbox(
+		bbox: GeoBbox2D,
+		include?: (i: number) => boolean,
+	): {
+		ids: Float64Array
+		positions: Float64Array
+	} {
+		console.time("Nodes.withinBbox")
+		const nodeCandidates = this.findIndexesWithinBbox(bbox)
+		const nodePositions = new Float64Array(nodeCandidates.length * 2)
+		const ids = new Float64Array(nodeCandidates.length)
+
+		let skipped = 0
+		nodeCandidates.forEach((nodeIndex, i) => {
+			if (include && !include(nodeIndex)) {
+				skipped++
+				return
+			}
+
+			const [lon, lat] = this.getNodeLonLat({ index: nodeIndex })
+			ids[i - skipped] = this.ids.at(nodeIndex)
+			nodePositions[(i - skipped) * 2] = lon
+			nodePositions[(i - skipped) * 2 + 1] = lat
+		})
+		console.timeEnd("Nodes.withinBbox")
+		return {
+			ids: ids.subarray(0, nodeCandidates.length - skipped),
+			positions: nodePositions.slice(0, (nodeCandidates.length - skipped) * 2),
+		}
+	}
+
+	/**
+	 * Get transferable objects for passing to another thread.
+	 * Only includes spatialIndex if it has been built.
+	 */
+	override transferables(): NodesTransferables {
+		const base = {
+			...super.transferables(),
+			lons: this.lons.array.buffer,
+			lats: this.lats.array.buffer,
+			bbox: this.bbox,
+		}
+		// Only include spatial index if it was built
+		if (this.spatialIndexBuilt) {
+			return { ...base, spatialIndex: this.spatialIndex.data }
+		}
+		return base
+	}
+
+	/**
+	 * Get the approximate memory requirements for a given number of nodes in bytes.
+	 */
+	static getBytesRequired(count: number) {
+		if (count === 0) return 0
+		const indexBytes = (count < 65536 ? 2 : 4) * count
+		// Spatial index stores coordinates in degrees (Float64Array)
+		const coordsBytes = count * 2 * Float64Array.BYTES_PER_ELEMENT
+		const padding = (8 - (indexBytes % 8)) % 8
+		const spatialIndexBytes = 8 + indexBytes + coordsBytes + padding
+
+		return (
+			Ids.getBytesRequired(count) +
+			Tags.getBytesRequired(count) +
+			count * Int32Array.BYTES_PER_ELEMENT + // lons (stored in microdegrees)
+			count * Int32Array.BYTES_PER_ELEMENT + // lats (stored in microdegrees)
+			spatialIndexBytes
+		)
+	}
+
+	/**
+	 * Update a ContentHasher with node-specific data (coordinates).
+	 */
+	override updateHash(hasher: ContentHasher): ContentHasher {
+		return super
+			.updateHash(hasher)
+			.update(this.lons.array)
+			.update(this.lats.array)
 	}
 }

package/src/osm.ts

@@ -0,0 +1,222 @@
+import type { OsmPbfHeaderBlock } from "@osmix/pbf"
+import type { GeoBbox2D } from "@osmix/shared/types"
+import { Nodes, type NodesTransferables } from "./nodes"
+import { Relations, type RelationsTransferables } from "./relations"
+import StringTable, { type StringTableTransferables } from "./stringtable"
+import type { BufferType } from "./typed-arrays"
+import { ContentHasher } from "./utils"
+import { Ways, type WaysTransferables } from "./ways"
+
+export interface OsmTransferables<T extends BufferType = BufferType> {
+	id: string
+	header: OsmPbfHeaderBlock
+	contentHash: string
+	stringTable: StringTableTransferables<T>
+	nodes: NodesTransferables<T>
+	ways: WaysTransferables<T>
+	relations: RelationsTransferables<T>
+}
+
+export interface OsmInfo {
+	id: string
+	bbox: GeoBbox2D
+	header: OsmPbfHeaderBlock
+	stats: {
+		nodes: number
+		ways: number
+		relations: number
+	}
+}
+
+export interface OsmOptions {
+	id: string
+	header: OsmPbfHeaderBlock
+}
+
+/**
+ * OSM Entity Index.
+ */
+export class Osm {
+	// Filename or ID of this OSM Entity index.
+	readonly id: string
+	readonly header: OsmPbfHeaderBlock
+
+	// Shared string lookup table for all nodes, ways, and relations
+	readonly stringTable: StringTable
+	readonly nodes: Nodes
+	readonly ways: Ways
+	readonly relations: Relations
+
+	private indexBuilt = false
+	private _contentHash = ""
+
+	/**
+	 * Create a new OSM Entity index.
+	 */
+	constructor(opts?: Partial<OsmOptions> | OsmTransferables | Osm) {
+		this.id = opts?.id ?? "unknown"
+		this.header = opts?.header ?? {
+			required_features: [],
+			optional_features: [],
+		}
+		if (opts && "stringTable" in opts) {
+			if (opts instanceof Osm) {
+				this.stringTable = new StringTable(opts.stringTable.transferables())
+				this.nodes = new Nodes(this.stringTable, opts.nodes.transferables())
+				this.ways = new Ways(
+					this.stringTable,
+					this.nodes,
+					opts.ways.transferables(),
+				)
+				this.relations = new Relations(
+					this.stringTable,
+					this.nodes,
+					this.ways,
+					opts.relations.transferables(),
+				)
+				this._contentHash = opts._contentHash
+				this.indexBuilt = true
+			} else {
+				this.stringTable = new StringTable(opts.stringTable)
+				this.nodes = new Nodes(this.stringTable, opts.nodes)
+				this.ways = new Ways(this.stringTable, this.nodes, opts.ways)
+				this.relations = new Relations(
+					this.stringTable,
+					this.nodes,
+					this.ways,
+					opts.relations,
+				)
+				this._contentHash = opts.contentHash
+				this.indexBuilt = true
+			}
+		} else {
+			this.stringTable = new StringTable()
+			this.nodes = new Nodes(this.stringTable)
+			this.ways = new Ways(this.stringTable, this.nodes)
+			this.relations = new Relations(this.stringTable, this.nodes, this.ways)
+		}
+	}
+
+	/**
+	 * Build the internal indexes for all entities.
+	 * Also computes the content hash after indexes are built.
+	 */
+	buildIndexes() {
+		this.stringTable.buildIndex()
+		this.nodes.buildIndex()
+		this.ways.buildIndex()
+		this.relations.buildIndex()
+		this.indexBuilt = true
+		// Compute content hash after indexes are built
+		this._contentHash = this.computeContentHash()
+	}
+
+	/**
+	 * Check if the index is built and ready for use.
+	 */
+	isReady() {
+		return (
+			this.nodes.isReady() &&
+			this.ways.isReady() &&
+			this.relations.isReady() &&
+			this.indexBuilt
+		)
+	}
+
+	/**
+	 * Build spatial indexes for all entities.
+	 */
+	buildSpatialIndexes() {
+		this.nodes.buildSpatialIndex()
+		this.ways.buildSpatialIndex()
+		this.relations.buildSpatialIndex()
+	}
+
+	/**
+	 * Check if spatial indexes have been built for all entity types.
+	 */
+	hasSpatialIndexes(): boolean {
+		return (
+			this.nodes.hasSpatialIndex() &&
+			this.ways.hasSpatialIndex() &&
+			this.relations.hasSpatialIndex()
+		)
+	}
+
+	/**
+	 * Get the bounding box of all entities in the OSM index.
+	 */
+	bbox(): GeoBbox2D {
+		return this.nodes.getBbox()
+	}
+
+	/**
+	 * Get information about the OSM index.
+	 */
+	info(): OsmInfo {
+		return {
+			id: this.id,
+			bbox: this.bbox(),
+			header: this.header,
+			stats: {
+				nodes: this.nodes.size,
+				ways: this.ways.size,
+				relations: this.relations.size,
+			},
+		}
+	}
+
+	/**
+	 * Get transferable objects for passing to another thread.
+	 */
+	transferables(): OsmTransferables {
+		return {
+			id: this.id,
+			header: this.header,
+			contentHash: this._contentHash,
+			stringTable: this.stringTable.transferables(),
+			nodes: this.nodes.transferables(),
+			ways: this.ways.transferables(),
+			relations: this.relations.transferables(),
+		}
+	}
+
+	/**
+	 * Get the content hash of this OSM dataset.
+	 * The hash is computed when indexes are built.
+	 *
+	 * @returns A hex string hash uniquely identifying the content.
+	 */
+	contentHash(): string {
+		return this._contentHash
+	}
+
+	/**
+	 * Check if this OSM dataset has identical content to another.
+	 * Uses the pre-computed content hash for fast comparison.
+	 *
+	 * @param other - The other Osm instance to compare with.
+	 * @returns True if both datasets have identical content.
+	 */
+	isEqual(other: Osm | null | undefined): boolean {
+		if (!other) return false
+		return this._contentHash === other._contentHash
+	}
+
+	/**
+	 * Compute a content hash of all underlying data.
+	 * This hash uniquely identifies the dataset content regardless of metadata.
+	 *
+	 * @returns A hex string hash of the content.
+	 */
+	private computeContentHash(): string {
+		const hasher = new ContentHasher()
+		// Hash string table first (shared by all entities)
+		this.stringTable.updateHash(hasher)
+		// Hash each entity collection
+		this.nodes.updateHash(hasher)
+		this.ways.updateHash(hasher)
+		this.relations.updateHash(hasher)
+		return hasher.digest()
+	}
+}