@osmix/core 0.0.2 → 0.1.1

package/src/ways.ts

@@ -1,7 +1,7 @@
-import type { OsmTags, OsmWay } from "@osmix/json"
 import type { OsmPbfWay } from "@osmix/pbf"
-import type { GeoBbox2D, LonLat } from "@osmix/shared/types"
+import type { GeoBbox2D, LonLat, OsmTags, OsmWay } from "@osmix/shared/types"
 import Flatbush from "flatbush"
+import { around as geoAround } from "geoflatbush"
 import { Entities, type EntitiesTransferables } from "./entities"
 import { type IdOrIndex, Ids } from "./ids"
 import type { Nodes } from "./nodes"
@@ -10,69 +10,77 @@ import { Tags } from "./tags"
 import {
 	BufferConstructor,
 	type BufferType,
-	CoordinateArrayType,
 	IdArrayType,
-	ResizeableTypedArray,
+	ResizeableTypedArray as RTA,
 } from "./typed-arrays"
+import type { ContentHasher } from "./utils"
 
-export interface WaysTransferables extends EntitiesTransferables {
-	refStart: BufferType
-	refCount: BufferType
-	refs: BufferType
-	bbox: BufferType
-	spatialIndex: BufferType
+export interface WaysTransferables<T extends BufferType = BufferType>
+	extends EntitiesTransferables<T> {
+	refStart: T
+	refCount: T
+	refs: T
+	bbox: T
+	/** Optional - can be rebuilt via buildSpatialIndex() */
+	spatialIndex?: T
 }
 
 export class Ways extends Entities<OsmWay> {
-	spatialIndex: Flatbush = new Flatbush(1)
+	private spatialIndex: Flatbush = new Flatbush(1)
+	// Track if spatial index was properly built (vs default empty)
+	private spatialIndexBuilt = false
 
-	refStart: ResizeableTypedArray<Uint32Array>
-	refCount: ResizeableTypedArray<Uint16Array> // Maximum 2,000 nodes per way
+	private refStart: RTA<Uint32Array>
+	private refCount: RTA<Uint16Array> // Maximum 2,000 nodes per way
 
 	// Node IDs
-	refs: ResizeableTypedArray<Float64Array>
+	private refs: RTA<Float64Array>
 
 	// Bounding box of the way in geographic coordinates
-	bbox: ResizeableTypedArray<Float64Array>
-
-	static from(stringTable: StringTable, wits: WaysTransferables) {
-		const idIndex = Ids.from(wits)
-		const tagIndex = Tags.from(stringTable, wits)
-		const wayIndex = new Ways(stringTable, idIndex, tagIndex)
-		wayIndex.refStart = ResizeableTypedArray.from(Uint32Array, wits.refStart)
-		wayIndex.refCount = ResizeableTypedArray.from(Uint16Array, wits.refCount)
-		wayIndex.refs = ResizeableTypedArray.from(IdArrayType, wits.refs)
-		wayIndex.bbox = ResizeableTypedArray.from(CoordinateArrayType, wits.bbox)
-		wayIndex.spatialIndex = Flatbush.from(wits.spatialIndex)
-		return wayIndex
-	}
+	private bbox: RTA<Float64Array>
 
-	constructor(stringTable: StringTable, idIndex?: Ids, tagIndex?: Tags) {
-		super("way", stringTable, idIndex, tagIndex)
-		this.refStart = new ResizeableTypedArray(Uint32Array)
-		this.refCount = new ResizeableTypedArray(Uint16Array)
-		this.refs = new ResizeableTypedArray(IdArrayType)
-		this.bbox = new ResizeableTypedArray(CoordinateArrayType)
-	}
+	// Node reference index
+	private nodes: Nodes
 
-	transferables(): WaysTransferables {
-		return {
-			refStart: this.refStart.array.buffer,
-			refCount: this.refCount.array.buffer,
-			refs: this.refs.array.buffer,
-			bbox: this.bbox.array.buffer,
-			spatialIndex: this.spatialIndex.data,
-			...this.ids.transferables(),
-			...this.tags.transferables(),
+	/**
+	 * Create a new Ways index.
+	 */
+	constructor(
+		stringTable: StringTable,
+		nodes: Nodes,
+		transferables?: WaysTransferables,
+	) {
+		if (transferables) {
+			super("way", new Ids(transferables), new Tags(stringTable, transferables))
+			this.refStart = RTA.from(Uint32Array, transferables.refStart)
+			this.refCount = RTA.from(Uint16Array, transferables.refCount)
+			this.refs = RTA.from(IdArrayType, transferables.refs)
+			this.bbox = RTA.from(Float64Array, transferables.bbox)
+			// Only load spatial index if provided (not stored in IndexedDB)
+			if (transferables.spatialIndex?.byteLength) {
+				this.spatialIndex = Flatbush.from(transferables.spatialIndex)
+				this.spatialIndexBuilt = true
+			}
+			this.indexBuilt = true
+		} else {
+			super("way", new Ids(), new Tags(stringTable))
+			this.refStart = new RTA(Uint32Array)
+			this.refCount = new RTA(Uint16Array)
+			this.refs = new RTA(IdArrayType)
+			this.bbox = new RTA(Float64Array)
 		}
+		this.nodes = nodes
 	}
 
+	/**
+	 * Add a single way to the index.
+	 */
 	addWay(way: OsmWay) {
-		this.ids.add(way.id)
-		this.tags.addTags(way.tags)
+		const wayIndex = this.addEntity(way.id, way.tags ?? {})
 		this.refStart.push(this.refs.length)
 		this.refCount.push(way.refs.length)
 		for (const ref of way.refs) this.refs.push(ref)
+		return wayIndex
 	}
 
 	/**
@@ -80,7 +88,7 @@ export class Ways extends Entities<OsmWay> {
 	 */
 	addWays(
 		ways: OsmPbfWay[],
-		blockStringIndexMap: Map<number, number>,
+		blockStringIndexMap: Uint32Array,
 		filter?: (way: OsmWay) => OsmWay | null,
 	) {
 		let added = 0
@@ -99,67 +107,108 @@ export class Ways extends Entities<OsmWay> {
 				: null
 			if (filter && filteredWay === null) continue
 
-			this.ids.add(way.id)
-			this.refStart.push(this.refs.length)
-			this.refCount.push(filteredWay?.refs.length ?? refs.length)
-			this.refs.pushMany(filteredWay?.refs ?? refs)
-
 			const tagKeys: number[] = way.keys.map((key) => {
-				const index = blockStringIndexMap.get(key)
+				const index = blockStringIndexMap[key]
 				if (index === undefined) throw Error("Tag key not found")
 				return index
 			})
 			const tagValues: number[] = way.vals.map((val) => {
-				const index = blockStringIndexMap.get(val)
+				const index = blockStringIndexMap[val]
 				if (index === undefined) throw Error("Tag value not found")
 				return index
 			})
-			this.tags.addTagKeysAndValues(tagKeys, tagValues)
+
+			this.addEntity(way.id, tagKeys, tagValues)
+			this.refStart.push(this.refs.length)
+			this.refCount.push(filteredWay?.refs.length ?? refs.length)
+			this.refs.pushMany(filteredWay?.refs ?? refs)
 			added++
 		}
 		return added
 	}
 
+	/**
+	 * Compact the internal arrays to free up memory.
+	 */
 	buildEntityIndex() {
 		this.refStart.compact()
 		this.refCount.compact()
 		this.refs.compact()
 	}
 
-	buildSpatialIndex(nodeIndex: Nodes) {
+	/**
+	 * Build the spatial index for ways.
+	 * If bbox data already exists (e.g., loaded from storage), reuses it.
+	 */
+	buildSpatialIndex() {
+		if (!this.nodes.isReady()) throw Error("Node index is not ready.")
+		if (this.size === 0) return this.spatialIndex
 		console.time("WayIndex.buildSpatialIndex")
+
 		this.spatialIndex = new Flatbush(
 			this.size,
 			128,
 			Float64Array,
 			BufferConstructor,
 		)
+
+		// If bbox already has data (loaded from storage), use it directly
+		const hasBboxData = this.bbox.length >= this.size * 4
 		for (let i = 0; i < this.size; i++) {
-			let minX = Number.POSITIVE_INFINITY
-			let minY = Number.POSITIVE_INFINITY
-			let maxX = Number.NEGATIVE_INFINITY
-			let maxY = Number.NEGATIVE_INFINITY
-			const start = this.refStart.at(i)
-			const count = this.refCount.at(i)
-			for (let j = start; j < start + count; j++) {
-				const refId = this.refs.at(j)
-				const [lon, lat] = nodeIndex.getNodeLonLat({ id: refId })
-				if (lon < minX) minX = lon
-				if (lon > maxX) maxX = lon
-				if (lat < minY) minY = lat
-				if (lat > maxY) maxY = lat
+			let minX: number
+			let minY: number
+			let maxX: number
+			let maxY: number
+
+			if (hasBboxData) {
+				// Use stored bbox values
+				minX = this.bbox.at(i * 4)
+				minY = this.bbox.at(i * 4 + 1)
+				maxX = this.bbox.at(i * 4 + 2)
+				maxY = this.bbox.at(i * 4 + 3)
+			} else {
+				// Calculate bbox from node coordinates
+				minX = Number.POSITIVE_INFINITY
+				minY = Number.POSITIVE_INFINITY
+				maxX = Number.NEGATIVE_INFINITY
+				maxY = Number.NEGATIVE_INFINITY
+				const start = this.refStart.at(i)
+				const count = this.refCount.at(i)
+				for (let j = start; j < start + count; j++) {
+					const refId = this.refs.at(j)
+					const [lon, lat] = this.nodes.getNodeLonLat({ id: refId })
+					if (lon < minX) minX = lon
+					if (lon > maxX) maxX = lon
+					if (lat < minY) minY = lat
+					if (lat > maxY) maxY = lat
+				}
+				this.bbox.push(minX)
+				this.bbox.push(minY)
+				this.bbox.push(maxX)
+				this.bbox.push(maxY)
 			}
-			this.bbox.push(minX)
-			this.bbox.push(minY)
-			this.bbox.push(maxX)
-			this.bbox.push(maxY)
 			this.spatialIndex.add(minX, minY, maxX, maxY)
 		}
+		// Compact bbox if we just calculated it
+		if (!hasBboxData) {
+			this.bbox.compact()
+		}
 		this.spatialIndex.finish()
+		this.spatialIndexBuilt = true
 		console.timeEnd("WayIndex.buildSpatialIndex")
 		return this.spatialIndex
 	}
 
+	/**
+	 * Check if the spatial index has been built.
+	 */
+	hasSpatialIndex(): boolean {
+		return this.spatialIndexBuilt
+	}
+
+	/**
+	 * Get the full way entity.
+	 */
 	getFullEntity(index: number, id: number, tags?: OsmTags): OsmWay {
 		return {
 			id,
@@ -168,13 +217,19 @@ export class Ways extends Entities<OsmWay> {
 		}
 	}
 
-	getRefIds(index: number) {
+	/**
+	 * Get the node IDs referenced by a way.
+	 */
+	getRefIds(index: number): number[] {
 		const start = this.refStart.at(index)
 		const count = this.refCount.at(index)
-		return this.refs.slice(start, start + count)
+		return Array.from(this.refs.slice(start, start + count))
 	}
 
-	getBbox(idOrIndex: IdOrIndex): GeoBbox2D {
+	/**
+	 * Get the bounding box of a way.
+	 */
+	getEntityBbox(idOrIndex: IdOrIndex): GeoBbox2D {
 		const index =
 			"index" in idOrIndex ? idOrIndex.index : this.ids.idOrIndex(idOrIndex)[0]
 		return [
@@ -185,32 +240,38 @@ export class Ways extends Entities<OsmWay> {
 		]
 	}
 
-	getLine(index: number, nodeIndex: Nodes) {
+	/**
+	 * Get the coordinates of a way as a flat array.
+	 */
+	getLine(index: number) {
 		const count = this.refCount.at(index)
 		const start = this.refStart.at(index)
 		const line = new Float64Array(count * 2)
 		for (let i = 0; i < count; i++) {
 			const ref = this.refs.at(start + i)
-			const [lon, lat] = nodeIndex.getNodeLonLat({ id: ref })
+			const [lon, lat] = this.nodes.getNodeLonLat({ id: ref })
 			line[i * 2] = lon
 			line[i * 2 + 1] = lat
 		}
 		return line
 	}
 
-	getCoordinates(index: number, nodeIndex: Nodes): LonLat[] {
+	/**
+	 * Get the coordinates of a way as an array of [lon, lat] pairs.
+	 */
+	getCoordinates(index: number): LonLat[] {
 		const count = this.refCount.at(index)
 		const start = this.refStart.at(index)
 		const coords: [number, number][] = []
 		for (let refIndex = start; refIndex < start + count; refIndex++) {
 			const ref = this.refs.at(refIndex)
-			const coord = nodeIndex.getNodeLonLat({ id: ref })
+			const coord = this.nodes.getNodeLonLat({ id: ref })
 			if (
 				coord === undefined ||
 				coord[0] === undefined ||
 				coord[1] === undefined
 			) {
-				console.error("node index has ref", nodeIndex.ids.has(ref))
+				console.error("node index has ref", this.nodes.ids.has(ref))
 				throw Error(
 					`Invalid coordinate for way id ${this.ids.at(index)}, index ${index}, node ref ${ref}, ref index ${refIndex}`,
 				)
@@ -220,7 +281,11 @@ export class Ways extends Entities<OsmWay> {
 		return coords
 	}
 
+	/**
+	 * Find way indexes that intersect a bounding box.
+	 */
 	intersects(bbox: GeoBbox2D, filterFn?: (index: number) => boolean): number[] {
+		if (this.size === 0) return []
 		return this.spatialIndex.search(
 			bbox[0],
 			bbox[1],
@@ -230,12 +295,123 @@ export class Ways extends Entities<OsmWay> {
 		)
 	}
 
+	/**
+	 * Find way indexes near a point using great-circle distance.
+	 * @param lon - Longitude in degrees.
+	 * @param lat - Latitude in degrees.
+	 * @param maxResults - Maximum number of results to return.
+	 * @param maxDistanceKm - Maximum distance in kilometers.
+	 * @returns Array of way indexes sorted by distance.
+	 */
 	neighbors(
-		x: number,
-		y: number,
+		lon: number,
+		lat: number,
 		maxResults?: number,
-		maxDistance?: number,
+		maxDistanceKm?: number,
 	): number[] {
-		return this.spatialIndex.neighbors(x, y, maxResults, maxDistance)
+		if (this.size === 0) return []
+		// Use geoflatbush for proper geographic distance calculations
+		return geoAround(this.spatialIndex, lon, lat, maxResults, maxDistanceKm)
+	}
+
+	/**
+	 * Get ways within a bounding box.
+	 */
+	withinBbox(
+		bbox: GeoBbox2D,
+		include?: (index: number) => boolean,
+	): {
+		ids: Float64Array
+		positions: Float64Array
+		startIndices: Uint32Array
+	} {
+		console.time("Ways.withinBbox")
+		const wayCandidates = this.intersects(bbox, include)
+		const ids = new Float64Array(wayCandidates.length)
+		const wayPositions: Float64Array[] = []
+		const wayStartIndices = new Uint32Array(wayCandidates.length + 1)
+		wayStartIndices[0] = 0
+
+		console.time("Ways.withinBbox.loop")
+		let size = 0
+		wayCandidates.forEach((wayIndex, i) => {
+			ids[i] = this.ids.at(wayIndex)
+			const way = this.getLine(wayIndex)
+			size += way.length
+			wayPositions.push(way)
+			const prevIndex = wayStartIndices[i]
+			if (prevIndex === undefined) throw Error("Previous index is undefined")
+			wayStartIndices[i + 1] = prevIndex + way.length / 2
+		})
+		console.timeEnd("Ways.withinBbox.loop")
+		const wayPositionsArray = new Float64Array(size)
+		let pIndex = 0
+		for (const way of wayPositions) {
+			wayPositionsArray.set(way, pIndex)
+			pIndex += way.length
+		}
+
+		console.timeEnd("Ways.withinBbox")
+		return {
+			ids,
+			positions: wayPositionsArray,
+			startIndices: wayStartIndices,
+		}
+	}
+
+	/**
+	 * Get transferable objects for passing to another thread.
+	 * Only includes spatialIndex if it has been built.
+	 */
+	override transferables(): WaysTransferables {
+		const base = {
+			...super.transferables(),
+			refStart: this.refStart.array.buffer,
+			refCount: this.refCount.array.buffer,
+			refs: this.refs.array.buffer,
+			bbox: this.bbox.array.buffer,
+		}
+		// Only include spatial index if it was built
+		if (this.spatialIndexBuilt) {
+			return { ...base, spatialIndex: this.spatialIndex.data }
+		}
+		return base
+	}
+
+	/**
+	 * Get the approximate memory requirements for a given number of ways in bytes.
+	 */
+	static getBytesRequired(count: number) {
+		if (count === 0) return 0
+		// Approximate nodes per way
+		let numNodes = count
+		let n = count
+		while (n !== 1) {
+			n = Math.ceil(n / 128)
+			numNodes += n
+		}
+		const indexBytes = (numNodes < 16384 ? 2 : 4) * numNodes
+		const boxesBytes = numNodes * 4 * Float64Array.BYTES_PER_ELEMENT
+		const spatialIndexBytes = 8 + indexBytes + boxesBytes
+
+		return (
+			Ids.getBytesRequired(count) +
+			Tags.getBytesRequired(count) +
+			count * Uint32Array.BYTES_PER_ELEMENT + // refStart
+			count * Uint16Array.BYTES_PER_ELEMENT + // refCount
+			count * 4 * Float64Array.BYTES_PER_ELEMENT + // bbox
+			spatialIndexBytes
+		)
+	}
+
+	/**
+	 * Update a ContentHasher with way-specific data (node references).
+	 */
+	override updateHash(hasher: ContentHasher): ContentHasher {
+		return super
+			.updateHash(hasher)
+			.update(this.refStart.array)
+			.update(this.refCount.array)
+			.update(this.refs.array)
 	}
 }

package/test/memory.test.ts

@@ -0,0 +1,38 @@
+import { describe, expect, it } from "bun:test"
+import { Nodes, Relations, Ways } from "../src"
+
+export function getOsmCoreMemoryRequirements(
+	nodeCount: number,
+	wayCount: number,
+	relationCount: number,
+) {
+	const nodesBytes = Nodes.getBytesRequired(nodeCount)
+	const waysBytes = Ways.getBytesRequired(wayCount)
+	const relationsBytes = Relations.getBytesRequired(relationCount)
+
+	return {
+		nodes: nodesBytes,
+		ways: waysBytes,
+		relations: relationsBytes,
+		total: nodesBytes + waysBytes + relationsBytes,
+	}
+}
+
+describe("Memory Requirements", () => {
+	it("computes memory requirements for small datasets", () => {
+		const breakdown = getOsmCoreMemoryRequirements(10, 2, 1)
+		expect(breakdown.nodes).toBeGreaterThan(0)
+		expect(breakdown.ways).toBeGreaterThan(0)
+		expect(breakdown.relations).toBeGreaterThan(0)
+		expect(breakdown.total).toBe(
+			breakdown.nodes + breakdown.ways + breakdown.relations,
+		)
+	})
+
+	it("computes memory requirements for large datasets", () => {
+		const breakdown = getOsmCoreMemoryRequirements(70000, 20000, 0)
+		expect(breakdown.nodes).toBeGreaterThan(0)
+		expect(breakdown.ways).toBeGreaterThan(0)
+		expect(breakdown.total).toBe(breakdown.nodes + breakdown.ways)
+	})
+})

package/test/mock-osm.ts

@@ -1,5 +1,5 @@
-import type { OsmNode } from "@osmix/json"
-import { Osmix } from "../src/osmix"
+import type { OsmNode } from "@osmix/shared/types"
+import { Osm } from "../src/osm"
 
 const YAKIM_LAT = 46.60207
 const YAKIM_LON = -120.505898
@@ -19,12 +19,12 @@ const node1: OsmNode = {
 	lon: YAKIM_LON - ONE_KM_LON, // approximately 1km west
 }
 
-function addBaseNodes(osm: Osmix) {
+function addBaseNodes(osm: Osm) {
 	osm.nodes.addNode(node0)
 	osm.nodes.addNode(node1)
 }
 
-function addBaseWays(osm: Osmix) {
+function addBaseWays(osm: Osm) {
 	osm.ways.addWay({
 		id: 1,
 		refs: [0, 1],
@@ -37,8 +37,8 @@ function addBaseWays(osm: Osmix) {
 /**
  * Create a base OSM with one way and two nodes.
  */
-export function createBaseOsm(): Osmix {
-	const base = new Osmix()
+export function createBaseOsm(): Osm {
+	const base = new Osm()
 	addBaseNodes(base)
 	base.nodes.buildIndex()
 	addBaseWays(base)
@@ -52,8 +52,8 @@ export function createBaseOsm(): Osmix {
  * - A way that has an overlapping node with the first way, but does not share a node and so starts disconnected
  * - A way that crosses over the first way, but does not share a node and so starts disconnected
  */
-export function createPatchOsm(): Osmix {
-	const osm = new Osmix()
+export function createPatchOsm(): Osm {
+	const osm = new Osm()
 
 	addBaseNodes(osm)
 

package/test/tags-search.test.ts

@@ -0,0 +1,60 @@
+import { describe, expect, it } from "bun:test"
+import type { OsmNode, OsmWay } from "@osmix/shared/types"
+import { Nodes } from "../src/nodes"
+import StringTable from "../src/stringtable"
+import { Ways } from "../src/ways"
+
+describe("Tags.search on Nodes", () => {
+	it("finds by key and key+value", () => {
+		const st = new StringTable()
+		const nodes = new Nodes(st)
+		const node1: OsmNode = { id: 1, lon: -1, lat: 1, tags: { curb: "yes" } }
+		const node2: OsmNode = { id: 2, lon: -2, lat: 2, tags: { curb: "no" } }
+		const node3: OsmNode = { id: 3, lon: -3, lat: 3 }
+		nodes.addNode(node1)
+		nodes.addNode(node2)
+		nodes.addNode(node3)
+		nodes.buildIndex()
+
+		const allCurb = nodes.search("curb")
+		expect(allCurb).toEqual([node1, node2])
+		const curbYes = nodes.search("curb", "yes")
+		expect(curbYes).toEqual([node1])
+		const curbNo = nodes.search("curb", "no")
+		expect(curbNo).toEqual([node2])
+		const none = nodes.search("highway")
+		expect(none).toEqual([])
+	})
+})
+
+describe("Tags.search on Ways", () => {
+	it("finds by key and key+value", () => {
+		const st = new StringTable()
+		const nodes = new Nodes(st)
+		const ways = new Ways(st, nodes)
+		const way1: OsmWay = {
+			id: 10,
+			refs: [1, 2],
+			tags: { highway: "residential" },
+		}
+		const way2: OsmWay = { id: 11, refs: [2, 3], tags: { area: "yes" } }
+		const way3: OsmWay = {
+			id: 12,
+			refs: [3, 4],
+			tags: { highway: "primary", area: "no" },
+		}
+		ways.addWay(way1)
+		ways.addWay(way2)
+		ways.addWay(way3)
+		ways.buildIndex()
+
+		const allHighway = ways.search("highway")
+		expect(allHighway).toEqual([way1, way3])
+		const areaYes = ways.search("area", "yes")
+		expect(areaYes).toEqual([way2])
+		const areaAll = ways.search("area")
+		expect(areaAll).toEqual([way2, way3])
+		const areaNo = ways.search("area", "no")
+		expect(areaNo).toEqual([way3])
+	})
+})

package/test/typed-arrays.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, it } from "vitest"
+import { describe, expect, it } from "bun:test"
 import { BufferConstructor, ResizeableTypedArray } from "../src/typed-arrays"
 
 describe("typed array helpers", () => {
@@ -106,9 +106,8 @@ describe("typed array helpers", () => {
 			expect(Array.from(arr.slice(0, arr.length))).toEqual([1, 2, 3, 4, 5])
 		})
 
-		it.runIf(typeof SharedArrayBuffer !== "undefined")(
-			"grows shared buffers in place",
-			() => {
+		if (typeof SharedArrayBuffer !== "undefined") {
+			it("grows shared buffers in place", () => {
 				const backing = new SharedArrayBuffer(8, {
 					maxByteLength: 32,
 				})
@@ -129,7 +128,7 @@ describe("typed array helpers", () => {
 				expect(arr.array.length).toBe(16)
 				expect(arr.buffer.byteLength).toBe(16)
 				expect(Array.from(arr.slice(0, 9))).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9])
-			},
-		)
+			})
+		}
 	})
 })