@osmix/core 0.0.1

package/src/ways.ts

@@ -0,0 +1,241 @@
+import type { OsmTags, OsmWay } from "@osmix/json"
+import type { OsmPbfWay } from "@osmix/pbf"
+import type { GeoBbox2D, LonLat } from "@osmix/shared/types"
+import Flatbush from "flatbush"
+import { Entities, type EntitiesTransferables } from "./entities"
+import { type IdOrIndex, Ids } from "./ids"
+import type { Nodes } from "./nodes"
+import type StringTable from "./stringtable"
+import { Tags } from "./tags"
+import {
+	BufferConstructor,
+	type BufferType,
+	CoordinateArrayType,
+	IdArrayType,
+	ResizeableTypedArray,
+} from "./typed-arrays"
+
+export interface WaysTransferables extends EntitiesTransferables {
+	refStart: BufferType
+	refCount: BufferType
+	refs: BufferType
+	bbox: BufferType
+	spatialIndex: BufferType
+}
+
+export class Ways extends Entities<OsmWay> {
+	spatialIndex: Flatbush = new Flatbush(1)
+
+	refStart: ResizeableTypedArray<Uint32Array>
+	refCount: ResizeableTypedArray<Uint16Array> // Maximum 2,000 nodes per way
+
+	// Node IDs
+	refs: ResizeableTypedArray<Float64Array>
+
+	// Bounding box of the way in geographic coordinates
+	bbox: ResizeableTypedArray<Float64Array>
+
+	static from(stringTable: StringTable, wits: WaysTransferables) {
+		const idIndex = Ids.from(wits)
+		const tagIndex = Tags.from(stringTable, wits)
+		const wayIndex = new Ways(stringTable, idIndex, tagIndex)
+		wayIndex.refStart = ResizeableTypedArray.from(Uint32Array, wits.refStart)
+		wayIndex.refCount = ResizeableTypedArray.from(Uint16Array, wits.refCount)
+		wayIndex.refs = ResizeableTypedArray.from(IdArrayType, wits.refs)
+		wayIndex.bbox = ResizeableTypedArray.from(CoordinateArrayType, wits.bbox)
+		wayIndex.spatialIndex = Flatbush.from(wits.spatialIndex)
+		return wayIndex
+	}
+
+	constructor(stringTable: StringTable, idIndex?: Ids, tagIndex?: Tags) {
+		super("way", stringTable, idIndex, tagIndex)
+		this.refStart = new ResizeableTypedArray(Uint32Array)
+		this.refCount = new ResizeableTypedArray(Uint16Array)
+		this.refs = new ResizeableTypedArray(IdArrayType)
+		this.bbox = new ResizeableTypedArray(CoordinateArrayType)
+	}
+
+	transferables(): WaysTransferables {
+		return {
+			refStart: this.refStart.array.buffer,
+			refCount: this.refCount.array.buffer,
+			refs: this.refs.array.buffer,
+			bbox: this.bbox.array.buffer,
+			spatialIndex: this.spatialIndex.data,
+			...this.ids.transferables(),
+			...this.tags.transferables(),
+		}
+	}
+
+	addWay(way: OsmWay) {
+		this.ids.add(way.id)
+		this.tags.addTags(way.tags)
+		this.refStart.push(this.refs.length)
+		this.refCount.push(way.refs.length)
+		for (const ref of way.refs) this.refs.push(ref)
+	}
+
+	/**
+	 * Bulk add ways directly from a PBF PrimitiveBlock.
+	 */
+	addWays(
+		ways: OsmPbfWay[],
+		blockStringIndexMap: Map<number, number>,
+		filter?: (way: OsmWay) => OsmWay | null,
+	) {
+		let added = 0
+		for (const way of ways) {
+			let prevRefId = 0
+			const refs = way.refs.map((refId) => {
+				prevRefId += refId
+				return prevRefId
+			})
+			const filteredWay = filter
+				? filter({
+						id: way.id,
+						refs,
+						tags: this.tags.getTagsFromIndices(way.keys, way.vals),
+					})
+				: null
+			if (filter && filteredWay === null) continue
+
+			this.ids.add(way.id)
+			this.refStart.push(this.refs.length)
+			this.refCount.push(filteredWay?.refs.length ?? refs.length)
+			this.refs.pushMany(filteredWay?.refs ?? refs)
+
+			const tagKeys: number[] = way.keys.map((key) => {
+				const index = blockStringIndexMap.get(key)
+				if (index === undefined) throw Error("Tag key not found")
+				return index
+			})
+			const tagValues: number[] = way.vals.map((val) => {
+				const index = blockStringIndexMap.get(val)
+				if (index === undefined) throw Error("Tag value not found")
+				return index
+			})
+			this.tags.addTagKeysAndValues(tagKeys, tagValues)
+			added++
+		}
+		return added
+	}
+
+	buildEntityIndex() {
+		this.refStart.compact()
+		this.refCount.compact()
+		this.refs.compact()
+	}
+
+	buildSpatialIndex(nodeIndex: Nodes) {
+		console.time("WayIndex.buildSpatialIndex")
+		this.spatialIndex = new Flatbush(
+			this.size,
+			128,
+			Float64Array,
+			BufferConstructor,
+		)
+		for (let i = 0; i < this.size; i++) {
+			let minX = Number.POSITIVE_INFINITY
+			let minY = Number.POSITIVE_INFINITY
+			let maxX = Number.NEGATIVE_INFINITY
+			let maxY = Number.NEGATIVE_INFINITY
+			const start = this.refStart.at(i)
+			const count = this.refCount.at(i)
+			for (let j = start; j < start + count; j++) {
+				const refId = this.refs.at(j)
+				const [lon, lat] = nodeIndex.getNodeLonLat({ id: refId })
+				if (lon < minX) minX = lon
+				if (lon > maxX) maxX = lon
+				if (lat < minY) minY = lat
+				if (lat > maxY) maxY = lat
+			}
+			this.bbox.push(minX)
+			this.bbox.push(minY)
+			this.bbox.push(maxX)
+			this.bbox.push(maxY)
+			this.spatialIndex.add(minX, minY, maxX, maxY)
+		}
+		this.spatialIndex.finish()
+		console.timeEnd("WayIndex.buildSpatialIndex")
+		return this.spatialIndex
+	}
+
+	getFullEntity(index: number, id: number, tags?: OsmTags): OsmWay {
+		return {
+			id,
+			refs: [...this.getRefIds(index)],
+			tags,
+		}
+	}
+
+	getRefIds(index: number) {
+		const start = this.refStart.at(index)
+		const count = this.refCount.at(index)
+		return this.refs.slice(start, start + count)
+	}
+
+	getBbox(idOrIndex: IdOrIndex): GeoBbox2D {
+		const index =
+			"index" in idOrIndex ? idOrIndex.index : this.ids.idOrIndex(idOrIndex)[0]
+		return [
+			this.bbox.at(index * 4),
+			this.bbox.at(index * 4 + 1),
+			this.bbox.at(index * 4 + 2),
+			this.bbox.at(index * 4 + 3),
+		]
+	}
+
+	getLine(index: number, nodeIndex: Nodes) {
+		const count = this.refCount.at(index)
+		const start = this.refStart.at(index)
+		const line = new Float64Array(count * 2)
+		for (let i = 0; i < count; i++) {
+			const ref = this.refs.at(start + i)
+			const [lon, lat] = nodeIndex.getNodeLonLat({ id: ref })
+			line[i * 2] = lon
+			line[i * 2 + 1] = lat
+		}
+		return line
+	}
+
+	getCoordinates(index: number, nodeIndex: Nodes): LonLat[] {
+		const count = this.refCount.at(index)
+		const start = this.refStart.at(index)
+		const coords: [number, number][] = []
+		for (let refIndex = start; refIndex < start + count; refIndex++) {
+			const ref = this.refs.at(refIndex)
+			const coord = nodeIndex.getNodeLonLat({ id: ref })
+			if (
+				coord === undefined ||
+				coord[0] === undefined ||
+				coord[1] === undefined
+			) {
+				console.error("node index has ref", nodeIndex.ids.has(ref))
+				throw Error(
+					`Invalid coordinate for way id ${this.ids.at(index)}, index ${index}, node ref ${ref}, ref index ${refIndex}`,
+				)
+			}
+			coords.push(coord)
+		}
+		return coords
+	}
+
+	intersects(bbox: GeoBbox2D, filterFn?: (index: number) => boolean): number[] {
+		return this.spatialIndex.search(
+			bbox[0],
+			bbox[1],
+			bbox[2],
+			bbox[3],
+			filterFn,
+		)
+	}
+
+	neighbors(
+		x: number,
+		y: number,
+		maxResults?: number,
+		maxDistance?: number,
+	): number[] {
+		return this.spatialIndex.neighbors(x, y, maxResults, maxDistance)
+	}
+}

package/test/extract.bench.ts

@@ -0,0 +1,29 @@
+import { getFixtureFile } from "@osmix/shared/test/fixtures"
+import type { GeoBbox2D } from "@osmix/shared/types"
+import { beforeAll, bench, describe } from "vitest"
+import { Osmix } from "../src/osmix"
+
+const MONACO_BBOX: GeoBbox2D = [7.4053929, 43.7232244, 7.4447259, 43.7543687]
+// const SEATTLE_BBOX: GeoBbox2D = [-122.33, 47.48, -122.29, 47.52]
+
+const BBOX = MONACO_BBOX
+const PBF = "monaco.pbf"
+
+let buffer: ArrayBufferLike
+
+beforeAll(async () => {
+	buffer = await getFixtureFile(PBF)
+})
+
+describe("simple extract benchmark", () => {
+	bench("two-step parse then extract", async () => {
+		const data = buffer.slice(0)
+		const full = await Osmix.fromPbf(data)
+		full.extract(BBOX)
+	})
+
+	bench("streaming extract during parse", async () => {
+		const data = buffer.slice(0)
+		await Osmix.fromPbf(data, { extractBbox: BBOX })
+	})
+})

package/test/extract.test.ts

@@ -0,0 +1,156 @@
+import { getFixtureFileReadStream } from "@osmix/shared/test/fixtures"
+import type { GeoBbox2D } from "@osmix/shared/types"
+import { assert, test } from "vitest"
+import { Osmix } from "../src/osmix"
+
+const TEST_BBOX: GeoBbox2D = [-0.1, -0.1, 1, 1]
+const SEATTLE_BBOX: GeoBbox2D = [-122.463226, 47.469878, -122.180328, 47.82883]
+
+function buildSourceOsm() {
+	const osm = new Osmix({ id: "source" })
+	osm.nodes.addNode({
+		id: 1,
+		lat: 0,
+		lon: 0,
+		tags: {
+			name: "inside-a",
+		},
+	})
+	osm.nodes.addNode({
+		id: 2,
+		lat: 0,
+		lon: 2,
+		tags: {
+			name: "outside-a",
+		},
+	})
+	osm.nodes.addNode({
+		id: 3,
+		lat: 0.5,
+		lon: 0.5,
+	})
+	osm.nodes.addNode({
+		id: 4,
+		lat: 0.5,
+		lon: 1.5,
+	})
+
+	osm.ways.addWay({
+		id: 10,
+		refs: [1, 2],
+		tags: {
+			highway: "residential",
+		},
+	})
+	osm.ways.addWay({
+		id: 11,
+		refs: [3, 4],
+		tags: {
+			highway: "service",
+		},
+	})
+
+	osm.relations.addRelation({
+		id: 20,
+		members: [
+			{
+				type: "way",
+				ref: 10,
+				role: "outer",
+			},
+		],
+		tags: {
+			type: "multipolygon",
+		},
+	})
+
+	osm.buildIndexes()
+	return osm
+}
+
+test("extract a BBOX while reading a PBF", async () => {
+	const source = buildSourceOsm()
+	const transform = new TransformStream<Uint8Array, ArrayBufferLike>()
+	const extract = new Osmix()
+	const extractPromise = extract.readPbf(transform.readable, {
+		extractBbox: TEST_BBOX,
+	})
+	await source.toPbfStream().pipeTo(transform.writable)
+	await extractPromise
+
+	assert.equal(extract.nodes.size, 2)
+	assert.isTrue(extract.nodes.ids.has(1))
+	assert.isTrue(extract.nodes.ids.has(3))
+	assert.isNotTrue(extract.nodes.ids.has(2))
+	assert.isNotTrue(extract.nodes.ids.has(4))
+
+	const truncatedWay = extract.ways.getById(10)
+	assert.exists(truncatedWay)
+	assert.deepEqual(truncatedWay?.refs, [1])
+
+	assert.isTrue(extract.ways.ids.has(11))
+	assert.isTrue(extract.relations.ids.has(20))
+})
+
+test("extract a BBOX after reading a PBF", async () => {
+	const source = buildSourceOsm()
+	const buffer = await source.toPbfBuffer()
+
+	const streaming = new Osmix()
+	await streaming.readPbf(buffer.slice(0), { extractBbox: TEST_BBOX })
+
+	const twoStepOsmix = new Osmix()
+	await twoStepOsmix.readPbf(buffer.slice(0))
+	const twoStep = twoStepOsmix.extract(TEST_BBOX)
+
+	assert.equal(streaming.nodes.size, twoStep.nodes.size)
+	assert.equal(streaming.ways.size, twoStep.ways.size)
+	assert.equal(streaming.relations.size, twoStep.relations.size)
+
+	assert.deepEqual(
+		Array.from(streaming.nodes.ids.sorted),
+		Array.from(twoStep.nodes.ids.sorted),
+	)
+	assert.deepEqual(
+		Array.from(streaming.ways.ids.sorted),
+		Array.from(twoStep.ways.ids.sorted),
+	)
+	assert.deepEqual(
+		Array.from(streaming.relations.ids.sorted),
+		Array.from(twoStep.relations.ids.sorted),
+	)
+
+	for (const nodeId of streaming.nodes.ids.sorted) {
+		const streamingNode = streaming.nodes.getById(nodeId)
+		const twoStepNode = twoStep.nodes.getById(nodeId)
+		assert.deepEqual(streamingNode, twoStepNode)
+	}
+
+	for (const wayId of streaming.ways.ids.sorted) {
+		const streamingWay = streaming.ways.getById(wayId)
+		const twoStepWay = twoStep.ways.getById(wayId)
+		assert.deepEqual(streamingWay, twoStepWay)
+	}
+
+	for (const relationId of streaming.relations.ids.sorted) {
+		const streamingRelation = streaming.relations.getById(relationId)
+		const twoStepRelation = twoStep.relations.getById(relationId)
+		assert.deepEqual(streamingRelation, twoStepRelation)
+	}
+})
+
+test.skip("extract from a large PBF", { timeout: 500_000 }, async () => {
+	const seattle = new Osmix({ logger: console.error })
+	await seattle.readPbf(getFixtureFileReadStream("usa.pbf"), {
+		extractBbox: SEATTLE_BBOX,
+	})
+
+	assert.deepEqual(
+		{
+			nodes: seattle.nodes.size,
+			ways: seattle.ways.size,
+			relations: seattle.relations.size,
+		},
+		{ nodes: 802_541, ways: 241_248, relations: 3_622 },
+	)
+})

package/test/mock-osm.ts

@@ -0,0 +1,140 @@
+import type { OsmNode } from "@osmix/json"
+import { Osmix } from "../src/osmix"
+
+const YAKIM_LAT = 46.60207
+const YAKIM_LON = -120.505898
+
+const ONE_KM_LON = 0.0131 // approximately
+const ONE_KM_LAT = 0.009
+
+const node0: OsmNode = {
+	id: 0,
+	lat: YAKIM_LAT,
+	lon: YAKIM_LON,
+}
+
+const node1: OsmNode = {
+	id: 1,
+	lat: YAKIM_LAT,
+	lon: YAKIM_LON - ONE_KM_LON, // approximately 1km west
+}
+
+function addBaseNodes(osm: Osmix) {
+	osm.nodes.addNode(node0)
+	osm.nodes.addNode(node1)
+}
+
+function addBaseWays(osm: Osmix) {
+	osm.ways.addWay({
+		id: 1,
+		refs: [0, 1],
+		tags: {
+			highway: "primary",
+		},
+	})
+}
+
+/**
+ * Create a base OSM with one way and two nodes.
+ */
+export function createBaseOsm(): Osmix {
+	const base = new Osmix()
+	addBaseNodes(base)
+	base.nodes.buildIndex()
+	addBaseWays(base)
+	base.buildIndexes()
+	return base
+}
+
+/**
+ * Create a patch OSM with:
+ * - The same way as the base OSM, with new tags
+ * - A way that has an overlapping node with the first way, but does not share a node and so starts disconnected
+ * - A way that crosses over the first way, but does not share a node and so starts disconnected
+ */
+export function createPatchOsm(): Osmix {
+	const osm = new Osmix()
+
+	addBaseNodes(osm)
+
+	// Add way the connects with base way node 0 (will replace it)
+	osm.nodes.addNode({
+		...node0,
+		id: 2,
+		tags: {
+			crossing: "yes",
+		},
+	})
+	osm.nodes.addNode({
+		...node1,
+		id: 3,
+		lon: YAKIM_LON + ONE_KM_LON, // ends 1km east of the center point
+	})
+
+	// Add nodes for disconnected way
+	osm.nodes.addNode({
+		id: 4,
+		lat: YAKIM_LAT - ONE_KM_LAT, // 1km south
+		lon: YAKIM_LON + ONE_KM_LON / 4, // 250m east
+	})
+	osm.nodes.addNode({
+		id: 5,
+		lat: YAKIM_LAT + ONE_KM_LAT, // 1km north
+		lon: YAKIM_LON + ONE_KM_LON / 4, // 250m east
+	})
+
+	// Add nodes for crossing way
+	osm.nodes.addNode({
+		id: 6,
+		lat: YAKIM_LAT - ONE_KM_LAT, // 1km south
+		lon: YAKIM_LON + ONE_KM_LON / 2, // 500m east
+	})
+	osm.nodes.addNode({
+		id: 7,
+		lat: YAKIM_LAT + ONE_KM_LAT, // 1km north
+		lon: YAKIM_LON + ONE_KM_LON / 2, // 500m east
+	})
+
+	osm.nodes.buildIndex()
+
+	// Add same base way with new tags
+	osm.ways.addWay({
+		id: 1,
+		refs: [0, 1],
+		tags: {
+			highway: "primary",
+			version: "2",
+		},
+	})
+
+	// Add way that overlaps with base way node 1. Node should be de-duplicated.
+	osm.ways.addWay({
+		id: 2,
+		refs: [2, 3],
+		tags: {
+			highway: "secondary",
+		},
+	})
+
+	// Add way that crosses way 2, and should generate 1 intersection.
+	osm.ways.addWay({
+		id: 3,
+		refs: [4, 5],
+		tags: {
+			highway: "primary",
+		},
+	})
+
+	// Add a way that crosses way 2, but has a tag indicating it is an underpass and should be left alone
+	osm.ways.addWay({
+		id: 4,
+		refs: [6, 7],
+		tags: {
+			highway: "underpass",
+			tunnel: "yes",
+		},
+	})
+
+	osm.buildIndexes()
+	return osm
+}

package/test/read.test.ts

@@ -0,0 +1,22 @@
+import {
+	getFixtureFile,
+	getFixtureFileReadStream,
+	PBFs,
+} from "@osmix/shared/test/fixtures"
+import { assert, beforeAll, describe, it } from "vitest"
+import { Osmix } from "../src/osmix"
+
+describe("read", () => {
+	describe.each(Object.entries(PBFs))("%s", async (name, pbf) => {
+		beforeAll(() => getFixtureFile(pbf.url))
+
+		it("into OSM class", async () => {
+			const fileStream = getFixtureFileReadStream(pbf.url)
+			const osm = await Osmix.fromPbf(fileStream, { id: name })
+			assert.equal(osm.nodes.size, pbf.nodes)
+			assert.equal(osm.stringTable.length, pbf.uniqueStrings)
+			assert.deepEqual(osm.nodes.getByIndex(0), pbf.node0)
+			assert.equal(osm.ways.size, pbf.ways)
+		})
+	})
+})