@osmix/core 0.0.1

package/src/relations.ts

@@ -0,0 +1,221 @@
+import type {
+	OsmEntityType,
+	OsmRelation,
+	OsmRelationMember,
+	OsmTags,
+} from "@osmix/json"
+import type { OsmPbfRelation } from "@osmix/pbf"
+import { assertValue } from "@osmix/shared/assert"
+import { Entities, type EntitiesTransferables } from "./entities"
+import { Ids } from "./ids"
+import type StringTable from "./stringtable"
+import { Tags } from "./tags"
+import {
+	type BufferType,
+	IdArrayType,
+	ResizeableTypedArray,
+} from "./typed-arrays"
+
+const RELATION_MEMBER_TYPES: OsmEntityType[] = ["node", "way", "relation"]
+
+export interface RelationsTransferables extends EntitiesTransferables {
+	memberStart: BufferType
+	memberCount: BufferType
+	memberRefs: BufferType
+	memberTypes: BufferType
+	memberRoles: BufferType
+}
+
+export class Relations extends Entities<OsmRelation> {
+	memberStart: ResizeableTypedArray<Uint32Array>
+	memberCount: ResizeableTypedArray<Uint16Array> // Maximum 65,535 members per relation
+
+	// Store the ID of the member because relations have other relations as members.
+	memberRefs: ResizeableTypedArray<Float64Array>
+	memberTypes: ResizeableTypedArray<Uint8Array>
+	memberRoles: ResizeableTypedArray<Uint32Array>
+
+	static from(stringTable: StringTable, rit: RelationsTransferables) {
+		const idIndex = Ids.from(rit)
+		const tagIndex = Tags.from(stringTable, rit)
+		const ri = new Relations(stringTable, idIndex, tagIndex)
+		ri.memberStart = ResizeableTypedArray.from(Uint32Array, rit.memberStart)
+		ri.memberCount = ResizeableTypedArray.from(Uint16Array, rit.memberCount)
+		ri.memberRefs = ResizeableTypedArray.from(IdArrayType, rit.memberRefs)
+		ri.memberTypes = ResizeableTypedArray.from(Uint8Array, rit.memberTypes)
+		ri.memberRoles = ResizeableTypedArray.from(Uint32Array, rit.memberRoles)
+		return ri
+	}
+
+	constructor(stringTable: StringTable, idIndex?: Ids, tagIndex?: Tags) {
+		super("relation", stringTable, idIndex, tagIndex)
+		this.memberStart = new ResizeableTypedArray(Uint32Array)
+		this.memberCount = new ResizeableTypedArray(Uint16Array)
+		this.memberRefs = new ResizeableTypedArray(IdArrayType)
+		this.memberTypes = new ResizeableTypedArray(Uint8Array)
+		this.memberRoles = new ResizeableTypedArray(Uint32Array)
+	}
+
+	transferables(): RelationsTransferables {
+		return {
+			memberStart: this.memberStart.array.buffer,
+			memberCount: this.memberCount.array.buffer,
+			memberRefs: this.memberRefs.array.buffer,
+			memberTypes: this.memberTypes.array.buffer,
+			memberRoles: this.memberRoles.array.buffer,
+			...this.ids.transferables(),
+			...this.tags.transferables(),
+		}
+	}
+
+	addRelation(relation: OsmRelation) {
+		this.ids.add(relation.id)
+		this.tags.addTags(relation.tags)
+		this.memberStart.push(this.memberRefs.length)
+		this.memberCount.push(relation.members.length)
+		for (const member of relation.members) {
+			this.memberRefs.push(member.ref)
+			this.memberTypes.push(RELATION_MEMBER_TYPES.indexOf(member.type))
+			this.memberRoles.push(this.stringTable.add(member.role ?? ""))
+		}
+	}
+
+	addRelations(
+		relations: OsmPbfRelation[],
+		blockStringIndexMap: Map<number, number>,
+		filter?: (relation: OsmRelation) => OsmRelation | null,
+	): number {
+		const blockToStringTable = (k: number) => {
+			const index = blockStringIndexMap.get(k)
+			if (index === undefined) throw Error("Tag key not found")
+			return index
+		}
+
+		let added = 0
+		for (const relation of relations) {
+			const members: OsmRelationMember[] = []
+			const memberRefs: number[] = []
+			const memberTypes: number[] = []
+			const memberRoles: number[] = []
+
+			let refId = 0
+			for (let i = 0; i < relation.memids.length; i++) {
+				const memid = relation.memids[i]
+				const roleSid = relation.roles_sid[i]
+				const typeIndex = relation.types[i]
+				assertValue(memid, "Relation member ID is undefined")
+				assertValue(roleSid, "Relation member role SID is undefined")
+				assertValue(typeIndex, "Relation member type is undefined")
+
+				refId += memid
+				const roleIndex = blockToStringTable(roleSid)
+				const type = RELATION_MEMBER_TYPES[typeIndex]
+				assertValue(type, "Relation member type not found")
+
+				if (filter) {
+					members.push({
+						type,
+						ref: refId,
+						role: this.stringTable.get(roleIndex),
+					})
+				}
+
+				memberRefs.push(refId)
+				memberTypes.push(typeIndex)
+				memberRoles.push(roleIndex)
+			}
+
+			const tagKeys: number[] = relation.keys.map(blockToStringTable)
+			const tagValues: number[] = relation.vals.map(blockToStringTable)
+
+			const filteredRelation = filter
+				? filter({
+						id: relation.id,
+						members,
+						tags: this.tags.getTagsFromIndices(tagKeys, tagValues),
+					})
+				: null
+			if (filter && filteredRelation === null) continue
+			added++
+
+			this.ids.add(relation.id)
+			this.memberStart.push(this.memberRefs.length)
+			this.memberCount.push(
+				filteredRelation?.members.length ?? memberRefs.length,
+			)
+			this.memberRefs.pushMany(
+				filteredRelation?.members.map((m) => m.ref) ?? memberRefs,
+			)
+			this.memberTypes.pushMany(
+				filteredRelation?.members.map((m) =>
+					RELATION_MEMBER_TYPES.indexOf(m.type),
+				) ?? memberTypes,
+			)
+			this.memberRoles.pushMany(
+				filteredRelation?.members.map((m) =>
+					this.stringTable.add(m.role ?? ""),
+				) ?? memberRoles,
+			)
+
+			this.tags.addTagKeysAndValues(tagKeys, tagValues)
+		}
+		return added
+	}
+
+	buildEntityIndex() {
+		this.memberStart.compact()
+		this.memberCount.compact()
+		this.memberRefs.compact()
+		this.memberTypes.compact()
+		this.memberRoles.compact()
+	}
+
+	getFullEntity(index: number, id: number, tags?: OsmTags): OsmRelation {
+		return {
+			id,
+			members: this.getMembersByIndex(index),
+			tags,
+		}
+	}
+
+	getMembersByIndex(
+		index: number,
+		relationMemberTypes = RELATION_MEMBER_TYPES,
+	) {
+		const start = this.memberStart.at(index)
+		const count = this.memberCount.at(index)
+		const members: OsmRelationMember[] = []
+		for (let i = start; i < start + count; i++) {
+			const ref = this.memberRefs.at(i)
+			const type = RELATION_MEMBER_TYPES[this.memberTypes.at(i)]
+			if (type === undefined) throw Error(`Member type not found: ${i}`)
+			if (!relationMemberTypes.includes(type)) continue
+			const role = this.stringTable.get(this.memberRoles.at(i))
+			members.push({ ref, type, role })
+		}
+		return members
+	}
+
+	includesMember(
+		index: number,
+		memberRef: number,
+		memberType: OsmEntityType,
+		memberRole?: string,
+	) {
+		const start = this.memberStart.at(index)
+		const count = this.memberCount.at(index)
+		for (let i = start; i < start + count; i++) {
+			const type = RELATION_MEMBER_TYPES[this.memberTypes.at(i)]
+			if (type !== memberType) continue
+			const ref = this.memberRefs.at(i)
+			if (ref !== memberRef) continue
+			if (
+				memberRole !== undefined &&
+				this.stringTable.get(this.memberRoles.at(i)) !== memberRole
+			)
+				continue
+			return true
+		}
+		return false
+	}
+}

package/src/stringtable.ts

@@ -0,0 +1,120 @@
+import type { OsmPbfBlock, OsmPbfStringTable } from "@osmix/pbf"
+import { type BufferType, ResizeableTypedArray } from "./typed-arrays"
+
+export interface StringTableTransferables {
+	bytes: BufferType
+	start: BufferType
+	count: BufferType
+}
+
+/**
+ * StringTable that can be used to store and retrieve strings as bytes.
+ */
+export default class StringTable {
+	private enc = new TextEncoder()
+	private dec = new TextDecoder() // UTF-8
+
+	// Serializable state
+	private bytes: ResizeableTypedArray<Uint8Array>
+	private start: ResizeableTypedArray<Uint32Array>
+	private count: ResizeableTypedArray<Uint16Array>
+
+	// Builder state
+	private stringToIndex = new Map<string, number>()
+
+	// Retrieval state
+	private indexToString = new Map<number, string>()
+
+	static from({ bytes, start, count }: StringTableTransferables): StringTable {
+		const builder = new StringTable()
+		builder.bytes = ResizeableTypedArray.from(Uint8Array, bytes)
+		builder.start = ResizeableTypedArray.from(Uint32Array, start)
+		builder.count = ResizeableTypedArray.from(Uint16Array, count)
+		return builder
+	}
+
+	constructor() {
+		this.bytes = new ResizeableTypedArray(Uint8Array)
+		this.start = new ResizeableTypedArray(Uint32Array)
+		this.count = new ResizeableTypedArray(Uint16Array)
+	}
+
+	transferables(): StringTableTransferables {
+		return {
+			bytes: this.bytes.array.buffer,
+			start: this.start.array.buffer,
+			count: this.count.array.buffer,
+		}
+	}
+
+	add(str: string): number {
+		const existingIndex = this.stringToIndex.get(str)
+		if (existingIndex !== undefined) return existingIndex
+		const startIndex = this.start.length
+		const encoded = this.enc.encode(str)
+		this.start.push(this.bytes.length)
+		this.bytes.pushMany(encoded)
+		this.count.push(encoded.length)
+		this.stringToIndex.set(str, startIndex)
+		return startIndex
+	}
+
+	/**
+	 * Decode all the strings in a primitive block and add them to the string table.
+	 * Return a mapping of block index -> string table index
+	 */
+	createBlockIndexMap(block: OsmPbfBlock) {
+		const map = new Map<number, number>()
+		for (let i = 0; i < block.stringtable.length; i++) {
+			const bytesString = block.stringtable[i]
+			const str = this.dec.decode(bytesString)
+			const index = this.add(str)
+			map.set(i, index)
+		}
+		return map
+	}
+
+	/**
+	 * TextDecoder.decode() does not support SharedArrayBuffer, so we need to first copy to a normal ArrayBuffer before decoding.
+	 */
+	get(index: number): string {
+		const string = this.indexToString.get(index)
+		if (string) return string
+		const bytes = this.getBytes(index)
+		const tempBuffer = new ArrayBuffer(bytes.byteLength)
+		const tempBytes = new Uint8Array(tempBuffer)
+		tempBytes.set(bytes)
+		const decoded = this.dec.decode(tempBytes)
+		this.indexToString.set(index, decoded)
+		return decoded
+	}
+
+	getBytes(index: number): Uint8Array {
+		if (index < 0 || index >= this.length)
+			throw Error(`String index out of range: ${index}`)
+		const start = this.start.at(index)
+		const count = this.count.at(index)
+		// Important: subarray creates a *view*, not a copy; TextDecoder cannot read from it directly.
+		return this.bytes.array.subarray(start, start + count)
+	}
+
+	get length() {
+		return this.start.length
+	}
+
+	buildIndex() {
+		this.bytes.compact()
+		this.start.compact()
+		this.count.compact()
+
+		this.stringToIndex.clear()
+	}
+
+	toOsmPbfStringTable(): OsmPbfStringTable {
+		const stringTable: OsmPbfStringTable = []
+		for (let i = 0; i < this.length; i++) {
+			stringTable.push(this.getBytes(i))
+		}
+		return stringTable
+	}
+}

package/src/tags.ts

@@ -0,0 +1,122 @@
+import type { OsmTags } from "@osmix/json"
+import type StringTable from "./stringtable"
+import { type BufferType, ResizeableTypedArray } from "./typed-arrays"
+
+export interface TagsTransferables {
+	tagStart: BufferType
+	tagCount: BufferType
+	tagKeys: BufferType
+	tagVals: BufferType
+}
+
+export class Tags {
+	private stringTable: StringTable
+	private tagStart: ResizeableTypedArray<Uint32Array>
+	private tagCount: ResizeableTypedArray<Uint8Array>
+	private tagKeys: ResizeableTypedArray<Uint32Array>
+	private tagVals: ResizeableTypedArray<Uint32Array>
+
+	private indexBuilt = false
+
+	static from(
+		stringTable: StringTable,
+		{ tagStart, tagCount, tagKeys, tagVals }: TagsTransferables,
+	) {
+		const tagIndex = new Tags(stringTable)
+		tagIndex.tagStart = ResizeableTypedArray.from(Uint32Array, tagStart)
+		tagIndex.tagCount = ResizeableTypedArray.from(Uint8Array, tagCount)
+		tagIndex.tagKeys = ResizeableTypedArray.from(Uint32Array, tagKeys)
+		tagIndex.tagVals = ResizeableTypedArray.from(Uint32Array, tagVals)
+		tagIndex.indexBuilt = true
+		return tagIndex
+	}
+
+	constructor(stringTable: StringTable) {
+		this.stringTable = stringTable
+		this.tagStart = new ResizeableTypedArray(Uint32Array)
+		this.tagCount = new ResizeableTypedArray(Uint8Array)
+		this.tagKeys = new ResizeableTypedArray(Uint32Array)
+		this.tagVals = new ResizeableTypedArray(Uint32Array)
+	}
+
+	transferables() {
+		return {
+			tagStart: this.tagStart.array.buffer,
+			tagCount: this.tagCount.array.buffer,
+			tagKeys: this.tagKeys.array.buffer,
+			tagVals: this.tagVals.array.buffer,
+		}
+	}
+
+	addTags(tags?: OsmTags) {
+		const tagKeys: number[] = []
+		const tagValues: number[] = []
+
+		if (tags) {
+			for (const [key, value] of Object.entries(tags)) {
+				tagKeys.push(this.stringTable.add(key))
+				tagValues.push(this.stringTable.add(String(value)))
+			}
+		}
+
+		this.addTagKeysAndValues(tagKeys, tagValues)
+	}
+
+	addTagKeysAndValues(keys: number[], values: number[]) {
+		this.tagStart.push(this.tagKeys.length)
+		this.tagCount.push(keys.length)
+		this.tagKeys.pushMany(keys)
+		this.tagVals.pushMany(values)
+	}
+
+	buildIndex() {
+		this.tagStart.compact()
+		this.tagCount.compact()
+		this.tagKeys.compact()
+		this.tagVals.compact()
+		this.indexBuilt = true
+	}
+
+	get isReady() {
+		return this.indexBuilt
+	}
+
+	cardinality(index: number): number {
+		return this.tagCount.at(index) ?? 0
+	}
+
+	getTags(index: number): OsmTags | undefined {
+		const tagCount = this.tagCount.at(index) ?? 0
+		if (tagCount === 0) return
+		const tagStart = this.tagStart.at(index) ?? 0
+		const tagKeyIndexes = this.tagKeys.array.slice(
+			tagStart,
+			tagStart + tagCount,
+		)
+		const tagValIndexes = this.tagVals.array.slice(
+			tagStart,
+			tagStart + tagCount,
+		)
+		const tags: OsmTags = {}
+		for (let i = 0; i < tagCount; i++) {
+			const keyIndex = tagKeyIndexes[i]
+			const valIndex = tagValIndexes[i]
+			if (keyIndex === undefined || valIndex === undefined)
+				throw Error("Tag key or value not found")
+			tags[this.stringTable.get(keyIndex)] = this.stringTable.get(valIndex)
+		}
+		return tags
+	}
+
+	getTagsFromIndices(keys: number[], values: number[]): OsmTags {
+		const tags: OsmTags = {}
+		for (let i = 0; i < keys.length; i++) {
+			const keyIndex = keys[i]
+			const valIndex = values[i]
+			if (keyIndex === undefined || valIndex === undefined)
+				throw Error("Tag key or value not found")
+			tags[this.stringTable.get(keyIndex)] = this.stringTable.get(valIndex)
+		}
+		return tags
+	}
+}

package/src/typed-arrays.ts

@@ -0,0 +1,164 @@
+/**
+ * Use SharedArrayBuffer if the runtime supports it, otherwise fall back to ArrayBuffer.
+ */
+export const BufferConstructor =
+	typeof SharedArrayBuffer !== "undefined"
+		? (SharedArrayBuffer as SharedArrayBufferConstructor)
+		: (ArrayBuffer as ArrayBufferConstructor)
+export type BufferType = InstanceType<typeof BufferConstructor>
+
+export type TypedArray<B extends BufferType = BufferType> =
+	| Int8Array<B>
+	| Uint8Array<B>
+	| Uint8ClampedArray<B>
+	| Int16Array<B>
+	| Uint16Array<B>
+	| Int32Array<B>
+	| Uint32Array<B>
+	| Float32Array<B>
+	| Float64Array<B>
+
+export interface TypedArrayConstructor<
+	T extends TypedArray<BufferType> = TypedArray<BufferType>,
+> {
+	new (buffer: BufferType): T
+	readonly BYTES_PER_ELEMENT: number
+}
+
+/**
+ * OSM IDs can be stored as 64-bit floating point numbers.
+ */
+export const IdArrayType = Float64Array
+
+/**
+ * When we are storing coordinates, we need to be able to store 64-bit floating point numbers.
+ * However, for benchmarking it is handy to test 32-bit floating point numbers.
+ */
+export const CoordinateArrayType = Float64Array
+
+/**
+ * When we are storing indexes into other arrays, we never need an index to exceed 2^32.
+ */
+export const IndexArrayType = Uint32Array
+
+/**
+ * Default buffer size.
+ */
+export const DEFAULT_BUFFER_SIZE = 2 ** 20 // 1 MiB
+
+export class ResizeableTypedArray<TA extends TypedArray> {
+	ArrayType: TypedArrayConstructor<TA>
+	array: TA
+	items = 0
+
+	buffer: BufferType
+	bufferSize: number
+	maxByteLength: number
+
+	BC: SharedArrayBufferConstructor | ArrayBufferConstructor
+
+	static from<TA extends TypedArray>(
+		ArrayType: TypedArrayConstructor<TA>,
+		buffer: BufferType,
+	) {
+		const rta = new ResizeableTypedArray<TA>(
+			ArrayType,
+			buffer instanceof SharedArrayBuffer ? SharedArrayBuffer : ArrayBuffer,
+		)
+		rta.buffer = buffer
+		rta.array = new ArrayType(buffer)
+		rta.items = rta.array.length
+		return rta
+	}
+
+	constructor(
+		ArrayType: TypedArrayConstructor<TA>,
+		BC:
+			| SharedArrayBufferConstructor
+			| ArrayBufferConstructor = BufferConstructor,
+	) {
+		this.ArrayType = ArrayType
+		this.bufferSize = DEFAULT_BUFFER_SIZE
+		this.maxByteLength = DEFAULT_BUFFER_SIZE * 2
+		this.BC = BC
+		this.buffer = new BC(this.bufferSize, {
+			maxByteLength: this.maxByteLength,
+		})
+		this.array = new this.ArrayType(this.buffer)
+	}
+
+	[Symbol.iterator](): ArrayIterator<number> {
+		return this.array[Symbol.iterator]()
+	}
+
+	expandArray() {
+		this.bufferSize *= 2
+		if (this.bufferSize > this.buffer.maxByteLength) {
+			this.maxByteLength *= 2
+			const newBuffer = new this.BC(this.bufferSize, {
+				maxByteLength: this.maxByteLength,
+			})
+			const newArray = new this.ArrayType(newBuffer)
+			newArray.set(this.array)
+			this.buffer = newBuffer
+			this.array = newArray
+		} else {
+			if (this.buffer instanceof SharedArrayBuffer && this.buffer.growable) {
+				this.buffer.grow(this.bufferSize)
+			} else if (this.buffer instanceof ArrayBuffer && this.buffer.resizable) {
+				this.buffer.resize(this.bufferSize)
+			} else {
+				throw Error("Buffer is not growable or resizable")
+			}
+		}
+	}
+
+	/**
+	 * Handles negative numbers
+	 */
+	at(index: number): number {
+		if (index < -this.length || index >= this.length)
+			throw Error(`Index out of bounds: ${index}. Length: ${this.length}`)
+		if (index < 0) return this.at(this.length + index)
+		const result = this.array[index]
+		if (result === undefined) throw Error(`No value at index: ${index}`)
+		return result
+	}
+
+	slice(start: number, end: number) {
+		return this.array.slice(start, end)
+	}
+
+	get length() {
+		return this.items
+	}
+
+	push(value: number): number {
+		if (this.length >= this.array.length) {
+			this.expandArray()
+		}
+		this.array[this.items++] = value
+		return this.length - 1
+	}
+
+	pushMany(values: number[] | TypedArray) {
+		while (this.length + values.length > this.array.length) this.expandArray()
+		this.array.set(values, this.length)
+		this.items += values.length
+	}
+
+	compact() {
+		if (this.buffer instanceof SharedArrayBuffer) {
+			this.buffer = this.buffer.slice(
+				0,
+				this.length * this.ArrayType.BYTES_PER_ELEMENT,
+			)
+		} else {
+			this.buffer = this.buffer.transferToFixedLength(
+				this.length * this.ArrayType.BYTES_PER_ELEMENT,
+			)
+		}
+		this.array = new this.ArrayType(this.buffer)
+		return this.array
+	}
+}

package/src/types/kdbush.d.ts

@@ -0,0 +1,29 @@
+declare module "kdbush" {
+	import type { BufferConstructor, BufferType } from "../typed-arrays"
+	export default class KDBush {
+		constructor(
+			length: number,
+			nodeSize: number,
+			ArrayType: Float64ArrayConstructor,
+			BufferConstructor: BufferConstructor,
+		)
+
+		/** Serialized index data buffer */
+		data: BufferType
+
+		/** Add a point before calling finish */
+		add(x: number, y: number): void
+
+		/** Finalize the index after adding all points */
+		finish(): void
+
+		/** Query points within a bounding box; returns internal point indexes */
+		range(minX: number, minY: number, maxX: number, maxY: number): number[]
+
+		/** Query points within a radius from (x, y); returns internal point indexes */
+		within(x: number, y: number, radius: number): number[]
+
+		/** Reconstruct an index from a serialized buffer */
+		static from(data: BufferType): KDBush
+	}
+}

package/src/utils.ts

@@ -0,0 +1,29 @@
+import type { GeoBbox2D, LonLat } from "@osmix/shared/types"
+
+export function throttle<T extends unknown[]>(
+	func: (...args: T) => void,
+	timeFrame: number,
+) {
+	let lastTime = 0
+	return (...args: T) => {
+		const now = Date.now()
+		if (now - lastTime >= timeFrame) {
+			func(...args)
+			lastTime = now
+		}
+	}
+}
+
+export function bboxFromLonLats(lonLats: LonLat[]): GeoBbox2D {
+	let minLon = Number.POSITIVE_INFINITY
+	let minLat = Number.POSITIVE_INFINITY
+	let maxLon = Number.NEGATIVE_INFINITY
+	let maxLat = Number.NEGATIVE_INFINITY
+	for (const [lon, lat] of lonLats) {
+		if (lon < minLon) minLon = lon
+		if (lat < minLat) minLat = lat
+		if (lon > maxLon) maxLon = lon
+		if (lat > maxLat) maxLat = lat
+	}
+	return [minLon, minLat, maxLon, maxLat]
+}