@osmix/core 0.0.1

package/CHANGELOG.md

@@ -0,0 +1,11 @@
+# @osmix/core
+
+## 0.0.1
+
+### Patch Changes
+
+- Initial release
+- Updated dependencies
+  - @osmix/json@0.0.1
+  - @osmix/pbf@0.0.1
+  - @osmix/shared@0.0.1

package/README.md

@@ -0,0 +1,160 @@
+# @osmix/core
+
+@osmix/core wraps the `Osmix` index: a typed-array OpenStreetMap engine that reads `.osm.pbf` streams, builds spatial indexes, and emits JSON, PBF, or raster tiles without leaving modern JavaScript runtimes.
+
+## Highlights
+
+- Ingest `.osm.pbf` sources into node, way, and relation stores backed by transferable typed arrays using readers from [`@osmix/pbf`](../pbf/README.md).
+- Run fast bounding-box searches with KDBush/Flatbush and convert matches straight to GeoJSON.
+- Trim extracts, write new PBF buffers, or stream entities to downstream tooling with [`@osmix/json`](../json/README.md) or [`@osmix/pbf`](../pbf/README.md).
+- Pair with [`@osmix/change`](../change/README.md) when you need deduplication, intersection, or merge pipelines.
+- Ship fully indexed datasets across workers via `transferables()` + `Osmix.from`.
+
+## Installation
+
+```sh
+npm install @osmix/core
+```
+
+## Usage
+
+### Load a PBF in one call
+
+```ts
+import { Osmix } from "@osmix/core"
+import { readFile } from "node:fs/promises"
+
+const osm = await Osmix.fromPbf(readFile("example.osm.pbf"))
+
+console.log(osm.nodes.size, osm.ways.size, osm.relations.size)
+```
+
+`fromPbf` accepts `ArrayBufferLike`, async (iterable) chunks, or Web `ReadableStream` inputs.
+
+### Stream with progress and extraction
+
+Pass a logger or set one later to receive throttled progress messages. Provide `extractBbox` to keep only features intersecting the desired bounds while parsing.
+
+```ts
+import { Osmix } from "@osmix/core"
+import { createReadStream } from "node:fs"
+
+const osm = new Osmix({ id: "planet-seattle" })
+osm.setLogger((message) => console.log(message))
+
+await osm.readPbf(createReadStream("planet.osm.pbf"), {
+	extractBbox: [-122.5, 47.45, -122.2, 47.75],
+})
+```
+
+### Build synthetic fixtures
+
+When generating test data, insert entities directly and call `buildIndexes()` before querying.
+
+```ts
+const osm = new Osmix({ id: "fixture" })
+osm.nodes.addNode({ id: 1, lon: -0.1, lat: 51.5, tags: { amenity: "cafe" } })
+osm.ways.addWay({ id: 10, refs: [1], tags: { highway: "service" } })
+osm.buildIndexes()
+```
+
+## Query the index
+
+- `osm.get("node", 123)` or `osm.getById("n123")` fetches specific entities.
+- `osm.nodes.sorted()` / `osm.ways.sorted()` iterate ids in ascending order.
+- `osm.getEntityGeoJson(entity)` returns a GeoJSON feature with coordinates resolved from the
+  indexed nodes.
+
+Spatial lookups emit compact typed arrays that can be transferred across Web Workers and used directly in `Deck.gl`:
+
+```ts
+const bbox = [-122.34, 47.57, -122.30, 47.61]
+const { ids, positions } = osm.getNodesInBbox(bbox)
+
+const { ids: wayIds, positions: wayCoords, startIndices } = osm.getWaysInBbox(bbox)
+```
+
+`positions` stores `[lon, lat]` pairs; `startIndices` splits `wayCoords` into per-way sequences.
+
+## Extract and export data
+
+```ts
+const downtown = osm.extract([-122.35, 47.60, -122.32, 47.62])
+
+const entityStream = downtown.toEntityStream() // header + entities
+const pbfStream = downtown.toPbfStream() // Uint8Array chunks
+const pbfBuffer = await downtown.toPbfBuffer()
+```
+
+`extract` returns a new, fully indexed `Osmix` instance with the header bbox updated.
+
+## Changesets and merging
+
+Change orchestration now lives in the dedicated [`@osmix/change`](../change/README.md) package. Install it alongside
+`@osmix/core` when you want to deduplicate entities, generate direct merges, or script intersection creation:
+
+```ts
+import { Osmix } from "@osmix/core"
+import { merge, OsmixChangeset } from "@osmix/change"
+import { readFile } from "node:fs/promises"
+
+const base = await Osmix.fromPbf(readFile("base.osm.pbf"))
+const patch = await Osmix.fromPbf(readFile("patch.osm.pbf"))
+
+const changeset = new OsmixChangeset(base)
+changeset.deduplicateWays(base.ways)
+
+const merged = await merge(base, patch, { directMerge: true })
+```
+
+Refer to the [`@osmix/change` README](../change/README.md) for the full API surface.
+
+## Transfer between threads
+
+Typed arrays keep the index transferable.
+
+```ts
+// main thread
+const payload = osm.transferables()
+const buffers = []
+const collect = (value: unknown) => {
+	if (value instanceof ArrayBuffer) buffers.push(value)
+	else if (ArrayBuffer.isView(value)) buffers.push(value.buffer)
+	else if (Array.isArray(value)) value.forEach(collect)
+	else if (value && typeof value === "object")
+		Object.values(value).forEach(collect)
+	return buffers
+}
+
+worker.postMessage(payload, { transfer: collect(payload) })
+
+// worker
+import { Osmix } from "@osmix/core"
+
+self.addEventListener("message", ({ data }) => {
+	const osm = Osmix.from(data)
+	// ready for queries immediately
+})
+```
+
+## API overview
+
+- `Osmix` – ingest PBF sources, build indexes, query entities, extract subsets, and emit JSON/PBF.
+- `Nodes` / `Ways` / `Relations` – typed-array backed stores exposed for advanced workflows.
+- `OsmixRasterTile` – rasterize indexed data into tile-sized buffers with lon/lat helpers.
+- Utilities – `throttle`, plus the shared type definitions used across the package.
+- Changesets and merge helpers now live in [`@osmix/change`](../change/README.md).
+
+## Environment and limitations
+
+- Requires runtimes with Web Streams, `TextEncoder`/`TextDecoder`, and zlib-compatible `CompressionStream`/`DecompressionStream` support (Bun, Node 20+, modern browsers).
+- `readPbf` expects dense-node blocks; PBFs that omit dense encodings are currently unsupported.
+- Filtering during ingest depends on node membership; emit nodes, then ways, then relations when supplying custom entity generators.
+
+## Development
+
+- `bun run test packages/core`
+- `bun run lint packages/core`
+- `bun run typecheck packages/core`
+
+Run `bun run check` at the repo root before publishing to ensure formatting, lint, and type coverage.

package/package.json

@@ -0,0 +1,53 @@
+{
+	"$schema": "https://json.schemastore.org/package",
+	"name": "@osmix/core",
+	"type": "module",
+	"version": "0.0.1",
+	"description": "An OSM PBF reader, writer, and merge tool written in TypeScript",
+	"main": "./src/index.ts",
+	"publishConfig": {
+		"access": "public",
+		"main": "./dist/index.js",
+		"types": "./dist/index.d.ts",
+		"exports": {
+			".": {
+				"import": "./dist/index.js",
+				"types": "./dist/index.d.ts"
+			}
+		},
+		"files": [
+			"dist",
+			"README.md",
+			"LICENSE"
+		]
+	},
+	"license": "MIT",
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/conveyal/osmix.git"
+	},
+	"homepage": "https://github.com/conveyal/osmix#readme",
+	"bugs": {
+		"url": "https://github.com/conveyal/osmix/issues"
+	},
+	"sideEffects": false,
+	"scripts": {
+		"build": "tsc",
+		"bench": "vitest bench",
+		"test": "vitest",
+		"typecheck": "tsc --noEmit"
+	},
+	"devDependencies": {
+		"@types/geojson": "catalog:",
+		"typescript": "catalog:",
+		"vitest": "catalog:"
+	},
+	"dependencies": {
+		"@osmix/json": "workspace:*",
+		"@osmix/pbf": "workspace:*",
+		"@osmix/shared": "workspace:*",
+		"flatbush": "^4.4.1",
+		"kdbush": "mourner/kdbush#16416cd",
+		"sweepline-intersections": "^1.5.0"
+	}
+}

package/src/entities.ts

@@ -0,0 +1,95 @@
+import type { OsmEntity, OsmEntityType, OsmTags } from "@osmix/json"
+import { type IdOrIndex, Ids, type IdsTransferables } from "./ids"
+import type StringTable from "./stringtable"
+import { Tags, type TagsTransferables } from "./tags"
+
+export interface EntitiesTransferables
+	extends IdsTransferables,
+		TagsTransferables {}
+
+export abstract class Entities<T extends OsmEntity> {
+	indexType: OsmEntityType
+	stringTable: StringTable
+	ids: Ids
+	tags: Tags
+
+	private indexBuilt = false
+
+	constructor(
+		indexType: OsmEntityType,
+		stringTable: StringTable,
+		ids?: Ids,
+		tags?: Tags,
+	) {
+		this.stringTable = stringTable
+		this.indexType = indexType
+		this.ids = ids ?? new Ids()
+		this.tags = tags ?? new Tags(stringTable)
+	}
+
+	get isReady() {
+		return this.ids.isReady && this.tags.isReady && this.indexBuilt
+	}
+
+	get size() {
+		return this.ids.size
+	}
+
+	abstract buildEntityIndex(): void
+
+	buildIndex() {
+		console.time(`${this.indexType}Index.finish`)
+		this.ids.buildIndex()
+		this.tags.buildIndex()
+		this.buildEntityIndex()
+		this.indexBuilt = true
+		console.timeEnd(`${this.indexType}Index.finish`)
+	}
+
+	abstract getFullEntity(index: number, id: number, tags?: OsmTags): T
+
+	get(idOrIndex: IdOrIndex): T | null {
+		const [index, id] = this.ids.idOrIndex(idOrIndex)
+		if (index === -1) return null
+		return this.getFullEntity(index, id, this.tags.getTags(index))
+	}
+
+	getByIndex(index: number): T {
+		return this.getFullEntity(
+			index,
+			this.ids.at(index),
+			this.tags.getTags(index),
+		)
+	}
+
+	getEntitiesByIndex(indexes: number[]): T[] {
+		const entities: T[] = []
+		for (const index of indexes) {
+			const entity = this.getByIndex(index)
+			if (entity) entities.push(entity)
+			else throw Error(`Entity not found at index ${index}`)
+		}
+		return entities
+	}
+
+	*[Symbol.iterator](): Generator<T> {
+		for (let i = 0; i < this.size; i++) {
+			yield this.getFullEntity(i, this.ids.at(i), this.tags.getTags(i))
+		}
+	}
+
+	getById(id: number): T | null {
+		const index = this.ids.getIndexFromId(id)
+		if (index !== -1)
+			return this.getFullEntity(index, id, this.tags.getTags(index))
+		return null
+	}
+
+	*sorted(): Generator<T> {
+		for (const id of this.ids.sorted) {
+			const index = this.ids.getIndexFromId(id)
+			if (index === -1) throw Error(`Entity not found at id ${id}`)
+			yield this.getFullEntity(index, id, this.tags.getTags(index))
+		}
+	}
+}

package/src/ids.ts

@@ -0,0 +1,210 @@
+import { assertValue } from "@osmix/shared/assert"
+import {
+	BufferConstructor,
+	type BufferType,
+	ResizeableTypedArray,
+} from "./typed-arrays"
+
+export type IdOrIndex = { id: number } | { index: number }
+
+const BLOCK_SIZE = 256
+
+export interface IdsTransferables {
+	ids: BufferType
+	sortedIds: BufferType
+	sortedIdPositionToIndex: BufferType
+	anchors: BufferType
+	idsAreSorted: boolean
+}
+
+/**
+ * Efficiently store and lookup IDs.
+ *
+ * This is used to store IDs in a sorted array and then use binary search to find the index of an ID.
+ * Maps max out at 2^32 IDs.
+ */
+export class Ids {
+	private ids: ResizeableTypedArray<Float64Array>
+	private indexBuilt = false
+	private idsAreSorted = true
+	private idsSorted: Float64Array
+	private sortedIdPositionToIndex: Uint32Array
+	private anchors: Float64Array
+
+	static from({
+		ids,
+		sortedIds,
+		sortedIdPositionToIndex,
+		anchors,
+		idsAreSorted,
+	}: IdsTransferables) {
+		const idIndex = new Ids()
+		idIndex.ids = ResizeableTypedArray.from(Float64Array, ids)
+		idIndex.idsSorted = new Float64Array(sortedIds)
+		idIndex.sortedIdPositionToIndex = new Uint32Array(sortedIdPositionToIndex)
+		idIndex.anchors = new Float64Array(anchors)
+		idIndex.indexBuilt = true
+		idIndex.idsAreSorted = idsAreSorted
+		return idIndex
+	}
+
+	constructor() {
+		this.ids = new ResizeableTypedArray(Float64Array)
+		this.idsSorted = new Float64Array(new BufferConstructor(0))
+		this.sortedIdPositionToIndex = new Uint32Array(new BufferConstructor(0))
+		this.anchors = new Float64Array(new BufferConstructor(0))
+	}
+
+	transferables(): IdsTransferables {
+		return {
+			ids: this.ids.array.buffer,
+			sortedIds: this.idsSorted.buffer,
+			sortedIdPositionToIndex: this.sortedIdPositionToIndex.buffer,
+			anchors: this.anchors.buffer,
+			idsAreSorted: this.idsAreSorted,
+		}
+	}
+
+	get size() {
+		return this.ids.length
+	}
+
+	get isReady() {
+		return this.indexBuilt
+	}
+
+	isSorted() {
+		return this.idsAreSorted
+	}
+
+	add(id: number): void {
+		if (this.indexBuilt) throw Error("ID index already built.")
+		if (this.ids.length > 0 && id < this.ids.at(-1)) this.idsAreSorted = false
+		this.ids.push(id)
+	}
+
+	at(index: number): number {
+		return this.ids.at(index)
+	}
+
+	has(id: number): boolean {
+		return this.getIndexFromId(id) !== -1
+	}
+
+	/**
+	 * Build the index of IDs to positions.
+	 *
+	 * If the IDs are not sorted, we need to sort them and build a new index.
+	 * If the IDs are sorted, we can use the existing index.
+	 */
+	buildIndex() {
+		if (this.indexBuilt) throw Error("ID index already build.")
+		this.ids.compact()
+		if (!this.idsAreSorted) {
+			console.warn("IDs were not sorted. Sorting now...")
+			// Build the sorted index
+			const idsBuffer = new BufferConstructor(
+				this.size * Float64Array.BYTES_PER_ELEMENT,
+			)
+			const posBuffer = new BufferConstructor(
+				this.size * Uint32Array.BYTES_PER_ELEMENT,
+			)
+			this.idsSorted = new Float64Array(idsBuffer)
+			this.sortedIdPositionToIndex = new Uint32Array(posBuffer)
+
+			// Fill and sort with positions.
+			for (let i = 0; i < this.size; i++) {
+				this.idsSorted[i] = this.ids.at(i)
+				this.sortedIdPositionToIndex[i] = i
+			}
+
+			// Sort by id, carrying position; use native sort on chunks or a custom radix/merge for stability.
+			// For simplicity:
+			const tmp: { id: number; pos: number }[] = Array.from(
+				{ length: this.size },
+				(_, i) => ({
+					id: this.idsSorted[i] as number,
+					pos: this.sortedIdPositionToIndex[i] as number,
+				}),
+			)
+			tmp.sort((a, b) => a.id - b.id)
+			tmp.forEach(({ id, pos }, i) => {
+				this.idsSorted[i] = id
+				this.sortedIdPositionToIndex[i] = pos
+			})
+		} else {
+			// Point to the same array
+			this.idsSorted = this.ids.array
+			// Create the sortedIdPositionToIndex array
+			const posBuffer = new BufferConstructor(
+				this.size * Uint32Array.BYTES_PER_ELEMENT,
+			)
+			this.sortedIdPositionToIndex = new Uint32Array(posBuffer)
+			for (let i = 0; i < this.size; i++) {
+				this.sortedIdPositionToIndex[i] = i
+			}
+		}
+
+		// Build anchors (every blockSize-th key)
+		const aLen = Math.ceil(this.size / BLOCK_SIZE)
+		const sab = new BufferConstructor(aLen * Float64Array.BYTES_PER_ELEMENT)
+		this.anchors = new Float64Array(sab, 0, aLen)
+		for (let j = 0; j < aLen; j++) {
+			const id = this.idsSorted[Math.min(j * BLOCK_SIZE, this.size - 1)]
+			assertValue(id, "ID is undefined")
+			this.anchors[j] = id
+		}
+
+		this.indexBuilt = true
+	}
+
+	// Lookup id → index
+	getIndexFromId(id: number): number {
+		if (!this.indexBuilt) throw Error("IdIndex not built.")
+
+		// binary search anchors
+		let lo = 0
+		let hi = this.anchors.length - 1
+		while (lo < hi) {
+			const mid = (lo + hi + 1) >>> 1
+			const anchor = this.anchors[mid]
+			assertValue(anchor, "Anchor is undefined")
+			if (anchor <= id) lo = mid
+			else hi = mid - 1
+		}
+		const start = lo * BLOCK_SIZE
+		const end = Math.min(start + BLOCK_SIZE, this.idsSorted.length)
+
+		// binary search within block
+		let l = start
+		let r = end - 1
+		while (l <= r) {
+			const m = (l + r) >>> 1
+			const v = this.idsSorted[m]
+			assertValue(v, "Value is undefined")
+			if (v === id) {
+				if (this.idsAreSorted) return m
+				const index = this.sortedIdPositionToIndex[m]
+				assertValue(index, "Position is undefined")
+				return index
+			}
+			if (v < id) l = m + 1
+			else r = m - 1
+		}
+
+		// ID not found
+		return -1
+	}
+
+	/**
+	 * Pass an ID or an index, get both.
+	 */
+	idOrIndex(i: IdOrIndex): [index: number, id: number] {
+		if ("id" in i) return [this.getIndexFromId(i.id), i.id]
+		return [i.index, this.at(i.index)]
+	}
+
+	get sorted() {
+		return this.idsSorted
+	}
+}

package/src/index.ts

@@ -0,0 +1,7 @@
+/// <reference path="./types/kdbush.d.ts" />
+
+export { Nodes } from "./nodes"
+export { Osmix } from "./osmix"
+export { Relations } from "./relations"
+export { throttle } from "./utils"
+export { Ways } from "./ways"