@osmix/pbf 0.0.1

package/test/utils.test.ts

@@ -0,0 +1,56 @@
+import { assert, describe, expect, it } from "vitest"
+import {
+	concatUint8,
+	toAsyncGenerator,
+	uint32BE,
+	webCompress,
+	webDecompress,
+} from "../src/utils"
+
+describe("utils", () => {
+	it("wraps values into an async generator", async () => {
+		const generator = toAsyncGenerator(3)
+		const first = await generator.next()
+		assert.deepEqual(first, { value: 3, done: false })
+		const done = await generator.next()
+		assert.deepEqual(done, { value: undefined, done: true })
+	})
+
+	it("consumes readable streams", async () => {
+		const stream = new ReadableStream<number>({
+			start(controller) {
+				controller.enqueue(1)
+				controller.enqueue(2)
+				controller.close()
+			},
+		})
+		const values: number[] = []
+		for await (const value of toAsyncGenerator(stream)) values.push(value)
+		assert.deepEqual(values, [1, 2])
+	})
+
+	it("throws on nullish inputs", async () => {
+		const invalidInput = null as unknown as never
+		await expect(toAsyncGenerator(invalidInput).next()).rejects.toThrow(
+			"Value is null",
+		)
+	})
+
+	it("concatenates Uint8Array segments", () => {
+		const a = Uint8Array.of(1, 2)
+		const b = Uint8Array.of(3)
+		assert.deepEqual(concatUint8(a, b), Uint8Array.of(1, 2, 3))
+	})
+
+	it("encodes big-endian 32-bit integers", () => {
+		assert.deepEqual(uint32BE(0x01020304), Uint8Array.of(1, 2, 3, 4))
+	})
+
+	it("compresses and decompresses data", async () => {
+		const input = new TextEncoder().encode("osmix") as Uint8Array<ArrayBuffer>
+		const compressed = await webCompress(input)
+		assert.notDeepEqual(compressed, input)
+		const decompressed = await webDecompress(compressed)
+		assert.deepEqual(decompressed, input)
+	})
+})

package/test/utils.ts

@@ -0,0 +1,65 @@
+import type { PbfFixture } from "@osmix/shared/test/fixtures"
+import { assert } from "vitest"
+import type {
+	OsmPbfBlock,
+	OsmPbfGroup,
+	OsmPbfHeaderBlock,
+} from "../src/proto/osmformat"
+
+export async function testOsmPbfReader(
+	osm: {
+		header: OsmPbfHeaderBlock
+		blocks: AsyncGenerator<OsmPbfBlock>
+	},
+	pbf: PbfFixture,
+) {
+	assert.deepEqual(osm.header.bbox, pbf.bbox)
+
+	const { onGroup, count } = createOsmEntityCounter()
+	for await (const block of osm.blocks)
+		for (const group of block.primitivegroup) onGroup(group)
+
+	assert.equal(count.nodes, pbf.nodes)
+	assert.equal(count.ways, pbf.ways)
+	assert.equal(count.relations, pbf.relations)
+	assert.equal(count.node0, pbf.node0.id)
+	assert.equal(count.way0, pbf.way0)
+	assert.equal(count.relation0, pbf.relation0)
+
+	return count
+}
+
+export function createOsmEntityCounter() {
+	const count = {
+		nodes: 0,
+		ways: 0,
+		relations: 0,
+		node0: -1,
+		way0: -1,
+		relation0: -1,
+	}
+
+	const onGroup = (group: OsmPbfGroup) => {
+		if (count.node0 === -1 && group.dense?.id?.[0] != null) {
+			count.node0 = group.dense.id[0]
+		}
+		if (count.way0 === -1 && group.ways?.[0]?.id != null) {
+			count.way0 = group.ways[0].id
+		}
+		if (count.relation0 === -1 && group.relations?.[0]?.id != null) {
+			count.relation0 = group.relations[0].id
+		}
+
+		count.nodes += group.nodes?.length ?? 0
+		if (group.dense) {
+			count.nodes += group.dense.id.length
+		}
+		count.ways += group.ways?.length ?? 0
+		count.relations += group.relations?.length ?? 0
+	}
+
+	return {
+		onGroup,
+		count,
+	}
+}

package/test/verify-pbf-reading.bun.test.ts

@@ -0,0 +1,39 @@
+import { getFixtureFile } from "@osmix/shared/test/fixtures"
+import { describe, expect, test } from "vitest"
+import { readOsmPbf } from "../src/pbf-to-blocks"
+import { isBun } from "../src/utils"
+
+describe.runIf(isBun())("PBF Reading with Bun", () => {
+	test("can read an actual OSM PBF file", async () => {
+		// Load a real PBF fixture
+		const pbfData = await getFixtureFile("monaco.pbf")
+
+		// Try to read it
+		const result = await readOsmPbf(pbfData)
+
+		// Basic sanity checks
+		expect(result.header).toBeDefined()
+		expect(result.header.required_features).toBeDefined()
+		expect(result.blocks).toBeDefined()
+
+		// Try to read at least one block
+		const firstBlock = await result.blocks.next()
+		expect(firstBlock.done).toBe(false)
+		expect(firstBlock.value).toBeDefined()
+
+		// If it's a primitive block, it should have primitivegroup
+		if ("primitivegroup" in firstBlock.value) {
+			expect(Array.isArray(firstBlock.value.primitivegroup)).toBe(true)
+		}
+	})
+
+	test("header contains expected OSM data", async () => {
+		const pbfData = await getFixtureFile("monaco.pbf")
+
+		const result = await readOsmPbf(pbfData)
+
+		// Check that header has the required OSM features
+		expect(result.header.required_features).toBeDefined()
+		expect(result.header.required_features.length).toBeGreaterThan(0)
+	})
+})

package/test/write.test.ts

@@ -0,0 +1,86 @@
+import { unlink } from "node:fs/promises"
+import {
+	getFixtureFile,
+	getFixtureFileReadStream,
+	getFixtureFileWriteStream,
+	getFixturePath,
+	PBFs,
+} from "@osmix/shared/test/fixtures"
+import { assert, describe, it } from "vitest"
+import {
+	OsmBlocksToPbfBytesTransformStream,
+	osmBlockToPbfBlobBytes,
+} from "../src/blocks-to-pbf"
+import {
+	OsmPbfBytesToBlocksTransformStream,
+	readOsmPbf,
+} from "../src/pbf-to-blocks"
+import { testOsmPbfReader } from "./utils"
+
+describe("write", () => {
+	describe.each(Object.entries(PBFs))("%s", (name, pbf) => {
+		it("to buffer", async () => {
+			const fileData = await getFixtureFile(pbf.url)
+			const osm = await readOsmPbf(fileData)
+
+			let node0: number | null = null
+			let way0: number | null = null
+			let relation0: number | null = null
+
+			// Write the PBF to an array buffer
+			let data = new Uint8Array(0)
+			const write = (chunk: Uint8Array) => {
+				const newData = new Uint8Array(data.length + chunk.length)
+				newData.set(data)
+				newData.set(chunk, data.length)
+				data = newData
+			}
+
+			write(await osmBlockToPbfBlobBytes(osm.header))
+			for await (const block of osm.blocks) {
+				for (const group of block.primitivegroup) {
+					if (node0 == null && group.dense?.id?.[0] != null) {
+						node0 = group.dense.id[0]
+					}
+					if (way0 == null && group.ways?.[0]?.id != null) {
+						way0 = group.ways[0].id
+					}
+					if (relation0 == null && group.relations?.[0]?.id != null) {
+						relation0 = group.relations[0].id
+					}
+				}
+				write(await osmBlockToPbfBlobBytes(block))
+			}
+
+			// Re-parse the new PBF and test
+			assert.exists(data.buffer)
+			// TODO: assert.equal(stream.buffer.byteLength, fileData.byteLength)
+			const osm2 = await readOsmPbf(data.buffer)
+
+			assert.deepEqual(osm.header, osm2.header)
+			const entities = await testOsmPbfReader(osm2, pbf)
+			assert.equal(entities.node0, node0)
+			assert.equal(entities.way0, way0)
+			assert.equal(entities.relation0, relation0)
+		})
+
+		it("to file", async () => {
+			const testFileName = `${name}-write-test.pbf`
+			const fileStream = getFixtureFileReadStream(pbf.url)
+
+			const fileWriteStream = getFixtureFileWriteStream(testFileName)
+			await fileStream
+				.pipeThrough(new OsmPbfBytesToBlocksTransformStream())
+				.pipeThrough(new OsmBlocksToPbfBytesTransformStream())
+				.pipeTo(fileWriteStream)
+
+			const testFileData = await getFixtureFile(pbf.url)
+			const testOsm = await readOsmPbf(testFileData)
+
+			assert.deepEqual(testOsm.header.bbox, pbf.bbox)
+			await testOsmPbfReader(testOsm, pbf)
+
+			await unlink(getFixturePath(testFileName))
+		})
+	})
+})

package/tsconfig.json

@@ -0,0 +1,9 @@
+{
+	"$schema": "https://json.schemastore.org/tsconfig",
+	"extends": "@osmix/shared/tsconfig/base.json",
+	"exclude": ["node_modules", "dist"],
+	"include": ["src", "test"],
+	"compilerOptions": {
+		"outDir": "./dist"
+	}
+}

package/vitest.config.ts

@@ -0,0 +1,7 @@
+import { defineConfig } from "vitest/config"
+
+export default defineConfig({
+	test: {
+		// exclude: ["**/node_modules/**", "**/*.bun.test.ts"],
+	},
+})