@osmix/pbf 0.0.1

package/test/helpers.ts

@@ -0,0 +1,66 @@
+import { osmBlockToPbfBlobBytes } from "../src/blocks-to-pbf"
+import type { OsmPbfBlock, OsmPbfHeaderBlock } from "../src/proto/osmformat"
+import { concatUint8 } from "../src/utils"
+
+const encoder = new TextEncoder()
+
+export function createSampleHeader(): OsmPbfHeaderBlock {
+	return {
+		bbox: { left: 0, right: 1, top: 1, bottom: 0 },
+		required_features: ["OsmSchema-V0.6"],
+		optional_features: ["DenseNodes"],
+		writingprogram: "osmix-tests",
+	}
+}
+
+export function createSamplePrimitiveBlock(): OsmPbfBlock {
+	return {
+		stringtable: [
+			encoder.encode(""),
+			encoder.encode("name"),
+			encoder.encode("value"),
+		],
+		primitivegroup: [
+			{
+				nodes: [],
+				dense: {
+					id: [1, 2],
+					lat: [1_000, 500],
+					lon: [1_500, 600],
+					keys_vals: [1, 2, 0],
+				},
+				ways: [
+					{
+						id: 10,
+						keys: [1],
+						vals: [2],
+						refs: [1, 1, 0],
+					},
+				],
+				relations: [],
+			},
+		],
+	} as const
+}
+
+export async function createSamplePbfFileBytes() {
+	const header = createSampleHeader()
+	const primitiveBlock = createSamplePrimitiveBlock()
+	const headerBytes = await osmBlockToPbfBlobBytes(header)
+	const primitiveBytes = await osmBlockToPbfBlobBytes(primitiveBlock)
+	return {
+		header,
+		primitiveBlock,
+		fileBytes: concatUint8(headerBytes, primitiveBytes),
+	}
+}
+
+export function isHeaderBlock(value: unknown): value is OsmPbfHeaderBlock {
+	return (
+		typeof value === "object" && value != null && "required_features" in value
+	)
+}
+
+export function isPrimitiveBlock(value: unknown): value is OsmPbfBlock {
+	return typeof value === "object" && value != null && "primitivegroup" in value
+}

package/test/pbf-to-blobs.test.ts

@@ -0,0 +1,85 @@
+import Pbf from "pbf"
+import { assert, describe, it } from "vitest"
+import { osmPbfBlobsToBlocksGenerator } from "../src/blobs-to-blocks"
+import { createOsmPbfBlobGenerator } from "../src/pbf-to-blobs"
+import { writeBlob, writeBlobHeader } from "../src/proto/fileformat"
+import { writeHeaderBlock } from "../src/proto/osmformat"
+import { concatUint8, uint32BE } from "../src/utils"
+import {
+	createSampleHeader,
+	createSamplePbfFileBytes,
+	isHeaderBlock,
+	isPrimitiveBlock,
+} from "./helpers"
+
+describe("createOsmPbfBlobGenerator", () => {
+	it("yields compressed blobs across fragmented chunks", async () => {
+		const { header, primitiveBlock, fileBytes } =
+			await createSamplePbfFileBytes()
+		const generate = createOsmPbfBlobGenerator()
+		const yielded: Uint8Array<ArrayBuffer>[] = []
+
+		let offset = 0
+		const chunkSizes = [1, 9]
+		for (const size of chunkSizes) {
+			const chunk = fileBytes.slice(offset, offset + size)
+			offset += size
+			for (const blob of generate(chunk)) yielded.push(blob)
+		}
+		if (offset < fileBytes.length) {
+			for (const blob of generate(fileBytes.slice(offset))) yielded.push(blob)
+		}
+
+		assert.equal(yielded.length, 2)
+
+		const blocks = osmPbfBlobsToBlocksGenerator(
+			(async function* () {
+				for (const blob of yielded) yield blob
+			})(),
+		)
+		const { value: headerBlock, done } = await blocks.next()
+		assert.isFalse(done)
+		if (!isHeaderBlock(headerBlock)) {
+			assert.fail("Expected first block to be a header")
+		}
+		assert.deepEqual(headerBlock.bbox, header.bbox)
+		assert.deepEqual(headerBlock.required_features, header.required_features)
+		assert.deepEqual(headerBlock.optional_features, header.optional_features)
+
+		const { value: primitive } = await blocks.next()
+		if (!isPrimitiveBlock(primitive)) {
+			assert.fail("Expected primitive block after header")
+		}
+		assert.lengthOf(
+			primitive.primitivegroup,
+			primitiveBlock.primitivegroup.length,
+		)
+		assert.exists(primitive.primitivegroup[0])
+		assert.exists(primitiveBlock.primitivegroup[0])
+		const dense = primitive.primitivegroup[0].dense
+		assert.exists(dense)
+		assert.deepEqual(dense?.id, primitiveBlock.primitivegroup[0].dense?.id)
+		assert.deepEqual(dense?.lat, primitiveBlock.primitivegroup[0].dense?.lat)
+		assert.deepEqual(dense?.lon, primitiveBlock.primitivegroup[0].dense?.lon)
+	})
+
+	it("throws when a blob omits zlib data", () => {
+		const headerBlock = createSampleHeader()
+		const headerPbf = new Pbf()
+		writeHeaderBlock(headerBlock, headerPbf)
+		const headerContent = headerPbf.finish()
+
+		const blobPbf = new Pbf()
+		writeBlob({ raw_size: headerContent.length, raw: headerContent }, blobPbf)
+		const blob = blobPbf.finish()
+
+		const blobHeaderPbf = new Pbf()
+		writeBlobHeader({ type: "OSMHeader", datasize: blob.length }, blobHeaderPbf)
+		const blobHeader = blobHeaderPbf.finish()
+
+		const chunk = concatUint8(uint32BE(blobHeader.byteLength), blobHeader, blob)
+		const generate = createOsmPbfBlobGenerator()
+		const iterator = generate(chunk)
+		assert.throws(() => iterator.next(), /Blob has no zlib data/)
+	})
+})

package/test/read.bench.ts

@@ -0,0 +1,42 @@
+import {
+	getFixtureFile,
+	getFixtureFileReadStream,
+	PBFs,
+} from "@osmix/shared/test/fixtures"
+import { assert, beforeAll, bench, describe } from "vitest"
+import {
+	OsmPbfBytesToBlocksTransformStream,
+	readOsmPbf,
+} from "../src/pbf-to-blocks"
+import { createOsmEntityCounter, testOsmPbfReader } from "./utils"
+
+describe.each(Object.entries(PBFs))("%s", (_name, pbf) => {
+	beforeAll(() => getFixtureFile(pbf.url))
+
+	bench("parse with generators", async () => {
+		const file = await getFixtureFile(pbf.url)
+		const osm = await readOsmPbf(file)
+
+		await testOsmPbfReader(osm, pbf)
+	})
+
+	bench("parse streaming", async () => {
+		const { onGroup, count } = createOsmEntityCounter()
+
+		await getFixtureFileReadStream(pbf.url)
+			.pipeThrough(new OsmPbfBytesToBlocksTransformStream())
+			.pipeTo(
+				new WritableStream({
+					write: (block) => {
+						if ("primitivegroup" in block) {
+							for (const group of block.primitivegroup) onGroup(group)
+						}
+					},
+				}),
+			)
+
+		assert.equal(count.nodes, pbf.nodes)
+		assert.equal(count.ways, pbf.ways)
+		assert.equal(count.relations, pbf.relations)
+	})
+})

package/test/read.test.ts

@@ -0,0 +1,45 @@
+import {
+	getFixtureFile,
+	getFixtureFileReadStream,
+	PBFs,
+} from "@osmix/shared/test/fixtures"
+import { assert, beforeAll, describe, it } from "vitest"
+import {
+	OsmPbfBytesToBlocksTransformStream,
+	readOsmPbf,
+} from "../src/pbf-to-blocks"
+import { createOsmEntityCounter, testOsmPbfReader } from "./utils"
+
+describe("read", () => {
+	describe.each(Object.entries(PBFs))("%s", async (_name, pbf) => {
+		beforeAll(() => getFixtureFile(pbf.url))
+
+		it("from stream", async () => {
+			const { onGroup, count } = createOsmEntityCounter()
+
+			await getFixtureFileReadStream(pbf.url)
+				.pipeThrough(new OsmPbfBytesToBlocksTransformStream())
+				.pipeTo(
+					new WritableStream({
+						write: (block) => {
+							if ("primitivegroup" in block) {
+								for (const group of block.primitivegroup) onGroup(group)
+							} else {
+								assert.deepEqual(block.bbox, pbf.bbox)
+							}
+						},
+					}),
+				)
+
+			assert.equal(count.nodes, pbf.nodes)
+			assert.equal(count.ways, pbf.ways)
+			assert.equal(count.relations, pbf.relations)
+		})
+
+		it("from buffer", async () => {
+			const fileData = await getFixtureFile(pbf.url)
+			const osm = await readOsmPbf(fileData)
+			await testOsmPbfReader(osm, pbf)
+		})
+	})
+})

package/test/streams.test.ts

@@ -0,0 +1,92 @@
+import { assert, describe, expect, it } from "vitest"
+import { OsmBlocksToPbfBytesTransformStream } from "../src/blocks-to-pbf"
+import { OsmPbfBytesToBlocksTransformStream } from "../src/pbf-to-blocks"
+import { concatUint8 } from "../src/utils"
+import {
+	createSamplePbfFileBytes,
+	createSamplePrimitiveBlock,
+	isHeaderBlock,
+	isPrimitiveBlock,
+} from "./helpers"
+
+describe("transform streams", () => {
+	it("requires the header to be written before data blocks", async () => {
+		const input = new ReadableStream({
+			start(controller) {
+				controller.enqueue(createSamplePrimitiveBlock())
+				controller.close()
+			},
+		})
+
+		await expect(
+			input
+				.pipeThrough(new OsmBlocksToPbfBytesTransformStream())
+				.pipeTo(new WritableStream()),
+		).rejects.toThrow("Header first in ReadableStream of blocks.")
+	})
+
+	it("serialises blocks into the expected PBF byte sequence", async () => {
+		const { header, primitiveBlock, fileBytes } =
+			await createSamplePbfFileBytes()
+		const chunks: Uint8Array[] = []
+
+		const input = new ReadableStream({
+			start(controller) {
+				controller.enqueue(header)
+				controller.enqueue(primitiveBlock)
+				controller.close()
+			},
+		})
+
+		await input.pipeThrough(new OsmBlocksToPbfBytesTransformStream()).pipeTo(
+			new WritableStream<Uint8Array>({
+				write(chunk) {
+					chunks.push(chunk)
+				},
+			}),
+		)
+
+		assert.deepEqual(concatUint8(...chunks), fileBytes)
+	})
+
+	it("parses streamed bytes back into header and primitive blocks", async () => {
+		const { header, primitiveBlock, fileBytes } =
+			await createSamplePbfFileBytes()
+		assert.exists(primitiveBlock.primitivegroup[0])
+		const blocks: unknown[] = []
+
+		const input = new ReadableStream({
+			start(controller) {
+				controller.enqueue(fileBytes.slice(0, 7).buffer)
+				controller.enqueue(fileBytes.slice(7).buffer)
+				controller.close()
+			},
+		})
+
+		await input.pipeThrough(new OsmPbfBytesToBlocksTransformStream()).pipeTo(
+			new WritableStream({
+				write(chunk) {
+					blocks.push(chunk)
+				},
+			}),
+		)
+
+		assert.equal(blocks.length, 2)
+		const headerBlock = blocks[0]
+		if (!isHeaderBlock(headerBlock)) {
+			assert.fail("Expected header block")
+		}
+		assert.deepEqual(headerBlock.bbox, header.bbox)
+		assert.deepEqual(headerBlock.required_features, header.required_features)
+		const block = blocks[1]
+		if (!isPrimitiveBlock(block)) {
+			assert.fail("Expected primitive block")
+		}
+		assert.lengthOf(block.primitivegroup, primitiveBlock.primitivegroup.length)
+		assert.exists(block.primitivegroup)
+		assert.exists(block.primitivegroup[0])
+		const dense = block.primitivegroup[0].dense
+		assert.exists(dense)
+		assert.deepEqual(dense.id, primitiveBlock.primitivegroup[0].dense?.id)
+	})
+})

package/test/utils.bun.test.ts

@@ -0,0 +1,327 @@
+import { describe, expect, test } from "vitest"
+import { concatUint8, isBun, toAsyncGenerator, uint32BE } from "../src/utils"
+
+/**
+ * Bun decompress.
+ */
+async function bunDecompress(
+	data: Uint8Array<ArrayBuffer>,
+): Promise<Uint8Array<ArrayBuffer>> {
+	const { inflate } = await import("node:zlib")
+	const result = await new Promise<ArrayBuffer>((resolve, reject) => {
+		inflate(data, (error, result) => {
+			if (error) reject(error)
+			else resolve(result.buffer as ArrayBuffer)
+		})
+	})
+	return new Uint8Array(result)
+}
+
+/**
+ * Bun compression.
+ */
+export async function bunCompress(
+	data: Uint8Array<ArrayBuffer>,
+): Promise<Uint8Array<ArrayBuffer>> {
+	const { deflate } = await import("node:zlib")
+	const result = await new Promise<ArrayBuffer>((resolve, reject) => {
+		deflate(data, (error, result) => {
+			if (error) reject(error)
+			else resolve(result.buffer as ArrayBuffer)
+		})
+	})
+	return new Uint8Array(result)
+}
+
+describe.runIf(isBun())("utils", () => {
+	test("wraps values into an async generator", async () => {
+		const generator = toAsyncGenerator(3)
+		const first = await generator.next()
+		expect(first).toEqual({ value: 3, done: false })
+		const done = await generator.next()
+		expect(done).toEqual({ value: undefined, done: true })
+	})
+
+	test("consumes readable streams", async () => {
+		const stream = new ReadableStream<number>({
+			start(controller) {
+				controller.enqueue(1)
+				controller.enqueue(2)
+				controller.close()
+			},
+		})
+		const values: number[] = []
+		for await (const value of toAsyncGenerator(stream)) values.push(value)
+		expect(values).toEqual([1, 2])
+	})
+
+	test("throws on nullish inputs", async () => {
+		const invalidInput = null as unknown as never
+		await expect(toAsyncGenerator(invalidInput).next()).rejects.toThrow(
+			"Value is null",
+		)
+	})
+
+	test("concatenates Uint8Array segments", () => {
+		const a = Uint8Array.of(1, 2)
+		const b = Uint8Array.of(3)
+		expect(concatUint8(a, b)).toEqual(Uint8Array.of(1, 2, 3))
+	})
+
+	test("encodes big-endian 32-bit integers", () => {
+		expect(uint32BE(0x01020304)).toEqual(Uint8Array.of(1, 2, 3, 4))
+	})
+
+	test("compresses and decompresses data with deflate", async () => {
+		const input = new TextEncoder().encode("osmix") as Uint8Array<ArrayBuffer>
+		const compressed = await bunCompress(input)
+		expect(compressed).not.toEqual(input)
+		const decompressed = await bunDecompress(compressed)
+		expect(decompressed).toEqual(input)
+	})
+
+	test("compresses and decompresses larger data", async () => {
+		const input = new TextEncoder().encode(
+			"a".repeat(1000),
+		) as Uint8Array<ArrayBuffer>
+		const compressed = await bunCompress(input)
+		expect(compressed.length).toBeLessThan(input.length)
+		const decompressed = await bunDecompress(compressed)
+		expect(decompressed).toEqual(input)
+	})
+
+	test("handles Uint8Array input", async () => {
+		const input = Uint8Array.of(1, 2, 3, 4, 5)
+		const compressed = await bunCompress(input)
+		const decompressed = await bunDecompress(compressed)
+		expect(decompressed).toEqual(input)
+	})
+
+	test("uses Bun runtime with Node.js zlib compatibility", () => {
+		// This test verifies that Bun is available in the runtime
+		expect(isBun()).toBe(true)
+	})
+
+	test("Node.js zlib methods work in Bun", async () => {
+		const { deflateSync, inflateSync } = await import("node:zlib")
+		const input = new TextEncoder().encode("test bun compression with zlib")
+		const compressed = deflateSync(input)
+		expect(compressed.length).toBeGreaterThan(0)
+		expect(compressed).not.toEqual(input)
+
+		const decompressed = inflateSync(compressed)
+		expect(new Uint8Array(decompressed)).toEqual(input)
+	})
+
+	test("compress/decompress are compatible with OSM PBF zlib format", async () => {
+		// Test that our compress/decompress functions produce zlib-compatible data
+		// This is critical for OSM PBF compatibility
+		const { deflateSync, inflateSync } = await import("node:zlib")
+		const input = new TextEncoder().encode(
+			"OSM PBF uses zlib format (deflate with headers)",
+		) as Uint8Array<ArrayBuffer>
+
+		// Compress with our function
+		const ourCompressed = await bunCompress(input)
+
+		// Decompress with Node.js zlib (what OSM PBF uses)
+		const decompressedWithNodeZlib = inflateSync(ourCompressed)
+		expect(new Uint8Array(decompressedWithNodeZlib)).toEqual(input)
+
+		// Compress with Node.js zlib
+		const nodeCompressed = deflateSync(input)
+
+		// Decompress with our function
+		const decompressedWithOurs = await bunDecompress(
+			new Uint8Array(nodeCompressed),
+		)
+		expect(decompressedWithOurs).toEqual(input)
+	})
+})
+
+describe.skip("CompressionStream polyfill", () => {
+	test("compresses data using deflate format", async () => {
+		const input = new TextEncoder().encode("test compression stream")
+		const compressor = new CompressionStream("deflate")
+
+		const writer = compressor.writable.getWriter()
+		writer.write(input)
+		writer.close()
+
+		const chunks: Uint8Array[] = []
+		const reader = compressor.readable.getReader()
+		while (true) {
+			const { done, value } = await reader.read()
+			if (done) break
+			chunks.push(value)
+		}
+
+		const compressed = concatUint8(...chunks)
+		expect(compressed.length).toBeGreaterThan(0)
+		expect(compressed).not.toEqual(input)
+
+		// Verify it's valid deflate data by decompressing
+		const decompressed = await bunDecompress(new Uint8Array(compressed))
+		expect(decompressed).toEqual(input)
+	})
+
+	test("returns proper Uint8Array<ArrayBuffer> instances", async () => {
+		const input = new TextEncoder().encode("type safety check")
+		const compressor = new CompressionStream("deflate")
+
+		const writer = compressor.writable.getWriter()
+		writer.write(input)
+		writer.close()
+
+		const reader = compressor.readable.getReader()
+		const { value } = await reader.read()
+
+		expect(value).toBeDefined()
+		if (!value) throw new Error("No value read")
+
+		// Verify it's a Uint8Array
+		expect(value).toBeInstanceOf(Uint8Array)
+		// Verify the buffer is an ArrayBuffer (not Buffer or SharedArrayBuffer)
+		expect(value.buffer).toBeInstanceOf(ArrayBuffer)
+		// Verify it's not a Node.js Buffer
+		expect(value.constructor.name).toBe("Uint8Array")
+	})
+
+	test("handles multiple writes", async () => {
+		const compressor = new CompressionStream("deflate")
+		const writer = compressor.writable.getWriter()
+
+		// Write multiple chunks
+		writer.write(new TextEncoder().encode("first "))
+		writer.write(new TextEncoder().encode("second "))
+		writer.write(new TextEncoder().encode("third"))
+		writer.close()
+
+		const chunks: Uint8Array[] = []
+		const reader = compressor.readable.getReader()
+		while (true) {
+			const { done, value } = await reader.read()
+			if (done) break
+			chunks.push(value)
+		}
+
+		const compressed = concatUint8(...chunks)
+		const decompressed = await bunDecompress(new Uint8Array(compressed))
+		expect(new TextDecoder().decode(decompressed)).toBe("first second third")
+	})
+})
+
+describe.skip("DecompressionStream polyfill", () => {
+	test("decompresses deflate data", async () => {
+		const input = new TextEncoder().encode(
+			"test decompression stream",
+		) as Uint8Array<ArrayBuffer>
+		const compressed = await bunCompress(input)
+
+		const decompressor = new DecompressionStream("deflate")
+		const writer = decompressor.writable.getWriter()
+		writer.write(compressed as Uint8Array<ArrayBuffer>)
+		writer.close()
+
+		const chunks: Uint8Array[] = []
+		const reader = decompressor.readable.getReader()
+		while (true) {
+			const { done, value } = await reader.read()
+			if (done) break
+			chunks.push(value)
+		}
+
+		const decompressed = concatUint8(...chunks)
+		expect(decompressed).toEqual(input)
+	})
+
+	test("returns proper Uint8Array<ArrayBuffer> instances", async () => {
+		const input = new TextEncoder().encode(
+			"type safety check",
+		) as Uint8Array<ArrayBuffer>
+		const compressed = await bunCompress(input)
+
+		const decompressor = new DecompressionStream("deflate")
+		const writer = decompressor.writable.getWriter()
+		writer.write(compressed as Uint8Array<ArrayBuffer>)
+		writer.close()
+
+		const reader = decompressor.readable.getReader()
+		const { value } = await reader.read()
+
+		expect(value).toBeDefined()
+		if (!value) throw new Error("No value read")
+
+		// Verify it's a Uint8Array
+		expect(value).toBeInstanceOf(Uint8Array)
+		// Verify the buffer is an ArrayBuffer (not Buffer or SharedArrayBuffer)
+		expect(value.buffer).toBeInstanceOf(ArrayBuffer)
+		// Verify it's not a Node.js Buffer
+		expect(value.constructor.name).toBe("Uint8Array")
+	})
+
+	test("handles chunked compressed data", async () => {
+		const input = new TextEncoder().encode(
+			"test chunked data",
+		) as Uint8Array<ArrayBuffer>
+		const compressed = await bunCompress(input)
+
+		const decompressor = new DecompressionStream("deflate")
+		const writer = decompressor.writable.getWriter()
+
+		// Write compressed data in chunks
+		const chunkSize = 5
+		for (let i = 0; i < compressed.length; i += chunkSize) {
+			const chunk = compressed.slice(i, i + chunkSize)
+			writer.write(chunk)
+		}
+		writer.close()
+
+		const chunks: Uint8Array[] = []
+		const reader = decompressor.readable.getReader()
+		while (true) {
+			const { done, value } = await reader.read()
+			if (done) break
+			chunks.push(value)
+		}
+
+		const decompressed = concatUint8(...chunks)
+		expect(decompressed).toEqual(input)
+	})
+
+	test("round-trip compression and decompression", async () => {
+		const input = new TextEncoder().encode("round trip test data")
+
+		// Compress
+		const compressor = new CompressionStream("deflate")
+		const compressorWriter = compressor.writable.getWriter()
+		compressorWriter.write(input)
+		compressorWriter.close()
+
+		const compressedChunks: Uint8Array[] = []
+		const compressorReader = compressor.readable.getReader()
+		while (true) {
+			const { done, value } = await compressorReader.read()
+			if (done) break
+			compressedChunks.push(value)
+		}
+		const compressed = concatUint8(...compressedChunks)
+
+		// Decompress
+		const decompressor = new DecompressionStream("deflate")
+		const decompressorWriter = decompressor.writable.getWriter()
+		decompressorWriter.write(new Uint8Array(compressed))
+		decompressorWriter.close()
+
+		const decompressedChunks: Uint8Array[] = []
+		const decompressorReader = decompressor.readable.getReader()
+		while (true) {
+			const { done, value } = await decompressorReader.read()
+			if (done) break
+			decompressedChunks.push(value)
+		}
+		const decompressed = concatUint8(...decompressedChunks)
+
+		expect(decompressed).toEqual(input)
+	})
+})