@osmix/pbf 0.0.1 → 0.0.3

package/dist/blobs-to-blocks.js

@@ -1,21 +0,0 @@
-import Pbf from "pbf";
-import { readHeaderBlock, readPrimitiveBlock } from "./proto/osmformat";
-import { decompress } from "./utils";
-/**
- * Decompresses raw OSM PBF blobs and yields typed header and primitive blocks.
- * Expects the first blob to contain the file header and streams the rest as data blocks.
- */
-export async function* osmPbfBlobsToBlocksGenerator(blobs) {
-    let headerRead = false;
-    for await (const blob of blobs) {
-        const decompressedBlob = await decompress(blob);
-        const pbf = new Pbf(decompressedBlob);
-        if (!headerRead) {
-            headerRead = true;
-            yield readHeaderBlock(pbf);
-        }
-        else {
-            yield readPrimitiveBlock(pbf);
-        }
-    }
-}

package/dist/blocks-to-pbf.d.ts

@@ -1,16 +0,0 @@
-import type { OsmPbfBlock, OsmPbfHeaderBlock } from "./proto/osmformat";
-/**
- * Serializes a header or primitive block into a spec-compliant compressed blob.
- * Automatically sets the blob type, compresses the payload, and enforces size guardrails.
- * @param block - Parsed header or primitive block to encode.
- * @returns BlobHeader length prefix + Blob bytes as a single Uint8Array.
- */
-export declare function osmBlockToPbfBlobBytes(block: OsmPbfBlock | OsmPbfHeaderBlock): Promise<Uint8Array<ArrayBufferLike>>;
-/**
- * Web `TransformStream` that converts OSM header/data blocks into PBF byte chunks.
- * Throws if the header is not the first block and reuses `osmBlockToPbfBlobBytes` for encoding.
- */
-export declare class OsmBlocksToPbfBytesTransformStream extends TransformStream<OsmPbfHeaderBlock | OsmPbfBlock, Uint8Array> {
-    headerEnqueued: boolean;
-    constructor();
-}

package/dist/index.d.ts

@@ -1,8 +0,0 @@
-export * from "./blobs-to-blocks";
-export * from "./blocks-to-pbf";
-export * from "./pbf-to-blobs";
-export * from "./pbf-to-blocks";
-export * from "./proto/fileformat";
-export * from "./proto/osmformat";
-export * from "./spec";
-export * from "./utils";

package/dist/index.js

@@ -1,8 +0,0 @@
-export * from "./blobs-to-blocks";
-export * from "./blocks-to-pbf";
-export * from "./pbf-to-blobs";
-export * from "./pbf-to-blocks";
-export * from "./proto/fileformat";
-export * from "./proto/osmformat";
-export * from "./spec";
-export * from "./utils";

package/dist/pbf-to-blobs.d.ts

@@ -1,6 +0,0 @@
-/**
- * Creates a stateful parser that slices incoming bytes into compressed OSM PBF blobs.
- * Works with buffers, iterables, or streams and yields `Uint8Array` payloads ready to decompress.
- * The first blob represents the file header; subsequent blobs hold primitive data.
- */
-export declare function createOsmPbfBlobGenerator(): (chunk: Uint8Array) => Generator<Uint8Array<ArrayBuffer>, void, unknown>;

package/dist/pbf-to-blocks.d.ts

@@ -1,20 +0,0 @@
-import { type OsmPbfBlock, type OsmPbfHeaderBlock } from "./proto/osmformat";
-import { type AsyncGeneratorValue } from "./utils";
-export declare const HEADER_LENGTH_BYTES = 4;
-/**
- * Parses OSM PBF bytes from buffers, streams, or generators into header + block iterators.
- * Returns the decoded header and a lazy async generator of primitive blocks.
- */
-export declare function readOsmPbf(data: AsyncGeneratorValue<ArrayBufferLike>): Promise<{
-    header: OsmPbfHeaderBlock;
-    blocks: AsyncGenerator<OsmPbfBlock>;
-}>;
-/**
- * Web `TransformStream` that turns raw PBF byte chunks into OSM header/data blocks.
- * Assumes the first decoded blob carries the header and emits it before any primitive blocks.
- */
-export declare class OsmPbfBytesToBlocksTransformStream extends TransformStream<ArrayBufferLike, OsmPbfHeaderBlock | OsmPbfBlock> {
-    generateBlobsFromChunk: (chunk: Uint8Array) => Generator<Uint8Array<ArrayBuffer>, void, unknown>;
-    header: OsmPbfHeaderBlock | null;
-    constructor();
-}

package/dist/pbf-to-blocks.js

@@ -1,53 +0,0 @@
-import Pbf from "pbf";
-import { osmPbfBlobsToBlocksGenerator } from "./blobs-to-blocks";
-import { createOsmPbfBlobGenerator } from "./pbf-to-blobs";
-import { readHeaderBlock, readPrimitiveBlock, } from "./proto/osmformat";
-import { decompress, toAsyncGenerator } from "./utils";
-export const HEADER_LENGTH_BYTES = 4;
-/**
- * Parses OSM PBF bytes from buffers, streams, or generators into header + block iterators.
- * Returns the decoded header and a lazy async generator of primitive blocks.
- */
-export async function readOsmPbf(data) {
-    const generateBlobsFromChunk = createOsmPbfBlobGenerator();
-    const blocks = osmPbfBlobsToBlocksGenerator((async function* () {
-        for await (const chunk of toAsyncGenerator(data)) {
-            for await (const blob of generateBlobsFromChunk(new Uint8Array(chunk))) {
-                yield blob;
-            }
-        }
-    })());
-    const header = (await blocks.next()).value;
-    if (header == null || !("required_features" in header)) {
-        throw Error("OSM PBF header block not found");
-    }
-    return {
-        header,
-        blocks: blocks,
-    };
-}
-/**
- * Web `TransformStream` that turns raw PBF byte chunks into OSM header/data blocks.
- * Assumes the first decoded blob carries the header and emits it before any primitive blocks.
- */
-export class OsmPbfBytesToBlocksTransformStream extends TransformStream {
-    generateBlobsFromChunk = createOsmPbfBlobGenerator();
-    header = null;
-    constructor() {
-        super({
-            transform: async (bytesChunk, controller) => {
-                for await (const rawBlobs of this.generateBlobsFromChunk(new Uint8Array(bytesChunk))) {
-                    const decompressed = await decompress(rawBlobs);
-                    const pbf = new Pbf(decompressed);
-                    if (this.header == null) {
-                        this.header = readHeaderBlock(pbf);
-                        controller.enqueue(this.header);
-                    }
-                    else {
-                        controller.enqueue(readPrimitiveBlock(pbf));
-                    }
-                }
-            },
-        });
-    }
-}

package/test/utils.bun.test.ts

@@ -1,327 +0,0 @@
-import { describe, expect, test } from "vitest"
-import { concatUint8, isBun, toAsyncGenerator, uint32BE } from "../src/utils"
-
-/**
- * Bun decompress.
- */
-async function bunDecompress(
-	data: Uint8Array<ArrayBuffer>,
-): Promise<Uint8Array<ArrayBuffer>> {
-	const { inflate } = await import("node:zlib")
-	const result = await new Promise<ArrayBuffer>((resolve, reject) => {
-		inflate(data, (error, result) => {
-			if (error) reject(error)
-			else resolve(result.buffer as ArrayBuffer)
-		})
-	})
-	return new Uint8Array(result)
-}
-
-/**
- * Bun compression.
- */
-export async function bunCompress(
-	data: Uint8Array<ArrayBuffer>,
-): Promise<Uint8Array<ArrayBuffer>> {
-	const { deflate } = await import("node:zlib")
-	const result = await new Promise<ArrayBuffer>((resolve, reject) => {
-		deflate(data, (error, result) => {
-			if (error) reject(error)
-			else resolve(result.buffer as ArrayBuffer)
-		})
-	})
-	return new Uint8Array(result)
-}
-
-describe.runIf(isBun())("utils", () => {
-	test("wraps values into an async generator", async () => {
-		const generator = toAsyncGenerator(3)
-		const first = await generator.next()
-		expect(first).toEqual({ value: 3, done: false })
-		const done = await generator.next()
-		expect(done).toEqual({ value: undefined, done: true })
-	})
-
-	test("consumes readable streams", async () => {
-		const stream = new ReadableStream<number>({
-			start(controller) {
-				controller.enqueue(1)
-				controller.enqueue(2)
-				controller.close()
-			},
-		})
-		const values: number[] = []
-		for await (const value of toAsyncGenerator(stream)) values.push(value)
-		expect(values).toEqual([1, 2])
-	})
-
-	test("throws on nullish inputs", async () => {
-		const invalidInput = null as unknown as never
-		await expect(toAsyncGenerator(invalidInput).next()).rejects.toThrow(
-			"Value is null",
-		)
-	})
-
-	test("concatenates Uint8Array segments", () => {
-		const a = Uint8Array.of(1, 2)
-		const b = Uint8Array.of(3)
-		expect(concatUint8(a, b)).toEqual(Uint8Array.of(1, 2, 3))
-	})
-
-	test("encodes big-endian 32-bit integers", () => {
-		expect(uint32BE(0x01020304)).toEqual(Uint8Array.of(1, 2, 3, 4))
-	})
-
-	test("compresses and decompresses data with deflate", async () => {
-		const input = new TextEncoder().encode("osmix") as Uint8Array<ArrayBuffer>
-		const compressed = await bunCompress(input)
-		expect(compressed).not.toEqual(input)
-		const decompressed = await bunDecompress(compressed)
-		expect(decompressed).toEqual(input)
-	})
-
-	test("compresses and decompresses larger data", async () => {
-		const input = new TextEncoder().encode(
-			"a".repeat(1000),
-		) as Uint8Array<ArrayBuffer>
-		const compressed = await bunCompress(input)
-		expect(compressed.length).toBeLessThan(input.length)
-		const decompressed = await bunDecompress(compressed)
-		expect(decompressed).toEqual(input)
-	})
-
-	test("handles Uint8Array input", async () => {
-		const input = Uint8Array.of(1, 2, 3, 4, 5)
-		const compressed = await bunCompress(input)
-		const decompressed = await bunDecompress(compressed)
-		expect(decompressed).toEqual(input)
-	})
-
-	test("uses Bun runtime with Node.js zlib compatibility", () => {
-		// This test verifies that Bun is available in the runtime
-		expect(isBun()).toBe(true)
-	})
-
-	test("Node.js zlib methods work in Bun", async () => {
-		const { deflateSync, inflateSync } = await import("node:zlib")
-		const input = new TextEncoder().encode("test bun compression with zlib")
-		const compressed = deflateSync(input)
-		expect(compressed.length).toBeGreaterThan(0)
-		expect(compressed).not.toEqual(input)
-
-		const decompressed = inflateSync(compressed)
-		expect(new Uint8Array(decompressed)).toEqual(input)
-	})
-
-	test("compress/decompress are compatible with OSM PBF zlib format", async () => {
-		// Test that our compress/decompress functions produce zlib-compatible data
-		// This is critical for OSM PBF compatibility
-		const { deflateSync, inflateSync } = await import("node:zlib")
-		const input = new TextEncoder().encode(
-			"OSM PBF uses zlib format (deflate with headers)",
-		) as Uint8Array<ArrayBuffer>
-
-		// Compress with our function
-		const ourCompressed = await bunCompress(input)
-
-		// Decompress with Node.js zlib (what OSM PBF uses)
-		const decompressedWithNodeZlib = inflateSync(ourCompressed)
-		expect(new Uint8Array(decompressedWithNodeZlib)).toEqual(input)
-
-		// Compress with Node.js zlib
-		const nodeCompressed = deflateSync(input)
-
-		// Decompress with our function
-		const decompressedWithOurs = await bunDecompress(
-			new Uint8Array(nodeCompressed),
-		)
-		expect(decompressedWithOurs).toEqual(input)
-	})
-})
-
-describe.skip("CompressionStream polyfill", () => {
-	test("compresses data using deflate format", async () => {
-		const input = new TextEncoder().encode("test compression stream")
-		const compressor = new CompressionStream("deflate")
-
-		const writer = compressor.writable.getWriter()
-		writer.write(input)
-		writer.close()
-
-		const chunks: Uint8Array[] = []
-		const reader = compressor.readable.getReader()
-		while (true) {
-			const { done, value } = await reader.read()
-			if (done) break
-			chunks.push(value)
-		}
-
-		const compressed = concatUint8(...chunks)
-		expect(compressed.length).toBeGreaterThan(0)
-		expect(compressed).not.toEqual(input)
-
-		// Verify it's valid deflate data by decompressing
-		const decompressed = await bunDecompress(new Uint8Array(compressed))
-		expect(decompressed).toEqual(input)
-	})
-
-	test("returns proper Uint8Array<ArrayBuffer> instances", async () => {
-		const input = new TextEncoder().encode("type safety check")
-		const compressor = new CompressionStream("deflate")
-
-		const writer = compressor.writable.getWriter()
-		writer.write(input)
-		writer.close()
-
-		const reader = compressor.readable.getReader()
-		const { value } = await reader.read()
-
-		expect(value).toBeDefined()
-		if (!value) throw new Error("No value read")
-
-		// Verify it's a Uint8Array
-		expect(value).toBeInstanceOf(Uint8Array)
-		// Verify the buffer is an ArrayBuffer (not Buffer or SharedArrayBuffer)
-		expect(value.buffer).toBeInstanceOf(ArrayBuffer)
-		// Verify it's not a Node.js Buffer
-		expect(value.constructor.name).toBe("Uint8Array")
-	})
-
-	test("handles multiple writes", async () => {
-		const compressor = new CompressionStream("deflate")
-		const writer = compressor.writable.getWriter()
-
-		// Write multiple chunks
-		writer.write(new TextEncoder().encode("first "))
-		writer.write(new TextEncoder().encode("second "))
-		writer.write(new TextEncoder().encode("third"))
-		writer.close()
-
-		const chunks: Uint8Array[] = []
-		const reader = compressor.readable.getReader()
-		while (true) {
-			const { done, value } = await reader.read()
-			if (done) break
-			chunks.push(value)
-		}
-
-		const compressed = concatUint8(...chunks)
-		const decompressed = await bunDecompress(new Uint8Array(compressed))
-		expect(new TextDecoder().decode(decompressed)).toBe("first second third")
-	})
-})
-
-describe.skip("DecompressionStream polyfill", () => {
-	test("decompresses deflate data", async () => {
-		const input = new TextEncoder().encode(
-			"test decompression stream",
-		) as Uint8Array<ArrayBuffer>
-		const compressed = await bunCompress(input)
-
-		const decompressor = new DecompressionStream("deflate")
-		const writer = decompressor.writable.getWriter()
-		writer.write(compressed as Uint8Array<ArrayBuffer>)
-		writer.close()
-
-		const chunks: Uint8Array[] = []
-		const reader = decompressor.readable.getReader()
-		while (true) {
-			const { done, value } = await reader.read()
-			if (done) break
-			chunks.push(value)
-		}
-
-		const decompressed = concatUint8(...chunks)
-		expect(decompressed).toEqual(input)
-	})
-
-	test("returns proper Uint8Array<ArrayBuffer> instances", async () => {
-		const input = new TextEncoder().encode(
-			"type safety check",
-		) as Uint8Array<ArrayBuffer>
-		const compressed = await bunCompress(input)
-
-		const decompressor = new DecompressionStream("deflate")
-		const writer = decompressor.writable.getWriter()
-		writer.write(compressed as Uint8Array<ArrayBuffer>)
-		writer.close()
-
-		const reader = decompressor.readable.getReader()
-		const { value } = await reader.read()
-
-		expect(value).toBeDefined()
-		if (!value) throw new Error("No value read")
-
-		// Verify it's a Uint8Array
-		expect(value).toBeInstanceOf(Uint8Array)
-		// Verify the buffer is an ArrayBuffer (not Buffer or SharedArrayBuffer)
-		expect(value.buffer).toBeInstanceOf(ArrayBuffer)
-		// Verify it's not a Node.js Buffer
-		expect(value.constructor.name).toBe("Uint8Array")
-	})
-
-	test("handles chunked compressed data", async () => {
-		const input = new TextEncoder().encode(
-			"test chunked data",
-		) as Uint8Array<ArrayBuffer>
-		const compressed = await bunCompress(input)
-
-		const decompressor = new DecompressionStream("deflate")
-		const writer = decompressor.writable.getWriter()
-
-		// Write compressed data in chunks
-		const chunkSize = 5
-		for (let i = 0; i < compressed.length; i += chunkSize) {
-			const chunk = compressed.slice(i, i + chunkSize)
-			writer.write(chunk)
-		}
-		writer.close()
-
-		const chunks: Uint8Array[] = []
-		const reader = decompressor.readable.getReader()
-		while (true) {
-			const { done, value } = await reader.read()
-			if (done) break
-			chunks.push(value)
-		}
-
-		const decompressed = concatUint8(...chunks)
-		expect(decompressed).toEqual(input)
-	})
-
-	test("round-trip compression and decompression", async () => {
-		const input = new TextEncoder().encode("round trip test data")
-
-		// Compress
-		const compressor = new CompressionStream("deflate")
-		const compressorWriter = compressor.writable.getWriter()
-		compressorWriter.write(input)
-		compressorWriter.close()
-
-		const compressedChunks: Uint8Array[] = []
-		const compressorReader = compressor.readable.getReader()
-		while (true) {
-			const { done, value } = await compressorReader.read()
-			if (done) break
-			compressedChunks.push(value)
-		}
-		const compressed = concatUint8(...compressedChunks)
-
-		// Decompress
-		const decompressor = new DecompressionStream("deflate")
-		const decompressorWriter = decompressor.writable.getWriter()
-		decompressorWriter.write(new Uint8Array(compressed))
-		decompressorWriter.close()
-
-		const decompressedChunks: Uint8Array[] = []
-		const decompressorReader = decompressor.readable.getReader()
-		while (true) {
-			const { done, value } = await decompressorReader.read()
-			if (done) break
-			decompressedChunks.push(value)
-		}
-		const decompressed = concatUint8(...decompressedChunks)
-
-		expect(decompressed).toEqual(input)
-	})
-})

package/test/verify-pbf-reading.bun.test.ts

@@ -1,39 +0,0 @@
-import { getFixtureFile } from "@osmix/shared/test/fixtures"
-import { describe, expect, test } from "vitest"
-import { readOsmPbf } from "../src/pbf-to-blocks"
-import { isBun } from "../src/utils"
-
-describe.runIf(isBun())("PBF Reading with Bun", () => {
-	test("can read an actual OSM PBF file", async () => {
-		// Load a real PBF fixture
-		const pbfData = await getFixtureFile("monaco.pbf")
-
-		// Try to read it
-		const result = await readOsmPbf(pbfData)
-
-		// Basic sanity checks
-		expect(result.header).toBeDefined()
-		expect(result.header.required_features).toBeDefined()
-		expect(result.blocks).toBeDefined()
-
-		// Try to read at least one block
-		const firstBlock = await result.blocks.next()
-		expect(firstBlock.done).toBe(false)
-		expect(firstBlock.value).toBeDefined()
-
-		// If it's a primitive block, it should have primitivegroup
-		if ("primitivegroup" in firstBlock.value) {
-			expect(Array.isArray(firstBlock.value.primitivegroup)).toBe(true)
-		}
-	})
-
-	test("header contains expected OSM data", async () => {
-		const pbfData = await getFixtureFile("monaco.pbf")
-
-		const result = await readOsmPbf(pbfData)
-
-		// Check that header has the required OSM features
-		expect(result.header.required_features).toBeDefined()
-		expect(result.header.required_features.length).toBeGreaterThan(0)
-	})
-})

package/vitest.config.ts

@@ -1,7 +0,0 @@
-import { defineConfig } from "vitest/config"
-
-export default defineConfig({
-	test: {
-		// exclude: ["**/node_modules/**", "**/*.bun.test.ts"],
-	},
-})