@osmix/pbf 0.0.1

package/src/blocks-to-pbf.ts

@@ -0,0 +1,98 @@
+import Pbf from "pbf"
+import { writeBlob, writeBlobHeader } from "./proto/fileformat"
+import type { OsmPbfBlock, OsmPbfHeaderBlock } from "./proto/osmformat"
+import { writeHeaderBlock, writePrimitiveBlock } from "./proto/osmformat"
+import {
+	MAX_BLOB_SIZE_BYTES,
+	MAX_HEADER_SIZE_BYTES,
+	RECOMMENDED_BLOB_SIZE_BYTES,
+	RECOMMENDED_HEADER_SIZE_BYTES,
+} from "./spec"
+import { concatUint8, uint32BE, webCompress } from "./utils"
+
+/**
+ * Serializes a header or primitive block into a spec-compliant compressed blob.
+ * Automatically sets the blob type, compresses the payload, and enforces size guardrails.
+ * @param block - Parsed header or primitive block to encode.
+ * @returns BlobHeader length prefix + Blob bytes as a single Uint8Array.
+ */
+export async function osmBlockToPbfBlobBytes(
+	block: OsmPbfBlock | OsmPbfHeaderBlock,
+	compress: (
+		data: Uint8Array<ArrayBuffer>,
+	) => Promise<Uint8Array<ArrayBuffer>> = webCompress,
+) {
+	const contentPbf = new Pbf()
+	let type: "OSMHeader" | "OSMData"
+	if ("primitivegroup" in block) {
+		type = "OSMData"
+		writePrimitiveBlock(block, contentPbf)
+	} else {
+		type = "OSMHeader"
+		writeHeaderBlock(block, contentPbf)
+	}
+	const contentData = contentPbf.finish() as Uint8Array<ArrayBuffer>
+	const raw_size = contentData.length
+	const compressedBuffer = await compress(contentData)
+
+	const blobPbf = new Pbf()
+	writeBlob(
+		{
+			raw_size,
+			zlib_data: compressedBuffer,
+		},
+		blobPbf,
+	)
+	const blob = blobPbf.finish()
+
+	const blobHeaderPbf = new Pbf()
+	writeBlobHeader(
+		{
+			type,
+			datasize: blob.length,
+		},
+		blobHeaderPbf,
+	)
+	const blobHeader = blobHeaderPbf.finish()
+	const blobHeaderSize = uint32BE(blobHeader.byteLength)
+
+	// Check the BlobHeader and Blob sizes, log error if over the recommended size, throw error if over the maximum size
+	if (blobHeader.byteLength > RECOMMENDED_HEADER_SIZE_BYTES) {
+		const sizeKiB = (blobHeader.byteLength / 1024).toFixed(2)
+		if (blobHeader.byteLength > MAX_HEADER_SIZE_BYTES) {
+			throw new Error(`BlobHeader is ${sizeKiB} KiB, the maximum size is 64KiB`)
+		}
+		console.warn(`BlobHeader is ${sizeKiB} KiB, the recommended size is 32KiB`)
+	}
+	if (blob.byteLength > RECOMMENDED_BLOB_SIZE_BYTES) {
+		const sizeMiB = (blob.byteLength / 1024 / 1024).toFixed(2)
+		if (blob.byteLength > MAX_BLOB_SIZE_BYTES) {
+			throw new Error(`Blob is ${sizeMiB} MiB, the maximum size is 32MiB`)
+		}
+		console.warn(`Blob is ${sizeMiB} MiB, the recommended size is 16MiB`)
+	}
+
+	return concatUint8(blobHeaderSize, blobHeader, blob)
+}
+
+/**
+ * Web `TransformStream` that converts OSM header/data blocks into PBF byte chunks.
+ * Throws if the header is not the first block and reuses `osmBlockToPbfBlobBytes` for encoding.
+ */
+export class OsmBlocksToPbfBytesTransformStream extends TransformStream<
+	OsmPbfHeaderBlock | OsmPbfBlock,
+	Uint8Array
+> {
+	headerEnqueued = false
+	constructor() {
+		super({
+			transform: async (block, controller) => {
+				if ("primitivegroup" in block && !this.headerEnqueued) {
+					throw Error("Header first in ReadableStream of blocks.")
+				}
+				this.headerEnqueued = true
+				controller.enqueue(await osmBlockToPbfBlobBytes(block))
+			},
+		})
+	}
+}

package/src/index.ts

@@ -0,0 +1,8 @@
+export * from "./blobs-to-blocks"
+export * from "./blocks-to-pbf"
+export * from "./pbf-to-blobs"
+export * from "./pbf-to-blocks"
+export * from "./proto/fileformat"
+export * from "./proto/osmformat"
+export * from "./spec"
+export * from "./utils"

package/src/pbf-to-blobs.ts

@@ -0,0 +1,56 @@
+import Pbf from "pbf"
+import { HEADER_LENGTH_BYTES } from "./pbf-to-blocks"
+import {
+	type OsmPbfBlobHeader,
+	readBlob,
+	readBlobHeader,
+} from "./proto/fileformat"
+
+/**
+ * Creates a stateful parser that slices incoming bytes into compressed OSM PBF blobs.
+ * Works with buffers, iterables, or streams and yields `Uint8Array` payloads ready to decompress.
+ * The first blob represents the file header; subsequent blobs hold primitive data.
+ */
+export function createOsmPbfBlobGenerator() {
+	let pbf: Pbf = new Pbf(new Uint8Array(0))
+	let state: "header-length" | "header" | "blob" = "header-length"
+	let bytesNeeded: number = HEADER_LENGTH_BYTES
+	let blobHeader: OsmPbfBlobHeader | null = null
+
+	/**
+	 * Feed the parser with the next chunk of bytes and yield any complete compressed blobs.
+	 */
+	return function* nextChunk(chunk: Uint8Array) {
+		const currentBuffer: Uint8Array = pbf.buf.slice(pbf.pos)
+		const tmpBuffer = new Uint8Array(
+			currentBuffer.buffer.byteLength + chunk.byteLength,
+		)
+		tmpBuffer.set(currentBuffer.subarray(0))
+		tmpBuffer.set(new Uint8Array(chunk), currentBuffer.byteLength)
+		pbf = new Pbf(tmpBuffer)
+
+		while (pbf.pos + bytesNeeded <= pbf.length) {
+			if (state === "header-length") {
+				const dataView = new DataView(pbf.buf.buffer)
+				bytesNeeded = dataView.getInt32(pbf.pos, false) // network byte order
+				pbf.pos += HEADER_LENGTH_BYTES
+				state = "header"
+			} else if (state === "header") {
+				blobHeader = readBlobHeader(pbf, pbf.pos + bytesNeeded)
+				bytesNeeded = blobHeader.datasize
+				state = "blob"
+			} else if (state === "blob") {
+				if (blobHeader == null) throw Error("Blob header has not been read")
+				const blob = readBlob(pbf, pbf.pos + bytesNeeded)
+				if (blob.zlib_data === undefined || blob.zlib_data.length === 0)
+					throw Error("Blob has no zlib data. Format is unsupported.")
+
+				yield blob.zlib_data as Uint8Array<ArrayBuffer>
+
+				state = "header-length"
+				bytesNeeded = HEADER_LENGTH_BYTES
+				blobHeader = null
+			}
+		}
+	}
+}

package/src/pbf-to-blocks.ts

@@ -0,0 +1,77 @@
+import Pbf from "pbf"
+import { osmPbfBlobsToBlocksGenerator } from "./blobs-to-blocks"
+import { createOsmPbfBlobGenerator } from "./pbf-to-blobs"
+import {
+	type OsmPbfBlock,
+	type OsmPbfHeaderBlock,
+	readHeaderBlock,
+	readPrimitiveBlock,
+} from "./proto/osmformat"
+import {
+	type AsyncGeneratorValue,
+	toAsyncGenerator,
+	webDecompress,
+} from "./utils"
+
+export const HEADER_LENGTH_BYTES = 4
+
+/**
+ * Parses OSM PBF bytes from buffers, streams, or generators into header + block iterators.
+ * Returns the decoded header and a lazy async generator of primitive blocks.
+ */
+export async function readOsmPbf(data: AsyncGeneratorValue<ArrayBufferLike>) {
+	const generateBlobsFromChunk = createOsmPbfBlobGenerator()
+	const blocks = osmPbfBlobsToBlocksGenerator(
+		(async function* () {
+			for await (const chunk of toAsyncGenerator(data)) {
+				for await (const blob of generateBlobsFromChunk(
+					new Uint8Array(chunk),
+				)) {
+					yield blob
+				}
+			}
+		})(),
+	)
+	const header = (await blocks.next()).value
+	if (header == null || !("required_features" in header)) {
+		throw Error("OSM PBF header block not found")
+	}
+	return {
+		header,
+		blocks: blocks as AsyncGenerator<OsmPbfBlock>,
+	}
+}
+
+/**
+ * Web `TransformStream` that turns raw PBF byte chunks into OSM header/data blocks.
+ * Assumes the first decoded blob carries the header and emits it before any primitive blocks.
+ */
+export class OsmPbfBytesToBlocksTransformStream extends TransformStream<
+	ArrayBufferLike,
+	OsmPbfHeaderBlock | OsmPbfBlock
+> {
+	generateBlobsFromChunk = createOsmPbfBlobGenerator()
+	header: OsmPbfHeaderBlock | null = null
+	constructor(
+		decompress: (
+			data: Uint8Array<ArrayBuffer>,
+		) => Promise<Uint8Array<ArrayBuffer>> = webDecompress,
+	) {
+		super({
+			transform: async (bytesChunk, controller) => {
+				for await (const rawBlobs of this.generateBlobsFromChunk(
+					new Uint8Array(bytesChunk),
+				)) {
+					const decompressed = await decompress(rawBlobs)
+					const pbf = new Pbf(decompressed)
+					if (this.header == null) {
+						this.header = readHeaderBlock(pbf)
+						controller.enqueue(this.header)
+					} else {
+						controller.enqueue(readPrimitiveBlock(pbf))
+					}
+				}
+			},
+		})
+	}
+}

package/src/proto/fileformat.proto

@@ -0,0 +1,68 @@
+/** Copyright (c) 2010 Scott A. Crosby. <scott@sacrosby.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+syntax = "proto2";
+
+option java_package = "crosby.binary";
+package OSMPBF;
+
+//protoc --java_out=../.. fileformat.proto
+
+
+//
+//  STORAGE LAYER: Storing primitives.
+//
+
+message Blob {
+  optional int32 raw_size = 2; // When compressed, the uncompressed size
+
+  oneof data {
+    bytes raw = 1; // No compression
+
+    // Possible compressed versions of the data.
+    bytes zlib_data = 3;
+
+    // For LZMA compressed data (optional)
+    bytes lzma_data = 4;
+
+    // Formerly used for bzip2 compressed data. Deprecated in 2010.
+    bytes OBSOLETE_bzip2_data = 5 [deprecated=true]; // Don't reuse this tag number.
+
+    // For LZ4 compressed data (optional)
+    bytes lz4_data = 6;
+
+    // For ZSTD compressed data (optional)
+    bytes zstd_data = 7;
+  }
+}
+
+/* A file contains an sequence of fileblock headers, each prefixed by
+their length in network byte order, followed by a data block
+containing the actual data. Types starting with a "_" are reserved.
+*/
+
+message BlobHeader {
+  required string type = 1;
+  optional bytes indexdata = 2;
+  required int32 datasize = 3;
+}
+

package/src/proto/fileformat.ts

@@ -0,0 +1,70 @@
+// code generated by @mapbox/pbf v4.0.1 and types merged in by hand
+import type Pbf from "pbf"
+
+export type OsmPbfBlob = {
+	raw_size?: number
+	raw?: Uint8Array
+	zlib_data?: Uint8Array
+}
+
+export type OsmPbfBlobHeader = {
+	type: "OSMHeader" | "OSMData"
+	datasize: number
+}
+
+/**
+ * Reads an `OsmPbfBlob` message from the current position in the Pbf reader.
+ */
+export function readBlob(pbf: Pbf, end?: number): OsmPbfBlob {
+	return pbf.readFields(
+		readBlobField,
+		{
+			raw_size: 0,
+		},
+		end,
+	)
+}
+/**
+ * Dispatches individual Blob fields based on their protobuf tag.
+ */
+function readBlobField(tag: number, obj: OsmPbfBlob, pbf: Pbf) {
+	if (tag === 2) obj.raw_size = pbf.readVarint(true)
+	else if (tag === 1) {
+		obj.raw = pbf.readBytes()
+	} else if (tag === 3) {
+		obj.zlib_data = pbf.readBytes()
+	}
+}
+/**
+ * Writes an `OsmPbfBlob` message to the provided Pbf writer.
+ */
+export function writeBlob(obj: OsmPbfBlob, pbf: Pbf) {
+	if (obj.raw_size) pbf.writeVarintField(2, obj.raw_size)
+	if (obj.raw != null) pbf.writeBytesField(1, obj.raw)
+	if (obj.zlib_data != null) pbf.writeBytesField(3, obj.zlib_data)
+}
+
+/**
+ * Reads an `OsmPbfBlobHeader` from the current position in the Pbf reader.
+ */
+export function readBlobHeader(pbf: Pbf, end?: number): OsmPbfBlobHeader {
+	return pbf.readFields(
+		readBlobHeaderField,
+		{ type: "OSMData", datasize: 0 },
+		end,
+	)
+}
+/**
+ * Dispatches individual BlobHeader fields based on their protobuf tag.
+ */
+function readBlobHeaderField(tag: number, obj: OsmPbfBlobHeader, pbf: Pbf) {
+	if (tag === 1) obj.type = pbf.readString() as OsmPbfBlobHeader["type"]
+	else if (tag === 3) obj.datasize = pbf.readVarint(true)
+}
+/**
+ * Writes an `OsmPbfBlobHeader` message to the provided Pbf writer.
+ */
+export function writeBlobHeader(obj: OsmPbfBlobHeader, pbf: Pbf) {
+	if (obj.type) pbf.writeStringField(1, obj.type)
+	if (obj.datasize) pbf.writeVarintField(3, obj.datasize)
+}

package/src/proto/osmformat.proto

@@ -0,0 +1,262 @@
+/** Copyright (c) 2010 Scott A. Crosby. <scott@sacrosby.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+syntax = "proto2";
+
+option java_package = "crosby.binary";
+package OSMPBF;
+
+/* OSM Binary file format
+
+This is the master schema file of the OSM binary file format. This
+file is designed to support limited random-access and future
+extendability.
+
+A binary OSM file consists of a sequence of FileBlocks (please see
+fileformat.proto). The first fileblock contains a serialized instance
+of HeaderBlock, followed by a sequence of PrimitiveBlock blocks that
+contain the primitives.
+
+Each primitiveblock is designed to be independently parsable. It
+contains a string table storing all strings in that block (keys and
+values in tags, roles in relations, usernames, etc.) as well as
+metadata containing the precision of coordinates or timestamps in that
+block.
+
+A primitiveblock contains a sequence of primitive groups, each
+containing primitives of the same type (nodes, densenodes, ways,
+relations). Coordinates are stored in signed 64-bit integers. Lat&lon
+are measured in units <granularity> nanodegrees. The default of
+granularity of 100 nanodegrees corresponds to about 1cm on the ground,
+and a full lat or lon fits into 32 bits.
+
+Converting an integer to a latitude or longitude uses the formula:
+$OUT = IN * granularity / 10**9$. Many encoding schemes use delta
+coding when representing nodes and relations.
+
+*/
+
+//////////////////////////////////////////////////////////////////////////
+//////////////////////////////////////////////////////////////////////////
+
+/* Contains the file header. */
+
+message HeaderBlock {
+  optional HeaderBBox bbox = 1;
+
+  /* Additional tags to aid in parsing this dataset */
+  repeated string required_features = 4;
+  repeated string optional_features = 5;
+
+  optional string writingprogram = 16;
+  optional string source = 17; // From the bbox field.
+
+  /* Tags that allow continuing an Osmosis replication */
+
+  // Replication timestamp, expressed in seconds since the epoch,
+  // otherwise the same value as in the "timestamp=..." field
+  // in the state.txt file used by Osmosis.
+  optional int64 osmosis_replication_timestamp = 32;
+
+  // Replication sequence number (sequenceNumber in state.txt).
+  optional int64 osmosis_replication_sequence_number = 33;
+
+  // Replication base URL (from Osmosis' configuration.txt file).
+  optional string osmosis_replication_base_url = 34;
+}
+
+
+/** The bounding box field in the OSM header. BBOX, as used in the OSM
+header. Units are always in nanodegrees -- they do not obey
+granularity rules. */
+
+message HeaderBBox {
+  required sint64 left = 1;
+  required sint64 right = 2;
+  required sint64 top = 3;
+  required sint64 bottom = 4;
+}
+
+
+///////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////
+
+
+message PrimitiveBlock {
+  required StringTable stringtable = 1;
+  repeated PrimitiveGroup primitivegroup = 2;
+
+  // Granularity, units of nanodegrees, used to store coordinates in this block.
+  optional int32 granularity = 17 [default=100];
+
+  // Offset value between the output coordinates and the granularity grid in units of nanodegrees.
+  optional int64 lat_offset = 19 [default=0];
+  optional int64 lon_offset = 20 [default=0];
+
+  // Granularity of dates, normally represented in units of milliseconds since the 1970 epoch.
+  optional int32 date_granularity = 18 [default=1000];
+}
+
+// Group of OSMPrimitives. All primitives in a group must be the same type.
+message PrimitiveGroup {
+  repeated Node nodes = 1;
+  optional DenseNodes dense = 2;
+  repeated Way ways = 3;
+  repeated Relation relations = 4;
+  repeated ChangeSet changesets = 5;
+}
+
+
+/** String table, contains the common strings in each block.
+
+ Note that we reserve index '0' as a delimiter, so the entry at that
+ index in the table is ALWAYS blank and unused.
+
+ */
+message StringTable {
+  repeated bytes s = 1;
+}
+
+/* Optional metadata that may be included into each primitive. */
+message Info {
+  optional int32 version = 1 [default = -1];
+  optional int64 timestamp = 2;
+  optional int64 changeset = 3;
+  optional int32 uid = 4;
+  optional uint32 user_sid = 5; // String IDs
+
+  // The visible flag is used to store history information. It indicates that
+  // the current object version has been created by a delete operation on the
+  // OSM API.
+  // When a writer sets this flag, it MUST add a required_features tag with
+  // value "HistoricalInformation" to the HeaderBlock.
+  // If this flag is not available for some object it MUST be assumed to be
+  // true if the file has the required_features tag "HistoricalInformation"
+  // set.
+  optional bool visible = 6;
+}
+
+/** Optional metadata that may be included into each primitive. Special dense format used in DenseNodes. */
+message DenseInfo {
+  repeated int32 version = 1 [packed = true];
+  repeated sint64 timestamp = 2 [packed = true]; // DELTA coded
+  repeated sint64 changeset = 3 [packed = true]; // DELTA coded
+  repeated sint32 uid = 4 [packed = true]; // DELTA coded
+  repeated sint32 user_sid = 5 [packed = true]; // String IDs for usernames. DELTA coded
+
+  // The visible flag is used to store history information. It indicates that
+  // the current object version has been created by a delete operation on the
+  // OSM API.
+  // When a writer sets this flag, it MUST add a required_features tag with
+  // value "HistoricalInformation" to the HeaderBlock.
+  // If this flag is not available for some object it MUST be assumed to be
+  // true if the file has the required_features tag "HistoricalInformation"
+  // set.
+  repeated bool visible = 6 [packed = true];
+}
+
+
+// This is kept for backwards compatibility but not used anywhere.
+message ChangeSet {
+  required int64 id = 1;
+}
+
+
+message Node {
+  required sint64 id = 1;
+
+  // Parallel arrays.
+  repeated uint32 keys = 2 [packed = true]; // String IDs.
+  repeated uint32 vals = 3 [packed = true]; // String IDs.
+
+  optional Info info = 4; // May be omitted in omitmeta
+
+  required sint64 lat = 8;
+  required sint64 lon = 9;
+}
+
+/* Used to densly represent a sequence of nodes that do not have any tags.
+
+We represent these nodes columnwise as five columns: ID's, lats, and
+lons, all delta coded. When metadata is not omitted,
+
+We encode keys & vals for all nodes as a single array of integers
+containing key-stringid and val-stringid, using a stringid of 0 as a
+delimiter between nodes.
+
+   ( (<keyid> <valid>)* '0' )*
+ */
+
+message DenseNodes {
+  repeated sint64 id = 1 [packed = true]; // DELTA coded
+
+  optional DenseInfo denseinfo = 5;
+
+  repeated sint64 lat = 8 [packed = true]; // DELTA coded
+  repeated sint64 lon = 9 [packed = true]; // DELTA coded
+
+  // Special packing of keys and vals into one array. May be empty if all nodes in this block are tagless.
+  repeated int32 keys_vals = 10 [packed = true];
+}
+
+
+message Way {
+  required int64 id = 1;
+
+  // Parallel arrays.
+  repeated uint32 keys = 2 [packed = true];
+  repeated uint32 vals = 3 [packed = true];
+
+  optional Info info = 4;
+
+  repeated sint64 refs = 8 [packed = true]; // DELTA coded
+
+  // The following two fields are optional. They are only used in a special
+  // format where node locations are also added to the ways. This makes the
+  // files larger, but allows creating way geometries directly.
+  //
+  // If this is used, you MUST set the optional_features tag "LocationsOnWays"
+  // and the number of values in refs, lat, and lon MUST be the same.
+  repeated sint64 lat = 9 [packed = true]; // DELTA coded, optional
+  repeated sint64 lon = 10 [packed = true]; // DELTA coded, optional
+}
+
+message Relation {
+  enum MemberType {
+    NODE = 0;
+    WAY = 1;
+    RELATION = 2;
+  }
+
+  required int64 id = 1;
+
+  // Parallel arrays.
+  repeated uint32 keys = 2 [packed = true];
+  repeated uint32 vals = 3 [packed = true];
+
+  optional Info info = 4;
+
+  // Parallel arrays
+  repeated int32 roles_sid = 8 [packed = true]; // This should have been defined as uint32 for consistency, but it is now too late to change it
+  repeated sint64 memids = 9 [packed = true]; // DELTA encoded
+  repeated MemberType types = 10 [packed = true];
+}