@osmix/geoparquet 0.1.0

package/test/monaco-parquet.test.ts

@@ -0,0 +1,249 @@
+import { describe, expect, it } from "bun:test"
+import { getFixturePath } from "@osmix/shared/test/fixtures"
+import { fromGeoParquet, GeoParquetOsmBuilder } from "../src"
+
+/**
+ * Integration tests using the Monaco highways GeoParquet fixture.
+ *
+ * The fixture was downloaded from LayerCake (https://openstreetmap.us/our-work/layercake/)
+ * using DuckDB with a bounding box filter for Monaco. The fixture has a flattened
+ * structure where each tag column is separate (highway, name, surface, etc.) rather
+ * than nested in a tags struct.
+ *
+ * Note: hyparquet automatically parses GeoParquet WKB geometry into GeoJSON objects.
+ *
+ * Monaco bounding box:
+ * - xmin: 7.409205, xmax: 7.448637
+ * - ymin: 43.72335, ymax: 43.75169
+ */
+
+describe("@osmix/geoparquet: Monaco highways fixture", () => {
+	const fixtureFile = () => Bun.file(getFixturePath("monaco.parquet"))
+	const getOsm = async () => fromGeoParquet(await fixtureFile().arrayBuffer())
+
+	it("should load the monaco.parquet fixture", async () => {
+		const file = fixtureFile()
+		expect(await file.exists()).toBe(true)
+
+		const size = file.size
+		expect(size).toBeGreaterThan(0)
+		expect(size).toBeLessThan(1_000_000) // Should be a small fixture
+	})
+
+	it("should read parquet rows with correct structure", async () => {
+		const builder = new GeoParquetOsmBuilder({ id: "monaco" }, { rowEnd: 5 })
+		const rows = await builder.readParquetRows(
+			await fixtureFile().arrayBuffer(),
+		)
+
+		expect(rows.length).toBe(5)
+
+		// Verify row structure
+		const firstRow = rows[0]!
+		expect(firstRow["id"]).toBeDefined()
+		expect(firstRow["geometry"]).toBeDefined()
+		// expect(firstRow.osm_type).toBeDefined()
+
+		// hyparquet parses GeoParquet WKB into GeoJSON automatically
+		expect(firstRow["geometry"].type).toBeDefined()
+		// Verify it's a geometry with coordinates (not a GeometryCollection)
+		expect(["Point", "LineString", "Polygon", "MultiPolygon"]).toContain(
+			firstRow["geometry"].type,
+		)
+	})
+
+	it("should convert highway features to OSM entities", async () => {
+		const osm = await getOsm()
+
+		// Should have created nodes and ways from the highway features
+		expect(osm.nodes.size).toBeGreaterThan(0)
+		expect(osm.ways.size).toBeGreaterThan(0)
+
+		// Monaco is small, so we expect reasonable numbers
+		// The highway extract should have a few hundred to a few thousand ways
+		expect(osm.ways.size).toBeGreaterThan(100)
+		expect(osm.ways.size).toBeLessThan(10_000)
+	})
+
+	it("should preserve highway tags from GeoParquet columns", async () => {
+		const osm = await getOsm()
+
+		// Find ways with highway tags
+		let highwayCount = 0
+		for (let i = 0; i < osm.ways.size; i++) {
+			const way = osm.ways.getByIndex(i)
+			if (way?.tags?.["highway"]) {
+				highwayCount++
+			}
+		}
+
+		// Most features should have highway tags since this is the highways layer
+		expect(highwayCount).toBeGreaterThan(0)
+	})
+
+	it("should have valid node coordinates within Monaco bounds", async () => {
+		const osm = await getOsm()
+
+		// Monaco bounding box (slightly expanded for node positions)
+		const minLon = 7.4
+		const maxLon = 7.45
+		const minLat = 43.72
+		const maxLat = 43.76
+
+		// Check that all nodes are within Monaco bounds
+		let validNodes = 0
+		for (let i = 0; i < osm.nodes.size; i++) {
+			const node = osm.nodes.getByIndex(i)
+			if (node) {
+				expect(node.lon).toBeGreaterThanOrEqual(minLon)
+				expect(node.lon).toBeLessThanOrEqual(maxLon)
+				expect(node.lat).toBeGreaterThanOrEqual(minLat)
+				expect(node.lat).toBeLessThanOrEqual(maxLat)
+				validNodes++
+			}
+		}
+
+		expect(validNodes).toBeGreaterThan(0)
+	})
+
+	it("should build spatial indexes after loading", async () => {
+		const osm = await getOsm()
+
+		// Indexes should be built
+		expect(osm.isReady()).toBe(true)
+		expect(osm.nodes.isReady()).toBe(true)
+		expect(osm.ways.isReady()).toBe(true)
+	})
+
+	it("should handle ways with multiple nodes", async () => {
+		const osm = await getOsm()
+		// Count ways with enough nodes
+		let validWays = 0
+		for (let i = 0; i < osm.ways.size; i++) {
+			const way = osm.ways.getByIndex(i)
+			if (way && way.refs.length >= 2) {
+				validWays++
+				// All node refs should resolve to actual nodes
+				for (const ref of way.refs) {
+					const node = osm.nodes.getById(ref)
+					expect(node).toBeDefined()
+				}
+			}
+		}
+
+		// Most ways should have at least 2 nodes (minimum for a LineString)
+		expect(validWays).toBeGreaterThan(0)
+	})
+
+	it("should handle maxRows option to limit features", async () => {
+		const builder = new GeoParquetOsmBuilder({ id: "monaco" }, { rowEnd: 10 })
+		const rows = await builder.readParquetRows(
+			await fixtureFile().arrayBuffer(),
+		)
+		builder.processGeoParquetRows(rows)
+		const osm = builder.buildOsm()
+
+		// Should only have processed 10 features
+		expect(rows.length).toBe(10)
+		// Note: Some rows may be Point geometries (crossings) which don't create ways
+		expect(osm.nodes.size + osm.ways.size).toBeGreaterThan(0)
+	})
+
+	it("should find common highway types in Monaco", async () => {
+		const osm = await getOsm()
+
+		const highwayTypes = new Map<string, number>()
+
+		for (let i = 0; i < osm.ways.size; i++) {
+			const way = osm.ways.getByIndex(i)
+			const highway = way?.tags?.["highway"]
+			if (highway && typeof highway === "string") {
+				highwayTypes.set(highway, (highwayTypes.get(highway) ?? 0) + 1)
+			}
+		}
+
+		// Monaco should have some common highway types
+		const allTypes = Array.from(highwayTypes.keys())
+		expect(allTypes.length).toBeGreaterThan(0)
+
+		// Common types that should exist in Monaco's dense urban area
+		const commonTypes = ["residential", "footway", "service", "path", "steps"]
+		const foundCommonTypes = commonTypes.filter((t) => highwayTypes.has(t))
+
+		// At least some common types should be present
+		expect(foundCommonTypes.length).toBeGreaterThan(0)
+	})
+
+	it("should handle surface and other highway attributes", async () => {
+		const osm = await getOsm()
+
+		// Check that some ways have additional attributes beyond just highway
+		let waysWithSurface = 0
+		let waysWithName = 0
+
+		for (let i = 0; i < osm.ways.size; i++) {
+			const way = osm.ways.getByIndex(i)
+			if (way?.tags) {
+				if (way.tags["surface"]) waysWithSurface++
+				if (way.tags["name"]) waysWithName++
+			}
+		}
+
+		// Some highways should have surface or name attributes
+		// (not all will, so we just check that at least some do)
+		expect(waysWithSurface + waysWithName).toBeGreaterThan(0)
+	})
+
+	it("should reuse nodes for connected highways", async () => {
+		const osm = await getOsm()
+
+		// Count how many ways reference the same nodes
+		const nodeRefCounts = new Map<number, number>()
+
+		for (let i = 0; i < osm.ways.size; i++) {
+			const way = osm.ways.getByIndex(i)
+			if (way) {
+				for (const ref of way.refs) {
+					nodeRefCounts.set(ref, (nodeRefCounts.get(ref) ?? 0) + 1)
+				}
+			}
+		}
+
+		// Some nodes should be shared between multiple ways (intersections)
+		const sharedNodes = Array.from(nodeRefCounts.values()).filter((c) => c > 1)
+		expect(sharedNodes.length).toBeGreaterThan(0)
+	})
+
+	it("should handle Point geometries (crossings)", async () => {
+		const osm = await getOsm()
+
+		// Find nodes with crossing tags (Point features become Nodes with tags)
+		let crossingCount = 0
+		for (let i = 0; i < osm.nodes.size; i++) {
+			const node = osm.nodes.getByIndex(i)
+			if (node?.tags?.["highway"] === "crossing") {
+				crossingCount++
+			}
+		}
+
+		// Monaco should have some pedestrian crossings
+		expect(crossingCount).toBeGreaterThan(0)
+	})
+
+	it("should count geometry types in the fixture", async () => {
+		const builder = new GeoParquetOsmBuilder({ id: "monaco" })
+		const rows = await builder.readParquetRows(
+			await fixtureFile().arrayBuffer(),
+		)
+
+		const typeCounts = new Map<string, number>()
+		for (const row of rows) {
+			const type = row["geometry"]?.type ?? "unknown"
+			typeCounts.set(type, (typeCounts.get(type) ?? 0) + 1)
+		}
+
+		// Highways layer should have mostly LineStrings (roads) and some Points (crossings)
+		expect(typeCounts.get("LineString") ?? 0).toBeGreaterThan(0)
+		expect(typeCounts.get("Point") ?? 0).toBeGreaterThan(0)
+	})
+})

package/test/wkb.test.ts

@@ -0,0 +1,296 @@
+import { describe, expect, it } from "bun:test"
+import { parseWkb } from "../src/wkb"
+
+/**
+ * Helper to create a WKB Point with properly encoded float64 coordinates.
+ */
+function createPointWkb(
+	lon: number,
+	lat: number,
+	littleEndian = true,
+): Uint8Array {
+	const buffer = new ArrayBuffer(21)
+	const view = new DataView(buffer)
+	let offset = 0
+
+	view.setUint8(offset, littleEndian ? 1 : 0)
+	offset += 1
+	view.setUint32(offset, 1, littleEndian) // Point type
+	offset += 4
+	view.setFloat64(offset, lon, littleEndian)
+	offset += 8
+	view.setFloat64(offset, lat, littleEndian)
+
+	return new Uint8Array(buffer)
+}
+
+describe("@osmix/geoparquet: WKB Parser", () => {
+	it("should parse a Point geometry (little endian)", () => {
+		const wkb = createPointWkb(-122.4194, 37.7749, true)
+
+		const geometry = parseWkb(wkb)
+
+		expect(geometry.type).toBe("Point")
+		if (geometry.type === "Point") {
+			expect(geometry.coordinates[0]).toBeCloseTo(-122.4194, 4)
+			expect(geometry.coordinates[1]).toBeCloseTo(37.7749, 4)
+		}
+	})
+
+	it("should parse a Point geometry (big endian)", () => {
+		const wkb = createPointWkb(-122.4194, 37.7749, false)
+
+		const geometry = parseWkb(wkb)
+
+		expect(geometry.type).toBe("Point")
+		if (geometry.type === "Point") {
+			expect(geometry.coordinates[0]).toBeCloseTo(-122.4194, 4)
+			expect(geometry.coordinates[1]).toBeCloseTo(37.7749, 4)
+		}
+	})
+
+	it("should parse a LineString geometry", () => {
+		// WKB LineString with 3 points
+		const buffer = new ArrayBuffer(1 + 4 + 4 + 3 * 16)
+		const view = new DataView(buffer)
+		let offset = 0
+
+		// Byte order: little endian
+		view.setUint8(offset, 1)
+		offset += 1
+
+		// Geometry type: LineString (2)
+		view.setUint32(offset, 2, true)
+		offset += 4
+
+		// Number of points: 3
+		view.setUint32(offset, 3, true)
+		offset += 4
+
+		// Point 1: -122.4194, 37.7749
+		view.setFloat64(offset, -122.4194, true)
+		offset += 8
+		view.setFloat64(offset, 37.7749, true)
+		offset += 8
+
+		// Point 2: -122.4094, 37.7849
+		view.setFloat64(offset, -122.4094, true)
+		offset += 8
+		view.setFloat64(offset, 37.7849, true)
+		offset += 8
+
+		// Point 3: -122.3994, 37.7949
+		view.setFloat64(offset, -122.3994, true)
+		offset += 8
+		view.setFloat64(offset, 37.7949, true)
+
+		const wkb = new Uint8Array(buffer)
+		const geometry = parseWkb(wkb)
+
+		expect(geometry.type).toBe("LineString")
+		if (geometry.type === "LineString") {
+			expect(geometry.coordinates).toHaveLength(3)
+			expect(geometry.coordinates[0]?.[0]).toBeCloseTo(-122.4194, 4)
+			expect(geometry.coordinates[0]?.[1]).toBeCloseTo(37.7749, 4)
+			expect(geometry.coordinates[1]?.[0]).toBeCloseTo(-122.4094, 4)
+			expect(geometry.coordinates[1]?.[1]).toBeCloseTo(37.7849, 4)
+			expect(geometry.coordinates[2]?.[0]).toBeCloseTo(-122.3994, 4)
+			expect(geometry.coordinates[2]?.[1]).toBeCloseTo(37.7949, 4)
+		}
+	})
+
+	it("should parse a Polygon geometry", () => {
+		// WKB Polygon with 1 ring of 5 points (closed)
+		const buffer = new ArrayBuffer(1 + 4 + 4 + 4 + 5 * 16)
+		const view = new DataView(buffer)
+		let offset = 0
+
+		// Byte order: little endian
+		view.setUint8(offset, 1)
+		offset += 1
+
+		// Geometry type: Polygon (3)
+		view.setUint32(offset, 3, true)
+		offset += 4
+
+		// Number of rings: 1
+		view.setUint32(offset, 1, true)
+		offset += 4
+
+		// Number of points in ring: 5
+		view.setUint32(offset, 5, true)
+		offset += 4
+
+		// Points (closed ring)
+		const coords = [
+			[-122.4194, 37.7749],
+			[-122.4094, 37.7749],
+			[-122.4094, 37.7849],
+			[-122.4194, 37.7849],
+			[-122.4194, 37.7749], // closed
+		]
+
+		for (const [lon, lat] of coords) {
+			view.setFloat64(offset, lon!, true)
+			offset += 8
+			view.setFloat64(offset, lat!, true)
+			offset += 8
+		}
+
+		const wkb = new Uint8Array(buffer)
+		const geometry = parseWkb(wkb)
+
+		expect(geometry.type).toBe("Polygon")
+		if (geometry.type === "Polygon") {
+			expect(geometry.coordinates).toHaveLength(1)
+			expect(geometry.coordinates[0]).toHaveLength(5)
+			expect(geometry.coordinates[0]?.[0]?.[0]).toBeCloseTo(-122.4194, 4)
+			expect(geometry.coordinates[0]?.[0]?.[1]).toBeCloseTo(37.7749, 4)
+		}
+	})
+
+	it("should parse a MultiPolygon geometry", () => {
+		// Helper to create polygon WKB
+		const createPolygonWkb = (
+			coords: [number, number][][],
+		): [ArrayBuffer, number] => {
+			const totalPoints = coords.reduce((sum, ring) => sum + ring.length, 0)
+			const size = 1 + 4 + 4 + coords.length * 4 + totalPoints * 16
+			const buffer = new ArrayBuffer(size)
+			const view = new DataView(buffer)
+			let offset = 0
+
+			// Byte order
+			view.setUint8(offset, 1)
+			offset += 1
+
+			// Polygon type
+			view.setUint32(offset, 3, true)
+			offset += 4
+
+			// Number of rings
+			view.setUint32(offset, coords.length, true)
+			offset += 4
+
+			for (const ring of coords) {
+				view.setUint32(offset, ring.length, true)
+				offset += 4
+				for (const [lon, lat] of ring) {
+					view.setFloat64(offset, lon, true)
+					offset += 8
+					view.setFloat64(offset, lat, true)
+					offset += 8
+				}
+			}
+
+			return [buffer, size]
+		}
+
+		// Create MultiPolygon with 2 polygons
+		const poly1Coords: [number, number][][] = [
+			[
+				[-122.4194, 37.7749],
+				[-122.4094, 37.7749],
+				[-122.4094, 37.7849],
+				[-122.4194, 37.7849],
+				[-122.4194, 37.7749],
+			],
+		]
+
+		const poly2Coords: [number, number][][] = [
+			[
+				[-122.3994, 37.7649],
+				[-122.3894, 37.7649],
+				[-122.3894, 37.7749],
+				[-122.3994, 37.7749],
+				[-122.3994, 37.7649],
+			],
+		]
+
+		const [poly1Buffer, poly1Size] = createPolygonWkb(poly1Coords)
+		const [poly2Buffer, poly2Size] = createPolygonWkb(poly2Coords)
+
+		// MultiPolygon header + 2 polygons
+		const multiBuffer = new ArrayBuffer(1 + 4 + 4 + poly1Size + poly2Size)
+		const multiView = new DataView(multiBuffer)
+		let offset = 0
+
+		// Byte order
+		multiView.setUint8(offset, 1)
+		offset += 1
+
+		// MultiPolygon type (6)
+		multiView.setUint32(offset, 6, true)
+		offset += 4
+
+		// Number of geometries
+		multiView.setUint32(offset, 2, true)
+		offset += 4
+
+		// Copy polygon 1
+		new Uint8Array(multiBuffer, offset, poly1Size).set(
+			new Uint8Array(poly1Buffer),
+		)
+		offset += poly1Size
+
+		// Copy polygon 2
+		new Uint8Array(multiBuffer, offset, poly2Size).set(
+			new Uint8Array(poly2Buffer),
+		)
+
+		const wkb = new Uint8Array(multiBuffer)
+		const geometry = parseWkb(wkb)
+
+		expect(geometry.type).toBe("MultiPolygon")
+		if (geometry.type === "MultiPolygon") {
+			expect(geometry.coordinates).toHaveLength(2)
+			expect(geometry.coordinates[0]).toHaveLength(1)
+			expect(geometry.coordinates[1]).toHaveLength(1)
+		}
+	})
+
+	it("should handle EWKB with SRID flag", () => {
+		// EWKB Point with SRID (SRID flag: 0x20000000)
+		const buffer = new ArrayBuffer(1 + 4 + 4 + 16)
+		const view = new DataView(buffer)
+		let offset = 0
+
+		// Byte order: little endian
+		view.setUint8(offset, 1)
+		offset += 1
+
+		// Geometry type: Point (1) with SRID flag (0x20000001)
+		view.setUint32(offset, 0x20000001, true)
+		offset += 4
+
+		// SRID: 4326
+		view.setUint32(offset, 4326, true)
+		offset += 4
+
+		// Point: -122.4194, 37.7749
+		view.setFloat64(offset, -122.4194, true)
+		offset += 8
+		view.setFloat64(offset, 37.7749, true)
+
+		const wkb = new Uint8Array(buffer)
+		const geometry = parseWkb(wkb)
+
+		expect(geometry.type).toBe("Point")
+		if (geometry.type === "Point") {
+			expect(geometry.coordinates[0]).toBeCloseTo(-122.4194, 4)
+			expect(geometry.coordinates[1]).toBeCloseTo(37.7749, 4)
+		}
+	})
+
+	it("should throw on unsupported geometry type", () => {
+		const wkb = new Uint8Array([
+			0x01, // little endian
+			0x64,
+			0x00,
+			0x00,
+			0x00, // Invalid type (100)
+		])
+
+		expect(() => parseWkb(wkb)).toThrow("Unsupported WKB geometry type: 100")
+	})
+})

package/tsconfig.json

@@ -0,0 +1,9 @@
+{
+	"$schema": "https://json.schemastore.org/tsconfig",
+	"extends": "@osmix/shared/tsconfig/base.json",
+	"exclude": ["node_modules", "dist"],
+	"include": ["src", "test"],
+	"compilerOptions": {
+		"outDir": "./dist"
+	}
+}