@truelies/osm-dybuf 0.2.6 → 0.3.0

package/README.md

@@ -3,6 +3,19 @@
 Gridex `osm.dybuf` parser utilities shared by the exporter, inspection scripts, and Next.js frontend.  
 This package wraps the shared schema tables (`schema_ids`) and exposes a high-level `parseOsmDybuf` helper.  
 It depends on the published [`dybuf`](https://www.npmjs.com/package/dybuf) runtime (≥ 0.4.2).
+Feature IDs are packed as `feature_id = sub_id * 64 + main_id` (both 0–63). Use `schema_ids` to decode feature strings.
+
+## Current Compatibility Status
+
+`parseOsmDybuf` now supports the current cloud formats and keeps a legacy fallback.
+
+Support matrix:
+
+- Flat V1 cell file (`osm.dybuf`): `version + payload` -> supported (`format="flat_v1"`)
+- V2 overview/bundle (`overview.dybuf`, `bundle.dybuf`): `version=2 + index entries` -> supported (`format="v2_indexed"`)
+- Legacy V1 with region chunks: `version + [region_id + region_payload]...` -> supported as fallback (`format="legacy_v1_regions"`)
+
+`inspect_dybuf.mjs` now prints by detected format.
 
 ## Usage
 
@@ -21,10 +34,25 @@ import {
 const data = await fetch("/tiles/08/213/161/osm.dybuf").then((res) => res.arrayBuffer());
 const cell = { level: 8, londex: 213, latdex: 161 };
 const parsed = parseOsmDybuf(data, cell);
+
+if (parsed.format === "flat_v1") {
+  console.log(parsed.features.length);
+} else if (parsed.format === "v2_indexed") {
+  console.log(parsed.entries.length);
+} else {
+  console.log(parsed.regions.length);
+}
 ```
 
 - CLI inspection (development only):
   ```bash
+  # Preferred: only provide file path.
+  # For v1 osm.dybuf, the tool auto-infers level/londex/latdex from .../<lv>/<x>/<y>/osm.dybuf
+  node js/inspect_dybuf.mjs \
+      --file /path/to/osm.dybuf \
+      --limit 5
+
+  # Optional fallback when path does not contain cell coordinates (needed for v1):
   node js/inspect_dybuf.mjs \
       --file /path/to/osm.dybuf \
       --level 8 --londex 213 --latdex 161 --limit 5

package/osm_dybuf.d.mts

@@ -10,7 +10,7 @@ export interface GeometryCoordinate {
 }
 
 export interface GeometrySegment {
-  role: string;
+  role: string | null;
   coordinates: GeometryCoordinate[];
 }
 
@@ -34,10 +34,36 @@ export interface RegionChunk {
   features: GridFeature[];
 }
 
-export interface OsmDybuf {
-  version: number;
+export interface V2IndexEntry {
+  cell: GridCellRef;
+  features: GridFeature[];
+}
+
+export interface FlatV1Dybuf {
+  version: 1;
+  format: "flat_v1";
+  cell: GridCellRef;
+  features: GridFeature[];
+  regions: RegionChunk[];
+}
+
+export interface LegacyV1Dybuf {
+  version: 1;
+  format: "legacy_v1_regions";
   regions: RegionChunk[];
 }
 
+export interface V2Dybuf {
+  version: 2;
+  format: "v2_indexed";
+  entries: V2IndexEntry[];
+}
+
+export type OsmDybuf = FlatV1Dybuf | LegacyV1Dybuf | V2Dybuf | {
+  version: number;
+  format: string;
+};
+
 export declare function parseOsmDybuf(data: ArrayBuffer | ArrayBufferView, cell: GridCellRef): OsmDybuf;
-export declare function decodeRegionPayload(payload: ArrayBuffer, cell: GridCellRef): GridFeature[];
+export declare function decodePayload(payload: ArrayBuffer | ArrayBufferView, cell: GridCellRef): GridFeature[];
+export declare function decodeRegionPayload(payload: ArrayBuffer | ArrayBufferView, cell: GridCellRef): GridFeature[];

package/osm_dybuf.mjs

@@ -1,11 +1,7 @@
 import {
   DyBuf,
-  TYPDEX_TYP_BOOL,
-  TYPDEX_TYP_INT,
   TYPDEX_TYP_UINT,
-  TYPDEX_TYP_STRING,
   TYPDEX_TYP_BYTES,
-  TYPDEX_TYP_ARRAY,
   TYPDEX_TYP_MAP,
 } from "dybuf";
 import {
@@ -106,11 +102,11 @@ function decodeGeometrySegments(bytes, cell) {
   const segmentCount = readVarUint(buf);
   for (let i = 0; i < segmentCount; i += 1) {
     const roleId = buf.getByte();
-    let role = "";
+    let role = null;
     if (roleId === SEGMENT_ROLE_ID_UNSUPPORTED) {
       role = "<unsupported>";
     } else if (roleId !== SEGMENT_ROLE_ID_UNSET) {
-      role = ID_TO_SEGMENT_ROLE[String(roleId)] ?? "";
+      role = ID_TO_SEGMENT_ROLE[String(roleId)] ?? "<unsupported>";
     }
     const pointCount = readVarUint(buf);
     const points = [];
@@ -192,14 +188,14 @@ function decodeGridPayload(bytes, cell) {
   };
 }
 
-function decodeRegionPayload(payloadBytes, cell) {
+function decodePayload(payloadBytes, cell) {
   const buf = new DyBuf(payloadBytes, false);
   const featureCount = readVarUint(buf);
   const features = [];
   for (let i = 0; i < featureCount; i += 1) {
     const featureNumericId = readVarUint(buf);
     const featureId =
-      ID_TO_FEATURE_KIND[featureNumericId] ?? `feature:${featureNumericId}`;
+      ID_TO_FEATURE_KIND[String(featureNumericId)] ?? `feature:${featureNumericId}`;
     const entryCount = readVarUint(buf);
     const elements = [];
     for (let j = 0; j < entryCount; j += 1) {
@@ -208,9 +204,48 @@ function decodeRegionPayload(payloadBytes, cell) {
     }
     features.push({ featureId, elements });
   }
+  if (buf.position() !== buf.limit()) {
+    throw new Error(
+      `Payload has trailing bytes (${buf.limit() - buf.position()} bytes remain)`
+    );
+  }
   return features;
 }
 
+function decodeLegacyRegions(bodyBytes, cell) {
+  const buf = new DyBuf(bodyBytes, false);
+  const regions = [];
+  while (buf.position() < buf.limit()) {
+    const regionId = readVarUint(buf);
+    const regionName = REGION_ID_TO_NAME[String(regionId)] ?? `region:${regionId}`;
+    const regionPayload = readVarBytes(buf);
+    const features = decodePayload(regionPayload, cell);
+    regions.push({ id: regionId, name: regionName, features });
+  }
+  return regions;
+}
+
+function decodeV2Entries(bodyBytes) {
+  const buf = new DyBuf(bodyBytes, false);
+  const count = readVarUint(buf);
+  const entries = [];
+  for (let i = 0; i < count; i += 1) {
+    const level = buf.getByte();
+    const londex = readVarInt(buf);
+    const latdex = readVarInt(buf);
+    const payloadBytes = readVarBytes(buf);
+    const cell = { level, londex, latdex };
+    const features = decodePayload(payloadBytes, cell);
+    entries.push({ cell, features });
+  }
+  if (buf.position() !== buf.limit()) {
+    throw new Error(
+      `V2 payload has trailing bytes (${buf.limit() - buf.position()} bytes remain)`
+    );
+  }
+  return entries;
+}
+
 /**
  * Parse a single cell's osm.dybuf payload.
  * @param {ArrayBuffer | ArrayBufferView} data
@@ -221,17 +256,51 @@ export function parseOsmDybuf(data, cell) {
     throw new TypeError("Cell metadata (level, londex, latdex) is required");
   }
   const buffer = toArrayBuffer(data);
-  const buf = new DyBuf(buffer, false);
-  const version = buf.getByte();
-  const regions = [];
-  while (buf.position() < buf.limit()) {
-    const regionId = readVarUint(buf);
-    const regionName = REGION_ID_TO_NAME[String(regionId)] ?? `region:${regionId}`;
-    const regionPayload = readVarBytes(buf);
-    const features = decodeRegionPayload(regionPayload, cell);
-    regions.push({ id: regionId, name: regionName, features });
+  const bytes = new Uint8Array(buffer);
+  if (bytes.length === 0) {
+    throw new Error("Empty dybuf payload");
+  }
+
+  const version = bytes[0];
+  const body = bytes.buffer.slice(bytes.byteOffset + 1, bytes.byteOffset + bytes.byteLength);
+
+  if (version === 2) {
+    return {
+      version,
+      format: "v2_indexed",
+      entries: decodeV2Entries(body),
+    };
+  }
+
+  if (version !== 1) {
+    throw new Error(`Unsupported dybuf version ${version}`);
+  }
+
+  // Prefer current flat V1: version + payload
+  try {
+    const features = decodePayload(body, cell);
+    return {
+      version,
+      format: "flat_v1",
+      cell: { level: cell.level, londex: cell.londex, latdex: cell.latdex },
+      features,
+      regions: [],
+    };
+  } catch (flatErr) {
+    // Fallback to legacy format: version + [region_id + region_payload]...
+    try {
+      const regions = decodeLegacyRegions(body, cell);
+      return {
+        version,
+        format: "legacy_v1_regions",
+        regions,
+      };
+    } catch (legacyErr) {
+      throw new Error(
+        `Unable to decode V1 dybuf as flat or legacy regions. flat=${String(flatErr)} legacy=${String(legacyErr)}`
+      );
+    }
   }
-  return { version, regions };
 }
 
-export { decodeRegionPayload, decodeGridPayload };
+export { decodePayload, decodeGridPayload, decodePayload as decodeRegionPayload };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@truelies/osm-dybuf",
-  "version": "0.2.6",
+  "version": "0.3.0",
   "description": "Gridex OSM DyBuf parser and schema utilities",
   "type": "module",
   "main": "./osm_dybuf.mjs",

package/schema_ids.d.mts

@@ -2,6 +2,7 @@ export declare const ENTITY_TYPE_IDS: Readonly<Record<string, number>>;
 export declare const ID_TO_ENTITY_TYPE: Readonly<Record<number, string>>;
 export declare const GEOMETRY_KIND_IDS: Readonly<Record<string, number>>;
 export declare const ID_TO_GEOMETRY_KIND: Readonly<Record<number, string>>;
+export declare const MAIN_FEATURE_IDS: Readonly<Record<string, number>>;
 export declare const FEATURE_KIND_IDS: Readonly<Record<string, number>>;
 export declare const ID_TO_FEATURE_KIND: Readonly<Record<number, string>>;
 export declare const REGION_NUMERIC_CODES: Readonly<Record<string, number>>;

package/schema_ids.mjs

@@ -21,20 +21,48 @@ export const GEOMETRY_KIND_IDS = Object.freeze({
 export const ID_TO_GEOMETRY_KIND = Object.freeze(
   Object.fromEntries(Object.entries(GEOMETRY_KIND_IDS).map(([k, v]) => [String(v), k]))
 );
-export const FEATURE_KIND_IDS = Object.freeze({
+export const MAIN_FEATURE_IDS = Object.freeze({
   country_land: 0,
-  admin_boundary: 100,
-  coastline: 200,
-  natural_area: 210,
-  waterbody: 220,
-  landuse: 230,
-  park: 240,
-  road_major: 300,
-  road_minor: 301,
-  waterway: 400,
-  place_label: 500,
+  admin_boundary: 1,
+  coastline: 2,
+  natural: 3,
+  waterbody: 4,
+  landuse: 5,
+  park: 6,
+  road: 7,
+  waterway: 8,
+  place_label: 9,
 });
 
+const FEATURE_KIND_CODES = Object.freeze({
+  country_land: ["country_land", 0],
+  admin_boundary: ["admin_boundary", 0],
+  coastline: ["coastline", 0],
+  natural_forest: ["natural", 0],
+  natural_vegetation: ["natural", 1],
+  natural_bare: ["natural", 2],
+  waterbody: ["waterbody", 0],
+  landuse: ["landuse", 0],
+  park: ["park", 0],
+  road_major: ["road", 0],
+  road_minor: ["road", 1],
+  road_local: ["road", 2],
+  road_trail: ["road", 3],
+  waterway_major: ["waterway", 0],
+  waterway_stream: ["waterway", 1],
+  waterway_minor: ["waterway", 2],
+  place_label: ["place_label", 0],
+});
+
+export const FEATURE_KIND_IDS = Object.freeze(
+  Object.fromEntries(
+    Object.entries(FEATURE_KIND_CODES).map(([kind, [main, sub]]) => [
+      kind,
+      sub * 64 + MAIN_FEATURE_IDS[main],
+    ])
+  )
+);
+
 export const REGION_NUMERIC_CODES = Object.freeze(REGION_CODES_RAW);
 
 export const ID_TO_FEATURE_KIND = Object.freeze(