@truelies/osm-dybuf 0.2.7 → 0.3.1

package/README.md

@@ -3,6 +3,19 @@
 Gridex `osm.dybuf` parser utilities shared by the exporter, inspection scripts, and Next.js frontend.  
 This package wraps the shared schema tables (`schema_ids`) and exposes a high-level `parseOsmDybuf` helper.  
 It depends on the published [`dybuf`](https://www.npmjs.com/package/dybuf) runtime (≥ 0.4.2).
+Feature IDs are packed as `feature_id = sub_id * 64 + main_id` (both 0–63). Use `schema_ids` to decode feature strings.
+
+## Current Compatibility Status
+
+`parseOsmDybuf` now supports the current cloud formats and keeps a legacy fallback.
+
+Support matrix:
+
+- Flat V1 cell file (`osm.dybuf`): `version + payload` -> supported (`format="flat_v1"`)
+- V2 overview/bundle (`overview.dybuf`, `bundle.dybuf`): `version=2 + index entries` -> supported (`format="v2_indexed"`)
+- Legacy V1 with region chunks: `version + [region_id + region_payload]...` -> supported as fallback (`format="legacy_v1_regions"`)
+
+`inspect_dybuf.mjs` now prints by detected format.
 
 ## Usage
 
@@ -21,10 +34,25 @@ import {
 const data = await fetch("/tiles/08/213/161/osm.dybuf").then((res) => res.arrayBuffer());
 const cell = { level: 8, londex: 213, latdex: 161 };
 const parsed = parseOsmDybuf(data, cell);
+
+if (parsed.format === "flat_v1") {
+  console.log(parsed.features.length);
+} else if (parsed.format === "v2_indexed") {
+  console.log(parsed.entries.length);
+} else {
+  console.log(parsed.regions.length);
+}
 ```
 
 - CLI inspection (development only):
   ```bash
+  # Preferred: only provide file path.
+  # For v1 osm.dybuf, the tool auto-infers level/londex/latdex from .../<lv>/<x>/<y>/osm.dybuf
+  node js/inspect_dybuf.mjs \
+      --file /path/to/osm.dybuf \
+      --limit 5
+
+  # Optional fallback when path does not contain cell coordinates (needed for v1):
   node js/inspect_dybuf.mjs \
       --file /path/to/osm.dybuf \
       --level 8 --londex 213 --latdex 161 --limit 5
@@ -32,7 +60,7 @@ const parsed = parseOsmDybuf(data, cell);
 
 ### Grid helpers (grid_index)
 
-`grid_index.ts` exposes the minimal Gridex helpers (integers, pole triangles) aligned with the exporter:
+`grid_index.mjs` exposes Gridex helpers (integers, pole triangles) aligned with the exporter:
 
 ```ts
 import { GridUnit, Gridex, INT_COORD_SCALE } from "@truelies/osm-dybuf/grid_index";
@@ -40,18 +68,44 @@ import { GridUnit, Gridex, INT_COORD_SCALE } from "@truelies/osm-dybuf/grid_inde
 const unit = new GridUnit(8); // level 8
 const cell = new Gridex(213, 161);
 const [minLon, minLat, maxLon, maxLat] = unit.boundsOfGrid(cell);
+
+// parent/child conversion (same rules as Python GridUnit)
+const children = unit.toLowerLevelGridexes(cell);   // level 9, 4 cells
+const parent = new GridUnit(9).toUpperLevelGridex(children[0]); // back to level 8
 ```
 
+- `toLowerLevelGridexes(gridex)`:
+  - returns 4 child cells at `level + 1`
+  - throws when current unit is already max level (`16`)
+- `toUpperLevelGridex(gridex)`:
+  - returns the parent cell at `level - 1`
+  - throws when current unit is level `0`
+
 ## Local Development
 
 ```bash
 cd js
 npm install      # installs dybuf runtime for this package
+npm test         # runs unit tests (node:test)
 ```
 
+Unit test example (`tests/grid_index.test.mjs`) covers:
+- `gridexesAt` near axis (no zero index cell)
+- `toLowerLevelGridexes` and `toUpperLevelGridex` roundtrip
+- level boundary errors (`lv0` has no upper level, `lv16` has no lower level)
+
 - `npm pack`: build a tarball so other repos can install via `npm install ./path/to/osm-dybuf-*.tgz`
 - `npm link`: link the package locally (`npm link` here, then `npm link @truelies/osm-dybuf` in the consumer repo)
-- `npm publish --access public`: publish to npm (requires the `@truelies` scope). If the scope is not available, rename `package.json` → `"name": "truelies-osm-dybuf"` and publish without a scope.
+- Publish to npm (requires `@truelies` scope):
+  ```bash
+  cd js
+  npm version <new-version> --no-git-tag-version
+  NPM_CONFIG_CACHE=/tmp/npm-cache npm login --auth-type=web --registry=https://registry.npmjs.org/ --scope=@truelies
+  NPM_CONFIG_CACHE=/tmp/npm-cache npm whoami
+  NPM_CONFIG_CACHE=/tmp/npm-cache npm publish --access public
+  ```
+  - If `~/.npm` has permission issues (root-owned cache/logs), keep using `NPM_CONFIG_CACHE=/tmp/npm-cache`.
+  - If the scope is not available, rename `package.json` → `"name": "truelies-osm-dybuf"` and publish without a scope.
 
 ## Files
 

package/grid_index.d.ts

@@ -31,6 +31,8 @@ export declare class GridUnit {
   makeGridex(londex: number, latdex: number): GridexAtLv;
   gridexesAt(longitudeInt: number, latitudeInt: number): GridexAtLv[];
   gridexesWithExt(longitudeInt: number, latitudeInt: number, ext?: number): GridexAtLv[];
+  toLowerLevelGridexes(gridex: Gridex): GridexAtLv[];
+  toUpperLevelGridex(gridex: Gridex): GridexAtLv;
   boundsOfGridFine(gridex: Gridex, applyTriangle?: boolean): [number, number, number, number];
   boundsOfGrid(gridex: Gridex, applyTriangle?: boolean): [number, number, number, number];
   gridexRangeForBounds(

package/grid_index.mjs

@@ -158,6 +158,43 @@ export class GridUnit {
     return result;
   }
 
+  toLowerLevelGridexes(gridex) {
+    if (this.level >= MAX_LEVEL) {
+      throw new Error(`No lower level exists for level ${this.level}`);
+    }
+    const lon = gridex.londex;
+    const lat = gridex.latdex;
+    const lonInc = lon > 0 ? -1 : 1;
+    const latInc = lat > 0 ? -1 : 1;
+    const childUnit = new GridUnit(this.level + 1);
+    return [
+      childUnit.makeGridex(lon * 2 + lonInc, lat * 2 + latInc),
+      childUnit.makeGridex(lon * 2 + lonInc, lat * 2),
+      childUnit.makeGridex(lon * 2, lat * 2 + latInc),
+      childUnit.makeGridex(lon * 2, lat * 2),
+    ];
+  }
+
+  toUpperLevelGridex(gridex) {
+    if (this.level <= 0) {
+      throw new Error("No upper level exists for level 0");
+    }
+    const lon = gridex.londex;
+    const lat = gridex.latdex;
+    const lonInc = lon > 0 ? -1 : 1;
+    const latInc = lat > 0 ? -1 : 1;
+    const upperLon =
+      lon > 0
+        ? Math.floor((lon - lonInc) / 2)
+        : Math.ceil((lon - lonInc) / 2);
+    const upperLat =
+      lat > 0
+        ? Math.floor((lat - latInc) / 2)
+        : Math.ceil((lat - latInc) / 2);
+    const upperUnit = new GridUnit(this.level - 1);
+    return upperUnit.makeGridex(upperLon, upperLat);
+  }
+
   boundsOfGridFine(gridex, applyTriangle = false) {
     const { londex, latdex } = gridex;
     const minLonFine = (londex - (londex > 0 ? 1 : 0)) * this.unitFine;

package/osm_dybuf.d.mts

@@ -10,7 +10,7 @@ export interface GeometryCoordinate {
 }
 
 export interface GeometrySegment {
-  role: string;
+  role: string | null;
   coordinates: GeometryCoordinate[];
 }
 
@@ -34,10 +34,36 @@ export interface RegionChunk {
   features: GridFeature[];
 }
 
-export interface OsmDybuf {
-  version: number;
+export interface V2IndexEntry {
+  cell: GridCellRef;
+  features: GridFeature[];
+}
+
+export interface FlatV1Dybuf {
+  version: 1;
+  format: "flat_v1";
+  cell: GridCellRef;
+  features: GridFeature[];
+  regions: RegionChunk[];
+}
+
+export interface LegacyV1Dybuf {
+  version: 1;
+  format: "legacy_v1_regions";
   regions: RegionChunk[];
 }
 
+export interface V2Dybuf {
+  version: 2;
+  format: "v2_indexed";
+  entries: V2IndexEntry[];
+}
+
+export type OsmDybuf = FlatV1Dybuf | LegacyV1Dybuf | V2Dybuf | {
+  version: number;
+  format: string;
+};
+
 export declare function parseOsmDybuf(data: ArrayBuffer | ArrayBufferView, cell: GridCellRef): OsmDybuf;
-export declare function decodeRegionPayload(payload: ArrayBuffer, cell: GridCellRef): GridFeature[];
+export declare function decodePayload(payload: ArrayBuffer | ArrayBufferView, cell: GridCellRef): GridFeature[];
+export declare function decodeRegionPayload(payload: ArrayBuffer | ArrayBufferView, cell: GridCellRef): GridFeature[];

package/osm_dybuf.mjs

@@ -1,11 +1,7 @@
 import {
   DyBuf,
-  TYPDEX_TYP_BOOL,
-  TYPDEX_TYP_INT,
   TYPDEX_TYP_UINT,
-  TYPDEX_TYP_STRING,
   TYPDEX_TYP_BYTES,
-  TYPDEX_TYP_ARRAY,
   TYPDEX_TYP_MAP,
 } from "dybuf";
 import {
@@ -106,11 +102,11 @@ function decodeGeometrySegments(bytes, cell) {
   const segmentCount = readVarUint(buf);
   for (let i = 0; i < segmentCount; i += 1) {
     const roleId = buf.getByte();
-    let role = "";
+    let role = null;
     if (roleId === SEGMENT_ROLE_ID_UNSUPPORTED) {
       role = "<unsupported>";
     } else if (roleId !== SEGMENT_ROLE_ID_UNSET) {
-      role = ID_TO_SEGMENT_ROLE[String(roleId)] ?? "";
+      role = ID_TO_SEGMENT_ROLE[String(roleId)] ?? "<unsupported>";
     }
     const pointCount = readVarUint(buf);
     const points = [];
@@ -192,14 +188,14 @@ function decodeGridPayload(bytes, cell) {
   };
 }
 
-function decodeRegionPayload(payloadBytes, cell) {
+function decodePayload(payloadBytes, cell) {
   const buf = new DyBuf(payloadBytes, false);
   const featureCount = readVarUint(buf);
   const features = [];
   for (let i = 0; i < featureCount; i += 1) {
     const featureNumericId = readVarUint(buf);
     const featureId =
-      ID_TO_FEATURE_KIND[featureNumericId] ?? `feature:${featureNumericId}`;
+      ID_TO_FEATURE_KIND[String(featureNumericId)] ?? `feature:${featureNumericId}`;
     const entryCount = readVarUint(buf);
     const elements = [];
     for (let j = 0; j < entryCount; j += 1) {
@@ -208,9 +204,48 @@ function decodeRegionPayload(payloadBytes, cell) {
     }
     features.push({ featureId, elements });
   }
+  if (buf.position() !== buf.limit()) {
+    throw new Error(
+      `Payload has trailing bytes (${buf.limit() - buf.position()} bytes remain)`
+    );
+  }
   return features;
 }
 
+function decodeLegacyRegions(bodyBytes, cell) {
+  const buf = new DyBuf(bodyBytes, false);
+  const regions = [];
+  while (buf.position() < buf.limit()) {
+    const regionId = readVarUint(buf);
+    const regionName = REGION_ID_TO_NAME[String(regionId)] ?? `region:${regionId}`;
+    const regionPayload = readVarBytes(buf);
+    const features = decodePayload(regionPayload, cell);
+    regions.push({ id: regionId, name: regionName, features });
+  }
+  return regions;
+}
+
+function decodeV2Entries(bodyBytes) {
+  const buf = new DyBuf(bodyBytes, false);
+  const count = readVarUint(buf);
+  const entries = [];
+  for (let i = 0; i < count; i += 1) {
+    const level = buf.getByte();
+    const londex = readVarInt(buf);
+    const latdex = readVarInt(buf);
+    const payloadBytes = readVarBytes(buf);
+    const cell = { level, londex, latdex };
+    const features = decodePayload(payloadBytes, cell);
+    entries.push({ cell, features });
+  }
+  if (buf.position() !== buf.limit()) {
+    throw new Error(
+      `V2 payload has trailing bytes (${buf.limit() - buf.position()} bytes remain)`
+    );
+  }
+  return entries;
+}
+
 /**
  * Parse a single cell's osm.dybuf payload.
  * @param {ArrayBuffer | ArrayBufferView} data
@@ -221,17 +256,51 @@ export function parseOsmDybuf(data, cell) {
     throw new TypeError("Cell metadata (level, londex, latdex) is required");
   }
   const buffer = toArrayBuffer(data);
-  const buf = new DyBuf(buffer, false);
-  const version = buf.getByte();
-  const regions = [];
-  while (buf.position() < buf.limit()) {
-    const regionId = readVarUint(buf);
-    const regionName = REGION_ID_TO_NAME[String(regionId)] ?? `region:${regionId}`;
-    const regionPayload = readVarBytes(buf);
-    const features = decodeRegionPayload(regionPayload, cell);
-    regions.push({ id: regionId, name: regionName, features });
+  const bytes = new Uint8Array(buffer);
+  if (bytes.length === 0) {
+    throw new Error("Empty dybuf payload");
+  }
+
+  const version = bytes[0];
+  const body = bytes.buffer.slice(bytes.byteOffset + 1, bytes.byteOffset + bytes.byteLength);
+
+  if (version === 2) {
+    return {
+      version,
+      format: "v2_indexed",
+      entries: decodeV2Entries(body),
+    };
+  }
+
+  if (version !== 1) {
+    throw new Error(`Unsupported dybuf version ${version}`);
+  }
+
+  // Prefer current flat V1: version + payload
+  try {
+    const features = decodePayload(body, cell);
+    return {
+      version,
+      format: "flat_v1",
+      cell: { level: cell.level, londex: cell.londex, latdex: cell.latdex },
+      features,
+      regions: [],
+    };
+  } catch (flatErr) {
+    // Fallback to legacy format: version + [region_id + region_payload]...
+    try {
+      const regions = decodeLegacyRegions(body, cell);
+      return {
+        version,
+        format: "legacy_v1_regions",
+        regions,
+      };
+    } catch (legacyErr) {
+      throw new Error(
+        `Unable to decode V1 dybuf as flat or legacy regions. flat=${String(flatErr)} legacy=${String(legacyErr)}`
+      );
+    }
   }
-  return { version, regions };
 }
 
-export { decodeRegionPayload, decodeGridPayload };
+export { decodePayload, decodeGridPayload, decodePayload as decodeRegionPayload };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@truelies/osm-dybuf",
-  "version": "0.2.7",
+  "version": "0.3.1",
   "description": "Gridex OSM DyBuf parser and schema utilities",
   "type": "module",
   "main": "./osm_dybuf.mjs",
@@ -39,6 +39,9 @@
   ],
   "author": "TrueLies",
   "license": "UNLICENSED",
+  "scripts": {
+    "test": "node --test tests/*.test.mjs"
+  },
   "dependencies": {
     "dybuf": "^0.4.2"
   }