maplibre-tilepackage-protocol 1.1.0 → 1.2.1

package/src/shared-promise-cache.js

@@ -80,6 +80,7 @@ export default class SharedPromiseCache {
     this.maxCacheEntries = maxCacheEntries;
     this.counter = 1;
     //this.decompress = decompress;
+    this.subdivided = new Map(); // key: sourceKey|z|x|y -> Uint8Array
   }
 
   async getHeader(source) {
@@ -174,6 +175,28 @@ export default class SharedPromiseCache {
     }
   }
 
+  getSubdivided(source, z, x, y) {
+    const key = `${source.getKey()}|${z}|${x}|${y}|Subdivided`;
+    return this.subdivided.get(key);
+  }
+
+  setSubdivided(source, z, x, y, bytes) {
+    const key = `${source.getKey()}|${z}|${x}|${y}|Subdivided`;
+    if (!this.subdivided.has(key)) {
+      this.subdivided.set(key, bytes);
+      // Optional lightweight prune if huge:
+      if (this.subdivided.size > this.maxCacheEntries * 2) {
+        // Remove oldest half (no usage tracking kept; iterate arbitrary)
+        let toRemove = Math.floor(this.subdivided.size / 2);
+        for (const k of this.subdivided.keys()) {
+          this.subdivided.delete(k);
+          toRemove--;
+          if (toRemove <= 0) break;
+        }
+      }
+    }
+  }
+
   async invalidate(source) {
     const key = source.getKey();
     if (this.invalidations.get(key)) {

package/src/source.js

@@ -18,9 +18,12 @@ export class Source {
  * see https://developer.mozilla.org/en-US/docs/Web/API/File_API
  */
 export class FileSource extends Source {
-  constructor(file) {
+  constructor(file, options) {
     super();
     this.file = file;
+    if (options && options.coverageCheck) {
+      this.coverageCheck = options.coverageCheck;
+    }
   }
 
   getKey() {
@@ -48,14 +51,23 @@ export class FileSource extends Source {
  * This also works around browser and storage-specific edge cases.
  */
 export class FetchSource extends Source {
-  constructor(url, customHeaders = new Headers()) {
+  constructor(url, options, customHeaders = new Headers()) {
     super();
     this.url = url;
     this.customHeaders = customHeaders;
     this.mustReload = false;
+    // Safe global detection for legacy environments without globalThis
+    const g =
+      typeof globalThis !== "undefined"
+        ? globalThis
+        : typeof window !== "undefined"
+        ? window
+        : typeof self !== "undefined"
+        ? self
+        : {};
     let userAgent = "";
-    if ("navigator" in globalThis) {
-      userAgent = globalThis.navigator.userAgent || "";
+    if (g && typeof g.navigator !== "undefined") {
+      userAgent = g.navigator.userAgent || "";
     }
     const isWindows = userAgent.indexOf("Windows") > -1;
     const isChromiumBased = /Chrome|Chromium|Edg|OPR|Brave/.test(userAgent);
@@ -63,6 +75,9 @@ export class FetchSource extends Source {
     if (isWindows && isChromiumBased) {
       this.chromeWindowsNoCache = true;
     }
+    if (options && options.coverageCheck) {
+      this.coverageCheck = options.coverageCheck;
+    }
   }
 
   getKey() {

package/src/tilecutter/IMPLEMENTATION_PLAN.md

@@ -0,0 +1,135 @@
+# Vector Tile Subdivision (Tilecutter) – Current Status & Future Work
+
+## Implemented (Current State)
+
+The following pieces are complete and in production code:
+
+- `subdivideVectorTile(...)` Phase A implementation using existing decode (`@mapbox/vector-tile`) and encode (`@maplibre/vt-pbf`) libraries.
+- Scale/offset math for arbitrary `dz = targetZ - parentZ` with containment validation.
+- Geometry transformation and clipping (buffer default 128) for points, lines, polygons.
+- Layer filtering via `includeLayers` (array or RegExp).
+- Stats returned (`featuresIn`, `featuresOut`, `dz`, `ms`).
+- Automatic parent selection via `coverageMap` (value=1) and subdivision fallback in `TilePackage.getZxyAttempt`.
+- In‑memory cache of synthesized tiles (`SharedPromiseCache.subdivided`) with simple half-prune heuristic.
+- Default `coverageCheck` enabled and propagated to both `FetchSource` and `FileSource`.
+- Debug instrumentation removed (warnings only for failures & high dz).
+
+## Deferred / Not Yet Implemented
+
+The items below represent remaining or future enhancements. They supersede earlier phased plan language.
+
+1. Unit Tests
+
+- Synthetic fixtures for point, line, polygon, multi-dz (e.g. dz=1,2,3).
+- Verify clipping correctness (edge touch, full outside, polygon ring closure).
+- Regression tests for parent containment validation and empty output tiles.
+
+2. Benchmarks
+
+- Measure average subdivision latency for typical dz values (1–5) and publish a README table.
+- Track memory footprint of cached tiles.
+
+3. Cache Improvements
+
+- Replace simple prune with true LRU (track lastUsed for subdivided tiles).
+- Optional byte-size cap (e.g. max 64 MB) + eviction on overflow.
+- Expose cache stats (hits, misses, evictions) via an API or debug callback.
+
+4. Max dz Guard
+
+- Configurable hard cap `maxDz` (skip synthesis beyond threshold; return empty tile or parent tile).
+- Distinct from warning-only `maxDzWarn` currently in function.
+
+5. Dependency Reduction (Minimal MVT Codec)
+
+- Custom lightweight decoder: parse layers, feature type, properties, geometry command stream.
+- Custom encoder for POINT, LINESTRING, POLYGON; property passthrough.
+- Goal: shrink bundle size and reduce external dependencies.
+
+6. Geometry Simplification (High dz)
+
+- Optional per-layer simplification tolerance parameter (Douglas-Peucker or grid-based).
+- Preserve topology for polygons; avoid slivers.
+
+7. Synthetic Metadata & Flags
+
+- Option to embed a small custom layer or feature attribute indicating `parentZ`/`dz` for diagnostics.
+- API return could include `synthetic: true` marker (currently implicit via stats presence).
+
+8. Parent Search Strategy Options
+
+- Fallback mode that uses nearest ancestor even if value != 1 (for packages lacking proper coverage semantics).
+- Configurable allowlist of coverage values (e.g. treat `1` or `"blob"` as valid parents).
+
+9. Observability & Metrics
+
+- Pluggable callback `onSubdivide(stats, context)`.
+- Accumulate moving averages; expose via `TilePackage.getSubdivisionMetrics()`.
+
+10. Memory & Concurrency Controls
+
+- Limit parallel subdivision operations (queue) to avoid main-thread spikes.
+- Optional Web Worker offload (future) for heavy dz.
+
+11. Error Handling Extensions
+
+- Graceful fallback: on subdivision failure, return blank tile vs warning.
+- Configurable retry for transient decode issues.
+
+12. Documentation Enhancements
+
+- README section: “Overzoom Mechanics & Tradeoffs”.
+- Guidance on choosing `maxDz` based on feature density.
+
+13. Security / Validation
+
+- Defensive checks on malformed PBF (guard against extremely large extents or geometry counts).
+
+14. Feature Count / Size Limits
+
+- Option `maxFeaturesOut` to truncate layers—report truncation in stats.
+
+15. Layer-Specific Options
+
+- Per-layer buffer overrides; per-layer simplification tolerance.
+
+## Updated API Roadmap (Proposed Additions)
+
+```ts
+interface SubdivideOptions {
+  buffer?: number; // existing
+  includeLayers?: string[] | RegExp; // existing
+  maxDzWarn?: number; // existing
+  maxDz?: number; // NEW hard cap
+  simplifyTolerance?: number; // NEW (all layers)
+  onStats?: (stats, ctx) => void; // NEW callback
+  maxFeaturesOut?: number; // NEW truncation limit
+  parentStrategy?: "coverageValue" | "nearestAncestor"; // NEW
+}
+```
+
+## Prioritization Suggestion (Next Steps)
+
+1. Implement unit tests (foundation for safe refactors).
+2. Add LRU cache + metrics collection.
+3. Introduce `maxDz` guard & benchmark to inform default.
+4. Optional: minimal codec prototype behind feature flag for size comparisons.
+5. Geometry simplification + truncation limits for high-density datasets.
+6. Parent strategy option for archives with unconventional coverage values.
+
+## Risks & Mitigations (Current)
+
+- High dz latency: mitigate with `maxDz` and caching.
+- Memory growth from cache: LRU + size cap.
+- Bundle size: address via minimal codec.
+- Incorrect clipping for edge cases: comprehensive unit tests.
+
+## Open Questions
+
+- Should synthetic tiles preserve empty layers (current: omit empty)?
+- Provide global toggle to disable stats collection for minimal overhead?
+- Standardize metadata flag naming (`vtpk_synthetic`?) if added.
+
+---
+
+This document now reflects current implementation and focuses on forward-looking tasks. Completed exploratory/phase details were removed for clarity.

package/src/tilecutter/subdivide.js

@@ -0,0 +1,247 @@
+// Phase A: Subdivide a parent vector tile into a higher zoom target tile.
+// Minimal dependency implementation using @mapbox/vector-tile for decode and @maplibre/vt-pbf for encode.
+// Public API: subdivideVectorTile(parentPbf, parentZ, parentX, parentY, targetZ, targetX, targetY, options)
+// Returns { data: Uint8Array, stats }
+
+import {
+  VectorTile,
+  VectorTileFeature,
+  VectorTileLayer,
+} from "@mapbox/vector-tile";
+import Protobuf from "pbf";
+import Point from "@mapbox/point-geometry";
+import { fromVectorTileJs } from "@maplibre/vt-pbf";
+
+/**
+ * Subdivide parent tile PBF into higher zoom target tile.
+ * @param {ArrayBuffer|Uint8Array} parentPbf - Raw parent tile PBF bytes.
+ * @param {number} parentZ
+ * @param {number} parentX
+ * @param {number} parentY
+ * @param {number} targetZ
+ * @param {number} targetX
+ * @param {number} targetY
+ * @param {object} options
+ * @param {number} [options.buffer=128] - Clip buffer.
+ * @param {string[]|RegExp} [options.includeLayers] - Layer filter.
+ * @param {number} [options.maxDzWarn=8] - Warn if dz larger.
+ * @returns {{ data: Uint8Array, stats: any }}
+ */
+export function subdivideVectorTile(
+  parentPbf,
+  parentZ,
+  parentX,
+  parentY,
+  targetZ,
+  targetX,
+  targetY,
+  options = {},
+) {
+  const t0 = performance.now ? performance.now() : Date.now();
+  const { buffer = 128, includeLayers, maxDzWarn = 8 } = options;
+  const dz = targetZ - parentZ;
+  if (dz <= 0) {
+    // Nothing to subdivide, caller should just use original bytes.
+    return { data: toUint8(parentPbf), stats: { dz, passthrough: true } };
+  }
+  if (dz > maxDzWarn && console && console.warn) {
+    console.warn(
+      `subdivideVectorTile: high dz=${dz}, performance may degrade.`,
+    );
+  }
+  // Validate containment.
+  const scale = Math.pow(2, dz);
+  const expectedParentX = Math.floor(targetX / scale);
+  const expectedParentY = Math.floor(targetY / scale);
+  if (expectedParentX !== parentX || expectedParentY !== parentY) {
+    throw new Error(
+      "Target tile not contained within parent tile coordinates.",
+    );
+  }
+
+  const vt = new VectorTile(new Protobuf(toUint8(parentPbf)));
+
+  const overzoomed = new OverzoomedVectorTile();
+  let featuresIn = 0;
+  let featuresOut = 0;
+  for (const layerName in vt.layers) {
+    if (!Object.prototype.hasOwnProperty.call(vt.layers, layerName)) continue;
+    if (includeLayers) {
+      if (Array.isArray(includeLayers) && !includeLayers.includes(layerName))
+        continue;
+      if (includeLayers instanceof RegExp && !includeLayers.test(layerName))
+        continue;
+    }
+    const sourceLayer = vt.layers[layerName];
+    const extent = sourceLayer.extent;
+    const offsetX = (targetX - parentX * scale) * extent;
+    const offsetY = (targetY - parentY * scale) * extent;
+
+    const featureWrappers = [];
+    for (let i = 0; i < sourceLayer.length; i++) {
+      const feature = sourceLayer.feature(i);
+      featuresIn++;
+      let geometry = feature.loadGeometry(); // Array of rings (Point[])
+      // Transform
+      for (const ring of geometry) {
+        for (const pt of ring) {
+          pt.x = pt.x * scale - offsetX;
+          pt.y = pt.y * scale - offsetY;
+        }
+      }
+      // Clip
+      geometry = clipGeometry(geometry, feature.type, -buffer, extent + buffer);
+      if (!geometry || geometry.length === 0) continue;
+      featureWrappers.push(
+        new OverzoomedFeatureWrapper(
+          feature.type,
+          geometry,
+          feature.properties,
+          feature.id,
+          extent,
+        ),
+      );
+      featuresOut++;
+    }
+    if (featureWrappers.length > 0) {
+      overzoomed.addLayer(
+        new OverzoomedTileLayer(featureWrappers, layerName, extent),
+      );
+    }
+  }
+  const pbf = fromVectorTileJs(overzoomed);
+  const data =
+    pbf.byteOffset === 0 && pbf.byteLength === pbf.buffer.byteLength
+      ? pbf
+      : new Uint8Array(pbf);
+  const t1 = performance.now ? performance.now() : Date.now();
+  return { data, stats: { dz, featuresIn, featuresOut, ms: t1 - t0 } };
+}
+
+// --- Internal wrappers (adapted from overzoomed-geojson-vector-tile.ts) ---
+class OverzoomedFeatureWrapper extends VectorTileFeature {
+  constructor(type, geometry, properties, id, extent) {
+    super(new Protobuf(), 0, extent, [], []);
+    this.type = type; // 1=Point,2=LineString,3=Polygon
+    this.properties = properties ? properties : {};
+    this.extent = extent;
+    this.myGeometry = geometry;
+    this.id = id;
+  }
+  loadGeometry() {
+    return this.myGeometry.map((ring) => ring.map((p) => new Point(p.x, p.y)));
+  }
+}
+
+class OverzoomedTileLayer extends VectorTileLayer {
+  constructor(features, name, extent) {
+    super(new Protobuf());
+    this._myFeatures = features;
+    this.name = name;
+    this.length = features.length;
+    this.extent = extent;
+    this.version = 2;
+  }
+  feature(i) {
+    return this._myFeatures[i];
+  }
+}
+
+class OverzoomedVectorTile {
+  constructor() {
+    this.layers = {};
+  }
+  addLayer(layer) {
+    this.layers[layer.name] = layer;
+  }
+}
+
+// --- Clipping helpers ---
+function clipGeometry(geometry, type, min, max) {
+  switch (type) {
+    case 1:
+      return clipPoints(geometry, min, max); // POINT
+    case 2:
+      return clipLinesWrapper(geometry, min, max, false); // LINESTRING
+    case 3:
+      return clipLinesWrapper(geometry, min, max, true); // POLYGON
+    default:
+      return [];
+  }
+}
+function clipPoints(geometry, min, max) {
+  const out = [];
+  for (const ring of geometry) {
+    for (const p of ring) {
+      if (p.x >= min && p.x <= max && p.y >= min && p.y <= max) {
+        out.push([p]);
+      }
+    }
+  }
+  return out;
+}
+function clipLinesWrapper(geometry, min, max, isPolygon) {
+  const out = [];
+  for (const line of geometry) {
+    const clipped = clipAxis(line, min, max, 0, isPolygon);
+    for (const seg of clipped) {
+      const clippedY = clipAxis(seg, min, max, 1, isPolygon);
+      for (const finalSeg of clippedY) out.push(finalSeg);
+    }
+  }
+  return out;
+}
+function clipAxis(line, start, end, axis, isPolygon) {
+  const intersect = axis === 0 ? intersectX : intersectY;
+  let slice = [];
+  const out = [];
+  for (let i = 0; i < line.length - 1; i++) {
+    const p1 = line[i];
+    const p2 = line[i + 1];
+    const a = axis === 0 ? p1.x : p1.y;
+    const b = axis === 0 ? p2.x : p2.y;
+    let exited = false;
+    if (a < start && b > start) slice.push(intersect(p1, p2, start));
+    else if (a > end && b < end) slice.push(intersect(p1, p2, end));
+    else if (a >= start && a <= end) slice.push(p1);
+    if (b < start && a >= start) {
+      slice.push(intersect(p1, p2, start));
+      exited = true;
+    }
+    if (b > end && a <= end) {
+      slice.push(intersect(p1, p2, end));
+      exited = true;
+    }
+    if (!isPolygon && exited) {
+      out.push(slice);
+      slice = [];
+    }
+  }
+  const last = line[line.length - 1];
+  const lastVal = axis === 0 ? last.x : last.y;
+  if (lastVal >= start && lastVal <= end) slice.push(last);
+  if (
+    isPolygon &&
+    slice.length > 0 &&
+    !slice[0].equals(slice[slice.length - 1])
+  ) {
+    slice.push(new Point(slice[0].x, slice[0].y));
+  }
+  if (slice.length > 0) out.push(slice);
+  return out;
+}
+function intersectX(p1, p2, x) {
+  const t = (x - p1.x) / (p2.x - p1.x);
+  return new Point(x, p1.y + (p2.y - p1.y) * t);
+}
+function intersectY(p1, p2, y) {
+  const t = (y - p1.y) / (p2.y - p1.y);
+  return new Point(p1.x + (p2.x - p1.x) * t, y);
+}
+
+function toUint8(data) {
+  if (data instanceof Uint8Array) return data;
+  return new Uint8Array(data);
+}
+
+export default subdivideVectorTile;