maplibre-tilepackage-protocol 1.0.0 → 1.2.0

package/rollup.config.mjs

@@ -0,0 +1,45 @@
+// rollup.config.mjs
+// node resolves makes npm packages be included in the bundle
+import { nodeResolve } from "@rollup/plugin-node-resolve";
+import commonjs from "@rollup/plugin-commonjs";
+
+const externals = [
+  "@mapbox/vector-tile",
+  "pbf",
+  "@mapbox/point-geometry",
+  "@maplibre/vt-pbf",
+  "maplibre-local-glyphs",
+];
+
+function onwarn(warning) {
+  if (warning.code === "CIRCULAR_DEPENDENCY") return;
+  console.warn(warning.message);
+}
+
+export default [
+  // ESM library build (dependencies external)
+  {
+    input: "src/index.js",
+    output: {
+      file: "dist/index.esm.js",
+      format: "esm",
+      sourcemap: true,
+    },
+    // Keep selected externals; bundle fflate (omit from externals)
+    external: externals,
+    plugins: [nodeResolve({ browser: true }), commonjs()],
+    onwarn,
+  },
+  // IIFE demo build (fully bundled)
+  {
+    input: "src/index.js",
+    output: {
+      file: "dist/bundle.js",
+      format: "iife",
+      name: "TilePackageProtocol",
+      sourcemap: true,
+    },
+    plugins: [nodeResolve({ browser: true }), commonjs()],
+    onwarn,
+  },
+];

package/src/get-header-and-filelist.js

@@ -59,6 +59,7 @@ export function jsonToHeader(json, files, etag) {
     maxLon: Number(json.extent.xmax),
     maxLat: Number(json.extent.ymax),
     files: files,
+    coverageMap: json.coverageMap,
     etag: etag,
   };
 }
@@ -239,11 +240,49 @@ export default async function getHeaderAndFileList(source) {
       root.metadataOffset = metadataOffset;
       root.metadataLength = tilePackageFiles["p12/metadata.json"].size;
     }
+
+    async function calculateCoverage(z, x, y, tilemap, coverageMap) {
+      z = z + 1;
+      x = x * 2;
+      y = y * 2;
+      if (coverageMap[z] == undefined) coverageMap[z] = {};
+      if (coverageMap[z][x] == undefined) coverageMap[z][x] = {};
+      if (coverageMap[z][x + 1] == undefined) coverageMap[z][x + 1] = {};
+      coverageMap[z][x][y] = tilemap[0];
+      if (isNaN(tilemap[0])) {
+        calculateCoverage(z, x, y, tilemap[0], coverageMap);
+      }
+      coverageMap[z][x + 1][y] = tilemap[1];
+      if (isNaN(tilemap[1])) {
+        calculateCoverage(z, x + 1, y, tilemap[1], coverageMap);
+      }
+      coverageMap[z][x][y + 1] = tilemap[2];
+      if (isNaN(tilemap[2])) {
+        calculateCoverage(z, x, y + 1, tilemap[2], coverageMap);
+      }
+      coverageMap[z][x + 1][y + 1] = tilemap[3];
+      if (isNaN(tilemap[3])) {
+        calculateCoverage(z, x + 1, y + 1, tilemap[3], coverageMap);
+      }
+    }
+    // Get and calculate the tilemap
+    if (source.coverageCheck > 0 && tilePackageFiles["p12/tilemap/root.json"]) {
+      const tilemap = await getJsonFromFile(
+        "p12/tilemap/root.json",
+        tilePackageFiles,
+        source,
+      );
+      const coverageMap = {};
+      coverageMap[0] = {};
+      coverageMap[0][0] = {};
+      coverageMap[0][0][0] = "blob";
+      calculateCoverage(0, 0, 0, tilemap.index, coverageMap);
+      root.coverageMap = coverageMap;
+    } else {
+      // Coverage map unavailable (either disabled or missing tilemap/root.json)
+    }
   }
-  //TODO read iteminfo.xml....
 
   const header = jsonToHeader(root, tilePackageFiles, resp.etag);
-
-  //return [header, [dirKey, rootDir.length, rootDir]];
   return [header, ""];
 }

package/src/index.js

@@ -1,6 +1,7 @@
 export * from "./maplibre-gl-js-protocol.js";
 export * from "./source.js";
 import { FetchSource } from "./source.js";
+import subdivideVectorTile from "./tilecutter/subdivide.js";
 import defaultDecompress from "./default-decompress.js";
 import getJsonFromFile from "./get-json-from-file.js";
 import SharedPromiseCache from "./shared-promise-cache.js";
@@ -11,12 +12,7 @@ function calculateFilename(z, x, y, header) {
   const baseCol = Math.floor(x / 128) * 128;
   const rowHex = baseRow.toString(16).padStart(4, "0");
   const colHex = baseCol.toString(16).padStart(4, "0");
-  let basePath;
-  if (header.type === "tpkx") {
-    basePath = `tile`;
-  } else {
-    basePath = `p12/tile`;
-  }
+  let basePath = header.type === "tpkx" ? "tile" : "p12/tile";
   return `${basePath}/L${zoom}/R${rowHex}C${colHex}.bundle`;
 }
 
@@ -26,10 +22,6 @@ function getTileInfoFromTileIndex(tileIndex, z, x, y) {
   return tileIndex[row][col];
 }
 
-/**
- * Error thrown when a response for TilePackage over HTTP does not match previous, cached parts of the archive.
- * The default TilePackage implementation will catch this error once internally and retry a request.
- */
 class EtagMismatch extends Error {
   constructor(message) {
     super(message);
@@ -38,24 +30,29 @@ class EtagMismatch extends Error {
 }
 
 export class TilePackage {
-  constructor(source, cache, decompress) {
+  constructor(source, options) {
+    // Default coverageCheck enabled; disable with coverageCheck:false
+    this.coverageCheck = 1;
+    if (options && options.coverageCheck === false) this.coverageCheck = 0;
+    // Max dz guard (skip subdivision beyond this): default 8
+    this.maxDz =
+      options && typeof options.maxDz === "number" ? options.maxDz : 8;
     if (typeof source === "string") {
-      this.source = new FetchSource(source);
+      const opts = this.coverageCheck
+        ? { coverageCheck: this.coverageCheck }
+        : undefined;
+      this.source = new FetchSource(source, opts);
     } else {
       this.source = source;
+      // Propagate default coverageCheck to existing Source instances if undefined
+      if (this.source && this.source.coverageCheck === undefined) {
+        this.source.coverageCheck = this.coverageCheck;
+      }
     }
-
-    if (decompress) {
-      this.decompress = decompress;
-    } else {
-      this.decompress = defaultDecompress;
-    }
-
-    if (cache) {
-      this.cache = cache;
-    } else {
-      this.cache = new SharedPromiseCache();
-    }
+    this.decompress =
+      options && options.decompress ? options.decompress : defaultDecompress;
+    this.cache =
+      options && options.cache ? options.cache : new SharedPromiseCache();
   }
 
   async getHeader() {
@@ -63,42 +60,99 @@ export class TilePackage {
   }
 
   async getZxyAttempt(z, x, y, signal) {
-    //y = (1 << z) - 1 - y; //TMS vs XYZ
     const header = await this.cache.getHeader(this.source);
-    if (z < header.minZoom || z > header.maxZoom) {
-      return undefined;
-    }
+    if (z < header.minZoom || z > header.maxZoom) return undefined;
     const file = calculateFilename(z, x, y, header);
-    if (!header.files[file]) {
-      return undefined;
-    }
-    const tileIndex = await this.cache.getTileIndex(
-      this.source,
-      file,
-      header,
-      signal,
-    );
-    const tileInfo = getTileInfoFromTileIndex(tileIndex, z, x, y);
-    if (tileInfo && tileInfo.tileSize > 0) {
-      const tileOffset =
-        header.files[file].absoluteOffset + tileInfo.tileOffset;
-      const resp = await this.source.getBytes(
-        tileOffset,
-        tileInfo.tileSize,
+    let tileInfo = null;
+    if (header.files[file]) {
+      const tileIndex = await this.cache.getTileIndex(
+        this.source,
+        file,
+        header,
         signal,
-        header.etag,
       );
-      const data = await this.decompress(resp.data, header.tileCompression);
-      return {
-        data: data,
-        cacheControl: resp.cacheControl,
-        expires: resp.expires,
-      };
-    } else {
-      //return await this.getZxyAttempt(z - 1, x / 2, y / 2, signal);
-      // Return undefined if the tile is not found
-      return undefined;
+      tileInfo = getTileInfoFromTileIndex(tileIndex, z, x, y);
+      if (tileInfo && tileInfo.tileSize > 0) {
+        const tileOffset =
+          header.files[file].absoluteOffset + tileInfo.tileOffset;
+        const resp = await this.source.getBytes(
+          tileOffset,
+          tileInfo.tileSize,
+          signal,
+          header.etag,
+        );
+        const data = await this.decompress(resp.data, header.tileCompression);
+        return { data, cacheControl: resp.cacheControl, expires: resp.expires };
+      }
+    }
+    if (header.packageType === "vtpk" && header.coverageMap) {
+      // Ascend to find parent with value 1
+      let pz = z,
+        px = x,
+        py = y,
+        foundParent = null;
+      while (pz > header.minZoom) {
+        pz -= 1;
+        px = Math.floor(px / 2);
+        py = Math.floor(py / 2);
+        const level = header.coverageMap[pz];
+        const val = level && level[px] ? level[px][py] : undefined;
+        if (val === 1) {
+          foundParent = { pz, px, py };
+          break;
+        }
+      }
+      if (foundParent) {
+        const parent = await this.getZxyAttempt(
+          foundParent.pz,
+          foundParent.px,
+          foundParent.py,
+          signal,
+        );
+        if (parent && parent.data) {
+          const cached = this.cache.getSubdivided(this.source, z, x, y);
+          if (cached) {
+            return {
+              data: cached,
+              cacheControl: parent.cacheControl,
+              expires: parent.expires,
+            };
+          }
+          const dz = z - foundParent.pz;
+          if (dz > this.maxDz) {
+            return undefined; // exceed hard cap
+          }
+          try {
+            const { data: overzoomed } = subdivideVectorTile(
+              parent.data,
+              foundParent.pz,
+              foundParent.px,
+              foundParent.py,
+              z,
+              x,
+              y,
+              { buffer: 128, maxDzWarn: this.maxDz - 1 },
+            );
+            this.cache.setSubdivided(this.source, z, x, y, overzoomed);
+            return {
+              data: overzoomed,
+              cacheControl: parent.cacheControl,
+              expires: parent.expires,
+            };
+          } catch (e) {
+            console.warn(
+              `[tilepackage subdivide] failed for z=${z} x=${x} y=${y}:`,
+              e,
+            );
+          }
+        } else {
+          console.warn(
+            `[tilepackage subdivide] parent tile missing at z=${foundParent.pz} x=${foundParent.px} y=${foundParent.py}`,
+          );
+        }
+      }
     }
+    return undefined;
   }
 
   async getZxy(z, x, y, signal) {
@@ -115,7 +169,6 @@ export class TilePackage {
 
   async getMetadataAttempt() {
     const header = await this.cache.getHeader(this.source);
-
     let metadata = {};
     if (header.packageType === "vtpk") {
       const resp = await this.source.getBytes(
@@ -124,12 +177,10 @@ export class TilePackage {
         undefined,
         header.etag,
       );
-
       const decoder = new TextDecoder("utf-8");
       metadata = JSON.parse(decoder.decode(resp.data));
     }
     metadata.name = header.name;
-
     return metadata;
   }
 
@@ -147,10 +198,7 @@ export class TilePackage {
 
   async getResourceAttempt(file, signal) {
     const header = await this.cache.getHeader(this.source);
-    if (!header.files[file]) {
-      return undefined;
-    }
-
+    if (!header.files[file]) return undefined;
     const resource = await this.cache.getResource(
       this.source,
       file,
@@ -180,37 +228,59 @@ export class TilePackage {
     const header = await this.cache.getHeader(this.source);
     const sourceKey = this.source.getKey();
     if (header.packageType === "vtpk") {
+      const metadata = await getJsonFromFile(
+        "p12/root.json",
+        header.files,
+        this.source,
+      );
       const style = await getJsonFromFile(
         "p12/resources/styles/root.json",
         header.files,
         this.source,
       );
-      style.sources.esri.url = `"tilepackage://${sourceKey}`;
-      style.sources.esri.maxzoom = header.maxZoom || 22;
+      if (style.sources && style.sources.esri) {
+        delete style.sources.esri.url;
+        style.sources.esri.tiles = [`tilepackage://${sourceKey}/{z}/{x}/{y}`];
+        style.sources.esri.type = style.sources.esri.type || "vector";
+        style.sources.esri.minzoom = header.minZoom || 0;
+        style.sources.esri.maxzoom = header.maxZoom || 22;
+      }
+      for (const k in style.sources) {
+        const src = style.sources[k];
+        if (
+          src &&
+          typeof src.url === "string" &&
+          src.url.startsWith("tilepackage://")
+        ) {
+          delete src.url;
+          if (!src.tiles)
+            src.tiles = [`tilepackage://${sourceKey}/{z}/{x}/{y}`];
+        }
+      }
+      if (metadata.copyrightText)
+        style.sources.esri.attribution = metadata.copyrightText;
       style.glyphs = `tilepackage://${sourceKey}/{fontstack}/{range}`;
       style.sprite = `tilepackage://${sourceKey}/sprite`;
-
+      if (this.debug)
+        console.debug(
+          "[tilepackage style] rewritten esri source",
+          style.sources.esri,
+        );
       return style;
-    } else {
-      return {
-        version: 8,
-        sources: {
-          esri: {
-            type: "raster",
-            tileSize: header.tileSize || 256,
-            url: `"tilepackage://${sourceKey}`,
-            maxzoom: header.maxZoom || 22,
-          },
-        },
-        layers: [
-          {
-            id: "tilepackageraster",
-            type: "raster",
-            source: "esri",
-          },
-        ],
-      };
     }
+    return {
+      version: 8,
+      sources: {
+        esri: {
+          type: "raster",
+          tileSize: header.tileSize || 256,
+          tiles: [`tilepackage://${sourceKey}/{z}/{x}/{y}`],
+          maxzoom: header.maxZoom || 22,
+          minzoom: header.minZoom || 0,
+        },
+      },
+      layers: [{ id: "tilepackageraster", type: "raster", source: "esri" }],
+    };
   }
 
   async getStyle() {
@@ -229,7 +299,6 @@ export class TilePackage {
     const header = await this.getHeader();
     const metadata = await this.getMetadata();
     const ext = header.tileType;
-
     const tileJson = {
       tilejson: "3.0.0",
       scheme: "xyz",
@@ -238,21 +307,11 @@ export class TilePackage {
       version: header.version,
       bounds: [header.minLon, header.minLat, header.maxLon, header.maxLat],
     };
-    if (metadata.vector_layers) {
-      tileJson.vector_layers = metadata.vector_layers;
-    }
-    if (metadata.attribution) {
-      tileJson.attribution = metadata.attribution;
-    }
-    if (metadata.description) {
-      tileJson.description = metadata.description;
-    }
-    if (header.minZoom) {
-      tileJson.minzoom = header.minZoom;
-    }
-    if (header.maxZoom) {
-      tileJson.maxzoom = header.maxZoom;
-    }
+    if (metadata.vector_layers) tileJson.vector_layers = metadata.vector_layers;
+    if (metadata.attribution) tileJson.attribution = metadata.attribution;
+    if (metadata.description) tileJson.description = metadata.description;
+    if (header.minZoom) tileJson.minzoom = header.minZoom;
+    if (header.maxZoom) tileJson.maxzoom = header.maxZoom;
     return tileJson;
   }
 }

package/src/maplibre-gl-js-protocol.js

@@ -1,5 +1,5 @@
 import { TilePackage } from "./index.js";
-import protocolGlyphs from "../protocol-glyphs.js";
+import protocolGlyphs from "maplibre-local-glyphs";
 
 function stringEndsIn(string, endings) {
   for (let i = 0; i < endings.length; i++) {
@@ -52,6 +52,7 @@ export class Protocol {
     this.errorOnMissingTile = options
       ? options.errorOnMissingTile || false
       : false;
+    this.debug = options ? options.debug || false : false;
     this.getData = async (params, abortController) => {
       if (params.type === "json") {
         let tilePackageUrl = params.url.substr(14); // TODO fix this to be more robust
@@ -91,9 +92,10 @@ export class Protocol {
         }
         // This is a request for a tile json
         if (this.metadata) {
-          return {
-            data: await instance.getTileJson(params.url),
-          };
+          const tj = await instance.getTileJson(params.url);
+          if (this.debug)
+            console.debug("[tilepackage] tilejson (metadata)", tj);
+          return { data: tj };
         }
         const h = await instance.getHeader();
         if (h.minLon >= h.maxLon || h.minLat >= h.maxLat) {
@@ -102,14 +104,15 @@ export class Protocol {
           );
         }
 
-        return {
-          data: {
-            tiles: [`${params.url}/{z}/{x}/{y}`],
-            minzoom: h.minZoom,
-            maxzoom: h.maxZoom,
-            bounds: [h.minLon, h.minLat, h.maxLon, h.maxLat],
-          },
+        const synthesized = {
+          tiles: [`${params.url}/{z}/{x}/{y}`],
+          minzoom: h.minZoom,
+          maxzoom: h.maxZoom,
+          bounds: [h.minLon, h.minLat, h.maxLon, h.maxLat],
         };
+        if (this.debug)
+          console.debug("[tilepackage] tilejson (synthesized)", synthesized);
+        return { data: synthesized };
         //}
       }
       // TODO handle other paths ???
@@ -197,6 +200,13 @@ export class Protocol {
       const y = result[4];
       const header = await instance.getHeader();
       const resp = await instance.getZxy(+z, +x, +y, abortController.signal);
+      if (this.debug)
+        console.debug("[tilepackage] tile fetch", {
+          z: +z,
+          x: +x,
+          y: +y,
+          found: !!resp,
+        });
       if (resp) {
         return {
           data: new Uint8Array(resp.data),
@@ -206,10 +216,13 @@ export class Protocol {
       }
       if (header.tileType === "pbf") {
         if (this.errorOnMissingTile) {
+          //*
           const e = new Error(
-            "Tile not found in Tile Package, this is normal for VTPK files with variable tile depth.",
-          ); // e.name is 'Error'
+            `Tile [${z},${x},${y}] not found in Tile Package, normal for VTPK with variable depth.`,
+          );
           e.name = "TileError";
+          if (this.debug)
+            console.debug("[tilepackage] missing tile", { z, x, y });
           throw e;
         }
         return { data: new Uint8Array() };
@@ -229,12 +242,15 @@ export class Protocol {
    */
   add(p) {
     this.tiles.set(p.source.getKey(), p);
+    if (this.debug)
+      console.debug("[tilepackage] add instance", p.source.getKey());
   }
 
   /**
    * Fetch a {@link TilePackage} instance by URL, for remote TilePackage instances.
    */
   get(url) {
+    if (this.debug) console.debug("[tilepackage] get instance", url);
     return this.tiles.get(url);
   }
 }

package/src/shared-promise-cache.js

@@ -80,6 +80,7 @@ export default class SharedPromiseCache {
     this.maxCacheEntries = maxCacheEntries;
     this.counter = 1;
     //this.decompress = decompress;
+    this.subdivided = new Map(); // key: sourceKey|z|x|y -> Uint8Array
   }
 
   async getHeader(source) {
@@ -174,6 +175,28 @@ export default class SharedPromiseCache {
     }
   }
 
+  getSubdivided(source, z, x, y) {
+    const key = `${source.getKey()}|${z}|${x}|${y}|Subdivided`;
+    return this.subdivided.get(key);
+  }
+
+  setSubdivided(source, z, x, y, bytes) {
+    const key = `${source.getKey()}|${z}|${x}|${y}|Subdivided`;
+    if (!this.subdivided.has(key)) {
+      this.subdivided.set(key, bytes);
+      // Optional lightweight prune if huge:
+      if (this.subdivided.size > this.maxCacheEntries * 2) {
+        // Remove oldest half (no usage tracking kept; iterate arbitrary)
+        let toRemove = Math.floor(this.subdivided.size / 2);
+        for (const k of this.subdivided.keys()) {
+          this.subdivided.delete(k);
+          toRemove--;
+          if (toRemove <= 0) break;
+        }
+      }
+    }
+  }
+
   async invalidate(source) {
     const key = source.getKey();
     if (this.invalidations.get(key)) {

package/src/source.js

@@ -18,9 +18,12 @@ export class Source {
  * see https://developer.mozilla.org/en-US/docs/Web/API/File_API
  */
 export class FileSource extends Source {
-  constructor(file) {
+  constructor(file, options) {
     super();
     this.file = file;
+    if (options && options.coverageCheck) {
+      this.coverageCheck = options.coverageCheck;
+    }
   }
 
   getKey() {
@@ -48,7 +51,7 @@ export class FileSource extends Source {
  * This also works around browser and storage-specific edge cases.
  */
 export class FetchSource extends Source {
-  constructor(url, customHeaders = new Headers()) {
+  constructor(url, options, customHeaders = new Headers()) {
     super();
     this.url = url;
     this.customHeaders = customHeaders;
@@ -63,6 +66,9 @@ export class FetchSource extends Source {
     if (isWindows && isChromiumBased) {
       this.chromeWindowsNoCache = true;
     }
+    if (options && options.coverageCheck) {
+      this.coverageCheck = options.coverageCheck;
+    }
   }
 
   getKey() {