@loaders.gl/tiles 5.0.0-alpha.11 → 5.0.0-alpha.13

package/src/pmtiles/pmtiles-source-loader.ts

@@ -0,0 +1,346 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {GeoArrowEncodingPreference, Schema} from '@loaders.gl/core/schema';
+import type {
+  CoreAPI,
+  SourceLoader,
+  VectorTile,
+  VectorTileSource,
+  GetTileParameters,
+  GetTileDataParameters,
+  ImageTileSource,
+  ImageType,
+  RangeRequestSchedulerProps
+} from '@loaders.gl/core/loader-utils';
+import {
+  DataSource,
+  DataSourceOptions,
+  resolvePath,
+  resolveXYZTileParameters,
+  waitForPromiseWithSignal
+} from '@loaders.gl/core/loader-utils';
+import {ImageBitmapLoader, ImageBitmapLoaderOptions} from '@loaders.gl/images';
+import {MLTLoader, MLTLoaderOptions} from '@loaders.gl/tiles/mlt';
+import {MVTLoader, MVTLoaderOptions, TileJSONLoaderOptions} from '@loaders.gl/tiles/mvt';
+import {PMTilesFormat} from './pmtiles-format';
+
+import * as pmtiles from 'pmtiles';
+const {PMTiles} = pmtiles;
+
+import type {PMTilesMetadata} from './lib/parse-pmtiles';
+import {parsePMTilesHeader} from './lib/parse-pmtiles';
+import {BlobSource} from './lib/blob-source';
+import {RangeRequestSource} from './lib/range-request-source';
+
+const VERSION = '1.0.0';
+
+/** Range request options for PMTiles URL sources. */
+export type PMTilesRangeRequestOptions = RangeRequestSchedulerProps & {
+  /** Reserved concurrency hint for range-request transports. */
+  maxConcurrentRequests?: number;
+};
+
+export type PMTilesSourceLoaderOptions = DataSourceOptions &
+  TileJSONLoaderOptions &
+  MLTLoaderOptions &
+  MVTLoaderOptions &
+  ImageBitmapLoaderOptions & {
+    /** Preferred encoding for Arrow geometry output. */
+    geoarrow?: {encodingPreference?: GeoArrowEncodingPreference};
+    pmtiles?: {
+      /** Explicit source CRS contract; PMTiles addressing alone does not establish geographic geometry. */
+      coordinateReferenceSystem?: 'EPSG:3857';
+      /** Logical XYZ pixel frame, independent of encoded tile dimensions or vector extent. */
+      tileSize?: number;
+      /** Shape of returned vector tile data. */
+      shape?: 'geojson-table' | 'columnar-table' | 'binary-geometry' | 'arrow-table';
+      /** Preferred encoding for Arrow geometry output. */
+      geoarrow?: {encodingPreference?: GeoArrowEncodingPreference};
+    };
+    rangeRequests?: PMTilesRangeRequestOptions;
+  };
+
+/**
+ * Creates vector tile data sources for PMTiles urls or blobs
+ */
+export const PMTilesSourceLoader = {
+  dataType: null as unknown as PMTilesTileSource,
+  batchType: null as never,
+  ...PMTilesFormat,
+  version: VERSION,
+  type: 'pmtiles',
+  fromUrl: true,
+  fromBlob: true,
+
+  options: {
+    pmtiles: {
+      shape: 'arrow-table'
+    }
+  },
+
+  defaultOptions: {
+    pmtiles: {}
+  },
+
+  testURL: (url: string) => url.endsWith('.pmtiles'),
+  createDataSource: (url: string | Blob, options: PMTilesSourceLoaderOptions, coreApi?: CoreAPI) =>
+    new PMTilesTileSource(url, options, coreApi)
+} as const satisfies SourceLoader<PMTilesTileSource>;
+
+/**
+ * A PMTiles data source
+ * @note Can be either a raster or vector tile source depending on the contents of the PMTiles file.
+ */
+export class PMTilesTileSource
+  extends DataSource<string | Blob, PMTilesSourceLoaderOptions>
+  implements ImageTileSource, VectorTileSource
+{
+  mimeType: string | null = null;
+  pmtiles: pmtiles.PMTiles;
+  metadata: Promise<PMTilesMetadata>;
+  private pendingTileRequests: PendingTileRequest[] = [];
+  private tileBatchTimer: ReturnType<typeof setTimeout> | null = null;
+
+  constructor(data: string | Blob, options: PMTilesSourceLoaderOptions, coreApi?: CoreAPI) {
+    super(data, options, PMTilesSourceLoader.defaultOptions, coreApi);
+    const rangeRequestOptions = options.rangeRequests;
+    const urlOrBlob =
+      typeof data === 'string'
+        ? new RangeRequestSource(resolvePath(data), {
+            ...rangeRequestOptions,
+            batchDelayMs: 0,
+            fetch: this.fetch
+          })
+        : new BlobSource(data, 'pmtiles');
+    this.pmtiles = new PMTiles(urlOrBlob);
+    this.getTileData = this.getTileData.bind(this);
+    this.metadata = this.getMetadata();
+  }
+
+  async getSchema(): Promise<Schema> {
+    return {fields: [], metadata: {}};
+  }
+
+  async getMetadata(): Promise<PMTilesMetadata> {
+    const pmtilesHeader = await this.pmtiles.getHeader();
+    const pmtilesMetadata = ((await this.pmtiles.getMetadata()) as Record<string, unknown>) || {};
+    const metadata: PMTilesMetadata = parsePMTilesHeader(
+      pmtilesHeader,
+      pmtilesMetadata,
+      {
+        includeFormatHeader: false,
+        coordinateReferenceSystem: this.options.pmtiles?.coordinateReferenceSystem,
+        tileSize: this.options.pmtiles?.tileSize
+      },
+      this.loadOptions
+    );
+    // Add additional attribution if necessary
+    if (this.options.attributions) {
+      metadata.attributions = [
+        ...(this.options.core?.attributions || []),
+        ...(metadata.attributions || [])
+      ];
+    }
+    if (metadata?.tileMIMEType) {
+      this.mimeType = metadata?.tileMIMEType;
+    }
+    return metadata;
+  }
+
+  async getTile(tileParams: GetTileParameters): Promise<ArrayBuffer | null> {
+    const resolved = resolveXYZTileParameters(
+      (await waitForPromiseWithSignal(this.metadata, tileParams.signal)).tileGrid,
+      tileParams
+    );
+    if (!resolved) return null;
+    const {x, y, z} = resolved;
+    let rangeResponse;
+    try {
+      rangeResponse = await this.getZxyBatched(z, x, y, tileParams.signal);
+    } catch (error) {
+      if (tileParams.signal?.aborted) throw tileParams.signal.reason;
+      if (error instanceof Error && error.name === 'AbortError') throw error;
+      this.reportError(error, `Failed to fetch tile ${this.url} ${JSON.stringify(tileParams)}`);
+      return null;
+    }
+    if (tileParams.signal?.aborted) throw tileParams.signal.reason;
+    const arrayBuffer = rangeResponse?.data;
+    if (!arrayBuffer) {
+      this.reportError(
+        new Error('Empty tile response'),
+        `Failed to fetch tile ${this.url} ${JSON.stringify(tileParams)}`
+      );
+      return null;
+    }
+    return arrayBuffer;
+  }
+
+  // Tile Source interface implementation: deck.gl compatible API
+  // TODO - currently only handles image tiles, not vector tiles
+
+  async getTileData(tileParams: GetTileDataParameters): Promise<any> {
+    const {x, y, z} = tileParams.index;
+    const metadata = await waitForPromiseWithSignal(this.metadata, tileParams.signal);
+    switch (metadata.tileMIMEType) {
+      case 'application/vnd.mapbox-vector-tile':
+      case 'application/vnd.maplibre-tile':
+        return await this.getVectorTile({
+          x,
+          y,
+          z,
+          layers: [],
+          tileMatrix: tileParams.tileMatrix,
+          signal: tileParams.signal
+        });
+      default:
+        return await this.getImageTile({
+          x,
+          y,
+          z,
+          layers: [],
+          tileMatrix: tileParams.tileMatrix,
+          signal: tileParams.signal
+        });
+    }
+  }
+
+  getTileBatch(tileParams: readonly GetTileParameters[]): readonly Promise<ArrayBuffer | null>[] {
+    return tileParams.map(tileParam => this.getTile(tileParam));
+  }
+
+  getTileDataBatch(tileParams: readonly GetTileDataParameters[]): readonly Promise<any>[] {
+    return tileParams.map(tileParam => this.getTileData(tileParam));
+  }
+
+  // ImageTileSource interface implementation
+
+  async getImageTile(tileParams: GetTileParameters): Promise<ImageType | null> {
+    const arrayBuffer = await this.getTile(tileParams);
+    return arrayBuffer
+      ? ((await this.coreApi.parse(arrayBuffer, ImageBitmapLoader, this.loadOptions)) as ImageType)
+      : null;
+  }
+
+  // VectorTileSource interface implementation
+
+  async getVectorTile(tileParams: GetTileParameters): Promise<VectorTile | null> {
+    const metadata = await waitForPromiseWithSignal(this.metadata, tileParams.signal);
+    const resolved = resolveXYZTileParameters(metadata.tileGrid, tileParams);
+    if (!resolved) return null;
+    tileParams = resolved;
+    const arrayBuffer = await this.getTile(tileParams);
+    const tileMIMEType = this.mimeType || metadata?.tileMIMEType;
+
+    if (tileMIMEType === 'application/vnd.maplibre-tile') {
+      const inheritedMLTOptions = (this.loadOptions as MLTLoaderOptions)?.mlt;
+      const loadOptions: MLTLoaderOptions = {
+        ...this.loadOptions,
+        mlt: {
+          ...inheritedMLTOptions,
+          shape:
+            getMLTShape(this.options.pmtiles?.shape) || inheritedMLTOptions?.shape || 'arrow-table',
+          coordinates: 'wgs84',
+          tileIndex: {x: tileParams.x, y: tileParams.y, z: tileParams.z},
+          layers: normalizeTileLayers(tileParams.layers) || inheritedMLTOptions?.layers,
+          geoarrow:
+            (this.options as PMTilesSourceLoaderOptions).geoarrow ||
+            this.options.pmtiles?.geoarrow ||
+            inheritedMLTOptions?.geoarrow
+        }
+      };
+
+      return arrayBuffer
+        ? ((await this.coreApi.parse(arrayBuffer, MLTLoader, loadOptions)) as VectorTile)
+        : null;
+    }
+
+    const inheritedMVTOptions = (this.loadOptions as MVTLoaderOptions)?.mvt;
+    const selectedLayers = normalizeTileLayers(tileParams.layers) || inheritedMVTOptions?.layers;
+    const loadOptions: MVTLoaderOptions = {
+      ...this.loadOptions,
+      mvt: {
+        ...inheritedMVTOptions,
+        shape: this.options.pmtiles?.shape || inheritedMVTOptions?.shape || 'arrow-table',
+        coordinates: 'wgs84',
+        tileIndex: {x: tileParams.x, y: tileParams.y, z: tileParams.z},
+        layers: selectedLayers,
+        geoarrow:
+          (this.options as PMTilesSourceLoaderOptions).geoarrow ||
+          this.options.pmtiles?.geoarrow ||
+          inheritedMVTOptions?.geoarrow
+      }
+    };
+
+    return arrayBuffer
+      ? ((await this.coreApi.parse(arrayBuffer, MVTLoader, loadOptions)) as VectorTile)
+      : null;
+  }
+
+  private getZxyBatched(
+    z: number,
+    x: number,
+    y: number,
+    signal?: AbortSignal
+  ): Promise<pmtiles.RangeResponse | undefined> {
+    if (this.data instanceof Blob) {
+      return this.pmtiles.getZxy(z, x, y, signal);
+    }
+
+    return new Promise((resolve, reject) => {
+      this.pendingTileRequests.push({z, x, y, signal, resolve, reject});
+      this.scheduleTileBatch();
+    });
+  }
+
+  private scheduleTileBatch(): void {
+    if (this.tileBatchTimer) {
+      return;
+    }
+
+    const batchDelayMs = this.options.rangeRequests?.batchDelayMs ?? 50;
+    this.tileBatchTimer = setTimeout(() => this.flushTileBatch(), batchDelayMs);
+  }
+
+  private flushTileBatch(): void {
+    if (this.tileBatchTimer) {
+      clearTimeout(this.tileBatchTimer);
+      this.tileBatchTimer = null;
+    }
+
+    const pendingTileRequests = this.pendingTileRequests;
+    this.pendingTileRequests = [];
+
+    for (const tileRequest of pendingTileRequests) {
+      this.pmtiles
+        .getZxy(tileRequest.z, tileRequest.x, tileRequest.y, tileRequest.signal)
+        .then(tileRequest.resolve, tileRequest.reject);
+    }
+  }
+}
+
+/** Normalizes a non-empty tile layer selection for the MVT decoder. */
+function normalizeTileLayers(layers?: string | string[]): string[] | undefined {
+  const normalizedLayers = typeof layers === 'string' ? [layers] : layers;
+  return normalizedLayers?.length ? normalizedLayers : undefined;
+}
+
+/** Returns a shape supported by the MLT decoder. */
+function getMLTShape(
+  shape?: NonNullable<PMTilesSourceLoaderOptions['pmtiles']>['shape']
+): NonNullable<MLTLoaderOptions['mlt']>['shape'] | undefined {
+  if (shape === 'columnar-table') {
+    throw new Error('PMTilesTileSource: columnar-table shape is not supported for MLT tiles');
+  }
+  return shape;
+}
+
+type PendingTileRequest = {
+  z: number;
+  x: number;
+  y: number;
+  signal?: AbortSignal;
+  resolve: (response: pmtiles.RangeResponse | undefined) => void;
+  reject: (error: unknown) => void;
+};

package/src/pmtiles/pmtiles-writer-types.ts

@@ -0,0 +1,45 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** One encoded tile, addressed using the XYZ convention (north-origin Y). */
+export interface PMTilesTile {
+  /** Tile zoom, an integer from 0 through 26. */
+  readonly z: number;
+  /** Tile column at this zoom. */
+  readonly x: number;
+  /** Tile row at this zoom. */
+  readonly y: number;
+  /** Encoded payload; views preserve their byte offset and length. */
+  readonly data: ArrayBuffer | ArrayBufferView | Blob;
+}
+
+/** Explicit finite tile enumeration; async iteration does not imply streaming output. */
+export type PMTilesTileInput = Iterable<PMTilesTile> | AsyncIterable<PMTilesTile>;
+
+/** A single payload format applies to every tile in the archive. */
+export type PMTilesTileType = 'mvt' | 'mlt' | 'png' | 'jpeg' | 'webp' | 'avif';
+
+/** Encoding controls for a bounded, in-memory PMTiles v3 archive. */
+export type PMTilesEncodeOptions = {
+  /** Declares the existing payload format; required, never inferred from opaque bytes. */
+  readonly tileType: PMTilesTileType;
+  /** Existing compression of all payloads; default none. Payloads are not recompressed. */
+  readonly tileCompression?: 'none' | 'gzip' | 'brotli' | 'zstd';
+  /** Directory and metadata compression; default none. */
+  readonly internalCompression?: 'none' | 'gzip';
+  /** JSON metadata. MVT requires TileJSON vector_layers with layer IDs and fields. */
+  readonly metadata?: Readonly<Record<string, unknown>>;
+  /** Geographic bounds [west, south, east, north]; defaults to selected tile footprints. */
+  readonly bounds?: readonly [number, number, number, number];
+  /** Initial view [longitude, latitude, zoom]; defaults to bounds midpoint and minimum zoom. */
+  readonly center?: readonly [number, number, number];
+  /** Maximum enumerated tiles; default 100,000. */
+  readonly maxTiles?: number;
+  /** Maximum archive bytes, including headers and directories; default 256 MiB. */
+  readonly maxOutputBytes?: number;
+  /** Maximum UTF-8 JSON metadata bytes before compression; default 1 MiB. */
+  readonly maxMetadataBytes?: number;
+  /** Cancellation checked during enumeration and between encoding stages. */
+  readonly signal?: AbortSignal;
+};

package/src/pmtiles/pmtiles-writer.ts

@@ -0,0 +1,26 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {WriterOptions, WriterWithEncoder} from '@loaders.gl/core/loader-utils';
+import {PMTilesFormat} from './pmtiles-format';
+import {encodePMTiles} from './lib/encode-pmtiles';
+import type {PMTilesEncodeOptions, PMTilesTileInput} from './pmtiles-writer-types';
+
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+/** Options for packaging already encoded tiles with the standard writer API. */
+export type PMTilesWriterOptions = WriterOptions & {
+  /** Archive format, metadata, compression declarations and resource budgets. */
+  pmtiles?: PMTilesEncodeOptions;
+};
+
+/** Bounded, portable PMTiles v3 writer; tile payloads are copied without transcoding. */
+export const PMTilesWriter = {
+  ...PMTilesFormat,
+  version: VERSION,
+  options: {},
+  encode: (tiles: PMTilesTileInput, options?: PMTilesWriterOptions): Promise<ArrayBuffer> =>
+    encodePMTiles(tiles, options?.pmtiles)
+} as const satisfies WriterWithEncoder<PMTilesTileInput, never, PMTilesWriterOptions>;