@loaders.gl/tiles 5.0.0-alpha.11 → 5.0.0-alpha.13

package/src/mvt/tilejson-loader-with-parser.ts

@@ -0,0 +1,38 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/core/loader-utils';
+import type {TileJSON} from './lib/parse-tilejson';
+import {parseTileJSON} from './lib/parse-tilejson';
+import {TileJSONLoader as TileJSONLoaderMetadata} from './tilejson-loader';
+
+const {preload: _TileJSONLoaderPreload, ...TileJSONLoaderMetadataWithoutPreload} =
+  TileJSONLoaderMetadata;
+
+export type TileJSONLoaderOptions = LoaderOptions & {
+  /** Options for the TileJSONLoaderWithParser */
+  tilejson?: {
+    /** Max number of unique values */
+    maxValues?: number;
+  };
+};
+
+/**
+ * Loader for TileJSON metadata
+ */
+export const TileJSONLoaderWithParser = {
+  ...TileJSONLoaderMetadataWithoutPreload,
+  worker: false,
+  parse: async (arrayBuffer: ArrayBuffer, options?: TileJSONLoaderOptions) => {
+    const jsonString = new TextDecoder().decode(arrayBuffer);
+    const json = JSON.parse(jsonString);
+    const tilejsonOptions = {...TileJSONLoaderWithParser.options.tilejson, ...options?.tilejson};
+    return parseTileJSON(json, tilejsonOptions) as TileJSON;
+  },
+  parseTextSync: (text: string, options?: TileJSONLoaderOptions) => {
+    const json = JSON.parse(text);
+    const tilejsonOptions = {...TileJSONLoaderWithParser.options.tilejson, ...options?.tilejson};
+    return parseTileJSON(json, tilejsonOptions) as TileJSON;
+  }
+} as const satisfies LoaderWithParser<TileJSON, never, TileJSONLoaderOptions>;

package/src/mvt/tilejson-loader.ts

@@ -0,0 +1,42 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Loader, LoaderOptions} from '@loaders.gl/core/loader-utils';
+import type {TileJSON} from './lib/parse-tilejson';
+import {TileJSONFormat} from './tilejson-format';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+export type TileJSONLoaderOptions = LoaderOptions & {
+  /** Options for the TileJSONLoader */
+  tilejson?: {
+    /** Max number of unique values */
+    maxValues?: number;
+  };
+};
+
+/** Preloads the parser-bearing TileJSON loader implementation. */
+async function preload() {
+  const {TileJSONLoaderWithParser} = await import('./tilejson-loader-with-parser');
+  return TileJSONLoaderWithParser;
+}
+
+/** Metadata-only loader for TileJSON metadata. */
+export const TileJSONLoader = {
+  dataType: null as unknown as TileJSON,
+  batchType: null as never,
+
+  ...TileJSONFormat,
+  module: 'pmtiles',
+  version: VERSION,
+  worker: true,
+  options: {
+    tilejson: {
+      maxValues: undefined
+    }
+  },
+  preload
+} as const satisfies Loader<TileJSON, never, TileJSONLoaderOptions>;

package/src/mvt/tilejson-zod-schema.ts

@@ -0,0 +1,199 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {z} from 'zod';
+
+/** One vector layer declaration in raw TileJSON metadata. */
+export interface TileJSONVectorLayer {
+  /** Stable identifier used by vector-tile features. */
+  id?: string;
+  /** Human-readable layer description. */
+  description?: string;
+  /** Declared field names mapped to producer-specific field type names. */
+  fields?: Record<string, string>;
+  /** Minimum zoom at which the layer is available. */
+  minzoom?: number;
+  /** Maximum zoom at which the layer is available. */
+  maxzoom?: number;
+  /** Additional TileJSON layer properties are preserved verbatim. */
+  [key: string]: unknown;
+}
+
+/**
+ * One raw, unparsed Tilestats layer attribute.
+ *
+ * @see https://github.com/mapbox/mapbox-geostats#output-the-stats
+ */
+export interface TilestatsLayerAttribute {
+  /** The name of this attribute. */
+  attribute?: string;
+  /**
+   * The inferred attribute type:
+   *
+   * - `string` if all values are strings or `null`.
+   * - `number` if all values are numbers or `null`.
+   * - `boolean` if all values are booleans or `null`.
+   * - `null` if the only value is `null`.
+   * - `mixed` if values have multiple types.
+   *
+   * Array and object values are coerced to strings by Mapbox Geostats.
+   */
+  type?: string;
+  /** Minimum value when any values are numbers. */
+  min?: number;
+  /** Maximum value when any values are numbers. */
+  max?: number;
+  /** Number of unique values. */
+  count?: number;
+  /** First 100 unique values. */
+  values?: unknown[];
+  /** Additional attribute statistics are preserved verbatim. */
+  [key: string]: unknown;
+}
+
+/**
+ * One raw, unparsed layer in a Tilestats document.
+ *
+ * @see https://github.com/mapbox/mapbox-geostats#output-the-stats
+ */
+export interface TilestatsLayer {
+  /** The name of this layer. */
+  layer?: string;
+  /** The number of features in this layer. */
+  count?: number;
+  /** The dominant geometry type in this layer. */
+  geometry?: string;
+  /** The number of unique attributes in this layer, up to 1000. */
+  attributeCount?: number;
+  /** Attribute statistics for this layer. */
+  attributes?: TilestatsLayerAttribute[];
+  /** Additional layer statistics are preserved verbatim. */
+  [key: string]: unknown;
+}
+
+/** Raw Tilestats metadata embedded in or accompanying TileJSON. */
+export interface Tilestats {
+  /** Number of layers represented by the metadata. */
+  layerCount?: number;
+  /** Per-layer statistics. */
+  layers?: TilestatsLayer[];
+  /** Additional Tilestats properties are preserved verbatim. */
+  [key: string]: unknown;
+}
+
+/**
+ * Raw TileJSON metadata accepted by loaders.gl before normalization.
+ *
+ * Known properties are optional because loaders.gl also accepts partial TileJSON-like metadata
+ * embedded in PMTiles and generated by tools such as Tippecanoe.
+ */
+export interface TileJSONMetadata {
+  /** TileJSON specification version. */
+  tilejson?: string;
+  /** Human-readable tileset name. */
+  name?: string;
+  /** Human-readable tileset description. */
+  description?: string;
+  /** Producer-specific tileset version. */
+  version?: string;
+  /** Tile indexing scheme. */
+  scheme?: 'xyz' | 'tms';
+  /** Tile URL templates. */
+  tiles?: string[];
+  /** Geographic bounds as an array or comma-separated string. */
+  bounds?: number[] | string;
+  /** Tippecanoe bounds adjusted to avoid crossing the antimeridian. */
+  antimeridian_adjusted_bounds?: number[] | string;
+  /** Default map center as an array or comma-separated string. */
+  center?: number[] | string;
+  /** Minimum zoom, optionally encoded as a string by metadata producers. */
+  minzoom?: number | string;
+  /** Maximum zoom, optionally encoded as a string by metadata producers. */
+  maxzoom?: number | string;
+  /** HTML attribution for the tileset. */
+  attribution?: string;
+  /** HTML legend for the tileset. */
+  legend?: string;
+  /** Vector layer declarations. */
+  vector_layers?: TileJSONVectorLayer[];
+  /** Inline Tilestats metadata. */
+  tilestats?: Tilestats;
+  /** Nested JSON metadata encoded as a string. */
+  json?: string;
+  /** Generating application. */
+  generator?: string;
+  /** Command-line or application options used by the generator. */
+  generator_options?: string;
+  /** Additional TileJSON properties are preserved verbatim. */
+  [key: string]: unknown;
+}
+
+/** Zod schema for one raw TileJSON vector layer. */
+export const TileJSONVectorLayerSchema = z
+  .object({
+    id: z.string().optional(),
+    description: z.string().optional(),
+    fields: z.record(z.string(), z.string()).optional(),
+    minzoom: z.number().optional(),
+    maxzoom: z.number().optional()
+  })
+  .passthrough() satisfies z.ZodType<TileJSONVectorLayer>;
+
+/** Zod schema for one raw Tilestats attribute. */
+export const TilestatsLayerAttributeSchema = z
+  .object({
+    attribute: z.string().optional(),
+    type: z.string().optional(),
+    min: z.number().optional(),
+    max: z.number().optional(),
+    count: z.number().nonnegative().optional(),
+    values: z.array(z.unknown()).optional()
+  })
+  .passthrough() satisfies z.ZodType<TilestatsLayerAttribute>;
+
+/** Zod schema for one raw Tilestats layer. */
+export const TilestatsLayerSchema = z
+  .object({
+    layer: z.string().optional(),
+    count: z.number().nonnegative().optional(),
+    geometry: z.string().optional(),
+    attributeCount: z.number().nonnegative().optional(),
+    attributes: z.array(TilestatsLayerAttributeSchema).optional()
+  })
+  .passthrough() satisfies z.ZodType<TilestatsLayer>;
+
+/** Zod schema for raw Tilestats metadata. */
+export const TilestatsSchema = z
+  .object({
+    layerCount: z.number().nonnegative().optional(),
+    layers: z.array(TilestatsLayerSchema).optional()
+  })
+  .passthrough() satisfies z.ZodType<Tilestats>;
+
+const numberArrayOrStringSchema = z.union([z.array(z.number()), z.string()]);
+const numberOrStringSchema = z.union([z.number(), z.string()]);
+
+/** Zod schema for raw TileJSON and compatible partial metadata accepted by loaders.gl. */
+export const TileJSONMetadataSchema = z
+  .object({
+    tilejson: z.string().optional(),
+    name: z.string().optional(),
+    description: z.string().optional(),
+    version: z.string().optional(),
+    scheme: z.enum(['xyz', 'tms']).optional(),
+    tiles: z.array(z.string()).optional(),
+    bounds: numberArrayOrStringSchema.optional(),
+    antimeridian_adjusted_bounds: numberArrayOrStringSchema.optional(),
+    center: numberArrayOrStringSchema.optional(),
+    minzoom: numberOrStringSchema.optional(),
+    maxzoom: numberOrStringSchema.optional(),
+    attribution: z.string().optional(),
+    legend: z.string().optional(),
+    vector_layers: z.array(TileJSONVectorLayerSchema).optional(),
+    tilestats: TilestatsSchema.optional(),
+    json: z.string().optional(),
+    generator: z.string().optional(),
+    generator_options: z.string().optional()
+  })
+  .passthrough() satisfies z.ZodType<TileJSONMetadata>;

package/src/mvt/unbundled.ts

@@ -0,0 +1,13 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+export type {TileJSONLoaderOptions} from './tilejson-loader';
+export {TileJSONLoader} from './tilejson-loader';
+
+export {MapStyleLoader} from './map-style-loader';
+
+export type {MVTLoaderOptions} from './mvt-loader';
+export {MVTLoader, MVTWorkerLoader} from './mvt-loader';
+export type {MVTGeoJSONLoaderOptions} from './mvt-geojson-loader-types';
+export {MVTGeoJSONLoader} from './mvt-geojson-loader-types';

package/src/mvt/workers/mvt-worker-node.ts

@@ -0,0 +1,9 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+import '@loaders.gl/core/polyfills';
+import {createLoaderWorker} from '@loaders.gl/core/loader-utils';
+import {MVTLoaderWithParser} from '../mvt-loader-with-parser';
+
+createLoaderWorker(MVTLoaderWithParser);

package/src/mvt/workers/mvt-worker.ts

@@ -0,0 +1,8 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+import {MVTLoaderWithParser} from '../mvt-loader-with-parser';
+import {createLoaderWorker} from '@loaders.gl/core/loader-utils';
+
+createLoaderWorker(MVTLoaderWithParser);

package/src/pmtiles/bundled.ts

@@ -0,0 +1,6 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+export type {PMTilesLoaderOptions} from './pmtiles-loader-with-parser';
+export {PMTilesLoaderWithParser as _PMTilesLoader} from './pmtiles-loader-with-parser';

package/src/pmtiles/geoarrow.ts

@@ -0,0 +1,188 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {MLTWriter} from '@loaders.gl/tiles/mlt';
+import type {MLTWriterOptions} from '@loaders.gl/tiles/mlt';
+import {MVTWriter} from '@loaders.gl/tiles/mvt';
+import type {
+  ArrowTableTileSourceInput,
+  ArrowTableTileSourceLoaderOptions,
+  MVTWriterOptions
+} from '@loaders.gl/tiles/mvt';
+import {ArrowTableVectorTileSource} from '@loaders.gl/tiles/mvt/arrow-table-tile-source-loader';
+import {getGeoMetadata} from '@loaders.gl/core/schema';
+import type {DataType} from '@loaders.gl/core/schema';
+import {encodePMTiles} from './lib/encode-pmtiles';
+import type {PMTilesEncodeOptions, PMTilesTile} from './pmtiles-writer-types';
+
+/** Explicit canonical XYZ address selected for a bounded export. */
+export type GeoArrowPMTilesTileIndex = Pick<PMTilesTile, 'x' | 'y' | 'z'>;
+
+/** Controls a single-layer GeoArrow-to-vector-tile-to-PMTiles export. */
+export type GeoArrowPMTilesOptions = {
+  /** Tile format; defaults to mvt for compatibility. */
+  readonly tileType?: 'mvt' | 'mlt';
+  /** MLT layer, scalar attributes, IDs and per-tile row limit. */
+  readonly mlt?: Pick<
+    NonNullable<MLTWriterOptions['mlt']>,
+    'layerName' | 'propertyColumns' | 'idColumn' | 'maxFeatures'
+  >;
+  /** Finite selected tile addresses, including empty tiles; no implicit global enumeration. */
+  readonly tiles: Iterable<GeoArrowPMTilesTileIndex> | AsyncIterable<GeoArrowPMTilesTileIndex>;
+  /** Existing tiler options. Output coordinates are always normalized tile-local XY. */
+  readonly table?: Omit<NonNullable<ArrowTableTileSourceLoaderOptions['table']>, 'coordinates'>;
+  /** Layer name, selected properties, feature IDs and per-tile row limit. */
+  readonly mvt?: Pick<
+    NonNullable<MVTWriterOptions['mvt']>,
+    'layerName' | 'propertyColumns' | 'idColumn' | 'maxFeatures'
+  >;
+  /** Archive limits, internal compression, bounds and auxiliary JSON metadata. */
+  readonly pmtiles?: Omit<PMTilesEncodeOptions, 'tileType' | 'tileCompression'>;
+  /** Maximum input rows checked before building the existing in-memory index; default 100,000. */
+  readonly maxInputRows?: number;
+};
+
+/**
+ * Reuses the existing GeoArrow tiler, encodes selected nonempty tiles as MVT v2 or MLT v1, and packages PMTiles v3.
+ * The input table/index and archive are in memory. Archive maxTiles also bounds all requested addresses.
+ * Cancellation is forwarded to packaging and checked around tile acquisition; synchronous indexing
+ * and per-tile encoding cannot be interrupted mid-call. No partial archive is returned.
+ */
+export async function encodeGeoArrowPMTiles(
+  input: ArrowTableTileSourceInput,
+  options: GeoArrowPMTilesOptions
+): Promise<ArrayBuffer> {
+  options = {
+    ...options,
+    table: {...options.table},
+    mlt: {...options.mlt, propertyColumns: options.mlt?.propertyColumns?.slice()},
+    mvt: {...options.mvt, propertyColumns: options.mvt?.propertyColumns?.slice()},
+    pmtiles: {...options.pmtiles}
+  };
+  const tileType = options.tileType ?? 'mvt';
+  if (!['mvt', 'mlt'].includes(tileType)) throw new Error('Unsupported GeoArrow PMTiles tileType');
+  const writerOptions = tileType === 'mlt' ? options.mlt : options.mvt;
+  const signal = options.pmtiles?.signal;
+  signal?.throwIfAborted();
+  const table = 'data' in input && 'shape' in input ? input.data : input;
+  const maxInputRows = options.maxInputRows ?? 100000;
+  const maxTiles = options.pmtiles?.maxTiles ?? 100000;
+  if (
+    !Number.isSafeInteger(maxInputRows) ||
+    maxInputRows < 1 ||
+    !Number.isSafeInteger(maxTiles) ||
+    maxTiles < 1
+  )
+    throw new Error('Invalid GeoArrow PMTiles input or requested tile limit');
+  if (!Number.isSafeInteger(table.numRows) || table.numRows > maxInputRows)
+    throw new Error('GeoArrow PMTiles input row limit exceeded');
+  const tiler = new ArrowTableVectorTileSource(input, {
+    table: {...options.table, coordinates: 'local'}
+  });
+  const schema = await tiler.getSchema();
+  signal?.throwIfAborted();
+  const geoMetadata = getGeoMetadata(schema.metadata);
+  const geometryColumn =
+    options.table?.geometryColumn ??
+    geoMetadata?.primary_column ??
+    schema.fields.find(field => field.metadata?.['ARROW:extension:name'] === 'geoarrow.wkb')!.name;
+  const propertyNames =
+    writerOptions?.propertyColumns ??
+    schema.fields
+      .filter(
+        field =>
+          field.name !== geometryColumn &&
+          !field.metadata?.['ARROW:extension:name']?.startsWith('geoarrow.') &&
+          !geoMetadata?.columns[field.name]
+      )
+      .map(field => field.name);
+  if (new Set(propertyNames).size !== propertyNames.length)
+    throw new Error('Duplicate vector tile property column');
+  const fields = Object.fromEntries(
+    propertyNames.flatMap(name => {
+      const field = schema.fields.find(field => field.name === name);
+      if (!field || name === geometryColumn)
+        throw new Error(`Unknown vector tile property column: ${name}`);
+      const type = getFieldType(field.type);
+      return type ? [[name, type]] : [];
+    })
+  );
+  const layerName = writerOptions?.layerName ?? 'features';
+  const metadata = {...options.pmtiles?.metadata, vector_layers: [{id: layerName, fields}]};
+  // TileJSON summary fields describe the selected export; the archive derives its bounds/zooms.
+  for (const name of ['bounds', 'minzoom', 'maxzoom', 'center', 'tiles', 'tilestats'])
+    delete (metadata as Record<string, unknown>)[name];
+  const maxZoom = (await tiler.getMetadata()).maxZoom!;
+  return encodePMTiles(encodeTiles(), {
+    ...options.pmtiles,
+    metadata,
+    tileType,
+    tileCompression: 'none'
+  });
+
+  /** Requests selected tiles sequentially and closes the caller's iterator on cancellation/failure. */
+  async function* encodeTiles(): AsyncIterable<PMTilesTile> {
+    const seen = new Set<string>();
+    let requested = 0;
+    for await (const tile of options.tiles) {
+      signal?.throwIfAborted();
+      if (++requested > maxTiles) throw new Error('GeoArrow PMTiles requested tile limit exceeded');
+      if (
+        !Number.isInteger(tile.z) ||
+        tile.z < 0 ||
+        tile.z > maxZoom ||
+        !Number.isInteger(tile.x) ||
+        !Number.isInteger(tile.y) ||
+        tile.x < 0 ||
+        tile.y < 0 ||
+        tile.x >= 2 ** tile.z ||
+        tile.y >= 2 ** tile.z
+      )
+        throw new Error('Invalid GeoArrow PMTiles selected XYZ address');
+      const key = `${tile.z}/${tile.x}/${tile.y}`;
+      if (seen.has(key)) throw new Error('Duplicate GeoArrow PMTiles selected XYZ address');
+      seen.add(key);
+      const arrowTile = await tiler.getTile(tile);
+      signal?.throwIfAborted();
+      if (!arrowTile) continue;
+      const data =
+        tileType === 'mlt'
+          ? MLTWriter.encodeSync(arrowTile, {
+              mlt: {
+                ...writerOptions,
+                layerName,
+                geometryColumn,
+                propertyColumns: propertyNames,
+                extent: options.table?.extent ?? 4096,
+                coordinates: 'local'
+              }
+            })
+          : MVTWriter.encodeSync(arrowTile, {
+              mvt: {
+                ...options.mvt,
+                layerName,
+                geometryColumn,
+                propertyColumns: propertyNames,
+                extent: options.table?.extent ?? 4096,
+                version: 2,
+                coordinates: 'local'
+              }
+            });
+      signal?.throwIfAborted();
+      if (data.byteLength) yield {...tile, data};
+    }
+  }
+}
+
+/** Maps supported scalar Arrow schema types to TileJSON, rejecting implicit attribute loss. */
+function getFieldType(type: DataType): 'Number' | 'String' | 'Boolean' | null {
+  if (typeof type === 'object' && type.type === 'dictionary') return getFieldType(type.dictionary);
+  if (type === 'null') return null;
+  if (type === 'bool') return 'Boolean';
+  if (type === 'utf8') return 'String';
+  if (typeof type === 'string' && /^(?:u?int|float)(?:8|16|32|64)?$/.test(type)) return 'Number';
+  throw new Error(
+    'Unsupported vector tile attribute type; explicitly select supported propertyColumns'
+  );
+}

package/src/pmtiles/index.ts

@@ -0,0 +1,27 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+export {PMTilesSourceLoader} from './pmtiles-source-loader';
+export {PMTilesFormat} from './pmtiles-format';
+
+export type {PMTilesMetadata} from './lib/parse-pmtiles';
+export type {PMTilesRangeRequestOptions, PMTilesSourceLoaderOptions} from './pmtiles-source-loader';
+export {PMTilesTileSource} from './pmtiles-source-loader';
+
+export {PMTilesLoader as _PMTilesLoader} from './pmtiles-loader';
+export type {PMTilesLoaderOptions} from './pmtiles-loader';
+
+export {PMTilesWriter} from './pmtiles-writer';
+export {encodePMTiles} from './lib/encode-pmtiles';
+export type {PMTilesWriterOptions} from './pmtiles-writer';
+export type {
+  PMTilesTile,
+  PMTilesTileInput,
+  PMTilesTileType,
+  PMTilesEncodeOptions
+} from './pmtiles-writer-types';
+
+export type {GeoArrowPMTilesOptions, GeoArrowPMTilesTileIndex} from './geoarrow';
+
+export type {RasterPMTilesOptions, RasterPMTilesSource} from './raster';

package/src/pmtiles/lib/blob-source.ts

@@ -0,0 +1,41 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as pmtiles from 'pmtiles';
+
+/**
+ * A PMTiles library compatible source that reads from blobs
+ * @deprecated TODO - reimplement as ReadableFileSource
+ * Use loaders.gl HTTP range requests instead
+ */
+export class BlobSource implements pmtiles.Source {
+  blob: Blob;
+  key: string;
+
+  constructor(blob: Blob, key: string) {
+    this.blob = blob;
+    this.key = key;
+  }
+
+  // TODO - how is this used?
+  getKey() {
+    // @ts-expect-error url is only defined on File subclass
+    return this.blob.url || '';
+  }
+
+  async getBytes(
+    offset: number,
+    length: number,
+    signal?: AbortSignal
+  ): Promise<pmtiles.RangeResponse> {
+    const slice = this.blob.slice(offset, offset + length);
+    const data = await slice.arrayBuffer();
+    return {
+      data
+      // etag: response.headers.get('ETag') || undefined,
+      // cacheControl: response.headers.get('Cache-Control') || undefined,
+      // expires: response.headers.get('Expires') || undefined
+    };
+  }
+}