@loaders.gl/tiles 5.0.0-alpha.11 → 5.0.0-alpha.13

package/src/mvt/lib/mapbox-vt-pbf/to-vector-tile.ts

@@ -0,0 +1,230 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+// Forked from https://github.com/mapbox/vt-pbf under MIT License Copyright (c) 2015 Anand Thakker
+
+import Pbf from 'pbf';
+import type {MVTTile} from '../mvt-pbf/mvt-types';
+import {writeMVT} from '../mvt-pbf/write-mvt-to-pbf';
+import GeoJSONWrapper from './geojson-wrapper';
+import type {GeoJSON, FeatureCollection, Geometry} from '@loaders.gl/core/schema';
+import {copyToArrayBuffer} from '@loaders.gl/core/loader-utils';
+
+export type FromGeojsonOptions = {
+  layerName?: string;
+  version?: number;
+  extent?: number;
+  tileIndex?: {x: number; y: number; z: number};
+};
+
+/**
+ * Serialize a map of geojson layers
+ * loaders.gl addition
+ *
+ * @param geojson
+ * @param [options] - An object specifying the vector-tile specification version and extent that were used to create `layers`.
+ * @param [options.extent=4096] - Extent of the vector tile
+ * @return  uncompressed, pbf-serialized tile data
+ */
+export function fromGeojson(geojson: FeatureCollection, options: FromGeojsonOptions): ArrayBuffer {
+  options = options || {};
+  geojson = normalizeGeojson(geojson);
+  const extent = options.extent || 4096;
+  const features = convertFeaturesToVectorTileFeatures(geojson.features, extent, options.tileIndex);
+  const layer = new GeoJSONWrapper(features, {...options, extent});
+  // TODO - this is broken
+  (layer as any).name = options.layerName || 'geojsonLayer';
+  (layer as any).version = options.version || 1;
+  (layer as any).extent = options.extent || 4096;
+
+  // @ts-expect-error
+  return fromVectorTileJs({layers: {[layer.name]: layer}});
+}
+
+/**
+ * Serialize a vector-tile-js-created tile to pbf
+ *
+ * @param tile
+ * @return  uncompressed, pbf-serialized tile data
+ */
+export function fromVectorTileJs(tile: MVTTile): ArrayBuffer {
+  const pbf = new Pbf();
+  writeMVT(tile, pbf);
+  const uint8Array = pbf.finish();
+  // TODO - make sure no byteOffsets/byteLenghts are used?
+  return copyToArrayBuffer(
+    uint8Array.buffer,
+    uint8Array.byteOffset,
+    uint8Array.byteOffset + uint8Array.byteLength
+  );
+}
+
+/**
+ * Serialized a geojson-vt-created tile to pbf.
+ *
+ * @param vtLayers - An object mapping layer names to geojson-vt-created vector tile objects
+ * @param  [options] - An object specifying the vector-tile specification version and extent that were used to create `layers`.
+ * @param  [options.version=1] - Version of vector-tile spec used
+ * @param  [options.extent=4096] - Extent of the vector tile
+ * @return uncompressed, pbf-serialized tile data
+ *
+export function fromGeojsonVt(vtLayers, options): ArrayBuffer {
+  options = options || {};
+  const layers = {};
+  for (const key in vtLayers) {
+    layers[key] = new GeoJSONWrapper(vtLayers[key].features, options);
+    layers[key].name = key;
+    layers[key].version = options.version;
+    layers[key].extent = options.extent;
+  }
+  return fromVectorTileJs({layers});
+}
+*/
+
+export function normalizeGeojson(geojson: GeoJSON): FeatureCollection {
+  if (geojson.type === 'FeatureCollection') {
+    return geojson;
+  }
+  // Array of features
+  if (Array.isArray(geojson)) {
+    return {
+      type: 'FeatureCollection',
+      features: geojson
+    };
+  }
+  // A single feature
+  if (geojson.type === 'Feature') {
+    return {
+      type: 'FeatureCollection',
+      features: [geojson]
+    };
+  }
+  throw new Error('Invalid GeoJSON object');
+}
+
+function convertFeaturesToVectorTileFeatures(
+  features,
+  extent: number,
+  tileIndex?: {x: number; y: number; z: number}
+) {
+  if (features.every(isVectorTileFeature)) {
+    return features;
+  }
+
+  return features.map(feature => convertFeatureToVectorTile(feature, extent, tileIndex));
+}
+
+function convertFeatureToVectorTile(
+  feature,
+  extent: number,
+  tileIndex?: {x: number; y: number; z: number}
+) {
+  const geometry = feature.geometry as Geometry;
+  const type = getVectorTileType(geometry.type);
+
+  return {
+    id: typeof feature.id === 'number' ? feature.id : undefined,
+    type,
+    geometry: projectGeometryToTileSpace(geometry, extent, tileIndex),
+    tags: feature.properties || {}
+  };
+}
+
+function projectGeometryToTileSpace(
+  geometry: Geometry,
+  extent: number,
+  tileIndex?: {x: number; y: number; z: number}
+) {
+  switch (geometry.type) {
+    case 'Point':
+      return [projectPointToTile(geometry.coordinates as number[], extent, tileIndex)];
+    case 'MultiPoint':
+      return geometry.coordinates.map(coord =>
+        projectPointToTile(coord as number[], extent, tileIndex)
+      );
+    case 'LineString':
+      return [
+        geometry.coordinates.map(coord => projectPointToTile(coord as number[], extent, tileIndex))
+      ];
+    case 'MultiLineString':
+      return geometry.coordinates.map(line =>
+        line.map(coord => projectPointToTile(coord as number[], extent, tileIndex))
+      );
+    case 'Polygon':
+      return geometry.coordinates.map(ring =>
+        ring.map(coord => projectPointToTile(coord as number[], extent, tileIndex))
+      );
+    case 'MultiPolygon':
+      return geometry.coordinates.flatMap(polygon =>
+        polygon.map(ring =>
+          ring.map(coord => projectPointToTile(coord as number[], extent, tileIndex))
+        )
+      );
+    default:
+      throw new Error(`Unsupported geometry type: ${geometry.type}`);
+  }
+}
+
+function projectPointToTile(
+  point: number[],
+  extent: number,
+  tileIndex?: {x: number; y: number; z: number}
+) {
+  if (isNormalizedPoint(point)) {
+    return [Math.round(point[0] * extent), Math.round(point[1] * extent)];
+  }
+
+  if (tileIndex && isLngLatPoint(point)) {
+    return projectLngLatToTile(point, tileIndex, extent);
+  }
+
+  return [Math.round(point[0]), Math.round(point[1])];
+}
+
+function isNormalizedPoint(point: number[]) {
+  return Math.abs(point[0]) <= 1 && Math.abs(point[1]) <= 1;
+}
+
+function isLngLatPoint(point: number[]) {
+  return Math.abs(point[0]) <= 180 && Math.abs(point[1]) <= 90;
+}
+
+function projectLngLatToTile(
+  point: number[],
+  tileIndex: {x: number; y: number; z: number},
+  extent: number
+) {
+  const [lng, lat] = point;
+  const {x, y, z} = tileIndex;
+  const size = extent * Math.pow(2, z);
+  const x0 = extent * x;
+  const y0 = extent * y;
+
+  const worldX = ((lng + 180) / 360) * size;
+  const worldY =
+    ((180 - (180 / Math.PI) * Math.log(Math.tan(Math.PI / 4 + (lat * Math.PI) / 180 / 2))) * size) /
+    360;
+
+  return [Math.round(worldX - x0), Math.round(worldY - y0)];
+}
+
+function isVectorTileFeature(feature): boolean {
+  return typeof feature?.type === 'number' && Array.isArray(feature.geometry);
+}
+
+function getVectorTileType(type: Geometry['type']): 1 | 2 | 3 {
+  switch (type) {
+    case 'Point':
+    case 'MultiPoint':
+      return 1;
+    case 'LineString':
+    case 'MultiLineString':
+      return 2;
+    case 'Polygon':
+    case 'MultiPolygon':
+      return 3;
+    default:
+      throw new Error(`Unknown geometry type: ${type}`);
+  }
+}

package/src/mvt/lib/mvt-pbf/mvt-constants.ts

@@ -0,0 +1,135 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+/**
+ * MVT spec constants
+ * @see https://github.com/mapbox/vector-tile-spec/blob/master/2.1/README.md
+ */
+export enum TileInfo {
+  /** repeated Layer */
+  layers = 3
+}
+
+/**
+ * MVT spec constants
+ * @see https://github.com/mapbox/vector-tile-spec/blob/master/2.1/README.md
+ * @note Layers are described in section 4.1 of the specification
+ */
+export enum LayerInfo {
+  /**
+   * Any compliant implementation must first read the version
+   * number encoded in this message and choose the correct
+   * implementation for this version number before proceeding to
+   * decode other parts of this message.
+   * required uint32  [ default = 1 ];
+   */
+  version = 15,
+
+  /** PBF: required string */
+  name = 1,
+
+  /** The actual features in this tile.
+   * PBF: repeated Feature
+   */
+  features = 2,
+
+  /**
+   * Dictionary encoding for keys
+   * PBF: repeated string
+   */
+  keys = 3,
+
+  /**
+   * Dictionary encoding for values
+   * PBF: repeated Value
+   */
+  values = 4,
+
+  /**
+   * Although this is an "optional" field it is required by the specification.
+   * See https://github.com/mapbox/vector-tile-spec/issues/47
+   * PBF: optional uint32 [ default = 4096 ];
+   */
+  extent = 5
+
+  // extensions 16 to max;
+}
+
+/**
+ * @see https://github.com/mapbox/vector-tile-spec/blob/master/2.1/README.md
+ * Features are described in section 4.2 of the specification
+ */
+export enum FeatureInfo {
+  /** optional uint64 [ default = 0 ]; */
+  id = 1,
+
+  /**
+   * Tags of this feature are encoded as repeated pairs of integers.
+   * A detailed description of tags is located in sections 4.2 and 4.4 of the specification
+   * repeated uint32  [ packed = true ];
+   */
+  tags = 2,
+
+  /**
+   * The type of geometry stored in this feature.
+   * GeomType  [ default = UNKNOWN ];
+   */
+  type = 3,
+
+  /**
+   * Contains a stream of commands and parameters (vertices).
+   * A detailed description on geometry encoding is located in
+   * section 4.3 of the specification.
+   * repeated uint32  [ packed = true ];
+   */
+  geometry = 4
+}
+
+/**
+ * GeomType is described in section 4.3.4 of the specification
+ * @see https://github.com/mapbox/vector-tile-spec/blob/master/2.1/README.md
+ * */
+export enum GeometryType {
+  UNKNOWN = 0,
+  POINT = 1,
+  LINESTRING = 2,
+  POLYGON = 3
+}
+
+/**
+ * Variant type encoding
+ * The use of values is described in section 4.1 of the specification
+ * @note Exactly one of these values must be present in a valid message
+ * @see https://github.com/mapbox/vector-tile-spec/blob/master/2.1/README.md
+ */
+export enum PropertyType {
+  /** string */
+  string_value = 1, //
+  /** float */
+  float_value = 2,
+  /** double */
+  double_value = 3,
+  /** int64 */
+  int_value = 4,
+  /** uint64 */
+  uint_value = 5,
+  /** sint64 */
+  sint_value = 6,
+  /** bool */
+  bool_value = 7
+  // extensions 8 to max;
+}
+
+/**
+ * "Turtle graphics" style geometry commands
+ * @see https://github.com/mapbox/vector-tile-spec/blob/master/2.1/README.md
+ */
+export enum Command {
+  /** 2 Parameters: dX, dY */
+  MoveTo = 1,
+  /** 2 Parameters dX, dY */
+  LineTo = 2,
+  /** No parameters */
+  ClosePath = 7
+}

package/src/mvt/lib/mvt-pbf/mvt-types.ts

@@ -0,0 +1,22 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+import type {Schema, FlatIndexedGeometry} from '@loaders.gl/core/schema';
+
+export type MVTTile = {
+  layers: Record<string, MVTLayer>;
+};
+
+export type MVTLayer = {
+  version: number;
+  name: string;
+  extent: number;
+  length: number;
+  schema: Schema;
+  columns: Record<string, (string | number | boolean | null)[]>;
+  idColumn: number[];
+  boundingBoxColumn: [number, number, number, number][];
+  geometryTypeColumn: number[];
+  geometryColumn: FlatIndexedGeometry[];
+};

package/src/mvt/lib/mvt-pbf/parse-geometry-from-pbf.ts

@@ -0,0 +1,285 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+// This code is inspired by https://github.com/mapbox/vector-tile-js under BSD 3-clause license.
+
+import type {
+  // Feature,
+  // FlatFeature,
+  // FlatGeometry,
+  FlatIndexedGeometry
+  // GeojsonGeometryInfo
+} from '@loaders.gl/core/schema';
+
+import Protobuf from 'pbf';
+
+import * as MVT from './mvt-constants';
+
+// import {
+//   classifyRings,
+//   classifyRingsFlat,
+//   projectToLngLat,
+//   projectToLngLatFlat,
+//   convertToLocalCoordinates,
+//   convertToLocalCoordinatesFlat
+// } from '../utils/geometry-utils';
+
+type GeometryCommand = {type: 'moveto' | 'lineto' | 'closepath'; x: number; y: number};
+
+export function* makeMVTGeometryCommandIterator(pbf: Protobuf): IterableIterator<GeometryCommand> {
+  const endPos = pbf.readVarint() + pbf.pos;
+  let cmd = 1;
+  let cmdLen: number;
+  let length = 0;
+
+  // Note: We reuse and return the same command object to avoid creating new objects
+  const command: GeometryCommand = {type: 'closepath', x: 0, y: 0};
+
+  while (pbf.pos < endPos) {
+    if (length <= 0) {
+      cmdLen = pbf.readVarint();
+      cmd = cmdLen & 0x7;
+      length = cmdLen >> 3;
+    }
+
+    length--;
+
+    switch (cmd as MVT.Command) {
+      case MVT.Command.MoveTo:
+        command.type = 'moveto';
+        command.x += pbf.readSVarint();
+        command.y += pbf.readSVarint();
+        yield command;
+        break;
+
+      case MVT.Command.LineTo:
+        command.type = 'lineto';
+        command.x += pbf.readSVarint();
+        command.y += pbf.readSVarint();
+        yield command;
+        break;
+
+      case MVT.Command.ClosePath:
+        command.type = 'moveto';
+        yield command;
+        break;
+
+      default:
+        throw new Error(`unknown command ${cmd}`);
+    }
+  }
+}
+
+/**
+ * Creates a bounding box from the feature geometry at the current position
+ * @todo - Reparses the entire PBF geometry - replace with a generic bounding box calculation on parsed data?
+ */
+export function readBoundingBoxFromPBF(pbf: Protobuf): [number, number, number, number] {
+  let x1 = Infinity;
+  let x2 = -Infinity;
+  let y1 = Infinity;
+  let y2 = -Infinity;
+
+  for (const command of makeMVTGeometryCommandIterator(pbf)) {
+    switch (command.type) {
+      case 'moveto':
+      case 'lineto':
+        if (command.x < x1) x1 = command.x;
+        if (command.x > x2) x2 = command.x;
+        if (command.y < y1) y1 = command.y;
+        if (command.y > y2) y2 = command.y;
+        break;
+
+      case 'closepath':
+      default:
+      // ignore
+    }
+  }
+
+  return [x1, y1, x2, y2];
+}
+
+/**
+ * Extract intermediate Flat GeoJSON
+ * data format, which maps closely to the binary data buffers.
+ * It is similar to GeoJSON, but rather than storing the coordinates
+ * in multidimensional arrays, we have a 1D `data` with all the
+ * coordinates, and then index into this using the `indices`
+ * parameter, e.g.
+ *
+ * geometry: {
+ *   type: 'Point', data: [1,2], indices: [0]
+ * }
+ * geometry: {
+ *   type: 'LineString', data: [1,2,3,4,...], indices: [0]
+ * }
+ * geometry: {
+ *   type: 'Polygon', data: [1,2,3,4,...], indices: [[0, 2]]
+ * }
+ * Thus the indices member lets us look up the relevant range
+ * from the data array.
+ * The Multi* versions of the above types share the same data
+ * structure, just with multiple elements in the indices array
+ */
+// eslint-disable-next-line complexity, max-statements
+export function loadFlatGeometryFromPBF(pbf: Protobuf): FlatIndexedGeometry {
+  // Note: I attempted to replace the `data` array with a
+  // Float32Array, but performance was worse, both using
+  // `set()` and direct index access. Also, we cannot
+  // know how large the buffer should be, so it would
+  // increase memory usage
+  const indices: number[] = []; // Indices where geometries start
+  const data: number[] = []; // Flat array of coordinate data
+  let i = 0;
+
+  for (const command of makeMVTGeometryCommandIterator(pbf)) {
+    switch (command.type) {
+      case 'moveto':
+        // New line
+        indices.push(i);
+        data.push(command.x, command.y);
+        i += 2;
+        break;
+
+      case 'lineto':
+        data.push(command.x, command.y);
+        i += 2;
+        break;
+
+      case 'closepath':
+        // Workaround for https://github.com/mapbox/mapnik-vector-tile/issues/90
+        if (i > 0) {
+          const start = indices[indices.length - 1]; // start index of polygon
+          data.push(data[start], data[start + 1]); // closePolygon
+          i += 2;
+        }
+        break;
+
+      default:
+      // just for eslint
+    }
+  }
+
+  return {data, indices};
+}
+
+/**
+ * Load a GeoJSON style Geometry from the raw PBF
+ *
+function readGeoJSONGeometryFromPBF(pbf): number[][][] {
+  const lines: number[][][] = [];
+  let line: number[][] | undefined;
+
+  for (const command of makeMVTGeometryCommandIterator(pbf)) {
+    switch (command.type) {
+      case 'moveto':
+        // moveTo
+        if (line) lines.push(line);
+        line = [];
+        if (line) line.push([command.x, command.y]);
+        break;
+
+      case 'lineto':
+        if (line) line.push([command.x, command.y]);
+        break;
+
+      case 'closepath':
+        // Workaround for https://github.com/mapbox/mapnik-vector-tile/issues/90
+        if (line) {
+          line.push(line[0].slice()); // closePolygon
+        }
+        break;
+
+      default:
+      // just for eslint
+    }
+  }
+
+  if (line) {
+    lines.push(line);
+  }
+
+  return lines;
+}
+*/
+
+// export class VectorTileFeature implements MVTFeature {
+//   properties: Record<string, string | number | boolean | null>;
+//   extent: any;
+//   type: number;
+//   id: number | null;
+//   _pbf: Protobuf;
+//   _geometryPos: number;
+//   _keys: string[];
+//   _values: (string | number | boolean | null)[];
+//   _geometryInfo: GeojsonGeometryInfo;
+
+//   static types = MVT_GEOMETRY_TYPES;
+
+//   // eslint-disable-next-line max-params
+//   constructor(
+//     pbf: Protobuf,
+//     end: number,
+//     extent: any,
+//     keys: string[],
+//     values: (string | number | boolean | null)[],
+//     geometryInfo?: GeojsonGeometryInfo
+//   ) {
+//     // Public
+//     this.properties = {};
+//     this.extent = extent;
+//     this.type = 0;
+//     this.id = null;
+
+//     // Private
+//     this._pbf = pbf;
+//     this._geometryPos = -1;
+//     this._keys = keys;
+//     this._values = values;
+
+//     // Only used by binary tiles
+//     this._geometryInfo = geometryInfo!;
+
+//     pbf.readFields(readFeatureFromPBF, this, end);
+//   }
+
+//   toGeoJSONFeature(
+//     coordinates: 'wgs84' | 'local',
+//     tileIndex?: {x: number; y: number; z: number}
+//   ): Feature {
+//     return readGeoJSONFeatureFromPBF(this, coordinates, this.extent, tileIndex);
+//   }
+
+//   /**
+//    *
+//    * @param options
+//    * @returns
+//    */
+//   toBinaryFeature(
+//     coordinates: 'wgs84' | 'local',
+//     tileIndex?: {x: number; y: number; z: number}
+//   ): FlatFeature {
+//     return readFlatFeatureFromPBF(this, coordinates, this.extent, tileIndex);
+//   }
+
+//   /** Read a bounding box from the feature */
+//   bbox() {
+//     this._pbf.pos = this._geometryPos;
+//     return readBoundingBoxFromPBF(this._pbf);
+//   }
+
+//   // Geometry helpers
+
+//   /** Parses protobuf data to nested "GeoJSON like" coordinates array */
+//   loadGeometry(): number[][][] {
+//     this._pbf.pos = this._geometryPos;
+//     return readGeoJSONGeometryFromPBF(this._pbf);
+//   }
+
+//   /** Parses protobuf data to an intermediate "Flat GeoJSON" data format */
+//   loadFlatGeometry(): FlatIndexedGeometry {
+//     this._pbf.pos = this._geometryPos;
+//     return loadFlatGeometryFromPBF(this._pbf);
+//   }
+// }