@loaders.gl/mvt 4.3.2 → 4.4.0-alpha.2

package/src/lib/pojo-parser/parse-mvt-from-pbf.ts

@@ -0,0 +1,310 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+// This code is inspired by https://github.com/mapbox/vector-tile-js under BSD 3-clause license.
+
+import Protobuf from 'pbf';
+import {Schema} from '@loaders.gl/schema';
+import type {MVTTile, MVTLayer} from './mvt-types';
+import * as MVT from './mvt-constants';
+import {readBoundingBoxFromPBF, loadFlatGeometryFromPBF} from './parse-geometry-from-pbf';
+
+export type MVTLayerData = {
+  /** Layer being built */
+  layer: MVTLayer;
+  currentFeature: number;
+  /** list of all keys in layer: Temporary, used when building up the layer */
+  keys: string[];
+  /** single list of all values in all columns - Temporary values used when building up the layer */
+  values: (string | number | boolean | null)[];
+  types: number[];
+  columnTypes: number[];
+  columnNullable: boolean[];
+  /** list of all feature start positions in the PBF - Temporary values used when building up the layer */
+  featurePositions: number[];
+  /** list of all geometry start positions in the PBF - Temporary values used when building up the layer */
+  geometryPositions: number[];
+};
+
+const DEFAULT_LAYER = {
+  version: 1,
+  name: '',
+  extent: 4096,
+  length: 0,
+  schema: {fields: [], metadata: {}},
+  columns: {},
+  idColumn: [],
+  geometryTypeColumn: [],
+  geometryColumn: [],
+  boundingBoxColumn: []
+} as const satisfies MVTLayer;
+
+const DEFAULT_LAYER_DATA = {
+  currentFeature: 0,
+  keys: [],
+  values: [],
+  types: [],
+  columnTypes: [],
+  columnNullable: [],
+  featurePositions: [],
+  geometryPositions: []
+};
+
+/** Parse an MVT tile from an ArrayBuffer */
+export function parseMVT(arrayBuffer: ArrayBuffer | Uint8Array): MVTTile {
+  const pbf = new Protobuf(arrayBuffer);
+  return parseMVTTile(pbf);
+}
+
+/** Parse an MVT tile from a PBF buffer */
+export function parseMVTTile(pbf: Protobuf, end?: number): MVTTile {
+  const tile = {layers: {}} satisfies MVTTile;
+  try {
+    pbf.readFields(readTileFieldFromPBF, tile, end);
+  } catch (error) {
+    // eslint-disable-next-line no-console
+    console.warn(error);
+  }
+  return tile;
+}
+
+/**
+ * Protobuf read callback for a top-level tile object's PBF tags
+ * @param tag
+ * @param layers
+ * @param pbf
+ */
+function readTileFieldFromPBF(tag: number, tile?: MVTTile, pbf?: Protobuf): void {
+  if (!pbf || !tile) {
+    return;
+  }
+
+  switch (tag as MVT.TileInfo) {
+    case MVT.TileInfo.layers:
+      // get the byte length and end of the layer
+      const byteLength = pbf.readVarint();
+      const end = byteLength + pbf.pos;
+
+      const layer = parseLayer(pbf, end);
+      tile.layers[layer.name] = layer;
+      break;
+    default:
+    // ignore? log?
+  }
+}
+
+/** Parse an MVT layer from BPF at current position */
+export function parseLayer(pbf: Protobuf, end: number): MVTLayer {
+  const layerData: MVTLayerData = {layer: {...DEFAULT_LAYER}, ...DEFAULT_LAYER_DATA};
+  pbf.readFields(readLayerFieldFromPBF, layerData, end);
+
+  // Read features
+  for (let featureIndex = 0; featureIndex < layerData.featurePositions.length; featureIndex++) {
+    // Determine start and end of feature in PBF
+    const featurePosition = layerData.featurePositions[featureIndex];
+
+    pbf.pos = featurePosition;
+    const byteLength = pbf.readVarint();
+    const end = byteLength + pbf.pos;
+
+    layerData.currentFeature = featureIndex;
+    pbf.readFields(readFeatureFieldFromPBF, layerData, end);
+    readBoundingBoxesFromPDF(pbf, layerData);
+    readGeometriesFromPBF(pbf, layerData);
+  }
+
+  // Post processing
+  const {layer} = layerData;
+  layer.length = layerData.featurePositions.length;
+  layer.schema = makeMVTSchema(layerData);
+  return layer;
+}
+
+/**
+ *
+ * @param tag
+ * @param layer
+ * @param pbf
+ */
+function readLayerFieldFromPBF(tag: number, layerData?: MVTLayerData, pbf?: Protobuf): void {
+  if (!layerData || !pbf) {
+    return;
+  }
+
+  switch (tag as MVT.LayerInfo) {
+    case MVT.LayerInfo.version:
+      layerData.layer.version = pbf.readVarint();
+      break;
+    case MVT.LayerInfo.name:
+      layerData.layer.name = pbf.readString();
+      break;
+    case MVT.LayerInfo.extent:
+      layerData.layer.extent = pbf.readVarint();
+      break;
+    case MVT.LayerInfo.features:
+      layerData.featurePositions.push(pbf.pos);
+      break;
+    case MVT.LayerInfo.keys:
+      layerData.keys.push(pbf.readString());
+      break;
+    case MVT.LayerInfo.values:
+      const [value, type] = parseValues(pbf);
+      layerData.values.push(value);
+      layerData.types.push(type);
+      break;
+    default:
+    // ignore? Log?
+  }
+}
+
+/**
+ * @param pbf
+ * @returns value
+ */
+function parseValues(pbf: Protobuf): [string | number | boolean | null, MVT.PropertyType] {
+  const end = pbf.readVarint() + pbf.pos;
+
+  let value: string | number | boolean | null = null;
+  // not a valid property type so we use it to detect multiple values
+  let type = -1 as MVT.PropertyType;
+
+  // TODO - can we have multiple values?
+  while (pbf.pos < end) {
+    if (type !== (-1 as MVT.PropertyType)) {
+      throw new Error('MVT: Multiple values not supported');
+    }
+    type = pbf.readVarint() >> 3;
+    value = readValueFromPBF(pbf, type);
+  }
+  return [value, type];
+}
+
+/** Read a type tagged value from the protobuf at current position */
+function readValueFromPBF(pbf: Protobuf, type: MVT.PropertyType): string | number | boolean | null {
+  switch (type) {
+    case MVT.PropertyType.string_value:
+      return pbf.readString();
+    case MVT.PropertyType.float_value:
+      return pbf.readFloat();
+    case MVT.PropertyType.double_value:
+      return pbf.readDouble();
+    case MVT.PropertyType.int_value:
+      return pbf.readVarint64();
+    case MVT.PropertyType.uint_value:
+      return pbf.readVarint();
+    case MVT.PropertyType.sint_value:
+      return pbf.readSVarint();
+    case MVT.PropertyType.bool_value:
+      return pbf.readBoolean();
+    default:
+      return null;
+  }
+}
+
+/**
+ *
+ * @param tag
+ * @param feature
+ * @param pbf
+ */
+function readFeatureFieldFromPBF(tag: number, layerData?: MVTLayerData, pbf?: Protobuf): void {
+  if (!pbf || !layerData) {
+    return;
+  }
+  switch (tag as MVT.FeatureInfo) {
+    case MVT.FeatureInfo.id:
+      const id = pbf.readVarint();
+      layerData.layer.idColumn[layerData.currentFeature] = id;
+      break;
+    case MVT.FeatureInfo.tags:
+      parseColumnValues(pbf, layerData);
+      break;
+    case MVT.FeatureInfo.type:
+      const type = pbf.readVarint();
+      layerData.layer.geometryTypeColumn[layerData.currentFeature] = type;
+      break;
+    case MVT.FeatureInfo.geometry:
+      layerData.geometryPositions[layerData.currentFeature] = pbf.pos;
+      break;
+    default:
+    // ignore? log?
+  }
+}
+
+/**
+ *
+ * @param pbf
+ * @param feature
+ */
+function parseColumnValues(pbf: Protobuf, layerData: MVTLayerData): void {
+  const end = pbf.readVarint() + pbf.pos;
+
+  while (pbf.pos < end) {
+    const keyIndex = pbf.readVarint();
+    const valueIndex = pbf.readVarint();
+    const columnName = layerData.keys[keyIndex];
+    const value = layerData.values[valueIndex];
+    layerData.columnTypes[columnName] = layerData.types[valueIndex];
+    layerData.columnNullable[columnName] ||= value === null;
+
+    layerData.layer.columns[columnName] ||= [];
+    layerData.layer.columns[columnName].push(value);
+  }
+}
+
+// Geometry readers
+
+function readBoundingBoxesFromPDF(pbf: Protobuf, layerData: MVTLayerData): void {
+  for (let row = 0; row < layerData.geometryPositions.length; row++) {
+    pbf.pos = layerData.geometryPositions[row];
+    const boundingBox = readBoundingBoxFromPBF(pbf);
+    layerData.layer.boundingBoxColumn[row] = boundingBox;
+  }
+}
+
+function readGeometriesFromPBF(pbf: Protobuf, layerData: MVTLayerData): void {
+  for (let row = 0; row < layerData.geometryPositions.length; row++) {
+    pbf.pos = layerData.geometryPositions[row];
+    const flatGeometry = loadFlatGeometryFromPBF(pbf);
+    layerData.layer.geometryColumn[row] = flatGeometry;
+  }
+}
+
+// Schema Builder
+
+function makeMVTSchema(layerData: MVTLayerData): Schema {
+  const {keys, columnTypes, columnNullable} = layerData;
+  const fields: Schema['fields'] = [];
+  for (let i = 0; i < keys.length; i++) {
+    const key = keys[i];
+    const nullable = columnNullable[key];
+    switch (columnTypes[key] as MVT.PropertyType) {
+      case MVT.PropertyType.string_value:
+        fields.push({name: keys[i], type: 'utf8', nullable});
+        break;
+      case MVT.PropertyType.float_value:
+        fields.push({name: keys[i], type: 'float32', nullable});
+        break;
+      case MVT.PropertyType.double_value:
+        fields.push({name: keys[i], type: 'float64', nullable});
+        break;
+      case MVT.PropertyType.int_value:
+        fields.push({name: keys[i], type: 'int32', nullable});
+        break;
+      case MVT.PropertyType.uint_value:
+        fields.push({name: keys[i], type: 'uint32', nullable});
+        break;
+      case MVT.PropertyType.sint_value:
+        fields.push({name: keys[i], type: 'int32', nullable});
+        break;
+      case MVT.PropertyType.bool_value:
+        fields.push({name: keys[i], type: 'bool', nullable});
+        break;
+      default:
+      // ignore?
+    }
+  }
+
+  return {fields, metadata: {}};
+}

package/src/lib/vector-tile/vector-tile-feature.ts

@@ -4,12 +4,8 @@
 
 // This code is forked from https://github.com/mapbox/vector-tile-js under BSD 3-clause license.
 
-import type {
-  Feature,
-  FlatFeature,
-  FlatIndexedGeometry,
-  GeojsonGeometryInfo
-} from '@loaders.gl/schema';
+import type {Feature, FlatFeature, FlatIndexedGeometry} from '@loaders.gl/schema';
+import type {GeojsonGeometryInfo} from '@loaders.gl/gis';
 import Protobuf from 'pbf';
 import {
   classifyRings,

package/src/lib/vector-tile/vector-tile-layer.ts

@@ -7,7 +7,7 @@
 
 import Protobuf from 'pbf';
 import {VectorTileFeature} from './vector-tile-feature';
-import {GeojsonGeometryInfo} from '@loaders.gl/schema';
+import {GeojsonGeometryInfo} from '@loaders.gl/gis';
 
 export class VectorTileLayer {
   version: number;

package/src/mvt-format.ts

@@ -0,0 +1,23 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+import type {Format} from '@loaders.gl/loader-utils';
+
+/**
+ * Worker loader for the Mapbox Vector Tile format
+ */
+export const MVTFormat = {
+  name: 'Mapbox Vector Tile',
+  id: 'mvt',
+  module: 'mvt',
+  // Note: ArcGIS uses '.pbf' extension and 'application/octet-stream'
+  extensions: ['mvt', 'pbf'],
+  mimeTypes: [
+    // https://www.iana.org/assignments/media-types/application/vnd.mapbox-vector-tile
+    'application/vnd.mapbox-vector-tile',
+    'application/x-protobuf'
+    // 'application/octet-stream'
+  ],
+  category: 'geometry'
+} as const satisfies Format;

package/src/mvt-loader.ts

@@ -5,6 +5,7 @@
 import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
 // import type {MVTOptions} from './lib/types';
 import {parseMVT} from './lib/parse-mvt';
+import {MVTFormat} from './mvt-format';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -37,23 +38,11 @@ export type MVTLoaderOptions = LoaderOptions & {
  * Worker loader for the Mapbox Vector Tile format
  */
 export const MVTWorkerLoader = {
+  ...MVTFormat,
   dataType: null as any,
   batchType: null as never,
-
-  name: 'Mapbox Vector Tile',
-  id: 'mvt',
-  module: 'mvt',
   version: VERSION,
-  // Note: ArcGIS uses '.pbf' extension and 'application/octet-stream'
-  extensions: ['mvt', 'pbf'],
-  mimeTypes: [
-    // https://www.iana.org/assignments/media-types/application/vnd.mapbox-vector-tile
-    'application/vnd.mapbox-vector-tile',
-    'application/x-protobuf'
-    // 'application/octet-stream'
-  ],
   worker: true,
-  category: 'geometry',
   options: {
     mvt: {
       shape: 'geojson',

package/src/mvt-source.ts

@@ -5,13 +5,13 @@
 import type {
   Source,
   ImageType,
-  DataSourceProps,
+  DataSourceOptions,
   ImageTileSource,
   VectorTileSource,
   GetTileParameters,
   GetTileDataParameters
 } from '@loaders.gl/loader-utils';
-import {DataSource, resolvePath} from '@loaders.gl/loader-utils';
+import {DataSource} from '@loaders.gl/loader-utils';
 import {ImageLoader, ImageLoaderOptions, getBinaryImageMetadata} from '@loaders.gl/images';
 import {
   MVTLoader,
@@ -20,32 +20,10 @@ import {
   TileJSON,
   TileJSONLoaderOptions
 } from '@loaders.gl/mvt';
-
-/** Creates an MVTTileSource */
-export const MVTSource = {
-  name: 'MVT',
-  id: 'mvt',
-  module: 'mvt',
-  version: '0.0.0',
-  extensions: ['mvt'],
-  mimeTypes: ['application/octet-stream'],
-  options: {
-    mvt: {
-      // TODO - add options here
-    }
-  },
-  type: 'mvt',
-  fromUrl: true,
-  fromBlob: false,
-
-  testURL: (url: string): boolean => true,
-  createDataSource(url: string, props: MVTTileSourceProps): MVTTileSource {
-    return new MVTTileSource(url, props);
-  }
-} as const satisfies Source<MVTTileSource, MVTTileSourceProps>;
+import {MVTFormat} from './mvt-format';
 
 /** Properties for a Mapbox Vector Tile Source */
-export type MVTTileSourceProps = DataSourceProps & {
+export type MVTSourceOptions = DataSourceOptions & {
   mvt?: {
     // TODO - add options here
     /** if not supplied, loads tilejson.json, If null does not load metadata */
@@ -59,6 +37,26 @@ export type MVTTileSourceProps = DataSourceProps & {
   };
 };
 
+/** Creates an MVTTileSource */
+export const MVTSource = {
+  ...MVTFormat,
+  version: '0.0.0',
+  type: 'mvt',
+  fromUrl: true,
+  fromBlob: false,
+
+  defaultOptions: {
+    mvt: {
+      // TODO - add options here
+    }
+  },
+
+  testURL: (url: string): boolean => true,
+  createDataSource(url: string, options: MVTSourceOptions): MVTTileSource {
+    return new MVTTileSource(url, options);
+  }
+} as const satisfies Source<MVTTileSource>;
+
 /**
  * MVT data source for Mapbox Vector Tiles v1.
  */
@@ -66,23 +64,20 @@ export type MVTTileSourceProps = DataSourceProps & {
  * A PMTiles data source
  * @note Can be either a raster or vector tile source depending on the contents of the PMTiles file.
  */
-export class MVTTileSource extends DataSource implements ImageTileSource, VectorTileSource {
-  readonly props: MVTTileSourceProps;
-  readonly url: string;
+export class MVTTileSource
+  extends DataSource<string, MVTSourceOptions>
+  implements ImageTileSource, VectorTileSource
+{
   readonly metadataUrl: string | null = null;
-  data: string;
   schema: 'tms' | 'xyz' | 'template' = 'tms';
   metadata: Promise<TileJSON | null>;
   extension: string;
   mimeType: string | null = null;
 
-  constructor(url: string, props: MVTTileSourceProps) {
-    super(props);
-    this.props = props;
-    this.url = resolvePath(url);
-    this.metadataUrl = props.mvt?.metadataUrl || `${this.url}/tilejson.json`;
-    this.extension = props.mvt?.extension || '.png';
-    this.data = this.url;
+  constructor(url: string, options: MVTSourceOptions) {
+    super(url, options, MVTSource.defaultOptions);
+    this.metadataUrl = options.mvt?.metadataUrl || `${this.url}/tilejson.json`;
+    this.extension = options.mvt?.extension || '.png';
 
     this.getTileData = this.getTileData.bind(this);
     this.metadata = this.getMetadata();
@@ -117,7 +112,7 @@ export class MVTTileSource extends DataSource implements ImageTileSource, Vector
     const metadata = TileJSONLoader.parseTextSync?.(tileJSON) || null;
 
     // TODO add metadata attributions
-    // metadata.attributions = [...this.props.attributions, ...(metadata.attributions || [])];
+    // metadata.attributions = [...this.options.attributions, ...(metadata.attributions || [])];
     // if (metadata?.mimeType) {
     //   this.mimeType = metadata?.tileMIMEType;
     // }