@loaders.gl/tiles 5.0.0-alpha.12 → 5.0.0-alpha.13

package/src/mvt/lib/mvt-pbf/parse-mvt-from-pbf.ts

@@ -0,0 +1,310 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+// This code is inspired by https://github.com/mapbox/vector-tile-js under BSD 3-clause license.
+
+import Protobuf from 'pbf';
+import {Schema} from '@loaders.gl/core/schema';
+import type {MVTTile, MVTLayer} from './mvt-types';
+import * as MVT from './mvt-constants';
+import {readBoundingBoxFromPBF, loadFlatGeometryFromPBF} from './parse-geometry-from-pbf';
+
+export type MVTLayerData = {
+  /** Layer being built */
+  layer: MVTLayer;
+  currentFeature: number;
+  /** list of all keys in layer: Temporary, used when building up the layer */
+  keys: string[];
+  /** single list of all values in all columns - Temporary values used when building up the layer */
+  values: (string | number | boolean | null)[];
+  types: number[];
+  columnTypes: number[];
+  columnNullable: boolean[];
+  /** list of all feature start positions in the PBF - Temporary values used when building up the layer */
+  featurePositions: number[];
+  /** list of all geometry start positions in the PBF - Temporary values used when building up the layer */
+  geometryPositions: number[];
+};
+
+const DEFAULT_LAYER = {
+  version: 1,
+  name: '',
+  extent: 4096,
+  length: 0,
+  schema: {fields: [], metadata: {}},
+  columns: {},
+  idColumn: [],
+  geometryTypeColumn: [],
+  geometryColumn: [],
+  boundingBoxColumn: []
+} as const satisfies MVTLayer;
+
+const DEFAULT_LAYER_DATA = {
+  currentFeature: 0,
+  keys: [],
+  values: [],
+  types: [],
+  columnTypes: [],
+  columnNullable: [],
+  featurePositions: [],
+  geometryPositions: []
+};
+
+/** Parse an MVT tile from an ArrayBuffer */
+export function parseMVT(arrayBuffer: ArrayBuffer | Uint8Array): MVTTile {
+  const pbf = new Protobuf(arrayBuffer);
+  return parseMVTTile(pbf);
+}
+
+/** Parse an MVT tile from a PBF buffer */
+export function parseMVTTile(pbf: Protobuf, end?: number): MVTTile {
+  const tile = {layers: {}} satisfies MVTTile;
+  try {
+    pbf.readFields(readTileFieldFromPBF, tile, end);
+  } catch (error) {
+    // eslint-disable-next-line no-console
+    console.warn(error);
+  }
+  return tile;
+}
+
+/**
+ * Protobuf read callback for a top-level tile object's PBF tags
+ * @param tag
+ * @param layers
+ * @param pbf
+ */
+function readTileFieldFromPBF(tag: number, tile?: MVTTile, pbf?: Protobuf): void {
+  if (!pbf || !tile) {
+    return;
+  }
+
+  switch (tag as MVT.TileInfo) {
+    case MVT.TileInfo.layers:
+      // get the byte length and end of the layer
+      const byteLength = pbf.readVarint();
+      const end = byteLength + pbf.pos;
+
+      const layer = parseLayer(pbf, end);
+      tile.layers[layer.name] = layer;
+      break;
+    default:
+    // ignore? log?
+  }
+}
+
+/** Parse an MVT layer from BPF at current position */
+export function parseLayer(pbf: Protobuf, end: number): MVTLayer {
+  const layerData: MVTLayerData = {layer: {...DEFAULT_LAYER}, ...DEFAULT_LAYER_DATA};
+  pbf.readFields(readLayerFieldFromPBF, layerData, end);
+
+  // Read features
+  for (let featureIndex = 0; featureIndex < layerData.featurePositions.length; featureIndex++) {
+    // Determine start and end of feature in PBF
+    const featurePosition = layerData.featurePositions[featureIndex];
+
+    pbf.pos = featurePosition;
+    const byteLength = pbf.readVarint();
+    const end = byteLength + pbf.pos;
+
+    layerData.currentFeature = featureIndex;
+    pbf.readFields(readFeatureFieldFromPBF, layerData, end);
+    readBoundingBoxesFromPDF(pbf, layerData);
+    readGeometriesFromPBF(pbf, layerData);
+  }
+
+  // Post processing
+  const {layer} = layerData;
+  layer.length = layerData.featurePositions.length;
+  layer.schema = makeMVTSchema(layerData);
+  return layer;
+}
+
+/**
+ *
+ * @param tag
+ * @param layer
+ * @param pbf
+ */
+function readLayerFieldFromPBF(tag: number, layerData?: MVTLayerData, pbf?: Protobuf): void {
+  if (!layerData || !pbf) {
+    return;
+  }
+
+  switch (tag as MVT.LayerInfo) {
+    case MVT.LayerInfo.version:
+      layerData.layer.version = pbf.readVarint();
+      break;
+    case MVT.LayerInfo.name:
+      layerData.layer.name = pbf.readString();
+      break;
+    case MVT.LayerInfo.extent:
+      layerData.layer.extent = pbf.readVarint();
+      break;
+    case MVT.LayerInfo.features:
+      layerData.featurePositions.push(pbf.pos);
+      break;
+    case MVT.LayerInfo.keys:
+      layerData.keys.push(pbf.readString());
+      break;
+    case MVT.LayerInfo.values:
+      const [value, type] = parseValues(pbf);
+      layerData.values.push(value);
+      layerData.types.push(type);
+      break;
+    default:
+    // ignore? Log?
+  }
+}
+
+/**
+ * @param pbf
+ * @returns value
+ */
+function parseValues(pbf: Protobuf): [string | number | boolean | null, MVT.PropertyType] {
+  const end = pbf.readVarint() + pbf.pos;
+
+  let value: string | number | boolean | null = null;
+  // not a valid property type so we use it to detect multiple values
+  let type = -1 as MVT.PropertyType;
+
+  // TODO - can we have multiple values?
+  while (pbf.pos < end) {
+    if (type !== (-1 as MVT.PropertyType)) {
+      throw new Error('MVT: Multiple values not supported');
+    }
+    type = pbf.readVarint() >> 3;
+    value = readValueFromPBF(pbf, type);
+  }
+  return [value, type];
+}
+
+/** Read a type tagged value from the protobuf at current position */
+function readValueFromPBF(pbf: Protobuf, type: MVT.PropertyType): string | number | boolean | null {
+  switch (type) {
+    case MVT.PropertyType.string_value:
+      return pbf.readString();
+    case MVT.PropertyType.float_value:
+      return pbf.readFloat();
+    case MVT.PropertyType.double_value:
+      return pbf.readDouble();
+    case MVT.PropertyType.int_value:
+      return pbf.readVarint64();
+    case MVT.PropertyType.uint_value:
+      return pbf.readVarint();
+    case MVT.PropertyType.sint_value:
+      return pbf.readSVarint();
+    case MVT.PropertyType.bool_value:
+      return pbf.readBoolean();
+    default:
+      return null;
+  }
+}
+
+/**
+ *
+ * @param tag
+ * @param feature
+ * @param pbf
+ */
+function readFeatureFieldFromPBF(tag: number, layerData?: MVTLayerData, pbf?: Protobuf): void {
+  if (!pbf || !layerData) {
+    return;
+  }
+  switch (tag as MVT.FeatureInfo) {
+    case MVT.FeatureInfo.id:
+      const id = pbf.readVarint();
+      layerData.layer.idColumn[layerData.currentFeature] = id;
+      break;
+    case MVT.FeatureInfo.tags:
+      parseColumnValues(pbf, layerData);
+      break;
+    case MVT.FeatureInfo.type:
+      const type = pbf.readVarint();
+      layerData.layer.geometryTypeColumn[layerData.currentFeature] = type;
+      break;
+    case MVT.FeatureInfo.geometry:
+      layerData.geometryPositions[layerData.currentFeature] = pbf.pos;
+      break;
+    default:
+    // ignore? log?
+  }
+}
+
+/**
+ *
+ * @param pbf
+ * @param feature
+ */
+function parseColumnValues(pbf: Protobuf, layerData: MVTLayerData): void {
+  const end = pbf.readVarint() + pbf.pos;
+
+  while (pbf.pos < end) {
+    const keyIndex = pbf.readVarint();
+    const valueIndex = pbf.readVarint();
+    const columnName = layerData.keys[keyIndex];
+    const value = layerData.values[valueIndex];
+    layerData.columnTypes[columnName] = layerData.types[valueIndex];
+    layerData.columnNullable[columnName] ||= value === null;
+
+    layerData.layer.columns[columnName] ||= [];
+    layerData.layer.columns[columnName].push(value);
+  }
+}
+
+// Geometry readers
+
+function readBoundingBoxesFromPDF(pbf: Protobuf, layerData: MVTLayerData): void {
+  for (let row = 0; row < layerData.geometryPositions.length; row++) {
+    pbf.pos = layerData.geometryPositions[row];
+    const boundingBox = readBoundingBoxFromPBF(pbf);
+    layerData.layer.boundingBoxColumn[row] = boundingBox;
+  }
+}
+
+function readGeometriesFromPBF(pbf: Protobuf, layerData: MVTLayerData): void {
+  for (let row = 0; row < layerData.geometryPositions.length; row++) {
+    pbf.pos = layerData.geometryPositions[row];
+    const flatGeometry = loadFlatGeometryFromPBF(pbf);
+    layerData.layer.geometryColumn[row] = flatGeometry;
+  }
+}
+
+// Schema Builder
+
+function makeMVTSchema(layerData: MVTLayerData): Schema {
+  const {keys, columnTypes, columnNullable} = layerData;
+  const fields: Schema['fields'] = [];
+  for (let i = 0; i < keys.length; i++) {
+    const key = keys[i];
+    const nullable = columnNullable[key];
+    switch (columnTypes[key] as MVT.PropertyType) {
+      case MVT.PropertyType.string_value:
+        fields.push({name: keys[i], type: 'utf8', nullable});
+        break;
+      case MVT.PropertyType.float_value:
+        fields.push({name: keys[i], type: 'float32', nullable});
+        break;
+      case MVT.PropertyType.double_value:
+        fields.push({name: keys[i], type: 'float64', nullable});
+        break;
+      case MVT.PropertyType.int_value:
+        fields.push({name: keys[i], type: 'int32', nullable});
+        break;
+      case MVT.PropertyType.uint_value:
+        fields.push({name: keys[i], type: 'uint32', nullable});
+        break;
+      case MVT.PropertyType.sint_value:
+        fields.push({name: keys[i], type: 'int32', nullable});
+        break;
+      case MVT.PropertyType.bool_value:
+        fields.push({name: keys[i], type: 'bool', nullable});
+        break;
+      default:
+      // ignore?
+    }
+  }
+
+  return {fields, metadata: {}};
+}

package/src/mvt/lib/mvt-pbf/write-mvt-to-pbf.ts

@@ -0,0 +1,162 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+// Forked from https://github.com/mapbox/vt-pbf under MIT License Copyright (c) 2015 Anand Thakker
+
+import Protobuf from 'pbf';
+import type {MVTTile, MVTLayer} from './mvt-types';
+import * as MVT from './mvt-constants';
+
+type WriteContext = {
+  layer: MVTLayer | any;
+  keys: string[];
+  values: unknown[];
+  keycache: Record<string, number>;
+  valuecache: Record<string, number>;
+  feature?: any;
+};
+
+export function writeMVT(tile: MVTTile, pbf?: Protobuf): void {
+  for (const key in tile.layers) {
+    const context: WriteContext = {
+      layer: tile.layers[key],
+      keys: [],
+      values: [],
+      keycache: Object.create(null),
+      valuecache: Object.create(null)
+    };
+
+    pbf!.writeMessage(MVT.TileInfo.layers, writeLayer, context);
+  }
+}
+
+function writeLayer(context: WriteContext, pbf?: Protobuf): void {
+  const {layer} = context;
+
+  pbf!.writeVarintField(MVT.LayerInfo.version, layer.version || 1);
+  pbf!.writeStringField(MVT.LayerInfo.name, layer.name || '');
+  pbf!.writeVarintField(MVT.LayerInfo.extent, layer.extent || 4096);
+
+  for (let i = 0; i < layer.length; i++) {
+    context.feature = layer.feature(i);
+    pbf!.writeMessage(MVT.LayerInfo.features, writeFeature, context);
+  }
+
+  const keys = context.keys;
+  for (let i = 0; i < keys.length; i++) {
+    pbf!.writeStringField(MVT.LayerInfo.keys, keys[i]);
+  }
+
+  const values = context.values;
+  for (let i = 0; i < values.length; i++) {
+    pbf!.writeMessage(MVT.LayerInfo.values, writeValue, values[i]);
+  }
+}
+
+function writeFeature(context: WriteContext, pbf?: Protobuf): void {
+  const feature = context.feature!;
+
+  if (feature.id !== undefined) {
+    pbf!.writeVarintField(MVT.FeatureInfo.id, feature.id);
+  }
+
+  pbf!.writeMessage(MVT.FeatureInfo.tags, writeProperties, context);
+  pbf!.writeVarintField(MVT.FeatureInfo.type, feature.type);
+  pbf!.writeMessage(MVT.FeatureInfo.geometry, writeGeometry, feature);
+}
+
+function writeProperties(context: WriteContext, pbf?: Protobuf): void {
+  const feature = context.feature!;
+  const {keys, values, keycache, valuecache} = context;
+
+  const entries = feature.propertyEntries
+    ? feature.propertyEntries()
+    : Object.entries(feature.properties);
+  for (const [key, propertyValue] of entries) {
+    let keyIndex = keycache[key];
+    if (typeof keyIndex === 'undefined') {
+      keys.push(key);
+      keyIndex = keys.length - 1;
+      keycache[key] = keyIndex;
+    }
+    pbf!.writeVarint(keyIndex);
+
+    let value = propertyValue;
+    const type = typeof value;
+    if (type !== 'string' && type !== 'boolean' && type !== 'number') {
+      value = JSON.stringify(value);
+    }
+    const valueKey = `${type}:${value}`;
+    let valueIndex = valuecache[valueKey];
+    if (typeof valueIndex === 'undefined') {
+      values.push(value);
+      valueIndex = values.length - 1;
+      valuecache[valueKey] = valueIndex;
+    }
+    pbf!.writeVarint(valueIndex);
+  }
+}
+
+function command(cmd, length) {
+  return (length << 3) + (cmd & 0x7);
+}
+
+/** Keeps signed 32-bit coordinate deltas unsigned on the wire. */
+function zigzag(num: number): number {
+  return ((num << 1) ^ (num >> 31)) >>> 0;
+}
+
+function writeGeometry(feature, pbf?: Protobuf): void {
+  const geometry = feature.loadGeometry();
+  const type = feature.type;
+  let x = 0;
+  let y = 0;
+  const rings = geometry.length;
+  for (let r = 0; r < rings; r++) {
+    const ring = geometry[r];
+    let count = 1;
+    if (type === 1) {
+      count = ring.length;
+    }
+    pbf!.writeVarint(command(1, count)); // moveto
+    // do not write polygon closing path as lineto
+    const lineCount = type === 3 ? ring.length - 1 : ring.length;
+    for (let i = 0; i < lineCount; i++) {
+      if (i === 1 && type !== 1) {
+        pbf!.writeVarint(command(2, lineCount - 1)); // lineto
+      }
+      const dx = ring[i].x - x;
+      const dy = ring[i].y - y;
+      pbf!.writeVarint(zigzag(dx));
+      pbf!.writeVarint(zigzag(dy));
+      x += dx;
+      y += dy;
+    }
+    if (type === 3) {
+      pbf!.writeVarint(command(7, 1)); // closepath
+    }
+  }
+}
+
+function writeValue(value: unknown, pbf?: Protobuf): void {
+  switch (typeof value) {
+    case 'string':
+      pbf!.writeStringField(1, value);
+      break;
+    case 'boolean':
+      pbf!.writeBooleanField(7, value);
+      break;
+    case 'number':
+      if (value % 1 !== 0 || (value < 0 && !Number.isSafeInteger(-value * 2 - 1))) {
+        pbf!.writeDoubleField(3, value);
+      } else if (value < 0) {
+        pbf!.writeSVarintField(6, value);
+      } else {
+        pbf!.writeVarintField(5, value);
+      }
+      break;
+    default:
+    // ignore
+  }
+}

package/src/mvt/lib/mvt-worker-transport.ts

@@ -0,0 +1,74 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+import type {DehydratedArrowTable, SplitArrowBuffersOptions} from '@loaders.gl/arrow/transport';
+import {dehydrateArrowTable, hydrateArrowTable} from '@loaders.gl/arrow/transport';
+import type {ArrowTable} from '@loaders.gl/core/schema';
+import type {MVTLoaderOptions} from '../mvt-loader';
+
+type SerializedArrowTable = Omit<ArrowTable, 'data'> & {
+  data: DehydratedArrowTable;
+};
+
+/**
+ * Serializes MVT worker results so Arrow tables survive structured clone.
+ * @param result Parser result.
+ * @param options MVT loader options.
+ * @returns Worker-safe parser result.
+ */
+export function serializeMVTWorkerResult(result: unknown, options?: MVTLoaderOptions): unknown {
+  if (isArrowTable(result)) {
+    const bufferCopyMode = getWorkerTransferBufferCopyMode(options);
+    return {
+      ...result,
+      data: dehydrateArrowTable(result.data, bufferCopyMode ? {copy: bufferCopyMode} : undefined)
+    };
+  }
+
+  return result;
+}
+
+/**
+ * Deserializes MVT worker results into real Arrow tables on the main thread.
+ * @param result Worker result.
+ * @returns Hydrated worker result.
+ */
+export function deserializeMVTWorkerResult(result: unknown): unknown {
+  if (isSerializedArrowTable(result)) {
+    return {
+      ...result,
+      data: hydrateArrowTable(result.data)
+    };
+  }
+
+  return result;
+}
+
+function isArrowTable(value: unknown): value is ArrowTable {
+  const table = value as ArrowTable;
+  return Boolean(
+    table &&
+      typeof table === 'object' &&
+      table.shape === 'arrow-table' &&
+      table.data &&
+      'batches' in table.data
+  );
+}
+
+function isSerializedArrowTable(value: unknown): value is SerializedArrowTable {
+  const table = value as SerializedArrowTable;
+  return Boolean(
+    table &&
+      typeof table === 'object' &&
+      table.shape === 'arrow-table' &&
+      table.data &&
+      table.data.transport === 'arrow-js'
+  );
+}
+
+function getWorkerTransferBufferCopyMode(
+  options?: MVTLoaderOptions
+): SplitArrowBuffersOptions['copy'] | undefined {
+  return options?.core?.workerTransferBufferCopy ?? options?.workerTransferBufferCopy;
+}

package/src/mvt/lib/parse-mvt-geojson.ts

@@ -0,0 +1,65 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {MVTGeoJSONFeature, MVTGeoJSONOptions, MVTGeoJSONTable} from './types';
+import Protobuf from 'pbf';
+import {VectorTile} from './vector-tile/vector-tile';
+
+/** Parse an MVT buffer into a GeoJSON feature collection without loading binary or Arrow converters. */
+export function parseMVTGeoJSON(
+  arrayBuffer: ArrayBuffer,
+  options: MVTGeoJSONOptions = {}
+): MVTGeoJSONTable {
+  const mvtOptions = options.mvt || {};
+  if (mvtOptions.coordinates === 'wgs84' && !mvtOptions.tileIndex) {
+    throw new Error('MVT Loader: WGS84 coordinates need tileIndex property');
+  }
+
+  const features: MVTGeoJSONFeature[] = [];
+  if (arrayBuffer.byteLength > 0) {
+    features.push(...parseToGeoJSONFeatures(arrayBuffer, mvtOptions));
+  }
+
+  return {shape: 'geojson-table', type: 'FeatureCollection', features};
+}
+
+/** Parse only the features from selected layers into GeoJSON without GIS or Arrow conversion code. */
+export function parseToGeoJSONFeatures(
+  arrayBuffer: ArrayBuffer,
+  mvtOptions: NonNullable<MVTGeoJSONOptions['mvt']>
+): MVTGeoJSONFeature[] {
+  if (arrayBuffer.byteLength <= 0) {
+    return [];
+  }
+  const tile = new VectorTile(new Protobuf(arrayBuffer));
+  const selectedLayers = Array.isArray(mvtOptions.layers)
+    ? mvtOptions.layers
+    : Object.keys(tile.layers);
+  const features: MVTGeoJSONFeature[] = [];
+
+  for (const layerName of selectedLayers) {
+    const vectorTileLayer = tile.layers[layerName];
+    if (!vectorTileLayer) {
+      continue;
+    }
+
+    for (let index = 0; index < vectorTileLayer.length; index++) {
+      const vectorTileFeature = vectorTileLayer.getGeoJSONFeature(index);
+      const feature: MVTGeoJSONFeature = vectorTileFeature.toGeoJSONFeature(
+        mvtOptions.coordinates || 'local',
+        mvtOptions.tileIndex
+      );
+      if (mvtOptions.layerProperty) {
+        feature.properties ||= {};
+        feature.properties[mvtOptions.layerProperty] = layerName;
+      }
+      if (mvtOptions.sourceLayer) {
+        feature.sourceLayer = layerName;
+      }
+      features.push(feature);
+    }
+  }
+
+  return features;
+}