@scoova/mgl 1.0.0

package/src/data/feature_index.ts

@@ -0,0 +1,337 @@
+import Point from '@mapbox/point-geometry';
+import {loadGeometry} from './load_geometry';
+import {toEvaluationFeature} from './evaluation_feature';
+import {EXTENT} from './extent';
+import {featureFilter} from '@maplibre/maplibre-gl-style-spec';
+import {TransferableGridIndex} from '../util/transferable_grid_index';
+import {DictionaryCoder} from '../util/dictionary_coder';
+import vt from '@mapbox/vector-tile';
+import Protobuf from 'pbf';
+import {GeoJSONFeature} from '../util/vectortile_to_geojson';
+import type {MapGeoJSONFeature} from '../util/vectortile_to_geojson';
+import {arraysIntersect, mapObject, extend} from '../util/util';
+import {OverscaledTileID} from '../source/tile_id';
+import {register} from '../util/web_worker_transfer';
+import {EvaluationParameters} from '../style/evaluation_parameters';
+import {SourceFeatureState} from '../source/source_state';
+import {polygonIntersectsBox} from '../util/intersection_tests';
+import {PossiblyEvaluated} from '../style/properties';
+import {FeatureIndexArray} from './array_types.g';
+import {mat4} from 'gl-matrix';
+
+import type {StyleLayer} from '../style/style_layer';
+import type {FeatureFilter, FeatureState, FilterSpecification, PromoteIdSpecification} from '@maplibre/maplibre-gl-style-spec';
+import type {Transform} from '../geo/transform';
+import type {VectorTileFeature, VectorTileLayer} from '@mapbox/vector-tile';
+
+type QueryParameters = {
+    scale: number;
+    pixelPosMatrix: mat4;
+    transform: Transform;
+    tileSize: number;
+    queryGeometry: Array<Point>;
+    cameraQueryGeometry: Array<Point>;
+    queryPadding: number;
+    params: {
+        filter: FilterSpecification;
+        layers: Array<string>;
+        availableImages: Array<string>;
+    };
+};
+
+/**
+ * An in memory index class to allow fast interaction with features
+ */
+export class FeatureIndex {
+    tileID: OverscaledTileID;
+    x: number;
+    y: number;
+    z: number;
+    grid: TransferableGridIndex;
+    grid3D: TransferableGridIndex;
+    featureIndexArray: FeatureIndexArray;
+    promoteId?: PromoteIdSpecification;
+
+    rawTileData: ArrayBuffer;
+    bucketLayerIDs: Array<Array<string>>;
+
+    vtLayers: {[_: string]: VectorTileLayer};
+    sourceLayerCoder: DictionaryCoder;
+
+    constructor(tileID: OverscaledTileID, promoteId?: PromoteIdSpecification | null) {
+        this.tileID = tileID;
+        this.x = tileID.canonical.x;
+        this.y = tileID.canonical.y;
+        this.z = tileID.canonical.z;
+        this.grid = new TransferableGridIndex(EXTENT, 16, 0);
+        this.grid3D = new TransferableGridIndex(EXTENT, 16, 0);
+        this.featureIndexArray = new FeatureIndexArray();
+        this.promoteId = promoteId;
+    }
+
+    insert(feature: VectorTileFeature, geometry: Array<Array<Point>>, featureIndex: number, sourceLayerIndex: number, bucketIndex: number, is3D?: boolean) {
+        const key = this.featureIndexArray.length;
+        this.featureIndexArray.emplaceBack(featureIndex, sourceLayerIndex, bucketIndex);
+
+        const grid = is3D ? this.grid3D : this.grid;
+
+        for (let r = 0; r < geometry.length; r++) {
+            const ring = geometry[r];
+
+            const bbox = [Infinity, Infinity, -Infinity, -Infinity];
+            for (let i = 0; i < ring.length; i++) {
+                const p = ring[i];
+                bbox[0] = Math.min(bbox[0], p.x);
+                bbox[1] = Math.min(bbox[1], p.y);
+                bbox[2] = Math.max(bbox[2], p.x);
+                bbox[3] = Math.max(bbox[3], p.y);
+            }
+
+            if (bbox[0] < EXTENT &&
+                bbox[1] < EXTENT &&
+                bbox[2] >= 0 &&
+                bbox[3] >= 0) {
+                grid.insert(key, bbox[0], bbox[1], bbox[2], bbox[3]);
+            }
+        }
+    }
+
+    loadVTLayers(): {[_: string]: VectorTileLayer} {
+        if (!this.vtLayers) {
+            this.vtLayers = new vt.VectorTile(new Protobuf(this.rawTileData)).layers;
+            this.sourceLayerCoder = new DictionaryCoder(this.vtLayers ? Object.keys(this.vtLayers).sort() : ['_geojsonTileLayer']);
+        }
+        return this.vtLayers;
+    }
+
+    // Finds non-symbol features in this tile at a particular position.
+    query(
+        args: QueryParameters,
+        styleLayers: {[_: string]: StyleLayer},
+        serializedLayers: {[_: string]: any},
+        sourceFeatureState: SourceFeatureState
+    ): {[_: string]: Array<{featureIndex: number; feature: GeoJSONFeature}>} {
+        this.loadVTLayers();
+
+        const params = args.params || {} as { filter: any; layers: string[]; availableImages: string[] },
+            pixelsToTileUnits = EXTENT / args.tileSize / args.scale,
+            filter = featureFilter(params.filter);
+
+        const queryGeometry = args.queryGeometry;
+        const queryPadding = args.queryPadding * pixelsToTileUnits;
+
+        const bounds = getBounds(queryGeometry);
+        const matching = this.grid.query(bounds.minX - queryPadding, bounds.minY - queryPadding, bounds.maxX + queryPadding, bounds.maxY + queryPadding);
+
+        const cameraBounds = getBounds(args.cameraQueryGeometry);
+        const matching3D = this.grid3D.query(
+            cameraBounds.minX - queryPadding, cameraBounds.minY - queryPadding, cameraBounds.maxX + queryPadding, cameraBounds.maxY + queryPadding,
+            (bx1, by1, bx2, by2) => {
+                return polygonIntersectsBox(args.cameraQueryGeometry, bx1 - queryPadding, by1 - queryPadding, bx2 + queryPadding, by2 + queryPadding);
+            });
+
+        for (const key of matching3D) {
+            matching.push(key);
+        }
+
+        matching.sort(topDownFeatureComparator);
+
+        const result = {};
+        let previousIndex;
+        for (let k = 0; k < matching.length; k++) {
+            const index = matching[k];
+
+            // don't check the same feature more than once
+            if (index === previousIndex) continue;
+            previousIndex = index;
+
+            const match = this.featureIndexArray.get(index);
+            let featureGeometry = null;
+            this.loadMatchingFeature(
+                result,
+                match.bucketIndex,
+                match.sourceLayerIndex,
+                match.featureIndex,
+                filter,
+                params.layers,
+                params.availableImages,
+                styleLayers,
+                serializedLayers,
+                sourceFeatureState,
+                (feature: VectorTileFeature, styleLayer: StyleLayer, featureState: FeatureState) => {
+                    if (!featureGeometry) {
+                        featureGeometry = loadGeometry(feature);
+                    }
+
+                    return styleLayer.queryIntersectsFeature(queryGeometry, feature, featureState, featureGeometry, this.z, args.transform, pixelsToTileUnits, args.pixelPosMatrix);
+                }
+            );
+        }
+
+        return result;
+    }
+
+    loadMatchingFeature(
+        result: {
+            [_: string]: Array<{
+                featureIndex: number;
+                feature: GeoJSONFeature;
+                intersectionZ?: boolean | number;
+            }>;
+        },
+        bucketIndex: number,
+        sourceLayerIndex: number,
+        featureIndex: number,
+        filter: FeatureFilter,
+        filterLayerIDs: Array<string>,
+        availableImages: Array<string>,
+        styleLayers: {[_: string]: StyleLayer},
+        serializedLayers: {[_: string]: any},
+        sourceFeatureState?: SourceFeatureState,
+        intersectionTest?: (
+            feature: VectorTileFeature,
+            styleLayer: StyleLayer,
+            featureState: any,
+            id: string | number | void
+        ) => boolean | number) {
+
+        const layerIDs = this.bucketLayerIDs[bucketIndex];
+        if (filterLayerIDs && !arraysIntersect(filterLayerIDs, layerIDs))
+            return;
+
+        const sourceLayerName = this.sourceLayerCoder.decode(sourceLayerIndex);
+        const sourceLayer = this.vtLayers[sourceLayerName];
+        const feature = sourceLayer.feature(featureIndex);
+
+        if (filter.needGeometry) {
+            const evaluationFeature = toEvaluationFeature(feature, true);
+            if (!filter.filter(new EvaluationParameters(this.tileID.overscaledZ), evaluationFeature, this.tileID.canonical)) {
+                return;
+            }
+        } else if (!filter.filter(new EvaluationParameters(this.tileID.overscaledZ), feature)) {
+            return;
+        }
+
+        const id = this.getId(feature, sourceLayerName);
+
+        for (let l = 0; l < layerIDs.length; l++) {
+            const layerID = layerIDs[l];
+
+            if (filterLayerIDs && filterLayerIDs.indexOf(layerID) < 0) {
+                continue;
+            }
+
+            const styleLayer = styleLayers[layerID];
+
+            if (!styleLayer) continue;
+
+            let featureState = {};
+            if (id && sourceFeatureState) {
+                // `feature-state` expression evaluation requires feature state to be available
+                featureState = sourceFeatureState.getState(styleLayer.sourceLayer || '_geojsonTileLayer', id);
+            }
+
+            const serializedLayer = extend({}, serializedLayers[layerID]);
+
+            serializedLayer.paint = evaluateProperties(serializedLayer.paint, styleLayer.paint, feature, featureState, availableImages);
+            serializedLayer.layout = evaluateProperties(serializedLayer.layout, styleLayer.layout, feature, featureState, availableImages);
+
+            const intersectionZ = !intersectionTest || intersectionTest(feature, styleLayer, featureState);
+            if (!intersectionZ) {
+                // Only applied for non-symbol features
+                continue;
+            }
+
+            const geojsonFeature = new GeoJSONFeature(feature, this.z, this.x, this.y, id) as MapGeoJSONFeature;
+            geojsonFeature.layer = serializedLayer;
+            let layerResult = result[layerID];
+            if (layerResult === undefined) {
+                layerResult = result[layerID] = [];
+            }
+            layerResult.push({featureIndex, feature: geojsonFeature, intersectionZ});
+        }
+    }
+
+    // Given a set of symbol indexes that have already been looked up,
+    // return a matching set of GeoJSONFeatures
+    lookupSymbolFeatures(symbolFeatureIndexes: Array<number>,
+        serializedLayers: {[_: string]: StyleLayer},
+        bucketIndex: number,
+        sourceLayerIndex: number,
+        filterSpec: FilterSpecification,
+        filterLayerIDs: Array<string>,
+        availableImages: Array<string>,
+        styleLayers: {[_: string]: StyleLayer}) {
+        const result = {};
+        this.loadVTLayers();
+
+        const filter = featureFilter(filterSpec);
+
+        for (const symbolFeatureIndex of symbolFeatureIndexes) {
+            this.loadMatchingFeature(
+                result,
+                bucketIndex,
+                sourceLayerIndex,
+                symbolFeatureIndex,
+                filter,
+                filterLayerIDs,
+                availableImages,
+                styleLayers,
+                serializedLayers
+            );
+
+        }
+        return result;
+    }
+
+    hasLayer(id: string) {
+        for (const layerIDs of this.bucketLayerIDs) {
+            for (const layerID of layerIDs) {
+                if (id === layerID) return true;
+            }
+        }
+
+        return false;
+    }
+
+    getId(feature: VectorTileFeature, sourceLayerId: string): string | number {
+        let id: string | number = feature.id;
+        if (this.promoteId) {
+            const propName = typeof this.promoteId === 'string' ? this.promoteId : this.promoteId[sourceLayerId];
+            id = feature.properties[propName] as string | number;
+            if (typeof id === 'boolean') id = Number(id);
+        }
+        return id;
+    }
+}
+
+register(
+    'FeatureIndex',
+    FeatureIndex,
+    {omit: ['rawTileData', 'sourceLayerCoder']}
+);
+
+function evaluateProperties(serializedProperties, styleLayerProperties, feature, featureState, availableImages) {
+    return mapObject(serializedProperties, (property, key) => {
+        const prop = styleLayerProperties instanceof PossiblyEvaluated ? styleLayerProperties.get(key) : null;
+        return prop && prop.evaluate ? prop.evaluate(feature, featureState, availableImages) : prop;
+    });
+}
+
+function getBounds(geometry: Array<Point>) {
+    let minX = Infinity;
+    let minY = Infinity;
+    let maxX = -Infinity;
+    let maxY = -Infinity;
+    for (const p of geometry) {
+        minX = Math.min(minX, p.x);
+        minY = Math.min(minY, p.y);
+        maxX = Math.max(maxX, p.x);
+        maxY = Math.max(maxY, p.y);
+    }
+    return {minX, minY, maxX, maxY};
+}
+
+function topDownFeatureComparator(a, b) {
+    return b - a;
+}

package/src/data/feature_position_map.test.ts

@@ -0,0 +1,33 @@
+import {FeaturePositionMap} from './feature_position_map';
+import {serialize, deserialize} from '../util/web_worker_transfer';
+
+describe('FeaturePositionMap', () => {
+    test('Can be queried after serialization/deserialization', () => {
+        const featureMap = new FeaturePositionMap();
+        featureMap.add(7, 1, 0, 1);
+        featureMap.add(3, 2, 1, 2);
+        featureMap.add(7, 3, 2, 3);
+        featureMap.add(4, 4, 3, 4);
+        featureMap.add(2, 5, 4, 5);
+        featureMap.add(7, 6, 5, 7);
+
+        const featureMap2 = deserialize(serialize(featureMap, [])) as FeaturePositionMap;
+
+        const compareIndex = (a, b) => a.index - b.index;
+
+        expect(featureMap2.getPositions(7).sort(compareIndex)).toEqual([
+            {index: 1, start: 0, end: 1},
+            {index: 3, start: 2, end: 3},
+            {index: 6, start: 5, end: 7}
+        ].sort(compareIndex));
+    });
+
+    test('Can not be queried before serialization/deserialization', () => {
+        const featureMap = new FeaturePositionMap();
+        featureMap.add(0, 1, 2, 3);
+
+        expect(() => {
+            featureMap.getPositions(0);
+        }).toThrow();
+    });
+});

package/src/data/feature_position_map.ts

@@ -0,0 +1,126 @@
+import murmur3 from 'murmurhash-js';
+import {register} from '../util/web_worker_transfer';
+
+type SerializedFeaturePositionMap = {
+    ids: Float64Array;
+    positions: Uint32Array;
+};
+
+type FeaturePosition = {
+    index: number;
+    start: number;
+    end: number;
+};
+
+// A transferable data structure that maps feature ids to their indices and buffer offsets
+export class FeaturePositionMap {
+    ids: Array<number>;
+    positions: Array<number>;
+    indexed: boolean;
+
+    constructor() {
+        this.ids = [];
+        this.positions = [];
+        this.indexed = false;
+    }
+
+    add(id: unknown, index: number, start: number, end: number) {
+        this.ids.push(getNumericId(id));
+        this.positions.push(index, start, end);
+    }
+
+    getPositions(id: unknown): Array<FeaturePosition> {
+        if (!this.indexed) throw new Error('Trying to get index, but feature positions are not indexed');
+
+        const intId = getNumericId(id);
+
+        // binary search for the first occurrence of id in this.ids;
+        // relies on ids/positions being sorted by id, which happens in serialization
+        let i = 0;
+        let j = this.ids.length - 1;
+        while (i < j) {
+            const m = (i + j) >> 1;
+            if (this.ids[m] >= intId) {
+                j = m;
+            } else {
+                i = m + 1;
+            }
+        }
+        const positions = [];
+        while (this.ids[i] === intId) {
+            const index = this.positions[3 * i];
+            const start = this.positions[3 * i + 1];
+            const end = this.positions[3 * i + 2];
+            positions.push({index, start, end});
+            i++;
+        }
+        return positions;
+    }
+
+    static serialize(map: FeaturePositionMap, transferables: Array<ArrayBuffer>): SerializedFeaturePositionMap {
+        const ids = new Float64Array(map.ids);
+        const positions = new Uint32Array(map.positions);
+
+        sort(ids, positions, 0, ids.length - 1);
+
+        if (transferables) {
+            transferables.push(ids.buffer, positions.buffer);
+        }
+
+        return {ids, positions};
+    }
+
+    static deserialize(obj: SerializedFeaturePositionMap): FeaturePositionMap {
+        const map = new FeaturePositionMap();
+        // after transferring, we only use these arrays statically (no pushes),
+        // so TypedArray vs Array distinction that flow points out doesn't matter
+        map.ids = (obj.ids as any);
+        map.positions = (obj.positions as any);
+        map.indexed = true;
+        return map;
+    }
+}
+
+function getNumericId(value: unknown) {
+    const numValue = +value;
+    if (!isNaN(numValue) && numValue <= Number.MAX_SAFE_INTEGER) {
+        return numValue;
+    }
+    return murmur3(String(value));
+}
+
+// custom quicksort that sorts ids, indices and offsets together (by ids)
+// uses Hoare partitioning & manual tail call optimization to avoid worst case scenarios
+function sort(ids, positions, left, right) {
+    while (left < right) {
+        const pivot = ids[(left + right) >> 1];
+        let i = left - 1;
+        let j = right + 1;
+
+        while (true) {
+            do i++; while (ids[i] < pivot);
+            do j--; while (ids[j] > pivot);
+            if (i >= j) break;
+            swap(ids, i, j);
+            swap(positions, 3 * i, 3 * j);
+            swap(positions, 3 * i + 1, 3 * j + 1);
+            swap(positions, 3 * i + 2, 3 * j + 2);
+        }
+
+        if (j - left < right - j) {
+            sort(ids, positions, left, j);
+            left = j + 1;
+        } else {
+            sort(ids, positions, j + 1, right);
+            right = j;
+        }
+    }
+}
+
+function swap(arr, i, j) {
+    const tmp = arr[i];
+    arr[i] = arr[j];
+    arr[j] = tmp;
+}
+
+register('FeaturePositionMap', FeaturePositionMap);

package/src/data/index_array_type.ts

@@ -0,0 +1,9 @@
+import {LineIndexArray, TriangleIndexArray, LineStripIndexArray} from './array_types.g';
+
+/**
+ * An index array stores Uint16 indices of vertexes in a corresponding vertex array. We use
+ * three kinds of index arrays: arrays storing groups of three indices, forming triangles;
+ * arrays storing pairs of indices, forming line segments; and arrays storing single indices,
+ * forming a line strip.
+ */
+export {LineIndexArray, TriangleIndexArray, LineStripIndexArray};

package/src/data/load_geometry.test.ts

@@ -0,0 +1,49 @@
+import fs from 'fs';
+import path from 'path';
+import Protobuf from 'pbf';
+import {VectorTile} from '@mapbox/vector-tile';
+import {loadGeometry} from './load_geometry';
+
+// Load a line feature from fixture tile.
+const vt = new VectorTile(new Protobuf(fs.readFileSync(path.resolve(__dirname, '../../test/unit/assets/mbsv5-6-18-23.vector.pbf'))));
+
+describe('loadGeometry', () => {
+    test('loadGeometry', () => {
+        const feature = vt.layers.road.feature(0);
+        const originalGeometry = feature.loadGeometry();
+        const scaledGeometry = loadGeometry(feature);
+        expect(scaledGeometry[0][0].x).toBe(originalGeometry[0][0].x * 2);
+        expect(scaledGeometry[0][0].y).toBe(originalGeometry[0][0].y * 2);
+    });
+
+    test('loadGeometry warns and clamps when exceeding extent', () => {
+        const feature = vt.layers.road.feature(0);
+        feature.extent = 2048;
+
+        let numWarnings = 0;
+
+        // Use a custom console.warn to count warnings
+        const warn = console.warn;
+        console.warn = (warning) => {
+            if (warning.match(/Geometry exceeds allowed extent, reduce your vector tile buffer size/)) {
+                numWarnings++;
+            }
+        };
+
+        const lines = loadGeometry(feature);
+
+        expect(numWarnings).toBe(1);
+
+        let maxValue = -Infinity;
+        for (const line of lines) {
+            for (const {x, y} of line) {
+                maxValue = Math.max(x, y, maxValue);
+            }
+        }
+        expect(maxValue).toBe(16383);
+
+        // Put it back
+        console.warn = warn;
+
+    });
+});

package/src/data/load_geometry.ts

@@ -0,0 +1,44 @@
+import {warnOnce, clamp} from '../util/util';
+
+import {EXTENT} from './extent';
+
+import type Point from '@mapbox/point-geometry';
+import type {VectorTileFeature} from '@mapbox/vector-tile';
+
+// These bounds define the minimum and maximum supported coordinate values.
+// While visible coordinates are within [0, EXTENT], tiles may theoretically
+// contain coordinates within [-Infinity, Infinity]. Our range is limited by the
+// number of bits used to represent the coordinate.
+const BITS = 15;
+const MAX = Math.pow(2, BITS - 1) - 1;
+const MIN = -MAX - 1;
+
+/**
+ * Loads a geometry from a VectorTileFeature and scales it to the common extent
+ * used internally.
+ * @param feature - the vector tile feature to load
+ */
+export function loadGeometry(feature: VectorTileFeature): Array<Array<Point>> {
+    const scale = EXTENT / feature.extent;
+    const geometry = feature.loadGeometry();
+    for (let r = 0; r < geometry.length; r++) {
+        const ring = geometry[r];
+        for (let p = 0; p < ring.length; p++) {
+            const point = ring[p];
+            // round here because mapbox-gl-native uses integers to represent
+            // points and we need to do the same to avoid renering differences.
+            const x = Math.round(point.x * scale);
+            const y = Math.round(point.y * scale);
+
+            point.x = clamp(x, MIN, MAX);
+            point.y = clamp(y, MIN, MAX);
+
+            if (x < point.x || x > point.x + 1 || y < point.y || y > point.y + 1) {
+                // warn when exceeding allowed extent except for the 1-px-off case
+                // https://github.com/mapbox/mapbox-gl-js/issues/8992
+                warnOnce('Geometry exceeds allowed extent, reduce your vector tile buffer size');
+            }
+        }
+    }
+    return geometry;
+}

package/src/data/pos3d_attributes.ts

@@ -0,0 +1,5 @@
+import {createLayout} from '../util/struct_array';
+
+export default createLayout([
+    {name: 'a_pos3d', type: 'Int16', components: 3}
+]);

package/src/data/pos_attributes.ts

@@ -0,0 +1,5 @@
+import {createLayout} from '../util/struct_array';
+
+export default createLayout([
+    {name: 'a_pos', type: 'Int16', components: 2}
+]);