@scoova/mgl 1.0.0

package/src/source/worker.ts

@@ -0,0 +1,301 @@
+import {Actor, ActorTarget, IActor} from '../util/actor';
+import {StyleLayerIndex} from '../style/style_layer_index';
+import {VectorTileWorkerSource} from './vector_tile_worker_source';
+import {RasterDEMTileWorkerSource} from './raster_dem_tile_worker_source';
+import {rtlWorkerPlugin, RTLTextPlugin} from './rtl_text_plugin_worker';
+import {GeoJSONWorkerSource, LoadGeoJSONParameters} from './geojson_worker_source';
+import {isWorker} from '../util/util';
+import {addProtocol, removeProtocol} from './protocol_crud';
+import {PluginState} from './rtl_text_plugin_status';
+import type {
+    WorkerSource,
+    WorkerSourceConstructor,
+    WorkerTileParameters,
+    WorkerDEMTileParameters,
+    TileParameters
+} from '../source/worker_source';
+
+import type {WorkerGlobalScopeInterface} from '../util/web_worker';
+import type {LayerSpecification} from '@maplibre/maplibre-gl-style-spec';
+import {
+    MessageType,
+    type ClusterIDAndSource,
+    type GetClusterLeavesParams,
+    type RemoveSourceParams,
+    type UpdateLayersParamaeters
+} from '../util/actor_messages';
+
+/**
+ * The Worker class responsible for background thread related execution
+ */
+export default class Worker {
+    self: WorkerGlobalScopeInterface & ActorTarget;
+    actor: Actor;
+    layerIndexes: {[_: string]: StyleLayerIndex};
+    availableImages: {[_: string]: Array<string>};
+    externalWorkerSourceTypes: { [_: string]: WorkerSourceConstructor };
+    /**
+     * This holds a cache for the already created worker source instances.
+     * The cache is build with the following hierarchy:
+     * [mapId][sourceType][sourceName]: worker source instance
+     * sourceType can be 'vector' for example
+     */
+    workerSources: {
+        [_: string]: {
+            [_: string]: {
+                [_: string]: WorkerSource;
+            };
+        };
+    };
+    /**
+     * This holds a cache for the already created DEM worker source instances.
+     * The cache is build with the following hierarchy:
+     * [mapId][sourceType]: DEM worker source instance
+     * sourceType can be 'raster-dem' for example
+     */
+    demWorkerSources: {
+        [_: string]: {
+            [_: string]: RasterDEMTileWorkerSource;
+        };
+    };
+    referrer: string;
+
+    constructor(self: WorkerGlobalScopeInterface & ActorTarget) {
+        this.self = self;
+        this.actor = new Actor(self);
+
+        this.layerIndexes = {};
+        this.availableImages = {};
+
+        this.workerSources = {};
+        this.demWorkerSources = {};
+        this.externalWorkerSourceTypes = {};
+
+        this.self.registerWorkerSource = (name: string, WorkerSource: WorkerSourceConstructor) => {
+            if (this.externalWorkerSourceTypes[name]) {
+                throw new Error(`Worker source with name "${name}" already registered.`);
+            }
+            this.externalWorkerSourceTypes[name] = WorkerSource;
+        };
+
+        this.self.addProtocol = addProtocol;
+        this.self.removeProtocol = removeProtocol;
+
+        // This is invoked by the RTL text plugin when the download via the `importScripts` call has finished, and the code has been parsed.
+        this.self.registerRTLTextPlugin = (rtlTextPlugin: RTLTextPlugin) => {
+            if (rtlWorkerPlugin.isParsed()) {
+                throw new Error('RTL text plugin already registered.');
+            }
+            rtlWorkerPlugin.setMethods(rtlTextPlugin);
+        };
+
+        this.actor.registerMessageHandler(MessageType.loadDEMTile, (mapId: string, params: WorkerDEMTileParameters) => {
+            return this._getDEMWorkerSource(mapId, params.source).loadTile(params);
+        });
+
+        this.actor.registerMessageHandler(MessageType.removeDEMTile, async (mapId: string, params: TileParameters) => {
+            this._getDEMWorkerSource(mapId, params.source).removeTile(params);
+        });
+
+        this.actor.registerMessageHandler(MessageType.getClusterExpansionZoom, async (mapId: string, params: ClusterIDAndSource) => {
+            return (this._getWorkerSource(mapId, params.type, params.source) as GeoJSONWorkerSource).getClusterExpansionZoom(params);
+        });
+
+        this.actor.registerMessageHandler(MessageType.getClusterChildren, async (mapId: string, params: ClusterIDAndSource) => {
+            return (this._getWorkerSource(mapId, params.type, params.source) as GeoJSONWorkerSource).getClusterChildren(params);
+        });
+
+        this.actor.registerMessageHandler(MessageType.getClusterLeaves, async (mapId: string, params: GetClusterLeavesParams) => {
+            return (this._getWorkerSource(mapId, params.type, params.source) as GeoJSONWorkerSource).getClusterLeaves(params);
+        });
+
+        this.actor.registerMessageHandler(MessageType.loadData, (mapId: string, params: LoadGeoJSONParameters) => {
+            return (this._getWorkerSource(mapId, params.type, params.source) as GeoJSONWorkerSource).loadData(params);
+        });
+
+        this.actor.registerMessageHandler(MessageType.getData, (mapId: string, params: LoadGeoJSONParameters) => {
+            return (this._getWorkerSource(mapId, params.type, params.source) as GeoJSONWorkerSource).getData();
+        });
+
+        this.actor.registerMessageHandler(MessageType.loadTile, (mapId: string, params: WorkerTileParameters) => {
+            return this._getWorkerSource(mapId, params.type, params.source).loadTile(params);
+        });
+
+        this.actor.registerMessageHandler(MessageType.reloadTile, (mapId: string, params: WorkerTileParameters) => {
+            return this._getWorkerSource(mapId, params.type, params.source).reloadTile(params);
+        });
+
+        this.actor.registerMessageHandler(MessageType.abortTile, (mapId: string, params: TileParameters) => {
+            return this._getWorkerSource(mapId, params.type, params.source).abortTile(params);
+        });
+
+        this.actor.registerMessageHandler(MessageType.removeTile, (mapId: string, params: TileParameters) => {
+            return this._getWorkerSource(mapId, params.type, params.source).removeTile(params);
+        });
+
+        this.actor.registerMessageHandler(MessageType.removeSource, async (mapId: string, params: RemoveSourceParams) => {
+            if (!this.workerSources[mapId] ||
+                !this.workerSources[mapId][params.type] ||
+                !this.workerSources[mapId][params.type][params.source]) {
+                return;
+            }
+
+            const worker = this.workerSources[mapId][params.type][params.source];
+            delete this.workerSources[mapId][params.type][params.source];
+
+            if (worker.removeSource !== undefined) {
+                worker.removeSource(params);
+            }
+        });
+
+        this.actor.registerMessageHandler(MessageType.removeMap, async (mapId: string) => {
+            delete this.layerIndexes[mapId];
+            delete this.availableImages[mapId];
+            delete this.workerSources[mapId];
+            delete this.demWorkerSources[mapId];
+        });
+
+        this.actor.registerMessageHandler(MessageType.setReferrer, async (_mapId: string, params: string) => {
+            this.referrer = params;
+        });
+
+        this.actor.registerMessageHandler(MessageType.syncRTLPluginState, (mapId: string, params: PluginState) => {
+            return this._syncRTLPluginState(mapId, params);
+        });
+
+        this.actor.registerMessageHandler(MessageType.importScript, async (_mapId: string, params: string) => {
+            this.self.importScripts(params);
+        });
+
+        this.actor.registerMessageHandler(MessageType.setImages, (mapId: string, params: string[]) => {
+            return this._setImages(mapId, params);
+        });
+
+        this.actor.registerMessageHandler(MessageType.updateLayers, async (mapId: string, params: UpdateLayersParamaeters) => {
+            this._getLayerIndex(mapId).update(params.layers, params.removedIds);
+        });
+
+        this.actor.registerMessageHandler(MessageType.setLayers, async (mapId: string, params: Array<LayerSpecification>) => {
+            this._getLayerIndex(mapId).replace(params);
+        });
+    }
+
+    private async _setImages(mapId: string, images: Array<string>): Promise<void> {
+        this.availableImages[mapId] = images;
+        for (const workerSource in this.workerSources[mapId]) {
+            const ws = this.workerSources[mapId][workerSource];
+            for (const source in ws) {
+                ws[source].availableImages = images;
+            }
+        }
+    }
+
+    private async _syncRTLPluginState(mapId: string, incomingState: PluginState): Promise<PluginState> {
+
+        // Parsed plugin cannot be changed, so just return its current state.
+        if (rtlWorkerPlugin.isParsed()) {
+            return rtlWorkerPlugin.getState();
+        }
+
+        if (incomingState.pluginStatus !== 'loading') {
+            // simply sync and done
+            rtlWorkerPlugin.setState(incomingState);
+            return incomingState;
+        }
+        const urlToLoad = incomingState.pluginURL;
+        this.self.importScripts(urlToLoad);
+        const complete = rtlWorkerPlugin.isParsed();
+        if (complete) {
+            const loadedState: PluginState = {
+                pluginStatus: 'loaded',
+                pluginURL: urlToLoad
+            };
+            rtlWorkerPlugin.setState(loadedState);
+            return loadedState;
+        }
+
+        // error case
+        rtlWorkerPlugin.setState({
+            pluginStatus: 'error',
+            pluginURL: ''
+        });
+        throw new Error(`RTL Text Plugin failed to import scripts from ${urlToLoad}`);
+    }
+
+    private _getAvailableImages(mapId: string) {
+        let availableImages = this.availableImages[mapId];
+
+        if (!availableImages) {
+            availableImages = [];
+        }
+
+        return availableImages;
+    }
+
+    private _getLayerIndex(mapId: string) {
+        let layerIndexes = this.layerIndexes[mapId];
+        if (!layerIndexes) {
+            layerIndexes = this.layerIndexes[mapId] = new StyleLayerIndex();
+        }
+        return layerIndexes;
+    }
+
+    /**
+     * This is basically a lazy initialization of a worker per mapId and sourceType and sourceName
+     * @param mapId - the mapId
+     * @param sourceType - the source type - 'vector' for example
+     * @param sourceName - the source name - 'osm' for example
+     * @returns a new instance or a cached one
+     */
+    private _getWorkerSource(mapId: string, sourceType: string, sourceName: string): WorkerSource {
+        if (!this.workerSources[mapId])
+            this.workerSources[mapId] = {};
+        if (!this.workerSources[mapId][sourceType])
+            this.workerSources[mapId][sourceType] = {};
+
+        if (!this.workerSources[mapId][sourceType][sourceName]) {
+            // use a wrapped actor so that we can attach a target mapId param
+            // to any messages invoked by the WorkerSource, this is very important when there are multiple maps
+            const actor: IActor = {
+                sendAsync: (message, abortController) => {
+                    message.targetMapId = mapId;
+                    return this.actor.sendAsync(message, abortController);
+                }
+            };
+            switch (sourceType) {
+                case 'vector':
+                    this.workerSources[mapId][sourceType][sourceName] = new VectorTileWorkerSource(actor, this._getLayerIndex(mapId), this._getAvailableImages(mapId));
+                    break;
+                case 'geojson':
+                    this.workerSources[mapId][sourceType][sourceName] = new GeoJSONWorkerSource(actor, this._getLayerIndex(mapId), this._getAvailableImages(mapId));
+                    break;
+                default:
+                    this.workerSources[mapId][sourceType][sourceName] = new (this.externalWorkerSourceTypes[sourceType])(actor, this._getLayerIndex(mapId), this._getAvailableImages(mapId));
+                    break;
+            }
+        }
+
+        return this.workerSources[mapId][sourceType][sourceName];
+    }
+
+    /**
+     * This is basically a lazy initialization of a worker per mapId and source
+     * @param mapId - the mapId
+     * @param sourceType - the source type - 'raster-dem' for example
+     * @returns a new instance or a cached one
+     */
+    private _getDEMWorkerSource(mapId: string, sourceType: string) {
+        if (!this.demWorkerSources[mapId])
+            this.demWorkerSources[mapId] = {};
+
+        if (!this.demWorkerSources[mapId][sourceType]) {
+            this.demWorkerSources[mapId][sourceType] = new RasterDEMTileWorkerSource();
+        }
+
+        return this.demWorkerSources[mapId][sourceType];
+    }
+}
+
+if (isWorker(self)) {
+    self.worker = new Worker(self);
+}

package/src/source/worker_source.ts

@@ -0,0 +1,117 @@
+import type {ExpiryData, RequestParameters} from '../util/ajax';
+import type {RGBAImage, AlphaImage} from '../util/image';
+import type {GlyphPositions} from '../render/glyph_atlas';
+import type {ImageAtlas} from '../render/image_atlas';
+import type {OverscaledTileID} from './tile_id';
+import type {Bucket} from '../data/bucket';
+import type {FeatureIndex} from '../data/feature_index';
+import type {CollisionBoxArray} from '../data/array_types.g';
+import type {DEMEncoding} from '../data/dem_data';
+import type {StyleGlyph} from '../style/style_glyph';
+import type {StyleImage} from '../style/style_image';
+import type {PromoteIdSpecification} from '@maplibre/maplibre-gl-style-spec';
+import type {RemoveSourceParams} from '../util/actor_messages';
+import type {IActor} from '../util/actor';
+import type {StyleLayerIndex} from '../style/style_layer_index';
+
+/**
+ * Parameters to identify a tile
+ */
+export type TileParameters = {
+    type: string;
+    source: string;
+    uid: string | number;
+};
+
+/**
+ * Parameters that are send when requesting to load a tile to the worker
+ */
+export type WorkerTileParameters = TileParameters & {
+    tileID: OverscaledTileID;
+    request?: RequestParameters;
+    zoom: number;
+    maxZoom?: number;
+    tileSize: number;
+    promoteId: PromoteIdSpecification;
+    pixelRatio: number;
+    showCollisionBoxes: boolean;
+    collectResourceTiming?: boolean;
+    returnDependencies?: boolean;
+};
+
+/**
+ * The paremeters needed in order to load a DEM tile
+ */
+export type WorkerDEMTileParameters = TileParameters & {
+    rawImageData: RGBAImage | ImageBitmap | ImageData;
+    encoding: DEMEncoding;
+    redFactor: number;
+    greenFactor: number;
+    blueFactor: number;
+    baseShift: number;
+};
+
+/**
+ * The worker tile's result type
+ */
+export type WorkerTileResult = ExpiryData & {
+    buckets: Array<Bucket>;
+    imageAtlas: ImageAtlas;
+    glyphAtlasImage: AlphaImage;
+    featureIndex: FeatureIndex;
+    collisionBoxArray: CollisionBoxArray;
+    rawTileData?: ArrayBuffer;
+    resourceTiming?: Array<PerformanceResourceTiming>;
+    // Only used for benchmarking:
+    glyphMap?: {
+        [_: string]: {
+            [_: number]: StyleGlyph;
+        };
+    } | null;
+    iconMap?: {
+        [_: string]: StyleImage;
+    } | null;
+    glyphPositions?: GlyphPositions | null;
+};
+
+/**
+ * This is how the @see {@link WorkerSource} constructor should look like.
+ */
+export interface WorkerSourceConstructor {
+    new (actor: IActor, layerIndex: StyleLayerIndex, availableImages: Array<string>): WorkerSource;
+}
+
+/**
+ * `WorkerSource` should be implemented by custom source types to provide code that can be run on the WebWorkers.
+ * Each of the methods has a relevant event that triggers it from the main thread with the relevant parameters.
+ * @see {@link Map#addSourceType}
+ */
+export interface WorkerSource {
+    availableImages: Array<string>;
+
+    /**
+     * Loads a tile from the given params and parse it into buckets ready to send
+     * back to the main thread for rendering.  Should call the callback with:
+     * `{ buckets, featureIndex, collisionIndex, rawTileData}`.
+     */
+    loadTile(params: WorkerTileParameters): Promise<WorkerTileResult>;
+    /**
+     * Re-parses a tile that has already been loaded.  Yields the same data as
+     * {@link WorkerSource#loadTile}.
+     */
+    reloadTile(params: WorkerTileParameters): Promise<WorkerTileResult>;
+    /**
+     * Aborts loading a tile that is in progress.
+     */
+    abortTile(params: TileParameters): Promise<void>;
+    /**
+     * Removes this tile from any local caches.
+     */
+    removeTile(params: TileParameters): Promise<void>;
+    /**
+     * Tells the WorkerSource to abort in-progress tasks and release resources.
+     * The foreground Source is responsible for ensuring that 'removeSource' is
+     * the last message sent to the WorkerSource.
+     */
+    removeSource?: (params: RemoveSourceParams) => Promise<void>;
+}

package/src/source/worker_tile.test.ts

@@ -0,0 +1,226 @@
+import {WorkerTile} from '../source/worker_tile';
+import {GeoJSONWrapper, Feature} from '../source/geojson_wrapper';
+import {OverscaledTileID} from '../source/tile_id';
+import {StyleLayerIndex} from '../style/style_layer_index';
+import {WorkerTileParameters} from './worker_source';
+import {VectorTile} from '@mapbox/vector-tile';
+
+function createWorkerTile() {
+    return new WorkerTile({
+        uid: '',
+        zoom: 0,
+        maxZoom: 20,
+        tileSize: 512,
+        source: 'source',
+        tileID: new OverscaledTileID(1, 0, 1, 1, 1),
+        overscaling: 1
+    } as any as WorkerTileParameters);
+}
+
+function createWrapper() {
+    return new GeoJSONWrapper([{
+        type: 1,
+        geometry: [0, 0],
+        tags: {}
+    } as any as Feature]);
+}
+
+describe('worker tile', () => {
+    test('WorkerTile#parse', async () => {
+        const layerIndex = new StyleLayerIndex([{
+            id: 'test',
+            source: 'source',
+            type: 'circle'
+        }]);
+
+        const tile = createWorkerTile();
+        const result = await tile.parse(createWrapper(), layerIndex, [], {} as any);
+        expect(result.buckets[0]).toBeTruthy();
+    });
+
+    test('WorkerTile#parse skips hidden layers', async () => {
+        const layerIndex = new StyleLayerIndex([{
+            id: 'test-hidden',
+            source: 'source',
+            type: 'fill',
+            layout: {visibility: 'none'}
+        }]);
+
+        const tile = createWorkerTile();
+        const result = await tile.parse(createWrapper(), layerIndex, [], {} as any);
+        expect(result.buckets).toHaveLength(0);
+    });
+
+    test('WorkerTile#parse skips layers without a corresponding source layer', async () => {
+        const layerIndex = new StyleLayerIndex([{
+            id: 'test',
+            source: 'source',
+            'source-layer': 'nonesuch',
+            type: 'fill'
+        }]);
+
+        const tile = createWorkerTile();
+        const result = await tile.parse({layers: {}}, layerIndex, [], {} as any);
+        expect(result.buckets).toHaveLength(0);
+    });
+
+    test('WorkerTile#parse warns once when encountering a v1 vector tile layer', async () => {
+        const layerIndex = new StyleLayerIndex([{
+            id: 'test',
+            source: 'source',
+            'source-layer': 'test',
+            type: 'fill'
+        }]);
+
+        const data = {
+            layers: {
+                test: {
+                    version: 1
+                }
+            }
+        } as any as VectorTile;
+
+        const spy = jest.spyOn(console, 'warn').mockImplementation(() => {});
+
+        const tile = createWorkerTile();
+        await tile.parse(data, layerIndex, [], {} as any);
+        expect(spy.mock.calls[0][0]).toMatch(/does not use vector tile spec v2/);
+    });
+
+    test('WorkerTile#parse would request all types of dependencies', async () => {
+        const tile = createWorkerTile();
+        const layerIndex = new StyleLayerIndex([{
+            id: '1',
+            type: 'fill',
+            source: 'source',
+            'source-layer': 'test',
+            paint: {
+                'fill-pattern': 'hello'
+            }
+        }, {
+            id: 'test',
+            source: 'source',
+            'source-layer': 'test',
+            type: 'symbol',
+            layout: {
+                'icon-image': 'hello',
+                'text-font': ['StandardFont-Bold'],
+                'text-field': '{name}'
+            }
+        }]);
+
+        const data = {
+            layers: {
+                test: {
+                    version: 2,
+                    name: 'test',
+                    extent: 8192,
+                    length: 1,
+                    feature: (featureIndex: number) => ({
+                        extent: 8192,
+                        type: 1,
+                        id: featureIndex,
+                        properties: {
+                            name: 'test'
+                        },
+                        loadGeometry () {
+                            return [[{x: 0, y: 0}]];
+                        }
+                    })
+                }
+            }
+        } as any as VectorTile;
+
+        const sendAsync = jest.fn().mockImplementation((message: {type: string; data: any}) => {
+            const response = message.type === 'getImages' ?
+                {'hello': {width: 1, height: 1, data: new Uint8Array([0])}} :
+                {'StandardFont-Bold': {width: 1, height: 1, data: new Uint8Array([0])}};
+            return Promise.resolve(response);
+        });
+
+        const actorMock = {
+            sendAsync
+        };
+        const result = await tile.parse(data, layerIndex, ['hello'], actorMock);
+        expect(result).toBeDefined();
+        expect(sendAsync).toHaveBeenCalledTimes(3);
+        expect(sendAsync).toHaveBeenCalledWith(expect.objectContaining({data: expect.objectContaining({'icons': ['hello'], 'type': 'icons'})}), expect.any(Object));
+        expect(sendAsync).toHaveBeenCalledWith(expect.objectContaining({data: expect.objectContaining({'icons': ['hello'], 'type': 'patterns'})}), expect.any(Object));
+        expect(sendAsync).toHaveBeenCalledWith(expect.objectContaining({data: expect.objectContaining({'source': 'source', 'type': 'glyphs', 'stacks': {'StandardFont-Bold': [101, 115, 116]}})}), expect.any(Object));
+    });
+
+    test('WorkerTile#parse would cancel and only event once on repeated reparsing', async () => {
+        const tile = createWorkerTile();
+        const layerIndex = new StyleLayerIndex([{
+            id: '1',
+            type: 'fill',
+            source: 'source',
+            'source-layer': 'test',
+            paint: {
+                'fill-pattern': 'hello'
+            }
+        }, {
+            id: 'test',
+            source: 'source',
+            'source-layer': 'test',
+            type: 'symbol',
+            layout: {
+                'icon-image': 'hello',
+                'text-font': ['StandardFont-Bold'],
+                'text-field': '{name}'
+            }
+        }]);
+
+        const data = {
+            layers: {
+                test: {
+                    version: 2,
+                    name: 'test',
+                    extent: 8192,
+                    length: 1,
+                    feature: (featureIndex: number) => ({
+                        extent: 8192,
+                        type: 1,
+                        id: featureIndex,
+                        properties: {
+                            name: 'test'
+                        },
+                        loadGeometry () {
+                            return [[{x: 0, y: 0}]];
+                        }
+                    })
+                }
+            }
+        } as any as VectorTile;
+
+        let cancelCount = 0;
+        const sendAsync = jest.fn().mockImplementation((message: {type: string; data: unknown}, abortController: AbortController) => {
+            return new Promise((resolve, _reject) => {
+                const res = setTimeout(() => {
+                    const response = message.type === 'getImages' ?
+                        {'hello': {width: 1, height: 1, data: new Uint8Array([0])}} :
+                        {'StandardFont-Bold': {width: 1, height: 1, data: new Uint8Array([0])}};
+                    resolve(response);
+                }
+                );
+                abortController.signal.addEventListener('abort', () => {
+                    cancelCount += 1;
+                    clearTimeout(res);
+                });
+            });
+        });
+
+        const actorMock = {
+            sendAsync
+        };
+        tile.parse(data, layerIndex, ['hello'], actorMock).then(() => expect(false).toBeTruthy());
+        tile.parse(data, layerIndex, ['hello'], actorMock).then(() => expect(false).toBeTruthy());
+        const result = await tile.parse(data, layerIndex, ['hello'], actorMock);
+        expect(result).toBeDefined();
+        expect(cancelCount).toBe(6);
+        expect(sendAsync).toHaveBeenCalledTimes(9);
+        expect(sendAsync).toHaveBeenCalledWith(expect.objectContaining({data: expect.objectContaining({'icons': ['hello'], 'type': 'icons'})}), expect.any(Object));
+        expect(sendAsync).toHaveBeenCalledWith(expect.objectContaining({data: expect.objectContaining({'icons': ['hello'], 'type': 'patterns'})}), expect.any(Object));
+        expect(sendAsync).toHaveBeenCalledWith(expect.objectContaining({data: expect.objectContaining({'source': 'source', 'type': 'glyphs', 'stacks': {'StandardFont-Bold': [101, 115, 116]}})}), expect.any(Object));
+    });
+});