@woosh/meep-engine 2.42.8 → 2.43.0

package/engine/asset/AssetManager.js

@@ -16,21 +16,8 @@ import { noop } from "../../core/function/Functions.js";
 import { Deque } from "../../core/collection/queue/Deque.js";
 import Task from "../../core/process/task/Task.js";
 import { TaskSignal } from "../../core/process/task/TaskSignal.js";
-
-class AssetRequest {
-    /**
-     *
-     * @param {function(asset:Asset)} successCallback
-     * @param {function(error:*)} failureCallback
-     * @param {function(loaded:number, total:number):void} progressCallback
-     * @constructor
-     */
-    constructor(successCallback, failureCallback, progressCallback) {
-        this.successCallback = successCallback;
-        this.failureCallback = failureCallback;
-        this.pogressCallback = progressCallback;
-    }
-}
+import { AssetRequest } from "./AssetRequest.js";
+import FastBinaryHeap from "../../core/collection/heap/FastBinaryHeap.js";
 
 /**
  * @enum {number}
@@ -69,6 +56,32 @@ class PendingAsset {
          */
         this.state = AssetLoadState.Initial;
     }
+
+    /**
+     * Returns aggregated priority of the pending asset based on highest priority of associated requests
+     * @return {number}
+     */
+    get priority() {
+        let max_priority = 0;
+
+        const requests = this.requests;
+        const n = requests.length;
+
+        if (n > 0) {
+            max_priority = requests[0].priority;
+        }
+
+        for (let i = 1; i < n; i++) {
+            const request = requests[i];
+            const priority = request.priority;
+
+            if (priority > max_priority) {
+                max_priority = request;
+            }
+        }
+
+        return max_priority;
+    }
 }
 
 class Response {
@@ -83,8 +96,18 @@ class Response {
     }
 }
 
+/**
+ * Used by the priority queue, so the priority is inverted as the BinaryHeap returns elements in ascending score order (from lowest)
+ * @param {PendingAsset} pending_asset
+ * @returns {number}
+ */
+function get_pending_asset_priority_score(pending_asset) {
+    return -pending_asset.priority;
+}
+
 
 /**
+ * Handles loading/generating assets
  * @class
  */
 export class AssetManager {
@@ -97,17 +120,44 @@ export class AssetManager {
         /**
          *
          * @type {HashMap<AssetDescription, Asset>}
+         * @private
          */
-        this.assets = new HashMap();
+        this.assets = new HashMap({
+            capacity: 1024
+        });
 
         /**
          *
          * @type {ObservedMap<AssetDescription, PendingAsset>}
+         * @private
          */
-        this.requestMap = new ObservedMap(new HashMap());
+        this.request_map = new ObservedMap(new HashMap());
+
+        /**
+         * Waiting queue for assets that haven't yet been scheduled (passed to relevant loader)
+         * @type {BinaryHeap<PendingAsset>}
+         * @private
+         */
+        this.__pending_asset_wait_queue = new FastBinaryHeap(get_pending_asset_priority_score);
+        /**
+         * Assets currently being processed by loaders
+         * @type {Set<PendingAsset>}
+         * @private
+         */
+        this.__pending_asset_active_load_set = new Set();
+
+        /**
+         * Maximum number of requests that can be handled at the same time
+         * Since most of the requests are handled over the network, this can help you control network concurrency,
+         * this can be useful to keep some network slots available for high-priority requests such as UI related ones
+         * TODO currently nested asset requests are not recognized, so if nesting level exceeds concurrency - it will lead to a deadlock
+         * @type {number}
+         */
+        this.load_concurrency = Infinity;
 
         /**
          * Registered loaders
+         * @private
          * @type {Object<AssetLoader>}
          */
         this.loaders = {};
@@ -115,19 +165,23 @@ export class AssetManager {
         /**
          * After asset is loaded, a chain of transforms can be applied to it, these are registered here
          * Transformers are executed in the order in which they are added
+         * Identified by asset "type" string
          * @type {Object<AssetTransformer[]>}
+         * @private
          */
         this.transformers = {};
 
         /**
          * Named links to specific assets. Useful to later re-mapping assets and having meaningful names for them
          * @type {Map<string, AssetDescription>}
+         * @private
          */
         this.aliases = new Map();
 
         /**
          * This will be added to asset path for actual network resolution
          * @type {string}
+         * @private
          */
         this.rootPath = '';
 
@@ -147,16 +201,12 @@ export class AssetManager {
         /**
          *
          * @type {HashSet<AssetDescription>}
+         * @private
          */
         this.failures = new HashSet();
-        /**
-         * @readonly
-         * @type {boolean}
-         */
-        this.isAssetManager = true;
 
         /**
-         *
+         * Queue of responses that are waiting to be dispatched
          * @type {Deque<Response>}
          * @private
          */
@@ -182,6 +232,11 @@ export class AssetManager {
             }
         });
 
+        /**
+         *
+         * @type {boolean}
+         * @private
+         */
         this.__is_running = false;
     }
 
@@ -195,6 +250,11 @@ export class AssetManager {
         this.__is_running = true;
     }
 
+    /**
+     *
+     * @param {number} [immediate] should be let pending work finish, or abort it and shut down immediately? Defaults to false (wait)
+     * @return {Promise<void>}
+     */
     async shutdown(immediate = false) {
         if (!this.__is_running) {
             return;
@@ -268,12 +328,12 @@ export class AssetManager {
             //create request object
             const assetRequest = new AssetRequest(callback, failure, progress);
             //submit request
-            this.submitRequest(path, type, assetRequest);
+            this.submitRequest(assetDescription, assetRequest);
         }
     }
 
     /**
-     *
+     * Manually add a fully resolved resource
      * @param {string} path
      * @param {string} type
      * @param {Asset} asset
@@ -281,6 +341,12 @@ export class AssetManager {
     insert(path, type, asset) {
         const assetDescription = new AssetDescription(path, type);
 
+        const existing_resource = this.assets.get(assetDescription);
+
+        if (existing_resource !== undefined && existing_resource !== asset) {
+            console.warn(`Another asset under ${assetDescription} already exists and will be replaced`);
+        }
+
         this.assets.set(assetDescription, asset);
     }
 
@@ -310,45 +376,84 @@ export class AssetManager {
 
     /**
      *
-     * @param {string} path
-     * @param {string} type
-     * @param {AssetRequest} request
+     * @param {PendingAsset} asset
+     * @private
+     */
+    __schedule_load(asset) {
+        if (this.__pending_asset_active_load_set.size < this.load_concurrency) {
+            this.__dispatch_pending_asset(asset);
+        } else {
+            this.__pending_asset_wait_queue.push(asset);
+        }
+    }
+
+    /**
+     * Asset has been loaded successfully, failed, or aborted
+     * @param {PendingAsset} pending_asset
      * @private
      */
-    submitRequest(path, type, request) {
-        const requestMap = this.requestMap;
+    __handle_asset_resolved(pending_asset) {
+        // console.log(`Asset resolved ${pending_asset.description}`); // DEBUG
 
-        const assetDescription = new AssetDescription(path, type);
+        const active_set = this.__pending_asset_active_load_set;
 
-        let pendingAsset = requestMap.get(assetDescription);
-        if (pendingAsset !== undefined) {
-            //already loading
-            pendingAsset.requests.push(request);
-            return;
-        }
+        this.request_map.delete(pending_asset.description);
+        active_set.delete(pending_asset);
 
-        pendingAsset = new PendingAsset(assetDescription);
+        const queue = this.__pending_asset_wait_queue;
 
-        requestMap.set(assetDescription, pendingAsset);
+        // schedule more if possible
+        while (!queue.isEmpty() && active_set.size < this.load_concurrency) {
+            const load = queue.pop();
+
+            this.__dispatch_pending_asset(load);
+        }
+    }
+
+    /**
+     * Dispatch load request to relevant loader
+     * @param {PendingAsset} pendingAsset
+     * @private
+     */
+    __dispatch_pending_asset(pendingAsset) {
+        // console.log(`Asset load dispatched ${pendingAsset.description}`); // DEBUG
 
         const requests = pendingAsset.requests;
-        requests.push(request);
+        const assetDescription = pendingAsset.description;
+        const type = assetDescription.type;
+        const path = assetDescription.path;
 
         const loader = this.loaders[type];
 
-        if (loader === void 0) {
-            if (typeof request.failureCallback === "function") {
-                request.failureCallback(`no loader exists for asset type '${type}', valid types are: ${Object.keys(this.loaders).join(', ')}`);
-            } else {
-                //uncaught
-                console.error("Uncaught asset load failure: No loader for asset type", type);
+        if (loader === undefined) {
+            let reported_error = false;
+            // no loader
+            for (let i = 0; i < requests.length; i++) {
+                const assetRequest = requests[i];
+
+                if (typeof assetRequest.failureCallback === "function") {
+                    assetRequest.failureCallback(`no loader exists for asset type '${type}', valid types are: ${Object.keys(this.loaders).join(', ')}`);
+                } else {
+                    //uncaught
+                    if (!reported_error) {
+                        console.error("Uncaught asset load failure: No loader for asset type", type);
+                        reported_error = true;
+                    }
+                }
             }
+
+            this.__handle_asset_resolved(pendingAsset);
+
             return;
         }
 
+        // mark as being loaded
+        this.__pending_asset_active_load_set.add(pendingAsset);
+
         const assets = this.assets;
         const failures = this.failures;
 
+
         /**
          *
          * @param {Asset} loaded_asset
@@ -392,8 +497,9 @@ export class AssetManager {
             //register asset
             assets.set(assetDescription, asset);
 
-            //clear callbacks
-            requestMap.delete(assetDescription);
+            //clear callbacks etc.
+            this.__handle_asset_resolved(pendingAsset);
+
 
             // process callbacks
             for (let i = 0; i < requests.length; i++) {
@@ -403,7 +509,7 @@ export class AssetManager {
 
         }
 
-        function failure(error) {
+        const failure = (error) => {
             if (pendingAsset.state === AssetLoadState.Failed) {
                 console.warn(`Asset already failed, this is a redundant invocation. AD: ${assetDescription}`);
                 return;
@@ -424,8 +530,9 @@ export class AssetManager {
                     console.error("Failed to execute asset failure callback", e);
                 }
             }
-            //clear callbacks
-            requestMap.delete(assetDescription);
+
+            //clear callbacks etc.
+            this.__handle_asset_resolved(pendingAsset);
 
             // record failure
             failures.add(assetDescription);
@@ -463,11 +570,42 @@ export class AssetManager {
             }
 
         } catch (e) {
-            console.error(`Loader failed on invocation. path=${path}, type=${type}`, request, 'Loader exception: ', e);
+            console.error(`Loader failed on invocation. path=${path}, type=${type}`, 'Loader exception: ', e);
             failure(e);
         }
     }
 
+    /**
+     *
+     * @param {AssetDescription} assetDescription
+     * @param {AssetRequest} request
+     * @private
+     */
+    submitRequest(assetDescription, request) {
+        const requestMap = this.request_map;
+
+        let pendingAsset = requestMap.get(assetDescription);
+        if (pendingAsset !== undefined) {
+            //already loading
+            pendingAsset.requests.push(request);
+
+            // update priority queue if necessary
+            this.__pending_asset_wait_queue.updateElementScore(pendingAsset);
+
+            return;
+        }
+
+        // not loading yet, lets create a load container and schedule it
+        pendingAsset = new PendingAsset(assetDescription);
+
+        requestMap.set(assetDescription, pendingAsset);
+
+        const requests = pendingAsset.requests;
+        requests.push(request);
+
+        this.__schedule_load(pendingAsset);
+    }
+
     /**
      *
      * @param {string} type
@@ -666,7 +804,7 @@ export class AssetManager {
     /**
      *
      * @param {string} alias
-     * @return {AssetDescription}
+     * @return {AssetDescription|undefined}
      */
     resolveAlias(alias) {
 
@@ -692,3 +830,9 @@ export class AssetManager {
         this.aliases.set(alias, assetDescription);
     }
 }
+
+/**
+ * @readonly
+ * @type {boolean}
+ */
+AssetManager.prototype.isAssetManager = true;

package/engine/asset/AssetRequest.js

@@ -0,0 +1,32 @@
+export class AssetRequest {
+    /**
+     *
+     * @param {function(asset:Asset)} successCallback
+     * @param {function(error:*)} failureCallback
+     * @param {function(loaded:number, total:number):void} progressCallback
+     * @constructor
+     */
+    constructor(successCallback, failureCallback, progressCallback) {
+        /**
+         *
+         * @type {function(Asset)}
+         */
+        this.successCallback = successCallback;
+        /**
+         *
+         * @type {function(*)}
+         */
+        this.failureCallback = failureCallback;
+        /**
+         *
+         * @type {function(number, number): void}
+         */
+        this.pogressCallback = progressCallback;
+
+        /**
+         * Higher priority requests should be handled first
+         * @type {number}
+         */
+        this.priority = 0;
+    }
+}

package/engine/asset/loaders/ArrayBufferLoader.js

@@ -14,84 +14,96 @@ export class ArrayBufferLoader extends AssetLoader {
         });
 
         fetch(request)
-            .then(response => {
-                if (response.status === 200 || response.status === 0) {
+            .then(handle_response)
+            .then(response_to_asset)
+            .catch(failure);
 
-                    // Some browsers return HTTP Status 0 when using non-http protocol
-                    // e.g. 'file://' or 'data://'. Handle as success.
+        /**
+         *
+         * @param {Response} response
+         * @return {Promise<void>}
+         */
+        async function response_to_asset(response) {
+            const arrayBuffer = await response.arrayBuffer();
 
-                    if (response.status === 0) {
+            const asset = new Asset(
+                function () {
+                    return arrayBuffer;
+                },
+                arrayBuffer.byteSize
+            );
 
-                        console.warn('HTTP Status 0 received.');
+            success(asset);
+        }
 
-                    }
+        /**
+         *
+         * @param {Response} response
+         * @return {Response}
+         */
+        function handle_response(response) {
+            if (!(response.status === 200 || response.status === 0)) {
 
-                    if (typeof ReadableStream === 'undefined' || response.body.getReader === undefined) {
+                throw Error(`fetch for "${response.url}" responded with ${response.status}: ${response.statusText}`);
 
-                        return response;
+            }
 
-                    }
+            // Some browsers return HTTP Status 0 when using non-http protocol
+            // e.g. 'file://' or 'data://'. Handle as success.
 
-                    const reader = response.body.getReader();
-                    const contentLength = response.headers.get('Content-Length');
-                    const total = contentLength ? parseInt(contentLength) : 0;
-                    const lengthComputable = total !== 0;
-                    let loaded = 0;
+            if (response.status === 0) {
 
-                    // periodically read data into the new stream tracking while download progress
-                    const stream = new ReadableStream({
-                        start(controller) {
+                console.warn('HTTP Status 0 received.');
 
-                            readData();
+            }
 
-                            function readData() {
+            if (typeof ReadableStream === 'undefined' || response.body.getReader === undefined) {
 
-                                reader.read().then(({ done, value }) => {
+                return response;
 
-                                    if (done) {
+            }
 
-                                        controller.close();
+            const reader = response.body.getReader();
+            const contentLength = response.headers.get('Content-Length');
+            const total = contentLength ? parseInt(contentLength) : 0;
+            const lengthComputable = total !== 0;
+            let loaded = 0;
 
-                                    } else {
+            // periodically read data into the new stream tracking while download progress
+            const stream = new ReadableStream({
+                start(controller) {
 
-                                        loaded += value.byteLength;
+                    readData();
 
-                                        progress(loaded, length);
+                    function readData() {
 
-                                        controller.enqueue(value);
-                                        readData();
+                        reader.read().then(({ done, value }) => {
 
-                                    }
+                            if (done) {
 
-                                });
+                                controller.close();
 
-                            }
+                            } else {
+
+                                loaded += value.byteLength;
 
-                        }
+                                progress(loaded, length);
 
-                    });
+                                controller.enqueue(value);
+                                readData();
 
-                    return new Response(stream);
+                            }
 
-                } else {
+                        });
 
-                    throw Error(`fetch for "${response.url}" responded with ${response.status}: ${response.statusText}`);
+                    }
 
                 }
-            })
-            .then(async response => {
-                const arrayBuffer = await response.arrayBuffer();
-
-                const asset = new Asset(
-                    function () {
-                        return arrayBuffer;
-                    },
-                    arrayBuffer.byteSize
-                );
-
-                success(asset);
-            })
-            .catch(failure);
+
+            });
+
+            return new Response(stream);
+        }
 
 
     }

package/engine/asset/loaders/image/png/PNG.js

@@ -360,8 +360,22 @@ PNG.prototype.getUint8Data_case3 = function () {
 
     //transparency
     if (transparency_lookup !== null) {
+        const transparency_lookup_size = transparency_lookup.length;
+
         for (let i = 0; i < pixel_count; i++) {
-            result_data[i * 4 + 3] = transparency_lookup[pixels[i * d]];
+            const pixel_index = pixels[i * d];
+
+            const result_address = i * 4 + 3;
+
+            if (pixel_index >= transparency_lookup_size) {
+                /*
+                 when sampling outside of lookup, value defaults to 255
+                 @see "tRNS" chunk in PNG 1.2 spec
+                 */
+                result_data[result_address] = 255;
+            } else {
+                result_data[result_address] = transparency_lookup[pixel_index];
+            }
         }
     }
 

package/engine/asset/loaders/image/png/PNGReader.js

@@ -267,10 +267,11 @@ PNGReader.prototype.decodePixels = function () {
 
     const inflator = new zlib.Inflate();
 
-    const chunk_count = this.dataChunks.length;
+    const chunks = this.dataChunks;
+    const chunk_count = chunks.length;
 
     for (let i = 0; i < chunk_count; i++) {
-        inflator.push(this.dataChunks[i]);
+        inflator.push(chunks[i]);
 
         if (inflator.err) {
             throw new Error(inflator.err);

package/engine/ecs/foliage/ecs/InstancedMeshUtils.js

@@ -7,9 +7,10 @@ import { InstancedFoliage } from "../InstancedFoliage.js";
 import { buildTreeOptimizationTask } from "../../../../core/bvh2/BVHTasks.js";
 import Task from "../../../../core/process/task/Task.js";
 import { TaskSignal } from "../../../../core/process/task/TaskSignal.js";
-import { countTask, promiseTask } from "../../../../core/process/task/TaskUtils.js";
 import { assert } from "../../../../core/assert.js";
 import { InstancedMeshLayer } from "./InstancedMeshLayer.js";
+import { countTask } from "../../../../core/process/task/util/countTask.js";
+import { promiseTask } from "../../../../core/process/task/util/promiseTask.js";
 
 /**
  * Convert all existing instanced mesh components to individual Transform+Mesh pairs

package/engine/ecs/storage/BinaryBufferDeSerializer.js

@@ -1,9 +1,9 @@
-import { emptyTask } from "../../../core/process/task/TaskUtils.js";
 import Task from "../../../core/process/task/Task.js";
 import { TaskSignal } from "../../../core/process/task/TaskSignal.js";
 import { assert } from "../../../core/assert.js";
 import { BinaryCollectionDeSerializer } from "./binary/collection/BinaryCollectionDeSerializer.js";
 import { BinaryObjectSerializationAdapter } from "./binary/object/BinaryObjectSerializationAdapter.js";
+import { emptyTask } from "../../../core/process/task/util/emptyTask.js";
 
 class BinaryBufferDeSerializer {
     constructor() {

package/engine/ecs/storage/JSONDeSerializer.js

@@ -4,8 +4,9 @@
 
 
 import Blueprint from '../Blueprint.js';
-import { countTask, emptyTask } from '../../../core/process/task/TaskUtils.js';
 import { assert } from "../../../core/assert.js";
+import { countTask } from "../../../core/process/task/util/countTask.js";
+import { emptyTask } from "../../../core/process/task/util/emptyTask.js";
 
 /**
  *

package/engine/ecs/terrain/ecs/splat/SplatMapOptimizer.js

@@ -6,8 +6,9 @@ import { colorizeGraphGreedyWeight } from "../../../../../core/graph/coloring/co
 import { validateGraphColoring } from "../../../../../core/graph/coloring/validateGraphColoring.js";
 import ConcurrentExecutor from "../../../../../core/process/executor/ConcurrentExecutor.js";
 import Task from "../../../../../core/process/task/Task.js";
-import { actionTask, countTask } from "../../../../../core/process/task/TaskUtils.js";
 import { TaskSignal } from "../../../../../core/process/task/TaskSignal.js";
+import { actionTask } from "../../../../../core/process/task/util/actionTask.js";
+import { countTask } from "../../../../../core/process/task/util/countTask.js";
 
 /**
  * We convert splat mapping into a graph of connecting material patches and solve the overlaps as a "Graph Coloring" problem