@loaders.gl/loader-utils 3.1.0-alpha.3 → 3.1.0-beta.2

package/dist/lib/request-utils/request-scheduler.js

@@ -1,146 +1,142 @@
-import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-import { Stats } from '@probe.gl/stats';
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const stats_1 = require("@probe.gl/stats");
 const STAT_QUEUED_REQUESTS = 'Queued Requests';
 const STAT_ACTIVE_REQUESTS = 'Active Requests';
 const STAT_CANCELLED_REQUESTS = 'Cancelled Requests';
 const STAT_QUEUED_REQUESTS_EVER = 'Queued Requests Ever';
 const STAT_ACTIVE_REQUESTS_EVER = 'Active Requests Ever';
 const DEFAULT_PROPS = {
-  id: 'request-scheduler',
-  throttleRequests: true,
-  maxRequests: 6
+    id: 'request-scheduler',
+    // Specifies if the request scheduler should throttle incoming requests, mainly for comparative testing
+    throttleRequests: true,
+    // The maximum number of simultaneous active requests. Un-throttled requests do not observe this limit.
+    maxRequests: 6
 };
-export default class RequestScheduler {
-  constructor(props = {}) {
-    _defineProperty(this, "props", void 0);
-
-    _defineProperty(this, "stats", void 0);
-
-    _defineProperty(this, "activeRequestCount", 0);
-
-    _defineProperty(this, "requestQueue", []);
-
-    _defineProperty(this, "requestMap", new Map());
-
-    _defineProperty(this, "deferredUpdate", null);
-
-    this.props = { ...DEFAULT_PROPS,
-      ...props
-    };
-    this.stats = new Stats({
-      id: this.props.id
-    });
-    this.stats.get(STAT_QUEUED_REQUESTS);
-    this.stats.get(STAT_ACTIVE_REQUESTS);
-    this.stats.get(STAT_CANCELLED_REQUESTS);
-    this.stats.get(STAT_QUEUED_REQUESTS_EVER);
-    this.stats.get(STAT_ACTIVE_REQUESTS_EVER);
-  }
-
-  scheduleRequest(handle, getPriority = () => 0) {
-    if (!this.props.throttleRequests) {
-      return Promise.resolve({
-        done: () => {}
-      });
+/**
+ * Used to issue a request, without having them "deeply queued" by the browser.
+ * @todo - Track requests globally, across multiple servers
+ */
+class RequestScheduler {
+    constructor(props = {}) {
+        this.activeRequestCount = 0;
+        /** Tracks the number of active requests and prioritizes/cancels queued requests. */
+        this.requestQueue = [];
+        this.requestMap = new Map();
+        this.deferredUpdate = null;
+        this.props = { ...DEFAULT_PROPS, ...props };
+        // Returns the statistics used by the request scheduler.
+        this.stats = new stats_1.Stats({ id: this.props.id });
+        this.stats.get(STAT_QUEUED_REQUESTS);
+        this.stats.get(STAT_ACTIVE_REQUESTS);
+        this.stats.get(STAT_CANCELLED_REQUESTS);
+        this.stats.get(STAT_QUEUED_REQUESTS_EVER);
+        this.stats.get(STAT_ACTIVE_REQUESTS_EVER);
     }
-
-    if (this.requestMap.has(handle)) {
-      return this.requestMap.get(handle);
-    }
-
-    const request = {
-      handle,
-      priority: 0,
-      getPriority
-    };
-    const promise = new Promise(resolve => {
-      request.resolve = resolve;
-      return request;
-    });
-    this.requestQueue.push(request);
-    this.requestMap.set(handle, promise);
-
-    this._issueNewRequests();
-
-    return promise;
-  }
-
-  _issueRequest(request) {
-    const {
-      handle,
-      resolve
-    } = request;
-    let isDone = false;
-
-    const done = () => {
-      if (!isDone) {
-        isDone = true;
-        this.requestMap.delete(handle);
-        this.activeRequestCount--;
-
+    /**
+     * Called by an application that wants to issue a request, without having it deeply queued by the browser
+     *
+     * When the returned promise resolved, it is OK for the application to issue a request.
+     * The promise resolves to an object that contains a `done` method.
+     * When the application's request has completed (or failed), the application must call the `done` function
+     *
+     * @param handle
+     * @param getPriority will be called when request "slots" open up,
+     *    allowing the caller to update priority or cancel the request
+     *    Highest priority executes first, priority < 0 cancels the request
+     * @returns a promise
+     *   - resolves to a object (with a `done` field) when the request can be issued without queueing,
+     *   - resolves to `null` if the request has been cancelled (by the callback return < 0).
+     *     In this case the application should not issue the request
+     */
+    scheduleRequest(handle, getPriority = () => 0) {
+        // Allows throttling to be disabled
+        if (!this.props.throttleRequests) {
+            return Promise.resolve({ done: () => { } });
+        }
+        // dedupe
+        if (this.requestMap.has(handle)) {
+            return this.requestMap.get(handle);
+        }
+        const request = { handle, priority: 0, getPriority };
+        const promise = new Promise((resolve) => {
+            // @ts-ignore
+            request.resolve = resolve;
+            return request;
+        });
+        this.requestQueue.push(request);
+        this.requestMap.set(handle, promise);
         this._issueNewRequests();
-      }
-    };
-
-    this.activeRequestCount++;
-    return resolve ? resolve({
-      done
-    }) : Promise.resolve({
-      done
-    });
-  }
-
-  _issueNewRequests() {
-    if (!this.deferredUpdate) {
-      this.deferredUpdate = setTimeout(() => this._issueNewRequestsAsync(), 0);
+        return promise;
+    }
+    // PRIVATE
+    _issueRequest(request) {
+        const { handle, resolve } = request;
+        let isDone = false;
+        const done = () => {
+            // can only be called once
+            if (!isDone) {
+                isDone = true;
+                // Stop tracking a request - it has completed, failed, cancelled etc
+                this.requestMap.delete(handle);
+                this.activeRequestCount--;
+                // A slot just freed up, see if any queued requests are waiting
+                this._issueNewRequests();
+            }
+        };
+        // Track this request
+        this.activeRequestCount++;
+        return resolve ? resolve({ done }) : Promise.resolve({ done });
     }
-  }
-
-  _issueNewRequestsAsync() {
-    this.deferredUpdate = null;
-    const freeSlots = Math.max(this.props.maxRequests - this.activeRequestCount, 0);
-
-    if (freeSlots === 0) {
-      return;
+    /** We check requests asynchronously, to prevent multiple updates */
+    _issueNewRequests() {
+        if (!this.deferredUpdate) {
+            this.deferredUpdate = setTimeout(() => this._issueNewRequestsAsync(), 0);
+        }
     }
-
-    this._updateAllRequests();
-
-    for (let i = 0; i < freeSlots; ++i) {
-      const request = this.requestQueue.shift();
-
-      if (request) {
-        this._issueRequest(request);
-      }
+    /** Refresh all requests  */
+    _issueNewRequestsAsync() {
+        // TODO - shouldn't we clear the timeout?
+        this.deferredUpdate = null;
+        const freeSlots = Math.max(this.props.maxRequests - this.activeRequestCount, 0);
+        if (freeSlots === 0) {
+            return;
+        }
+        this._updateAllRequests();
+        // Resolve pending promises for the top-priority requests
+        for (let i = 0; i < freeSlots; ++i) {
+            const request = this.requestQueue.shift();
+            if (request) {
+                this._issueRequest(request); // eslint-disable-line @typescript-eslint/no-floating-promises
+            }
+        }
+        // Uncomment to debug
+        // console.log(`${freeSlots} free slots, ${this.requestQueue.length} queued requests`);
     }
-  }
-
-  _updateAllRequests() {
-    const requestQueue = this.requestQueue;
-
-    for (let i = 0; i < requestQueue.length; ++i) {
-      const request = requestQueue[i];
-
-      if (!this._updateRequest(request)) {
-        requestQueue.splice(i, 1);
-        this.requestMap.delete(request.handle);
-        i--;
-      }
+    /** Ensure all requests have updated priorities, and that no longer valid requests are cancelled */
+    _updateAllRequests() {
+        const requestQueue = this.requestQueue;
+        for (let i = 0; i < requestQueue.length; ++i) {
+            const request = requestQueue[i];
+            if (!this._updateRequest(request)) {
+                // Remove the element and make sure to adjust the counter to account for shortened array
+                requestQueue.splice(i, 1);
+                this.requestMap.delete(request.handle);
+                i--;
+            }
+        }
+        // Sort the remaining requests based on priority
+        requestQueue.sort((a, b) => a.priority - b.priority);
     }
-
-    requestQueue.sort((a, b) => a.priority - b.priority);
-  }
-
-  _updateRequest(request) {
-    request.priority = request.getPriority(request.handle);
-
-    if (request.priority < 0) {
-      request.resolve(null);
-      return false;
+    /** Update a single request by calling the callback */
+    _updateRequest(request) {
+        request.priority = request.getPriority(request.handle); // eslint-disable-line callback-return
+        // by returning a negative priority, the callback cancels the request
+        if (request.priority < 0) {
+            request.resolve(null);
+            return false;
+        }
+        return true;
     }
-
-    return true;
-  }
-
 }
-//# sourceMappingURL=request-scheduler.js.map
+exports.default = RequestScheduler;

package/dist/lib/worker-loader-utils/create-loader-worker.d.ts

@@ -0,0 +1,7 @@
+import type { LoaderWithParser } from '../../types';
+/**
+ * Set up a WebWorkerGlobalScope to talk with the main thread
+ * @param loader
+ */
+export declare function createLoaderWorker(loader: LoaderWithParser): void;
+//# sourceMappingURL=create-loader-worker.d.ts.map

package/dist/lib/worker-loader-utils/create-loader-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"create-loader-worker.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-loader-utils/create-loader-worker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,aAAa,CAAC;AAMlD;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,gBAAgB,QA+B1D"}

package/dist/lib/worker-loader-utils/create-loader-worker.js

@@ -1,102 +1,97 @@
-import { WorkerBody } from '@loaders.gl/worker-utils';
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createLoaderWorker = void 0;
+const worker_utils_1 = require("@loaders.gl/worker-utils");
+// import {validateLoaderVersion} from './validate-loader-version.js';
 let requestId = 0;
-export function createLoaderWorker(loader) {
-  if (typeof self === 'undefined') {
-    return;
-  }
-
-  WorkerBody.onmessage = async (type, payload) => {
-    switch (type) {
-      case 'process':
-        try {
-          const {
-            input,
-            options = {}
-          } = payload;
-          const result = await parseData({
-            loader,
-            arrayBuffer: input,
-            options,
-            context: {
-              parse: parseOnMainThread
-            }
-          });
-          WorkerBody.postMessage('done', {
-            result
-          });
-        } catch (error) {
-          const message = error instanceof Error ? error.message : '';
-          WorkerBody.postMessage('error', {
-            error: message
-          });
-        }
-
-        break;
-
-      default:
+/**
+ * Set up a WebWorkerGlobalScope to talk with the main thread
+ * @param loader
+ */
+function createLoaderWorker(loader) {
+    // Check that we are actually in a worker thread
+    if (typeof self === 'undefined') {
+        return;
     }
-  };
+    worker_utils_1.WorkerBody.onmessage = async (type, payload) => {
+        switch (type) {
+            case 'process':
+                try {
+                    // validateLoaderVersion(loader, data.source.split('@')[1]);
+                    const { input, options = {} } = payload;
+                    const result = await parseData({
+                        loader,
+                        arrayBuffer: input,
+                        options,
+                        context: {
+                            parse: parseOnMainThread
+                        }
+                    });
+                    worker_utils_1.WorkerBody.postMessage('done', { result });
+                }
+                catch (error) {
+                    const message = error instanceof Error ? error.message : '';
+                    worker_utils_1.WorkerBody.postMessage('error', { error: message });
+                }
+                break;
+            default:
+        }
+    };
 }
-
+exports.createLoaderWorker = createLoaderWorker;
 function parseOnMainThread(arrayBuffer, options) {
-  return new Promise((resolve, reject) => {
-    const id = requestId++;
-
-    const onMessage = (type, payload) => {
-      if (payload.id !== id) {
-        return;
-      }
-
-      switch (type) {
-        case 'done':
-          WorkerBody.removeEventListener(onMessage);
-          resolve(payload.result);
-          break;
-
-        case 'error':
-          WorkerBody.removeEventListener(onMessage);
-          reject(payload.error);
-          break;
-
-        default:
-      }
-    };
-
-    WorkerBody.addEventListener(onMessage);
-    const payload = {
-      id,
-      input: arrayBuffer,
-      options
-    };
-    WorkerBody.postMessage('process', payload);
-  });
+    return new Promise((resolve, reject) => {
+        const id = requestId++;
+        /**
+         */
+        const onMessage = (type, payload) => {
+            if (payload.id !== id) {
+                // not ours
+                return;
+            }
+            switch (type) {
+                case 'done':
+                    worker_utils_1.WorkerBody.removeEventListener(onMessage);
+                    resolve(payload.result);
+                    break;
+                case 'error':
+                    worker_utils_1.WorkerBody.removeEventListener(onMessage);
+                    reject(payload.error);
+                    break;
+                default:
+                // ignore
+            }
+        };
+        worker_utils_1.WorkerBody.addEventListener(onMessage);
+        // Ask the main thread to decode data
+        const payload = { id, input: arrayBuffer, options };
+        worker_utils_1.WorkerBody.postMessage('process', payload);
+    });
 }
-
-async function parseData({
-  loader,
-  arrayBuffer,
-  options,
-  context
-}) {
-  let data;
-  let parser;
-
-  if (loader.parseSync || loader.parse) {
-    data = arrayBuffer;
-    parser = loader.parseSync || loader.parse;
-  } else if (loader.parseTextSync) {
-    const textDecoder = new TextDecoder();
-    data = textDecoder.decode(arrayBuffer);
-    parser = loader.parseTextSync;
-  } else {
-    throw new Error(`Could not load data with ${loader.name} loader`);
-  }
-
-  options = { ...options,
-    modules: loader && loader.options && loader.options.modules || {},
-    worker: false
-  };
-  return await parser(data, { ...options
-  }, context, loader);
+// TODO - Support byteOffset and byteLength (enabling parsing of embedded binaries without copies)
+// TODO - Why not support async loader.parse* funcs here?
+// TODO - Why not reuse a common function instead of reimplementing loader.parse* selection logic? Keeping loader small?
+// TODO - Lack of appropriate parser functions can be detected when we create worker, no need to wait until parse
+async function parseData({ loader, arrayBuffer, options, context }) {
+    let data;
+    let parser;
+    if (loader.parseSync || loader.parse) {
+        data = arrayBuffer;
+        parser = loader.parseSync || loader.parse;
+    }
+    else if (loader.parseTextSync) {
+        const textDecoder = new TextDecoder();
+        data = textDecoder.decode(arrayBuffer);
+        parser = loader.parseTextSync;
+    }
+    else {
+        throw new Error(`Could not load data with ${loader.name} loader`);
+    }
+    // TODO - proper merge in of loader options...
+    options = {
+        ...options,
+        modules: (loader && loader.options && loader.options.modules) || {},
+        worker: false
+    };
+    return await parser(data, { ...options }, context, loader);
 }
-//# sourceMappingURL=create-loader-worker.js.map

package/dist/lib/worker-loader-utils/parse-with-worker.d.ts

@@ -0,0 +1,15 @@
+import type { Loader, LoaderOptions, LoaderContext } from '../../types';
+/**
+ * Determines if a loader can parse with worker
+ * @param loader
+ * @param options
+ */
+export declare function canParseWithWorker(loader: Loader, options?: LoaderOptions): boolean | "" | undefined;
+/**
+ * this function expects that the worker function sends certain messages,
+ * this can be automated if the worker is wrapper by a call to createLoaderWorker in @loaders.gl/loader-utils.
+ */
+export declare function parseWithWorker(loader: Loader, data: any, options?: LoaderOptions, context?: LoaderContext, parseOnMainThread?: (arrayBuffer: ArrayBuffer, options: {
+    [key: string]: any;
+}) => Promise<void>): Promise<any>;
+//# sourceMappingURL=parse-with-worker.d.ts.map

package/dist/lib/worker-loader-utils/parse-with-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-with-worker.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-loader-utils/parse-with-worker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,MAAM,EAAE,aAAa,EAAE,aAAa,EAAC,MAAM,aAAa,CAAC;AAGtE;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,4BAMzE;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,GAAG,EACT,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,aAAa,EACvB,iBAAiB,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,EAAE;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,gBA2B/F"}

package/dist/lib/worker-loader-utils/parse-with-worker.js

@@ -1,64 +1,73 @@
-import { WorkerFarm, getWorkerURL } from '@loaders.gl/worker-utils';
-export function canParseWithWorker(loader, options) {
-  if (!WorkerFarm.isSupported()) {
-    return false;
-  }
-
-  return loader.worker && (options === null || options === void 0 ? void 0 : options.worker);
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseWithWorker = exports.canParseWithWorker = void 0;
+const worker_utils_1 = require("@loaders.gl/worker-utils");
+/**
+ * Determines if a loader can parse with worker
+ * @param loader
+ * @param options
+ */
+function canParseWithWorker(loader, options) {
+    if (!worker_utils_1.WorkerFarm.isSupported()) {
+        return false;
+    }
+    return loader.worker && options?.worker;
 }
-export async function parseWithWorker(loader, data, options, context, parseOnMainThread) {
-  const name = loader.id;
-  const url = getWorkerURL(loader, options);
-  const workerFarm = WorkerFarm.getWorkerFarm(options);
-  const workerPool = workerFarm.getWorkerPool({
-    name,
-    url
-  });
-  options = JSON.parse(JSON.stringify(options));
-  const job = await workerPool.startJob('process-on-worker', onMessage.bind(null, parseOnMainThread));
-  job.postMessage('process', {
-    input: data,
-    options
-  });
-  const result = await job.result;
-  return await result.result;
+exports.canParseWithWorker = canParseWithWorker;
+/**
+ * this function expects that the worker function sends certain messages,
+ * this can be automated if the worker is wrapper by a call to createLoaderWorker in @loaders.gl/loader-utils.
+ */
+async function parseWithWorker(loader, data, options, context, parseOnMainThread) {
+    const name = loader.id; // TODO
+    const url = (0, worker_utils_1.getWorkerURL)(loader, options);
+    const workerFarm = worker_utils_1.WorkerFarm.getWorkerFarm(options);
+    const workerPool = workerFarm.getWorkerPool({ name, url });
+    // options.log object contains functions which cannot be transferred
+    // TODO - decide how to handle logging on workers
+    options = JSON.parse(JSON.stringify(options));
+    const job = await workerPool.startJob('process-on-worker', 
+    // @ts-expect-error
+    onMessage.bind(null, parseOnMainThread) // eslint-disable-line @typescript-eslint/no-misused-promises
+    );
+    job.postMessage('process', {
+        // @ts-ignore
+        input: data,
+        options
+    });
+    const result = await job.result;
+    // TODO - what is going on here?
+    return await result.result;
 }
-
+exports.parseWithWorker = parseWithWorker;
+/**
+ * Handle worker's responses to the main thread
+ * @param job
+ * @param type
+ * @param payload
+ */
 async function onMessage(parseOnMainThread, job, type, payload) {
-  switch (type) {
-    case 'done':
-      job.done(payload);
-      break;
-
-    case 'error':
-      job.error(payload.error);
-      break;
-
-    case 'process':
-      const {
-        id,
-        input,
-        options
-      } = payload;
-
-      try {
-        const result = await parseOnMainThread(input, options);
-        job.postMessage('done', {
-          id,
-          result
-        });
-      } catch (error) {
-        const message = error instanceof Error ? error.message : 'unknown error';
-        job.postMessage('error', {
-          id,
-          error: message
-        });
-      }
-
-      break;
-
-    default:
-      console.warn(`parse-with-worker unknown message ${type}`);
-  }
+    switch (type) {
+        case 'done':
+            job.done(payload);
+            break;
+        case 'error':
+            job.error(new Error(payload.error));
+            break;
+        case 'process':
+            // Worker is asking for main thread to parseO
+            const { id, input, options } = payload;
+            try {
+                const result = await parseOnMainThread(input, options);
+                job.postMessage('done', { id, result });
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : 'unknown error';
+                job.postMessage('error', { id, error: message });
+            }
+            break;
+        default:
+            // eslint-disable-next-line
+            console.warn(`parse-with-worker unknown message ${type}`);
+    }
 }
-//# sourceMappingURL=parse-with-worker.js.map