@loaders.gl/core 5.0.0-alpha.5 → 5.0.0-alpha.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/dist/dist.dev.js +379 -41
  2. package/dist/dist.min.js +3 -3
  3. package/dist/index.cjs +385 -42
  4. package/dist/index.cjs.map +3 -3
  5. package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
  6. package/dist/lib/api/parse-in-batches.js +6 -2
  7. package/dist/lib/api/parse-in-batches.js.map +1 -1
  8. package/dist/lib/api/parse.js +2 -2
  9. package/dist/lib/api/parse.js.map +1 -1
  10. package/dist/lib/init.js +1 -1
  11. package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
  12. package/dist/lib/loader-utils/get-fetch-function.js +2 -1
  13. package/dist/lib/loader-utils/get-fetch-function.js.map +1 -1
  14. package/dist/lib/loader-utils/loader-context.d.ts +1 -1
  15. package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
  16. package/dist/lib/loader-utils/loader-context.js +6 -3
  17. package/dist/lib/loader-utils/loader-context.js.map +1 -1
  18. package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
  19. package/dist/lib/loader-utils/option-defaults.js +1 -0
  20. package/dist/lib/loader-utils/option-defaults.js.map +1 -1
  21. package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
  22. package/dist/lib/loader-utils/option-utils.js +12 -1
  23. package/dist/lib/loader-utils/option-utils.js.map +1 -1
  24. package/dist/null-loader.js +1 -1
  25. package/dist/null-worker-node.js +29 -3
  26. package/dist/null-worker.js +29 -3
  27. package/package.json +6 -6
  28. package/src/lib/api/parse-in-batches.ts +11 -2
  29. package/src/lib/api/parse.ts +2 -2
  30. package/src/lib/loader-utils/get-fetch-function.ts +2 -1
  31. package/src/lib/loader-utils/loader-context.ts +6 -3
  32. package/src/lib/loader-utils/option-defaults.ts +1 -0
  33. package/src/lib/loader-utils/option-utils.ts +13 -1
package/dist/index.cjs CHANGED
@@ -401,6 +401,10 @@ var WorkerThread = class {
401
401
  name;
402
402
  source;
403
403
  url;
404
+ /** Lazily resolves the classic worker URL used as a fallback. */
405
+ getUrl;
406
+ /** Creates a browser Worker directly. */
407
+ loadWorker;
404
408
  terminated = false;
405
409
  worker;
406
410
  onMessage;
@@ -411,11 +415,13 @@ var WorkerThread = class {
411
415
  return typeof Worker !== "undefined" && isBrowser2 || typeof NodeWorker !== "undefined" && !isBrowser2;
412
416
  }
413
417
  constructor(props) {
414
- const { name, source, url } = props;
415
- assert2(source || url);
418
+ const { name, source, url, getUrl, loadWorker } = props;
419
+ assert2(source || url || getUrl || loadWorker);
416
420
  this.name = name;
417
421
  this.source = source;
418
422
  this.url = url;
423
+ this.getUrl = getUrl;
424
+ this.loadWorker = loadWorker;
419
425
  this.onMessage = NOOP;
420
426
  this.onError = (error) => console.log(error);
421
427
  this.worker = isBrowser2 ? this._createBrowserWorker() : this._createNodeWorker();
@@ -461,7 +467,7 @@ var WorkerThread = class {
461
467
  */
462
468
  _getErrorFromErrorEvent(event) {
463
469
  let message = "Failed to load ";
464
- message += `worker ${this.name} from ${this.url}. `;
470
+ message += `worker ${this.name} from ${this.url || "built-in worker factory"}. `;
465
471
  if (event.message) {
466
472
  message += `${event.message} in `;
467
473
  }
@@ -474,8 +480,26 @@ var WorkerThread = class {
474
480
  * Creates a worker thread on the browser
475
481
  */
476
482
  _createBrowserWorker() {
477
- this._loadableURL = getLoadableWorkerURL({ source: this.source, url: this.url });
483
+ var _a;
484
+ if (this.loadWorker) {
485
+ try {
486
+ const worker2 = this.loadWorker();
487
+ if (worker2) {
488
+ return this._initializeBrowserWorker(worker2);
489
+ }
490
+ } catch (error) {
491
+ if (!this.source && !this.url && !this.getUrl) {
492
+ throw error;
493
+ }
494
+ }
495
+ }
496
+ const url = this.url || ((_a = this.getUrl) == null ? void 0 : _a.call(this));
497
+ this._loadableURL = getLoadableWorkerURL({ source: this.source, url });
478
498
  const worker = new Worker(this._loadableURL, { name: this.name });
499
+ return this._initializeBrowserWorker(worker);
500
+ }
501
+ /** Attaches protocol handlers to a browser worker. */
502
+ _initializeBrowserWorker(worker) {
479
503
  worker.onmessage = (event) => {
480
504
  if (!event.data) {
481
505
  this.onError(new Error("No data received"));
@@ -525,6 +549,10 @@ var WorkerPool = class {
525
549
  source;
526
550
  // | Function;
527
551
  url;
552
+ /** Lazily resolves a classic worker fallback URL. */
553
+ getUrl;
554
+ /** Creates a browser Worker directly. */
555
+ loadWorker;
528
556
  maxConcurrency = 1;
529
557
  maxMobileConcurrency = 1;
530
558
  onDebug = () => {
@@ -546,6 +574,8 @@ var WorkerPool = class {
546
574
  constructor(props) {
547
575
  this.source = props.source;
548
576
  this.url = props.url;
577
+ this.getUrl = props.getUrl;
578
+ this.loadWorker = props.loadWorker;
549
579
  this.setProps(props);
550
580
  }
551
581
  /**
@@ -655,7 +685,13 @@ var WorkerPool = class {
655
685
  if (this.count < this._getMaxConcurrency()) {
656
686
  this.count++;
657
687
  const name = `${this.name.toLowerCase()} (#${this.count} of ${this.maxConcurrency})`;
658
- return new WorkerThread({ name, source: this.source, url: this.url });
688
+ return new WorkerThread({
689
+ name,
690
+ source: this.source,
691
+ url: this.url,
692
+ getUrl: this.getUrl,
693
+ loadWorker: this.loadWorker
694
+ });
659
695
  }
660
696
  return null;
661
697
  }
@@ -672,6 +708,8 @@ var DEFAULT_PROPS = {
672
708
  onDebug: () => {
673
709
  }
674
710
  };
711
+ var loadWorkerIds = /* @__PURE__ */ new WeakMap();
712
+ var nextLoadWorkerId = 1;
675
713
  var WorkerFarm = class _WorkerFarm {
676
714
  props;
677
715
  workerPools = /* @__PURE__ */ new Map();
@@ -715,24 +753,28 @@ var WorkerFarm = class _WorkerFarm {
715
753
  }
716
754
  /**
717
755
  * Returns a worker pool for the specified worker
718
- * @param options - only used first time for a specific worker name
719
- * @param options.name - the name of the worker - used to identify worker pool
720
- * @param options.url -
721
- * @param options.source -
756
+ * @param options Worker target used to identify and initialize the pool.
757
+ * @param options.name Worker name used as the base pool identity.
758
+ * @param options.url Worker script URL.
759
+ * @param options.source Inline worker source.
760
+ * @param options.loadWorker Built-in browser worker factory.
722
761
  * @example
723
762
  * const job = WorkerFarm.getWorkerFarm().getWorkerPool({name, url}).startJob(...);
724
763
  */
725
764
  getWorkerPool(options) {
726
- const { name, source, url } = options;
727
- let workerPool = this.workerPools.get(name);
765
+ const { name, source, url, getUrl, loadWorker } = options;
766
+ const workerPoolKey = getWorkerPoolKey(options);
767
+ let workerPool = this.workerPools.get(workerPoolKey);
728
768
  if (!workerPool) {
729
769
  workerPool = new WorkerPool({
730
770
  name,
731
771
  source,
732
- url
772
+ url,
773
+ getUrl,
774
+ loadWorker
733
775
  });
734
776
  workerPool.setProps(this._getWorkerPoolProps());
735
- this.workerPools.set(name, workerPool);
777
+ this.workerPools.set(workerPoolKey, workerPool);
736
778
  }
737
779
  return workerPool;
738
780
  }
@@ -745,6 +787,91 @@ var WorkerFarm = class _WorkerFarm {
745
787
  };
746
788
  }
747
789
  };
790
+ function getWorkerPoolKey(options) {
791
+ const { name, source, url, urlKey, loadWorker } = options;
792
+ const sourceKey = source ? `source:${source}` : "";
793
+ const urlKeyPart = url || urlKey ? `url:${url || urlKey}` : "";
794
+ const loadWorkerKey = loadWorker ? `loadWorker:${getLoadWorkerId(loadWorker)}` : "";
795
+ return [name, sourceKey, urlKeyPart, loadWorkerKey].filter(Boolean).join("|");
796
+ }
797
+ function getLoadWorkerId(loadWorker) {
798
+ let loadWorkerId = loadWorkerIds.get(loadWorker);
799
+ if (!loadWorkerId) {
800
+ loadWorkerId = nextLoadWorkerId++;
801
+ loadWorkerIds.set(loadWorker, loadWorkerId);
802
+ }
803
+ return loadWorkerId;
804
+ }
805
+
806
+ // ../worker-utils/src/lib/async-queue/async-queue.ts
807
+ var AsyncQueue = class {
808
+ _values;
809
+ _settlers;
810
+ _closed;
811
+ constructor() {
812
+ this._values = [];
813
+ this._settlers = [];
814
+ this._closed = false;
815
+ }
816
+ /** Return an async iterator for this queue */
817
+ [Symbol.asyncIterator]() {
818
+ return this;
819
+ }
820
+ /** Push a new value - the async iterator will yield a promise resolved to this value */
821
+ push(value) {
822
+ return this.enqueue(value);
823
+ }
824
+ /**
825
+ * Push a new value - the async iterator will yield a promise resolved to this value
826
+ * Add an error - the async iterator will yield a promise rejected with this value
827
+ */
828
+ enqueue(value) {
829
+ if (this._closed) {
830
+ throw new Error("Closed");
831
+ }
832
+ if (this._settlers.length > 0) {
833
+ if (this._values.length > 0) {
834
+ throw new Error("Illegal internal state");
835
+ }
836
+ const settler = this._settlers.shift();
837
+ if (value instanceof Error) {
838
+ settler.reject(value);
839
+ } else {
840
+ settler.resolve({ value });
841
+ }
842
+ } else {
843
+ this._values.push(value);
844
+ }
845
+ }
846
+ /** Indicate that we not waiting for more values - The async iterator will be done */
847
+ close() {
848
+ while (this._settlers.length > 0) {
849
+ const settler = this._settlers.shift();
850
+ settler.resolve({ done: true });
851
+ }
852
+ this._closed = true;
853
+ }
854
+ // ITERATOR IMPLEMENTATION
855
+ /** @returns a Promise for an IteratorResult */
856
+ next() {
857
+ if (this._values.length > 0) {
858
+ const value = this._values.shift();
859
+ if (value instanceof Error) {
860
+ return Promise.reject(value);
861
+ }
862
+ return Promise.resolve({ done: false, value });
863
+ }
864
+ if (this._closed) {
865
+ if (this._settlers.length > 0) {
866
+ throw new Error("Illegal internal state");
867
+ }
868
+ return Promise.resolve({ done: true, value: void 0 });
869
+ }
870
+ return new Promise((resolve2, reject) => {
871
+ this._settlers.push({ resolve: resolve2, reject });
872
+ });
873
+ }
874
+ };
748
875
 
749
876
  // ../worker-utils/src/lib/worker-api/get-worker-url.ts
750
877
  var warnedWorkerVersionFallbacks = /* @__PURE__ */ new Set();
@@ -752,35 +879,42 @@ function getWorkerName(worker) {
752
879
  const warning = worker.version !== VERSION2 ? ` (worker-utils@${VERSION2})` : "";
753
880
  return `${worker.name}@${worker.version}${warning}`;
754
881
  }
755
- function getWorkerURL(worker, options = {}) {
882
+ function getCustomWorkerURL(worker, options = {}) {
756
883
  var _a;
757
884
  const workerOptions = options[worker.id] || {};
758
- const workerFile = isBrowser2 ? `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
885
+ const workerFile = isBrowser2 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
759
886
  let url = workerOptions.workerUrl;
760
- if (!url && typeof worker.worker === "string") {
761
- url = worker.worker;
762
- }
763
887
  if (!url && worker.id === "compression") {
764
888
  url = options.workerUrl;
765
889
  }
766
890
  const workerType = options._workerType || ((_a = options == null ? void 0 : options.core) == null ? void 0 : _a._workerType);
767
- if (workerType === "test") {
891
+ if (!url && workerType === "test") {
768
892
  if (isBrowser2) {
769
893
  url = `modules/${worker.module}/dist/${workerFile}`;
770
894
  } else {
771
895
  url = `modules/${worker.module}/src/workers/${worker.id}-worker-node.ts`;
772
896
  }
773
897
  }
774
- if (!url) {
775
- let version2 = worker.version;
776
- if (version2 === "latest") {
777
- version2 = NPM_TAG;
778
- }
779
- const versionTag = version2 ? `@${version2}` : "";
780
- url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
898
+ return url || null;
899
+ }
900
+ function getWorkerURL(worker, options = {}) {
901
+ const customWorkerUrl = getCustomWorkerURL(worker, options);
902
+ const descriptorWorkerUrl = typeof worker.worker === "string" ? worker.worker : null;
903
+ const url = customWorkerUrl || descriptorWorkerUrl || getDefaultWorkerURL(worker, true);
904
+ assert2(url);
905
+ return url;
906
+ }
907
+ function getDefaultWorkerURL(worker, warn = false) {
908
+ const workerFile = isBrowser2 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
909
+ let version2 = worker.version;
910
+ if (version2 === "latest") {
911
+ version2 = NPM_TAG;
912
+ }
913
+ const versionTag = version2 ? `@${version2}` : "";
914
+ const url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
915
+ if (warn) {
781
916
  warnIfUsingNpmTagFallback(worker, url);
782
917
  }
783
- assert2(url);
784
918
  return url;
785
919
  }
786
920
  function warnIfUsingNpmTagFallback(worker, url) {
@@ -803,19 +937,16 @@ function canProcessOnWorker(worker, options) {
803
937
  return false;
804
938
  }
805
939
  const workerOptions = options == null ? void 0 : options[worker.id];
806
- return Boolean((worker.worker || (workerOptions == null ? void 0 : workerOptions.workerUrl)) && (options == null ? void 0 : options.worker));
940
+ return Boolean(
941
+ (worker.worker || worker.loadWorker || (workerOptions == null ? void 0 : workerOptions.workerUrl)) && (options == null ? void 0 : options.worker)
942
+ );
807
943
  }
808
944
  async function processOnWorker(worker, data, options = {}, context = {}, jobContext = {}) {
809
945
  var _a, _b, _c;
810
946
  throwIfAborted(options.signal);
811
947
  const name = getWorkerName(worker);
812
948
  const workerFarm = WorkerFarm.getWorkerFarm(options);
813
- const { source } = options;
814
- const workerPoolProps = { name, source };
815
- if (!source) {
816
- workerPoolProps.url = getWorkerURL(worker, options);
817
- }
818
- const workerPool = workerFarm.getWorkerPool(workerPoolProps);
949
+ const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
819
950
  const jobName = options.jobName || worker.name;
820
951
  const job = await workerPool.startJob(
821
952
  jobName,
@@ -844,6 +975,115 @@ async function processOnWorker(worker, data, options = {}, context = {}, jobCont
844
975
  (_c = options.signal) == null ? void 0 : _c.removeEventListener("abort", abortJob);
845
976
  }
846
977
  }
978
+ function processOnWorkerInBatches(worker, input, options = {}, context = {}, jobContext = {}) {
979
+ return processOnWorkerInBatchesIterator(worker, input, options, context, jobContext);
980
+ }
981
+ async function* processOnWorkerInBatchesIterator(worker, input, options, context, jobContext) {
982
+ var _a, _b, _c;
983
+ throwIfAborted(options.signal);
984
+ const name = getWorkerName(worker);
985
+ const workerFarm = WorkerFarm.getWorkerFarm(options);
986
+ const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
987
+ const outputBatches = new AsyncQueue();
988
+ const inputIterator = getAsyncIterator(input);
989
+ let outputFinished = false;
990
+ let inputFinished = false;
991
+ let inputRequest = Promise.resolve();
992
+ const batchJobFailed = new Error("Worker batch job failed");
993
+ const finishOutput = () => {
994
+ if (!outputFinished) {
995
+ outputFinished = true;
996
+ outputBatches.close();
997
+ }
998
+ };
999
+ const failOutput = (_error) => {
1000
+ if (!outputFinished) {
1001
+ outputFinished = true;
1002
+ outputBatches.enqueue(batchJobFailed);
1003
+ outputBatches.close();
1004
+ }
1005
+ };
1006
+ const job = await workerPool.startJob(
1007
+ options.jobName || worker.name,
1008
+ (activeJob, type, payload) => {
1009
+ switch (type) {
1010
+ case "input-request":
1011
+ inputRequest = inputRequest.then(async () => {
1012
+ const nextBatch = await inputIterator.next();
1013
+ if (!activeJob.isRunning) {
1014
+ return;
1015
+ }
1016
+ if (nextBatch.done) {
1017
+ inputFinished = true;
1018
+ activeJob.postMessage("input-done", {});
1019
+ } else {
1020
+ activeJob.postMessage("input-batch", { input: nextBatch.value });
1021
+ }
1022
+ }).catch((error) => activeJob.abort(error));
1023
+ break;
1024
+ case "output-batch":
1025
+ outputBatches.push(payload.result);
1026
+ break;
1027
+ case "done":
1028
+ activeJob.done(payload);
1029
+ finishOutput();
1030
+ break;
1031
+ case "error": {
1032
+ const error = new Error(payload.error || "Worker batch processing failed");
1033
+ activeJob.error(error);
1034
+ failOutput(error);
1035
+ break;
1036
+ }
1037
+ case "process":
1038
+ void onMessage(context, activeJob, type, payload);
1039
+ break;
1040
+ default: {
1041
+ const error = new Error(`Unexpected worker batch message: ${type}`);
1042
+ activeJob.error(error);
1043
+ failOutput(error);
1044
+ break;
1045
+ }
1046
+ }
1047
+ }
1048
+ );
1049
+ const abortJob = () => job.abort(getAbortReason(options.signal));
1050
+ (_a = options.signal) == null ? void 0 : _a.addEventListener("abort", abortJob, { once: true });
1051
+ if ((_b = options.signal) == null ? void 0 : _b.aborted) {
1052
+ abortJob();
1053
+ }
1054
+ void job.result.catch(failOutput);
1055
+ try {
1056
+ const { signal: _signal, ...workerOptions } = options;
1057
+ if (job.isRunning) {
1058
+ job.postMessage("process-in-batches", {
1059
+ options: getTransferListForWriter(workerOptions),
1060
+ context: getTransferListForWriter(jobContext)
1061
+ });
1062
+ }
1063
+ try {
1064
+ for await (const outputBatch of outputBatches) {
1065
+ yield outputBatch;
1066
+ if (job.isRunning) {
1067
+ job.postMessage("output-ack", {});
1068
+ }
1069
+ }
1070
+ } catch (error) {
1071
+ if (error === batchJobFailed) {
1072
+ await job.result;
1073
+ }
1074
+ throw error;
1075
+ }
1076
+ await job.result;
1077
+ } finally {
1078
+ (_c = options.signal) == null ? void 0 : _c.removeEventListener("abort", abortJob);
1079
+ if (job.isRunning) {
1080
+ job.abort(createAbortError2("Worker batch iterator was closed"));
1081
+ }
1082
+ if (!inputFinished) {
1083
+ closeInputIterator(inputIterator);
1084
+ }
1085
+ }
1086
+ }
847
1087
  function throwIfAborted(signal) {
848
1088
  if (!(signal == null ? void 0 : signal.aborted)) {
849
1089
  return;
@@ -858,6 +1098,39 @@ function createAbortError2(message) {
858
1098
  error.name = "AbortError";
859
1099
  return error;
860
1100
  }
1101
+ function getAsyncIterator(input) {
1102
+ if (Symbol.asyncIterator in input) {
1103
+ return input[Symbol.asyncIterator]();
1104
+ }
1105
+ return (async function* iterateSynchronously() {
1106
+ yield* input;
1107
+ })();
1108
+ }
1109
+ function closeInputIterator(inputIterator) {
1110
+ var _a;
1111
+ try {
1112
+ void Promise.resolve((_a = inputIterator.return) == null ? void 0 : _a.call(inputIterator)).catch(() => void 0);
1113
+ } catch {
1114
+ }
1115
+ }
1116
+ function getWorkerPoolTarget(worker, options, name) {
1117
+ if (options.source) {
1118
+ return { name, source: options.source };
1119
+ }
1120
+ const customWorkerUrl = getCustomWorkerURL(worker, options);
1121
+ if (customWorkerUrl) {
1122
+ return { name, url: customWorkerUrl };
1123
+ }
1124
+ if (isBrowser2 && worker.loadWorker) {
1125
+ return {
1126
+ name,
1127
+ loadWorker: worker.loadWorker,
1128
+ getUrl: () => getWorkerURL(worker, options),
1129
+ urlKey: getDefaultWorkerURL(worker)
1130
+ };
1131
+ }
1132
+ return { name, url: getWorkerURL(worker, options) };
1133
+ }
861
1134
  async function onMessage(context, job, type, payload) {
862
1135
  switch (type) {
863
1136
  case "done":
@@ -911,12 +1184,30 @@ function canParseWithWorker(loader, options) {
911
1184
  }
912
1185
  return Boolean(canProcessOnWorker(loader, workerOptions));
913
1186
  }
1187
+ function shouldParseWithWorker(loader, data, options, context, estimateLoader = loader) {
1188
+ var _a, _b;
1189
+ if (!canParseWithWorker(loader, options)) {
1190
+ return false;
1191
+ }
1192
+ if (((_a = options == null ? void 0 : options.core) == null ? void 0 : _a.worker) !== "auto" || !estimateLoader.getWorkerEstimate) {
1193
+ return true;
1194
+ }
1195
+ try {
1196
+ const estimate = estimateLoader.getWorkerEstimate(data, options, context);
1197
+ if (estimate === void 0 || !Number.isFinite(estimate) || estimate < 0 || estimate > 1) {
1198
+ return true;
1199
+ }
1200
+ return estimate >= (((_b = options.core) == null ? void 0 : _b.workerThreshold) ?? 0.1);
1201
+ } catch {
1202
+ return true;
1203
+ }
1204
+ }
914
1205
  async function parseWithWorker(loader, data, options, context, parseOnMainThread, signal) {
915
1206
  const workerSignal = signal || getWorkerAbortSignal(options);
916
1207
  const result = await processOnWorker(
917
1208
  loader,
918
1209
  data,
919
- { ...getWorkerOptions(options), signal: workerSignal },
1210
+ { ...getWorkerOptions(options, loader), signal: workerSignal },
920
1211
  {
921
1212
  process: async (input, processOptions, _workerContext, parseContext) => {
922
1213
  if (!parseOnMainThread) {
@@ -935,6 +1226,32 @@ async function parseWithWorker(loader, data, options, context, parseOnMainThread
935
1226
  );
936
1227
  return isLoaderWithWorkerResultDeserializer(loader) ? loader.deserializeWorkerResult(result, options, context) : result;
937
1228
  }
1229
+ async function* parseWithWorkerInBatches(loader, inputIterator, options, context, parseOnMainThread, signal) {
1230
+ const workerSignal = signal || getWorkerAbortSignal(options);
1231
+ const outputIterator = processOnWorkerInBatches(
1232
+ loader,
1233
+ inputIterator,
1234
+ { ...getWorkerOptions(options), signal: workerSignal },
1235
+ {
1236
+ process: async (input, processOptions, _workerContext, parseContext) => {
1237
+ if (!parseOnMainThread) {
1238
+ throw new Error("Worker not set up to parse on main thread");
1239
+ }
1240
+ const mainThreadContext = context ? { ...context, ...parseContext || {} } : void 0;
1241
+ return await callParseOnMainThread(
1242
+ parseOnMainThread,
1243
+ input,
1244
+ processOptions,
1245
+ mainThreadContext
1246
+ );
1247
+ }
1248
+ },
1249
+ getSerializableLoaderContext(context)
1250
+ );
1251
+ for await (const batch of outputIterator) {
1252
+ yield loader.deserializeWorkerBatch ? loader.deserializeWorkerBatch(batch, options, context) : batch;
1253
+ }
1254
+ }
938
1255
  function getWorkerAbortSignal(options) {
939
1256
  var _a;
940
1257
  const parquetSignal = (_a = options == null ? void 0 : options.parquet) == null ? void 0 : _a.signal;
@@ -949,12 +1266,19 @@ function callParseOnMainThread(parseOnMainThread, input, options, context) {
949
1266
  }
950
1267
  return parseOnMainThread(input, void 0, options, context);
951
1268
  }
952
- function getWorkerOptions(options = {}) {
1269
+ function getWorkerOptions(options = {}, loader) {
953
1270
  const serializedOptions = JSON.parse(JSON.stringify(options));
954
- return {
1271
+ const workerOptions = {
955
1272
  ...serializedOptions.core,
956
1273
  ...serializedOptions
957
1274
  };
1275
+ if (workerOptions.worker === "auto") {
1276
+ workerOptions.worker = true;
1277
+ }
1278
+ if (loader) {
1279
+ workerOptions._workerLoaderId = loader.id;
1280
+ }
1281
+ return workerOptions;
958
1282
  }
959
1283
  function getSerializableLoaderContext(context) {
960
1284
  if (!context) {
@@ -2029,6 +2353,7 @@ var DEFAULT_LOADER_OPTIONS = {
2029
2353
  CDN: "https://unpkg.com/@loaders.gl",
2030
2354
  worker: true,
2031
2355
  // By default, use worker if provided by loader.
2356
+ workerThreshold: 0.1,
2032
2357
  maxConcurrency: 3,
2033
2358
  // How many worker instances should be created for each loader.
2034
2359
  maxMobileConcurrency: 1,
@@ -2104,6 +2429,7 @@ var CORE_LOADER_OPTION_KEYS = [
2104
2429
  "useLocalLibraries",
2105
2430
  "CDN",
2106
2431
  "worker",
2432
+ "workerThreshold",
2107
2433
  "maxConcurrency",
2108
2434
  "maxMobileConcurrency",
2109
2435
  "reuseWorkers",
@@ -2144,7 +2470,10 @@ function normalizeOptions(options, loader, loaders, url) {
2144
2470
  loaders = loaders || [];
2145
2471
  loaders = Array.isArray(loaders) ? loaders : [loaders];
2146
2472
  validateOptions(options, loaders);
2147
- return normalizeLoaderOptions(normalizeOptionsInternal(loader, options, url));
2473
+ validateWorkerThreshold(options);
2474
+ const normalizedOptions = normalizeLoaderOptions(normalizeOptionsInternal(loader, options, url));
2475
+ validateWorkerThreshold(normalizedOptions);
2476
+ return normalizedOptions;
2148
2477
  }
2149
2478
  function normalizeLoaderOptions(options) {
2150
2479
  const normalized = cloneLoaderOptions(options);
@@ -2168,6 +2497,13 @@ function validateOptions(options, loaders) {
2168
2497
  validateOptionsObject(idOptions, loader.id, loaderOptions, deprecatedOptions, loaders);
2169
2498
  }
2170
2499
  }
2500
+ function validateWorkerThreshold(options) {
2501
+ var _a;
2502
+ const threshold = (_a = options.core) == null ? void 0 : _a.workerThreshold;
2503
+ if (threshold !== void 0 && (!Number.isFinite(threshold) || threshold < 0 || threshold > 1)) {
2504
+ throw new Error("core.workerThreshold must be a finite number between 0 and 1");
2505
+ }
2506
+ }
2171
2507
  function validateOptionsObject(options, id, defaultOptions, deprecatedOptions, loaders) {
2172
2508
  const loaderName = id || "Top level";
2173
2509
  const prefix = id ? `${id}.` : "";
@@ -2872,7 +3208,8 @@ function getFetchFunction(options, context) {
2872
3208
  if (typeof fetchOption === "function") {
2873
3209
  fetchFunction = fetchOption;
2874
3210
  } else if (isObject(fetchOption)) {
2875
- fetchFunction = (url, requestOptions) => fetchFile(url, mergeFetchOptions(fetchOption, requestOptions));
3211
+ const baseFetch = (context == null ? void 0 : context.fetch) || fetchFile;
3212
+ fetchFunction = (url, requestOptions) => baseFetch(url, mergeFetchOptions(fetchOption, requestOptions));
2876
3213
  } else if (context == null ? void 0 : context.fetch) {
2877
3214
  fetchFunction = context.fetch;
2878
3215
  } else {
@@ -3167,6 +3504,9 @@ async function parseToOutputIterator(loader, data, options, context) {
3167
3504
  const inputIterator = await getAsyncIterableFromData(data, options);
3168
3505
  const transformedIterator = await applyInputTransforms(inputIterator, ((_a = options == null ? void 0 : options.core) == null ? void 0 : _a.transforms) || []);
3169
3506
  if (loader.parseInBatches) {
3507
+ if (loader.id === "csv" && isBrowser && canParseWithWorker(loader, options)) {
3508
+ return parseWithWorkerInBatches(loader, transformedIterator, options, context, parse);
3509
+ }
3170
3510
  return loader.parseInBatches(transformedIterator, options, context);
3171
3511
  }
3172
3512
  return parseChunkInBatches(transformedIterator, loader, options, context);
@@ -3337,7 +3677,10 @@ var coreApi = Object.freeze({
3337
3677
  // dist/lib/loader-utils/loader-context.js
3338
3678
  function getLoaderContext(context, options, parentContext) {
3339
3679
  if (parentContext) {
3340
- return parentContext;
3680
+ return {
3681
+ ...parentContext,
3682
+ fetch: getFetchFunction(options, parentContext)
3683
+ };
3341
3684
  }
3342
3685
  const newContext = {
3343
3686
  ...context,
@@ -3408,7 +3751,7 @@ async function parseWithLoader(loader, data, options, context) {
3408
3751
  context.response = { headers, ok, redirected, status, statusText, type, url };
3409
3752
  }
3410
3753
  const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
3411
- if (canParseWithWorker(loaderWithParser, options)) {
3754
+ if (shouldParseWithWorker(loaderWithParser, data, options, context, loader)) {
3412
3755
  data = await getArrayBufferFromData(data, options);
3413
3756
  return await parseWithWorker(loaderWithParser, data, options, context, parse);
3414
3757
  }
@@ -3651,7 +3994,7 @@ function makeStream(source, options) {
3651
3994
  }
3652
3995
 
3653
3996
  // dist/null-loader.js
3654
- var VERSION4 = true ? "5.0.0-alpha.5" : "latest";
3997
+ var VERSION4 = true ? "5.0.0-alpha.6" : "latest";
3655
3998
  var NullWorkerLoader = {
3656
3999
  dataType: null,
3657
4000
  batchType: null,