@loaders.gl/core 5.0.0-alpha.13 → 5.0.0-alpha.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/dist.dev.js +190 -63
  2. package/dist/dist.min.js +3 -3
  3. package/dist/index.cjs +205 -72
  4. package/dist/index.cjs.map +2 -2
  5. package/dist/lib/api/parse-file.js +2 -0
  6. package/dist/lib/api/parse-file.js.map +1 -1
  7. package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
  8. package/dist/lib/api/parse-in-batches.js +5 -4
  9. package/dist/lib/api/parse-in-batches.js.map +1 -1
  10. package/dist/lib/api/parse.js +4 -2
  11. package/dist/lib/api/parse.js.map +1 -1
  12. package/dist/lib/init.js +1 -1
  13. package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
  14. package/dist/lib/loader-utils/get-fetch-function.js +3 -1
  15. package/dist/lib/loader-utils/get-fetch-function.js.map +1 -1
  16. package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
  17. package/dist/lib/loader-utils/loader-context.js +4 -0
  18. package/dist/lib/loader-utils/loader-context.js.map +1 -1
  19. package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
  20. package/dist/lib/loader-utils/option-defaults.js +1 -0
  21. package/dist/lib/loader-utils/option-defaults.js.map +1 -1
  22. package/dist/lib/loader-utils/option-utils.js +1 -1
  23. package/dist/lib/loader-utils/option-utils.js.map +1 -1
  24. package/dist/loader-utils/index.cjs +231 -92
  25. package/dist/loader-utils/index.cjs.map +2 -2
  26. package/dist/loader-utils/index.d.ts +1 -1
  27. package/dist/loader-utils/index.d.ts.map +1 -1
  28. package/dist/loader-utils/index.js +1 -1
  29. package/dist/loader-utils/index.js.map +1 -1
  30. package/dist/loader-utils/json-loader.d.ts +1 -1
  31. package/dist/loader-utils/json-loader.js +2 -2
  32. package/dist/loader-utils/lib/log-utils/log.js +1 -1
  33. package/dist/loader-utils/lib/request-utils/request-credentials.d.ts +3 -1
  34. package/dist/loader-utils/lib/request-utils/request-credentials.d.ts.map +1 -1
  35. package/dist/loader-utils/lib/request-utils/request-credentials.js +10 -16
  36. package/dist/loader-utils/lib/request-utils/request-credentials.js.map +1 -1
  37. package/dist/loader-utils/lib/worker-loader-utils/create-loader-worker.d.ts +3 -0
  38. package/dist/loader-utils/lib/worker-loader-utils/create-loader-worker.d.ts.map +1 -1
  39. package/dist/loader-utils/lib/worker-loader-utils/create-loader-worker.js +13 -12
  40. package/dist/loader-utils/lib/worker-loader-utils/create-loader-worker.js.map +1 -1
  41. package/dist/loader-utils/lib/worker-loader-utils/parse-with-worker.d.ts +6 -0
  42. package/dist/loader-utils/lib/worker-loader-utils/parse-with-worker.d.ts.map +1 -1
  43. package/dist/loader-utils/lib/worker-loader-utils/parse-with-worker.js +38 -7
  44. package/dist/loader-utils/lib/worker-loader-utils/parse-with-worker.js.map +1 -1
  45. package/dist/loader-utils/loader-types.d.ts +12 -4
  46. package/dist/loader-utils/loader-types.d.ts.map +1 -1
  47. package/dist/loader-utils/loader-types.js.map +1 -1
  48. package/dist/null-loader.js +1 -1
  49. package/dist/null-worker-node.js +60 -15
  50. package/dist/null-worker.js +59 -15
  51. package/dist/schema-utils/lib/table/tables/convert-table.d.ts +1 -1
  52. package/dist/schema-utils/lib/table/tables/convert-table.js +1 -1
  53. package/dist/worker-utils/index.cjs +170 -49
  54. package/dist/worker-utils/index.cjs.map +2 -2
  55. package/dist/worker-utils/index.d.ts +2 -2
  56. package/dist/worker-utils/index.d.ts.map +1 -1
  57. package/dist/worker-utils/index.js +1 -1
  58. package/dist/worker-utils/index.js.map +1 -1
  59. package/dist/worker-utils/lib/env-utils/version.js +2 -2
  60. package/dist/worker-utils/lib/worker-api/create-worker.d.ts.map +1 -1
  61. package/dist/worker-utils/lib/worker-api/create-worker.js +40 -2
  62. package/dist/worker-utils/lib/worker-api/create-worker.js.map +1 -1
  63. package/dist/worker-utils/lib/worker-api/get-worker-url.d.ts +2 -0
  64. package/dist/worker-utils/lib/worker-api/get-worker-url.d.ts.map +1 -1
  65. package/dist/worker-utils/lib/worker-api/get-worker-url.js +7 -11
  66. package/dist/worker-utils/lib/worker-api/get-worker-url.js.map +1 -1
  67. package/dist/worker-utils/lib/worker-api/process-on-worker.d.ts +6 -0
  68. package/dist/worker-utils/lib/worker-api/process-on-worker.d.ts.map +1 -1
  69. package/dist/worker-utils/lib/worker-api/process-on-worker.js +64 -27
  70. package/dist/worker-utils/lib/worker-api/process-on-worker.js.map +1 -1
  71. package/dist/worker-utils/lib/worker-farm/worker-job.d.ts.map +1 -1
  72. package/dist/worker-utils/lib/worker-farm/worker-job.js +3 -2
  73. package/dist/worker-utils/lib/worker-farm/worker-job.js.map +1 -1
  74. package/dist/worker-utils/lib/worker-farm/worker-pool.d.ts +1 -1
  75. package/dist/worker-utils/lib/worker-farm/worker-pool.d.ts.map +1 -1
  76. package/dist/worker-utils/lib/worker-farm/worker-pool.js +65 -12
  77. package/dist/worker-utils/lib/worker-farm/worker-pool.js.map +1 -1
  78. package/dist/worker-utils/null-worker-node.js +36 -2
  79. package/dist/worker-utils/null-worker-node.js.map +2 -2
  80. package/dist/worker-utils/null-worker.js +36 -2
  81. package/dist/worker-utils/null-worker.js.map +2 -2
  82. package/dist/worker-utils/types.d.ts +25 -4
  83. package/dist/worker-utils/types.d.ts.map +1 -1
  84. package/package.json +4 -4
  85. package/src/lib/api/parse-file.ts +2 -0
  86. package/src/lib/api/parse-in-batches.ts +5 -5
  87. package/src/lib/api/parse.ts +4 -2
  88. package/src/lib/loader-utils/get-fetch-function.ts +4 -2
  89. package/src/lib/loader-utils/loader-context.ts +4 -0
  90. package/src/lib/loader-utils/option-defaults.ts +1 -0
  91. package/src/lib/loader-utils/option-utils.ts +1 -1
  92. package/src/loader-utils/index.ts +1 -0
  93. package/src/loader-utils/json-loader.ts +1 -1
  94. package/src/loader-utils/lib/request-utils/request-credentials.ts +10 -12
  95. package/src/loader-utils/lib/worker-loader-utils/create-loader-worker.ts +38 -15
  96. package/src/loader-utils/lib/worker-loader-utils/parse-with-worker.ts +52 -7
  97. package/src/loader-utils/loader-types.ts +16 -4
  98. package/src/schema-utils/lib/table/tables/convert-table.ts +1 -1
  99. package/src/worker-utils/index.ts +3 -1
  100. package/src/worker-utils/lib/worker-api/create-worker.ts +41 -2
  101. package/src/worker-utils/lib/worker-api/get-worker-url.ts +13 -14
  102. package/src/worker-utils/lib/worker-api/process-on-worker.ts +128 -73
  103. package/src/worker-utils/lib/worker-farm/worker-job.ts +4 -2
  104. package/src/worker-utils/lib/worker-farm/worker-pool.ts +65 -12
  105. package/src/worker-utils/types.ts +26 -3
package/dist/index.cjs CHANGED
@@ -243,6 +243,7 @@ var WorkerJob = class {
243
243
  * @param data any data structure, ideally consisting mostly of transferrable objects
244
244
  */
245
245
  postMessage(type, payload) {
246
+ if (!this.isRunning) return;
246
247
  this.workerThread.postMessage({
247
248
  source: "loaders.gl",
248
249
  // Lets worker ignore unrelated messages
@@ -278,9 +279,10 @@ var WorkerJob = class {
278
279
  if (!this.isRunning) {
279
280
  return;
280
281
  }
281
- const error = reason ?? createAbortError(`Worker job "${this.name}" was aborted`);
282
282
  this.workerThread.destroy();
283
- this.error(error);
283
+ this.error(
284
+ arguments.length ? reason : createAbortError(`Worker job "${this.name}" was aborted`)
285
+ );
284
286
  }
285
287
  };
286
288
  function createAbortError(message) {
@@ -585,9 +587,13 @@ var WorkerPool = class {
585
587
  * @note Can free up significant memory
586
588
  */
587
589
  destroy(reason = new Error("Worker pool was destroyed")) {
590
+ this.isDestroyed = true;
588
591
  this.activeJobs.forEach((job) => job.abort(reason));
589
592
  const queuedJobs = this.jobQueue.splice(0);
590
- queuedJobs.forEach((queuedJob) => queuedJob.onReject(reason));
593
+ queuedJobs.forEach((queuedJob) => {
594
+ queuedJob.removeAbortListener();
595
+ queuedJob.onReject(reason);
596
+ });
591
597
  this.idleQueue.forEach((worker) => worker.destroy());
592
598
  this.isDestroyed = true;
593
599
  }
@@ -609,12 +615,33 @@ var WorkerPool = class {
609
615
  this.onDebug = props.onDebug;
610
616
  }
611
617
  }
612
- async startJob(name, onMessage2 = (job, type, data) => job.done(data), onError = (job, error) => job.error(error)) {
618
+ async startJob(name, onMessage2 = (job, type, data) => job.done(data), onError = (job, error) => job.error(error), signal) {
619
+ if (signal == null ? void 0 : signal.aborted) {
620
+ throw signal.reason;
621
+ }
613
622
  if (this.isDestroyed) {
614
623
  return Promise.reject(new Error("Worker pool was destroyed"));
615
624
  }
616
625
  const startPromise = new Promise((onStart, onReject) => {
617
- this.jobQueue.push({ name, onMessage: onMessage2, onError, onStart, onReject });
626
+ const abortQueuedJob = () => {
627
+ const index = this.jobQueue.indexOf(queuedJob);
628
+ if (index >= 0) {
629
+ this.jobQueue.splice(index, 1);
630
+ queuedJob.removeAbortListener();
631
+ onReject(signal == null ? void 0 : signal.reason);
632
+ }
633
+ };
634
+ const queuedJob = {
635
+ name,
636
+ onMessage: onMessage2,
637
+ onError,
638
+ onStart,
639
+ onReject,
640
+ signal,
641
+ removeAbortListener: () => signal == null ? void 0 : signal.removeEventListener("abort", abortQueuedJob)
642
+ };
643
+ this.jobQueue.push(queuedJob);
644
+ signal == null ? void 0 : signal.addEventListener("abort", abortQueuedJob, { once: true });
618
645
  return this;
619
646
  });
620
647
  this._startQueuedJob();
@@ -626,10 +653,20 @@ var WorkerPool = class {
626
653
  * Called when job is started and whenever a worker returns to the idleQueue
627
654
  */
628
655
  async _startQueuedJob() {
656
+ var _a, _b, _c;
629
657
  if (!this.jobQueue.length) {
630
658
  return;
631
659
  }
632
- const workerThread = this._getAvailableWorker();
660
+ let workerThread;
661
+ try {
662
+ workerThread = this._getAvailableWorker();
663
+ } catch (error) {
664
+ const queuedJob2 = this.jobQueue.shift();
665
+ queuedJob2 == null ? void 0 : queuedJob2.removeAbortListener();
666
+ queuedJob2 == null ? void 0 : queuedJob2.onReject(error);
667
+ void this._startQueuedJob();
668
+ return;
669
+ }
633
670
  if (!workerThread) {
634
671
  return;
635
672
  }
@@ -642,18 +679,30 @@ var WorkerPool = class {
642
679
  backlog: this.jobQueue.length
643
680
  });
644
681
  const job = new WorkerJob(queuedJob.name, workerThread);
682
+ queuedJob.removeAbortListener();
683
+ const abortJob = () => {
684
+ var _a2;
685
+ return job.abort((_a2 = queuedJob.signal) == null ? void 0 : _a2.reason);
686
+ };
687
+ (_a = queuedJob.signal) == null ? void 0 : _a.addEventListener("abort", abortJob, { once: true });
688
+ if ((_b = queuedJob.signal) == null ? void 0 : _b.aborted) abortJob();
645
689
  this.activeJobs.add(job);
646
690
  workerThread.ref();
647
- workerThread.onMessage = (data) => queuedJob.onMessage(job, data.type, data.payload);
691
+ workerThread.onMessage = (data) => {
692
+ if (job.isRunning) queuedJob.onMessage(job, data.type, data.payload);
693
+ };
648
694
  workerThread.onError = (error) => queuedJob.onError(job, error);
649
695
  queuedJob.onStart(job);
650
696
  try {
651
697
  await job.result;
652
698
  } catch {
653
699
  } finally {
700
+ (_c = queuedJob.signal) == null ? void 0 : _c.removeEventListener("abort", abortJob);
654
701
  this.activeJobs.delete(job);
655
702
  this.returnWorkerToQueue(workerThread);
656
703
  }
704
+ } else {
705
+ this.returnWorkerToQueue(workerThread);
657
706
  }
658
707
  }
659
708
  /**
@@ -695,13 +744,18 @@ var WorkerPool = class {
695
744
  if (this.count < this._getMaxConcurrency()) {
696
745
  this.count++;
697
746
  const name = `${this.name.toLowerCase()} (#${this.count} of ${this.maxConcurrency})`;
698
- return new WorkerThread({
699
- name,
700
- source: this.source,
701
- url: this.url,
702
- getUrl: this.getUrl,
703
- loadWorker: this.loadWorker
704
- });
747
+ try {
748
+ return new WorkerThread({
749
+ name,
750
+ source: this.source,
751
+ url: this.url,
752
+ getUrl: this.getUrl,
753
+ loadWorker: this.loadWorker
754
+ });
755
+ } catch (error) {
756
+ this.count--;
757
+ throw error;
758
+ }
705
759
  }
706
760
  return null;
707
761
  }
@@ -892,12 +946,8 @@ function getWorkerName(worker) {
892
946
  }
893
947
  function getCustomWorkerURL(worker, options = {}) {
894
948
  var _a, _b;
895
- const workerOptions = options[worker.id] || {};
896
949
  const workerFile = isBrowser2 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
897
- let url = workerOptions.workerUrl;
898
- if (!url && worker.id === "compression") {
899
- url = options.workerUrl;
900
- }
950
+ let url = getConfiguredWorkerURL(worker, options);
901
951
  const workerType = options._workerType || ((_a = options == null ? void 0 : options.core) == null ? void 0 : _a._workerType);
902
952
  if (!url && workerType === "test") {
903
953
  if (isBrowser2) {
@@ -910,6 +960,10 @@ function getCustomWorkerURL(worker, options = {}) {
910
960
  }
911
961
  return url || null;
912
962
  }
963
+ function getConfiguredWorkerURL(worker, options = {}) {
964
+ const workerOptions = options[worker.id] || {};
965
+ return workerOptions.workerUrl || (worker.id === "compression" ? options.workerUrl : null) || null;
966
+ }
913
967
  function getWorkerURL(worker, options = {}) {
914
968
  const customWorkerUrl = getCustomWorkerURL(worker, options);
915
969
  const descriptorWorkerUrl = typeof worker.worker === "string" ? worker.worker : null;
@@ -954,8 +1008,15 @@ function canProcessOnWorker(worker, options) {
954
1008
  (worker.worker || worker.loadWorker || (workerOptions == null ? void 0 : workerOptions.workerUrl)) && (options == null ? void 0 : options.worker)
955
1009
  );
956
1010
  }
1011
+ function canProcessOnWorkerInBatches(worker, options = {}) {
1012
+ if (!canProcessOnWorker(worker, options)) return false;
1013
+ if (options.workerInBatches !== void 0) return options.workerInBatches;
1014
+ return Boolean(
1015
+ worker.workerInBatches && !options.source && !getConfiguredWorkerURL(worker, options)
1016
+ );
1017
+ }
957
1018
  async function processOnWorker(worker, data, options = {}, context = {}, jobContext = {}) {
958
- var _a, _b, _c;
1019
+ var _a;
959
1020
  throwIfAborted(options.signal);
960
1021
  const name = getWorkerName(worker);
961
1022
  const workerFarm = WorkerFarm.getWorkerFarm(options);
@@ -964,15 +1025,13 @@ async function processOnWorker(worker, data, options = {}, context = {}, jobCont
964
1025
  const job = await workerPool.startJob(
965
1026
  jobName,
966
1027
  // eslint-disable-next-line
967
- onMessage.bind(null, context)
1028
+ onMessage.bind(null, context, options),
1029
+ void 0,
1030
+ options.signal
968
1031
  );
969
- const abortJob = () => job.abort(getAbortReason(options.signal));
970
- (_a = options.signal) == null ? void 0 : _a.addEventListener("abort", abortJob, { once: true });
971
- if ((_b = options.signal) == null ? void 0 : _b.aborted) {
972
- abortJob();
973
- }
974
1032
  try {
975
- const { signal: _signal, ...workerOptions } = options;
1033
+ if ((_a = options.signal) == null ? void 0 : _a.aborted) job.abort(getAbortReason(options.signal));
1034
+ const { signal: _signal, onProgress: _onProgress, ...workerOptions } = options;
976
1035
  const transferableOptions = getTransferListForWriter(workerOptions);
977
1036
  const transferableContext = getTransferListForWriter(jobContext);
978
1037
  if (job.isRunning) {
@@ -984,15 +1043,15 @@ async function processOnWorker(worker, data, options = {}, context = {}, jobCont
984
1043
  }
985
1044
  const result = await job.result;
986
1045
  return result.result;
987
- } finally {
988
- (_c = options.signal) == null ? void 0 : _c.removeEventListener("abort", abortJob);
1046
+ } catch (error) {
1047
+ job.abort(error);
1048
+ throw error;
989
1049
  }
990
1050
  }
991
1051
  function processOnWorkerInBatches(worker, input, options = {}, context = {}, jobContext = {}) {
992
1052
  return processOnWorkerInBatchesIterator(worker, input, options, context, jobContext);
993
1053
  }
994
1054
  async function* processOnWorkerInBatchesIterator(worker, input, options, context, jobContext) {
995
- var _a, _b, _c;
996
1055
  throwIfAborted(options.signal);
997
1056
  const name = getWorkerName(worker);
998
1057
  const workerFarm = WorkerFarm.getWorkerFarm(options);
@@ -1001,8 +1060,15 @@ async function* processOnWorkerInBatchesIterator(worker, input, options, context
1001
1060
  const inputIterator = getAsyncIterator(input);
1002
1061
  let outputFinished = false;
1003
1062
  let inputFinished = false;
1063
+ let inputCleanupRequested = false;
1004
1064
  let inputRequest = Promise.resolve();
1005
1065
  const batchJobFailed = new Error("Worker batch job failed");
1066
+ const closeJobInput = () => {
1067
+ if (!inputFinished && !inputCleanupRequested) {
1068
+ inputCleanupRequested = true;
1069
+ closeInputIterator(inputIterator);
1070
+ }
1071
+ };
1006
1072
  const finishOutput = () => {
1007
1073
  if (!outputFinished) {
1008
1074
  outputFinished = true;
@@ -1020,8 +1086,12 @@ async function* processOnWorkerInBatchesIterator(worker, input, options, context
1020
1086
  options.jobName || worker.name,
1021
1087
  (activeJob, type, payload) => {
1022
1088
  switch (type) {
1089
+ case "progress":
1090
+ reportProgress(activeJob, options, payload);
1091
+ break;
1023
1092
  case "input-request":
1024
1093
  inputRequest = inputRequest.then(async () => {
1094
+ if (!activeJob.isRunning) return;
1025
1095
  const nextBatch = await inputIterator.next();
1026
1096
  if (!activeJob.isRunning) {
1027
1097
  return;
@@ -1048,7 +1118,7 @@ async function* processOnWorkerInBatchesIterator(worker, input, options, context
1048
1118
  break;
1049
1119
  }
1050
1120
  case "process":
1051
- void onMessage(context, activeJob, type, payload);
1121
+ void onMessage(context, options, activeJob, type, payload);
1052
1122
  break;
1053
1123
  default: {
1054
1124
  const error = new Error(`Unexpected worker batch message: ${type}`);
@@ -1057,16 +1127,19 @@ async function* processOnWorkerInBatchesIterator(worker, input, options, context
1057
1127
  break;
1058
1128
  }
1059
1129
  }
1060
- }
1061
- );
1062
- const abortJob = () => job.abort(getAbortReason(options.signal));
1063
- (_a = options.signal) == null ? void 0 : _a.addEventListener("abort", abortJob, { once: true });
1064
- if ((_b = options.signal) == null ? void 0 : _b.aborted) {
1065
- abortJob();
1066
- }
1067
- void job.result.catch(failOutput);
1130
+ },
1131
+ void 0,
1132
+ options.signal
1133
+ ).catch((error) => {
1134
+ closeJobInput();
1135
+ throw error;
1136
+ });
1137
+ void job.result.catch((error) => {
1138
+ failOutput(error);
1139
+ closeJobInput();
1140
+ });
1068
1141
  try {
1069
- const { signal: _signal, ...workerOptions } = options;
1142
+ const { signal: _signal, onProgress: _onProgress, ...workerOptions } = options;
1070
1143
  if (job.isRunning) {
1071
1144
  job.postMessage("process-in-batches", {
1072
1145
  options: getTransferListForWriter(workerOptions),
@@ -1088,13 +1161,10 @@ async function* processOnWorkerInBatchesIterator(worker, input, options, context
1088
1161
  }
1089
1162
  await job.result;
1090
1163
  } finally {
1091
- (_c = options.signal) == null ? void 0 : _c.removeEventListener("abort", abortJob);
1092
1164
  if (job.isRunning) {
1093
1165
  job.abort(createAbortError2("Worker batch iterator was closed"));
1094
1166
  }
1095
- if (!inputFinished) {
1096
- closeInputIterator(inputIterator);
1097
- }
1167
+ closeJobInput();
1098
1168
  }
1099
1169
  }
1100
1170
  function throwIfAborted(signal) {
@@ -1104,7 +1174,7 @@ function throwIfAborted(signal) {
1104
1174
  throw getAbortReason(signal);
1105
1175
  }
1106
1176
  function getAbortReason(signal) {
1107
- return (signal == null ? void 0 : signal.reason) ?? createAbortError2("Worker job was aborted");
1177
+ return signal ? signal.reason : createAbortError2("Worker job was aborted");
1108
1178
  }
1109
1179
  function createAbortError2(message) {
1110
1180
  const error = new Error(message);
@@ -1144,8 +1214,11 @@ function getWorkerPoolTarget(worker, options, name) {
1144
1214
  }
1145
1215
  return { name, url: getWorkerURL(worker, options) };
1146
1216
  }
1147
- async function onMessage(context, job, type, payload) {
1217
+ async function onMessage(context, options, job, type, payload) {
1148
1218
  switch (type) {
1219
+ case "progress":
1220
+ reportProgress(job, options, payload);
1221
+ break;
1149
1222
  case "done":
1150
1223
  job.done(payload);
1151
1224
  break;
@@ -1153,13 +1226,18 @@ async function onMessage(context, job, type, payload) {
1153
1226
  job.error(new Error(payload.error));
1154
1227
  break;
1155
1228
  case "process":
1156
- const { id, input, options } = payload;
1229
+ const { id, input, options: processOptions } = payload;
1157
1230
  try {
1158
1231
  if (!context.process) {
1159
1232
  job.postMessage("error", { id, error: "Worker not set up to process on main thread" });
1160
1233
  return;
1161
1234
  }
1162
- const result = await context.process(input, options, void 0, payload.context || {});
1235
+ const result = await context.process(
1236
+ input,
1237
+ { ...processOptions, signal: options.signal, onProgress: options.onProgress },
1238
+ void 0,
1239
+ payload.context || {}
1240
+ );
1163
1241
  job.postMessage("done", { id, result });
1164
1242
  } catch (error) {
1165
1243
  const message = error instanceof Error ? error.message : "unknown error";
@@ -1170,6 +1248,15 @@ async function onMessage(context, job, type, payload) {
1170
1248
  console.warn(`process-on-worker: unknown message ${type}`);
1171
1249
  }
1172
1250
  }
1251
+ function reportProgress(job, options, payload) {
1252
+ var _a;
1253
+ if (!job.isRunning || !payload.progress) return;
1254
+ try {
1255
+ (_a = options.onProgress) == null ? void 0 : _a.call(options, payload.progress);
1256
+ } catch (error) {
1257
+ job.abort(error);
1258
+ }
1259
+ }
1173
1260
 
1174
1261
  // src/worker-utils/lib/worker-api/validate-worker-version.ts
1175
1262
  function validateWorkerVersion(worker, coreVersion = VERSION2) {
@@ -1204,6 +1291,15 @@ function canParseWithWorker(loader, options) {
1204
1291
  }
1205
1292
  return Boolean(canProcessOnWorker(loader, workerOptions));
1206
1293
  }
1294
+ function canParseWithWorkerInBatches(loader, options) {
1295
+ const parquetOptions = options == null ? void 0 : options.parquet;
1296
+ if (loader.id === "parquet" && typeof (parquetOptions == null ? void 0 : parquetOptions.keyRetriever) === "function") {
1297
+ return false;
1298
+ }
1299
+ return Boolean(
1300
+ loader.workerInBatches && canParseWithWorker(loader, options) && canProcessOnWorkerInBatches(loader, getWorkerOptions(options))
1301
+ );
1302
+ }
1207
1303
  function shouldParseWithWorker(loader, data, options, context, estimateLoader = loader) {
1208
1304
  var _a, _b;
1209
1305
  if (!canParseWithWorker(loader, options)) {
@@ -1223,11 +1319,15 @@ function shouldParseWithWorker(loader, data, options, context, estimateLoader =
1223
1319
  }
1224
1320
  }
1225
1321
  async function parseWithWorker(loader, data, options, context, parseOnMainThread, signal) {
1226
- const workerSignal = signal || getWorkerAbortSignal(options);
1322
+ const workerSignal = signal || getWorkerAbortSignal(options) || (context == null ? void 0 : context.signal);
1227
1323
  const result = await processOnWorker(
1228
1324
  loader,
1229
1325
  data,
1230
- { ...getWorkerOptions(options, loader), signal: workerSignal },
1326
+ {
1327
+ ...getWorkerOptions(options, loader),
1328
+ signal: workerSignal,
1329
+ onProgress: (options == null ? void 0 : options.onProgress) || (context == null ? void 0 : context.reportProgress)
1330
+ },
1231
1331
  {
1232
1332
  process: async (input, processOptions, _workerContext, parseContext) => {
1233
1333
  if (!parseOnMainThread) {
@@ -1249,11 +1349,15 @@ async function parseWithWorker(loader, data, options, context, parseOnMainThread
1249
1349
  return isLoaderWithWorkerResultDeserializer(loader) ? loader.deserializeWorkerResult(result, options, context) : result;
1250
1350
  }
1251
1351
  async function* parseWithWorkerInBatches(loader, inputIterator, options, context, parseOnMainThread, signal) {
1252
- const workerSignal = signal || getWorkerAbortSignal(options);
1352
+ const workerSignal = signal || getWorkerAbortSignal(options) || (context == null ? void 0 : context.signal);
1253
1353
  const outputIterator = processOnWorkerInBatches(
1254
1354
  loader,
1255
1355
  inputIterator,
1256
- { ...getWorkerOptions(options), signal: workerSignal },
1356
+ {
1357
+ ...getWorkerOptions(options, loader),
1358
+ signal: workerSignal,
1359
+ onProgress: (options == null ? void 0 : options.onProgress) || (context == null ? void 0 : context.reportProgress)
1360
+ },
1257
1361
  {
1258
1362
  process: async (input, processOptions, _workerContext, parseContext) => {
1259
1363
  if (!parseOnMainThread) {
@@ -1278,6 +1382,7 @@ async function* parseWithWorkerInBatches(loader, inputIterator, options, context
1278
1382
  }
1279
1383
  function getWorkerAbortSignal(options) {
1280
1384
  var _a;
1385
+ if (options == null ? void 0 : options.signal) return options.signal;
1281
1386
  const parquetSignal = (_a = options == null ? void 0 : options.parquet) == null ? void 0 : _a.signal;
1282
1387
  if (typeof AbortSignal !== "undefined" && parquetSignal instanceof AbortSignal) {
1283
1388
  return parquetSignal;
@@ -1322,10 +1427,15 @@ function resolveWorkerLoaders(loader, loaderIds, loaderIsArray, context, options
1322
1427
  function getWorkerOptions(options = {}, loader) {
1323
1428
  const optionsToSerialize = {
1324
1429
  ...options,
1430
+ signal: void 0,
1431
+ onProgress: void 0,
1325
1432
  ...options.core ? { core: { ...options.core, authentications: void 0 } } : {}
1326
1433
  };
1327
1434
  const serializedOptions = JSON.parse(
1328
- JSON.stringify(optionsToSerialize, (_key, value) => typeof value === "bigint" ? null : value)
1435
+ JSON.stringify(optionsToSerialize, (_key, value) => {
1436
+ if (typeof AbortSignal !== "undefined" && value instanceof AbortSignal) return void 0;
1437
+ return typeof value === "bigint" ? null : value;
1438
+ })
1329
1439
  );
1330
1440
  restoreBigIntOptions(optionsToSerialize, serializedOptions);
1331
1441
  const workerOptions = {
@@ -1352,7 +1462,17 @@ function getSerializableLoaderContext(context) {
1352
1462
  if (!context) {
1353
1463
  return {};
1354
1464
  }
1355
- const { fetch: fetch2, loaders, coreApi: coreApi2, _parse, _parseSync, _parseInBatches, ...serializableContext } = context;
1465
+ const {
1466
+ fetch: fetch2,
1467
+ loaders,
1468
+ coreApi: coreApi2,
1469
+ signal,
1470
+ reportProgress: reportProgress2,
1471
+ _parse,
1472
+ _parseSync,
1473
+ _parseInBatches,
1474
+ ...serializableContext
1475
+ } = context;
1356
1476
  return JSON.parse(JSON.stringify(serializableContext));
1357
1477
  }
1358
1478
  function shouldParseCSVWithWorker(options) {
@@ -1773,7 +1893,7 @@ function createAuthenticatedFetch(options) {
1773
1893
  const authenticatedFetch = async (url, requestOptions = {}) => {
1774
1894
  var _a, _b, _c, _d, _e, _f, _g;
1775
1895
  requestOptions = mergeFetchOptions(fetchOptions, requestOptions);
1776
- if ((_a = requestOptions.signal) == null ? void 0 : _a.aborted) throw createAbortError3();
1896
+ (_a = requestOptions.signal) == null ? void 0 : _a.throwIfAborted();
1777
1897
  if (!credentials.length) return baseFetch(url, requestOptions);
1778
1898
  const resolvedURL = parseHTTPURL(url);
1779
1899
  const authorization = resolvedURL ? await authorizeRequest(resolvedURL, requestOptions, tokenCredentials) : { url, options: requestOptions, credentials: [] };
@@ -1792,7 +1912,7 @@ function createAuthenticatedFetch(options) {
1792
1912
  }
1793
1913
  );
1794
1914
  if (!refreshableCredentials.length && !shouldRetrySignature) return response;
1795
- if ((_e = requestOptions.signal) == null ? void 0 : _e.aborted) throw createAbortError3();
1915
+ (_e = requestOptions.signal) == null ? void 0 : _e.throwIfAborted();
1796
1916
  const refreshedTokens = /* @__PURE__ */ new Map();
1797
1917
  await Promise.all(
1798
1918
  refreshableCredentials.map(async ({ credential }) => {
@@ -1803,7 +1923,7 @@ function createAuthenticatedFetch(options) {
1803
1923
  if (!refreshedTokens.size && !shouldRetrySignature) return response;
1804
1924
  await ((_f = response.body) == null ? void 0 : _f.cancel().catch(() => {
1805
1925
  }));
1806
- if ((_g = requestOptions.signal) == null ? void 0 : _g.aborted) throw createAbortError3();
1926
+ (_g = requestOptions.signal) == null ? void 0 : _g.throwIfAborted();
1807
1927
  const replayAuthorization = resolvedURL ? await authorizeRequest(
1808
1928
  resolvedURL,
1809
1929
  requestOptions,
@@ -1844,7 +1964,7 @@ async function authenticateRequest(request, credentials, reason, response) {
1844
1964
  for (const credential of credentials) {
1845
1965
  const origin = getRequestOrigin(request.url);
1846
1966
  if (!origin || !normalizeRequestOrigins(credential.origins).includes(origin)) continue;
1847
- if (signal == null ? void 0 : signal.aborted) throw createAbortError3();
1967
+ signal == null ? void 0 : signal.throwIfAborted();
1848
1968
  const authenticatedRequest = await credential.authenticate({
1849
1969
  url: request.url,
1850
1970
  options: { ...request.options, headers: new Headers(request.options.headers) },
@@ -1860,7 +1980,7 @@ async function authenticateRequest(request, credentials, reason, response) {
1860
1980
  };
1861
1981
  appliedCredentials.push(credential);
1862
1982
  }
1863
- if (signal == null ? void 0 : signal.aborted) throw createAbortError3();
1983
+ signal == null ? void 0 : signal.throwIfAborted();
1864
1984
  return { ...request, credentials: appliedCredentials };
1865
1985
  }
1866
1986
  function getRequestOrigin(url) {
@@ -1950,9 +2070,6 @@ function isReplayableRequest(options) {
1950
2070
  if (!["GET", "HEAD", "OPTIONS", "PUT", "DELETE"].includes(method)) return false;
1951
2071
  return !(options.body && typeof ReadableStream !== "undefined" && options.body instanceof ReadableStream);
1952
2072
  }
1953
- function createAbortError3() {
1954
- return new DOMException("The operation was aborted.", "AbortError");
1955
- }
1956
2073
 
1957
2074
  // src/loader-utils/lib/path-utils/file-aliases.ts
1958
2075
  var pathPrefix = "";
@@ -2597,6 +2714,7 @@ var DEFAULT_LOADER_OPTIONS = {
2597
2714
  worker: true,
2598
2715
  // By default, use worker if provided by loader.
2599
2716
  workerThreshold: 0.1,
2717
+ workerInBatches: void 0,
2600
2718
  maxConcurrency: 3,
2601
2719
  // How many worker instances should be created for each loader.
2602
2720
  maxMobileConcurrency: 1,
@@ -2773,7 +2891,7 @@ function validateOptionsObject(options, id, defaultOptions, deprecatedOptions, l
2773
2891
  const prefix = id ? `${id}.` : "";
2774
2892
  for (const key in options) {
2775
2893
  const isSubOptions = !id && isObject(options[key]);
2776
- const isBaseUriOption = key === "baseUri" && !id;
2894
+ const isBaseUriOption = !id && ["baseUri", "signal", "onProgress"].includes(key);
2777
2895
  const isWorkerUrlOption = key === "workerUrl" && id;
2778
2896
  if (!(key in defaultOptions) && !isBaseUriOption && !isWorkerUrlOption) {
2779
2897
  if (key in deprecatedOptions) {
@@ -3478,11 +3596,13 @@ function getFetchFunction(options, context) {
3478
3596
  } else {
3479
3597
  fetchFunction = fetchFile;
3480
3598
  }
3599
+ const signal = loaderOptions.signal || (context == null ? void 0 : context.signal);
3600
+ const requestOptions = isObject(fetchOption) ? fetchOption : void 0;
3481
3601
  return createAuthenticatedFetch({
3482
3602
  fetch: fetchFunction,
3483
3603
  credentials: ((_b = loaderOptions.core) == null ? void 0 : _b.credentials) || [],
3484
3604
  authentications: (_c = loaderOptions.core) == null ? void 0 : _c.authentications,
3485
- fetchOptions: isObject(fetchOption) ? fetchOption : void 0
3605
+ fetchOptions: signal ? { ...requestOptions, signal } : requestOptions
3486
3606
  });
3487
3607
  }
3488
3608
 
@@ -3686,13 +3806,14 @@ async function bindSubloaders(loader, options, url) {
3686
3806
 
3687
3807
  // dist/lib/api/parse-file.js
3688
3808
  async function parseFile(file, loaders, options, context) {
3689
- var _a, _b;
3809
+ var _a, _b, _c;
3690
3810
  if (loaders && !Array.isArray(loaders) && !isLoaderObject(loaders)) {
3691
3811
  context = void 0;
3692
3812
  options = loaders;
3693
3813
  loaders = void 0;
3694
3814
  }
3695
3815
  options = options || {};
3816
+ (_a = options.signal || (context == null ? void 0 : context.signal)) == null ? void 0 : _a.throwIfAborted();
3696
3817
  const url = file.url || "";
3697
3818
  const typedLoaders = loaders;
3698
3819
  const candidateLoaders = getLoadersFromContext(typedLoaders, context);
@@ -3704,7 +3825,7 @@ async function parseFile(file, loaders, options, context) {
3704
3825
  throw new Error(`${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parseFile().`);
3705
3826
  }
3706
3827
  const normalizedOptions = normalizeOptions(options, loader, candidateLoaders, url);
3707
- const authenticationLoader = !loader.getAuthentications && ((_b = (_a = normalizedOptions.core) == null ? void 0 : _a.credentials) == null ? void 0 : _b.length) ? await getLoaderImplementation(loader, normalizedOptions, url || (context == null ? void 0 : context.url)) : loader;
3828
+ const authenticationLoader = !loader.getAuthentications && ((_c = (_b = normalizedOptions.core) == null ? void 0 : _b.credentials) == null ? void 0 : _c.length) ? await getLoaderImplementation(loader, normalizedOptions, url || (context == null ? void 0 : context.url)) : loader;
3708
3829
  const strictOptions = await resolveLoaderAuthenticationOptions(authenticationLoader, url, normalizedOptions);
3709
3830
  context = getLoaderContext(
3710
3831
  // @ts-expect-error candidateLoaders may be a single forced loader
@@ -3715,6 +3836,8 @@ async function parseFile(file, loaders, options, context) {
3715
3836
  return await parseFileWithLoader(loader, file, strictOptions, context);
3716
3837
  }
3717
3838
  async function parseFileWithLoader(loader, file, options, context) {
3839
+ var _a;
3840
+ (_a = options.signal || context.signal) == null ? void 0 : _a.throwIfAborted();
3718
3841
  validateWorkerVersion(loader);
3719
3842
  options = mergeOptions(loader.options, options);
3720
3843
  const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
@@ -3775,22 +3898,24 @@ function parseWithLoaderSync(loader, data, options, context) {
3775
3898
 
3776
3899
  // dist/lib/api/parse-in-batches.js
3777
3900
  async function parseInBatches(data, loaders, options, context) {
3778
- var _a, _b;
3901
+ var _a, _b, _c, _d;
3779
3902
  const loaderArray = Array.isArray(loaders) ? loaders : void 0;
3780
3903
  if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
3781
3904
  context = void 0;
3782
3905
  options = loaders;
3783
3906
  loaders = void 0;
3784
3907
  }
3908
+ (_a = (options == null ? void 0 : options.signal) || (context == null ? void 0 : context.signal)) == null ? void 0 : _a.throwIfAborted();
3785
3909
  data = await data;
3786
3910
  options = options || {};
3911
+ (_b = options.signal || (context == null ? void 0 : context.signal)) == null ? void 0 : _b.throwIfAborted();
3787
3912
  const url = getResourceUrl(data);
3788
3913
  const loader = await selectLoader(data, loaders, options);
3789
3914
  if (!loader) {
3790
3915
  return [];
3791
3916
  }
3792
3917
  const normalizedOptions = normalizeOptions(options, loader, loaderArray, url);
3793
- const authenticationLoader = !loader.getAuthentications && ((_b = (_a = normalizedOptions.core) == null ? void 0 : _a.credentials) == null ? void 0 : _b.length) ? await getLoaderImplementation(loader, normalizedOptions, url || (context == null ? void 0 : context.url)) : loader;
3918
+ const authenticationLoader = !loader.getAuthentications && ((_d = (_c = normalizedOptions.core) == null ? void 0 : _c.credentials) == null ? void 0 : _d.length) ? await getLoaderImplementation(loader, normalizedOptions, url || (context == null ? void 0 : context.url)) : loader;
3794
3919
  const strictOptions = await resolveLoaderAuthenticationOptions(authenticationLoader, url || "", normalizedOptions);
3795
3920
  context = getLoaderContext({ url, _parseInBatches: parseInBatches, _parse: parse, loaders: loaderArray }, strictOptions, context || null);
3796
3921
  const loaderWithParser = await getLoaderImplementation(loader, strictOptions, context.url);
@@ -3824,7 +3949,7 @@ async function parseToOutputIterator(loader, data, options, context) {
3824
3949
  const inputIterator = await getAsyncIterableFromData(data, options);
3825
3950
  const transformedIterator = await applyInputTransforms(inputIterator, ((_a = options == null ? void 0 : options.core) == null ? void 0 : _a.transforms) || []);
3826
3951
  if (loader.parseInBatches) {
3827
- if (loader.id === "csv" && isBrowser && canParseWithWorker(loader, options)) {
3952
+ if (canParseWithWorkerInBatches(loader, options)) {
3828
3953
  return parseWithWorkerInBatches(loader, transformedIterator, options, context, parse);
3829
3954
  }
3830
3955
  return loader.parseInBatches(transformedIterator, options, context);
@@ -4050,6 +4175,8 @@ function getLoaderContext(context, options, parentContext) {
4050
4175
  if (parentContext) {
4051
4176
  return {
4052
4177
  ...parentContext,
4178
+ signal: options.signal || parentContext.signal,
4179
+ reportProgress: options.onProgress || parentContext.reportProgress,
4053
4180
  coreApi: parentContext.coreApi || context.coreApi || coreApi,
4054
4181
  _parse: parentContext._parse || context._parse,
4055
4182
  fetch: getFetchFunction(options, parentContext)
@@ -4057,6 +4184,8 @@ function getLoaderContext(context, options, parentContext) {
4057
4184
  }
4058
4185
  const newContext = {
4059
4186
  ...context,
4187
+ signal: options.signal || context.signal,
4188
+ reportProgress: options.onProgress || context.reportProgress,
4060
4189
  fetch: getFetchFunction(options, context),
4061
4190
  coreApi: context.coreApi || coreApi
4062
4191
  };
@@ -4089,14 +4218,16 @@ function getLoadersFromContext(loaders, context) {
4089
4218
 
4090
4219
  // dist/lib/api/parse.js
4091
4220
  async function parse(data, loaders, options, context) {
4092
- var _a, _b;
4221
+ var _a, _b, _c, _d;
4093
4222
  if (loaders && !Array.isArray(loaders) && !isLoaderObject(loaders)) {
4094
4223
  context = void 0;
4095
4224
  options = loaders;
4096
4225
  loaders = void 0;
4097
4226
  }
4227
+ (_a = (options == null ? void 0 : options.signal) || (context == null ? void 0 : context.signal)) == null ? void 0 : _a.throwIfAborted();
4098
4228
  data = await data;
4099
4229
  options = options || {};
4230
+ (_b = options.signal || (context == null ? void 0 : context.signal)) == null ? void 0 : _b.throwIfAborted();
4100
4231
  const url = getResourceUrl(data);
4101
4232
  const typedLoaders = loaders;
4102
4233
  const candidateLoaders = getLoadersFromContext(typedLoaders, context);
@@ -4108,7 +4239,7 @@ async function parse(data, loaders, options, context) {
4108
4239
  throw new Error(`${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parse().`);
4109
4240
  }
4110
4241
  const normalizedOptions = normalizeOptions(options, loader, candidateLoaders, url);
4111
- const authenticationLoader = !loader.getAuthentications && ((_b = (_a = normalizedOptions.core) == null ? void 0 : _a.credentials) == null ? void 0 : _b.length) ? await getLoaderImplementation(loader, normalizedOptions, url || (context == null ? void 0 : context.url)) : loader;
4242
+ const authenticationLoader = !loader.getAuthentications && ((_d = (_c = normalizedOptions.core) == null ? void 0 : _c.credentials) == null ? void 0 : _d.length) ? await getLoaderImplementation(loader, normalizedOptions, url || (context == null ? void 0 : context.url)) : loader;
4112
4243
  const strictOptions = await resolveLoaderAuthenticationOptions(authenticationLoader, url || "", normalizedOptions);
4113
4244
  context = getLoaderContext(
4114
4245
  // @ts-expect-error
@@ -4119,6 +4250,8 @@ async function parse(data, loaders, options, context) {
4119
4250
  return await parseWithLoader(loader, data, strictOptions, context);
4120
4251
  }
4121
4252
  async function parseWithLoader(loader, data, options, context) {
4253
+ var _a;
4254
+ (_a = options.signal || context.signal) == null ? void 0 : _a.throwIfAborted();
4122
4255
  validateWorkerVersion(loader);
4123
4256
  options = mergeOptions(loader.options, options);
4124
4257
  if (isResponse(data)) {
@@ -4386,7 +4519,7 @@ function makeStream(source, options) {
4386
4519
  }
4387
4520
 
4388
4521
  // dist/null-loader.js
4389
- var VERSION4 = true ? "5.0.0-alpha.13" : "latest";
4522
+ var VERSION4 = true ? "5.0.0-alpha.14" : "latest";
4390
4523
  var NullWorkerLoader = {
4391
4524
  dataType: null,
4392
4525
  batchType: null,