@loaders.gl/core 5.0.0-alpha.4 → 5.0.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.dev.js +379 -41
- package/dist/dist.min.js +3 -3
- package/dist/index.cjs +385 -42
- package/dist/index.cjs.map +3 -3
- package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
- package/dist/lib/api/parse-in-batches.js +6 -2
- package/dist/lib/api/parse-in-batches.js.map +1 -1
- package/dist/lib/api/parse.js +2 -2
- package/dist/lib/api/parse.js.map +1 -1
- package/dist/lib/init.js +1 -1
- package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
- package/dist/lib/loader-utils/get-fetch-function.js +2 -1
- package/dist/lib/loader-utils/get-fetch-function.js.map +1 -1
- package/dist/lib/loader-utils/loader-context.d.ts +1 -1
- package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
- package/dist/lib/loader-utils/loader-context.js +6 -3
- package/dist/lib/loader-utils/loader-context.js.map +1 -1
- package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-defaults.js +1 -0
- package/dist/lib/loader-utils/option-defaults.js.map +1 -1
- package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-utils.js +12 -1
- package/dist/lib/loader-utils/option-utils.js.map +1 -1
- package/dist/null-loader.js +1 -1
- package/dist/null-worker-node.js +29 -3
- package/dist/null-worker.js +29 -3
- package/package.json +6 -6
- package/src/lib/api/parse-in-batches.ts +11 -2
- package/src/lib/api/parse.ts +2 -2
- package/src/lib/loader-utils/get-fetch-function.ts +2 -1
- package/src/lib/loader-utils/loader-context.ts +6 -3
- package/src/lib/loader-utils/option-defaults.ts +1 -0
- package/src/lib/loader-utils/option-utils.ts +13 -1
|
@@ -17,6 +17,7 @@ export const DEFAULT_LOADER_OPTIONS = {
|
|
|
17
17
|
useLocalLibraries: false,
|
|
18
18
|
CDN: 'https://unpkg.com/@loaders.gl',
|
|
19
19
|
worker: true, // By default, use worker if provided by loader.
|
|
20
|
+
workerThreshold: 0.1,
|
|
20
21
|
maxConcurrency: 3, // How many worker instances should be created for each loader.
|
|
21
22
|
maxMobileConcurrency: 1, // How many worker instances should be created for each loader on mobile devices.
|
|
22
23
|
reuseWorkers: isBrowser, // By default reuse workers in browser (Node.js refuses to terminate if browsers are running)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"option-defaults.js","sourceRoot":"","sources":["../../../src/lib/loader-utils/option-defaults.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAGpC,OAAO,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAC,UAAU,EAAC,qBAAkB;AAErC,MAAM,CAAC,MAAM,sBAAsB,GAAkB;IACnD,IAAI,EAAE;QACJ,OAAO,EAAE,SAAS;QAClB,UAAU;QACV,KAAK,EAAE,IAAI;QACX,QAAQ,EAAE,SAAS;QACnB,gBAAgB,EAAE,SAAS;QAC3B,uBAAuB,EAAE,SAAS;QAClC,OAAO,EAAE,KAAK;QACd,KAAK,EAAE,SAAS;QAChB,GAAG,EAAE,IAAI,UAAU,EAAE,EAAE,yEAAyE;QAChG,iBAAiB,EAAE,KAAK;QAExB,GAAG,EAAE,+BAA+B;QACpC,MAAM,EAAE,IAAI,EAAE,gDAAgD;QAC9D,cAAc,EAAE,CAAC,EAAE,+DAA+D;QAClF,oBAAoB,EAAE,CAAC,EAAE,iFAAiF;QAC1G,YAAY,EAAE,SAAS,EAAE,6FAA6F;QACtH,YAAY,EAAE,KAAK,EAAE,yCAAyC;QAC9D,WAAW,EAAE,EAAE,EAAE,0CAA0C;QAE3D,KAAK,EAAE,CAAC;QACR,QAAQ,EAAE,CAAC;QACX,SAAS,EAAE,MAAM;QACjB,eAAe,EAAE,CAAC;QAClB,QAAQ,EAAE,KAAK,EAAE,qFAAqF;QACtG,UAAU,EAAE,EAAE;KACf;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,6BAA6B;IAC7B,OAAO,EAAE,cAAc;IACvB,KAAK,EAAE,YAAY;IACnB,QAAQ,EAAE,eAAe;IACzB,gBAAgB,EAAE,uBAAuB;IACzC,uBAAuB,EAAE,8BAA8B;IACvD,OAAO,EAAE,cAAc;IACvB,GAAG,EAAE,UAAU;IACf,iBAAiB,EAAE,wBAAwB;IAE3C,GAAG,EAAE,UAAU;IACf,MAAM,EAAE,aAAa;IACrB,cAAc,EAAE,qBAAqB;IACrC,oBAAoB,EAAE,2BAA2B;IACjD,YAAY,EAAE,mBAAmB;IACjC,YAAY,EAAE,kBAAkB;IAChC,WAAW,EAAE,kBAAkB;IAC/B,OAAO,EAAE,kBAAkB;IAE3B,KAAK,EAAE,YAAY;IACnB,QAAQ,EAAE,eAAe;IACzB,KAAK,EAAE,YAAY;IACnB,SAAS,EAAE,gBAAgB;IAC3B,eAAe,EAAE,sBAAsB;IACvC,QAAQ,EAAE,eAAe;IACzB,UAAU,EAAE,iBAAiB;IAE7B,qBAAqB;IACrB,MAAM,EAAE,SAAS;IACjB,QAAQ,EAAE,kBAAkB;IAC5B,GAAG,EAAE,cAAc;IAEnB,6CAA6C;IAC7C,MAAM,EAAE,mBAAmB;IAC3B,OAAO,EAAE,oBAAoB;IAC7B,IAAI,EAAE,iBAAiB;IACvB,IAAI,EAAE,iBAAiB;IACvB,WAAW,EAAE,wBAAwB;IACrC,KAAK,EAAE,kBAAkB;IACzB,QAAQ,EAAE,qBAAqB;IAC/B,QAAQ,EAAE,qBAAqB;IAC/B,cAAc,EAAE,2BAA2B;IAC3C,SAAS,EAAE,sBAAsB;IACjC,SAAS,EAAE,sBAAsB;IACjC,MAAM,EAAE,mBAAmB;CAC5B,CAAC"}
|
|
1
|
+
{"version":3,"file":"option-defaults.js","sourceRoot":"","sources":["../../../src/lib/loader-utils/option-defaults.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAGpC,OAAO,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAC,UAAU,EAAC,qBAAkB;AAErC,MAAM,CAAC,MAAM,sBAAsB,GAAkB;IACnD,IAAI,EAAE;QACJ,OAAO,EAAE,SAAS;QAClB,UAAU;QACV,KAAK,EAAE,IAAI;QACX,QAAQ,EAAE,SAAS;QACnB,gBAAgB,EAAE,SAAS;QAC3B,uBAAuB,EAAE,SAAS;QAClC,OAAO,EAAE,KAAK;QACd,KAAK,EAAE,SAAS;QAChB,GAAG,EAAE,IAAI,UAAU,EAAE,EAAE,yEAAyE;QAChG,iBAAiB,EAAE,KAAK;QAExB,GAAG,EAAE,+BAA+B;QACpC,MAAM,EAAE,IAAI,EAAE,gDAAgD;QAC9D,eAAe,EAAE,GAAG;QACpB,cAAc,EAAE,CAAC,EAAE,+DAA+D;QAClF,oBAAoB,EAAE,CAAC,EAAE,iFAAiF;QAC1G,YAAY,EAAE,SAAS,EAAE,6FAA6F;QACtH,YAAY,EAAE,KAAK,EAAE,yCAAyC;QAC9D,WAAW,EAAE,EAAE,EAAE,0CAA0C;QAE3D,KAAK,EAAE,CAAC;QACR,QAAQ,EAAE,CAAC;QACX,SAAS,EAAE,MAAM;QACjB,eAAe,EAAE,CAAC;QAClB,QAAQ,EAAE,KAAK,EAAE,qFAAqF;QACtG,UAAU,EAAE,EAAE;KACf;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,6BAA6B;IAC7B,OAAO,EAAE,cAAc;IACvB,KAAK,EAAE,YAAY;IACnB,QAAQ,EAAE,eAAe;IACzB,gBAAgB,EAAE,uBAAuB;IACzC,uBAAuB,EAAE,8BAA8B;IACvD,OAAO,EAAE,cAAc;IACvB,GAAG,EAAE,UAAU;IACf,iBAAiB,EAAE,wBAAwB;IAE3C,GAAG,EAAE,UAAU;IACf,MAAM,EAAE,aAAa;IACrB,cAAc,EAAE,qBAAqB;IACrC,oBAAoB,EAAE,2BAA2B;IACjD,YAAY,EAAE,mBAAmB;IACjC,YAAY,EAAE,kBAAkB;IAChC,WAAW,EAAE,kBAAkB;IAC/B,OAAO,EAAE,kBAAkB;IAE3B,KAAK,EAAE,YAAY;IACnB,QAAQ,EAAE,eAAe;IACzB,KAAK,EAAE,YAAY;IACnB,SAAS,EAAE,gBAAgB;IAC3B,eAAe,EAAE,sBAAsB;IACvC,QAAQ,EAAE,eAAe;IACzB,UAAU,EAAE,iBAAiB;IAE7B,qBAAqB;IACrB,MAAM,EAAE,SAAS;IACjB,QAAQ,EAAE,kBAAkB;IAC5B,GAAG,EAAE,cAAc;IAEnB,6CAA6C;IAC7C,MAAM,EAAE,mBAAmB;IAC3B,OAAO,EAAE,oBAAoB;IAC7B,IAAI,EAAE,iBAAiB;IACvB,IAAI,EAAE,iBAAiB;IACvB,WAAW,EAAE,wBAAwB;IACrC,KAAK,EAAE,kBAAkB;IACzB,QAAQ,EAAE,qBAAqB;IAC/B,QAAQ,EAAE,qBAAqB;IAC/B,cAAc,EAAE,2BAA2B;IAC3C,SAAS,EAAE,sBAAsB;IACjC,SAAS,EAAE,sBAAsB;IACjC,MAAM,EAAE,mBAAmB;CAC5B,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"option-utils.d.ts","sourceRoot":"","sources":["../../../src/lib/loader-utils/option-utils.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,MAAM,EACN,aAAa,EAIb,mBAAmB,EAEpB,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"option-utils.d.ts","sourceRoot":"","sources":["../../../src/lib/loader-utils/option-utils.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,MAAM,EACN,aAAa,EAIb,mBAAmB,EAEpB,MAAM,0BAA0B,CAAC;AAgClC;;GAEG;AACH,KAAK,iBAAiB,GAAG;IACvB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,aAAa,EAAE,aAAa,CAAC;CAC9B,CAAC;AAEF;;;GAGG;AACH,wBAAgB,oBAAoB,IAAI,iBAAiB,CAWxD;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,IAAI,mBAAmB,CAQ5D;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI,CAO7D;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,aAAa,EACtB,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,MAAM,EAAE,EAClB,GAAG,CAAC,EAAE,MAAM,GACX,mBAAmB,CASrB;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,aAAa,GAAG,mBAAmB,CAYlF"}
|
|
@@ -16,6 +16,7 @@ const CORE_LOADER_OPTION_KEYS = [
|
|
|
16
16
|
'useLocalLibraries',
|
|
17
17
|
'CDN',
|
|
18
18
|
'worker',
|
|
19
|
+
'workerThreshold',
|
|
19
20
|
'maxConcurrency',
|
|
20
21
|
'maxMobileConcurrency',
|
|
21
22
|
'reuseWorkers',
|
|
@@ -82,7 +83,10 @@ export function normalizeOptions(options, loader, loaders, url) {
|
|
|
82
83
|
loaders = loaders || [];
|
|
83
84
|
loaders = Array.isArray(loaders) ? loaders : [loaders];
|
|
84
85
|
validateOptions(options, loaders);
|
|
85
|
-
|
|
86
|
+
validateWorkerThreshold(options);
|
|
87
|
+
const normalizedOptions = normalizeLoaderOptions(normalizeOptionsInternal(loader, options, url));
|
|
88
|
+
validateWorkerThreshold(normalizedOptions);
|
|
89
|
+
return normalizedOptions;
|
|
86
90
|
}
|
|
87
91
|
/**
|
|
88
92
|
* Returns a copy of the provided options with deprecated top-level core fields moved into `core`
|
|
@@ -122,6 +126,13 @@ function validateOptions(options, loaders) {
|
|
|
122
126
|
validateOptionsObject(idOptions, loader.id, loaderOptions, deprecatedOptions, loaders);
|
|
123
127
|
}
|
|
124
128
|
}
|
|
129
|
+
/** Validates the automatic worker-selection threshold. */
|
|
130
|
+
function validateWorkerThreshold(options) {
|
|
131
|
+
const threshold = options.core?.workerThreshold;
|
|
132
|
+
if (threshold !== undefined && (!Number.isFinite(threshold) || threshold < 0 || threshold > 1)) {
|
|
133
|
+
throw new Error('core.workerThreshold must be a finite number between 0 and 1');
|
|
134
|
+
}
|
|
135
|
+
}
|
|
125
136
|
// eslint-disable-next-line max-params, complexity
|
|
126
137
|
function validateOptionsObject(options, id, defaultOptions, deprecatedOptions, loaders) {
|
|
127
138
|
const loaderName = id || 'Top level';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"option-utils.js","sourceRoot":"","sources":["../../../src/lib/loader-utils/option-utils.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,EAGL,iBAAiB,EACjB,YAAY,EACZ,QAAQ,EAER,IAAI,EACL,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAC,QAAQ,EAAE,OAAO,EAAC,qBAAkB;AAC5C,OAAO,EAAC,sBAAsB,EAAE,sBAAsB,EAAC,6BAA0B;AACjF,OAAO,EAAC,gBAAgB,EAAC,8BAA2B;AAEpD,MAAM,uBAAuB,GAAG;IAC9B,SAAS;IACT,OAAO;IACP,UAAU;IACV,kBAAkB;IAClB,yBAAyB;IACzB,SAAS;IACT,KAAK;IACL,mBAAmB;IACnB,KAAK;IACL,QAAQ;IACR,gBAAgB;IAChB,sBAAsB;IACtB,cAAc;IACd,0BAA0B;IAC1B,cAAc;IACd,aAAa;IACb,OAAO;IACP,UAAU;IACV,OAAO;IACP,WAAW;IACX,iBAAiB;IACjB,UAAU;IACV,YAAY;CACJ,CAAC;AAUX;;;GAGG;AACH,MAAM,UAAU,oBAAoB;IAClC,aAAa;IACb,UAAU,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,IAAI,EAAE,CAAC;IAC9C,aAAa;IACb,MAAM,EAAC,OAAO,EAAC,GAAG,UAAU,CAAC;IAE7B,8EAA8E;IAC9E,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,OAAO,CAAC,MAAM,GAAG,EAAE,CAAC;IACtB,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,CAAC;AACxB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB;IACpC,MAAM,KAAK,GAAG,oBAAoB,EAAE,CAAC;IACrC,oEAAoE;IACpE,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa,IAAI;QAC3C,GAAG,sBAAsB;QACzB,IAAI,EAAE,EAAC,GAAG,sBAAsB,CAAC,IAAI,EAAC;KACvC,CAAC;IACF,OAAO,sBAAsB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;AACrD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAsB;IACrD,MAAM,KAAK,GAAG,oBAAoB,EAAE,CAAC;IACrC,MAAM,aAAa,GAAG,sBAAsB,EAAE,CAAC;IAC/C,+CAA+C;IAC/C,KAAK,CAAC,aAAa,GAAG,wBAAwB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IACvE,2CAA2C;IAC3C,iBAAiB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;AACrC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAC9B,OAAsB,EACtB,MAAc,EACd,OAAkB,EAClB,GAAY;IAEZ,OAAO,GAAG,OAAO,IAAI,EAAE,CAAC;IACxB,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAEvD,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAClC,OAAO,sBAAsB,CAAC,wBAAwB,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"option-utils.js","sourceRoot":"","sources":["../../../src/lib/loader-utils/option-utils.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,EAGL,iBAAiB,EACjB,YAAY,EACZ,QAAQ,EAER,IAAI,EACL,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAC,QAAQ,EAAE,OAAO,EAAC,qBAAkB;AAC5C,OAAO,EAAC,sBAAsB,EAAE,sBAAsB,EAAC,6BAA0B;AACjF,OAAO,EAAC,gBAAgB,EAAC,8BAA2B;AAEpD,MAAM,uBAAuB,GAAG;IAC9B,SAAS;IACT,OAAO;IACP,UAAU;IACV,kBAAkB;IAClB,yBAAyB;IACzB,SAAS;IACT,KAAK;IACL,mBAAmB;IACnB,KAAK;IACL,QAAQ;IACR,iBAAiB;IACjB,gBAAgB;IAChB,sBAAsB;IACtB,cAAc;IACd,0BAA0B;IAC1B,cAAc;IACd,aAAa;IACb,OAAO;IACP,UAAU;IACV,OAAO;IACP,WAAW;IACX,iBAAiB;IACjB,UAAU;IACV,YAAY;CACJ,CAAC;AAUX;;;GAGG;AACH,MAAM,UAAU,oBAAoB;IAClC,aAAa;IACb,UAAU,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,IAAI,EAAE,CAAC;IAC9C,aAAa;IACb,MAAM,EAAC,OAAO,EAAC,GAAG,UAAU,CAAC;IAE7B,8EAA8E;IAC9E,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,OAAO,CAAC,MAAM,GAAG,EAAE,CAAC;IACtB,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,CAAC;AACxB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB;IACpC,MAAM,KAAK,GAAG,oBAAoB,EAAE,CAAC;IACrC,oEAAoE;IACpE,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa,IAAI;QAC3C,GAAG,sBAAsB;QACzB,IAAI,EAAE,EAAC,GAAG,sBAAsB,CAAC,IAAI,EAAC;KACvC,CAAC;IACF,OAAO,sBAAsB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;AACrD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAsB;IACrD,MAAM,KAAK,GAAG,oBAAoB,EAAE,CAAC;IACrC,MAAM,aAAa,GAAG,sBAAsB,EAAE,CAAC;IAC/C,+CAA+C;IAC/C,KAAK,CAAC,aAAa,GAAG,wBAAwB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IACvE,2CAA2C;IAC3C,iBAAiB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;AACrC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAC9B,OAAsB,EACtB,MAAc,EACd,OAAkB,EAClB,GAAY;IAEZ,OAAO,GAAG,OAAO,IAAI,EAAE,CAAC;IACxB,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAEvD,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAClC,uBAAuB,CAAC,OAAO,CAAC,CAAC;IACjC,MAAM,iBAAiB,GAAG,sBAAsB,CAAC,wBAAwB,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IACjG,uBAAuB,CAAC,iBAAiB,CAAC,CAAC;IAC3C,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,OAAsB;IAC3D,MAAM,UAAU,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC/C,mCAAmC,CAAC,UAAU,CAAC,CAAC;IAChD,KAAK,MAAM,GAAG,IAAI,uBAAuB,EAAE,CAAC;QAC1C,IAAI,UAAU,CAAC,IAAI,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;YAC1D,OAAQ,UAAsC,CAAC,GAAG,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IACD,IAAI,UAAU,CAAC,IAAI,IAAI,UAAU,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACjE,OAAQ,UAAkB,CAAC,OAAO,CAAC;IACrC,CAAC;IACD,OAAO,UAAiC,CAAC;AAC3C,CAAC;AAED,mBAAmB;AAEnB;;;;GAIG;AACH,SAAS,eAAe,CAAC,OAAsB,EAAE,OAAiB;IAChE,0BAA0B;IAC1B,qBAAqB,CAAC,OAAO,EAAE,IAAI,EAAE,sBAAsB,EAAE,sBAAsB,EAAE,OAAO,CAAC,CAAC;IAC9F,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,yEAAyE;QACzE,MAAM,SAAS,GACZ,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAA6B,IAAI,EAAE,CAAC;QAErE,sFAAsF;QACtF,MAAM,aAAa,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1E,MAAM,iBAAiB,GACrB,CAAC,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAE1E,mCAAmC;QACnC,aAAa;QACb,qBAAqB,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,EAAE,aAAa,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAED,0DAA0D;AAC1D,SAAS,uBAAuB,CAAC,OAAsB;IACrD,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,EAAE,eAAe,CAAC;IAChD,IAAI,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC;QAC/F,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;AACH,CAAC;AAED,kDAAkD;AAClD,SAAS,qBAAqB,CAC5B,OAAsB,EACtB,EAAiB,EACjB,cAAuC,EACvC,iBAA0C,EAC1C,OAAiB;IAEjB,MAAM,UAAU,GAAG,EAAE,IAAI,WAAW,CAAC;IACrC,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAElC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,kFAAkF;QAClF,MAAM,YAAY,GAAG,CAAC,EAAE,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;QACnD,MAAM,eAAe,GAAG,GAAG,KAAK,SAAS,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,iBAAiB,GAAG,GAAG,KAAK,WAAW,IAAI,EAAE,CAAC;QACpD,wHAAwH;QACxH,IAAI,CAAC,CAAC,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACvE,6BAA6B;YAC7B,IAAI,GAAG,IAAI,iBAAiB,EAAE,CAAC;gBAC7B,IAAI,QAAQ,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;oBACvB,QAAQ,CAAC,IAAI,CACX,GAAG,UAAU,oBAAoB,MAAM,GAAG,GAAG,iCAAiC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CACzG,EAAE,CAAC;gBACN,CAAC;YACH,CAAC;iBAAM,IAAI,CAAC,YAAY,EAAE,CAAC;gBACzB,IAAI,QAAQ,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;oBACvB,MAAM,UAAU,GAAG,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;oBACnD,QAAQ,CAAC,IAAI,CACX,GAAG,UAAU,oBAAoB,MAAM,GAAG,GAAG,sBAAsB,UAAU,EAAE,CAChF,EAAE,CAAC;gBACN,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiB,EAAE,OAAiB;IAC7D,MAAM,kBAAkB,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IACnD,IAAI,cAAc,GAAG,EAAE,CAAC;IACxB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACjC,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;gBACtB,OAAO,kBAAkB,MAAM,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC;YACjD,CAAC;YACD,MAAM,YAAY,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;YACvC,MAAM,cAAc,GAClB,kBAAkB,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,YAAY,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;YAC7F,IAAI,cAAc,EAAE,CAAC;gBACnB,cAAc,GAAG,cAAc,IAAI,kBAAkB,MAAM,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC;YAC7E,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,cAAc,CAAC;AACxB,CAAC;AAED,SAAS,wBAAwB,CAC/B,MAAc,EACd,OAAsB,EACtB,GAAY;IAEZ,MAAM,oBAAoB,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IAElD,MAAM,aAAa,GAAG,EAAC,GAAG,oBAAoB,EAAC,CAAC;IAChD,IAAI,oBAAoB,CAAC,IAAI,EAAE,CAAC;QAC9B,aAAa,CAAC,IAAI,GAAG,EAAC,GAAG,oBAAoB,CAAC,IAAI,EAAC,CAAC;IACtD,CAAC;IACD,mCAAmC,CAAC,aAAa,CAAC,CAAC;IAEnD,8DAA8D;IAC9D,IAAI,aAAa,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,EAAE,CAAC;QACrC,aAAa,CAAC,IAAI,GAAG,EAAC,GAAG,aAAa,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,OAAO,EAAE,EAAC,CAAC;IACnE,CAAC;IAED,iBAAiB,CAAC,aAAa,EAAE,sBAAsB,CAAC,sBAAsB,EAAE,CAAC,CAAC,CAAC;IAEnF,MAAM,WAAW,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IACpD,iBAAiB,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;IAC9C,qBAAqB,CAAC,aAAa,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;IAE1D,aAAa,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;IAClC,4BAA4B,CAAC,aAAa,CAAC,CAAC;IAE5C,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,qBAAqB,CAC5B,aAA4B,EAC5B,WAA0B,EAC1B,MAAc;IAEd,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC;IAC5C,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO;IACT,CAAC;IAED,MAAM,oBAAoB,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACzD,IAAI,CAAC,YAAY,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,oBAAoB,CAAC,EAAE,CAAC;QAC9E,OAAO;IACT,CAAC;IAED,MAAM,yBAAyB,GAAG,sBAAsB,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACtE,IAAI,YAAY,CAAC,yBAAyB,CAAC,IAAI,OAAO,IAAI,yBAAyB,EAAE,CAAC;QACpF,OAAO;IACT,CAAC;IAED,MAAM,uBAAuB,GAAG,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvD,IAAI,YAAY,CAAC,uBAAuB,CAAC,IAAI,OAAO,IAAI,uBAAuB,EAAE,CAAC;QAChF,OAAO;IACT,CAAC;IAED,MAAM,mBAAmB,GAAG,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrD,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG;QACzB,GAAG,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,KAAK,EAAE,SAAS;KACjB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,aAA4B,EAAE,OAAsB;IAC7E,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,2BAA2B;QAC3B,mFAAmF;QACnF,IAAI,GAAG,IAAI,OAAO,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,YAAY,CAAC,KAAK,CAAC,IAAI,YAAY,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC5D,aAAa,CAAC,GAAG,CAAC,GAAG;oBACnB,GAAI,aAAa,CAAC,GAAG,CAAY;oBACjC,GAAI,OAAO,CAAC,GAAG,CAAY;iBAC5B,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,aAAa,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;QACD,mGAAmG;IACrG,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,SAAS,aAAa,CAAC,OAAsB,EAAE,GAAY;IACzD,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO;IACT,CAAC;IACD,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,EAAE,OAAO,KAAK,SAAS,CAAC;IAC3D,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,OAAO,CAAC,IAAI,KAAK,EAAE,CAAC;QACpB,OAAO,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7D,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAsB;IAChD,MAAM,aAAa,GAAG,EAAC,GAAG,OAAO,EAAC,CAAC;IACnC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,aAAa,CAAC,IAAI,GAAG,EAAC,GAAG,OAAO,CAAC,IAAI,EAAC,CAAC;IACzC,CAAC;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;;;GAIG;AACH,SAAS,mCAAmC,CAAC,OAAsB;IACjE,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,OAAO,CAAC,IAAI,KAAK,EAAE,CAAC;QACpB,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YACvC,OAAO,CAAC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QACzC,CAAC;IACH,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,uBAAuB,EAAE,CAAC;QAC1C,IAAK,OAAmC,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;YAC5D,OAAO,CAAC,IAAI,KAAK,EAAE,CAAC;YACpB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;YACjC,MAAM,UAAU,GAAG,WAAsC,CAAC;YAC1D,yGAAyG;YACzG,iCAAiC;YACjC,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;gBAClC,UAAU,CAAC,GAAG,CAAC,GAAI,OAAmC,CAAC,GAAG,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;IACH,CAAC;IAED,sGAAsG;IACtG,MAAM,eAAe,GAAI,OAAe,CAAC,OAAO,CAAC;IACjD,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QAClC,OAAO,CAAC,IAAI,KAAK,EAAE,CAAC;QACpB,IAAI,OAAO,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC,WAAW,GAAG,eAAe,CAAC;QAC7C,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,4BAA4B,CAAC,OAAsB;IAC1D,MAAM,WAAW,GAAG,OAAO,CAAC,IAA2C,CAAC;IACxE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,uBAAuB,EAAE,CAAC;QAC1C,IAAI,WAAW,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;YAClC,OAAmC,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC"}
|
package/dist/null-loader.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
5
5
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
6
|
-
const VERSION = typeof "5.0.0-alpha.
|
|
6
|
+
const VERSION = typeof "5.0.0-alpha.6" !== 'undefined' ? "5.0.0-alpha.6" : 'latest';
|
|
7
7
|
/**
|
|
8
8
|
* Loads any data and returns null (or optionally passes through data unparsed)
|
|
9
9
|
*/
|
package/dist/null-worker-node.js
CHANGED
|
@@ -345,9 +345,22 @@ function processOnMainThread(arrayBuffer, options = {}, jobContext = {}) {
|
|
|
345
345
|
}
|
|
346
346
|
|
|
347
347
|
// ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
|
|
348
|
-
async function createLoaderWorker(loader) {
|
|
348
|
+
async function createLoaderWorker(loader, selectLoader = () => loader) {
|
|
349
349
|
await createWorker(
|
|
350
|
-
(input, options, workerContext, loaderContext) => processLoaderWorkerData(
|
|
350
|
+
(input, options, workerContext, loaderContext) => processLoaderWorkerData(
|
|
351
|
+
selectLoader(options || {}),
|
|
352
|
+
input,
|
|
353
|
+
options,
|
|
354
|
+
workerContext,
|
|
355
|
+
loaderContext
|
|
356
|
+
),
|
|
357
|
+
(inputIterator, options, workerContext, loaderContext) => processLoaderWorkerBatches(
|
|
358
|
+
selectLoader(options || {}),
|
|
359
|
+
inputIterator,
|
|
360
|
+
options,
|
|
361
|
+
workerContext,
|
|
362
|
+
loaderContext
|
|
363
|
+
)
|
|
351
364
|
);
|
|
352
365
|
}
|
|
353
366
|
async function processLoaderWorkerData(loader, input, options = {}, workerContext, loaderContext = {}) {
|
|
@@ -363,6 +376,19 @@ async function processLoaderWorkerData(loader, input, options = {}, workerContex
|
|
|
363
376
|
});
|
|
364
377
|
return loader.serializeWorkerResult ? loader.serializeWorkerResult(result, options, loaderContext) : result;
|
|
365
378
|
}
|
|
379
|
+
async function* processLoaderWorkerBatches(loader, inputIterator, options = {}, workerContext, loaderContext = {}) {
|
|
380
|
+
if (!loader.parseInBatches) {
|
|
381
|
+
throw new Error(`${loader.id} loader does not support batched parsing`);
|
|
382
|
+
}
|
|
383
|
+
const resultIterator = loader.parseInBatches(inputIterator, options, {
|
|
384
|
+
...loaderContext,
|
|
385
|
+
coreApi: createWorkerCoreApi(),
|
|
386
|
+
_parse: createParseOnMainThread(workerContext == null ? void 0 : workerContext.process)
|
|
387
|
+
});
|
|
388
|
+
for await (const batch of resultIterator) {
|
|
389
|
+
yield loader.serializeWorkerBatch ? loader.serializeWorkerBatch(batch, options, loaderContext) : batch;
|
|
390
|
+
}
|
|
391
|
+
}
|
|
366
392
|
function createWorkerCoreApi() {
|
|
367
393
|
const unavailable = (methodName) => () => {
|
|
368
394
|
throw new Error(`context.coreApi.${methodName} is unavailable inside worker loaders.`);
|
|
@@ -441,7 +467,7 @@ async function parseData({
|
|
|
441
467
|
}
|
|
442
468
|
|
|
443
469
|
// src/null-loader.ts
|
|
444
|
-
var VERSION = true ? "5.0.0-alpha.
|
|
470
|
+
var VERSION = true ? "5.0.0-alpha.6" : "latest";
|
|
445
471
|
var NullLoader = {
|
|
446
472
|
dataType: null,
|
|
447
473
|
batchType: null,
|
package/dist/null-worker.js
CHANGED
|
@@ -310,9 +310,22 @@
|
|
|
310
310
|
}
|
|
311
311
|
|
|
312
312
|
// ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
|
|
313
|
-
async function createLoaderWorker(loader) {
|
|
313
|
+
async function createLoaderWorker(loader, selectLoader = () => loader) {
|
|
314
314
|
await createWorker(
|
|
315
|
-
(input, options, workerContext, loaderContext) => processLoaderWorkerData(
|
|
315
|
+
(input, options, workerContext, loaderContext) => processLoaderWorkerData(
|
|
316
|
+
selectLoader(options || {}),
|
|
317
|
+
input,
|
|
318
|
+
options,
|
|
319
|
+
workerContext,
|
|
320
|
+
loaderContext
|
|
321
|
+
),
|
|
322
|
+
(inputIterator, options, workerContext, loaderContext) => processLoaderWorkerBatches(
|
|
323
|
+
selectLoader(options || {}),
|
|
324
|
+
inputIterator,
|
|
325
|
+
options,
|
|
326
|
+
workerContext,
|
|
327
|
+
loaderContext
|
|
328
|
+
)
|
|
316
329
|
);
|
|
317
330
|
}
|
|
318
331
|
async function processLoaderWorkerData(loader, input, options = {}, workerContext, loaderContext = {}) {
|
|
@@ -328,6 +341,19 @@
|
|
|
328
341
|
});
|
|
329
342
|
return loader.serializeWorkerResult ? loader.serializeWorkerResult(result, options, loaderContext) : result;
|
|
330
343
|
}
|
|
344
|
+
async function* processLoaderWorkerBatches(loader, inputIterator, options = {}, workerContext, loaderContext = {}) {
|
|
345
|
+
if (!loader.parseInBatches) {
|
|
346
|
+
throw new Error(`${loader.id} loader does not support batched parsing`);
|
|
347
|
+
}
|
|
348
|
+
const resultIterator = loader.parseInBatches(inputIterator, options, {
|
|
349
|
+
...loaderContext,
|
|
350
|
+
coreApi: createWorkerCoreApi(),
|
|
351
|
+
_parse: createParseOnMainThread(workerContext?.process)
|
|
352
|
+
});
|
|
353
|
+
for await (const batch of resultIterator) {
|
|
354
|
+
yield loader.serializeWorkerBatch ? loader.serializeWorkerBatch(batch, options, loaderContext) : batch;
|
|
355
|
+
}
|
|
356
|
+
}
|
|
331
357
|
function createWorkerCoreApi() {
|
|
332
358
|
const unavailable = (methodName) => () => {
|
|
333
359
|
throw new Error(`context.coreApi.${methodName} is unavailable inside worker loaders.`);
|
|
@@ -406,7 +432,7 @@
|
|
|
406
432
|
}
|
|
407
433
|
|
|
408
434
|
// src/null-loader.ts
|
|
409
|
-
var VERSION = true ? "5.0.0-alpha.
|
|
435
|
+
var VERSION = true ? "5.0.0-alpha.6" : "latest";
|
|
410
436
|
var NullLoader = {
|
|
411
437
|
dataType: null,
|
|
412
438
|
batchType: null,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loaders.gl/core",
|
|
3
|
-
"version": "5.0.0-alpha.
|
|
3
|
+
"version": "5.0.0-alpha.6",
|
|
4
4
|
"description": "The core API for working with loaders.gl loaders and writers",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -58,11 +58,11 @@
|
|
|
58
58
|
"build-worker-node": "esbuild src/workers/null-worker.ts --outfile=dist/null-worker-node.js --bundle --platform=node --target=node16 --define:__VERSION__=\\\"$npm_package_version\\\""
|
|
59
59
|
},
|
|
60
60
|
"dependencies": {
|
|
61
|
-
"@loaders.gl/loader-utils": "5.0.0-alpha.
|
|
62
|
-
"@loaders.gl/schema": "5.0.0-alpha.
|
|
63
|
-
"@loaders.gl/schema-utils": "5.0.0-alpha.
|
|
64
|
-
"@loaders.gl/worker-utils": "5.0.0-alpha.
|
|
61
|
+
"@loaders.gl/loader-utils": "5.0.0-alpha.6",
|
|
62
|
+
"@loaders.gl/schema": "5.0.0-alpha.6",
|
|
63
|
+
"@loaders.gl/schema-utils": "5.0.0-alpha.6",
|
|
64
|
+
"@loaders.gl/worker-utils": "5.0.0-alpha.6",
|
|
65
65
|
"@probe.gl/log": "^4.1.1"
|
|
66
66
|
},
|
|
67
|
-
"gitHead": "
|
|
67
|
+
"gitHead": "9b9ea7738e0fe63e55c19c136283705f99e0b9bf"
|
|
68
68
|
}
|
|
@@ -19,7 +19,12 @@ import type {
|
|
|
19
19
|
LoaderArrayBatchType,
|
|
20
20
|
TransformBatches
|
|
21
21
|
} from '@loaders.gl/loader-utils';
|
|
22
|
-
import {
|
|
22
|
+
import {
|
|
23
|
+
canParseWithWorker,
|
|
24
|
+
concatenateArrayBuffersAsync,
|
|
25
|
+
isBrowser,
|
|
26
|
+
parseWithWorkerInBatches
|
|
27
|
+
} from '@loaders.gl/loader-utils';
|
|
23
28
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
24
29
|
import {normalizeOptions} from '../loader-utils/option-utils';
|
|
25
30
|
import {getLoaderContext} from '../loader-utils/loader-context';
|
|
@@ -177,8 +182,12 @@ async function parseToOutputIterator(
|
|
|
177
182
|
options?.core?.transforms || []
|
|
178
183
|
);
|
|
179
184
|
|
|
180
|
-
//
|
|
185
|
+
// Stream supported browser batch parsers through one stateful worker session.
|
|
181
186
|
if (loader.parseInBatches) {
|
|
187
|
+
// The loader-level adapter is intentionally piloted with CSV Arrow batches first.
|
|
188
|
+
if (loader.id === 'csv' && isBrowser && canParseWithWorker(loader, options)) {
|
|
189
|
+
return parseWithWorkerInBatches(loader, transformedIterator, options, context, parse);
|
|
190
|
+
}
|
|
182
191
|
return loader.parseInBatches(transformedIterator, options, context);
|
|
183
192
|
}
|
|
184
193
|
|
package/src/lib/api/parse.ts
CHANGED
|
@@ -18,7 +18,7 @@ import type {
|
|
|
18
18
|
} from '@loaders.gl/loader-utils';
|
|
19
19
|
import {
|
|
20
20
|
parseWithWorker,
|
|
21
|
-
|
|
21
|
+
shouldParseWithWorker,
|
|
22
22
|
mergeOptions,
|
|
23
23
|
isResponse,
|
|
24
24
|
isBlob,
|
|
@@ -156,7 +156,7 @@ async function parseWithLoader(
|
|
|
156
156
|
|
|
157
157
|
const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
|
|
158
158
|
|
|
159
|
-
if (
|
|
159
|
+
if (shouldParseWithWorker(loaderWithParser, data, options, context, loader)) {
|
|
160
160
|
data = await getArrayBufferFromData(data, options);
|
|
161
161
|
return await parseWithWorker(loaderWithParser, data, options, context, parse);
|
|
162
162
|
}
|
|
@@ -27,8 +27,9 @@ export function getFetchFunction(
|
|
|
27
27
|
if (typeof fetchOption === 'function') {
|
|
28
28
|
fetchFunction = fetchOption;
|
|
29
29
|
} else if (isObject(fetchOption)) {
|
|
30
|
+
const baseFetch = context?.fetch || fetchFile;
|
|
30
31
|
fetchFunction = (url, requestOptions) =>
|
|
31
|
-
|
|
32
|
+
baseFetch(url, mergeFetchOptions(fetchOption as RequestInit, requestOptions));
|
|
32
33
|
} else if (context?.fetch) {
|
|
33
34
|
fetchFunction = context.fetch;
|
|
34
35
|
} else {
|
|
@@ -19,17 +19,20 @@ type LoaderContextProps = Omit<LoaderContext, 'fetch' | 'coreApi'> &
|
|
|
19
19
|
*
|
|
20
20
|
* @param context
|
|
21
21
|
* @param options
|
|
22
|
-
* @param
|
|
22
|
+
* @param parentContext
|
|
23
23
|
*/
|
|
24
24
|
export function getLoaderContext(
|
|
25
25
|
context: LoaderContextProps,
|
|
26
26
|
options: StrictLoaderOptions,
|
|
27
27
|
parentContext: LoaderContext | null
|
|
28
28
|
): LoaderContext {
|
|
29
|
-
// For recursive calls,
|
|
29
|
+
// For recursive calls, preserve the context while applying this loader's options
|
|
30
30
|
// TODO - add any additional loaders to context?
|
|
31
31
|
if (parentContext) {
|
|
32
|
-
return
|
|
32
|
+
return {
|
|
33
|
+
...parentContext,
|
|
34
|
+
fetch: getFetchFunction(options, parentContext)
|
|
35
|
+
};
|
|
33
36
|
}
|
|
34
37
|
|
|
35
38
|
const newContext: LoaderContext = {
|
|
@@ -21,6 +21,7 @@ export const DEFAULT_LOADER_OPTIONS: LoaderOptions = {
|
|
|
21
21
|
|
|
22
22
|
CDN: 'https://unpkg.com/@loaders.gl',
|
|
23
23
|
worker: true, // By default, use worker if provided by loader.
|
|
24
|
+
workerThreshold: 0.1,
|
|
24
25
|
maxConcurrency: 3, // How many worker instances should be created for each loader.
|
|
25
26
|
maxMobileConcurrency: 1, // How many worker instances should be created for each loader on mobile devices.
|
|
26
27
|
reuseWorkers: isBrowser, // By default reuse workers in browser (Node.js refuses to terminate if browsers are running)
|
|
@@ -26,6 +26,7 @@ const CORE_LOADER_OPTION_KEYS = [
|
|
|
26
26
|
'useLocalLibraries',
|
|
27
27
|
'CDN',
|
|
28
28
|
'worker',
|
|
29
|
+
'workerThreshold',
|
|
29
30
|
'maxConcurrency',
|
|
30
31
|
'maxMobileConcurrency',
|
|
31
32
|
'reuseWorkers',
|
|
@@ -111,7 +112,10 @@ export function normalizeOptions(
|
|
|
111
112
|
loaders = Array.isArray(loaders) ? loaders : [loaders];
|
|
112
113
|
|
|
113
114
|
validateOptions(options, loaders);
|
|
114
|
-
|
|
115
|
+
validateWorkerThreshold(options);
|
|
116
|
+
const normalizedOptions = normalizeLoaderOptions(normalizeOptionsInternal(loader, options, url));
|
|
117
|
+
validateWorkerThreshold(normalizedOptions);
|
|
118
|
+
return normalizedOptions;
|
|
115
119
|
}
|
|
116
120
|
|
|
117
121
|
/**
|
|
@@ -159,6 +163,14 @@ function validateOptions(options: LoaderOptions, loaders: Loader[]): void {
|
|
|
159
163
|
}
|
|
160
164
|
}
|
|
161
165
|
|
|
166
|
+
/** Validates the automatic worker-selection threshold. */
|
|
167
|
+
function validateWorkerThreshold(options: LoaderOptions): void {
|
|
168
|
+
const threshold = options.core?.workerThreshold;
|
|
169
|
+
if (threshold !== undefined && (!Number.isFinite(threshold) || threshold < 0 || threshold > 1)) {
|
|
170
|
+
throw new Error('core.workerThreshold must be a finite number between 0 and 1');
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
162
174
|
// eslint-disable-next-line max-params, complexity
|
|
163
175
|
function validateOptionsObject(
|
|
164
176
|
options: LoaderOptions,
|