@loaders.gl/loader-utils 4.3.4 → 4.4.0-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/README.md +8 -0
  2. package/dist/format-types.d.ts +24 -0
  3. package/dist/format-types.d.ts.map +1 -0
  4. package/dist/format-types.js +5 -0
  5. package/dist/format-types.js.map +1 -0
  6. package/dist/index.cjs +201 -307
  7. package/dist/index.cjs.map +4 -4
  8. package/dist/index.d.ts +16 -17
  9. package/dist/index.d.ts.map +1 -1
  10. package/dist/index.js +6 -9
  11. package/dist/index.js.map +1 -0
  12. package/dist/json-loader.js +2 -1
  13. package/dist/json-loader.js.map +1 -0
  14. package/dist/lib/binary-utils/array-buffer-utils.d.ts +2 -2
  15. package/dist/lib/binary-utils/array-buffer-utils.d.ts.map +1 -1
  16. package/dist/lib/binary-utils/array-buffer-utils.js +1 -0
  17. package/dist/lib/binary-utils/array-buffer-utils.js.map +1 -0
  18. package/dist/lib/binary-utils/dataview-copy-utils.js +1 -0
  19. package/dist/lib/binary-utils/dataview-copy-utils.js.map +1 -0
  20. package/dist/lib/binary-utils/get-first-characters.d.ts +1 -1
  21. package/dist/lib/binary-utils/get-first-characters.d.ts.map +1 -1
  22. package/dist/lib/binary-utils/get-first-characters.js +1 -0
  23. package/dist/lib/binary-utils/get-first-characters.js.map +1 -0
  24. package/dist/lib/binary-utils/memory-conversion-utils.d.ts +10 -4
  25. package/dist/lib/binary-utils/memory-conversion-utils.d.ts.map +1 -1
  26. package/dist/lib/binary-utils/memory-conversion-utils.js +40 -4
  27. package/dist/lib/binary-utils/memory-conversion-utils.js.map +1 -0
  28. package/dist/lib/binary-utils/memory-copy-utils.js +1 -0
  29. package/dist/lib/binary-utils/memory-copy-utils.js.map +1 -0
  30. package/dist/lib/env-utils/assert.js +1 -0
  31. package/dist/lib/env-utils/assert.js.map +1 -0
  32. package/dist/lib/env-utils/globals.js +1 -0
  33. package/dist/lib/env-utils/globals.js.map +1 -0
  34. package/dist/lib/file-provider/data-view-file.d.ts.map +1 -1
  35. package/dist/lib/file-provider/data-view-file.js +3 -1
  36. package/dist/lib/file-provider/data-view-file.js.map +1 -0
  37. package/dist/lib/file-provider/file-handle-file.js +1 -0
  38. package/dist/lib/file-provider/file-handle-file.js.map +1 -0
  39. package/dist/lib/file-provider/file-provider-interface.js +1 -0
  40. package/dist/lib/file-provider/file-provider-interface.js.map +1 -0
  41. package/dist/lib/file-provider/file-provider.js +1 -0
  42. package/dist/lib/file-provider/file-provider.js.map +1 -0
  43. package/dist/lib/files/blob-file.js +1 -0
  44. package/dist/lib/files/blob-file.js.map +1 -0
  45. package/dist/lib/files/file.js +1 -0
  46. package/dist/lib/files/file.js.map +1 -0
  47. package/dist/lib/files/http-file.js +1 -0
  48. package/dist/lib/files/http-file.js.map +1 -0
  49. package/dist/lib/files/node-file-facade.js +1 -0
  50. package/dist/lib/files/node-file-facade.js.map +1 -0
  51. package/dist/lib/files/sources.js +1 -0
  52. package/dist/lib/files/sources.js.map +1 -0
  53. package/dist/lib/filesystems/filesystem.js +1 -0
  54. package/dist/lib/filesystems/filesystem.js.map +1 -0
  55. package/dist/lib/filesystems/node-filesystem-facade.js +1 -0
  56. package/dist/lib/filesystems/node-filesystem-facade.js.map +1 -0
  57. package/dist/lib/iterators/async-iteration.d.ts +14 -14
  58. package/dist/lib/iterators/async-iteration.d.ts.map +1 -1
  59. package/dist/lib/iterators/async-iteration.js +68 -16
  60. package/dist/lib/iterators/async-iteration.js.map +1 -0
  61. package/dist/lib/iterators/text-iterators.js +2 -1
  62. package/dist/lib/iterators/text-iterators.js.map +1 -0
  63. package/dist/lib/javascript-utils/is-type.d.ts +59 -0
  64. package/dist/lib/javascript-utils/is-type.d.ts.map +1 -0
  65. package/dist/lib/javascript-utils/is-type.js +68 -0
  66. package/dist/lib/javascript-utils/is-type.js.map +1 -0
  67. package/dist/lib/log-utils/log.js +2 -1
  68. package/dist/lib/log-utils/log.js.map +1 -0
  69. package/dist/lib/module-utils/js-module-utils.js +1 -0
  70. package/dist/lib/module-utils/js-module-utils.js.map +1 -0
  71. package/dist/lib/node/buffer.browser.js +1 -0
  72. package/dist/lib/node/buffer.browser.js.map +1 -0
  73. package/dist/lib/node/buffer.js +1 -0
  74. package/dist/lib/node/buffer.js.map +1 -0
  75. package/dist/lib/node/fs.browser.js +1 -0
  76. package/dist/lib/node/fs.browser.js.map +1 -0
  77. package/dist/lib/node/promisify.js +1 -0
  78. package/dist/lib/node/promisify.js.map +1 -0
  79. package/dist/lib/node/stream.browser.js +1 -0
  80. package/dist/lib/node/stream.browser.js.map +1 -0
  81. package/dist/lib/node/stream.js +1 -0
  82. package/dist/lib/node/stream.js.map +1 -0
  83. package/dist/lib/option-utils/merge-options.d.ts +12 -0
  84. package/dist/lib/option-utils/merge-options.d.ts.map +1 -0
  85. package/dist/lib/option-utils/{merge-loader-options.js → merge-options.js} +5 -1
  86. package/dist/lib/option-utils/merge-options.js.map +1 -0
  87. package/dist/lib/parser-utils/parse-json.js +1 -0
  88. package/dist/lib/parser-utils/parse-json.js.map +1 -0
  89. package/dist/lib/path-utils/file-aliases.js +1 -0
  90. package/dist/lib/path-utils/file-aliases.js.map +1 -0
  91. package/dist/lib/path-utils/get-cwd.js +1 -0
  92. package/dist/lib/path-utils/get-cwd.js.map +1 -0
  93. package/dist/lib/path-utils/path.js +2 -1
  94. package/dist/lib/path-utils/path.js.map +1 -0
  95. package/dist/lib/request-utils/request-scheduler.d.ts +4 -0
  96. package/dist/lib/request-utils/request-scheduler.d.ts.map +1 -1
  97. package/dist/lib/request-utils/request-scheduler.js +15 -0
  98. package/dist/lib/request-utils/request-scheduler.js.map +1 -0
  99. package/dist/lib/sources/data-source.d.ts +26 -14
  100. package/dist/lib/sources/data-source.d.ts.map +1 -1
  101. package/dist/lib/sources/data-source.js +32 -9
  102. package/dist/lib/sources/data-source.js.map +1 -0
  103. package/dist/lib/sources/image-source.d.ts +1 -4
  104. package/dist/lib/sources/image-source.d.ts.map +1 -1
  105. package/dist/lib/sources/image-source.js +3 -2
  106. package/dist/lib/sources/image-source.js.map +1 -0
  107. package/dist/lib/sources/image-tile-source.d.ts +2 -1
  108. package/dist/lib/sources/image-tile-source.d.ts.map +1 -1
  109. package/dist/lib/sources/image-tile-source.js +1 -0
  110. package/dist/lib/sources/image-tile-source.js.map +1 -0
  111. package/dist/lib/sources/tile-source-adapter.d.ts +3 -3
  112. package/dist/lib/sources/tile-source-adapter.d.ts.map +1 -1
  113. package/dist/lib/sources/tile-source-adapter.js +1 -1
  114. package/dist/lib/sources/tile-source-adapter.js.map +1 -0
  115. package/dist/lib/sources/tile-source.d.ts +3 -5
  116. package/dist/lib/sources/tile-source.d.ts.map +1 -1
  117. package/dist/lib/sources/tile-source.js +1 -0
  118. package/dist/lib/sources/tile-source.js.map +1 -0
  119. package/dist/lib/sources/utils/image-type.js +1 -0
  120. package/dist/lib/sources/utils/image-type.js.map +1 -0
  121. package/dist/lib/sources/utils/utils.js +2 -1
  122. package/dist/lib/sources/utils/utils.js.map +1 -0
  123. package/dist/lib/sources/vector-source.d.ts +2 -4
  124. package/dist/lib/sources/vector-source.d.ts.map +1 -1
  125. package/dist/lib/sources/vector-source.js +2 -2
  126. package/dist/lib/sources/vector-source.js.map +1 -0
  127. package/dist/lib/sources/vector-tile-source.d.ts +2 -2
  128. package/dist/lib/sources/vector-tile-source.d.ts.map +1 -1
  129. package/dist/lib/sources/vector-tile-source.js +1 -0
  130. package/dist/lib/sources/vector-tile-source.js.map +1 -0
  131. package/dist/lib/worker-loader-utils/create-loader-worker.js +5 -1
  132. package/dist/lib/worker-loader-utils/create-loader-worker.js.map +1 -0
  133. package/dist/lib/worker-loader-utils/encode-with-worker.d.ts +1 -1
  134. package/dist/lib/worker-loader-utils/encode-with-worker.d.ts.map +1 -1
  135. package/dist/lib/worker-loader-utils/encode-with-worker.js +5 -2
  136. package/dist/lib/worker-loader-utils/encode-with-worker.js.map +1 -0
  137. package/dist/lib/worker-loader-utils/parse-with-worker.d.ts +1 -1
  138. package/dist/lib/worker-loader-utils/parse-with-worker.d.ts.map +1 -1
  139. package/dist/lib/worker-loader-utils/parse-with-worker.js +6 -3
  140. package/dist/lib/worker-loader-utils/parse-with-worker.js.map +1 -0
  141. package/dist/loader-types.d.ts +99 -74
  142. package/dist/loader-types.d.ts.map +1 -1
  143. package/dist/loader-types.js +1 -0
  144. package/dist/loader-types.js.map +1 -0
  145. package/dist/source-types.d.ts +20 -8
  146. package/dist/source-types.d.ts.map +1 -1
  147. package/dist/source-types.js +2 -0
  148. package/dist/source-types.js.map +1 -0
  149. package/dist/types.d.ts +9 -4
  150. package/dist/types.d.ts.map +1 -1
  151. package/dist/types.js +1 -0
  152. package/dist/types.js.map +1 -0
  153. package/dist/workers/json-worker.js +1 -0
  154. package/dist/workers/json-worker.js.map +1 -0
  155. package/dist/writer-types.d.ts +22 -15
  156. package/dist/writer-types.d.ts.map +1 -1
  157. package/dist/writer-types.js +1 -0
  158. package/dist/writer-types.js.map +1 -0
  159. package/package.json +4 -7
  160. package/src/format-types.ts +27 -0
  161. package/src/index.ts +51 -17
  162. package/src/lib/binary-utils/array-buffer-utils.ts +3 -3
  163. package/src/lib/binary-utils/get-first-characters.ts +1 -1
  164. package/src/lib/binary-utils/memory-conversion-utils.ts +58 -8
  165. package/src/lib/file-provider/data-view-file.ts +6 -1
  166. package/src/lib/iterators/async-iteration.ts +94 -19
  167. package/src/lib/iterators/text-iterators.ts +1 -1
  168. package/src/lib/javascript-utils/is-type.ts +130 -0
  169. package/src/lib/node/buffer.ts +1 -1
  170. package/src/lib/option-utils/{merge-loader-options.ts → merge-options.ts} +10 -6
  171. package/src/lib/path-utils/path.ts +1 -1
  172. package/src/lib/request-utils/request-scheduler.ts +15 -0
  173. package/src/lib/sources/data-source.ts +53 -19
  174. package/src/lib/sources/image-source.ts +2 -7
  175. package/src/lib/sources/image-tile-source.ts +2 -4
  176. package/src/lib/sources/tile-source-adapter.ts +4 -4
  177. package/src/lib/sources/tile-source.ts +3 -9
  178. package/src/lib/sources/utils/utils.ts +1 -1
  179. package/src/lib/sources/vector-source.ts +2 -6
  180. package/src/lib/sources/vector-tile-source.ts +2 -5
  181. package/src/lib/worker-loader-utils/create-loader-worker.ts +4 -1
  182. package/src/lib/worker-loader-utils/encode-with-worker.ts +5 -2
  183. package/src/lib/worker-loader-utils/parse-with-worker.ts +5 -3
  184. package/src/loader-types.ts +126 -98
  185. package/src/source-types.ts +35 -10
  186. package/src/types.ts +17 -9
  187. package/src/writer-types.ts +25 -20
  188. package/dist/lib/option-utils/merge-loader-options.d.ts +0 -9
  189. package/dist/lib/option-utils/merge-loader-options.d.ts.map +0 -1
@@ -2,34 +2,68 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import type {LoaderOptions} from '@loaders.gl/loader-utils';
5
+ import type {Loader, StrictLoaderOptions} from '../../loader-types';
6
+ import type {RequiredOptions} from '../option-utils/merge-options';
7
+ import {mergeOptions} from '../option-utils/merge-options';
8
+ import {resolvePath} from '../path-utils/file-aliases';
6
9
 
7
10
  /** Common properties for all data sources */
8
- export type DataSourceProps = {
9
- url?: string | Blob;
10
- /** LoaderOptions provide an option to override `fetch`. Will also be passed to any sub loaders */
11
- loadOptions?: LoaderOptions;
12
- };
11
+ export type DataSourceOptions = Partial<{
12
+ core: {
13
+ /** Allows application to specify which source should be selected. Matches `Source.type`. Defaults to 'auto' */
14
+ type?: string;
15
+ /** Any dataset attributions (in case underlying metadata does not include attributions) */
16
+ attributions?: string[];
17
+ /** LoaderOptions provide an option to override `fetch`. Will also be passed to any sub loaders */
18
+ loadOptions?: StrictLoaderOptions;
19
+ /** Make additional loaders available to the data source */
20
+ loaders?: Loader[];
21
+ };
22
+ [key: string]: Record<string, unknown>;
23
+ }>;
13
24
 
14
25
  /** base class of all data sources */
15
- export abstract class DataSource<PropsT extends DataSourceProps = DataSourceProps> {
16
- abstract data: unknown;
26
+ export abstract class DataSource<DataT, OptionsT extends DataSourceOptions> {
27
+ static defaultOptions: Required<DataSourceOptions> = {
28
+ core: {
29
+ type: 'auto',
30
+ attributions: [],
31
+ loadOptions: {},
32
+ loaders: []
33
+ }
34
+ };
35
+
36
+ optionsType?: OptionsT & DataSourceOptions;
37
+ options: Required<OptionsT & DataSourceOptions>;
38
+ readonly data: DataT;
39
+ readonly url: string;
40
+
41
+ /** The actual load options, if calling a loaders.gl loader */
42
+ loadOptions: StrictLoaderOptions;
17
43
  /** A resolved fetch function extracted from loadOptions prop */
18
44
  fetch: (url: string, options?: RequestInit) => Promise<Response>;
19
- /** The actual load options, if calling a loaders.gl loader */
20
- loadOptions: LoaderOptions;
21
45
  _needsRefresh: boolean = true;
22
46
 
23
- props: PropsT;
24
-
25
- constructor(props: PropsT) {
26
- this.props = {...props};
27
- this.loadOptions = {...props.loadOptions};
47
+ constructor(
48
+ data: DataT,
49
+ options: OptionsT,
50
+ defaultOptions?: Omit<RequiredOptions<OptionsT>, 'core'>
51
+ ) {
52
+ if (defaultOptions) {
53
+ // @ts-expect-error Typescript gets confused
54
+ this.options = mergeOptions({...defaultOptions, core: DataSource.defaultOptions}, options);
55
+ } else {
56
+ // @ts-expect-error
57
+ this.options = {...options};
58
+ }
59
+ this.data = data;
60
+ this.url = typeof data === 'string' ? resolvePath(data) : '';
61
+ this.loadOptions = {...this.options.core?.loadOptions};
28
62
  this.fetch = getFetchFunction(this.loadOptions);
29
63
  }
30
64
 
31
- setProps(props: PropsT) {
32
- this.props = Object.assign(this.props, props);
65
+ setProps(options: OptionsT) {
66
+ this.options = Object.assign(this.options, options);
33
67
  // TODO - add a shallow compare to avoid setting refresh if no change?
34
68
  this.setNeedsRefresh();
35
69
  }
@@ -59,8 +93,8 @@ export abstract class DataSource<PropsT extends DataSourceProps = DataSourceProp
59
93
  * @param options
60
94
  * @param context
61
95
  */
62
- export function getFetchFunction(options?: LoaderOptions) {
63
- const fetchFunction = options?.fetch;
96
+ export function getFetchFunction(options?: StrictLoaderOptions) {
97
+ const fetchFunction = options?.core?.fetch;
64
98
 
65
99
  // options.fetch can be a function
66
100
  if (fetchFunction && typeof fetchFunction === 'function') {
@@ -2,19 +2,14 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import type {DataSourceProps} from './data-source';
6
- import {DataSource} from './data-source';
7
5
  // TODO - can we import from schema?
8
6
  import {ImageType} from './utils/image-type';
9
-
10
- export type ImageSourceProps = DataSourceProps;
7
+ // TODO - remove (this breaks WMS layer)
11
8
 
12
9
  /**
13
10
  * ImageSource - data sources that allow images to be queried by (geospatial) extents
14
11
  */
15
- export abstract class ImageSource<
16
- PropsT extends ImageSourceProps = ImageSourceProps
17
- > extends DataSource<PropsT> {
12
+ export abstract class ImageSource {
18
13
  static type: string = 'template';
19
14
  static testURL = (url: string): boolean => false;
20
15
 
@@ -16,9 +16,7 @@ export type ImageTileSourceProps = TileSourceProps;
16
16
  * MapTileSource - data sources that allow data to be queried by (geospatial) tile
17
17
  * @note If geospatial, bounding box is expected to be in web mercator coordinates
18
18
  */
19
- export interface ImageTileSource<
20
- PropsT extends TileSourceProps = TileSourceProps,
21
- MetadataT extends TileSourceMetadata = TileSourceMetadata
22
- > extends TileSource<PropsT, MetadataT> {
19
+ export interface ImageTileSource extends TileSource {
20
+ getMetadata(): Promise<TileSourceMetadata>;
23
21
  getImageTile(parameters: GetTileParameters): Promise<ImageType | null>;
24
22
  }
@@ -9,12 +9,12 @@ import {ImageSource, ImageSourceMetadata} from './image-source';
9
9
  * @note
10
10
  * - If geospatial, bounding box is expected to be in web mercator coordinates
11
11
  */
12
- // @ts-expect-error TODO - does not implement all DataSource members
13
- export class TileSourceAdapter implements TileSource<ImageSourceMetadata> {
14
- readonly viewportSource: ImageSource;
15
- constructor(source: ImageSource) {
12
+ export class TileSourceAdapter<ImageSourceT extends ImageSource> implements TileSource {
13
+ readonly viewportSource: ImageSourceT;
14
+ constructor(source: ImageSourceT) {
16
15
  this.viewportSource = source;
17
16
  }
17
+
18
18
  async getMetadata(): Promise<ImageSourceMetadata> {
19
19
  return await this.viewportSource.getMetadata();
20
20
  }
@@ -2,25 +2,19 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import type {DataSourceProps} from './data-source';
6
- import {DataSource} from './data-source';
7
-
8
5
  /**
9
6
  * Props for a TileSource
10
7
  */
11
- export type TileSourceProps = DataSourceProps;
8
+ export type TileSourceProps = {};
12
9
 
13
10
  /**
14
11
  * MapTileSource - data sources that allow data to be queried by (geospatial) extents
15
12
  * @note
16
13
  * - If geospatial, bounding box is expected to be in web mercator coordinates
17
14
  */
18
- export interface TileSource<
19
- PropsT extends TileSourceProps = TileSourceProps,
20
- MetadataT extends TileSourceMetadata = TileSourceMetadata
21
- > extends DataSource<PropsT> {
15
+ export interface TileSource {
22
16
  // extends DataSource {
23
- getMetadata(): Promise<MetadataT>;
17
+ getMetadata(): Promise<TileSourceMetadata>;
24
18
  /** Flat parameters */
25
19
  getTile(parameters: GetTileParameters): Promise<unknown | null>;
26
20
  /** deck.gl compatibility: TileLayer and MTVLayer */
@@ -12,7 +12,7 @@ import type {LoaderOptions} from '@loaders.gl/loader-utils';
12
12
  * @param context
13
13
  */
14
14
  export function getFetchFunction(options?: LoaderOptions) {
15
- const fetchFunction = options?.fetch;
15
+ const fetchFunction = options?.core?.fetch;
16
16
 
17
17
  // options.fetch can be a function
18
18
  if (fetchFunction && typeof fetchFunction === 'function') {
@@ -3,19 +3,15 @@
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
5
  import type {Schema, GeoJSONTable, BinaryFeatureCollection} from '@loaders.gl/schema';
6
- import type {DataSourceProps} from './data-source';
7
- import {DataSource} from './data-source';
8
6
 
9
- export type VectorSourceProps = DataSourceProps;
7
+ export type VectorSourceProps = {};
10
8
 
11
9
  /**
12
10
  * VectorSource - data sources that allow features to be queried by (geospatial) extents
13
11
  * @note
14
12
  * - If geospatial, bounding box is expected to be in web mercator coordinates
15
13
  */
16
- export abstract class VectorSource<
17
- PropsT extends VectorSourceProps = VectorSourceProps
18
- > extends DataSource<PropsT> {
14
+ export abstract class VectorSource {
19
15
  static type: string = 'template';
20
16
  static testURL = (url: string): boolean => false;
21
17
 
@@ -3,7 +3,7 @@
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
5
  import type {Schema, Feature, BinaryFeatureCollection} from '@loaders.gl/schema';
6
- import {TileSource, TileSourceProps, TileSourceMetadata, GetTileParameters} from './tile-source';
6
+ import {TileSource, TileSourceProps, GetTileParameters} from './tile-source';
7
7
  import type {GetTileDataParameters} from './tile-source';
8
8
 
9
9
  export type VectorTile = unknown;
@@ -14,10 +14,7 @@ export type VectorTileSourceProps = TileSourceProps;
14
14
  * VectorTileSource - data sources that allow data to be queried by (geospatial) tile
15
15
  * @note If geospatial, bounding box is expected to be in web mercator coordinates
16
16
  */
17
- export interface VectorTileSource<
18
- PropsT extends VectorTileSourceProps = VectorTileSourceProps,
19
- MetadataT extends TileSourceMetadata = TileSourceMetadata
20
- > extends TileSource<PropsT, MetadataT> {
17
+ export interface VectorTileSource extends TileSource {
21
18
  getSchema(): Promise<Schema>;
22
19
  getVectorTile(parameters: GetTileParameters): Promise<VectorTile | null>;
23
20
  getTileData(
@@ -117,7 +117,10 @@ async function parseData({
117
117
  options = {
118
118
  ...options,
119
119
  modules: (loader && loader.options && loader.options.modules) || {},
120
- worker: false
120
+ core: {
121
+ ...options.core,
122
+ worker: false
123
+ }
121
124
  };
122
125
 
123
126
  return await parser(data, {...options}, context, loader);
@@ -12,10 +12,13 @@ export function canEncodeWithWorker(writer: Writer, options?: WriterOptions) {
12
12
  return false;
13
13
  }
14
14
 
15
+ const nodeWorkers = options?._nodeWorkers ?? options?.core?._nodeWorkers;
16
+ const useWorkers = options?.worker ?? options?.core?.worker;
17
+
15
18
  // Node workers are still experimental
16
- if (!isBrowser && !options?._nodeWorkers) {
19
+ if (!isBrowser && !nodeWorkers) {
17
20
  return false;
18
21
  }
19
22
 
20
- return writer.worker && options?.worker;
23
+ return Boolean(writer.worker && useWorkers);
21
24
  }
@@ -19,11 +19,13 @@ export function canParseWithWorker(loader: Loader, options?: LoaderOptions) {
19
19
  }
20
20
 
21
21
  // Node workers are still experimental
22
- if (!isBrowser && !options?._nodeWorkers) {
22
+ const nodeWorkers = options?._nodeWorkers ?? options?.core?._nodeWorkers;
23
+ if (!isBrowser && !nodeWorkers) {
23
24
  return false;
24
25
  }
25
26
 
26
- return loader.worker && options?.worker;
27
+ const useWorkers = options?.worker ?? options?.core?.worker;
28
+ return Boolean(loader.worker && useWorkers);
27
29
  }
28
30
 
29
31
  /**
@@ -40,7 +42,7 @@ export async function parseWithWorker(
40
42
  const name = loader.id; // TODO
41
43
  const url = getWorkerURL(loader, options);
42
44
 
43
- const workerFarm = WorkerFarm.getWorkerFarm(options);
45
+ const workerFarm = WorkerFarm.getWorkerFarm(options?.core);
44
46
  const workerPool = workerFarm.getWorkerPool({name, url});
45
47
 
46
48
  // options.log object contains functions which cannot be transferred
@@ -2,105 +2,132 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import {
6
- FetchLike,
7
- TransformBatches /* , DataType, SyncDataType, BatchableDataType */
8
- } from './types';
5
+ import type {Format} from './format-types';
6
+ import {FetchLike, TransformBatches} from './types';
9
7
  import {ReadableFile} from './lib/files/file';
10
8
 
11
9
  // LOADERS
12
10
 
11
+ /**
12
+ * Core Loader Options
13
+ */
14
+ export type StrictLoaderOptions = {
15
+ core?: {
16
+ /** Base URI for resolving relative paths */
17
+ baseUri?: string;
18
+ /** fetch options or a custom fetch function */
19
+ fetch?: typeof fetch | FetchLike | RequestInit | null;
20
+ /** Do not throw on errors */
21
+ nothrow?: boolean;
22
+
23
+ /** loader selection, search first for supplied mimeType */
24
+ mimeType?: string;
25
+ /** loader selection, provide fallback mimeType is server does not provide */
26
+ fallbackMimeType?: string;
27
+ /** loader selection, avoid searching registered loaders */
28
+ ignoreRegisteredLoaders?: boolean;
29
+
30
+ // general
31
+ /** Experimental: Supply a logger to the parser */
32
+ log?: any;
33
+
34
+ // batched parsing
35
+
36
+ /** Size of each batch. `auto` matches batches to size of incoming chunks */
37
+ batchSize?: number | 'auto';
38
+ /** Minimal amount of time between batches */
39
+ batchDebounceMs?: number;
40
+ /** Stop loading after a given number of rows (compare SQL limit clause) */
41
+ limit?: 0;
42
+ /** Experimental: Stop loading after reaching */
43
+ _limitMB?: 0;
44
+ /** Generate metadata batches */
45
+ metadata?: boolean;
46
+ /** Transforms to run on incoming batches */
47
+ transforms?: TransformBatches[];
48
+
49
+ // module loading
50
+
51
+ /** Force to load WASM libraries from local file system in NodeJS or from loaders.gl CDN in a web browser */
52
+ useLocalLibraries?: boolean;
53
+
54
+ // workers
55
+
56
+ /** CDN load workers from */
57
+ CDN?: string | null;
58
+ /** Set to `false` to disable workers */
59
+ worker?: boolean;
60
+ /** Number of concurrent workers (per loader) on desktop browser */
61
+ maxConcurrency?: number;
62
+ /** Number of concurrent workers (per loader) on mobile browsers */
63
+ maxMobileConcurrency?: number;
64
+ /** Set to `false` to prevent reuse workers */
65
+ reuseWorkers?: boolean;
66
+ /** Whether to use workers under Node.js (experimental) */
67
+ _nodeWorkers?: boolean;
68
+ /** set to 'test' to run local worker */
69
+ _workerType?: string;
70
+ };
71
+
72
+ /** Any additional JS libraries */
73
+ modules?: Record<string, unknown>;
74
+
75
+ // Accept other keys (loader options objects, e.g. `options.csv`, `options.json` ...)
76
+ [loaderId: string]: Record<string, unknown> | undefined;
77
+ };
78
+
13
79
  /**
14
80
  * Core Loader Options
15
81
  */
16
82
  export type LoaderOptions = {
17
- /** fetch options or a custom fetch function */
18
- fetch?: typeof fetch | FetchLike | RequestInit | null;
19
- /** Do not throw on errors */
20
- nothrow?: boolean;
83
+ core?: StrictLoaderOptions['core'];
84
+ modules?: StrictLoaderOptions['modules'];
21
85
 
22
- /** loader selection, search first for supplied mimeType */
86
+ // Deprecated top-level aliases for core options
87
+ /** @deprecated Use options.core.baseUri */
88
+ baseUri?: string;
89
+ /** @deprecated Use options.core.fetch */
90
+ fetch?: typeof fetch | FetchLike | RequestInit | null;
91
+ /** @deprecated Use options.core.mimeType */
23
92
  mimeType?: string;
24
- /** loader selection, provide fallback mimeType is server does not provide */
93
+ /** @deprecated Use options.core.fallbackMimeType */
25
94
  fallbackMimeType?: string;
26
- /** loader selection, avoid searching registered loaders */
95
+ /** @deprecated Use options.core.ignoreRegisteredLoaders */
27
96
  ignoreRegisteredLoaders?: boolean;
28
-
29
- // general
30
- /** Experimental: Supply a logger to the parser */
97
+ /** @deprecated Use options.core.nothrow */
98
+ nothrow?: boolean;
99
+ /** @deprecated Use options.core.log */
31
100
  log?: any;
32
-
33
- // batched parsing
34
-
35
- /** Size of each batch. `auto` matches batches to size of incoming chunks */
36
- batchSize?: number | 'auto';
37
- /** Minimal amount of time between batches */
38
- batchDebounceMs?: number;
39
- /** Stop loading after a given number of rows (compare SQL limit clause) */
40
- limit?: 0;
41
- /** Experimental: Stop loading after reaching */
42
- _limitMB?: 0;
43
- /** Generate metadata batches */
44
- metadata?: boolean;
45
- /** Transforms to run on incoming batches */
46
- transforms?: TransformBatches[];
47
-
48
- // module loading
49
-
50
- /** Any additional JS libraries */
51
- modules?: Record<string, any>;
52
- /** Force to load WASM libraries from local file system in NodeJS or from loaders.gl CDN in a web browser */
101
+ /** @deprecated Use options.core.useLocalLibraries */
53
102
  useLocalLibraries?: boolean;
54
-
55
- // workers
56
-
57
- /** CDN load workers from */
103
+ /** @deprecated Use options.core.CDN */
58
104
  CDN?: string | null;
59
- /** Set to `false` to disable workers */
105
+ /** @deprecated Use options.core.worker */
60
106
  worker?: boolean;
61
- /** Number of concurrent workers (per loader) on desktop browser */
107
+ /** @deprecated Use options.core.maxConcurrency */
62
108
  maxConcurrency?: number;
63
- /** Number of concurrent workers (per loader) on mobile browsers */
109
+ /** @deprecated Use options.core.maxMobileConcurrency */
64
110
  maxMobileConcurrency?: number;
65
- /** Set to `false` to prevent reuse workers */
111
+ /** @deprecated Use options.core.reuseWorkers */
66
112
  reuseWorkers?: boolean;
67
- /** Whether to use workers under Node.js (experimental) */
113
+ /** @deprecated Use options.core._nodeWorkers */
68
114
  _nodeWorkers?: boolean;
69
- /** set to 'test' to run local worker */
115
+ /** @deprecated Use options.core._workerType */
70
116
  _workerType?: string;
71
-
72
- /** @deprecated `options.batchType` removed, Use `options.<loader>.type` instead */
73
- batchType?: 'row' | 'columnar' | 'arrow';
74
- /** @deprecated `options.throw removed`, Use `options.nothrow` instead */
75
- throws?: boolean;
76
- /** @deprecated `options.dataType` no longer used */
77
- dataType?: never;
78
- /** @deprecated `options.uri` no longer used */
79
- uri?: never;
80
- /** @deprecated Use `options.fetch.method` */
81
- method?: never;
82
- /** @deprecated Use `options.fetch.headers` */
83
- headers?: never;
84
- /** @deprecated Use `options.fetch.body` */
85
- body?: never;
86
- /** @deprecated Use `options.fetch.mode` */
87
- mode?: never;
88
- /** @deprecated Use `options.fetch.credentials` */
89
- credentials?: never;
90
- /** @deprecated Use `options.fetch.cache` */
91
- cache?: never;
92
- /** @deprecated Use `options.fetch.redirect` */
93
- redirect?: never;
94
- /** @deprecated Use `options.fetch.referrer` */
95
- referrer?: never;
96
- /** @deprecated Use `options.fetch.referrerPolicy` */
97
- referrerPolicy?: never;
98
- /** @deprecated Use `options.fetch.integrity` */
99
- integrity?: never;
100
- /** @deprecated Use `options.fetch.keepalive` */
101
- keepalive?: never;
102
- /** @deprecated Use `options.fetch.signal` */
103
- signal?: never;
117
+ /** @deprecated Use options.core._workerType */
118
+ _worker?: string;
119
+ /** @deprecated Use options.core.limit */
120
+ limit?: 0;
121
+ /** @deprecated Use options.core._limitMB */
122
+ _limitMB?: 0;
123
+ /** @deprecated Use options.core.batchSize */
124
+ batchSize?: number | 'auto';
125
+ /** @deprecated Use options.core.batchDebounceMs */
126
+ batchDebounceMs?: number;
127
+ /** @deprecated Use options.core.metadata */
128
+ metadata?: boolean;
129
+ /** @deprecated Use options.core.transforms */
130
+ transforms?: TransformBatches[];
104
131
 
105
132
  // Accept other keys (loader options objects, e.g. `options.csv`, `options.json` ...)
106
133
  [loaderId: string]: unknown;
@@ -113,7 +140,7 @@ type PreloadOptions = {
113
140
  /**
114
141
  * A worker loader definition that can be used with `@loaders.gl/core` functions
115
142
  */
116
- export type Loader<DataT = any, BatchT = any, LoaderOptionsT = LoaderOptions> = {
143
+ export type Loader<DataT = any, BatchT = any, LoaderOptionsT = StrictLoaderOptions> = Format & {
117
144
  /** The result type of this loader */
118
145
  dataType?: DataT;
119
146
  /** The batched result type of this loader */
@@ -123,6 +150,11 @@ export type Loader<DataT = any, BatchT = any, LoaderOptionsT = LoaderOptions> =
123
150
  options: LoaderOptionsT;
124
151
  /** Deprecated Options */
125
152
  deprecatedOptions?: Record<string, string | Record<string, string>>;
153
+ /** Version should be injected by build tools */
154
+ version: string;
155
+ /** A boolean, or a URL */
156
+ worker?: string | boolean;
157
+ // end Worker
126
158
 
127
159
  /** Human readable name */
128
160
  name: string;
@@ -130,19 +162,12 @@ export type Loader<DataT = any, BatchT = any, LoaderOptionsT = LoaderOptions> =
130
162
  id: string;
131
163
  /** module is used to generate worker threads, need to be the module directory name */
132
164
  module: string;
133
- /** Version should be injected by build tools */
134
- version: string;
135
- /** A boolean, or a URL */
136
- worker?: string | boolean;
137
- // end Worker
138
-
139
165
  /** Which category does this loader belong to */
140
166
  category?: string;
141
167
  /** File extensions that are potential matches with this loader. */
142
168
  extensions: string[];
143
169
  /** MIMETypes that indicate a match with this loader. @note Some MIMETypes are generic and supported by many loaders */
144
170
  mimeTypes: string[];
145
-
146
171
  /** Is the input of this loader binary */
147
172
  binary?: boolean;
148
173
  /** Is the input of this loader text */
@@ -161,11 +186,11 @@ export type Loader<DataT = any, BatchT = any, LoaderOptionsT = LoaderOptions> =
161
186
  * A "bundled" loader definition that can be used with `@loaders.gl/core` functions
162
187
  * If a worker loader is supported it will also be supported.
163
188
  */
164
- export type LoaderWithParser<DataT = any, BatchT = any, LoaderOptionsT = LoaderOptions> = Loader<
165
- DataT,
166
- BatchT,
167
- LoaderOptionsT
168
- > & {
189
+ export type LoaderWithParser<
190
+ DataT = any,
191
+ BatchT = any,
192
+ LoaderOptionsT = StrictLoaderOptions
193
+ > = Loader<DataT, BatchT, LoaderOptionsT> & {
169
194
  /** Perform actions before load. @deprecated Not officially supported. */
170
195
  preload?: Preload;
171
196
  /** Parse asynchronously and atomically from an arraybuffer */
@@ -188,7 +213,9 @@ export type LoaderWithParser<DataT = any, BatchT = any, LoaderOptionsT = LoaderO
188
213
  ) => DataT;
189
214
  /** Parse batches of data from an iterator (e.g. fetch stream), return an iterator that yield parsed batches. */
190
215
  parseInBatches?: (
191
- iterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
216
+ iterator:
217
+ | AsyncIterable<ArrayBufferLike | ArrayBufferView>
218
+ | Iterable<ArrayBufferLike | ArrayBufferView>,
192
219
  options?: LoaderOptionsT,
193
220
  context?: LoaderContext
194
221
  ) => AsyncIterable<BatchT>;
@@ -295,12 +322,13 @@ type Preload = (url: string, options?: PreloadOptions) => any;
295
322
 
296
323
  /** Typescript helper to extract options type from a loader type */
297
324
  export type LoaderOptionsType<T = Loader> =
298
- T extends Loader<any, any, infer Options> ? Options : never;
325
+ T extends Loader<unknown, unknown, infer Options> ? Options : never;
299
326
  /** Typescript helper to extract data type from a loader type */
300
327
  export type LoaderReturnType<T = Loader> =
301
- T extends Loader<infer Return, any, any> ? Return : never;
328
+ T extends Loader<infer Return, unknown, unknown> ? Return : never;
302
329
  /** Typescript helper to extract batch type from a loader type */
303
- export type LoaderBatchType<T = Loader> = T extends Loader<any, infer Batch, any> ? Batch : never;
330
+ export type LoaderBatchType<T = Loader> =
331
+ T extends Loader<unknown, infer Batch, unknown> ? Batch : never;
304
332
 
305
333
  /** Typescript helper to extract options type from an array of loader types */
306
334
  export type LoaderArrayOptionsType<LoadersT extends Loader[] = Loader[]> =
@@ -317,7 +345,7 @@ export type LoaderArrayBatchType<LoadersT extends Loader[] = Loader[]> =
317
345
  */
318
346
  export async function parseFromContext<
319
347
  LoaderT extends Loader,
320
- OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
348
+ OptionsT extends StrictLoaderOptions = LoaderOptionsType<LoaderT>
321
349
  >(
322
350
  data: ArrayBuffer,
323
351
  loader: LoaderT,
@@ -357,7 +385,7 @@ export async function parseFromContext(
357
385
  */
358
386
  export function parseSyncFromContext<
359
387
  LoaderT extends Loader,
360
- OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
388
+ OptionsT extends StrictLoaderOptions = LoaderOptionsType<LoaderT>
361
389
  >(
362
390
  data: ArrayBuffer,
363
391
  loader: LoaderT,
@@ -375,7 +403,7 @@ export function parseSyncFromContext<
375
403
  */
376
404
  export async function parseInBatchesFromContext<
377
405
  LoaderT extends Loader,
378
- OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
406
+ OptionsT extends StrictLoaderOptions = LoaderOptionsType<LoaderT>
379
407
  >(
380
408
  data: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer> | Response,
381
409
  loader: LoaderT,