@loaders.gl/scientific 5.0.0-beta.1 → 5.0.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/README.md +5 -2
  2. package/dist/index.cjs +78 -4110
  3. package/dist/index.cjs.map +4 -3
  4. package/dist/index.d.ts +4 -0
  5. package/dist/index.d.ts.map +1 -1
  6. package/dist/index.js +2 -0
  7. package/dist/index.js.map +1 -1
  8. package/dist/lib/parsers/parse-npy.d.ts +15 -0
  9. package/dist/lib/parsers/parse-npy.d.ts.map +1 -0
  10. package/dist/lib/parsers/parse-npy.js +81 -0
  11. package/dist/lib/parsers/parse-npy.js.map +1 -0
  12. package/dist/netcdf/bundled.cjs +1 -2
  13. package/dist/netcdf/index.cjs +1 -2
  14. package/dist/netcdf/index.cjs.map +2 -2
  15. package/dist/netcdf/netcdf-loader-with-parser.cjs +1 -2
  16. package/dist/netcdf/netcdf-loader-with-parser.js +1 -1
  17. package/dist/netcdf/netcdf-loader-with-parser.js.map +1 -1
  18. package/dist/netcdf/netcdf-loader.cjs +5 -4
  19. package/dist/netcdf/netcdf-loader.cjs.map +7 -0
  20. package/dist/netcdf/netcdf-loader.js +1 -1
  21. package/dist/netcdf/netcdf-source-loader-types.js +1 -1
  22. package/dist/netcdf/netcdf-source-loader.js +1 -1
  23. package/dist/netcdf/unbundled.cjs +1 -2
  24. package/dist/netcdf.min.js +3 -3
  25. package/dist/npy-format.d.ts +12 -0
  26. package/dist/npy-format.d.ts.map +1 -0
  27. package/dist/npy-format.js +15 -0
  28. package/dist/npy-format.js.map +1 -0
  29. package/dist/npy-loader-types.cjs +80 -0
  30. package/dist/npy-loader-types.cjs.map +7 -0
  31. package/dist/npy-loader-types.d.ts +73 -0
  32. package/dist/npy-loader-types.d.ts.map +1 -0
  33. package/dist/npy-loader-types.js +34 -0
  34. package/dist/npy-loader-types.js.map +1 -0
  35. package/dist/npy-loader.cjs +273 -0
  36. package/dist/npy-loader.cjs.map +6 -0
  37. package/dist/npy-loader.d.ts +27 -0
  38. package/dist/npy-loader.d.ts.map +1 -0
  39. package/dist/npy-loader.js +15 -0
  40. package/dist/npy-loader.js.map +1 -0
  41. package/dist/npy-worker.d.ts +2 -0
  42. package/dist/npy-worker.d.ts.map +1 -0
  43. package/dist/npy-worker.js +1057 -0
  44. package/dist/npy-worker.js.map +1 -0
  45. package/dist/vtk/index.cjs +4 -4118
  46. package/dist/vtk/index.cjs.map +2 -2
  47. package/dist/vtk/vtk-xml.js +4 -4
  48. package/dist/vtk/vtk-xml.js.map +1 -1
  49. package/dist/workers/npy-worker.d.ts +2 -0
  50. package/dist/workers/npy-worker.d.ts.map +1 -0
  51. package/dist/workers/npy-worker.js +7 -0
  52. package/dist/workers/npy-worker.js.map +1 -0
  53. package/dist/zarr/geo-zarr-source-loader.d.ts.map +1 -1
  54. package/dist/zarr/geo-zarr-source-loader.js +162 -43
  55. package/dist/zarr/geo-zarr-source-loader.js.map +1 -1
  56. package/dist/zarr/index.cjs +194 -43
  57. package/dist/zarr/index.cjs.map +2 -2
  58. package/dist/zarr/lib/spatial-multiscales.d.ts +13 -2
  59. package/dist/zarr/lib/spatial-multiscales.d.ts.map +1 -1
  60. package/dist/zarr/lib/spatial-multiscales.js +94 -8
  61. package/dist/zarr/lib/spatial-multiscales.js.map +1 -1
  62. package/dist/zarr/ome-zarr-source-loader.js +1 -1
  63. package/dist/zarr/zarr-loader-types.cjs +161 -0
  64. package/dist/zarr/zarr-loader-types.cjs.map +7 -0
  65. package/dist/zarr/zarr-loader-types.js +1 -1
  66. package/dist/zarr.dev.js +194 -43
  67. package/dist/zarr.min.js +40 -40
  68. package/package.json +42 -7
  69. package/src/index.ts +5 -0
  70. package/src/lib/parsers/parse-npy.ts +123 -0
  71. package/src/netcdf/netcdf-loader-with-parser.ts +1 -1
  72. package/src/npy-format.ts +17 -0
  73. package/src/npy-loader-types.ts +50 -0
  74. package/src/npy-loader.ts +21 -0
  75. package/src/npy-worker.ts +6 -0
  76. package/src/vtk/vtk-xml.ts +4 -4
  77. package/src/workers/npy-worker.ts +8 -0
  78. package/src/zarr/geo-zarr-source-loader.ts +195 -44
  79. package/src/zarr/lib/spatial-multiscales.ts +104 -8
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@loaders.gl/scientific",
3
- "version": "5.0.0-beta.1",
4
- "description": "Scientific data format loaders with metadata roots and explicit format subpaths",
3
+ "version": "5.0.0-beta.4",
4
+ "description": "Scientific NumPy arrays and sources for NetCDF, Zarr, GRIB2, HDF5, and VTK",
5
5
  "license": "MIT",
6
6
  "type": "module",
7
7
  "publishConfig": {
@@ -63,6 +63,19 @@
63
63
  "types": "./dist/vector-field/index.d.ts",
64
64
  "import": "./dist/vector-field/index.js",
65
65
  "require": "./dist/vector-field/index.cjs"
66
+ },
67
+ "./npy-worker.js": {
68
+ "import": "./dist/npy-worker.js"
69
+ },
70
+ "./npy-loader": {
71
+ "types": "./dist/npy-loader.d.ts",
72
+ "import": "./dist/npy-loader.js",
73
+ "require": "./dist/npy-loader.cjs"
74
+ },
75
+ "./npy-loader-types": {
76
+ "types": "./dist/npy-loader-types.d.ts",
77
+ "import": "./dist/npy-loader-types.js",
78
+ "require": "./dist/npy-loader-types.cjs"
66
79
  }
67
80
  },
68
81
  "sideEffects": false,
@@ -75,7 +88,7 @@
75
88
  "third-party-licenses"
76
89
  ],
77
90
  "scripts": {
78
- "pre-build": "npm run build-bundle && npm run build-bundle-dev && npm run build-cjs-netcdf && npm run build-cjs-zarr && npm run build-cjs-grib2 && npm run build-cjs-hdf5 && npm run build-cjs-vtk && npm run build-cjs-vector-field",
91
+ "pre-build": "npm run build-bundle && npm run build-bundle-dev && npm run build-cjs-netcdf && npm run build-cjs-zarr && npm run build-cjs-grib2 && npm run build-cjs-hdf5 && npm run build-cjs-vtk && npm run build-cjs-vector-field && npm run build-npy-worker && npm run build-metadata",
79
92
  "build-bundle": "ocular-bundle ./netcdf-bundle.ts --output=dist/netcdf.min.js && ocular-bundle ./zarr-bundle.ts --output=dist/zarr.min.js",
80
93
  "build-bundle-dev": "ocular-bundle ./netcdf-bundle.ts --env=dev --output=dist/netcdf.dev.js && ocular-bundle ./zarr-bundle.ts --env=dev --output=dist/zarr.dev.js",
81
94
  "build-cjs-netcdf": "esbuild src/netcdf/*.ts --bundle --format=cjs --platform=node --target=node22 --outdir=dist/netcdf --out-extension:.js=.cjs --external:@loaders.gl/* --external:@math.gl/*",
@@ -83,11 +96,14 @@
83
96
  "build-cjs-grib2": "esbuild src/grib2/index.ts src/scientific-array.ts --bundle --format=cjs --platform=node --target=node22 --outdir=dist --out-extension:.js=.cjs --external:@loaders.gl/*",
84
97
  "build-cjs-hdf5": "esbuild src/hdf5/index.ts --bundle --format=cjs --platform=node --target=node22 --outfile=dist/hdf5/index.cjs --external:@loaders.gl/*",
85
98
  "build-cjs-vtk": "esbuild src/vtk/index.ts --bundle --format=cjs --platform=node --target=node22 --outfile=dist/vtk/index.cjs --external:@loaders.gl/*",
86
- "build-cjs-vector-field": "esbuild src/vector-field/index.ts --bundle --format=cjs --platform=node --target=node22 --outfile=dist/vector-field/index.cjs --external:@loaders.gl/*"
99
+ "build-cjs-vector-field": "esbuild src/vector-field/index.ts --bundle --format=cjs --platform=node --target=node22 --outfile=dist/vector-field/index.cjs --external:@loaders.gl/*",
100
+ "build-npy-worker": "esbuild src/workers/npy-worker.ts --outfile=dist/npy-worker.js --target=esnext --bundle --define:__VERSION__=\\\"$npm_package_version\\\"",
101
+ "build-metadata": "node ../../scripts/build-scientific-metadata.mjs"
87
102
  },
88
103
  "dependencies": {
89
- "@math.gl/geospatial": "5.0.0-beta.1",
90
- "fast-xml-parser": "^5.3.6",
104
+ "@loaders.gl/documents": "5.0.0-beta.4",
105
+ "@math.gl/core": "5.0.0-beta.4",
106
+ "@math.gl/geospatial": "5.0.0-beta.4",
91
107
  "h5wasm": "^0.10.3",
92
108
  "zarr": "^0.5.0",
93
109
  "zarrita": "~0.7.1"
@@ -95,5 +111,24 @@
95
111
  "peerDependencies": {
96
112
  "@loaders.gl/core": "~5.0.0-alpha.0"
97
113
  },
98
- "gitHead": "89312301ed9e8ec89a58501c052a38284d1570c1"
114
+ "keywords": [
115
+ "loaders.gl",
116
+ "NetCDF",
117
+ "NetCDF-4",
118
+ "Zarr",
119
+ "GeoZarr",
120
+ "OME-Zarr",
121
+ "SpatialData",
122
+ "GRIB2",
123
+ "HDF5",
124
+ "VTK",
125
+ "VTI",
126
+ "VTR",
127
+ "VTS",
128
+ "VTU",
129
+ "VTP",
130
+ "cdf",
131
+ "nc"
132
+ ],
133
+ "gitHead": "c9420f06e870fa353889d55b684e3588d24574c2"
99
134
  }
package/src/index.ts CHANGED
@@ -60,3 +60,8 @@ export type {
60
60
  ScientificSelection,
61
61
  ScientificSourceCapabilities
62
62
  } from './scientific-array';
63
+
64
+ export {NPYLoader, NPYWorkerLoader} from './npy-loader-types';
65
+ export type {NPYLoaderOptions} from './npy-loader-types';
66
+ export type {NPYTile} from './lib/parsers/parse-npy';
67
+ export {NPYFormat} from './npy-format';
@@ -0,0 +1,123 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ // import type {TextureLevel} from '@loaders.gl/core/schema';
6
+ import type {TypedArray} from '@math.gl/core';
7
+ // import {TypedArrayConstructor} from "@math.gl/core";
8
+
9
+ // TODO move to math.gl
10
+ type TypedArrayConstructor =
11
+ | typeof Int8Array
12
+ | typeof Uint8Array
13
+ | typeof Int16Array
14
+ | typeof Uint16Array
15
+ | typeof Int32Array
16
+ | typeof Uint32Array
17
+ | typeof Float32Array
18
+ | typeof Float64Array;
19
+
20
+ const a = new Uint32Array([0x12345678]);
21
+ const b = new Uint8Array(a.buffer, a.byteOffset, a.byteLength);
22
+ const isLittleEndian = !(b[0] === 0x12);
23
+
24
+ const LITTLE_ENDIAN_OS = isLittleEndian;
25
+
26
+ /** One numpy "tile" */
27
+ export type NPYTile = {
28
+ /** tile header */
29
+ header: NumpyHeader;
30
+ /** data in tile */
31
+ data: TypedArray;
32
+ };
33
+
34
+ type NumpyHeader = {
35
+ descr: string;
36
+ shape: number[];
37
+ };
38
+
39
+ /**
40
+ * The basic string format consists of 3 characters:
41
+ * 1. a character describing the byteorder of the data (<: little-endian, >: big-endian, |: not-relevant)
42
+ * 2. a character code giving the basic type of the array
43
+ * 3. an integer providing the number of bytes the type uses.
44
+ * https://numpy.org/doc/stable/reference/arrays.interface.html
45
+ *
46
+ * Here I only include the second and third characters, and check endianness separately
47
+ */
48
+ const DTYPES: Record<string, TypedArrayConstructor> = {
49
+ u1: Uint8Array,
50
+ i1: Int8Array,
51
+ u2: Uint16Array,
52
+ i2: Int16Array,
53
+ u4: Uint32Array,
54
+ i4: Int32Array,
55
+ f4: Float32Array,
56
+ f8: Float64Array
57
+ };
58
+
59
+ export function parseNPY(arrayBuffer: ArrayBuffer, options?: {}): NPYTile {
60
+ const view = new DataView(arrayBuffer);
61
+ const {header, headerEndOffset} = parseHeader(view);
62
+
63
+ const numpyType = header.descr;
64
+ const ArrayType = DTYPES[numpyType.slice(1, 3)];
65
+ if (!ArrayType) {
66
+ throw new Error(`Unimplemented type ${numpyType}`);
67
+ }
68
+
69
+ const nArrayElements = header.shape?.reduce((a: number, b: number): number => a * b);
70
+ const arrayByteLength = nArrayElements * ArrayType.BYTES_PER_ELEMENT;
71
+
72
+ if (arrayBuffer.byteLength < headerEndOffset + arrayByteLength) {
73
+ throw new Error('Buffer overflow');
74
+ }
75
+ const data = new ArrayType(arrayBuffer.slice(headerEndOffset, headerEndOffset + arrayByteLength));
76
+
77
+ // Swap endianness if needed
78
+ if ((numpyType[0] === '>' && LITTLE_ENDIAN_OS) || (numpyType[0] === '<' && !LITTLE_ENDIAN_OS)) {
79
+ throw new Error('Incorrect endianness');
80
+ }
81
+
82
+ return {
83
+ data,
84
+ header
85
+ };
86
+ }
87
+
88
+ /**
89
+ * Parse NPY header
90
+ *
91
+ * @param view
92
+ * @return
93
+ */
94
+ function parseHeader(view: DataView): {header: NumpyHeader; headerEndOffset: number} {
95
+ const majorVersion = view.getUint8(6);
96
+ // const minorVersion = view.getUint8(7);
97
+
98
+ let offset = 8;
99
+ let headerLength: number;
100
+ if (majorVersion >= 2) {
101
+ headerLength = view.getUint32(offset, true);
102
+ offset += 4;
103
+ } else {
104
+ headerLength = view.getUint16(offset, true);
105
+ offset += 2;
106
+ }
107
+
108
+ const encoding = majorVersion <= 2 ? 'latin1' : 'utf-8';
109
+ const decoder = new TextDecoder(encoding);
110
+ const headerArray = new Uint8Array(view.buffer, offset, headerLength);
111
+ const headerText = decoder.decode(headerArray);
112
+ offset += headerLength;
113
+
114
+ const header = JSON.parse(
115
+ headerText
116
+ .replace(/'/g, '"')
117
+ .replace('False', 'false')
118
+ .replace('(', '[')
119
+ .replace(/,*\),*/g, ']')
120
+ );
121
+
122
+ return {header, headerEndOffset: offset};
123
+ }
@@ -5,7 +5,7 @@
5
5
  import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/core/loader-utils';
6
6
  import type {NetCDFHeader} from './netcdfjs/netcdf-types';
7
7
  import {NetCDFReader} from './netcdfjs/netcdf-reader';
8
- import {NetCDFWorkerLoader as NetCDFWorkerLoaderMetadata} from './netcdf-loader';
8
+ import {NetCDFLoader as NetCDFWorkerLoaderMetadata} from './netcdf-loader';
9
9
  import {NetCDFLoader as NetCDFLoaderMetadata} from './netcdf-loader';
10
10
 
11
11
  const {preload: _NetCDFWorkerLoaderPreload, ...NetCDFWorkerLoaderMetadataWithoutPreload} =
@@ -0,0 +1,17 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import type {Format} from '@loaders.gl/core/loader-utils';
6
+
7
+ /** NumPy binary array format. */
8
+ export const NPYFormat = {
9
+ name: 'NPY',
10
+ id: 'npy',
11
+ module: 'scientific',
12
+ encoding: 'binary',
13
+ format: 'npy',
14
+ extensions: ['npy'],
15
+ mimeTypes: [],
16
+ binary: true
17
+ } as const satisfies Format;
@@ -0,0 +1,50 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import type {Loader, LoaderOptions} from '@loaders.gl/core/loader-utils';
6
+ import {NPYFormat} from './npy-format';
7
+ // @ts-ignore __VERSION__ is injected by the package build.
8
+ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
9
+ import type {NPYTile} from './lib/parsers/parse-npy';
10
+
11
+ // \x93NUMPY
12
+ const NPY_MAGIC_NUMBER = new Uint8Array([147, 78, 85, 77, 80, 89]);
13
+
14
+ /** NPYLoader for numpy tiles */
15
+ export type NPYLoaderOptions = LoaderOptions & {
16
+ /** NPYLoader for numpy tiles */
17
+ npy?: {
18
+ /** Override the URL to the worker bundle (by default loads from unpkg.com) */
19
+ workerUrl?: string;
20
+ };
21
+ };
22
+
23
+ /** Preloads the parser-bearing NPY loader implementation. */
24
+ async function preload() {
25
+ const {NPYLoaderWithParser} = await import('@loaders.gl/scientific/npy-loader');
26
+ return NPYLoaderWithParser;
27
+ }
28
+
29
+ /** Metadata-only worker loader for numpy "tiles". */
30
+ export const NPYLoader = {
31
+ ...NPYFormat,
32
+ dataType: null as any as NPYTile,
33
+ batchType: null as never,
34
+
35
+ name: 'NPY',
36
+ id: 'npy',
37
+ module: 'scientific',
38
+ version: VERSION,
39
+ worker: true,
40
+ extensions: ['npy'],
41
+ mimeTypes: [],
42
+ tests: [NPY_MAGIC_NUMBER.buffer],
43
+ options: {
44
+ npy: {}
45
+ },
46
+ preload
47
+ } as const satisfies Loader<NPYTile, never, NPYLoaderOptions>;
48
+
49
+ /** @deprecated Use NPYLoader. */
50
+ export const NPYWorkerLoader = NPYLoader;
@@ -0,0 +1,21 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/core/loader-utils';
6
+ import {parseNPY, type NPYTile} from './lib/parsers/parse-npy';
7
+ import type {NPYLoaderOptions} from './npy-loader-types';
8
+ import {NPYLoader as NPYLoaderMetadata} from './npy-loader-types';
9
+
10
+ const {preload: _preload, ...metadata} = NPYLoaderMetadata;
11
+
12
+ export type {NPYLoaderOptions} from './npy-loader-types';
13
+
14
+ /**
15
+ * Loader for numpy "tiles"
16
+ */
17
+ export const NPYLoaderWithParser = {
18
+ ...metadata,
19
+ parseSync: parseNPY,
20
+ parse: async (arrayBuffer: ArrayBuffer, options?: LoaderOptions) => parseNPY(arrayBuffer, options)
21
+ } as const satisfies LoaderWithParser<NPYTile, never, NPYLoaderOptions>;
@@ -0,0 +1,6 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ // Classic-worker entrypoint that lazily loads the ESM worker implementation.
6
+ import(/* @vite-ignore */ new URL('./workers/npy-worker.js', import.meta.url).toString());
@@ -2,7 +2,7 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import {XMLParser} from 'fast-xml-parser';
5
+ import {parseXMLSync} from '@loaders.gl/documents/xml-parser';
6
6
 
7
7
  /** Numeric arrays supported by the VTK XML reader without precision loss. */
8
8
  export type VTKXMLNumericArray =
@@ -175,7 +175,7 @@ function extentDimensions(value: unknown): [number, number, number] | undefined
175
175
  */
176
176
  export function parseVTKXML(xml: string): VTKXMLDataset {
177
177
  check(xml.length <= 512 * 1024 * 1024, 'XML document too large');
178
- const parser = new XMLParser({
178
+ const XML_OPTIONS = {
179
179
  ignoreAttributes: false,
180
180
  attributeNamePrefix: '',
181
181
  textNodeName: '#text',
@@ -183,8 +183,8 @@ export function parseVTKXML(xml: string): VTKXMLDataset {
183
183
  parseAttributeValue: false,
184
184
  processEntities: false,
185
185
  trimValues: true
186
- });
187
- const root = parser.parse(xml)?.VTKFile as Element | undefined;
186
+ };
187
+ const root = parseXMLSync(xml, XML_OPTIONS)?.VTKFile as Element | undefined;
188
188
  check(root, 'missing VTKFile root');
189
189
  const type = String(root.type ?? '');
190
190
  check(
@@ -0,0 +1,8 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import {NPYLoaderWithParser} from '../npy-loader';
6
+ import {createLoaderWorker} from '@loaders.gl/core/loader-utils';
7
+
8
+ createLoaderWorker(NPYLoaderWithParser);