@loaders.gl/arrow 3.1.3 → 4.0.0-alpha.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/dist/arrow-loader.js +18 -23
  2. package/dist/arrow-loader.js.map +1 -0
  3. package/dist/arrow-worker.js +1 -1
  4. package/dist/arrow-writer.js +15 -19
  5. package/dist/arrow-writer.js.map +1 -0
  6. package/dist/bundle.js +2 -2
  7. package/dist/bundle.js.map +1 -0
  8. package/dist/index.js +15 -27
  9. package/dist/index.js.map +1 -0
  10. package/dist/lib/arrow-table-batch.js +63 -54
  11. package/dist/lib/arrow-table-batch.js.map +1 -0
  12. package/dist/lib/encode-arrow.js +25 -37
  13. package/dist/lib/encode-arrow.js.map +1 -0
  14. package/dist/lib/parse-arrow-in-batches.js +25 -42
  15. package/dist/lib/parse-arrow-in-batches.js.map +1 -0
  16. package/dist/lib/parse-arrow-sync.js +40 -37
  17. package/dist/lib/parse-arrow-sync.js.map +1 -0
  18. package/dist/types.js +6 -7
  19. package/dist/types.js.map +1 -0
  20. package/dist/workers/arrow-worker.js +4 -5
  21. package/dist/workers/arrow-worker.js.map +1 -0
  22. package/package.json +7 -10
  23. package/dist/es5/arrow-loader.js +0 -29
  24. package/dist/es5/arrow-loader.js.map +0 -1
  25. package/dist/es5/arrow-writer.js +0 -27
  26. package/dist/es5/arrow-writer.js.map +0 -1
  27. package/dist/es5/bundle.js +0 -7
  28. package/dist/es5/bundle.js.map +0 -1
  29. package/dist/es5/index.js +0 -84
  30. package/dist/es5/index.js.map +0 -1
  31. package/dist/es5/lib/arrow-table-batch.js +0 -129
  32. package/dist/es5/lib/arrow-table-batch.js.map +0 -1
  33. package/dist/es5/lib/encode-arrow.js +0 -54
  34. package/dist/es5/lib/encode-arrow.js.map +0 -1
  35. package/dist/es5/lib/parse-arrow-in-batches.js +0 -198
  36. package/dist/es5/lib/parse-arrow-in-batches.js.map +0 -1
  37. package/dist/es5/lib/parse-arrow-sync.js +0 -53
  38. package/dist/es5/lib/parse-arrow-sync.js.map +0 -1
  39. package/dist/es5/types.js +0 -14
  40. package/dist/es5/types.js.map +0 -1
  41. package/dist/es5/workers/arrow-worker.js +0 -8
  42. package/dist/es5/workers/arrow-worker.js.map +0 -1
  43. package/dist/esm/arrow-loader.js +0 -21
  44. package/dist/esm/arrow-loader.js.map +0 -1
  45. package/dist/esm/arrow-writer.js +0 -18
  46. package/dist/esm/arrow-writer.js.map +0 -1
  47. package/dist/esm/bundle.js +0 -5
  48. package/dist/esm/bundle.js.map +0 -1
  49. package/dist/esm/index.js +0 -16
  50. package/dist/esm/index.js.map +0 -1
  51. package/dist/esm/lib/arrow-table-batch.js +0 -71
  52. package/dist/esm/lib/arrow-table-batch.js.map +0 -1
  53. package/dist/esm/lib/encode-arrow.js +0 -28
  54. package/dist/esm/lib/encode-arrow.js.map +0 -1
  55. package/dist/esm/lib/parse-arrow-in-batches.js +0 -30
  56. package/dist/esm/lib/parse-arrow-in-batches.js.map +0 -1
  57. package/dist/esm/lib/parse-arrow-sync.js +0 -44
  58. package/dist/esm/lib/parse-arrow-sync.js.map +0 -1
  59. package/dist/esm/types.js +0 -7
  60. package/dist/esm/types.js.map +0 -1
  61. package/dist/esm/workers/arrow-worker.js +0 -4
  62. package/dist/esm/workers/arrow-worker.js.map +0 -1
@@ -1,26 +1,21 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports._typecheckArrowLoader = exports.ArrowLoader = void 0;
4
- // __VERSION__ is injected by babel-plugin-version-inline
5
- // @ts-ignore TS2304: Cannot find name '__VERSION__'.
6
- const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
1
+ const VERSION = typeof "4.0.0-alpha.5" !== 'undefined' ? "4.0.0-alpha.5" : 'latest';
7
2
  const DEFAULT_ARROW_LOADER_OPTIONS = {
8
- arrow: {
9
- shape: 'columnar-table'
10
- }
3
+ arrow: {
4
+ shape: 'columnar-table'
5
+ }
11
6
  };
12
- /** ArrowJS table loader */
13
- exports.ArrowLoader = {
14
- name: 'Apache Arrow',
15
- id: 'arrow',
16
- module: 'arrow',
17
- version: VERSION,
18
- worker: true,
19
- category: 'table',
20
- extensions: ['arrow', 'feather'],
21
- mimeTypes: ['application/octet-stream'],
22
- binary: true,
23
- tests: ['ARROW'],
24
- options: DEFAULT_ARROW_LOADER_OPTIONS
7
+ export const ArrowLoader = {
8
+ name: 'Apache Arrow',
9
+ id: 'arrow',
10
+ module: 'arrow',
11
+ version: VERSION,
12
+ worker: true,
13
+ category: 'table',
14
+ extensions: ['arrow', 'feather'],
15
+ mimeTypes: ['application/octet-stream'],
16
+ binary: true,
17
+ tests: ['ARROW'],
18
+ options: DEFAULT_ARROW_LOADER_OPTIONS
25
19
  };
26
- exports._typecheckArrowLoader = exports.ArrowLoader;
20
+ export const _typecheckArrowLoader = ArrowLoader;
21
+ //# sourceMappingURL=arrow-loader.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/arrow-loader.ts"],"names":["VERSION","DEFAULT_ARROW_LOADER_OPTIONS","arrow","shape","ArrowLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckArrowLoader"],"mappings":"AAIA,MAAMA,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AAQA,MAAMC,4BAA4B,GAAG;AACnCC,EAAAA,KAAK,EAAE;AACLC,IAAAA,KAAK,EAAE;AADF;AAD4B,CAArC;AAOA,OAAO,MAAMC,WAAW,GAAG;AACzBC,EAAAA,IAAI,EAAE,cADmB;AAEzBC,EAAAA,EAAE,EAAE,OAFqB;AAGzBC,EAAAA,MAAM,EAAE,OAHiB;AAIzBC,EAAAA,OAAO,EAAER,OAJgB;AAKzBS,EAAAA,MAAM,EAAE,IALiB;AAMzBC,EAAAA,QAAQ,EAAE,OANe;AAOzBC,EAAAA,UAAU,EAAE,CAAC,OAAD,EAAU,SAAV,CAPa;AAQzBC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CARc;AASzBC,EAAAA,MAAM,EAAE,IATiB;AAUzBC,EAAAA,KAAK,EAAE,CAAC,OAAD,CAVkB;AAWzBC,EAAAA,OAAO,EAAEd;AAXgB,CAApB;AAcP,OAAO,MAAMe,qBAA6B,GAAGZ,WAAtC","sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ArrowLoaderOptions = LoaderOptions & {\n arrow?: {\n shape: 'arrow-table' | 'columnar-table' | 'row-table' | 'array-row-table' | 'object-row-table';\n };\n};\n\nconst DEFAULT_ARROW_LOADER_OPTIONS = {\n arrow: {\n shape: 'columnar-table'\n }\n};\n\n/** ArrowJS table loader */\nexport const ArrowLoader = {\n name: 'Apache Arrow',\n id: 'arrow',\n module: 'arrow',\n version: VERSION,\n worker: true,\n category: 'table',\n extensions: ['arrow', 'feather'],\n mimeTypes: ['application/octet-stream'],\n binary: true,\n tests: ['ARROW'],\n options: DEFAULT_ARROW_LOADER_OPTIONS\n};\n\nexport const _typecheckArrowLoader: Loader = ArrowLoader;\n"],"file":"arrow-loader.js"}
@@ -14940,7 +14940,7 @@ return true;`);
14940
14940
  }
14941
14941
 
14942
14942
  // src/arrow-loader.ts
14943
- var VERSION = typeof __VERSION__ !== "undefined" ? __VERSION__ : "latest";
14943
+ var VERSION = true ? "4.0.0-alpha.5" : "latest";
14944
14944
  var DEFAULT_ARROW_LOADER_OPTIONS = {
14945
14945
  arrow: {
14946
14946
  shape: "columnar-table"
@@ -1,22 +1,18 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ArrowWriter = void 0;
4
- const encode_arrow_1 = require("./lib/encode-arrow");
5
- // __VERSION__ is injected by babel-plugin-version-inline
6
- // @ts-ignore TS2304: Cannot find name '__VERSION__'.
7
- const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
8
- /** Apache Arrow writer */
9
- exports.ArrowWriter = {
10
- name: 'Apache Arrow',
11
- id: 'arrow',
12
- module: 'arrow',
13
- version: VERSION,
14
- extensions: ['arrow', 'feather'],
15
- mimeTypes: ['application/octet-stream'],
16
- encodeSync,
17
- binary: true,
18
- options: {}
1
+ import { encodeArrowSync } from './lib/encode-arrow';
2
+ const VERSION = typeof "4.0.0-alpha.5" !== 'undefined' ? "4.0.0-alpha.5" : 'latest';
3
+ export const ArrowWriter = {
4
+ name: 'Apache Arrow',
5
+ id: 'arrow',
6
+ module: 'arrow',
7
+ version: VERSION,
8
+ extensions: ['arrow', 'feather'],
9
+ mimeTypes: ['application/octet-stream'],
10
+ encodeSync,
11
+ binary: true,
12
+ options: {}
19
13
  };
14
+
20
15
  function encodeSync(data, options) {
21
- return (0, encode_arrow_1.encodeArrowSync)(data);
16
+ return encodeArrowSync(data);
22
17
  }
18
+ //# sourceMappingURL=arrow-writer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/arrow-writer.ts"],"names":["encodeArrowSync","VERSION","ArrowWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data"],"mappings":"AAEA,SAAQA,eAAR,QAA8B,oBAA9B;AAIA,MAAMC,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AAKA,OAAO,MAAMC,WAAW,GAAG;AACzBC,EAAAA,IAAI,EAAE,cADmB;AAEzBC,EAAAA,EAAE,EAAE,OAFqB;AAGzBC,EAAAA,MAAM,EAAE,OAHiB;AAIzBC,EAAAA,OAAO,EAAEL,OAJgB;AAKzBM,EAAAA,UAAU,EAAE,CAAC,OAAD,EAAU,SAAV,CALa;AAMzBC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CANc;AAOzBC,EAAAA,UAPyB;AAQzBC,EAAAA,MAAM,EAAE,IARiB;AASzBC,EAAAA,OAAO,EAAE;AATgB,CAApB;;AAYP,SAASF,UAAT,CAAoBG,IAApB,EAA0BD,OAA1B,EAAwD;AACtD,SAAOX,eAAe,CAACY,IAAD,CAAtB;AACD","sourcesContent":["// import type {Writer} from '@loaders.gl/loader-utils';\nimport type {WriterOptions} from '@loaders.gl/loader-utils';\nimport {encodeArrowSync} from './lib/encode-arrow';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\ntype ArrowWriterOptions = WriterOptions;\n\n/** Apache Arrow writer */\nexport const ArrowWriter = {\n name: 'Apache Arrow',\n id: 'arrow',\n module: 'arrow',\n version: VERSION,\n extensions: ['arrow', 'feather'],\n mimeTypes: ['application/octet-stream'],\n encodeSync,\n binary: true,\n options: {}\n};\n\nfunction encodeSync(data, options?: ArrowWriterOptions) {\n return encodeArrowSync(data);\n}\n"],"file":"arrow-writer.js"}
package/dist/bundle.js CHANGED
@@ -1,5 +1,5 @@
1
- "use strict";
2
- // @ts-nocheck
3
1
  const moduleExports = require('./index');
2
+
4
3
  globalThis.loaders = globalThis.loaders || {};
5
4
  module.exports = Object.assign(globalThis.loaders, moduleExports);
5
+ //# sourceMappingURL=bundle.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/bundle.ts"],"names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"mappings":"AACA,MAAMA,aAAa,GAAGC,OAAO,CAAC,SAAD,CAA7B;;AACAC,UAAU,CAACC,OAAX,GAAqBD,UAAU,CAACC,OAAX,IAAsB,EAA3C;AACAC,MAAM,CAACC,OAAP,GAAiBC,MAAM,CAACC,MAAP,CAAcL,UAAU,CAACC,OAAzB,EAAkCH,aAAlC,CAAjB","sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"file":"bundle.js"}
package/dist/index.js CHANGED
@@ -1,28 +1,16 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
1
+ import { ArrowLoader as ArrowWorkerLoader } from './arrow-loader';
2
+ import parseSync from './lib/parse-arrow-sync';
3
+ import { parseArrowInBatches } from './lib/parse-arrow-in-batches';
4
+ import { TableBatchBuilder } from '@loaders.gl/schema';
5
+ import ArrowTableBatchAggregator from './lib/arrow-table-batch';
6
+ TableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;
7
+ export { VECTOR_TYPES } from './types';
8
+ export { ArrowWriter } from './arrow-writer';
9
+ export { ArrowWorkerLoader };
10
+ export const ArrowLoader = { ...ArrowWorkerLoader,
11
+ parse: async (arraybuffer, options) => parseSync(arraybuffer, options),
12
+ parseSync,
13
+ parseInBatches: parseArrowInBatches
4
14
  };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports._typecheckArrowLoader = exports.ArrowLoader = exports.ArrowWorkerLoader = exports.ArrowWriter = exports.VECTOR_TYPES = void 0;
7
- const arrow_loader_1 = require("./arrow-loader");
8
- Object.defineProperty(exports, "ArrowWorkerLoader", { enumerable: true, get: function () { return arrow_loader_1.ArrowLoader; } });
9
- const parse_arrow_sync_1 = __importDefault(require("./lib/parse-arrow-sync"));
10
- const parse_arrow_in_batches_1 = require("./lib/parse-arrow-in-batches");
11
- const schema_1 = require("@loaders.gl/schema");
12
- const arrow_table_batch_1 = __importDefault(require("./lib/arrow-table-batch"));
13
- // Make the ArrowBatch type available
14
- schema_1.TableBatchBuilder.ArrowBatch = arrow_table_batch_1.default;
15
- // Types
16
- var types_1 = require("./types");
17
- Object.defineProperty(exports, "VECTOR_TYPES", { enumerable: true, get: function () { return types_1.VECTOR_TYPES; } });
18
- // Arrow writer
19
- var arrow_writer_1 = require("./arrow-writer");
20
- Object.defineProperty(exports, "ArrowWriter", { enumerable: true, get: function () { return arrow_writer_1.ArrowWriter; } });
21
- /** ArrowJS table loader */
22
- exports.ArrowLoader = {
23
- ...arrow_loader_1.ArrowLoader,
24
- parse: async (arraybuffer, options) => (0, parse_arrow_sync_1.default)(arraybuffer, options),
25
- parseSync: parse_arrow_sync_1.default,
26
- parseInBatches: parse_arrow_in_batches_1.parseArrowInBatches
27
- };
28
- exports._typecheckArrowLoader = exports.ArrowLoader;
15
+ export const _typecheckArrowLoader = ArrowLoader;
16
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/index.ts"],"names":["ArrowLoader","ArrowWorkerLoader","parseSync","parseArrowInBatches","TableBatchBuilder","ArrowTableBatchAggregator","ArrowBatch","VECTOR_TYPES","ArrowWriter","parse","arraybuffer","options","parseInBatches","_typecheckArrowLoader"],"mappings":"AAEA,SAAQA,WAAW,IAAIC,iBAAvB,QAA+C,gBAA/C;AACA,OAAOC,SAAP,MAAsB,wBAAtB;AACA,SAAQC,mBAAR,QAAkC,8BAAlC;AAEA,SAAQC,iBAAR,QAAgC,oBAAhC;AACA,OAAOC,yBAAP,MAAsC,yBAAtC;AAGAD,iBAAiB,CAACE,UAAlB,GAA+BD,yBAA/B;AAGA,SAAQE,YAAR,QAA2B,SAA3B;AAIA,SAAQC,WAAR,QAA0B,gBAA1B;AAKA,SAAQP,iBAAR;AAGA,OAAO,MAAMD,WAA6B,GAAG,EAC3C,GAAGC,iBADwC;AAE3CQ,EAAAA,KAAK,EAAE,OAAOC,WAAP,EAAiCC,OAAjC,KACLT,SAAS,CAACQ,WAAD,EAAcC,OAAd,CAHgC;AAI3CT,EAAAA,SAJ2C;AAK3CU,EAAAA,cAAc,EAAET;AAL2B,CAAtC;AAQP,OAAO,MAAMU,qBAAuC,GAAGb,WAAhD","sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\nimport type {ArrowLoaderOptions} from './arrow-loader';\nimport {ArrowLoader as ArrowWorkerLoader} from './arrow-loader';\nimport parseSync from './lib/parse-arrow-sync';\nimport {parseArrowInBatches} from './lib/parse-arrow-in-batches';\n\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport ArrowTableBatchAggregator from './lib/arrow-table-batch';\n\n// Make the ArrowBatch type available\nTableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;\n\n// Types\nexport {VECTOR_TYPES} from './types';\n\n// Arrow writer\n\nexport {ArrowWriter} from './arrow-writer';\n\n// Arrow loader\n\nexport type {ArrowLoaderOptions};\nexport {ArrowWorkerLoader};\n\n/** ArrowJS table loader */\nexport const ArrowLoader: LoaderWithParser = {\n ...ArrowWorkerLoader,\n parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n parseSync(arraybuffer, options),\n parseSync,\n parseInBatches: parseArrowInBatches\n};\n\nexport const _typecheckArrowLoader: LoaderWithParser = ArrowLoader;\n"],"file":"index.js"}
@@ -1,62 +1,71 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const apache_arrow_1 = require("apache-arrow");
4
- const schema_1 = require("@loaders.gl/schema");
5
- class ArrowTableBatchAggregator extends schema_1.ColumnarTableBatchAggregator {
6
- constructor(schema, options) {
7
- super(schema, options);
8
- this.arrowSchema = null;
9
- }
10
- getBatch() {
11
- const batch = super.getBatch();
12
- if (batch) {
13
- // Get the arrow schema
14
- this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
15
- // Get arrow format vectors
16
- const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
17
- // Create the record batch
18
- // new RecordBatch(schema, numRows, vectors, ...);
19
- const recordBatch = new apache_arrow_1.RecordBatch(this.arrowSchema, batch.length, arrowVectors);
20
- return {
21
- shape: 'arrow-table',
22
- batchType: 'data',
23
- data: recordBatch,
24
- length: batch.length
25
- };
26
- }
27
- return null;
1
+ import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
2
+ import { Schema, Field, RecordBatch, Float32Vector, Float32 } from 'apache-arrow';
3
+ import { ColumnarTableBatchAggregator } from '@loaders.gl/schema';
4
+ export default class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
5
+ constructor(schema, options) {
6
+ super(schema, options);
7
+
8
+ _defineProperty(this, "arrowSchema", void 0);
9
+
10
+ this.arrowSchema = null;
11
+ }
12
+
13
+ getBatch() {
14
+ const batch = super.getBatch();
15
+
16
+ if (batch) {
17
+ this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
18
+ const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
19
+ const recordBatch = new RecordBatch(this.arrowSchema, batch.length, arrowVectors);
20
+ return {
21
+ shape: 'arrow-table',
22
+ batchType: 'data',
23
+ data: recordBatch,
24
+ length: batch.length
25
+ };
28
26
  }
27
+
28
+ return null;
29
+ }
30
+
29
31
  }
30
- exports.default = ArrowTableBatchAggregator;
31
- // Convert from a simple loaders.gl schema to an Arrow schema
32
+
32
33
  function getArrowSchema(schema) {
33
- const arrowFields = [];
34
- for (const key in schema) {
35
- const field = schema[key];
36
- if (field.type === Float32Array) {
37
- const metadata = field; // just store the original field as metadata
38
- // arrow: new Field(name, nullable, metadata)
39
- const arrowField = new apache_arrow_1.Field(field.name, new apache_arrow_1.Float32(), field.nullable, metadata);
40
- arrowFields.push(arrowField);
41
- }
42
- }
43
- if (arrowFields.length === 0) {
44
- throw new Error('No arrow convertible fields');
34
+ const arrowFields = [];
35
+
36
+ for (const key in schema) {
37
+ const field = schema[key];
38
+
39
+ if (field.type === Float32Array) {
40
+ const metadata = field;
41
+ const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);
42
+ arrowFields.push(arrowField);
45
43
  }
46
- return new apache_arrow_1.Schema(arrowFields);
44
+ }
45
+
46
+ if (arrowFields.length === 0) {
47
+ throw new Error('No arrow convertible fields');
48
+ }
49
+
50
+ return new Schema(arrowFields);
47
51
  }
48
- // Convert from simple loaders.gl arrays to arrow vectors
52
+
49
53
  function getArrowVectors(arrowSchema, data) {
50
- const arrowVectors = [];
51
- for (const field of arrowSchema.fields) {
52
- const vector = data[field.name];
53
- if (vector instanceof Float32Array) {
54
- const arrowVector = apache_arrow_1.Float32Vector.from(vector);
55
- arrowVectors.push(arrowVector);
56
- }
57
- }
58
- if (arrowSchema.fields.length !== arrowVectors.length) {
59
- throw new Error('Some columns not arrow convertible');
54
+ const arrowVectors = [];
55
+
56
+ for (const field of arrowSchema.fields) {
57
+ const vector = data[field.name];
58
+
59
+ if (vector instanceof Float32Array) {
60
+ const arrowVector = Float32Vector.from(vector);
61
+ arrowVectors.push(arrowVector);
60
62
  }
61
- return arrowVectors;
63
+ }
64
+
65
+ if (arrowSchema.fields.length !== arrowVectors.length) {
66
+ throw new Error('Some columns not arrow convertible');
67
+ }
68
+
69
+ return arrowVectors;
62
70
  }
71
+ //# sourceMappingURL=arrow-table-batch.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../src/lib/arrow-table-batch.ts"],"names":["Schema","Field","RecordBatch","Float32Vector","Float32","ColumnarTableBatchAggregator","ArrowTableBatchAggregator","constructor","schema","options","arrowSchema","getBatch","batch","getArrowSchema","arrowVectors","getArrowVectors","data","recordBatch","length","shape","batchType","arrowFields","key","field","type","Float32Array","metadata","arrowField","name","nullable","push","Error","fields","vector","arrowVector","from"],"mappings":";AACA,SAAQA,MAAR,EAAgBC,KAAhB,EAAuBC,WAAvB,EAAoCC,aAApC,EAAmDC,OAAnD,QAAiE,cAAjE;AACA,SAAQC,4BAAR,QAA2C,oBAA3C;AAEA,eAAe,MAAMC,yBAAN,SAAwCD,4BAAxC,CAAqE;AAGlFE,EAAAA,WAAW,CAACC,MAAD,EAASC,OAAT,EAAkB;AAC3B,UAAMD,MAAN,EAAcC,OAAd;;AAD2B;;AAE3B,SAAKC,WAAL,GAAmB,IAAnB;AACD;;AAEDC,EAAAA,QAAQ,GAA2B;AACjC,UAAMC,KAAK,GAAG,MAAMD,QAAN,EAAd;;AACA,QAAIC,KAAJ,EAAW;AAET,WAAKF,WAAL,GAAmB,KAAKA,WAAL,IAAoBG,cAAc,CAACD,KAAK,CAACJ,MAAP,CAArD;AAEA,YAAMM,YAAY,GAAGC,eAAe,CAAC,KAAKL,WAAN,EAAmBE,KAAK,CAACI,IAAzB,CAApC;AAGA,YAAMC,WAAW,GAAG,IAAIf,WAAJ,CAAgB,KAAKQ,WAArB,EAAkCE,KAAK,CAACM,MAAxC,EAAgDJ,YAAhD,CAApB;AACA,aAAO;AACLK,QAAAA,KAAK,EAAE,aADF;AAELC,QAAAA,SAAS,EAAE,MAFN;AAGLJ,QAAAA,IAAI,EAAEC,WAHD;AAILC,QAAAA,MAAM,EAAEN,KAAK,CAACM;AAJT,OAAP;AAMD;;AAED,WAAO,IAAP;AACD;;AA3BiF;;AA+BpF,SAASL,cAAT,CAAwBL,MAAxB,EAAgC;AAC9B,QAAMa,WAAoB,GAAG,EAA7B;;AACA,OAAK,MAAMC,GAAX,IAAkBd,MAAlB,EAA0B;AACxB,UAAMe,KAAK,GAAGf,MAAM,CAACc,GAAD,CAApB;;AACA,QAAIC,KAAK,CAACC,IAAN,KAAeC,YAAnB,EAAiC;AAC/B,YAAMC,QAAQ,GAAGH,KAAjB;AAEA,YAAMI,UAAU,GAAG,IAAI1B,KAAJ,CAAUsB,KAAK,CAACK,IAAhB,EAAsB,IAAIxB,OAAJ,EAAtB,EAAqCmB,KAAK,CAACM,QAA3C,EAAqDH,QAArD,CAAnB;AACAL,MAAAA,WAAW,CAACS,IAAZ,CAAiBH,UAAjB;AACD;AACF;;AACD,MAAIN,WAAW,CAACH,MAAZ,KAAuB,CAA3B,EAA8B;AAC5B,UAAM,IAAIa,KAAJ,CAAU,6BAAV,CAAN;AACD;;AAED,SAAO,IAAI/B,MAAJ,CAAWqB,WAAX,CAAP;AACD;;AAGD,SAASN,eAAT,CAAyBL,WAAzB,EAAsCM,IAAtC,EAA4C;AAC1C,QAAMF,YAAmB,GAAG,EAA5B;;AACA,OAAK,MAAMS,KAAX,IAAoBb,WAAW,CAACsB,MAAhC,EAAwC;AACtC,UAAMC,MAAM,GAAGjB,IAAI,CAACO,KAAK,CAACK,IAAP,CAAnB;;AACA,QAAIK,MAAM,YAAYR,YAAtB,EAAoC;AAClC,YAAMS,WAAW,GAAG/B,aAAa,CAACgC,IAAd,CAAmBF,MAAnB,CAApB;AACAnB,MAAAA,YAAY,CAACgB,IAAb,CAAkBI,WAAlB;AACD;AACF;;AACD,MAAIxB,WAAW,CAACsB,MAAZ,CAAmBd,MAAnB,KAA8BJ,YAAY,CAACI,MAA/C,EAAuD;AACrD,UAAM,IAAIa,KAAJ,CAAU,oCAAV,CAAN;AACD;;AACD,SAAOjB,YAAP;AACD","sourcesContent":["import type {ArrowTableBatch} from '@loaders.gl/schema';\nimport {Schema, Field, RecordBatch, Float32Vector, Float32} from 'apache-arrow';\nimport {ColumnarTableBatchAggregator} from '@loaders.gl/schema';\n\nexport default class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {\n arrowSchema: Schema | null;\n\n constructor(schema, options) {\n super(schema, options);\n this.arrowSchema = null;\n }\n\n getBatch(): ArrowTableBatch | null {\n const batch = super.getBatch();\n if (batch) {\n // Get the arrow schema\n this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);\n // Get arrow format vectors\n const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);\n // Create the record batch\n // new RecordBatch(schema, numRows, vectors, ...);\n const recordBatch = new RecordBatch(this.arrowSchema, batch.length, arrowVectors);\n return {\n shape: 'arrow-table',\n batchType: 'data',\n data: recordBatch,\n length: batch.length\n };\n }\n\n return null;\n }\n}\n\n// Convert from a simple loaders.gl schema to an Arrow schema\nfunction getArrowSchema(schema) {\n const arrowFields: Field[] = [];\n for (const key in schema) {\n const field = schema[key];\n if (field.type === Float32Array) {\n const metadata = field; // just store the original field as metadata\n // arrow: new Field(name, nullable, metadata)\n const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);\n arrowFields.push(arrowField);\n }\n }\n if (arrowFields.length === 0) {\n throw new Error('No arrow convertible fields');\n }\n\n return new Schema(arrowFields);\n}\n\n// Convert from simple loaders.gl arrays to arrow vectors\nfunction getArrowVectors(arrowSchema, data) {\n const arrowVectors: any[] = [];\n for (const field of arrowSchema.fields) {\n const vector = data[field.name];\n if (vector instanceof Float32Array) {\n const arrowVector = Float32Vector.from(vector);\n arrowVectors.push(arrowVector);\n }\n }\n if (arrowSchema.fields.length !== arrowVectors.length) {\n throw new Error('Some columns not arrow convertible');\n }\n return arrowVectors;\n}\n"],"file":"arrow-table-batch.js"}
@@ -1,40 +1,28 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.encodeArrowSync = void 0;
4
- const apache_arrow_1 = require("apache-arrow");
5
- const types_1 = require("../types");
6
- /**
7
- * Encodes set of arrays into the Apache Arrow columnar format
8
- * https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
9
- * @param data - columns data
10
- * @param options - the writer options
11
- * @returns - encoded ArrayBuffer
12
- */
13
- function encodeArrowSync(data) {
14
- const vectors = [];
15
- const arrayNames = [];
16
- for (const arrayData of data) {
17
- arrayNames.push(arrayData.name);
18
- const arrayVector = createVector(arrayData.array, arrayData.type);
19
- vectors.push(arrayVector);
20
- }
21
- const table = apache_arrow_1.Table.new(vectors, arrayNames);
22
- const arrowBuffer = table.serialize();
23
- return arrowBuffer;
1
+ import { Table, FloatVector, DateVector } from 'apache-arrow';
2
+ import { VECTOR_TYPES } from '../types';
3
+ export function encodeArrowSync(data) {
4
+ const vectors = [];
5
+ const arrayNames = [];
6
+
7
+ for (const arrayData of data) {
8
+ arrayNames.push(arrayData.name);
9
+ const arrayVector = createVector(arrayData.array, arrayData.type);
10
+ vectors.push(arrayVector);
11
+ }
12
+
13
+ const table = Table.new(vectors, arrayNames);
14
+ const arrowBuffer = table.serialize();
15
+ return arrowBuffer;
24
16
  }
25
- exports.encodeArrowSync = encodeArrowSync;
26
- /**
27
- * Create Arrow Vector from given data and vector type
28
- * @param array {import('../types').AnyArrayType} - columns data
29
- * @param type {number} - the writer options
30
- * @return a vector of one of vector's types defined in the Apache Arrow library
31
- */
17
+
32
18
  function createVector(array, type) {
33
- switch (type) {
34
- case types_1.VECTOR_TYPES.DATE:
35
- return apache_arrow_1.DateVector.from(array);
36
- case types_1.VECTOR_TYPES.FLOAT:
37
- default:
38
- return apache_arrow_1.FloatVector.from(array);
39
- }
19
+ switch (type) {
20
+ case VECTOR_TYPES.DATE:
21
+ return DateVector.from(array);
22
+
23
+ case VECTOR_TYPES.FLOAT:
24
+ default:
25
+ return FloatVector.from(array);
26
+ }
40
27
  }
28
+ //# sourceMappingURL=encode-arrow.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../src/lib/encode-arrow.ts"],"names":["Table","FloatVector","DateVector","VECTOR_TYPES","encodeArrowSync","data","vectors","arrayNames","arrayData","push","name","arrayVector","createVector","array","type","table","new","arrowBuffer","serialize","DATE","from","FLOAT"],"mappings":"AAAA,SAAQA,KAAR,EAAeC,WAAf,EAA4BC,UAA5B,QAA6C,cAA7C;AACA,SAAsBC,YAAtB,QAAyC,UAAzC;AAeA,OAAO,SAASC,eAAT,CAAyBC,IAAzB,EAA2D;AAChE,QAAMC,OAAc,GAAG,EAAvB;AACA,QAAMC,UAAoB,GAAG,EAA7B;;AACA,OAAK,MAAMC,SAAX,IAAwBH,IAAxB,EAA8B;AAC5BE,IAAAA,UAAU,CAACE,IAAX,CAAgBD,SAAS,CAACE,IAA1B;AACA,UAAMC,WAAW,GAAGC,YAAY,CAACJ,SAAS,CAACK,KAAX,EAAkBL,SAAS,CAACM,IAA5B,CAAhC;AACAR,IAAAA,OAAO,CAACG,IAAR,CAAaE,WAAb;AACD;;AACD,QAAMI,KAAK,GAAGf,KAAK,CAACgB,GAAN,CAAUV,OAAV,EAAmBC,UAAnB,CAAd;AACA,QAAMU,WAAW,GAAGF,KAAK,CAACG,SAAN,EAApB;AACA,SAAOD,WAAP;AACD;;AAQD,SAASL,YAAT,CAAsBC,KAAtB,EAA6BC,IAA7B,EAAmC;AACjC,UAAQA,IAAR;AACE,SAAKX,YAAY,CAACgB,IAAlB;AACE,aAAOjB,UAAU,CAACkB,IAAX,CAAgBP,KAAhB,CAAP;;AACF,SAAKV,YAAY,CAACkB,KAAlB;AACA;AACE,aAAOpB,WAAW,CAACmB,IAAZ,CAAiBP,KAAjB,CAAP;AALJ;AAOD","sourcesContent":["import {Table, FloatVector, DateVector} from 'apache-arrow';\nimport {AnyArrayType, VECTOR_TYPES} from '../types';\n\ntype ColumnarTable = {\n name: string;\n array: AnyArrayType;\n type: number;\n}[];\n\n/**\n * Encodes set of arrays into the Apache Arrow columnar format\n * https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format\n * @param data - columns data\n * @param options - the writer options\n * @returns - encoded ArrayBuffer\n */\nexport function encodeArrowSync(data: ColumnarTable): ArrayBuffer {\n const vectors: any[] = [];\n const arrayNames: string[] = [];\n for (const arrayData of data) {\n arrayNames.push(arrayData.name);\n const arrayVector = createVector(arrayData.array, arrayData.type);\n vectors.push(arrayVector);\n }\n const table = Table.new(vectors, arrayNames);\n const arrowBuffer = table.serialize();\n return arrowBuffer;\n}\n\n/**\n * Create Arrow Vector from given data and vector type\n * @param array {import('../types').AnyArrayType} - columns data\n * @param type {number} - the writer options\n * @return a vector of one of vector's types defined in the Apache Arrow library\n */\nfunction createVector(array, type) {\n switch (type) {\n case VECTOR_TYPES.DATE:\n return DateVector.from(array);\n case VECTOR_TYPES.FLOAT:\n default:\n return FloatVector.from(array);\n }\n}\n"],"file":"encode-arrow.js"}
@@ -1,47 +1,30 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.parseArrowInBatches = void 0;
4
- // TODO - this import defeats the sophisticated typescript checking in ArrowJS
5
- const apache_arrow_1 = require("apache-arrow");
6
- // import {isIterable} from '@loaders.gl/core';
7
- /**
8
- */
9
- function parseArrowInBatches(asyncIterator) {
10
- // Creates the appropriate RecordBatchReader subclasses from the input
11
- // This will also close the underlying source in case of early termination or errors
12
- // As an optimization, return a non-async iterator
13
- /*
14
- if (isIterable(readers)) {
15
- function* makeArrowIterator() {
16
- for (const reader of readers) {
17
- for (const batch of reader) {
18
- yield processBatch(batch, reader);
19
- }
20
- break; // only processing one stream of batches
21
- }
1
+ import { RecordBatchReader } from 'apache-arrow';
2
+ export function parseArrowInBatches(asyncIterator) {
3
+ async function* makeArrowAsyncIterator() {
4
+ const readers = RecordBatchReader.readAll(asyncIterator);
5
+
6
+ for await (const reader of readers) {
7
+ for await (const batch of reader) {
8
+ yield processBatch(batch);
22
9
  }
23
- const arrowIterator = makeArrowIterator();
10
+
11
+ break;
24
12
  }
25
- */
26
- async function* makeArrowAsyncIterator() {
27
- const readers = apache_arrow_1.RecordBatchReader.readAll(asyncIterator);
28
- for await (const reader of readers) {
29
- for await (const batch of reader) {
30
- yield processBatch(batch);
31
- }
32
- break; // only processing one stream of batches
33
- }
34
- }
35
- return makeArrowAsyncIterator();
13
+ }
14
+
15
+ return makeArrowAsyncIterator();
36
16
  }
37
- exports.parseArrowInBatches = parseArrowInBatches;
17
+
38
18
  function processBatch(batch) {
39
- const values = {
40
- metadata: batch.schema.metadata,
41
- length: batch.length
42
- };
43
- batch.schema.fields.forEach(({ name }, index) => {
44
- values[name] = batch.getChildAt(index).toArray();
45
- });
46
- return values;
19
+ const values = {
20
+ metadata: batch.schema.metadata,
21
+ length: batch.length
22
+ };
23
+ batch.schema.fields.forEach(({
24
+ name
25
+ }, index) => {
26
+ values[name] = batch.getChildAt(index).toArray();
27
+ });
28
+ return values;
47
29
  }
30
+ //# sourceMappingURL=parse-arrow-in-batches.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../src/lib/parse-arrow-in-batches.ts"],"names":["RecordBatchReader","parseArrowInBatches","asyncIterator","makeArrowAsyncIterator","readers","readAll","reader","batch","processBatch","values","metadata","schema","length","fields","forEach","name","index","getChildAt","toArray"],"mappings":"AACA,SAAQA,iBAAR,QAAgC,cAAhC;AAKA,OAAO,SAASC,mBAAT,CACLC,aADK,EAEe;AAmBpB,kBAAgBC,sBAAhB,GAAyC;AACvC,UAAMC,OAAO,GAAGJ,iBAAiB,CAACK,OAAlB,CAA0BH,aAA1B,CAAhB;;AACA,eAAW,MAAMI,MAAjB,IAA2BF,OAA3B,EAAoC;AAClC,iBAAW,MAAMG,KAAjB,IAA0BD,MAA1B,EAAkC;AAChC,cAAME,YAAY,CAACD,KAAD,CAAlB;AACD;;AACD;AACD;AACF;;AACD,SAAOJ,sBAAsB,EAA7B;AACD;;AAED,SAASK,YAAT,CAAsBD,KAAtB,EAA6B;AAC3B,QAAME,MAAM,GAAG;AACbC,IAAAA,QAAQ,EAAEH,KAAK,CAACI,MAAN,CAAaD,QADV;AAEbE,IAAAA,MAAM,EAAEL,KAAK,CAACK;AAFD,GAAf;AAIAL,EAAAA,KAAK,CAACI,MAAN,CAAaE,MAAb,CAAoBC,OAApB,CAA4B,CAAC;AAACC,IAAAA;AAAD,GAAD,EAASC,KAAT,KAAmB;AAC7CP,IAAAA,MAAM,CAACM,IAAD,CAAN,GAAeR,KAAK,CAACU,UAAN,CAAiBD,KAAjB,EAAwBE,OAAxB,EAAf;AACD,GAFD;AAGA,SAAOT,MAAP;AACD","sourcesContent":["// TODO - this import defeats the sophisticated typescript checking in ArrowJS\nimport {RecordBatchReader} from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<any> {\n // Creates the appropriate RecordBatchReader subclasses from the input\n // This will also close the underlying source in case of early termination or errors\n\n // As an optimization, return a non-async iterator\n /*\n if (isIterable(readers)) {\n function* makeArrowIterator() {\n for (const reader of readers) {\n for (const batch of reader) {\n yield processBatch(batch, reader);\n }\n break; // only processing one stream of batches\n }\n }\n const arrowIterator = makeArrowIterator();\n }\n */\n\n async function* makeArrowAsyncIterator() {\n const readers = RecordBatchReader.readAll(asyncIterator);\n for await (const reader of readers) {\n for await (const batch of reader) {\n yield processBatch(batch);\n }\n break; // only processing one stream of batches\n }\n }\n return makeArrowAsyncIterator();\n}\n\nfunction processBatch(batch) {\n const values = {\n metadata: batch.schema.metadata,\n length: batch.length\n };\n batch.schema.fields.forEach(({name}, index) => {\n values[name] = batch.getChildAt(index).toArray();\n });\n return values;\n}\n"],"file":"parse-arrow-in-batches.js"}
@@ -1,41 +1,44 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const apache_arrow_1 = require("apache-arrow");
4
- // Parses arrow to a columnar table
5
- function parseArrowSync(arrayBuffer, options) {
6
- const arrowTable = apache_arrow_1.Table.from([new Uint8Array(arrayBuffer)]);
7
- // Extract columns
8
- // TODO - avoid calling `getColumn` on columns we are not interested in?
9
- // Add options object?
10
- const columnarTable = {};
11
- arrowTable.schema.fields.forEach((field) => {
12
- // This (is intended to) coalesce all record batches into a single typed array
13
- const arrowColumn = arrowTable.getColumn(field.name);
14
- const values = arrowColumn.toArray();
15
- columnarTable[field.name] = values;
16
- });
17
- switch (options?.arrow?.shape) {
18
- case 'arrow-table':
19
- return arrowTable;
20
- case 'object-row-table':
21
- return convertColumnarToRowFormatTable(columnarTable);
22
- case 'columnar-table':
23
- default:
24
- return columnarTable;
25
- }
1
+ import { Table } from 'apache-arrow';
2
+ export default function parseArrowSync(arrayBuffer, options) {
3
+ var _options$arrow;
4
+
5
+ const arrowTable = Table.from([new Uint8Array(arrayBuffer)]);
6
+ const columnarTable = {};
7
+ arrowTable.schema.fields.forEach(field => {
8
+ const arrowColumn = arrowTable.getColumn(field.name);
9
+ const values = arrowColumn.toArray();
10
+ columnarTable[field.name] = values;
11
+ });
12
+
13
+ switch (options === null || options === void 0 ? void 0 : (_options$arrow = options.arrow) === null || _options$arrow === void 0 ? void 0 : _options$arrow.shape) {
14
+ case 'arrow-table':
15
+ return arrowTable;
16
+
17
+ case 'object-row-table':
18
+ return convertColumnarToRowFormatTable(columnarTable);
19
+
20
+ case 'columnar-table':
21
+ default:
22
+ return columnarTable;
23
+ }
26
24
  }
27
- exports.default = parseArrowSync;
25
+
28
26
  function convertColumnarToRowFormatTable(columnarTable) {
29
- const tableKeys = Object.keys(columnarTable);
30
- const tableRowsCount = columnarTable[tableKeys[0]].length;
31
- const rowFormatTable = [];
32
- for (let index = 0; index < tableRowsCount; index++) {
33
- const tableItem = {};
34
- for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
35
- const fieldName = tableKeys[keyIndex];
36
- tableItem[fieldName] = columnarTable[fieldName][index];
37
- }
38
- rowFormatTable.push(tableItem);
27
+ const tableKeys = Object.keys(columnarTable);
28
+ const tableRowsCount = columnarTable[tableKeys[0]].length;
29
+ const rowFormatTable = [];
30
+
31
+ for (let index = 0; index < tableRowsCount; index++) {
32
+ const tableItem = {};
33
+
34
+ for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
35
+ const fieldName = tableKeys[keyIndex];
36
+ tableItem[fieldName] = columnarTable[fieldName][index];
39
37
  }
40
- return rowFormatTable;
38
+
39
+ rowFormatTable.push(tableItem);
40
+ }
41
+
42
+ return rowFormatTable;
41
43
  }
44
+ //# sourceMappingURL=parse-arrow-sync.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../src/lib/parse-arrow-sync.ts"],"names":["Table","parseArrowSync","arrayBuffer","options","arrowTable","from","Uint8Array","columnarTable","schema","fields","forEach","field","arrowColumn","getColumn","name","values","toArray","arrow","shape","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"mappings":"AACA,SAAQA,KAAR,QAAoB,cAApB;AAGA,eAAe,SAASC,cAAT,CAAwBC,WAAxB,EAAqCC,OAArC,EAAmE;AAAA;;AAChF,QAAMC,UAAU,GAAGJ,KAAK,CAACK,IAAN,CAAW,CAAC,IAAIC,UAAJ,CAAeJ,WAAf,CAAD,CAAX,CAAnB;AAMA,QAAMK,aAAa,GAAG,EAAtB;AAEAH,EAAAA,UAAU,CAACI,MAAX,CAAkBC,MAAlB,CAAyBC,OAAzB,CAAkCC,KAAD,IAAW;AAE1C,UAAMC,WAAW,GAAGR,UAAU,CAACS,SAAX,CAAqBF,KAAK,CAACG,IAA3B,CAApB;AACA,UAAMC,MAAM,GAAGH,WAAW,CAACI,OAAZ,EAAf;AACAT,IAAAA,aAAa,CAACI,KAAK,CAACG,IAAP,CAAb,GAA4BC,MAA5B;AACD,GALD;;AAOA,UAAQZ,OAAR,aAAQA,OAAR,yCAAQA,OAAO,CAAEc,KAAjB,mDAAQ,eAAgBC,KAAxB;AACE,SAAK,aAAL;AACE,aAAOd,UAAP;;AACF,SAAK,kBAAL;AACE,aAAOe,+BAA+B,CAACZ,aAAD,CAAtC;;AACF,SAAK,gBAAL;AACA;AACE,aAAOA,aAAP;AAPJ;AASD;;AAED,SAASY,+BAAT,CAAyCZ,aAAzC,EAAwD;AACtD,QAAMa,SAAS,GAAGC,MAAM,CAACC,IAAP,CAAYf,aAAZ,CAAlB;AACA,QAAMgB,cAAc,GAAGhB,aAAa,CAACa,SAAS,CAAC,CAAD,CAAV,CAAb,CAA4BI,MAAnD;AACA,QAAMC,cAAoB,GAAG,EAA7B;;AAEA,OAAK,IAAIC,KAAK,GAAG,CAAjB,EAAoBA,KAAK,GAAGH,cAA5B,EAA4CG,KAAK,EAAjD,EAAqD;AACnD,UAAMC,SAAS,GAAG,EAAlB;;AACA,SAAK,IAAIC,QAAQ,GAAG,CAApB,EAAuBA,QAAQ,GAAGR,SAAS,CAACI,MAA5C,EAAoDI,QAAQ,EAA5D,EAAgE;AAC9D,YAAMC,SAAS,GAAGT,SAAS,CAACQ,QAAD,CAA3B;AACAD,MAAAA,SAAS,CAACE,SAAD,CAAT,GAAuBtB,aAAa,CAACsB,SAAD,CAAb,CAAyBH,KAAzB,CAAvB;AACD;;AACDD,IAAAA,cAAc,CAACK,IAAf,CAAoBH,SAApB;AACD;;AACD,SAAOF,cAAP;AACD","sourcesContent":["import type {ArrowLoaderOptions} from '../arrow-loader';\nimport {Table} from 'apache-arrow';\n\n// Parses arrow to a columnar table\nexport default function parseArrowSync(arrayBuffer, options?: ArrowLoaderOptions) {\n const arrowTable = Table.from([new Uint8Array(arrayBuffer)]);\n\n // Extract columns\n\n // TODO - avoid calling `getColumn` on columns we are not interested in?\n // Add options object?\n const columnarTable = {};\n\n arrowTable.schema.fields.forEach((field) => {\n // This (is intended to) coalesce all record batches into a single typed array\n const arrowColumn = arrowTable.getColumn(field.name);\n const values = arrowColumn.toArray();\n columnarTable[field.name] = values;\n });\n\n switch (options?.arrow?.shape) {\n case 'arrow-table':\n return arrowTable;\n case 'object-row-table':\n return convertColumnarToRowFormatTable(columnarTable);\n case 'columnar-table':\n default:\n return columnarTable;\n }\n}\n\nfunction convertColumnarToRowFormatTable(columnarTable) {\n const tableKeys = Object.keys(columnarTable);\n const tableRowsCount = columnarTable[tableKeys[0]].length;\n const rowFormatTable: {}[] = [];\n\n for (let index = 0; index < tableRowsCount; index++) {\n const tableItem = {};\n for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n const fieldName = tableKeys[keyIndex];\n tableItem[fieldName] = columnarTable[fieldName][index];\n }\n rowFormatTable.push(tableItem);\n }\n return rowFormatTable;\n}\n"],"file":"parse-arrow-sync.js"}
package/dist/types.js CHANGED
@@ -1,8 +1,7 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.VECTOR_TYPES = void 0;
4
- var VECTOR_TYPES;
1
+ export let VECTOR_TYPES;
2
+
5
3
  (function (VECTOR_TYPES) {
6
- VECTOR_TYPES[VECTOR_TYPES["FLOAT"] = 0] = "FLOAT";
7
- VECTOR_TYPES[VECTOR_TYPES["DATE"] = 1] = "DATE";
8
- })(VECTOR_TYPES = exports.VECTOR_TYPES || (exports.VECTOR_TYPES = {}));
4
+ VECTOR_TYPES[VECTOR_TYPES["FLOAT"] = 0] = "FLOAT";
5
+ VECTOR_TYPES[VECTOR_TYPES["DATE"] = 1] = "DATE";
6
+ })(VECTOR_TYPES || (VECTOR_TYPES = {}));
7
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/types.ts"],"names":["VECTOR_TYPES"],"mappings":"AAgBA,WAAYA,YAAZ;;WAAYA,Y;AAAAA,EAAAA,Y,CAAAA,Y;AAAAA,EAAAA,Y,CAAAA,Y;GAAAA,Y,KAAAA,Y","sourcesContent":["type TypedIntArray =\n | Int8Array\n | Uint8Array\n | Int16Array\n | Uint16Array\n | Int32Array\n | Uint32Array\n | Int32Array\n | Uint32Array;\n\ntype TypedFloatArray = Float32Array | Float64Array;\n\ntype TypedArray = TypedIntArray | TypedFloatArray;\n\nexport type AnyArrayType = Array<any> | TypedArray;\n\nexport enum VECTOR_TYPES {\n FLOAT,\n DATE\n}\n"],"file":"types.js"}
@@ -1,5 +1,4 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const loader_utils_1 = require("@loaders.gl/loader-utils");
4
- const index_1 = require("../index");
5
- (0, loader_utils_1.createLoaderWorker)(index_1.ArrowLoader);
1
+ import { createLoaderWorker } from '@loaders.gl/loader-utils';
2
+ import { ArrowLoader } from '../index';
3
+ createLoaderWorker(ArrowLoader);
4
+ //# sourceMappingURL=arrow-worker.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../src/workers/arrow-worker.ts"],"names":["createLoaderWorker","ArrowLoader"],"mappings":"AAAA,SAAQA,kBAAR,QAAiC,0BAAjC;AACA,SAAQC,WAAR,QAA0B,UAA1B;AAEAD,kBAAkB,CAACC,WAAD,CAAlB","sourcesContent":["import {createLoaderWorker} from '@loaders.gl/loader-utils';\nimport {ArrowLoader} from '../index';\n\ncreateLoaderWorker(ArrowLoader);\n"],"file":"arrow-worker.js"}