@loaders.gl/arrow 3.3.0-alpha.5 → 3.3.0-alpha.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow-worker.js +1288 -1304
- package/dist/dist.min.js +1289 -1305
- package/dist/es5/arrow-loader.js +2 -1
- package/dist/es5/arrow-loader.js.map +1 -1
- package/dist/es5/arrow-writer.js +1 -3
- package/dist/es5/arrow-writer.js.map +1 -1
- package/dist/es5/bundle.js +0 -1
- package/dist/es5/bundle.js.map +1 -1
- package/dist/es5/index.js +8 -25
- package/dist/es5/index.js.map +1 -1
- package/dist/es5/lib/arrow-table-batch.js +1 -36
- package/dist/es5/lib/arrow-table-batch.js.map +1 -1
- package/dist/es5/lib/encode-arrow.js +1 -12
- package/dist/es5/lib/encode-arrow.js.map +1 -1
- package/dist/es5/lib/parse-arrow-in-batches.js +62 -117
- package/dist/es5/lib/parse-arrow-in-batches.js.map +1 -1
- package/dist/es5/lib/parse-arrow-sync.js +0 -11
- package/dist/es5/lib/parse-arrow-sync.js.map +1 -1
- package/dist/es5/types.js +0 -1
- package/dist/es5/types.js.map +1 -1
- package/dist/es5/workers/arrow-worker.js +0 -2
- package/dist/es5/workers/arrow-worker.js.map +1 -1
- package/dist/esm/arrow-loader.js +3 -1
- package/dist/esm/arrow-loader.js.map +1 -1
- package/dist/esm/arrow-writer.js +4 -2
- package/dist/esm/arrow-writer.js.map +1 -1
- package/dist/esm/bundle.js +1 -1
- package/dist/esm/bundle.js.map +1 -1
- package/dist/esm/index.js +7 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/lib/arrow-table-batch.js +0 -14
- package/dist/esm/lib/arrow-table-batch.js.map +1 -1
- package/dist/esm/lib/encode-arrow.js +0 -3
- package/dist/esm/lib/encode-arrow.js.map +1 -1
- package/dist/esm/lib/parse-arrow-in-batches.js +7 -6
- package/dist/esm/lib/parse-arrow-in-batches.js.map +1 -1
- package/dist/esm/lib/parse-arrow-sync.js +2 -9
- package/dist/esm/lib/parse-arrow-sync.js.map +1 -1
- package/dist/esm/types.js +0 -1
- package/dist/esm/types.js.map +1 -1
- package/dist/esm/workers/arrow-worker.js.map +1 -1
- package/dist/lib/parse-arrow-in-batches.d.ts.map +1 -1
- package/dist/lib/parse-arrow-in-batches.js +1 -0
- package/package.json +4 -4
- package/src/lib/parse-arrow-in-batches.ts +1 -0
package/dist/es5/arrow-loader.js
CHANGED
|
@@ -4,12 +4,13 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
value: true
|
|
5
5
|
});
|
|
6
6
|
exports._typecheckArrowLoader = exports.ArrowLoader = void 0;
|
|
7
|
-
var VERSION = typeof "3.3.0-alpha.
|
|
7
|
+
var VERSION = typeof "3.3.0-alpha.7" !== 'undefined' ? "3.3.0-alpha.7" : 'latest';
|
|
8
8
|
var DEFAULT_ARROW_LOADER_OPTIONS = {
|
|
9
9
|
arrow: {
|
|
10
10
|
shape: 'columnar-table'
|
|
11
11
|
}
|
|
12
12
|
};
|
|
13
|
+
|
|
13
14
|
var ArrowLoader = {
|
|
14
15
|
name: 'Apache Arrow',
|
|
15
16
|
id: 'arrow',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"arrow-loader.js","names":["VERSION","DEFAULT_ARROW_LOADER_OPTIONS","arrow","shape","ArrowLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckArrowLoader"],"sources":["../../src/arrow-loader.ts"],"sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ArrowLoaderOptions = LoaderOptions & {\n arrow?: {\n shape: 'arrow-table' | 'columnar-table' | 'row-table' | 'array-row-table' | 'object-row-table';\n };\n};\n\nconst DEFAULT_ARROW_LOADER_OPTIONS = {\n arrow: {\n shape: 'columnar-table'\n }\n};\n\n/** ArrowJS table loader */\nexport const ArrowLoader = {\n name: 'Apache Arrow',\n id: 'arrow',\n module: 'arrow',\n version: VERSION,\n worker: true,\n category: 'table',\n extensions: ['arrow', 'feather'],\n mimeTypes: [\n 'application/vnd.apache.arrow.file',\n 'application/vnd.apache.arrow.stream',\n 'application/octet-stream'\n ],\n binary: true,\n tests: ['ARROW'],\n options: DEFAULT_ARROW_LOADER_OPTIONS\n};\n\nexport const _typecheckArrowLoader: Loader = ArrowLoader;\n"],"mappings":";;;;;;AAIA,IAAMA,OAAO,GAAG,sBAAkB,KAAK,WAAW,qBAAiB,QAAQ;AAQ3E,IAAMC,4BAA4B,GAAG;EACnCC,KAAK,EAAE;IACLC,KAAK,EAAE;EACT;AACF,CAAC;;AAGM,IAAMC,WAAW,GAAG;EACzBC,IAAI,EAAE,cAAc;EACpBC,EAAE,EAAE,OAAO;EACXC,MAAM,EAAE,OAAO;EACfC,OAAO,EAAER,OAAO;EAChBS,MAAM,EAAE,IAAI;EACZC,QAAQ,EAAE,OAAO;EACjBC,UAAU,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC;EAChCC,SAAS,EAAE,CACT,mCAAmC,EACnC,qCAAqC,EACrC,0BAA0B,CAC3B;EACDC,MAAM,EAAE,IAAI;EACZC,KAAK,EAAE,CAAC,OAAO,CAAC;EAChBC,OAAO,EAAEd;AACX,CAAC;AAAC;AAEK,IAAMe,qBAA6B,GAAGZ,WAAW;AAAC"}
|
package/dist/es5/arrow-writer.js
CHANGED
|
@@ -4,10 +4,9 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
value: true
|
|
5
5
|
});
|
|
6
6
|
exports.ArrowWriter = void 0;
|
|
7
|
-
|
|
8
7
|
var _encodeArrow = require("./lib/encode-arrow");
|
|
9
8
|
|
|
10
|
-
var VERSION = typeof "3.3.0-alpha.
|
|
9
|
+
var VERSION = typeof "3.3.0-alpha.7" !== 'undefined' ? "3.3.0-alpha.7" : 'latest';
|
|
11
10
|
var ArrowWriter = {
|
|
12
11
|
name: 'Apache Arrow',
|
|
13
12
|
id: 'arrow',
|
|
@@ -20,7 +19,6 @@ var ArrowWriter = {
|
|
|
20
19
|
options: {}
|
|
21
20
|
};
|
|
22
21
|
exports.ArrowWriter = ArrowWriter;
|
|
23
|
-
|
|
24
22
|
function encodeSync(data, options) {
|
|
25
23
|
return (0, _encodeArrow.encodeArrowSync)(data);
|
|
26
24
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"arrow-writer.js","names":["VERSION","ArrowWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","encodeArrowSync"],"sources":["../../src/arrow-writer.ts"],"sourcesContent":["// import type {Writer} from '@loaders.gl/loader-utils';\nimport type {WriterOptions} from '@loaders.gl/loader-utils';\nimport {encodeArrowSync} from './lib/encode-arrow';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\ntype ArrowWriterOptions = WriterOptions;\n\n/** Apache Arrow writer */\nexport const ArrowWriter = {\n name: 'Apache Arrow',\n id: 'arrow',\n module: 'arrow',\n version: VERSION,\n extensions: ['arrow', 'feather'],\n mimeTypes: [\n 'application/vnd.apache.arrow.file',\n 'application/vnd.apache.arrow.stream',\n 'application/octet-stream'\n ],\n encodeSync,\n binary: true,\n options: {}\n};\n\nfunction encodeSync(data, options?: ArrowWriterOptions) {\n return encodeArrowSync(data);\n}\n"],"mappings":";;;;;;AAEA;;AAIA,IAAMA,OAAO,GAAG,sBAAkB,KAAK,WAAW,qBAAiB,QAAQ;AAKpE,IAAMC,WAAW,GAAG;EACzBC,IAAI,EAAE,cAAc;EACpBC,EAAE,EAAE,OAAO;EACXC,MAAM,EAAE,OAAO;EACfC,OAAO,EAAEL,OAAO;EAChBM,UAAU,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC;EAChCC,SAAS,EAAE,CACT,mCAAmC,EACnC,qCAAqC,EACrC,0BAA0B,CAC3B;EACDC,UAAU,EAAVA,UAAU;EACVC,MAAM,EAAE,IAAI;EACZC,OAAO,EAAE,CAAC;AACZ,CAAC;AAAC;AAEF,SAASF,UAAU,CAACG,IAAI,EAAED,OAA4B,EAAE;EACtD,OAAO,IAAAE,4BAAe,EAACD,IAAI,CAAC;AAC9B"}
|
package/dist/es5/bundle.js
CHANGED
package/dist/es5/bundle.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"bundle.js","names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"sources":["../../src/bundle.ts"],"sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"mappings":";;AACA,IAAMA,aAAa,GAAGC,OAAO,CAAC,SAAS,CAAC;AACxCC,UAAU,CAACC,OAAO,GAAGD,UAAU,CAACC,OAAO,IAAI,CAAC,CAAC;AAC7CC,MAAM,CAACC,OAAO,GAAGC,MAAM,CAACC,MAAM,CAACL,UAAU,CAACC,OAAO,EAAEH,aAAa,CAAC"}
|
package/dist/es5/index.js
CHANGED
|
@@ -1,54 +1,41 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
4
|
Object.defineProperty(exports, "__esModule", {
|
|
6
5
|
value: true
|
|
7
6
|
});
|
|
7
|
+
exports.ArrowLoader = void 0;
|
|
8
8
|
Object.defineProperty(exports, "ArrowWorkerLoader", {
|
|
9
9
|
enumerable: true,
|
|
10
10
|
get: function get() {
|
|
11
11
|
return _arrowLoader.ArrowLoader;
|
|
12
12
|
}
|
|
13
13
|
});
|
|
14
|
-
Object.defineProperty(exports, "
|
|
14
|
+
Object.defineProperty(exports, "ArrowWriter", {
|
|
15
15
|
enumerable: true,
|
|
16
16
|
get: function get() {
|
|
17
|
-
return
|
|
17
|
+
return _arrowWriter.ArrowWriter;
|
|
18
18
|
}
|
|
19
19
|
});
|
|
20
|
-
Object.defineProperty(exports, "
|
|
20
|
+
Object.defineProperty(exports, "VECTOR_TYPES", {
|
|
21
21
|
enumerable: true,
|
|
22
22
|
get: function get() {
|
|
23
|
-
return
|
|
23
|
+
return _types.VECTOR_TYPES;
|
|
24
24
|
}
|
|
25
25
|
});
|
|
26
|
-
exports._typecheckArrowLoader =
|
|
27
|
-
|
|
26
|
+
exports._typecheckArrowLoader = void 0;
|
|
28
27
|
var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
|
|
29
|
-
|
|
30
28
|
var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
|
|
31
|
-
|
|
32
29
|
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
33
|
-
|
|
34
30
|
var _arrowLoader = require("./arrow-loader");
|
|
35
|
-
|
|
36
31
|
var _parseArrowSync = _interopRequireDefault(require("./lib/parse-arrow-sync"));
|
|
37
|
-
|
|
38
32
|
var _parseArrowInBatches = require("./lib/parse-arrow-in-batches");
|
|
39
|
-
|
|
40
33
|
var _schema = require("@loaders.gl/schema");
|
|
41
|
-
|
|
42
34
|
var _arrowTableBatch = _interopRequireDefault(require("./lib/arrow-table-batch"));
|
|
43
|
-
|
|
44
35
|
var _types = require("./types");
|
|
45
|
-
|
|
46
36
|
var _arrowWriter = require("./arrow-writer");
|
|
47
|
-
|
|
48
|
-
function
|
|
49
|
-
|
|
50
|
-
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
|
|
51
|
-
|
|
37
|
+
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
|
|
38
|
+
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
|
|
52
39
|
_schema.TableBatchBuilder.ArrowBatch = _arrowTableBatch.default;
|
|
53
40
|
|
|
54
41
|
var ArrowLoader = _objectSpread(_objectSpread({}, _arrowLoader.ArrowLoader), {}, {
|
|
@@ -59,7 +46,6 @@ var ArrowLoader = _objectSpread(_objectSpread({}, _arrowLoader.ArrowLoader), {},
|
|
|
59
46
|
switch (_context.prev = _context.next) {
|
|
60
47
|
case 0:
|
|
61
48
|
return _context.abrupt("return", (0, _parseArrowSync.default)(arraybuffer, options));
|
|
62
|
-
|
|
63
49
|
case 1:
|
|
64
50
|
case "end":
|
|
65
51
|
return _context.stop();
|
|
@@ -67,17 +53,14 @@ var ArrowLoader = _objectSpread(_objectSpread({}, _arrowLoader.ArrowLoader), {},
|
|
|
67
53
|
}
|
|
68
54
|
}, _callee);
|
|
69
55
|
}));
|
|
70
|
-
|
|
71
56
|
function parse(_x, _x2) {
|
|
72
57
|
return _parse.apply(this, arguments);
|
|
73
58
|
}
|
|
74
|
-
|
|
75
59
|
return parse;
|
|
76
60
|
}(),
|
|
77
61
|
parseSync: _parseArrowSync.default,
|
|
78
62
|
parseInBatches: _parseArrowInBatches.parseArrowInBatches
|
|
79
63
|
});
|
|
80
|
-
|
|
81
64
|
exports.ArrowLoader = ArrowLoader;
|
|
82
65
|
var _typecheckArrowLoader = ArrowLoader;
|
|
83
66
|
exports._typecheckArrowLoader = _typecheckArrowLoader;
|
package/dist/es5/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"index.js","names":["TableBatchBuilder","ArrowBatch","ArrowTableBatchAggregator","ArrowLoader","ArrowWorkerLoader","parse","arraybuffer","options","parseSync","parseInBatches","parseArrowInBatches","_typecheckArrowLoader"],"sources":["../../src/index.ts"],"sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\nimport type {ArrowLoaderOptions} from './arrow-loader';\nimport {ArrowLoader as ArrowWorkerLoader} from './arrow-loader';\nimport parseSync from './lib/parse-arrow-sync';\nimport {parseArrowInBatches} from './lib/parse-arrow-in-batches';\n\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport ArrowTableBatchAggregator from './lib/arrow-table-batch';\n\n// Make the ArrowBatch type available\nTableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;\n\n// Types\nexport {VECTOR_TYPES} from './types';\n\n// Arrow writer\n\nexport {ArrowWriter} from './arrow-writer';\n\n// Arrow loader\n\nexport type {ArrowLoaderOptions};\nexport {ArrowWorkerLoader};\n\n/** ArrowJS table loader */\nexport const ArrowLoader: LoaderWithParser = {\n ...ArrowWorkerLoader,\n parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n parseSync(arraybuffer, options),\n parseSync,\n parseInBatches: parseArrowInBatches\n};\n\nexport const _typecheckArrowLoader: LoaderWithParser = ArrowLoader;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA;AACA;AACA;AAEA;AACA;AAMA;AAIA;AAA2C;AAAA;AAP3CA,yBAAiB,CAACC,UAAU,GAAGC,wBAAyB;;AAejD,IAAMC,WAA6B,mCACrCC,wBAAiB;EACpBC,KAAK;IAAA,uEAAE,iBAAOC,WAAwB,EAAEC,OAA4B;MAAA;QAAA;UAAA;YAAA;cAAA,iCAClE,IAAAC,uBAAS,EAACF,WAAW,EAAEC,OAAO,CAAC;YAAA;YAAA;cAAA;UAAA;QAAA;MAAA;IAAA;IAAA;MAAA;IAAA;IAAA;EAAA;EACjCC,SAAS,EAATA,uBAAS;EACTC,cAAc,EAAEC;AAAmB,EACpC;AAAC;AAEK,IAAMC,qBAAuC,GAAGR,WAAW;AAAC"}
|
|
@@ -1,62 +1,40 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
4
|
Object.defineProperty(exports, "__esModule", {
|
|
6
5
|
value: true
|
|
7
6
|
});
|
|
8
7
|
exports.default = void 0;
|
|
9
|
-
|
|
10
8
|
var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
|
|
11
|
-
|
|
12
9
|
var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
|
|
13
|
-
|
|
14
10
|
var _assertThisInitialized2 = _interopRequireDefault(require("@babel/runtime/helpers/assertThisInitialized"));
|
|
15
|
-
|
|
16
11
|
var _get2 = _interopRequireDefault(require("@babel/runtime/helpers/get"));
|
|
17
|
-
|
|
18
12
|
var _inherits2 = _interopRequireDefault(require("@babel/runtime/helpers/inherits"));
|
|
19
|
-
|
|
20
13
|
var _possibleConstructorReturn2 = _interopRequireDefault(require("@babel/runtime/helpers/possibleConstructorReturn"));
|
|
21
|
-
|
|
22
14
|
var _getPrototypeOf2 = _interopRequireDefault(require("@babel/runtime/helpers/getPrototypeOf"));
|
|
23
|
-
|
|
24
15
|
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
25
|
-
|
|
26
16
|
var _apacheArrow = require("apache-arrow");
|
|
27
|
-
|
|
28
17
|
var _schema = require("@loaders.gl/schema");
|
|
29
|
-
|
|
30
18
|
function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
|
|
31
|
-
|
|
32
19
|
function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
|
|
33
|
-
|
|
34
20
|
function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
|
|
35
|
-
|
|
36
21
|
function _createSuper(Derived) { var hasNativeReflectConstruct = _isNativeReflectConstruct(); return function _createSuperInternal() { var Super = (0, _getPrototypeOf2.default)(Derived), result; if (hasNativeReflectConstruct) { var NewTarget = (0, _getPrototypeOf2.default)(this).constructor; result = Reflect.construct(Super, arguments, NewTarget); } else { result = Super.apply(this, arguments); } return (0, _possibleConstructorReturn2.default)(this, result); }; }
|
|
37
|
-
|
|
38
22
|
function _isNativeReflectConstruct() { if (typeof Reflect === "undefined" || !Reflect.construct) return false; if (Reflect.construct.sham) return false; if (typeof Proxy === "function") return true; try { Boolean.prototype.valueOf.call(Reflect.construct(Boolean, [], function () {})); return true; } catch (e) { return false; } }
|
|
39
|
-
|
|
40
23
|
var ArrowTableBatchAggregator = function (_ColumnarTableBatchAg) {
|
|
41
24
|
(0, _inherits2.default)(ArrowTableBatchAggregator, _ColumnarTableBatchAg);
|
|
42
|
-
|
|
43
25
|
var _super = _createSuper(ArrowTableBatchAggregator);
|
|
44
|
-
|
|
45
26
|
function ArrowTableBatchAggregator(schema, options) {
|
|
46
27
|
var _this;
|
|
47
|
-
|
|
48
28
|
(0, _classCallCheck2.default)(this, ArrowTableBatchAggregator);
|
|
49
29
|
_this = _super.call(this, schema, options);
|
|
50
30
|
(0, _defineProperty2.default)((0, _assertThisInitialized2.default)(_this), "arrowSchema", void 0);
|
|
51
31
|
_this.arrowSchema = null;
|
|
52
32
|
return _this;
|
|
53
33
|
}
|
|
54
|
-
|
|
55
34
|
(0, _createClass2.default)(ArrowTableBatchAggregator, [{
|
|
56
35
|
key: "getBatch",
|
|
57
36
|
value: function getBatch() {
|
|
58
37
|
var batch = (0, _get2.default)((0, _getPrototypeOf2.default)(ArrowTableBatchAggregator.prototype), "getBatch", this).call(this);
|
|
59
|
-
|
|
60
38
|
if (batch) {
|
|
61
39
|
this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
|
|
62
40
|
var arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
|
|
@@ -68,49 +46,38 @@ var ArrowTableBatchAggregator = function (_ColumnarTableBatchAg) {
|
|
|
68
46
|
length: batch.length
|
|
69
47
|
};
|
|
70
48
|
}
|
|
71
|
-
|
|
72
49
|
return null;
|
|
73
50
|
}
|
|
74
51
|
}]);
|
|
75
52
|
return ArrowTableBatchAggregator;
|
|
76
53
|
}(_schema.ColumnarTableBatchAggregator);
|
|
77
|
-
|
|
78
54
|
exports.default = ArrowTableBatchAggregator;
|
|
79
|
-
|
|
80
55
|
function getArrowSchema(schema) {
|
|
81
56
|
var arrowFields = [];
|
|
82
|
-
|
|
83
57
|
for (var key in schema) {
|
|
84
58
|
var field = schema[key];
|
|
85
|
-
|
|
86
59
|
if (field.type === Float32Array) {
|
|
87
60
|
var metadata = field;
|
|
88
61
|
var arrowField = new _apacheArrow.Field(field.name, new _apacheArrow.Float32(), field.nullable, metadata);
|
|
89
62
|
arrowFields.push(arrowField);
|
|
90
63
|
}
|
|
91
64
|
}
|
|
92
|
-
|
|
93
65
|
if (arrowFields.length === 0) {
|
|
94
66
|
throw new Error('No arrow convertible fields');
|
|
95
67
|
}
|
|
96
|
-
|
|
97
68
|
return new _apacheArrow.Schema(arrowFields);
|
|
98
69
|
}
|
|
99
70
|
|
|
100
71
|
function getArrowVectors(arrowSchema, data) {
|
|
101
72
|
var arrowVectors = [];
|
|
102
|
-
|
|
103
73
|
var _iterator = _createForOfIteratorHelper(arrowSchema.fields),
|
|
104
|
-
|
|
105
|
-
|
|
74
|
+
_step;
|
|
106
75
|
try {
|
|
107
76
|
for (_iterator.s(); !(_step = _iterator.n()).done;) {
|
|
108
77
|
var field = _step.value;
|
|
109
78
|
var vector = data[field.name];
|
|
110
|
-
|
|
111
79
|
if (vector instanceof Float32Array) {
|
|
112
80
|
var arrowVector = _apacheArrow.Float32Vector.from(vector);
|
|
113
|
-
|
|
114
81
|
arrowVectors.push(arrowVector);
|
|
115
82
|
}
|
|
116
83
|
}
|
|
@@ -119,11 +86,9 @@ function getArrowVectors(arrowSchema, data) {
|
|
|
119
86
|
} finally {
|
|
120
87
|
_iterator.f();
|
|
121
88
|
}
|
|
122
|
-
|
|
123
89
|
if (arrowSchema.fields.length !== arrowVectors.length) {
|
|
124
90
|
throw new Error('Some columns not arrow convertible');
|
|
125
91
|
}
|
|
126
|
-
|
|
127
92
|
return arrowVectors;
|
|
128
93
|
}
|
|
129
94
|
//# sourceMappingURL=arrow-table-batch.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"arrow-table-batch.js","names":["ArrowTableBatchAggregator","schema","options","arrowSchema","batch","getArrowSchema","arrowVectors","getArrowVectors","data","recordBatch","RecordBatch","length","shape","batchType","ColumnarTableBatchAggregator","arrowFields","key","field","type","Float32Array","metadata","arrowField","Field","name","Float32","nullable","push","Error","Schema","fields","vector","arrowVector","Float32Vector","from"],"sources":["../../../src/lib/arrow-table-batch.ts"],"sourcesContent":["import type {ArrowTableBatch} from '@loaders.gl/schema';\nimport {Schema, Field, RecordBatch, Float32Vector, Float32} from 'apache-arrow';\nimport {ColumnarTableBatchAggregator} from '@loaders.gl/schema';\n\nexport default class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {\n arrowSchema: Schema | null;\n\n constructor(schema, options) {\n super(schema, options);\n this.arrowSchema = null;\n }\n\n getBatch(): ArrowTableBatch | null {\n const batch = super.getBatch();\n if (batch) {\n // Get the arrow schema\n this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);\n // Get arrow format vectors\n const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);\n // Create the record batch\n // new RecordBatch(schema, numRows, vectors, ...);\n const recordBatch = new RecordBatch(this.arrowSchema, batch.length, arrowVectors);\n return {\n shape: 'arrow-table',\n batchType: 'data',\n data: recordBatch,\n length: batch.length\n };\n }\n\n return null;\n }\n}\n\n// Convert from a simple loaders.gl schema to an Arrow schema\nfunction getArrowSchema(schema) {\n const arrowFields: Field[] = [];\n for (const key in schema) {\n const field = schema[key];\n if (field.type === Float32Array) {\n const metadata = field; // just store the original field as metadata\n // arrow: new Field(name, nullable, metadata)\n const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);\n arrowFields.push(arrowField);\n }\n }\n if (arrowFields.length === 0) {\n throw new Error('No arrow convertible fields');\n }\n\n return new Schema(arrowFields);\n}\n\n// Convert from simple loaders.gl arrays to arrow vectors\nfunction getArrowVectors(arrowSchema, data) {\n const arrowVectors: any[] = [];\n for (const field of arrowSchema.fields) {\n const vector = data[field.name];\n if (vector instanceof Float32Array) {\n const arrowVector = Float32Vector.from(vector);\n arrowVectors.push(arrowVector);\n }\n }\n if (arrowSchema.fields.length !== arrowVectors.length) {\n throw new Error('Some columns not arrow convertible');\n }\n return arrowVectors;\n}\n"],"mappings":";;;;;;;;;;;;;;;AACA;AACA;AAAgE;AAAA;AAAA;AAAA;AAAA;AAAA,IAE3CA,yBAAyB;EAAA;EAAA;EAG5C,mCAAYC,MAAM,EAAEC,OAAO,EAAE;IAAA;IAAA;IAC3B,0BAAMD,MAAM,EAAEC,OAAO;IAAE;IACvB,MAAKC,WAAW,GAAG,IAAI;IAAC;EAC1B;EAAC;IAAA;IAAA,OAED,oBAAmC;MACjC,IAAMC,KAAK,sHAAmB;MAC9B,IAAIA,KAAK,EAAE;QAET,IAAI,CAACD,WAAW,GAAG,IAAI,CAACA,WAAW,IAAIE,cAAc,CAACD,KAAK,CAACH,MAAM,CAAC;QAEnE,IAAMK,YAAY,GAAGC,eAAe,CAAC,IAAI,CAACJ,WAAW,EAAEC,KAAK,CAACI,IAAI,CAAC;QAGlE,IAAMC,WAAW,GAAG,IAAIC,wBAAW,CAAC,IAAI,CAACP,WAAW,EAAEC,KAAK,CAACO,MAAM,EAAEL,YAAY,CAAC;QACjF,OAAO;UACLM,KAAK,EAAE,aAAa;UACpBC,SAAS,EAAE,MAAM;UACjBL,IAAI,EAAEC,WAAW;UACjBE,MAAM,EAAEP,KAAK,CAACO;QAChB,CAAC;MACH;MAEA,OAAO,IAAI;IACb;EAAC;EAAA;AAAA,EA3BoDG,oCAA4B;AAAA;AA+BnF,SAAST,cAAc,CAACJ,MAAM,EAAE;EAC9B,IAAMc,WAAoB,GAAG,EAAE;EAC/B,KAAK,IAAMC,GAAG,IAAIf,MAAM,EAAE;IACxB,IAAMgB,KAAK,GAAGhB,MAAM,CAACe,GAAG,CAAC;IACzB,IAAIC,KAAK,CAACC,IAAI,KAAKC,YAAY,EAAE;MAC/B,IAAMC,QAAQ,GAAGH,KAAK;MAEtB,IAAMI,UAAU,GAAG,IAAIC,kBAAK,CAACL,KAAK,CAACM,IAAI,EAAE,IAAIC,oBAAO,EAAE,EAAEP,KAAK,CAACQ,QAAQ,EAAEL,QAAQ,CAAC;MACjFL,WAAW,CAACW,IAAI,CAACL,UAAU,CAAC;IAC9B;EACF;EACA,IAAIN,WAAW,CAACJ,MAAM,KAAK,CAAC,EAAE;IAC5B,MAAM,IAAIgB,KAAK,CAAC,6BAA6B,CAAC;EAChD;EAEA,OAAO,IAAIC,mBAAM,CAACb,WAAW,CAAC;AAChC;;AAGA,SAASR,eAAe,CAACJ,WAAW,EAAEK,IAAI,EAAE;EAC1C,IAAMF,YAAmB,GAAG,EAAE;EAAC,2CACXH,WAAW,CAAC0B,MAAM;IAAA;EAAA;IAAtC,oDAAwC;MAAA,IAA7BZ,KAAK;MACd,IAAMa,MAAM,GAAGtB,IAAI,CAACS,KAAK,CAACM,IAAI,CAAC;MAC/B,IAAIO,MAAM,YAAYX,YAAY,EAAE;QAClC,IAAMY,WAAW,GAAGC,0BAAa,CAACC,IAAI,CAACH,MAAM,CAAC;QAC9CxB,YAAY,CAACoB,IAAI,CAACK,WAAW,CAAC;MAChC;IACF;EAAC;IAAA;EAAA;IAAA;EAAA;EACD,IAAI5B,WAAW,CAAC0B,MAAM,CAAClB,MAAM,KAAKL,YAAY,CAACK,MAAM,EAAE;IACrD,MAAM,IAAIgB,KAAK,CAAC,oCAAoC,CAAC;EACvD;EACA,OAAOrB,YAAY;AACrB"}
|
|
@@ -4,24 +4,16 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
value: true
|
|
5
5
|
});
|
|
6
6
|
exports.encodeArrowSync = encodeArrowSync;
|
|
7
|
-
|
|
8
7
|
var _apacheArrow = require("apache-arrow");
|
|
9
|
-
|
|
10
8
|
var _types = require("../types");
|
|
11
|
-
|
|
12
9
|
function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
|
|
13
|
-
|
|
14
10
|
function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
|
|
15
|
-
|
|
16
11
|
function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
|
|
17
|
-
|
|
18
12
|
function encodeArrowSync(data) {
|
|
19
13
|
var vectors = [];
|
|
20
14
|
var arrayNames = [];
|
|
21
|
-
|
|
22
15
|
var _iterator = _createForOfIteratorHelper(data),
|
|
23
|
-
|
|
24
|
-
|
|
16
|
+
_step;
|
|
25
17
|
try {
|
|
26
18
|
for (_iterator.s(); !(_step = _iterator.n()).done;) {
|
|
27
19
|
var arrayData = _step.value;
|
|
@@ -34,9 +26,7 @@ function encodeArrowSync(data) {
|
|
|
34
26
|
} finally {
|
|
35
27
|
_iterator.f();
|
|
36
28
|
}
|
|
37
|
-
|
|
38
29
|
var table = _apacheArrow.Table.new(vectors, arrayNames);
|
|
39
|
-
|
|
40
30
|
var arrowBuffer = table.serialize();
|
|
41
31
|
return arrowBuffer;
|
|
42
32
|
}
|
|
@@ -45,7 +35,6 @@ function createVector(array, type) {
|
|
|
45
35
|
switch (type) {
|
|
46
36
|
case _types.VECTOR_TYPES.DATE:
|
|
47
37
|
return _apacheArrow.DateVector.from(array);
|
|
48
|
-
|
|
49
38
|
case _types.VECTOR_TYPES.FLOAT:
|
|
50
39
|
default:
|
|
51
40
|
return _apacheArrow.FloatVector.from(array);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"encode-arrow.js","names":["encodeArrowSync","data","vectors","arrayNames","arrayData","push","name","arrayVector","createVector","array","type","table","Table","new","arrowBuffer","serialize","VECTOR_TYPES","DATE","DateVector","from","FLOAT","FloatVector"],"sources":["../../../src/lib/encode-arrow.ts"],"sourcesContent":["import {Table, FloatVector, DateVector} from 'apache-arrow';\nimport {AnyArrayType, VECTOR_TYPES} from '../types';\n\ntype ColumnarTable = {\n name: string;\n array: AnyArrayType;\n type: number;\n}[];\n\n/**\n * Encodes set of arrays into the Apache Arrow columnar format\n * https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format\n * @param data - columns data\n * @param options - the writer options\n * @returns - encoded ArrayBuffer\n */\nexport function encodeArrowSync(data: ColumnarTable): ArrayBuffer {\n const vectors: any[] = [];\n const arrayNames: string[] = [];\n for (const arrayData of data) {\n arrayNames.push(arrayData.name);\n const arrayVector = createVector(arrayData.array, arrayData.type);\n vectors.push(arrayVector);\n }\n const table = Table.new(vectors, arrayNames);\n const arrowBuffer = table.serialize();\n return arrowBuffer;\n}\n\n/**\n * Create Arrow Vector from given data and vector type\n * @param array {import('../types').AnyArrayType} - columns data\n * @param type {number} - the writer options\n * @return a vector of one of vector's types defined in the Apache Arrow library\n */\nfunction createVector(array, type) {\n switch (type) {\n case VECTOR_TYPES.DATE:\n return DateVector.from(array);\n case VECTOR_TYPES.FLOAT:\n default:\n return FloatVector.from(array);\n }\n}\n"],"mappings":";;;;;;AAAA;AACA;AAAoD;AAAA;AAAA;AAe7C,SAASA,eAAe,CAACC,IAAmB,EAAe;EAChE,IAAMC,OAAc,GAAG,EAAE;EACzB,IAAMC,UAAoB,GAAG,EAAE;EAAC,2CACRF,IAAI;IAAA;EAAA;IAA5B,oDAA8B;MAAA,IAAnBG,SAAS;MAClBD,UAAU,CAACE,IAAI,CAACD,SAAS,CAACE,IAAI,CAAC;MAC/B,IAAMC,WAAW,GAAGC,YAAY,CAACJ,SAAS,CAACK,KAAK,EAAEL,SAAS,CAACM,IAAI,CAAC;MACjER,OAAO,CAACG,IAAI,CAACE,WAAW,CAAC;IAC3B;EAAC;IAAA;EAAA;IAAA;EAAA;EACD,IAAMI,KAAK,GAAGC,kBAAK,CAACC,GAAG,CAACX,OAAO,EAAEC,UAAU,CAAC;EAC5C,IAAMW,WAAW,GAAGH,KAAK,CAACI,SAAS,EAAE;EACrC,OAAOD,WAAW;AACpB;;AAQA,SAASN,YAAY,CAACC,KAAK,EAAEC,IAAI,EAAE;EACjC,QAAQA,IAAI;IACV,KAAKM,mBAAY,CAACC,IAAI;MACpB,OAAOC,uBAAU,CAACC,IAAI,CAACV,KAAK,CAAC;IAC/B,KAAKO,mBAAY,CAACI,KAAK;IACvB;MACE,OAAOC,wBAAW,CAACF,IAAI,CAACV,KAAK,CAAC;EAAC;AAErC"}
|