@loaders.gl/schema 3.3.0-alpha.1 → 3.3.0-alpha.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/category/common.d.ts +3 -3
- package/dist/category/common.d.ts.map +1 -1
- package/dist/category/gis.d.ts +23 -23
- package/dist/category/gis.d.ts.map +1 -1
- package/dist/category/image/image.d.ts +3 -3
- package/dist/category/image/image.d.ts.map +1 -1
- package/dist/category/mesh/convert-mesh.d.ts +1 -1
- package/dist/category/mesh/convert-mesh.d.ts.map +1 -1
- package/dist/category/mesh/mesh-types.d.ts +4 -4
- package/dist/category/mesh/mesh-types.d.ts.map +1 -1
- package/dist/category/mesh/mesh-utils.d.ts +2 -2
- package/dist/category/mesh/mesh-utils.d.ts.map +1 -1
- package/dist/category/table/table-types.d.ts +8 -8
- package/dist/category/table/table-types.d.ts.map +1 -1
- package/dist/category/texture/texture.d.ts +3 -3
- package/dist/category/texture/texture.d.ts.map +1 -1
- package/dist/es5/bundle.js +0 -1
- package/dist/es5/bundle.js.map +1 -1
- package/dist/es5/category/common.js.map +1 -1
- package/dist/es5/category/gis.js.map +1 -1
- package/dist/es5/category/image/image.js.map +1 -1
- package/dist/es5/category/mesh/convert-mesh.js +2 -12
- package/dist/es5/category/mesh/convert-mesh.js.map +1 -1
- package/dist/es5/category/mesh/deduce-mesh-schema.js +1 -10
- package/dist/es5/category/mesh/deduce-mesh-schema.js.map +1 -1
- package/dist/es5/category/mesh/mesh-to-arrow-table.js +2 -9
- package/dist/es5/category/mesh/mesh-to-arrow-table.js.map +1 -1
- package/dist/es5/category/mesh/mesh-types.js.map +1 -1
- package/dist/es5/category/mesh/mesh-utils.js +1 -6
- package/dist/es5/category/mesh/mesh-utils.js.map +1 -1
- package/dist/es5/category/table/deduce-table-schema.js +0 -11
- package/dist/es5/category/table/deduce-table-schema.js.map +1 -1
- package/dist/es5/category/table/table-types.js.map +1 -1
- package/dist/es5/category/texture/texture.js.map +1 -1
- package/dist/es5/index.js +100 -112
- package/dist/es5/index.js.map +1 -1
- package/dist/es5/lib/arrow/arrow-like-type-utils.js +0 -10
- package/dist/es5/lib/arrow/arrow-like-type-utils.js.map +1 -1
- package/dist/es5/lib/arrow/arrow-type-utils.js +0 -19
- package/dist/es5/lib/arrow/arrow-type-utils.js.map +1 -1
- package/dist/es5/lib/arrow/get-type-info.js +0 -6
- package/dist/es5/lib/arrow/get-type-info.js.map +1 -1
- package/dist/es5/lib/batches/base-table-batch-aggregator.js +0 -13
- package/dist/es5/lib/batches/base-table-batch-aggregator.js.map +1 -1
- package/dist/es5/lib/batches/columnar-table-batch-aggregator.js +6 -23
- package/dist/es5/lib/batches/columnar-table-batch-aggregator.js.map +1 -1
- package/dist/es5/lib/batches/row-table-batch-aggregator.js +0 -13
- package/dist/es5/lib/batches/row-table-batch-aggregator.js.map +1 -1
- package/dist/es5/lib/batches/table-batch-aggregator.js.map +1 -1
- package/dist/es5/lib/batches/table-batch-builder.js +21 -37
- package/dist/es5/lib/batches/table-batch-builder.js.map +1 -1
- package/dist/es5/lib/schema/impl/enum.js +0 -1
- package/dist/es5/lib/schema/impl/enum.js.map +1 -1
- package/dist/es5/lib/schema/impl/field.js +0 -7
- package/dist/es5/lib/schema/impl/field.js.map +1 -1
- package/dist/es5/lib/schema/impl/schema.js +6 -35
- package/dist/es5/lib/schema/impl/schema.js.map +1 -1
- package/dist/es5/lib/schema/impl/type.js +74 -248
- package/dist/es5/lib/schema/impl/type.js.map +1 -1
- package/dist/es5/lib/schema/schema.js +70 -74
- package/dist/es5/lib/schema/schema.js.map +1 -1
- package/dist/es5/lib/schema-utils/deduce-column-type.js +0 -3
- package/dist/es5/lib/schema-utils/deduce-column-type.js.map +1 -1
- package/dist/es5/lib/utils/assert.js +0 -1
- package/dist/es5/lib/utils/assert.js.map +1 -1
- package/dist/es5/lib/utils/async-queue.js +10 -50
- package/dist/es5/lib/utils/async-queue.js.map +1 -1
- package/dist/es5/lib/utils/row-utils.js +1 -10
- package/dist/es5/lib/utils/row-utils.js.map +1 -1
- package/dist/es5/types.js.map +1 -1
- package/dist/esm/bundle.js +1 -1
- package/dist/esm/bundle.js.map +1 -1
- package/dist/esm/category/common.js.map +1 -1
- package/dist/esm/category/gis.js.map +1 -1
- package/dist/esm/category/image/image.js.map +1 -1
- package/dist/esm/category/mesh/convert-mesh.js +1 -5
- package/dist/esm/category/mesh/convert-mesh.js.map +1 -1
- package/dist/esm/category/mesh/deduce-mesh-schema.js +2 -6
- package/dist/esm/category/mesh/deduce-mesh-schema.js.map +1 -1
- package/dist/esm/category/mesh/mesh-to-arrow-table.js +1 -3
- package/dist/esm/category/mesh/mesh-to-arrow-table.js.map +1 -1
- package/dist/esm/category/mesh/mesh-types.js.map +1 -1
- package/dist/esm/category/mesh/mesh-utils.js +3 -5
- package/dist/esm/category/mesh/mesh-utils.js.map +1 -1
- package/dist/esm/category/table/deduce-table-schema.js +2 -11
- package/dist/esm/category/table/deduce-table-schema.js.map +1 -1
- package/dist/esm/category/table/table-types.js.map +1 -1
- package/dist/esm/category/texture/texture.js.map +1 -1
- package/dist/esm/index.js +5 -0
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/lib/arrow/arrow-like-type-utils.js +0 -8
- package/dist/esm/lib/arrow/arrow-like-type-utils.js.map +1 -1
- package/dist/esm/lib/arrow/arrow-type-utils.js +0 -16
- package/dist/esm/lib/arrow/arrow-type-utils.js.map +1 -1
- package/dist/esm/lib/arrow/get-type-info.js +0 -3
- package/dist/esm/lib/arrow/get-type-info.js.map +1 -1
- package/dist/esm/lib/batches/base-table-batch-aggregator.js +0 -16
- package/dist/esm/lib/batches/base-table-batch-aggregator.js.map +1 -1
- package/dist/esm/lib/batches/columnar-table-batch-aggregator.js +0 -20
- package/dist/esm/lib/batches/columnar-table-batch-aggregator.js.map +1 -1
- package/dist/esm/lib/batches/row-table-batch-aggregator.js +0 -17
- package/dist/esm/lib/batches/row-table-batch-aggregator.js.map +1 -1
- package/dist/esm/lib/batches/table-batch-aggregator.js.map +1 -1
- package/dist/esm/lib/batches/table-batch-builder.js +4 -35
- package/dist/esm/lib/batches/table-batch-builder.js.map +1 -1
- package/dist/esm/lib/schema/impl/enum.js +2 -1
- package/dist/esm/lib/schema/impl/enum.js.map +1 -1
- package/dist/esm/lib/schema/impl/field.js +3 -10
- package/dist/esm/lib/schema/impl/field.js.map +1 -1
- package/dist/esm/lib/schema/impl/schema.js +9 -20
- package/dist/esm/lib/schema/impl/schema.js.map +1 -1
- package/dist/esm/lib/schema/impl/type.js +19 -113
- package/dist/esm/lib/schema/impl/type.js.map +1 -1
- package/dist/esm/lib/schema/schema.js +3 -1
- package/dist/esm/lib/schema/schema.js.map +1 -1
- package/dist/esm/lib/schema-utils/deduce-column-type.js +2 -2
- package/dist/esm/lib/schema-utils/deduce-column-type.js.map +1 -1
- package/dist/esm/lib/utils/assert.js +1 -0
- package/dist/esm/lib/utils/assert.js.map +1 -1
- package/dist/esm/lib/utils/async-queue.js +3 -25
- package/dist/esm/lib/utils/async-queue.js.map +1 -1
- package/dist/esm/lib/utils/row-utils.js +2 -8
- package/dist/esm/lib/utils/row-utils.js.map +1 -1
- package/dist/esm/types.js.map +1 -1
- package/dist/index.js +1 -1
- package/dist/lib/batches/columnar-table-batch-aggregator.d.ts +1 -1
- package/dist/lib/batches/columnar-table-batch-aggregator.d.ts.map +1 -1
- package/dist/lib/batches/row-table-batch-aggregator.js +1 -1
- package/dist/lib/batches/table-batch-builder.d.ts +2 -2
- package/dist/lib/batches/table-batch-builder.d.ts.map +1 -1
- package/dist/lib/schema/impl/schema.d.ts +1 -1
- package/dist/lib/schema/impl/schema.d.ts.map +1 -1
- package/dist/lib/schema/impl/type.d.ts +4 -4
- package/dist/lib/schema/impl/type.d.ts.map +1 -1
- package/dist/lib/schema-utils/deduce-column-type.js +1 -1
- package/dist/types.d.ts +4 -4
- package/dist/types.d.ts.map +1 -1
- package/package.json +3 -4
|
@@ -5,66 +5,47 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
5
5
|
});
|
|
6
6
|
exports.getArrowType = getArrowType;
|
|
7
7
|
exports.getArrowVector = getArrowVector;
|
|
8
|
-
|
|
9
8
|
var _Arrow = require("apache-arrow/Arrow.dom");
|
|
10
|
-
|
|
11
9
|
function getArrowType(array) {
|
|
12
10
|
switch (array.constructor) {
|
|
13
11
|
case Int8Array:
|
|
14
12
|
return new _Arrow.Int8();
|
|
15
|
-
|
|
16
13
|
case Uint8Array:
|
|
17
14
|
return new _Arrow.Uint8();
|
|
18
|
-
|
|
19
15
|
case Int16Array:
|
|
20
16
|
return new _Arrow.Int16();
|
|
21
|
-
|
|
22
17
|
case Uint16Array:
|
|
23
18
|
return new _Arrow.Uint16();
|
|
24
|
-
|
|
25
19
|
case Int32Array:
|
|
26
20
|
return new _Arrow.Int32();
|
|
27
|
-
|
|
28
21
|
case Uint32Array:
|
|
29
22
|
return new _Arrow.Uint32();
|
|
30
|
-
|
|
31
23
|
case Float32Array:
|
|
32
24
|
return new _Arrow.Float32();
|
|
33
|
-
|
|
34
25
|
case Float64Array:
|
|
35
26
|
return new _Arrow.Float64();
|
|
36
|
-
|
|
37
27
|
default:
|
|
38
28
|
throw new Error('array type not supported');
|
|
39
29
|
}
|
|
40
30
|
}
|
|
41
|
-
|
|
42
31
|
function getArrowVector(array) {
|
|
43
32
|
switch (array.constructor) {
|
|
44
33
|
case Int8Array:
|
|
45
34
|
return _Arrow.Int8Vector.from(array);
|
|
46
|
-
|
|
47
35
|
case Uint8Array:
|
|
48
36
|
return _Arrow.Uint8Vector.from(array);
|
|
49
|
-
|
|
50
37
|
case Int16Array:
|
|
51
38
|
return _Arrow.Int16Vector.from(array);
|
|
52
|
-
|
|
53
39
|
case Uint16Array:
|
|
54
40
|
return _Arrow.Uint16Vector.from(array);
|
|
55
|
-
|
|
56
41
|
case Int32Array:
|
|
57
42
|
return _Arrow.Int32Vector.from(array);
|
|
58
|
-
|
|
59
43
|
case Uint32Array:
|
|
60
44
|
return _Arrow.Uint32Vector.from(array);
|
|
61
|
-
|
|
62
45
|
case Float32Array:
|
|
63
46
|
return _Arrow.Float32Vector.from(array);
|
|
64
|
-
|
|
65
47
|
case Float64Array:
|
|
66
48
|
return _Arrow.Float64Vector.from(array);
|
|
67
|
-
|
|
68
49
|
default:
|
|
69
50
|
throw new Error('array type not supported');
|
|
70
51
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"arrow-type-utils.js","names":["getArrowType","array","constructor","Int8Array","Int8","Uint8Array","Uint8","Int16Array","Int16","Uint16Array","Uint16","Int32Array","Int32","Uint32Array","Uint32","Float32Array","Float32","Float64Array","Float64","Error","getArrowVector","Int8Vector","from","Uint8Vector","Int16Vector","Uint16Vector","Int32Vector","Uint32Vector","Float32Vector","Float64Vector"],"sources":["../../../../src/lib/arrow/arrow-type-utils.ts"],"sourcesContent":["import type {TypedArray} from '../../types';\nimport {\n DataType,\n Float32,\n Float64,\n Int16,\n Int32,\n Int8,\n Uint16,\n Uint32,\n Uint8,\n Int8Vector,\n Uint8Vector,\n Int16Vector,\n Uint16Vector,\n Int32Vector,\n Uint32Vector,\n Float32Vector,\n Float64Vector\n} from 'apache-arrow/Arrow.dom';\nimport {AbstractVector} from 'apache-arrow/vector';\n\nexport function getArrowType(array: TypedArray): DataType {\n switch (array.constructor) {\n case Int8Array:\n return new Int8();\n case Uint8Array:\n return new Uint8();\n case Int16Array:\n return new Int16();\n case Uint16Array:\n return new Uint16();\n case Int32Array:\n return new Int32();\n case Uint32Array:\n return new Uint32();\n case Float32Array:\n return new Float32();\n case Float64Array:\n return new Float64();\n default:\n throw new Error('array type not supported');\n }\n}\n\nexport function getArrowVector(array: TypedArray): AbstractVector {\n switch (array.constructor) {\n case Int8Array:\n return Int8Vector.from(array);\n case Uint8Array:\n return Uint8Vector.from(array);\n case Int16Array:\n return Int16Vector.from(array);\n case Uint16Array:\n return Uint16Vector.from(array);\n case Int32Array:\n return Int32Vector.from(array);\n case Uint32Array:\n return Uint32Vector.from(array);\n case Float32Array:\n return Float32Vector.from(array);\n case Float64Array:\n return Float64Vector.from(array);\n default:\n throw new Error('array type not supported');\n }\n}\n"],"mappings":";;;;;;;AACA;AAqBO,SAASA,YAAY,CAACC,KAAiB,EAAY;EACxD,QAAQA,KAAK,CAACC,WAAW;IACvB,KAAKC,SAAS;MACZ,OAAO,IAAIC,WAAI,EAAE;IACnB,KAAKC,UAAU;MACb,OAAO,IAAIC,YAAK,EAAE;IACpB,KAAKC,UAAU;MACb,OAAO,IAAIC,YAAK,EAAE;IACpB,KAAKC,WAAW;MACd,OAAO,IAAIC,aAAM,EAAE;IACrB,KAAKC,UAAU;MACb,OAAO,IAAIC,YAAK,EAAE;IACpB,KAAKC,WAAW;MACd,OAAO,IAAIC,aAAM,EAAE;IACrB,KAAKC,YAAY;MACf,OAAO,IAAIC,cAAO,EAAE;IACtB,KAAKC,YAAY;MACf,OAAO,IAAIC,cAAO,EAAE;IACtB;MACE,MAAM,IAAIC,KAAK,CAAC,0BAA0B,CAAC;EAAC;AAElD;AAEO,SAASC,cAAc,CAACnB,KAAiB,EAAkB;EAChE,QAAQA,KAAK,CAACC,WAAW;IACvB,KAAKC,SAAS;MACZ,OAAOkB,iBAAU,CAACC,IAAI,CAACrB,KAAK,CAAC;IAC/B,KAAKI,UAAU;MACb,OAAOkB,kBAAW,CAACD,IAAI,CAACrB,KAAK,CAAC;IAChC,KAAKM,UAAU;MACb,OAAOiB,kBAAW,CAACF,IAAI,CAACrB,KAAK,CAAC;IAChC,KAAKQ,WAAW;MACd,OAAOgB,mBAAY,CAACH,IAAI,CAACrB,KAAK,CAAC;IACjC,KAAKU,UAAU;MACb,OAAOe,kBAAW,CAACJ,IAAI,CAACrB,KAAK,CAAC;IAChC,KAAKY,WAAW;MACd,OAAOc,mBAAY,CAACL,IAAI,CAACrB,KAAK,CAAC;IACjC,KAAKc,YAAY;MACf,OAAOa,oBAAa,CAACN,IAAI,CAACrB,KAAK,CAAC;IAClC,KAAKgB,YAAY;MACf,OAAOY,oBAAa,CAACP,IAAI,CAACrB,KAAK,CAAC;IAClC;MACE,MAAM,IAAIkB,KAAK,CAAC,0BAA0B,CAAC;EAAC;AAElD"}
|
|
@@ -4,9 +4,7 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
value: true
|
|
5
5
|
});
|
|
6
6
|
exports.getTypeInfo = getTypeInfo;
|
|
7
|
-
|
|
8
7
|
var _schema = require("../schema/schema");
|
|
9
|
-
|
|
10
8
|
function getTypeInfo(arrowTypeLike) {
|
|
11
9
|
return {
|
|
12
10
|
typeId: arrowTypeLike.typeId,
|
|
@@ -16,18 +14,14 @@ function getTypeInfo(arrowTypeLike) {
|
|
|
16
14
|
precision: arrowTypeLike.precision
|
|
17
15
|
};
|
|
18
16
|
}
|
|
19
|
-
|
|
20
17
|
var ReverseType = null;
|
|
21
|
-
|
|
22
18
|
function getTypeKey(typeKey) {
|
|
23
19
|
if (!ReverseType) {
|
|
24
20
|
ReverseType = {};
|
|
25
|
-
|
|
26
21
|
for (var _key in _schema.Type) {
|
|
27
22
|
ReverseType[_schema.Type[_key]] = _key;
|
|
28
23
|
}
|
|
29
24
|
}
|
|
30
|
-
|
|
31
25
|
return ReverseType[typeKey];
|
|
32
26
|
}
|
|
33
27
|
//# sourceMappingURL=get-type-info.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"get-type-info.js","names":["getTypeInfo","arrowTypeLike","typeId","ArrayType","typeName","toString","typeEnumName","getTypeKey","precision","ReverseType","typeKey","key","Type"],"sources":["../../../../src/lib/arrow/get-type-info.ts"],"sourcesContent":["import {Type} from '../schema/schema';\nimport {AnyArray} from '../../types';\n\n/**\n * Gets type information from an Arrow type object or \"mock\" Arrow type object\n * @param arrowTypeLike Arrow Type or type object of similar shape\n */\nexport function getTypeInfo(arrowTypeLike: any): {\n typeId: Type;\n ArrayType: AnyArray;\n typeName: string;\n typeEnumName?: string;\n precision?: number;\n} {\n return {\n typeId: arrowTypeLike.typeId,\n ArrayType: arrowTypeLike.ArrayType,\n typeName: arrowTypeLike.toString(),\n typeEnumName: getTypeKey(arrowTypeLike.typeId),\n precision: arrowTypeLike.precision\n };\n}\n\nlet ReverseType: {[key: string]: string} | null = null;\n\nfunction getTypeKey(typeKey) {\n if (!ReverseType) {\n ReverseType = {};\n for (const key in Type) {\n ReverseType[Type[key]] = key;\n }\n }\n\n return ReverseType[typeKey];\n}\n"],"mappings":";;;;;;AAAA;AAOO,SAASA,WAAW,CAACC,aAAkB,EAM5C;EACA,OAAO;IACLC,MAAM,EAAED,aAAa,CAACC,MAAM;IAC5BC,SAAS,EAAEF,aAAa,CAACE,SAAS;IAClCC,QAAQ,EAAEH,aAAa,CAACI,QAAQ,EAAE;IAClCC,YAAY,EAAEC,UAAU,CAACN,aAAa,CAACC,MAAM,CAAC;IAC9CM,SAAS,EAAEP,aAAa,CAACO;EAC3B,CAAC;AACH;AAEA,IAAIC,WAA2C,GAAG,IAAI;AAEtD,SAASF,UAAU,CAACG,OAAO,EAAE;EAC3B,IAAI,CAACD,WAAW,EAAE;IAChBA,WAAW,GAAG,CAAC,CAAC;IAChB,KAAK,IAAME,IAAG,IAAIC,YAAI,EAAE;MACtBH,WAAW,CAACG,YAAI,CAACD,IAAG,CAAC,CAAC,GAAGA,IAAG;IAC9B;EACF;EAEA,OAAOF,WAAW,CAACC,OAAO,CAAC;AAC7B"}
|
|
@@ -1,20 +1,14 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
4
|
Object.defineProperty(exports, "__esModule", {
|
|
6
5
|
value: true
|
|
7
6
|
});
|
|
8
7
|
exports.default = void 0;
|
|
9
|
-
|
|
10
8
|
var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
|
|
11
|
-
|
|
12
9
|
var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
|
|
13
|
-
|
|
14
10
|
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
15
|
-
|
|
16
11
|
var DEFAULT_ROW_COUNT = 100;
|
|
17
|
-
|
|
18
12
|
var RowTableBatchAggregator = function () {
|
|
19
13
|
function RowTableBatchAggregator(schema, options) {
|
|
20
14
|
(0, _classCallCheck2.default)(this, RowTableBatchAggregator);
|
|
@@ -29,13 +23,11 @@ var RowTableBatchAggregator = function () {
|
|
|
29
23
|
|
|
30
24
|
if (!Array.isArray(schema)) {
|
|
31
25
|
this._headers = [];
|
|
32
|
-
|
|
33
26
|
for (var key in schema) {
|
|
34
27
|
this._headers[schema[key].index] = schema[key].name;
|
|
35
28
|
}
|
|
36
29
|
}
|
|
37
30
|
}
|
|
38
|
-
|
|
39
31
|
(0, _createClass2.default)(RowTableBatchAggregator, [{
|
|
40
32
|
key: "rowCount",
|
|
41
33
|
value: function rowCount() {
|
|
@@ -47,7 +39,6 @@ var RowTableBatchAggregator = function () {
|
|
|
47
39
|
if (Number.isFinite(cursor)) {
|
|
48
40
|
this.cursor = cursor;
|
|
49
41
|
}
|
|
50
|
-
|
|
51
42
|
this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
|
|
52
43
|
this.rows[this.length] = row;
|
|
53
44
|
this.length++;
|
|
@@ -58,7 +49,6 @@ var RowTableBatchAggregator = function () {
|
|
|
58
49
|
if (Number.isFinite(cursor)) {
|
|
59
50
|
this.cursor = cursor;
|
|
60
51
|
}
|
|
61
|
-
|
|
62
52
|
this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
|
|
63
53
|
this.rows[this.length] = row;
|
|
64
54
|
this.length++;
|
|
@@ -67,11 +57,9 @@ var RowTableBatchAggregator = function () {
|
|
|
67
57
|
key: "getBatch",
|
|
68
58
|
value: function getBatch() {
|
|
69
59
|
var rows = this.rows;
|
|
70
|
-
|
|
71
60
|
if (!rows) {
|
|
72
61
|
return null;
|
|
73
62
|
}
|
|
74
|
-
|
|
75
63
|
rows = rows.slice(0, this.length);
|
|
76
64
|
this.rows = null;
|
|
77
65
|
var batch = {
|
|
@@ -87,6 +75,5 @@ var RowTableBatchAggregator = function () {
|
|
|
87
75
|
}]);
|
|
88
76
|
return RowTableBatchAggregator;
|
|
89
77
|
}();
|
|
90
|
-
|
|
91
78
|
exports.default = RowTableBatchAggregator;
|
|
92
79
|
//# sourceMappingURL=base-table-batch-aggregator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"base-table-batch-aggregator.js","names":["DEFAULT_ROW_COUNT","RowTableBatchAggregator","schema","options","Array","isArray","_headers","key","index","name","length","row","cursor","Number","isFinite","rows","slice","batch","shape","batchType","data"],"sources":["../../../../src/lib/batches/base-table-batch-aggregator.ts"],"sourcesContent":["import type {Schema} from '../schema/schema';\nimport type {TableBatch} from '../../category/table/table-types';\nimport {TableBatchAggregator, TableBatchOptions} from './table-batch-aggregator';\n\nconst DEFAULT_ROW_COUNT = 100;\n\nexport default class RowTableBatchAggregator implements TableBatchAggregator {\n schema: Schema;\n options: TableBatchOptions;\n\n length: number = 0;\n rows: any[] | null = null;\n cursor: number = 0;\n private _headers: string[] = [];\n\n constructor(schema: Schema, options: TableBatchOptions) {\n this.options = options;\n this.schema = schema;\n\n // schema is an array if there're no headers\n // object if there are headers\n if (!Array.isArray(schema)) {\n this._headers = [];\n for (const key in schema) {\n this._headers[schema[key].index] = schema[key].name;\n }\n }\n }\n\n rowCount(): number {\n return this.length;\n }\n\n addArrayRow(row: any[], cursor?: number): void {\n if (Number.isFinite(cursor)) {\n this.cursor = cursor as number;\n }\n\n this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);\n this.rows[this.length] = row;\n this.length++;\n }\n\n addObjectRow(row: {[columnName: string]: any}, cursor?: number): void {\n if (Number.isFinite(cursor)) {\n this.cursor = cursor as number;\n }\n\n this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);\n this.rows[this.length] = row;\n this.length++;\n }\n\n getBatch(): TableBatch | null {\n let rows = this.rows;\n if (!rows) {\n return null;\n }\n\n rows = rows.slice(0, this.length);\n this.rows = null;\n\n const batch: TableBatch = {\n shape: this.options.shape,\n batchType: 'data',\n data: rows,\n length: this.length,\n schema: this.schema,\n cursor: this.cursor\n };\n\n return batch;\n }\n}\n"],"mappings":";;;;;;;;;;AAIA,IAAMA,iBAAiB,GAAG,GAAG;AAAC,IAETC,uBAAuB;EAS1C,iCAAYC,MAAc,EAAEC,OAA0B,EAAE;IAAA;IAAA;IAAA;IAAA,8CALvC,CAAC;IAAA,4CACG,IAAI;IAAA,8CACR,CAAC;IAAA,gDACW,EAAE;IAG7B,IAAI,CAACA,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACD,MAAM,GAAGA,MAAM;;IAIpB,IAAI,CAACE,KAAK,CAACC,OAAO,CAACH,MAAM,CAAC,EAAE;MAC1B,IAAI,CAACI,QAAQ,GAAG,EAAE;MAClB,KAAK,IAAMC,GAAG,IAAIL,MAAM,EAAE;QACxB,IAAI,CAACI,QAAQ,CAACJ,MAAM,CAACK,GAAG,CAAC,CAACC,KAAK,CAAC,GAAGN,MAAM,CAACK,GAAG,CAAC,CAACE,IAAI;MACrD;IACF;EACF;EAAC;IAAA;IAAA,OAED,oBAAmB;MACjB,OAAO,IAAI,CAACC,MAAM;IACpB;EAAC;IAAA;IAAA,OAED,qBAAYC,GAAU,EAAEC,MAAe,EAAQ;MAC7C,IAAIC,MAAM,CAACC,QAAQ,CAACF,MAAM,CAAC,EAAE;QAC3B,IAAI,CAACA,MAAM,GAAGA,MAAgB;MAChC;MAEA,IAAI,CAACG,IAAI,GAAG,IAAI,CAACA,IAAI,IAAI,IAAIX,KAAK,CAACJ,iBAAiB,CAAC;MACrD,IAAI,CAACe,IAAI,CAAC,IAAI,CAACL,MAAM,CAAC,GAAGC,GAAG;MAC5B,IAAI,CAACD,MAAM,EAAE;IACf;EAAC;IAAA;IAAA,OAED,sBAAaC,GAAgC,EAAEC,MAAe,EAAQ;MACpE,IAAIC,MAAM,CAACC,QAAQ,CAACF,MAAM,CAAC,EAAE;QAC3B,IAAI,CAACA,MAAM,GAAGA,MAAgB;MAChC;MAEA,IAAI,CAACG,IAAI,GAAG,IAAI,CAACA,IAAI,IAAI,IAAIX,KAAK,CAACJ,iBAAiB,CAAC;MACrD,IAAI,CAACe,IAAI,CAAC,IAAI,CAACL,MAAM,CAAC,GAAGC,GAAG;MAC5B,IAAI,CAACD,MAAM,EAAE;IACf;EAAC;IAAA;IAAA,OAED,oBAA8B;MAC5B,IAAIK,IAAI,GAAG,IAAI,CAACA,IAAI;MACpB,IAAI,CAACA,IAAI,EAAE;QACT,OAAO,IAAI;MACb;MAEAA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,CAAC,EAAE,IAAI,CAACN,MAAM,CAAC;MACjC,IAAI,CAACK,IAAI,GAAG,IAAI;MAEhB,IAAME,KAAiB,GAAG;QACxBC,KAAK,EAAE,IAAI,CAACf,OAAO,CAACe,KAAK;QACzBC,SAAS,EAAE,MAAM;QACjBC,IAAI,EAAEL,IAAI;QACVL,MAAM,EAAE,IAAI,CAACA,MAAM;QACnBR,MAAM,EAAE,IAAI,CAACA,MAAM;QACnBU,MAAM,EAAE,IAAI,CAACA;MACf,CAAC;MAED,OAAOK,KAAK;IACd;EAAC;EAAA;AAAA;AAAA"}
|
|
@@ -1,22 +1,15 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
4
|
Object.defineProperty(exports, "__esModule", {
|
|
6
5
|
value: true
|
|
7
6
|
});
|
|
8
7
|
exports.default = void 0;
|
|
9
|
-
|
|
10
8
|
var _slicedToArray2 = _interopRequireDefault(require("@babel/runtime/helpers/slicedToArray"));
|
|
11
|
-
|
|
12
9
|
var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
|
|
13
|
-
|
|
14
10
|
var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
|
|
15
|
-
|
|
16
11
|
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
17
|
-
|
|
18
12
|
var DEFAULT_ROW_COUNT = 100;
|
|
19
|
-
|
|
20
13
|
var ColumnarTableBatchAggregator = function () {
|
|
21
14
|
function ColumnarTableBatchAggregator(schema, options) {
|
|
22
15
|
(0, _classCallCheck2.default)(this, ColumnarTableBatchAggregator);
|
|
@@ -25,10 +18,8 @@ var ColumnarTableBatchAggregator = function () {
|
|
|
25
18
|
(0, _defineProperty2.default)(this, "allocated", 0);
|
|
26
19
|
(0, _defineProperty2.default)(this, "columns", {});
|
|
27
20
|
this.schema = schema;
|
|
28
|
-
|
|
29
21
|
this._reallocateColumns();
|
|
30
22
|
}
|
|
31
|
-
|
|
32
23
|
(0, _createClass2.default)(ColumnarTableBatchAggregator, [{
|
|
33
24
|
key: "rowCount",
|
|
34
25
|
value: function rowCount() {
|
|
@@ -38,31 +29,25 @@ var ColumnarTableBatchAggregator = function () {
|
|
|
38
29
|
key: "addArrayRow",
|
|
39
30
|
value: function addArrayRow(row) {
|
|
40
31
|
this._reallocateColumns();
|
|
41
|
-
|
|
42
32
|
var i = 0;
|
|
43
|
-
|
|
44
33
|
for (var fieldName in this.columns) {
|
|
45
34
|
this.columns[fieldName][this.length] = row[i++];
|
|
46
35
|
}
|
|
47
|
-
|
|
48
36
|
this.length++;
|
|
49
37
|
}
|
|
50
38
|
}, {
|
|
51
39
|
key: "addObjectRow",
|
|
52
40
|
value: function addObjectRow(row) {
|
|
53
41
|
this._reallocateColumns();
|
|
54
|
-
|
|
55
42
|
for (var fieldName in row) {
|
|
56
43
|
this.columns[fieldName][this.length] = row[fieldName];
|
|
57
44
|
}
|
|
58
|
-
|
|
59
45
|
this.length++;
|
|
60
46
|
}
|
|
61
47
|
}, {
|
|
62
48
|
key: "getBatch",
|
|
63
49
|
value: function getBatch() {
|
|
64
50
|
this._pruneColumns();
|
|
65
|
-
|
|
66
51
|
var columns = Array.isArray(this.schema) ? this.columns : {};
|
|
67
52
|
|
|
68
53
|
if (!Array.isArray(this.schema)) {
|
|
@@ -71,7 +56,6 @@ var ColumnarTableBatchAggregator = function () {
|
|
|
71
56
|
columns[field.name] = this.columns[field.index];
|
|
72
57
|
}
|
|
73
58
|
}
|
|
74
|
-
|
|
75
59
|
this.columns = {};
|
|
76
60
|
var batch = {
|
|
77
61
|
shape: 'columnar-table',
|
|
@@ -82,21 +66,22 @@ var ColumnarTableBatchAggregator = function () {
|
|
|
82
66
|
};
|
|
83
67
|
return batch;
|
|
84
68
|
}
|
|
69
|
+
|
|
85
70
|
}, {
|
|
86
71
|
key: "_reallocateColumns",
|
|
87
|
-
value:
|
|
72
|
+
value:
|
|
73
|
+
|
|
74
|
+
function _reallocateColumns() {
|
|
88
75
|
if (this.length < this.allocated) {
|
|
89
76
|
return;
|
|
90
77
|
}
|
|
91
78
|
|
|
92
79
|
this.allocated = this.allocated > 0 ? this.allocated *= 2 : DEFAULT_ROW_COUNT;
|
|
93
80
|
this.columns = {};
|
|
94
|
-
|
|
95
81
|
for (var fieldName in this.schema) {
|
|
96
82
|
var field = this.schema[fieldName];
|
|
97
83
|
var ArrayType = field.type || Float32Array;
|
|
98
84
|
var oldColumn = this.columns[field.index];
|
|
99
|
-
|
|
100
85
|
if (oldColumn && ArrayBuffer.isView(oldColumn)) {
|
|
101
86
|
var typedArray = new ArrayType(this.allocated);
|
|
102
87
|
typedArray.set(oldColumn);
|
|
@@ -114,15 +99,13 @@ var ColumnarTableBatchAggregator = function () {
|
|
|
114
99
|
value: function _pruneColumns() {
|
|
115
100
|
for (var _i = 0, _Object$entries = Object.entries(this.columns); _i < _Object$entries.length; _i++) {
|
|
116
101
|
var _Object$entries$_i = (0, _slicedToArray2.default)(_Object$entries[_i], 2),
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
102
|
+
_columnName = _Object$entries$_i[0],
|
|
103
|
+
column = _Object$entries$_i[1];
|
|
120
104
|
this.columns[_columnName] = column.slice(0, this.length);
|
|
121
105
|
}
|
|
122
106
|
}
|
|
123
107
|
}]);
|
|
124
108
|
return ColumnarTableBatchAggregator;
|
|
125
109
|
}();
|
|
126
|
-
|
|
127
110
|
exports.default = ColumnarTableBatchAggregator;
|
|
128
111
|
//# sourceMappingURL=columnar-table-batch-aggregator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"columnar-table-batch-aggregator.js","names":["DEFAULT_ROW_COUNT","ColumnarTableBatchAggregator","schema","options","_reallocateColumns","length","row","i","fieldName","columns","_pruneColumns","Array","isArray","field","name","index","batch","shape","batchType","data","allocated","ArrayType","type","Float32Array","oldColumn","ArrayBuffer","isView","typedArray","set","Object","entries","columnName","column","slice"],"sources":["../../../../src/lib/batches/columnar-table-batch-aggregator.ts"],"sourcesContent":["import type {Schema} from '../schema/schema';\nimport type {ColumnarTableBatch, ArrowTableBatch} from '../../category/table/table-types';\nimport {TableBatchAggregator} from './table-batch-aggregator';\n\ntype ColumnarTableBatchOptions = {};\n\nconst DEFAULT_ROW_COUNT = 100;\n\nexport default class ColumnarTableBatchAggregator implements TableBatchAggregator {\n schema: Schema;\n length: number = 0;\n allocated: number = 0;\n columns: {[columnName: string]: any[]} = {};\n\n constructor(schema: Schema, options: ColumnarTableBatchOptions) {\n this.schema = schema;\n this._reallocateColumns();\n }\n\n rowCount(): number {\n return this.length;\n }\n\n addArrayRow(row: any[]) {\n // If user keeps pushing rows beyond batch size, reallocate\n this._reallocateColumns();\n let i = 0;\n // TODO what if no csv header, columns not populated?\n for (const fieldName in this.columns) {\n this.columns[fieldName][this.length] = row[i++];\n }\n this.length++;\n }\n\n addObjectRow(row: {[columnName: string]: any}): void {\n // If user keeps pushing rows beyond batch size, reallocate\n this._reallocateColumns();\n for (const fieldName in row) {\n this.columns[fieldName][this.length] = row[fieldName];\n }\n this.length++;\n }\n\n getBatch(): ColumnarTableBatch | ArrowTableBatch | null {\n this._pruneColumns();\n const columns = Array.isArray(this.schema) ? this.columns : {};\n\n // schema is an array if there're no headers\n // object if there are headers\n // columns should match schema format\n if (!Array.isArray(this.schema)) {\n for (const fieldName in this.schema) {\n const field = this.schema[fieldName];\n columns[field.name] = this.columns[field.index];\n }\n }\n\n this.columns = {};\n\n const batch: ColumnarTableBatch = {\n shape: 'columnar-table',\n batchType: 'data',\n data: columns,\n schema: this.schema,\n length: this.length\n };\n\n return batch;\n }\n\n // HELPERS\n\n _reallocateColumns() {\n if (this.length < this.allocated) {\n return;\n }\n\n // @ts-ignore TODO\n this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;\n this.columns = {};\n\n for (const fieldName in this.schema) {\n const field = this.schema[fieldName];\n const ArrayType = field.type || Float32Array;\n const oldColumn = this.columns[field.index];\n\n if (oldColumn && ArrayBuffer.isView(oldColumn)) {\n // Copy the old data to the new array\n const typedArray = new ArrayType(this.allocated);\n typedArray.set(oldColumn);\n this.columns[field.index] = typedArray;\n } else if (oldColumn) {\n // Plain array\n oldColumn.length = this.allocated;\n this.columns[field.index] = oldColumn;\n } else {\n // Create new\n this.columns[field.index] = new ArrayType(this.allocated);\n }\n }\n }\n\n _pruneColumns() {\n for (const [columnName, column] of Object.entries(this.columns)) {\n this.columns[columnName] = column.slice(0, this.length);\n }\n }\n}\n"],"mappings":";;;;;;;;;;;AAMA,IAAMA,iBAAiB,GAAG,GAAG;AAAC,IAETC,4BAA4B;EAM/C,sCAAYC,MAAc,EAAEC,OAAkC,EAAE;IAAA;IAAA;IAAA,8CAJ/C,CAAC;IAAA,iDACE,CAAC;IAAA,+CACoB,CAAC,CAAC;IAGzC,IAAI,CAACD,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACE,kBAAkB,EAAE;EAC3B;EAAC;IAAA;IAAA,OAED,oBAAmB;MACjB,OAAO,IAAI,CAACC,MAAM;IACpB;EAAC;IAAA;IAAA,OAED,qBAAYC,GAAU,EAAE;MAEtB,IAAI,CAACF,kBAAkB,EAAE;MACzB,IAAIG,CAAC,GAAG,CAAC;MAET,KAAK,IAAMC,SAAS,IAAI,IAAI,CAACC,OAAO,EAAE;QACpC,IAAI,CAACA,OAAO,CAACD,SAAS,CAAC,CAAC,IAAI,CAACH,MAAM,CAAC,GAAGC,GAAG,CAACC,CAAC,EAAE,CAAC;MACjD;MACA,IAAI,CAACF,MAAM,EAAE;IACf;EAAC;IAAA;IAAA,OAED,sBAAaC,GAAgC,EAAQ;MAEnD,IAAI,CAACF,kBAAkB,EAAE;MACzB,KAAK,IAAMI,SAAS,IAAIF,GAAG,EAAE;QAC3B,IAAI,CAACG,OAAO,CAACD,SAAS,CAAC,CAAC,IAAI,CAACH,MAAM,CAAC,GAAGC,GAAG,CAACE,SAAS,CAAC;MACvD;MACA,IAAI,CAACH,MAAM,EAAE;IACf;EAAC;IAAA;IAAA,OAED,oBAAwD;MACtD,IAAI,CAACK,aAAa,EAAE;MACpB,IAAMD,OAAO,GAAGE,KAAK,CAACC,OAAO,CAAC,IAAI,CAACV,MAAM,CAAC,GAAG,IAAI,CAACO,OAAO,GAAG,CAAC,CAAC;;MAK9D,IAAI,CAACE,KAAK,CAACC,OAAO,CAAC,IAAI,CAACV,MAAM,CAAC,EAAE;QAC/B,KAAK,IAAMM,SAAS,IAAI,IAAI,CAACN,MAAM,EAAE;UACnC,IAAMW,KAAK,GAAG,IAAI,CAACX,MAAM,CAACM,SAAS,CAAC;UACpCC,OAAO,CAACI,KAAK,CAACC,IAAI,CAAC,GAAG,IAAI,CAACL,OAAO,CAACI,KAAK,CAACE,KAAK,CAAC;QACjD;MACF;MAEA,IAAI,CAACN,OAAO,GAAG,CAAC,CAAC;MAEjB,IAAMO,KAAyB,GAAG;QAChCC,KAAK,EAAE,gBAAgB;QACvBC,SAAS,EAAE,MAAM;QACjBC,IAAI,EAAEV,OAAO;QACbP,MAAM,EAAE,IAAI,CAACA,MAAM;QACnBG,MAAM,EAAE,IAAI,CAACA;MACf,CAAC;MAED,OAAOW,KAAK;IACd;;EAAC;IAAA;IAAA;;IAID,8BAAqB;MACnB,IAAI,IAAI,CAACX,MAAM,GAAG,IAAI,CAACe,SAAS,EAAE;QAChC;MACF;;MAGA,IAAI,CAACA,SAAS,GAAG,IAAI,CAACA,SAAS,GAAG,CAAC,GAAI,IAAI,CAACA,SAAS,IAAI,CAAC,GAAIpB,iBAAiB;MAC/E,IAAI,CAACS,OAAO,GAAG,CAAC,CAAC;MAEjB,KAAK,IAAMD,SAAS,IAAI,IAAI,CAACN,MAAM,EAAE;QACnC,IAAMW,KAAK,GAAG,IAAI,CAACX,MAAM,CAACM,SAAS,CAAC;QACpC,IAAMa,SAAS,GAAGR,KAAK,CAACS,IAAI,IAAIC,YAAY;QAC5C,IAAMC,SAAS,GAAG,IAAI,CAACf,OAAO,CAACI,KAAK,CAACE,KAAK,CAAC;QAE3C,IAAIS,SAAS,IAAIC,WAAW,CAACC,MAAM,CAACF,SAAS,CAAC,EAAE;UAE9C,IAAMG,UAAU,GAAG,IAAIN,SAAS,CAAC,IAAI,CAACD,SAAS,CAAC;UAChDO,UAAU,CAACC,GAAG,CAACJ,SAAS,CAAC;UACzB,IAAI,CAACf,OAAO,CAACI,KAAK,CAACE,KAAK,CAAC,GAAGY,UAAU;QACxC,CAAC,MAAM,IAAIH,SAAS,EAAE;UAEpBA,SAAS,CAACnB,MAAM,GAAG,IAAI,CAACe,SAAS;UACjC,IAAI,CAACX,OAAO,CAACI,KAAK,CAACE,KAAK,CAAC,GAAGS,SAAS;QACvC,CAAC,MAAM;UAEL,IAAI,CAACf,OAAO,CAACI,KAAK,CAACE,KAAK,CAAC,GAAG,IAAIM,SAAS,CAAC,IAAI,CAACD,SAAS,CAAC;QAC3D;MACF;IACF;EAAC;IAAA;IAAA,OAED,yBAAgB;MACd,mCAAmCS,MAAM,CAACC,OAAO,CAAC,IAAI,CAACrB,OAAO,CAAC,qCAAE;QAA5D;UAAOsB,WAAU;UAAEC,MAAM;QAC5B,IAAI,CAACvB,OAAO,CAACsB,WAAU,CAAC,GAAGC,MAAM,CAACC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC5B,MAAM,CAAC;MACzD;IACF;EAAC;EAAA;AAAA;AAAA"}
|
|
@@ -1,22 +1,16 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
4
|
Object.defineProperty(exports, "__esModule", {
|
|
6
5
|
value: true
|
|
7
6
|
});
|
|
8
7
|
exports.default = void 0;
|
|
9
|
-
|
|
10
8
|
var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
|
|
11
|
-
|
|
12
9
|
var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
|
|
13
|
-
|
|
14
10
|
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
15
|
-
|
|
16
11
|
var _rowUtils = require("../utils/row-utils");
|
|
17
12
|
|
|
18
13
|
var DEFAULT_ROW_COUNT = 100;
|
|
19
|
-
|
|
20
14
|
var RowTableBatchAggregator = function () {
|
|
21
15
|
function RowTableBatchAggregator(schema, options) {
|
|
22
16
|
(0, _classCallCheck2.default)(this, RowTableBatchAggregator);
|
|
@@ -32,13 +26,11 @@ var RowTableBatchAggregator = function () {
|
|
|
32
26
|
|
|
33
27
|
if (!Array.isArray(schema)) {
|
|
34
28
|
this._headers = [];
|
|
35
|
-
|
|
36
29
|
for (var key in schema) {
|
|
37
30
|
this._headers[schema[key].index] = schema[key].name;
|
|
38
31
|
}
|
|
39
32
|
}
|
|
40
33
|
}
|
|
41
|
-
|
|
42
34
|
(0, _createClass2.default)(RowTableBatchAggregator, [{
|
|
43
35
|
key: "rowCount",
|
|
44
36
|
value: function rowCount() {
|
|
@@ -56,7 +48,6 @@ var RowTableBatchAggregator = function () {
|
|
|
56
48
|
var rowObject = (0, _rowUtils.convertToObjectRow)(row, this._headers);
|
|
57
49
|
this.addObjectRow(rowObject, cursor);
|
|
58
50
|
break;
|
|
59
|
-
|
|
60
51
|
case 'array-row-table':
|
|
61
52
|
this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);
|
|
62
53
|
this.arrayRows[this.length] = row;
|
|
@@ -76,7 +67,6 @@ var RowTableBatchAggregator = function () {
|
|
|
76
67
|
var rowArray = (0, _rowUtils.convertToArrayRow)(row, this._headers);
|
|
77
68
|
this.addArrayRow(rowArray, cursor);
|
|
78
69
|
break;
|
|
79
|
-
|
|
80
70
|
case 'object-row-table':
|
|
81
71
|
this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);
|
|
82
72
|
this.objectRows[this.length] = row;
|
|
@@ -88,11 +78,9 @@ var RowTableBatchAggregator = function () {
|
|
|
88
78
|
key: "getBatch",
|
|
89
79
|
value: function getBatch() {
|
|
90
80
|
var rows = this.arrayRows || this.objectRows;
|
|
91
|
-
|
|
92
81
|
if (!rows) {
|
|
93
82
|
return null;
|
|
94
83
|
}
|
|
95
|
-
|
|
96
84
|
rows = rows.slice(0, this.length);
|
|
97
85
|
this.arrayRows = null;
|
|
98
86
|
this.objectRows = null;
|
|
@@ -108,6 +96,5 @@ var RowTableBatchAggregator = function () {
|
|
|
108
96
|
}]);
|
|
109
97
|
return RowTableBatchAggregator;
|
|
110
98
|
}();
|
|
111
|
-
|
|
112
99
|
exports.default = RowTableBatchAggregator;
|
|
113
100
|
//# sourceMappingURL=row-table-batch-aggregator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"row-table-batch-aggregator.js","names":["DEFAULT_ROW_COUNT","RowTableBatchAggregator","schema","options","Array","isArray","_headers","key","index","name","length","row","cursor","Number","isFinite","shape","rowObject","convertToObjectRow","addObjectRow","arrayRows","rowArray","convertToArrayRow","addArrayRow","objectRows","rows","slice","batchType","data"],"sources":["../../../../src/lib/batches/row-table-batch-aggregator.ts"],"sourcesContent":["import type {Schema} from '../schema/schema';\nimport type {TableBatch} from '../../category/table/table-types';\n// import type {ArrayRowTableBatch, ObjectRowTableBatch} from '../../category/table';\nimport {convertToArrayRow, convertToObjectRow} from '../utils/row-utils';\nimport {TableBatchAggregator, TableBatchOptions} from './table-batch-aggregator';\n\nconst DEFAULT_ROW_COUNT = 100;\n\nexport default class RowTableBatchAggregator implements TableBatchAggregator {\n schema: Schema;\n options: TableBatchOptions;\n\n length: number = 0;\n objectRows: {[columnName: string]: any} | null = null;\n arrayRows: any[] | null = null;\n cursor: number = 0;\n private _headers: string[] = [];\n\n constructor(schema: Schema, options: TableBatchOptions) {\n this.options = options;\n this.schema = schema;\n\n // schema is an array if there're no headers\n // object if there are headers\n if (!Array.isArray(schema)) {\n this._headers = [];\n for (const key in schema) {\n this._headers[schema[key].index] = schema[key].name;\n }\n }\n }\n\n rowCount(): number {\n return this.length;\n }\n\n addArrayRow(row: any[], cursor?: number): void {\n if (Number.isFinite(cursor)) {\n this.cursor = cursor as number;\n }\n\n // eslint-disable-next-line default-case\n switch (this.options.shape) {\n case 'object-row-table':\n const rowObject = convertToObjectRow(row, this._headers);\n this.addObjectRow(rowObject, cursor);\n break;\n case 'array-row-table':\n this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);\n this.arrayRows[this.length] = row;\n this.length++;\n break;\n }\n }\n\n addObjectRow(row: {[columnName: string]: any}, cursor?: number): void {\n if (Number.isFinite(cursor)) {\n this.cursor = cursor as number;\n }\n\n // eslint-disable-next-line default-case\n switch (this.options.shape) {\n case 'array-row-table':\n const rowArray = convertToArrayRow(row, this._headers);\n this.addArrayRow(rowArray, cursor);\n break;\n case 'object-row-table':\n this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);\n this.objectRows[this.length] = row;\n this.length++;\n break;\n }\n }\n\n getBatch(): TableBatch | null {\n let rows = this.arrayRows || this.objectRows;\n if (!rows) {\n return null;\n }\n\n rows = rows.slice(0, this.length);\n this.arrayRows = null;\n this.objectRows = null;\n\n return {\n shape: this.options.shape,\n batchType: 'data',\n data: rows,\n length: this.length,\n schema: this.schema,\n cursor: this.cursor\n };\n }\n}\n"],"mappings":";;;;;;;;;;AAGA;;AAGA,IAAMA,iBAAiB,GAAG,GAAG;AAAC,IAETC,uBAAuB;EAU1C,iCAAYC,MAAc,EAAEC,OAA0B,EAAE;IAAA;IAAA;IAAA;IAAA,8CANvC,CAAC;IAAA,kDAC+B,IAAI;IAAA,iDAC3B,IAAI;IAAA,8CACb,CAAC;IAAA,gDACW,EAAE;IAG7B,IAAI,CAACA,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACD,MAAM,GAAGA,MAAM;;IAIpB,IAAI,CAACE,KAAK,CAACC,OAAO,CAACH,MAAM,CAAC,EAAE;MAC1B,IAAI,CAACI,QAAQ,GAAG,EAAE;MAClB,KAAK,IAAMC,GAAG,IAAIL,MAAM,EAAE;QACxB,IAAI,CAACI,QAAQ,CAACJ,MAAM,CAACK,GAAG,CAAC,CAACC,KAAK,CAAC,GAAGN,MAAM,CAACK,GAAG,CAAC,CAACE,IAAI;MACrD;IACF;EACF;EAAC;IAAA;IAAA,OAED,oBAAmB;MACjB,OAAO,IAAI,CAACC,MAAM;IACpB;EAAC;IAAA;IAAA,OAED,qBAAYC,GAAU,EAAEC,MAAe,EAAQ;MAC7C,IAAIC,MAAM,CAACC,QAAQ,CAACF,MAAM,CAAC,EAAE;QAC3B,IAAI,CAACA,MAAM,GAAGA,MAAgB;MAChC;;MAGA,QAAQ,IAAI,CAACT,OAAO,CAACY,KAAK;QACxB,KAAK,kBAAkB;UACrB,IAAMC,SAAS,GAAG,IAAAC,4BAAkB,EAACN,GAAG,EAAE,IAAI,CAACL,QAAQ,CAAC;UACxD,IAAI,CAACY,YAAY,CAACF,SAAS,EAAEJ,MAAM,CAAC;UACpC;QACF,KAAK,iBAAiB;UACpB,IAAI,CAACO,SAAS,GAAG,IAAI,CAACA,SAAS,IAAI,IAAIf,KAAK,CAACJ,iBAAiB,CAAC;UAC/D,IAAI,CAACmB,SAAS,CAAC,IAAI,CAACT,MAAM,CAAC,GAAGC,GAAG;UACjC,IAAI,CAACD,MAAM,EAAE;UACb;MAAM;IAEZ;EAAC;IAAA;IAAA,OAED,sBAAaC,GAAgC,EAAEC,MAAe,EAAQ;MACpE,IAAIC,MAAM,CAACC,QAAQ,CAACF,MAAM,CAAC,EAAE;QAC3B,IAAI,CAACA,MAAM,GAAGA,MAAgB;MAChC;;MAGA,QAAQ,IAAI,CAACT,OAAO,CAACY,KAAK;QACxB,KAAK,iBAAiB;UACpB,IAAMK,QAAQ,GAAG,IAAAC,2BAAiB,EAACV,GAAG,EAAE,IAAI,CAACL,QAAQ,CAAC;UACtD,IAAI,CAACgB,WAAW,CAACF,QAAQ,EAAER,MAAM,CAAC;UAClC;QACF,KAAK,kBAAkB;UACrB,IAAI,CAACW,UAAU,GAAG,IAAI,CAACA,UAAU,IAAI,IAAInB,KAAK,CAACJ,iBAAiB,CAAC;UACjE,IAAI,CAACuB,UAAU,CAAC,IAAI,CAACb,MAAM,CAAC,GAAGC,GAAG;UAClC,IAAI,CAACD,MAAM,EAAE;UACb;MAAM;IAEZ;EAAC;IAAA;IAAA,OAED,oBAA8B;MAC5B,IAAIc,IAAI,GAAG,IAAI,CAACL,SAAS,IAAI,IAAI,CAACI,UAAU;MAC5C,IAAI,CAACC,IAAI,EAAE;QACT,OAAO,IAAI;MACb;MAEAA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,CAAC,EAAE,IAAI,CAACf,MAAM,CAAC;MACjC,IAAI,CAACS,SAAS,GAAG,IAAI;MACrB,IAAI,CAACI,UAAU,GAAG,IAAI;MAEtB,OAAO;QACLR,KAAK,EAAE,IAAI,CAACZ,OAAO,CAACY,KAAK;QACzBW,SAAS,EAAE,MAAM;QACjBC,IAAI,EAAEH,IAAI;QACVd,MAAM,EAAE,IAAI,CAACA,MAAM;QACnBR,MAAM,EAAE,IAAI,CAACA,MAAM;QACnBU,MAAM,EAAE,IAAI,CAACA;MACf,CAAC;IACH;EAAC;EAAA;AAAA;AAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"table-batch-aggregator.js","names":[],"sources":["../../../../src/lib/batches/table-batch-aggregator.ts"],"sourcesContent":["import type {Schema} from '../schema/schema';\nimport type {TableBatch} from '../../category/table/table-types';\n\nexport interface TableBatchOptions {\n batchSize: number | string;\n [key: string]: any;\n}\n\nexport interface TableBatchConstructor {\n new (schema: Schema, options: TableBatchOptions): TableBatchAggregator;\n}\n\n/**\n * TableBatchBuilder delegates batch building to this interface\n */\nexport interface TableBatchAggregator {\n /** Number of rows */\n rowCount(): number;\n /** Add one row */\n addArrayRow(row: any[]): void;\n /** Add one row */\n addObjectRow(row: {[columnName: string]: any}): void;\n /** return a batch object */\n getBatch(): TableBatch | null;\n}\n"],"mappings":""}
|
|
@@ -1,28 +1,18 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
4
|
Object.defineProperty(exports, "__esModule", {
|
|
6
5
|
value: true
|
|
7
6
|
});
|
|
8
7
|
exports.default = void 0;
|
|
9
|
-
|
|
10
8
|
var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
|
|
11
|
-
|
|
12
9
|
var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
|
|
13
|
-
|
|
14
10
|
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
15
|
-
|
|
16
11
|
var _baseTableBatchAggregator = _interopRequireDefault(require("./base-table-batch-aggregator"));
|
|
17
|
-
|
|
18
12
|
var _rowTableBatchAggregator = _interopRequireDefault(require("./row-table-batch-aggregator"));
|
|
19
|
-
|
|
20
13
|
var _columnarTableBatchAggregator = _interopRequireDefault(require("./columnar-table-batch-aggregator"));
|
|
21
|
-
|
|
22
|
-
function
|
|
23
|
-
|
|
24
|
-
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
|
|
25
|
-
|
|
14
|
+
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
|
|
15
|
+
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
|
|
26
16
|
var DEFAULT_OPTIONS = {
|
|
27
17
|
shape: 'array-row-table',
|
|
28
18
|
batchSize: 'auto',
|
|
@@ -48,72 +38,68 @@ var TableBatchBuilder = function () {
|
|
|
48
38
|
this.schema = schema;
|
|
49
39
|
this.options = _objectSpread(_objectSpread({}, DEFAULT_OPTIONS), options);
|
|
50
40
|
}
|
|
51
|
-
|
|
52
41
|
(0, _createClass2.default)(TableBatchBuilder, [{
|
|
53
42
|
key: "limitReached",
|
|
54
43
|
value: function limitReached() {
|
|
55
44
|
var _this$options, _this$options2;
|
|
56
|
-
|
|
57
45
|
if (Boolean((_this$options = this.options) === null || _this$options === void 0 ? void 0 : _this$options.limit) && this.totalLength >= this.options.limit) {
|
|
58
46
|
return true;
|
|
59
47
|
}
|
|
60
|
-
|
|
61
48
|
if (Boolean((_this$options2 = this.options) === null || _this$options2 === void 0 ? void 0 : _this$options2._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
|
|
62
49
|
return true;
|
|
63
50
|
}
|
|
64
|
-
|
|
65
51
|
return false;
|
|
66
52
|
}
|
|
53
|
+
|
|
67
54
|
}, {
|
|
68
55
|
key: "addRow",
|
|
69
|
-
value:
|
|
56
|
+
value:
|
|
57
|
+
function addRow(row) {
|
|
70
58
|
if (this.limitReached()) {
|
|
71
59
|
return;
|
|
72
60
|
}
|
|
73
|
-
|
|
74
61
|
this.totalLength++;
|
|
75
62
|
this.rowBytes = this.rowBytes || this._estimateRowMB(row);
|
|
76
63
|
this.totalBytes += this.rowBytes;
|
|
77
|
-
|
|
78
64
|
if (Array.isArray(row)) {
|
|
79
65
|
this.addArrayRow(row);
|
|
80
66
|
} else {
|
|
81
67
|
this.addObjectRow(row);
|
|
82
68
|
}
|
|
83
69
|
}
|
|
70
|
+
|
|
84
71
|
}, {
|
|
85
72
|
key: "addArrayRow",
|
|
86
|
-
value:
|
|
73
|
+
value:
|
|
74
|
+
function addArrayRow(row) {
|
|
87
75
|
if (!this.aggregator) {
|
|
88
76
|
var TableBatchType = this._getTableBatchType();
|
|
89
|
-
|
|
90
77
|
this.aggregator = new TableBatchType(this.schema, this.options);
|
|
91
78
|
}
|
|
92
|
-
|
|
93
79
|
this.aggregator.addArrayRow(row);
|
|
94
80
|
}
|
|
81
|
+
|
|
95
82
|
}, {
|
|
96
83
|
key: "addObjectRow",
|
|
97
|
-
value:
|
|
84
|
+
value:
|
|
85
|
+
function addObjectRow(row) {
|
|
98
86
|
if (!this.aggregator) {
|
|
99
87
|
var TableBatchType = this._getTableBatchType();
|
|
100
|
-
|
|
101
88
|
this.aggregator = new TableBatchType(this.schema, this.options);
|
|
102
89
|
}
|
|
103
|
-
|
|
104
90
|
this.aggregator.addObjectRow(row);
|
|
105
91
|
}
|
|
92
|
+
|
|
106
93
|
}, {
|
|
107
94
|
key: "chunkComplete",
|
|
108
|
-
value:
|
|
95
|
+
value:
|
|
96
|
+
function chunkComplete(chunk) {
|
|
109
97
|
if (chunk instanceof ArrayBuffer) {
|
|
110
98
|
this.bytesUsed += chunk.byteLength;
|
|
111
99
|
}
|
|
112
|
-
|
|
113
100
|
if (typeof chunk === 'string') {
|
|
114
101
|
this.bytesUsed += chunk.length;
|
|
115
102
|
}
|
|
116
|
-
|
|
117
103
|
this.isChunkComplete = true;
|
|
118
104
|
}
|
|
119
105
|
}, {
|
|
@@ -126,9 +112,12 @@ var TableBatchBuilder = function () {
|
|
|
126
112
|
value: function getFinalBatch(options) {
|
|
127
113
|
return this._getBatch(options);
|
|
128
114
|
}
|
|
115
|
+
|
|
129
116
|
}, {
|
|
130
117
|
key: "_estimateRowMB",
|
|
131
|
-
value:
|
|
118
|
+
value:
|
|
119
|
+
|
|
120
|
+
function _estimateRowMB(row) {
|
|
132
121
|
return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
|
|
133
122
|
}
|
|
134
123
|
}, {
|
|
@@ -154,9 +143,11 @@ var TableBatchBuilder = function () {
|
|
|
154
143
|
this.lastBatchEmittedMs = Date.now();
|
|
155
144
|
return true;
|
|
156
145
|
}
|
|
146
|
+
|
|
157
147
|
}, {
|
|
158
148
|
key: "_getBatch",
|
|
159
|
-
value:
|
|
149
|
+
value:
|
|
150
|
+
function _getBatch(options) {
|
|
160
151
|
if (!this.aggregator) {
|
|
161
152
|
return null;
|
|
162
153
|
}
|
|
@@ -164,7 +155,6 @@ var TableBatchBuilder = function () {
|
|
|
164
155
|
if (options !== null && options !== void 0 && options.bytesUsed) {
|
|
165
156
|
this.bytesUsed = options.bytesUsed;
|
|
166
157
|
}
|
|
167
|
-
|
|
168
158
|
var normalizedBatch = this.aggregator.getBatch();
|
|
169
159
|
normalizedBatch.count = this.batchCount;
|
|
170
160
|
normalizedBatch.bytesUsed = this.bytesUsed;
|
|
@@ -179,21 +169,16 @@ var TableBatchBuilder = function () {
|
|
|
179
169
|
switch (this.options.shape) {
|
|
180
170
|
case 'row-table':
|
|
181
171
|
return _baseTableBatchAggregator.default;
|
|
182
|
-
|
|
183
172
|
case 'array-row-table':
|
|
184
173
|
case 'object-row-table':
|
|
185
174
|
return _rowTableBatchAggregator.default;
|
|
186
|
-
|
|
187
175
|
case 'columnar-table':
|
|
188
176
|
return _columnarTableBatchAggregator.default;
|
|
189
|
-
|
|
190
177
|
case 'arrow-table':
|
|
191
178
|
if (!TableBatchBuilder.ArrowBatch) {
|
|
192
179
|
throw new Error(ERR_MESSAGE);
|
|
193
180
|
}
|
|
194
|
-
|
|
195
181
|
return TableBatchBuilder.ArrowBatch;
|
|
196
|
-
|
|
197
182
|
default:
|
|
198
183
|
throw new Error(ERR_MESSAGE);
|
|
199
184
|
}
|
|
@@ -201,7 +186,6 @@ var TableBatchBuilder = function () {
|
|
|
201
186
|
}]);
|
|
202
187
|
return TableBatchBuilder;
|
|
203
188
|
}();
|
|
204
|
-
|
|
205
189
|
exports.default = TableBatchBuilder;
|
|
206
190
|
(0, _defineProperty2.default)(TableBatchBuilder, "ArrowBatch", void 0);
|
|
207
191
|
//# sourceMappingURL=table-batch-builder.js.map
|