@loaders.gl/schema 3.1.3 → 4.0.0-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle.js +2 -2
- package/dist/bundle.js.map +1 -0
- package/dist/category/common.d.ts +1 -2
- package/dist/category/common.d.ts.map +1 -1
- package/dist/category/common.js +2 -10
- package/dist/{es5/category → category}/common.js.map +0 -0
- package/dist/category/gis.js +2 -2
- package/dist/{es5/category → category}/gis.js.map +0 -0
- package/dist/category/image/image.d.ts +18 -0
- package/dist/category/image/image.d.ts.map +1 -0
- package/dist/category/image/image.js +2 -0
- package/dist/{esm/category/gis.js.map → category/image/image.js.map} +1 -1
- package/dist/category/mesh/convert-mesh.js +31 -37
- package/dist/category/mesh/convert-mesh.js.map +1 -0
- package/dist/category/mesh/deduce-mesh-schema.js +37 -57
- package/dist/category/mesh/deduce-mesh-schema.js.map +1 -0
- package/dist/category/mesh/mesh-to-arrow-table.d.ts +10 -0
- package/dist/category/mesh/mesh-to-arrow-table.d.ts.map +1 -1
- package/dist/category/mesh/mesh-to-arrow-table.js +15 -28
- package/dist/category/mesh/mesh-to-arrow-table.js.map +1 -0
- package/dist/category/mesh/mesh-types.js +2 -2
- package/dist/{es5/category → category}/mesh/mesh-types.js.map +0 -0
- package/dist/category/mesh/mesh-utils.js +35 -48
- package/dist/category/mesh/mesh-utils.js.map +1 -0
- package/dist/category/table/deduce-table-schema.js +39 -94
- package/dist/category/table/deduce-table-schema.js.map +1 -0
- package/dist/category/table/table-types.d.ts +1 -3
- package/dist/category/table/table-types.d.ts.map +1 -1
- package/dist/category/table/table-types.js +2 -2
- package/dist/{es5/category → category}/table/table-types.js.map +0 -0
- package/dist/category/texture/texture.d.ts +18 -0
- package/dist/category/texture/texture.d.ts.map +1 -0
- package/dist/category/texture/texture.js +2 -0
- package/dist/{esm/category/common.js.map → category/texture/texture.js.map} +1 -1
- package/dist/dist.min.js +15078 -208
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +13 -75
- package/dist/index.js.map +1 -0
- package/dist/lib/arrow/arrow-like-type-utils.js +31 -26
- package/dist/lib/arrow/arrow-like-type-utils.js.map +1 -0
- package/dist/lib/arrow/arrow-type-utils.d.ts +5 -0
- package/dist/lib/arrow/arrow-type-utils.d.ts.map +1 -1
- package/dist/lib/arrow/arrow-type-utils.js +20 -28
- package/dist/lib/arrow/arrow-type-utils.js.map +1 -0
- package/dist/lib/arrow/get-type-info.js +19 -23
- package/dist/lib/arrow/get-type-info.js.map +1 -0
- package/dist/lib/batches/base-table-batch-aggregator.js +68 -52
- package/dist/lib/batches/base-table-batch-aggregator.js.map +1 -0
- package/dist/lib/batches/columnar-table-batch-aggregator.js +92 -82
- package/dist/lib/batches/columnar-table-batch-aggregator.js.map +1 -0
- package/dist/lib/batches/row-table-batch-aggregator.js +88 -72
- package/dist/lib/batches/row-table-batch-aggregator.js.map +1 -0
- package/dist/lib/batches/table-batch-aggregator.js +2 -2
- package/dist/{es5/lib → lib}/batches/table-batch-aggregator.js.map +0 -0
- package/dist/lib/batches/table-batch-builder.js +166 -137
- package/dist/lib/batches/table-batch-builder.js.map +1 -0
- package/dist/lib/schema/impl/enum.js +48 -96
- package/dist/lib/schema/impl/enum.js.map +1 -0
- package/dist/lib/schema/impl/field.js +34 -31
- package/dist/lib/schema/impl/field.js.map +1 -0
- package/dist/lib/schema/impl/schema.js +81 -72
- package/dist/lib/schema/impl/schema.js.map +1 -0
- package/dist/lib/schema/impl/type.js +490 -454
- package/dist/lib/schema/impl/type.js.map +1 -0
- package/dist/lib/schema/schema.js +5 -90
- package/dist/lib/schema/schema.js.map +1 -0
- package/dist/lib/schema-utils/deduce-column-type.js +20 -89
- package/dist/lib/schema-utils/deduce-column-type.js.map +1 -0
- package/dist/lib/utils/assert.js +5 -11
- package/dist/lib/utils/assert.js.map +1 -0
- package/dist/lib/utils/async-queue.js +109 -82
- package/dist/lib/utils/async-queue.js.map +1 -0
- package/dist/lib/utils/row-utils.js +33 -31
- package/dist/lib/utils/row-utils.js.map +1 -0
- package/dist/types.js +2 -2
- package/dist/{es5/types.js.map → types.js.map} +0 -0
- package/package.json +6 -5
- package/src/category/common.ts +1 -2
- package/src/category/image/image.ts +19 -0
- package/src/category/mesh/convert-mesh.ts +6 -6
- package/src/category/mesh/mesh-to-arrow-table.ts +2 -4
- package/src/category/table/table-types.ts +1 -4
- package/src/category/texture/texture.ts +28 -0
- package/src/index.ts +6 -1
- package/src/lib/arrow/arrow-type-utils.ts +0 -2
- package/dist/es5/bundle.js +0 -7
- package/dist/es5/bundle.js.map +0 -1
- package/dist/es5/category/common.js +0 -2
- package/dist/es5/category/gis.js +0 -2
- package/dist/es5/category/mesh/convert-mesh.js +0 -43
- package/dist/es5/category/mesh/convert-mesh.js.map +0 -1
- package/dist/es5/category/mesh/deduce-mesh-schema.js +0 -54
- package/dist/es5/category/mesh/deduce-mesh-schema.js.map +0 -1
- package/dist/es5/category/mesh/mesh-to-arrow-table.js +0 -2
- package/dist/es5/category/mesh/mesh-to-arrow-table.js.map +0 -1
- package/dist/es5/category/mesh/mesh-types.js +0 -2
- package/dist/es5/category/mesh/mesh-utils.js +0 -47
- package/dist/es5/category/mesh/mesh-utils.js.map +0 -1
- package/dist/es5/category/table/deduce-table-schema.js +0 -58
- package/dist/es5/category/table/deduce-table-schema.js.map +0 -1
- package/dist/es5/category/table/table-types.js +0 -2
- package/dist/es5/index.js +0 -344
- package/dist/es5/index.js.map +0 -1
- package/dist/es5/lib/arrow/arrow-like-type-utils.js +0 -40
- package/dist/es5/lib/arrow/arrow-like-type-utils.js.map +0 -1
- package/dist/es5/lib/arrow/arrow-type-utils.js +0 -2
- package/dist/es5/lib/arrow/arrow-type-utils.js.map +0 -1
- package/dist/es5/lib/arrow/get-type-info.js +0 -33
- package/dist/es5/lib/arrow/get-type-info.js.map +0 -1
- package/dist/es5/lib/batches/base-table-batch-aggregator.js +0 -92
- package/dist/es5/lib/batches/base-table-batch-aggregator.js.map +0 -1
- package/dist/es5/lib/batches/columnar-table-batch-aggregator.js +0 -128
- package/dist/es5/lib/batches/columnar-table-batch-aggregator.js.map +0 -1
- package/dist/es5/lib/batches/row-table-batch-aggregator.js +0 -113
- package/dist/es5/lib/batches/row-table-batch-aggregator.js.map +0 -1
- package/dist/es5/lib/batches/table-batch-aggregator.js +0 -2
- package/dist/es5/lib/batches/table-batch-builder.js +0 -207
- package/dist/es5/lib/batches/table-batch-builder.js.map +0 -1
- package/dist/es5/lib/schema/impl/enum.js +0 -56
- package/dist/es5/lib/schema/impl/enum.js.map +0 -1
- package/dist/es5/lib/schema/impl/field.js +0 -56
- package/dist/es5/lib/schema/impl/field.js.map +0 -1
- package/dist/es5/lib/schema/impl/schema.js +0 -168
- package/dist/es5/lib/schema/impl/schema.js.map +0 -1
- package/dist/es5/lib/schema/impl/type.js +0 -927
- package/dist/es5/lib/schema/impl/type.js.map +0 -1
- package/dist/es5/lib/schema/schema.js +0 -236
- package/dist/es5/lib/schema/schema.js.map +0 -1
- package/dist/es5/lib/schema-utils/deduce-column-type.js +0 -32
- package/dist/es5/lib/schema-utils/deduce-column-type.js.map +0 -1
- package/dist/es5/lib/utils/assert.js +0 -13
- package/dist/es5/lib/utils/assert.js.map +0 -1
- package/dist/es5/lib/utils/async-queue.js +0 -213
- package/dist/es5/lib/utils/async-queue.js.map +0 -1
- package/dist/es5/lib/utils/row-utils.js +0 -44
- package/dist/es5/lib/utils/row-utils.js.map +0 -1
- package/dist/es5/types.js +0 -2
- package/dist/esm/bundle.js +0 -5
- package/dist/esm/bundle.js.map +0 -1
- package/dist/esm/category/common.js +0 -2
- package/dist/esm/category/gis.js +0 -2
- package/dist/esm/category/mesh/convert-mesh.js +0 -26
- package/dist/esm/category/mesh/convert-mesh.js.map +0 -1
- package/dist/esm/category/mesh/deduce-mesh-schema.js +0 -42
- package/dist/esm/category/mesh/deduce-mesh-schema.js.map +0 -1
- package/dist/esm/category/mesh/mesh-to-arrow-table.js +0 -2
- package/dist/esm/category/mesh/mesh-to-arrow-table.js.map +0 -1
- package/dist/esm/category/mesh/mesh-types.js +0 -2
- package/dist/esm/category/mesh/mesh-types.js.map +0 -1
- package/dist/esm/category/mesh/mesh-utils.js +0 -38
- package/dist/esm/category/mesh/mesh-utils.js.map +0 -1
- package/dist/esm/category/table/deduce-table-schema.js +0 -51
- package/dist/esm/category/table/deduce-table-schema.js.map +0 -1
- package/dist/esm/category/table/table-types.js +0 -2
- package/dist/esm/category/table/table-types.js.map +0 -1
- package/dist/esm/index.js +0 -13
- package/dist/esm/index.js.map +0 -1
- package/dist/esm/lib/arrow/arrow-like-type-utils.js +0 -32
- package/dist/esm/lib/arrow/arrow-like-type-utils.js.map +0 -1
- package/dist/esm/lib/arrow/arrow-type-utils.js +0 -2
- package/dist/esm/lib/arrow/arrow-type-utils.js.map +0 -1
- package/dist/esm/lib/arrow/get-type-info.js +0 -24
- package/dist/esm/lib/arrow/get-type-info.js.map +0 -1
- package/dist/esm/lib/batches/base-table-batch-aggregator.js +0 -74
- package/dist/esm/lib/batches/base-table-batch-aggregator.js.map +0 -1
- package/dist/esm/lib/batches/columnar-table-batch-aggregator.js +0 -100
- package/dist/esm/lib/batches/columnar-table-batch-aggregator.js.map +0 -1
- package/dist/esm/lib/batches/row-table-batch-aggregator.js +0 -95
- package/dist/esm/lib/batches/row-table-batch-aggregator.js.map +0 -1
- package/dist/esm/lib/batches/table-batch-aggregator.js +0 -2
- package/dist/esm/lib/batches/table-batch-aggregator.js.map +0 -1
- package/dist/esm/lib/batches/table-batch-builder.js +0 -182
- package/dist/esm/lib/batches/table-batch-builder.js.map +0 -1
- package/dist/esm/lib/schema/impl/enum.js +0 -49
- package/dist/esm/lib/schema/impl/enum.js.map +0 -1
- package/dist/esm/lib/schema/impl/field.js +0 -35
- package/dist/esm/lib/schema/impl/field.js.map +0 -1
- package/dist/esm/lib/schema/impl/schema.js +0 -92
- package/dist/esm/lib/schema/impl/schema.js.map +0 -1
- package/dist/esm/lib/schema/impl/type.js +0 -498
- package/dist/esm/lib/schema/impl/type.js.map +0 -1
- package/dist/esm/lib/schema/schema.js +0 -5
- package/dist/esm/lib/schema/schema.js.map +0 -1
- package/dist/esm/lib/schema-utils/deduce-column-type.js +0 -23
- package/dist/esm/lib/schema-utils/deduce-column-type.js.map +0 -1
- package/dist/esm/lib/utils/assert.js +0 -6
- package/dist/esm/lib/utils/assert.js.map +0 -1
- package/dist/esm/lib/utils/async-queue.js +0 -119
- package/dist/esm/lib/utils/async-queue.js.map +0 -1
- package/dist/esm/lib/utils/row-utils.js +0 -35
- package/dist/esm/lib/utils/row-utils.js.map +0 -1
- package/dist/esm/types.js +0 -2
- package/dist/esm/types.js.map +0 -1
|
@@ -1,90 +1,100 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1
|
+
import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
|
|
3
2
|
const DEFAULT_ROW_COUNT = 100;
|
|
4
|
-
class ColumnarTableBatchAggregator {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
3
|
+
export default class ColumnarTableBatchAggregator {
|
|
4
|
+
constructor(schema, options) {
|
|
5
|
+
_defineProperty(this, "schema", void 0);
|
|
6
|
+
|
|
7
|
+
_defineProperty(this, "length", 0);
|
|
8
|
+
|
|
9
|
+
_defineProperty(this, "allocated", 0);
|
|
10
|
+
|
|
11
|
+
_defineProperty(this, "columns", {});
|
|
12
|
+
|
|
13
|
+
this.schema = schema;
|
|
14
|
+
|
|
15
|
+
this._reallocateColumns();
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
rowCount() {
|
|
19
|
+
return this.length;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
addArrayRow(row) {
|
|
23
|
+
this._reallocateColumns();
|
|
24
|
+
|
|
25
|
+
let i = 0;
|
|
26
|
+
|
|
27
|
+
for (const fieldName in this.columns) {
|
|
28
|
+
this.columns[fieldName][this.length] = row[i++];
|
|
11
29
|
}
|
|
12
|
-
|
|
13
|
-
|
|
30
|
+
|
|
31
|
+
this.length++;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
addObjectRow(row) {
|
|
35
|
+
this._reallocateColumns();
|
|
36
|
+
|
|
37
|
+
for (const fieldName in row) {
|
|
38
|
+
this.columns[fieldName][this.length] = row[fieldName];
|
|
14
39
|
}
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
40
|
+
|
|
41
|
+
this.length++;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
getBatch() {
|
|
45
|
+
this._pruneColumns();
|
|
46
|
+
|
|
47
|
+
const columns = Array.isArray(this.schema) ? this.columns : {};
|
|
48
|
+
|
|
49
|
+
if (!Array.isArray(this.schema)) {
|
|
50
|
+
for (const fieldName in this.schema) {
|
|
51
|
+
const field = this.schema[fieldName];
|
|
52
|
+
columns[field.name] = this.columns[field.index];
|
|
53
|
+
}
|
|
24
54
|
}
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
55
|
+
|
|
56
|
+
this.columns = {};
|
|
57
|
+
const batch = {
|
|
58
|
+
shape: 'columnar-table',
|
|
59
|
+
batchType: 'data',
|
|
60
|
+
data: columns,
|
|
61
|
+
schema: this.schema,
|
|
62
|
+
length: this.length
|
|
63
|
+
};
|
|
64
|
+
return batch;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
_reallocateColumns() {
|
|
68
|
+
if (this.length < this.allocated) {
|
|
69
|
+
return;
|
|
32
70
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
this.columns =
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
};
|
|
53
|
-
return batch;
|
|
71
|
+
|
|
72
|
+
this.allocated = this.allocated > 0 ? this.allocated *= 2 : DEFAULT_ROW_COUNT;
|
|
73
|
+
this.columns = {};
|
|
74
|
+
|
|
75
|
+
for (const fieldName in this.schema) {
|
|
76
|
+
const field = this.schema[fieldName];
|
|
77
|
+
const ArrayType = field.type || Float32Array;
|
|
78
|
+
const oldColumn = this.columns[field.index];
|
|
79
|
+
|
|
80
|
+
if (oldColumn && ArrayBuffer.isView(oldColumn)) {
|
|
81
|
+
const typedArray = new ArrayType(this.allocated);
|
|
82
|
+
typedArray.set(oldColumn);
|
|
83
|
+
this.columns[field.index] = typedArray;
|
|
84
|
+
} else if (oldColumn) {
|
|
85
|
+
oldColumn.length = this.allocated;
|
|
86
|
+
this.columns[field.index] = oldColumn;
|
|
87
|
+
} else {
|
|
88
|
+
this.columns[field.index] = new ArrayType(this.allocated);
|
|
89
|
+
}
|
|
54
90
|
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
// @ts-ignore TODO
|
|
61
|
-
this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;
|
|
62
|
-
this.columns = {};
|
|
63
|
-
for (const fieldName in this.schema) {
|
|
64
|
-
const field = this.schema[fieldName];
|
|
65
|
-
const ArrayType = field.type || Float32Array;
|
|
66
|
-
const oldColumn = this.columns[field.index];
|
|
67
|
-
if (oldColumn && ArrayBuffer.isView(oldColumn)) {
|
|
68
|
-
// Copy the old data to the new array
|
|
69
|
-
const typedArray = new ArrayType(this.allocated);
|
|
70
|
-
typedArray.set(oldColumn);
|
|
71
|
-
this.columns[field.index] = typedArray;
|
|
72
|
-
}
|
|
73
|
-
else if (oldColumn) {
|
|
74
|
-
// Plain array
|
|
75
|
-
oldColumn.length = this.allocated;
|
|
76
|
-
this.columns[field.index] = oldColumn;
|
|
77
|
-
}
|
|
78
|
-
else {
|
|
79
|
-
// Create new
|
|
80
|
-
this.columns[field.index] = new ArrayType(this.allocated);
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
_pruneColumns() {
|
|
85
|
-
for (const [columnName, column] of Object.entries(this.columns)) {
|
|
86
|
-
this.columns[columnName] = column.slice(0, this.length);
|
|
87
|
-
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
_pruneColumns() {
|
|
94
|
+
for (const [columnName, column] of Object.entries(this.columns)) {
|
|
95
|
+
this.columns[columnName] = column.slice(0, this.length);
|
|
88
96
|
}
|
|
97
|
+
}
|
|
98
|
+
|
|
89
99
|
}
|
|
90
|
-
|
|
100
|
+
//# sourceMappingURL=columnar-table-batch-aggregator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../../src/lib/batches/columnar-table-batch-aggregator.ts"],"names":["DEFAULT_ROW_COUNT","ColumnarTableBatchAggregator","constructor","schema","options","_reallocateColumns","rowCount","length","addArrayRow","row","i","fieldName","columns","addObjectRow","getBatch","_pruneColumns","Array","isArray","field","name","index","batch","shape","batchType","data","allocated","ArrayType","type","Float32Array","oldColumn","ArrayBuffer","isView","typedArray","set","columnName","column","Object","entries","slice"],"mappings":";AAMA,MAAMA,iBAAiB,GAAG,GAA1B;AAEA,eAAe,MAAMC,4BAAN,CAAmE;AAMhFC,EAAAA,WAAW,CAACC,MAAD,EAAiBC,OAAjB,EAAqD;AAAA;;AAAA,oCAJ/C,CAI+C;;AAAA,uCAH5C,CAG4C;;AAAA,qCAFvB,EAEuB;;AAC9D,SAAKD,MAAL,GAAcA,MAAd;;AACA,SAAKE,kBAAL;AACD;;AAEDC,EAAAA,QAAQ,GAAW;AACjB,WAAO,KAAKC,MAAZ;AACD;;AAEDC,EAAAA,WAAW,CAACC,GAAD,EAAa;AAEtB,SAAKJ,kBAAL;;AACA,QAAIK,CAAC,GAAG,CAAR;;AAEA,SAAK,MAAMC,SAAX,IAAwB,KAAKC,OAA7B,EAAsC;AACpC,WAAKA,OAAL,CAAaD,SAAb,EAAwB,KAAKJ,MAA7B,IAAuCE,GAAG,CAACC,CAAC,EAAF,CAA1C;AACD;;AACD,SAAKH,MAAL;AACD;;AAEDM,EAAAA,YAAY,CAACJ,GAAD,EAAyC;AAEnD,SAAKJ,kBAAL;;AACA,SAAK,MAAMM,SAAX,IAAwBF,GAAxB,EAA6B;AAC3B,WAAKG,OAAL,CAAaD,SAAb,EAAwB,KAAKJ,MAA7B,IAAuCE,GAAG,CAACE,SAAD,CAA1C;AACD;;AACD,SAAKJ,MAAL;AACD;;AAEDO,EAAAA,QAAQ,GAAgD;AACtD,SAAKC,aAAL;;AACA,UAAMH,OAAO,GAAGI,KAAK,CAACC,OAAN,CAAc,KAAKd,MAAnB,IAA6B,KAAKS,OAAlC,GAA4C,EAA5D;;AAKA,QAAI,CAACI,KAAK,CAACC,OAAN,CAAc,KAAKd,MAAnB,CAAL,EAAiC;AAC/B,WAAK,MAAMQ,SAAX,IAAwB,KAAKR,MAA7B,EAAqC;AACnC,cAAMe,KAAK,GAAG,KAAKf,MAAL,CAAYQ,SAAZ,CAAd;AACAC,QAAAA,OAAO,CAACM,KAAK,CAACC,IAAP,CAAP,GAAsB,KAAKP,OAAL,CAAaM,KAAK,CAACE,KAAnB,CAAtB;AACD;AACF;;AAED,SAAKR,OAAL,GAAe,EAAf;AAEA,UAAMS,KAAyB,GAAG;AAChCC,MAAAA,KAAK,EAAE,gBADyB;AAEhCC,MAAAA,SAAS,EAAE,MAFqB;AAGhCC,MAAAA,IAAI,EAAEZ,OAH0B;AAIhCT,MAAAA,MAAM,EAAE,KAAKA,MAJmB;AAKhCI,MAAAA,MAAM,EAAE,KAAKA;AALmB,KAAlC;AAQA,WAAOc,KAAP;AACD;;AAIDhB,EAAAA,kBAAkB,GAAG;AACnB,QAAI,KAAKE,MAAL,GAAc,KAAKkB,SAAvB,EAAkC;AAChC;AACD;;AAGD,SAAKA,SAAL,GAAiB,KAAKA,SAAL,GAAiB,CAAjB,GAAsB,KAAKA,SAAL,IAAkB,CAAxC,GAA6CzB,iBAA9D;AACA,SAAKY,OAAL,GAAe,EAAf;;AAEA,SAAK,MAAMD,SAAX,IAAwB,KAAKR,MAA7B,EAAqC;AACnC,YAAMe,KAAK,GAAG,KAAKf,MAAL,CAAYQ,SAAZ,CAAd;AACA,YAAMe,SAAS,GAAGR,KAAK,CAACS,IAAN,IAAcC,YAAhC;AACA,YAAMC,SAAS,GAAG,KAAKjB,OAAL,CAAaM,KAAK,CAACE,KAAnB,CAAlB;;AAEA,UAAIS,SAAS,IAAIC,WAAW,CAACC,MAAZ,CAAmBF,SAAnB,CAAjB,EAAgD;AAE9C,cAAMG,UAAU,GAAG,IAAIN,SAAJ,CAAc,KAAKD,SAAnB,CAAnB;AACAO,QAAAA,UAAU,CAACC,GAAX,CAAeJ,SAAf;AACA,aAAKjB,OAAL,CAAaM,KAAK,CAACE,KAAnB,IAA4BY,UAA5B;AACD,OALD,MAKO,IAAIH,SAAJ,EAAe;AAEpBA,QAAAA,SAAS,CAACtB,MAAV,GAAmB,KAAKkB,SAAxB;AACA,aAAKb,OAAL,CAAaM,KAAK,CAACE,KAAnB,IAA4BS,SAA5B;AACD,OAJM,MAIA;AAEL,aAAKjB,OAAL,CAAaM,KAAK,CAACE,KAAnB,IAA4B,IAAIM,SAAJ,CAAc,KAAKD,SAAnB,CAA5B;AACD;AACF;AACF;;AAEDV,EAAAA,aAAa,GAAG;AACd,SAAK,MAAM,CAACmB,UAAD,EAAaC,MAAb,CAAX,IAAmCC,MAAM,CAACC,OAAP,CAAe,KAAKzB,OAApB,CAAnC,EAAiE;AAC/D,WAAKA,OAAL,CAAasB,UAAb,IAA2BC,MAAM,CAACG,KAAP,CAAa,CAAb,EAAgB,KAAK/B,MAArB,CAA3B;AACD;AACF;;AAlG+E","sourcesContent":["import type {Schema} from '../schema/schema';\nimport type {ColumnarTableBatch, ArrowTableBatch} from '../../category/table/table-types';\nimport {TableBatchAggregator} from './table-batch-aggregator';\n\ntype ColumnarTableBatchOptions = {};\n\nconst DEFAULT_ROW_COUNT = 100;\n\nexport default class ColumnarTableBatchAggregator implements TableBatchAggregator {\n schema: Schema;\n length: number = 0;\n allocated: number = 0;\n columns: {[columnName: string]: any[]} = {};\n\n constructor(schema: Schema, options: ColumnarTableBatchOptions) {\n this.schema = schema;\n this._reallocateColumns();\n }\n\n rowCount(): number {\n return this.length;\n }\n\n addArrayRow(row: any[]) {\n // If user keeps pushing rows beyond batch size, reallocate\n this._reallocateColumns();\n let i = 0;\n // TODO what if no csv header, columns not populated?\n for (const fieldName in this.columns) {\n this.columns[fieldName][this.length] = row[i++];\n }\n this.length++;\n }\n\n addObjectRow(row: {[columnName: string]: any}): void {\n // If user keeps pushing rows beyond batch size, reallocate\n this._reallocateColumns();\n for (const fieldName in row) {\n this.columns[fieldName][this.length] = row[fieldName];\n }\n this.length++;\n }\n\n getBatch(): ColumnarTableBatch | ArrowTableBatch | null {\n this._pruneColumns();\n const columns = Array.isArray(this.schema) ? this.columns : {};\n\n // schema is an array if there're no headers\n // object if there are headers\n // columns should match schema format\n if (!Array.isArray(this.schema)) {\n for (const fieldName in this.schema) {\n const field = this.schema[fieldName];\n columns[field.name] = this.columns[field.index];\n }\n }\n\n this.columns = {};\n\n const batch: ColumnarTableBatch = {\n shape: 'columnar-table',\n batchType: 'data',\n data: columns,\n schema: this.schema,\n length: this.length\n };\n\n return batch;\n }\n\n // HELPERS\n\n _reallocateColumns() {\n if (this.length < this.allocated) {\n return;\n }\n\n // @ts-ignore TODO\n this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;\n this.columns = {};\n\n for (const fieldName in this.schema) {\n const field = this.schema[fieldName];\n const ArrayType = field.type || Float32Array;\n const oldColumn = this.columns[field.index];\n\n if (oldColumn && ArrayBuffer.isView(oldColumn)) {\n // Copy the old data to the new array\n const typedArray = new ArrayType(this.allocated);\n typedArray.set(oldColumn);\n this.columns[field.index] = typedArray;\n } else if (oldColumn) {\n // Plain array\n oldColumn.length = this.allocated;\n this.columns[field.index] = oldColumn;\n } else {\n // Create new\n this.columns[field.index] = new ArrayType(this.allocated);\n }\n }\n }\n\n _pruneColumns() {\n for (const [columnName, column] of Object.entries(this.columns)) {\n this.columns[columnName] = column.slice(0, this.length);\n }\n }\n}\n"],"file":"columnar-table-batch-aggregator.js"}
|
|
@@ -1,79 +1,95 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
// import type {ArrayRowTableBatch, ObjectRowTableBatch} from '../../category/table.js';
|
|
4
|
-
const row_utils_1 = require("../utils/row-utils");
|
|
1
|
+
import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
|
|
2
|
+
import { convertToArrayRow, convertToObjectRow } from '../utils/row-utils';
|
|
5
3
|
const DEFAULT_ROW_COUNT = 100;
|
|
6
|
-
class RowTableBatchAggregator {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
4
|
+
export default class RowTableBatchAggregator {
|
|
5
|
+
constructor(schema, options) {
|
|
6
|
+
_defineProperty(this, "schema", void 0);
|
|
7
|
+
|
|
8
|
+
_defineProperty(this, "options", void 0);
|
|
9
|
+
|
|
10
|
+
_defineProperty(this, "length", 0);
|
|
11
|
+
|
|
12
|
+
_defineProperty(this, "objectRows", null);
|
|
13
|
+
|
|
14
|
+
_defineProperty(this, "arrayRows", null);
|
|
15
|
+
|
|
16
|
+
_defineProperty(this, "cursor", 0);
|
|
17
|
+
|
|
18
|
+
_defineProperty(this, "_headers", []);
|
|
19
|
+
|
|
20
|
+
this.options = options;
|
|
21
|
+
this.schema = schema;
|
|
22
|
+
|
|
23
|
+
if (!Array.isArray(schema)) {
|
|
24
|
+
this._headers = [];
|
|
25
|
+
|
|
26
|
+
for (const key in schema) {
|
|
27
|
+
this._headers[schema[key].index] = schema[key].name;
|
|
28
|
+
}
|
|
23
29
|
}
|
|
24
|
-
|
|
25
|
-
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
rowCount() {
|
|
33
|
+
return this.length;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
addArrayRow(row, cursor) {
|
|
37
|
+
if (Number.isFinite(cursor)) {
|
|
38
|
+
this.cursor = cursor;
|
|
26
39
|
}
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
this.arrayRows[this.length] = row;
|
|
40
|
-
this.length++;
|
|
41
|
-
break;
|
|
42
|
-
}
|
|
40
|
+
|
|
41
|
+
switch (this.options.shape) {
|
|
42
|
+
case 'object-row-table':
|
|
43
|
+
const rowObject = convertToObjectRow(row, this._headers);
|
|
44
|
+
this.addObjectRow(rowObject, cursor);
|
|
45
|
+
break;
|
|
46
|
+
|
|
47
|
+
case 'array-row-table':
|
|
48
|
+
this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);
|
|
49
|
+
this.arrayRows[this.length] = row;
|
|
50
|
+
this.length++;
|
|
51
|
+
break;
|
|
43
52
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
switch (this.options.shape) {
|
|
50
|
-
case 'array-row-table':
|
|
51
|
-
const rowArray = (0, row_utils_1.convertToArrayRow)(row, this._headers);
|
|
52
|
-
this.addArrayRow(rowArray, cursor);
|
|
53
|
-
break;
|
|
54
|
-
case 'object-row-table':
|
|
55
|
-
this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);
|
|
56
|
-
this.objectRows[this.length] = row;
|
|
57
|
-
this.length++;
|
|
58
|
-
break;
|
|
59
|
-
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
addObjectRow(row, cursor) {
|
|
56
|
+
if (Number.isFinite(cursor)) {
|
|
57
|
+
this.cursor = cursor;
|
|
60
58
|
}
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
length: this.length,
|
|
74
|
-
schema: this.schema,
|
|
75
|
-
cursor: this.cursor
|
|
76
|
-
};
|
|
59
|
+
|
|
60
|
+
switch (this.options.shape) {
|
|
61
|
+
case 'array-row-table':
|
|
62
|
+
const rowArray = convertToArrayRow(row, this._headers);
|
|
63
|
+
this.addArrayRow(rowArray, cursor);
|
|
64
|
+
break;
|
|
65
|
+
|
|
66
|
+
case 'object-row-table':
|
|
67
|
+
this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);
|
|
68
|
+
this.objectRows[this.length] = row;
|
|
69
|
+
this.length++;
|
|
70
|
+
break;
|
|
77
71
|
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
getBatch() {
|
|
75
|
+
let rows = this.arrayRows || this.objectRows;
|
|
76
|
+
|
|
77
|
+
if (!rows) {
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
rows = rows.slice(0, this.length);
|
|
82
|
+
this.arrayRows = null;
|
|
83
|
+
this.objectRows = null;
|
|
84
|
+
return {
|
|
85
|
+
shape: this.options.shape,
|
|
86
|
+
batchType: 'data',
|
|
87
|
+
data: rows,
|
|
88
|
+
length: this.length,
|
|
89
|
+
schema: this.schema,
|
|
90
|
+
cursor: this.cursor
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
78
94
|
}
|
|
79
|
-
|
|
95
|
+
//# sourceMappingURL=row-table-batch-aggregator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../../src/lib/batches/row-table-batch-aggregator.ts"],"names":["convertToArrayRow","convertToObjectRow","DEFAULT_ROW_COUNT","RowTableBatchAggregator","constructor","schema","options","Array","isArray","_headers","key","index","name","rowCount","length","addArrayRow","row","cursor","Number","isFinite","shape","rowObject","addObjectRow","arrayRows","rowArray","objectRows","getBatch","rows","slice","batchType","data"],"mappings":";AAGA,SAAQA,iBAAR,EAA2BC,kBAA3B,QAAoD,oBAApD;AAGA,MAAMC,iBAAiB,GAAG,GAA1B;AAEA,eAAe,MAAMC,uBAAN,CAA8D;AAU3EC,EAAAA,WAAW,CAACC,MAAD,EAAiBC,OAAjB,EAA6C;AAAA;;AAAA;;AAAA,oCANvC,CAMuC;;AAAA,wCALP,IAKO;;AAAA,uCAJ9B,IAI8B;;AAAA,oCAHvC,CAGuC;;AAAA,sCAF3B,EAE2B;;AACtD,SAAKA,OAAL,GAAeA,OAAf;AACA,SAAKD,MAAL,GAAcA,MAAd;;AAIA,QAAI,CAACE,KAAK,CAACC,OAAN,CAAcH,MAAd,CAAL,EAA4B;AAC1B,WAAKI,QAAL,GAAgB,EAAhB;;AACA,WAAK,MAAMC,GAAX,IAAkBL,MAAlB,EAA0B;AACxB,aAAKI,QAAL,CAAcJ,MAAM,CAACK,GAAD,CAAN,CAAYC,KAA1B,IAAmCN,MAAM,CAACK,GAAD,CAAN,CAAYE,IAA/C;AACD;AACF;AACF;;AAEDC,EAAAA,QAAQ,GAAW;AACjB,WAAO,KAAKC,MAAZ;AACD;;AAEDC,EAAAA,WAAW,CAACC,GAAD,EAAaC,MAAb,EAAoC;AAC7C,QAAIC,MAAM,CAACC,QAAP,CAAgBF,MAAhB,CAAJ,EAA6B;AAC3B,WAAKA,MAAL,GAAcA,MAAd;AACD;;AAGD,YAAQ,KAAKX,OAAL,CAAac,KAArB;AACE,WAAK,kBAAL;AACE,cAAMC,SAAS,GAAGpB,kBAAkB,CAACe,GAAD,EAAM,KAAKP,QAAX,CAApC;AACA,aAAKa,YAAL,CAAkBD,SAAlB,EAA6BJ,MAA7B;AACA;;AACF,WAAK,iBAAL;AACE,aAAKM,SAAL,GAAiB,KAAKA,SAAL,IAAkB,IAAIhB,KAAJ,CAAUL,iBAAV,CAAnC;AACA,aAAKqB,SAAL,CAAe,KAAKT,MAApB,IAA8BE,GAA9B;AACA,aAAKF,MAAL;AACA;AATJ;AAWD;;AAEDQ,EAAAA,YAAY,CAACN,GAAD,EAAmCC,MAAnC,EAA0D;AACpE,QAAIC,MAAM,CAACC,QAAP,CAAgBF,MAAhB,CAAJ,EAA6B;AAC3B,WAAKA,MAAL,GAAcA,MAAd;AACD;;AAGD,YAAQ,KAAKX,OAAL,CAAac,KAArB;AACE,WAAK,iBAAL;AACE,cAAMI,QAAQ,GAAGxB,iBAAiB,CAACgB,GAAD,EAAM,KAAKP,QAAX,CAAlC;AACA,aAAKM,WAAL,CAAiBS,QAAjB,EAA2BP,MAA3B;AACA;;AACF,WAAK,kBAAL;AACE,aAAKQ,UAAL,GAAkB,KAAKA,UAAL,IAAmB,IAAIlB,KAAJ,CAAUL,iBAAV,CAArC;AACA,aAAKuB,UAAL,CAAgB,KAAKX,MAArB,IAA+BE,GAA/B;AACA,aAAKF,MAAL;AACA;AATJ;AAWD;;AAEDY,EAAAA,QAAQ,GAAsB;AAC5B,QAAIC,IAAI,GAAG,KAAKJ,SAAL,IAAkB,KAAKE,UAAlC;;AACA,QAAI,CAACE,IAAL,EAAW;AACT,aAAO,IAAP;AACD;;AAEDA,IAAAA,IAAI,GAAGA,IAAI,CAACC,KAAL,CAAW,CAAX,EAAc,KAAKd,MAAnB,CAAP;AACA,SAAKS,SAAL,GAAiB,IAAjB;AACA,SAAKE,UAAL,GAAkB,IAAlB;AAEA,WAAO;AACLL,MAAAA,KAAK,EAAE,KAAKd,OAAL,CAAac,KADf;AAELS,MAAAA,SAAS,EAAE,MAFN;AAGLC,MAAAA,IAAI,EAAEH,IAHD;AAILb,MAAAA,MAAM,EAAE,KAAKA,MAJR;AAKLT,MAAAA,MAAM,EAAE,KAAKA,MALR;AAMLY,MAAAA,MAAM,EAAE,KAAKA;AANR,KAAP;AAQD;;AApF0E","sourcesContent":["import type {Schema} from '../schema/schema';\nimport type {TableBatch} from '../../category/table/table-types';\n// import type {ArrayRowTableBatch, ObjectRowTableBatch} from '../../category/table';\nimport {convertToArrayRow, convertToObjectRow} from '../utils/row-utils';\nimport {TableBatchAggregator, TableBatchOptions} from './table-batch-aggregator';\n\nconst DEFAULT_ROW_COUNT = 100;\n\nexport default class RowTableBatchAggregator implements TableBatchAggregator {\n schema: Schema;\n options: TableBatchOptions;\n\n length: number = 0;\n objectRows: {[columnName: string]: any} | null = null;\n arrayRows: any[] | null = null;\n cursor: number = 0;\n private _headers: string[] = [];\n\n constructor(schema: Schema, options: TableBatchOptions) {\n this.options = options;\n this.schema = schema;\n\n // schema is an array if there're no headers\n // object if there are headers\n if (!Array.isArray(schema)) {\n this._headers = [];\n for (const key in schema) {\n this._headers[schema[key].index] = schema[key].name;\n }\n }\n }\n\n rowCount(): number {\n return this.length;\n }\n\n addArrayRow(row: any[], cursor?: number): void {\n if (Number.isFinite(cursor)) {\n this.cursor = cursor as number;\n }\n\n // eslint-disable-next-line default-case\n switch (this.options.shape) {\n case 'object-row-table':\n const rowObject = convertToObjectRow(row, this._headers);\n this.addObjectRow(rowObject, cursor);\n break;\n case 'array-row-table':\n this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);\n this.arrayRows[this.length] = row;\n this.length++;\n break;\n }\n }\n\n addObjectRow(row: {[columnName: string]: any}, cursor?: number): void {\n if (Number.isFinite(cursor)) {\n this.cursor = cursor as number;\n }\n\n // eslint-disable-next-line default-case\n switch (this.options.shape) {\n case 'array-row-table':\n const rowArray = convertToArrayRow(row, this._headers);\n this.addArrayRow(rowArray, cursor);\n break;\n case 'object-row-table':\n this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);\n this.objectRows[this.length] = row;\n this.length++;\n break;\n }\n }\n\n getBatch(): TableBatch | null {\n let rows = this.arrayRows || this.objectRows;\n if (!rows) {\n return null;\n }\n\n rows = rows.slice(0, this.length);\n this.arrayRows = null;\n this.objectRows = null;\n\n return {\n shape: this.options.shape,\n batchType: 'data',\n data: rows,\n length: this.length,\n schema: this.schema,\n cursor: this.cursor\n };\n }\n}\n"],"file":"row-table-batch-aggregator.js"}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
export {};
|
|
2
|
+
//# sourceMappingURL=table-batch-aggregator.js.map
|
|
File without changes
|