@loaders.gl/schema 3.0.13 → 3.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es5/bundle.js +2 -2
- package/dist/es5/bundle.js.map +1 -1
- package/dist/es5/category/mesh/mesh-utils.js +16 -16
- package/dist/es5/category/mesh/mesh-utils.js.map +1 -1
- package/dist/es5/index.js +46 -46
- package/dist/es5/lib/schema/impl/enum.js +1 -1
- package/dist/es5/lib/schema/impl/field.js +32 -19
- package/dist/es5/lib/schema/impl/field.js.map +1 -1
- package/dist/es5/lib/schema/impl/schema.js +119 -54
- package/dist/es5/lib/schema/impl/schema.js.map +1 -1
- package/dist/es5/lib/schema/impl/type.js +693 -375
- package/dist/es5/lib/schema/impl/type.js.map +1 -1
- package/dist/es5/lib/schema/index.js +36 -36
- package/dist/es5/lib/schema-utils/deduce-table-schema.js +9 -9
- package/dist/es5/lib/schema-utils/deduce-table-schema.js.map +1 -1
- package/dist/es5/lib/schema-utils/get-type-info.js +3 -3
- package/dist/es5/lib/schema-utils/get-type-info.js.map +1 -1
- package/dist/es5/lib/table/base-table-batch-aggregator.js +53 -42
- package/dist/es5/lib/table/base-table-batch-aggregator.js.map +1 -1
- package/dist/es5/lib/table/columnar-table-batch-aggregator.js +90 -71
- package/dist/es5/lib/table/columnar-table-batch-aggregator.js.map +1 -1
- package/dist/es5/lib/table/row-table-batch-aggregator.js +70 -59
- package/dist/es5/lib/table/row-table-batch-aggregator.js.map +1 -1
- package/dist/es5/lib/table/table-batch-builder.js +133 -113
- package/dist/es5/lib/table/table-batch-builder.js.map +1 -1
- package/dist/es5/lib/utils/async-queue.js +164 -81
- package/dist/es5/lib/utils/async-queue.js.map +1 -1
- package/dist/es5/lib/utils/row-utils.js +4 -4
- package/dist/es5/lib/utils/row-utils.js.map +1 -1
- package/package.json +2 -2
|
@@ -7,6 +7,10 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
7
7
|
});
|
|
8
8
|
exports.default = void 0;
|
|
9
9
|
|
|
10
|
+
var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
|
|
11
|
+
|
|
12
|
+
var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
|
|
13
|
+
|
|
10
14
|
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
11
15
|
|
|
12
16
|
var _baseTableBatchAggregator = _interopRequireDefault(require("./base-table-batch-aggregator"));
|
|
@@ -15,17 +19,22 @@ var _rowTableBatchAggregator = _interopRequireDefault(require("./row-table-batch
|
|
|
15
19
|
|
|
16
20
|
var _columnarTableBatchAggregator = _interopRequireDefault(require("./columnar-table-batch-aggregator"));
|
|
17
21
|
|
|
18
|
-
|
|
22
|
+
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
|
|
23
|
+
|
|
24
|
+
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
|
|
25
|
+
|
|
26
|
+
var DEFAULT_OPTIONS = {
|
|
19
27
|
shape: 'array-row-table',
|
|
20
28
|
batchSize: 'auto',
|
|
21
29
|
batchDebounceMs: 0,
|
|
22
30
|
limit: 0,
|
|
23
31
|
_limitMB: 0
|
|
24
32
|
};
|
|
25
|
-
|
|
33
|
+
var ERR_MESSAGE = 'TableBatchBuilder';
|
|
26
34
|
|
|
27
|
-
|
|
28
|
-
|
|
35
|
+
var TableBatchBuilder = function () {
|
|
36
|
+
function TableBatchBuilder(schema, options) {
|
|
37
|
+
(0, _classCallCheck2.default)(this, TableBatchBuilder);
|
|
29
38
|
(0, _defineProperty2.default)(this, "schema", void 0);
|
|
30
39
|
(0, _defineProperty2.default)(this, "options", void 0);
|
|
31
40
|
(0, _defineProperty2.default)(this, "aggregator", null);
|
|
@@ -37,150 +46,161 @@ class TableBatchBuilder {
|
|
|
37
46
|
(0, _defineProperty2.default)(this, "totalBytes", 0);
|
|
38
47
|
(0, _defineProperty2.default)(this, "rowBytes", 0);
|
|
39
48
|
this.schema = schema;
|
|
40
|
-
this.options = {
|
|
41
|
-
...options
|
|
42
|
-
};
|
|
49
|
+
this.options = _objectSpread(_objectSpread({}, DEFAULT_OPTIONS), options);
|
|
43
50
|
}
|
|
44
51
|
|
|
45
|
-
|
|
46
|
-
|
|
52
|
+
(0, _createClass2.default)(TableBatchBuilder, [{
|
|
53
|
+
key: "limitReached",
|
|
54
|
+
value: function limitReached() {
|
|
55
|
+
var _this$options, _this$options2;
|
|
47
56
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
if (Boolean((_this$options2 = this.options) === null || _this$options2 === void 0 ? void 0 : _this$options2._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
|
|
53
|
-
return true;
|
|
54
|
-
}
|
|
57
|
+
if (Boolean((_this$options = this.options) === null || _this$options === void 0 ? void 0 : _this$options.limit) && this.totalLength >= this.options.limit) {
|
|
58
|
+
return true;
|
|
59
|
+
}
|
|
55
60
|
|
|
56
|
-
|
|
57
|
-
|
|
61
|
+
if (Boolean((_this$options2 = this.options) === null || _this$options2 === void 0 ? void 0 : _this$options2._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
|
|
62
|
+
return true;
|
|
63
|
+
}
|
|
58
64
|
|
|
59
|
-
|
|
60
|
-
if (this.limitReached()) {
|
|
61
|
-
return;
|
|
65
|
+
return false;
|
|
62
66
|
}
|
|
67
|
+
}, {
|
|
68
|
+
key: "addRow",
|
|
69
|
+
value: function addRow(row) {
|
|
70
|
+
if (this.limitReached()) {
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
63
73
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
74
|
+
this.totalLength++;
|
|
75
|
+
this.rowBytes = this.rowBytes || this._estimateRowMB(row);
|
|
76
|
+
this.totalBytes += this.rowBytes;
|
|
67
77
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
78
|
+
if (Array.isArray(row)) {
|
|
79
|
+
this.addArrayRow(row);
|
|
80
|
+
} else {
|
|
81
|
+
this.addObjectRow(row);
|
|
82
|
+
}
|
|
72
83
|
}
|
|
73
|
-
}
|
|
84
|
+
}, {
|
|
85
|
+
key: "addArrayRow",
|
|
86
|
+
value: function addArrayRow(row) {
|
|
87
|
+
if (!this.aggregator) {
|
|
88
|
+
var TableBatchType = this._getTableBatchType();
|
|
74
89
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
const TableBatchType = this._getTableBatchType();
|
|
90
|
+
this.aggregator = new TableBatchType(this.schema, this.options);
|
|
91
|
+
}
|
|
78
92
|
|
|
79
|
-
this.aggregator
|
|
93
|
+
this.aggregator.addArrayRow(row);
|
|
80
94
|
}
|
|
95
|
+
}, {
|
|
96
|
+
key: "addObjectRow",
|
|
97
|
+
value: function addObjectRow(row) {
|
|
98
|
+
if (!this.aggregator) {
|
|
99
|
+
var TableBatchType = this._getTableBatchType();
|
|
81
100
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
addObjectRow(row) {
|
|
86
|
-
if (!this.aggregator) {
|
|
87
|
-
const TableBatchType = this._getTableBatchType();
|
|
101
|
+
this.aggregator = new TableBatchType(this.schema, this.options);
|
|
102
|
+
}
|
|
88
103
|
|
|
89
|
-
this.aggregator
|
|
104
|
+
this.aggregator.addObjectRow(row);
|
|
90
105
|
}
|
|
106
|
+
}, {
|
|
107
|
+
key: "chunkComplete",
|
|
108
|
+
value: function chunkComplete(chunk) {
|
|
109
|
+
if (chunk instanceof ArrayBuffer) {
|
|
110
|
+
this.bytesUsed += chunk.byteLength;
|
|
111
|
+
}
|
|
91
112
|
|
|
92
|
-
|
|
93
|
-
|
|
113
|
+
if (typeof chunk === 'string') {
|
|
114
|
+
this.bytesUsed += chunk.length;
|
|
115
|
+
}
|
|
94
116
|
|
|
95
|
-
|
|
96
|
-
if (chunk instanceof ArrayBuffer) {
|
|
97
|
-
this.bytesUsed += chunk.byteLength;
|
|
117
|
+
this.isChunkComplete = true;
|
|
98
118
|
}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
119
|
+
}, {
|
|
120
|
+
key: "getFullBatch",
|
|
121
|
+
value: function getFullBatch(options) {
|
|
122
|
+
return this._isFull() ? this._getBatch(options) : null;
|
|
102
123
|
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
getFullBatch(options) {
|
|
108
|
-
return this._isFull() ? this._getBatch(options) : null;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
getFinalBatch(options) {
|
|
112
|
-
return this._getBatch(options);
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
_estimateRowMB(row) {
|
|
116
|
-
return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
_isFull() {
|
|
120
|
-
if (!this.aggregator || this.aggregator.rowCount() === 0) {
|
|
121
|
-
return false;
|
|
124
|
+
}, {
|
|
125
|
+
key: "getFinalBatch",
|
|
126
|
+
value: function getFinalBatch(options) {
|
|
127
|
+
return this._getBatch(options);
|
|
122
128
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
129
|
+
}, {
|
|
130
|
+
key: "_estimateRowMB",
|
|
131
|
+
value: function _estimateRowMB(row) {
|
|
132
|
+
return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
|
|
133
|
+
}
|
|
134
|
+
}, {
|
|
135
|
+
key: "_isFull",
|
|
136
|
+
value: function _isFull() {
|
|
137
|
+
if (!this.aggregator || this.aggregator.rowCount() === 0) {
|
|
126
138
|
return false;
|
|
127
139
|
}
|
|
128
|
-
} else if (this.options.batchSize > this.aggregator.rowCount()) {
|
|
129
|
-
return false;
|
|
130
|
-
}
|
|
131
140
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
141
|
+
if (this.options.batchSize === 'auto') {
|
|
142
|
+
if (!this.isChunkComplete) {
|
|
143
|
+
return false;
|
|
144
|
+
}
|
|
145
|
+
} else if (this.options.batchSize > this.aggregator.rowCount()) {
|
|
146
|
+
return false;
|
|
147
|
+
}
|
|
135
148
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
}
|
|
149
|
+
if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
|
|
150
|
+
return false;
|
|
151
|
+
}
|
|
140
152
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
return
|
|
153
|
+
this.isChunkComplete = false;
|
|
154
|
+
this.lastBatchEmittedMs = Date.now();
|
|
155
|
+
return true;
|
|
144
156
|
}
|
|
157
|
+
}, {
|
|
158
|
+
key: "_getBatch",
|
|
159
|
+
value: function _getBatch(options) {
|
|
160
|
+
if (!this.aggregator) {
|
|
161
|
+
return null;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
if (options !== null && options !== void 0 && options.bytesUsed) {
|
|
165
|
+
this.bytesUsed = options.bytesUsed;
|
|
166
|
+
}
|
|
145
167
|
|
|
146
|
-
|
|
147
|
-
|
|
168
|
+
var normalizedBatch = this.aggregator.getBatch();
|
|
169
|
+
normalizedBatch.count = this.batchCount;
|
|
170
|
+
normalizedBatch.bytesUsed = this.bytesUsed;
|
|
171
|
+
Object.assign(normalizedBatch, options);
|
|
172
|
+
this.batchCount++;
|
|
173
|
+
this.aggregator = null;
|
|
174
|
+
return normalizedBatch;
|
|
148
175
|
}
|
|
176
|
+
}, {
|
|
177
|
+
key: "_getTableBatchType",
|
|
178
|
+
value: function _getTableBatchType() {
|
|
179
|
+
switch (this.options.shape) {
|
|
180
|
+
case 'row-table':
|
|
181
|
+
return _baseTableBatchAggregator.default;
|
|
149
182
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
Object.assign(normalizedBatch, options);
|
|
154
|
-
this.batchCount++;
|
|
155
|
-
this.aggregator = null;
|
|
156
|
-
return normalizedBatch;
|
|
157
|
-
}
|
|
183
|
+
case 'array-row-table':
|
|
184
|
+
case 'object-row-table':
|
|
185
|
+
return _rowTableBatchAggregator.default;
|
|
158
186
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
case 'row-table':
|
|
162
|
-
return _baseTableBatchAggregator.default;
|
|
187
|
+
case 'columnar-table':
|
|
188
|
+
return _columnarTableBatchAggregator.default;
|
|
163
189
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
190
|
+
case 'arrow-table':
|
|
191
|
+
if (!TableBatchBuilder.ArrowBatch) {
|
|
192
|
+
throw new Error(ERR_MESSAGE);
|
|
193
|
+
}
|
|
167
194
|
|
|
168
|
-
|
|
169
|
-
return _columnarTableBatchAggregator.default;
|
|
195
|
+
return TableBatchBuilder.ArrowBatch;
|
|
170
196
|
|
|
171
|
-
|
|
172
|
-
if (!TableBatchBuilder.ArrowBatch) {
|
|
197
|
+
default:
|
|
173
198
|
throw new Error(ERR_MESSAGE);
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
return TableBatchBuilder.ArrowBatch;
|
|
177
|
-
|
|
178
|
-
default:
|
|
179
|
-
throw new Error(ERR_MESSAGE);
|
|
199
|
+
}
|
|
180
200
|
}
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
}
|
|
201
|
+
}]);
|
|
202
|
+
return TableBatchBuilder;
|
|
203
|
+
}();
|
|
184
204
|
|
|
185
205
|
exports.default = TableBatchBuilder;
|
|
186
206
|
(0, _defineProperty2.default)(TableBatchBuilder, "ArrowBatch", void 0);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../src/lib/table/table-batch-builder.ts"],"names":["DEFAULT_OPTIONS","shape","batchSize","batchDebounceMs","limit","_limitMB","ERR_MESSAGE","TableBatchBuilder","constructor","schema","options","Date","now","limitReached","Boolean","totalLength","totalBytes","addRow","row","rowBytes","_estimateRowMB","Array","isArray","addArrayRow","addObjectRow","aggregator","TableBatchType","_getTableBatchType","chunkComplete","chunk","ArrayBuffer","bytesUsed","byteLength","length","isChunkComplete","getFullBatch","_isFull","_getBatch","getFinalBatch","Object","keys","rowCount","lastBatchEmittedMs","normalizedBatch","getBatch","count","batchCount","assign","BaseTableBatchAggregator","RowTableBatchAggregator","ColumnarTableBatchAggregator","ArrowBatch","Error"],"mappings":";;;;;;;;;;;AAGA;;AACA;;AACA;;AAgBA,MAAMA,eAAmD,GAAG;AAC1DC,EAAAA,KAAK,EAAE,iBADmD;AAE1DC,EAAAA,SAAS,EAAE,MAF+C;AAG1DC,EAAAA,eAAe,EAAE,CAHyC;AAI1DC,EAAAA,KAAK,EAAE,CAJmD;AAK1DC,EAAAA,QAAQ,EAAE;AALgD,CAA5D;AAQA,MAAMC,WAAW,GAAG,mBAApB;;AAGe,MAAMC,iBAAN,CAAwB;AAerCC,EAAAA,WAAW,CAACC,MAAD,EAAiBC,OAAjB,EAAqD;AAAA;AAAA;AAAA,sDAXd,IAWc;AAAA,sDAVnC,CAUmC;AAAA,qDATpC,CASoC;AAAA,2DAR7B,KAQ6B;AAAA,8DAP3BC,IAAI,CAACC,GAAL,EAO2B;AAAA,uDANlC,CAMkC;AAAA,sDALnC,CAKmC;AAAA,oDAJrC,CAIqC;AAC9D,SAAKH,MAAL,GAAcA,MAAd;AACA,SAAKC,OAAL,GAAe,EAAC,GAAGV,eAAJ;AAAqB,SAAGU;AAAxB,KAAf;AACD;;AAEDG,EAAAA,YAAY,GAAY;AAAA;;AACtB,QAAIC,OAAO,kBAAC,KAAKJ,OAAN,kDAAC,cAAcN,KAAf,CAAP,IAAgC,KAAKW,WAAL,IAAoB,KAAKL,OAAL,CAAaN,KAArE,EAA4E;AAC1E,aAAO,IAAP;AACD;;AACD,QAAIU,OAAO,mBAAC,KAAKJ,OAAN,mDAAC,eAAcL,QAAf,CAAP,IAAmC,KAAKW,UAAL,GAAkB,GAAlB,IAAyB,KAAKN,OAAL,CAAaL,QAA7E,EAAuF;AACrF,aAAO,IAAP;AACD;;AACD,WAAO,KAAP;AACD;;AAGDY,EAAAA,MAAM,CAACC,GAAD,EAAiD;AACrD,QAAI,KAAKL,YAAL,EAAJ,EAAyB;AACvB;AACD;;AACD,SAAKE,WAAL;AACA,SAAKI,QAAL,GAAgB,KAAKA,QAAL,IAAiB,KAAKC,cAAL,CAAoBF,GAApB,CAAjC;AACA,SAAKF,UAAL,IAAmB,KAAKG,QAAxB;;AACA,QAAIE,KAAK,CAACC,OAAN,CAAcJ,GAAd,CAAJ,EAAwB;AACtB,WAAKK,WAAL,CAAiBL,GAAjB;AACD,KAFD,MAEO;AACL,WAAKM,YAAL,CAAkBN,GAAlB;AACD;AACF;;AAGSK,EAAAA,WAAW,CAACL,GAAD,EAAa;AAChC,QAAI,CAAC,KAAKO,UAAV,EAAsB;AACpB,YAAMC,cAAc,GAAG,KAAKC,kBAAL,EAAvB;;AACA,WAAKF,UAAL,GAAkB,IAAIC,cAAJ,CAAmB,KAAKjB,MAAxB,EAAgC,KAAKC,OAArC,CAAlB;AACD;;AACD,SAAKe,UAAL,CAAgBF,WAAhB,CAA4BL,GAA5B;AACD;;AAGSM,EAAAA,YAAY,CAACN,GAAD,EAAyC;AAC7D,QAAI,CAAC,KAAKO,UAAV,EAAsB;AACpB,YAAMC,cAAc,GAAG,KAAKC,kBAAL,EAAvB;;AACA,WAAKF,UAAL,GAAkB,IAAIC,cAAJ,CAAmB,KAAKjB,MAAxB,EAAgC,KAAKC,OAArC,CAAlB;AACD;;AACD,SAAKe,UAAL,CAAgBD,YAAhB,CAA6BN,GAA7B;AACD;;AAGDU,EAAAA,aAAa,CAACC,KAAD,EAAoC;AAC/C,QAAIA,KAAK,YAAYC,WAArB,EAAkC;AAChC,WAAKC,SAAL,IAAkBF,KAAK,CAACG,UAAxB;AACD;;AACD,QAAI,OAAOH,KAAP,KAAiB,QAArB,EAA+B;AAC7B,WAAKE,SAAL,IAAkBF,KAAK,CAACI,MAAxB;AACD;;AACD,SAAKC,eAAL,GAAuB,IAAvB;AACD;;AAEDC,EAAAA,YAAY,CAACzB,OAAD,EAA+C;AACzD,WAAO,KAAK0B,OAAL,KAAiB,KAAKC,SAAL,CAAe3B,OAAf,CAAjB,GAA2C,IAAlD;AACD;;AAED4B,EAAAA,aAAa,CAAC5B,OAAD,EAA+C;AAC1D,WAAO,KAAK2B,SAAL,CAAe3B,OAAf,CAAP;AACD;;AAIDU,EAAAA,cAAc,CAACF,GAAD,EAAM;AAClB,WAAOG,KAAK,CAACC,OAAN,CAAcJ,GAAd,IAAqBA,GAAG,CAACe,MAAJ,GAAa,CAAlC,GAAsCM,MAAM,CAACC,IAAP,CAAYtB,GAAZ,EAAiBe,MAAjB,GAA0B,CAAvE;AACD;;AAEOG,EAAAA,OAAO,GAAY;AAEzB,QAAI,CAAC,KAAKX,UAAN,IAAoB,KAAKA,UAAL,CAAgBgB,QAAhB,OAA+B,CAAvD,EAA0D;AACxD,aAAO,KAAP;AACD;;AAID,QAAI,KAAK/B,OAAL,CAAaR,SAAb,KAA2B,MAA/B,EAAuC;AACrC,UAAI,CAAC,KAAKgC,eAAV,EAA2B;AACzB,eAAO,KAAP;AACD;AACF,KAJD,MAIO,IAAI,KAAKxB,OAAL,CAAaR,SAAb,GAAyB,KAAKuB,UAAL,CAAgBgB,QAAhB,EAA7B,EAAyD;AAC9D,aAAO,KAAP;AACD;;AAGD,QAAI,KAAK/B,OAAL,CAAaP,eAAb,GAA+BQ,IAAI,CAACC,GAAL,KAAa,KAAK8B,kBAArD,EAAyE;AACvE,aAAO,KAAP;AACD;;AAGD,SAAKR,eAAL,GAAuB,KAAvB;AACA,SAAKQ,kBAAL,GAA0B/B,IAAI,CAACC,GAAL,EAA1B;AACA,WAAO,IAAP;AACD;;AAKOyB,EAAAA,SAAS,CAAC3B,OAAD,EAA+C;AAC9D,QAAI,CAAC,KAAKe,UAAV,EAAsB;AACpB,aAAO,IAAP;AACD;;AAGD,QAAIf,OAAJ,aAAIA,OAAJ,eAAIA,OAAO,CAAEqB,SAAb,EAAwB;AACtB,WAAKA,SAAL,GAAiBrB,OAAO,CAACqB,SAAzB;AACD;;AACD,UAAMY,eAAe,GAAG,KAAKlB,UAAL,CAAgBmB,QAAhB,EAAxB;AACAD,IAAAA,eAAe,CAACE,KAAhB,GAAwB,KAAKC,UAA7B;AACAH,IAAAA,eAAe,CAACZ,SAAhB,GAA4B,KAAKA,SAAjC;AACAQ,IAAAA,MAAM,CAACQ,MAAP,CAAcJ,eAAd,EAA+BjC,OAA/B;AAEA,SAAKoC,UAAL;AACA,SAAKrB,UAAL,GAAkB,IAAlB;AACA,WAAOkB,eAAP;AACD;;AAEOhB,EAAAA,kBAAkB,GAA0B;AAClD,YAAQ,KAAKjB,OAAL,CAAaT,KAArB;AACE,WAAK,WAAL;AACE,eAAO+C,iCAAP;;AACF,WAAK,iBAAL;AACA,WAAK,kBAAL;AACE,eAAOC,gCAAP;;AACF,WAAK,gBAAL;AACE,eAAOC,qCAAP;;AACF,WAAK,aAAL;AACE,YAAI,CAAC3C,iBAAiB,CAAC4C,UAAvB,EAAmC;AACjC,gBAAM,IAAIC,KAAJ,CAAU9C,WAAV,CAAN;AACD;;AACD,eAAOC,iBAAiB,CAAC4C,UAAzB;;AACF;AACE,cAAM,IAAIC,KAAJ,CAAU9C,WAAV,CAAN;AAdJ;AAgBD;;AA1JoC;;;8BAAlBC,iB","sourcesContent":["import type {Schema} from '../schema';\nimport type {TableBatch} from '../../category/table';\nimport type {TableBatchAggregator, TableBatchConstructor} from './table-batch-aggregator';\nimport BaseTableBatchAggregator from './base-table-batch-aggregator';\nimport RowTableBatchAggregator from './row-table-batch-aggregator';\nimport ColumnarTableBatchAggregator from './columnar-table-batch-aggregator';\n\n// TODO define interface instead\ntype TableBatchBuilderOptions = {\n shape: 'row-table' | 'array-row-table' | 'object-row-table' | 'columnar-table' | 'arrow-table';\n batchSize?: number | 'auto';\n batchDebounceMs?: number;\n limit: number;\n _limitMB: number;\n};\n\ntype GetBatchOptions = {\n bytesUsed?: number;\n [key: string]: any;\n};\n\nconst DEFAULT_OPTIONS: Required<TableBatchBuilderOptions> = {\n shape: 'array-row-table',\n batchSize: 'auto',\n batchDebounceMs: 0,\n limit: 0,\n _limitMB: 0\n};\n\nconst ERR_MESSAGE = 'TableBatchBuilder';\n\n/** Incrementally builds batches from a stream of rows */\nexport default class TableBatchBuilder {\n schema: Schema;\n options: Required<TableBatchBuilderOptions>;\n\n private aggregator: TableBatchAggregator | null = null;\n private batchCount: number = 0;\n private bytesUsed: number = 0;\n private isChunkComplete: boolean = false;\n private lastBatchEmittedMs: number = Date.now();\n private totalLength: number = 0;\n private totalBytes: number = 0;\n private rowBytes: number = 0;\n\n static ArrowBatch?: TableBatchConstructor;\n\n constructor(schema: Schema, options?: TableBatchBuilderOptions) {\n this.schema = schema;\n this.options = {...DEFAULT_OPTIONS, ...options};\n }\n\n limitReached(): boolean {\n if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {\n return true;\n }\n if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {\n return true;\n }\n return false;\n }\n\n /** @deprecated Use addArrayRow or addObjectRow */\n addRow(row: any[] | {[columnName: string]: any}): void {\n if (this.limitReached()) {\n return;\n }\n this.totalLength++;\n this.rowBytes = this.rowBytes || this._estimateRowMB(row);\n this.totalBytes += this.rowBytes;\n if (Array.isArray(row)) {\n this.addArrayRow(row);\n } else {\n this.addObjectRow(row);\n }\n }\n\n /** Add one row to the batch */\n protected addArrayRow(row: any[]) {\n if (!this.aggregator) {\n const TableBatchType = this._getTableBatchType();\n this.aggregator = new TableBatchType(this.schema, this.options);\n }\n this.aggregator.addArrayRow(row);\n }\n\n /** Add one row to the batch */\n protected addObjectRow(row: {[columnName: string]: any}): void {\n if (!this.aggregator) {\n const TableBatchType = this._getTableBatchType();\n this.aggregator = new TableBatchType(this.schema, this.options);\n }\n this.aggregator.addObjectRow(row);\n }\n\n /** Mark an incoming raw memory chunk has completed */\n chunkComplete(chunk: ArrayBuffer | string): void {\n if (chunk instanceof ArrayBuffer) {\n this.bytesUsed += chunk.byteLength;\n }\n if (typeof chunk === 'string') {\n this.bytesUsed += chunk.length;\n }\n this.isChunkComplete = true;\n }\n\n getFullBatch(options?: GetBatchOptions): TableBatch | null {\n return this._isFull() ? this._getBatch(options) : null;\n }\n\n getFinalBatch(options?: GetBatchOptions): TableBatch | null {\n return this._getBatch(options);\n }\n\n // INTERNAL\n\n _estimateRowMB(row) {\n return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;\n }\n\n private _isFull(): boolean {\n // No batch, not ready\n if (!this.aggregator || this.aggregator.rowCount() === 0) {\n return false;\n }\n\n // if batchSize === 'auto' we wait for chunk to complete\n // if batchSize === number, ensure we have enough rows\n if (this.options.batchSize === 'auto') {\n if (!this.isChunkComplete) {\n return false;\n }\n } else if (this.options.batchSize > this.aggregator.rowCount()) {\n return false;\n }\n\n // Debounce batches\n if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {\n return false;\n }\n\n // Emit batch\n this.isChunkComplete = false;\n this.lastBatchEmittedMs = Date.now();\n return true;\n }\n\n /**\n * bytesUsed can be set via chunkComplete or via getBatch*\n */\n private _getBatch(options?: GetBatchOptions): TableBatch | null {\n if (!this.aggregator) {\n return null;\n }\n\n // TODO - this can overly increment bytes used?\n if (options?.bytesUsed) {\n this.bytesUsed = options.bytesUsed;\n }\n const normalizedBatch = this.aggregator.getBatch() as TableBatch;\n normalizedBatch.count = this.batchCount;\n normalizedBatch.bytesUsed = this.bytesUsed;\n Object.assign(normalizedBatch, options);\n\n this.batchCount++;\n this.aggregator = null;\n return normalizedBatch;\n }\n\n private _getTableBatchType(): TableBatchConstructor {\n switch (this.options.shape) {\n case 'row-table':\n return BaseTableBatchAggregator;\n case 'array-row-table':\n case 'object-row-table':\n return RowTableBatchAggregator;\n case 'columnar-table':\n return ColumnarTableBatchAggregator;\n case 'arrow-table':\n if (!TableBatchBuilder.ArrowBatch) {\n throw new Error(ERR_MESSAGE);\n }\n return TableBatchBuilder.ArrowBatch;\n default:\n throw new Error(ERR_MESSAGE);\n }\n }\n}\n"],"file":"table-batch-builder.js"}
|
|
1
|
+
{"version":3,"sources":["../../../../src/lib/table/table-batch-builder.ts"],"names":["DEFAULT_OPTIONS","shape","batchSize","batchDebounceMs","limit","_limitMB","ERR_MESSAGE","TableBatchBuilder","schema","options","Date","now","Boolean","totalLength","totalBytes","row","limitReached","rowBytes","_estimateRowMB","Array","isArray","addArrayRow","addObjectRow","aggregator","TableBatchType","_getTableBatchType","chunk","ArrayBuffer","bytesUsed","byteLength","length","isChunkComplete","_isFull","_getBatch","Object","keys","rowCount","lastBatchEmittedMs","normalizedBatch","getBatch","count","batchCount","assign","BaseTableBatchAggregator","RowTableBatchAggregator","ColumnarTableBatchAggregator","ArrowBatch","Error"],"mappings":";;;;;;;;;;;;;;;AAGA;;AACA;;AACA;;;;;;AAgBA,IAAMA,eAAmD,GAAG;AAC1DC,EAAAA,KAAK,EAAE,iBADmD;AAE1DC,EAAAA,SAAS,EAAE,MAF+C;AAG1DC,EAAAA,eAAe,EAAE,CAHyC;AAI1DC,EAAAA,KAAK,EAAE,CAJmD;AAK1DC,EAAAA,QAAQ,EAAE;AALgD,CAA5D;AAQA,IAAMC,WAAW,GAAG,mBAApB;;IAGqBC,iB;AAenB,6BAAYC,MAAZ,EAA4BC,OAA5B,EAAgE;AAAA;AAAA;AAAA;AAAA,sDAXd,IAWc;AAAA,sDAVnC,CAUmC;AAAA,qDATpC,CASoC;AAAA,2DAR7B,KAQ6B;AAAA,8DAP3BC,IAAI,CAACC,GAAL,EAO2B;AAAA,uDANlC,CAMkC;AAAA,sDALnC,CAKmC;AAAA,oDAJrC,CAIqC;AAC9D,SAAKH,MAAL,GAAcA,MAAd;AACA,SAAKC,OAAL,mCAAmBT,eAAnB,GAAuCS,OAAvC;AACD;;;;WAED,wBAAwB;AAAA;;AACtB,UAAIG,OAAO,kBAAC,KAAKH,OAAN,kDAAC,cAAcL,KAAf,CAAP,IAAgC,KAAKS,WAAL,IAAoB,KAAKJ,OAAL,CAAaL,KAArE,EAA4E;AAC1E,eAAO,IAAP;AACD;;AACD,UAAIQ,OAAO,mBAAC,KAAKH,OAAN,mDAAC,eAAcJ,QAAf,CAAP,IAAmC,KAAKS,UAAL,GAAkB,GAAlB,IAAyB,KAAKL,OAAL,CAAaJ,QAA7E,EAAuF;AACrF,eAAO,IAAP;AACD;;AACD,aAAO,KAAP;AACD;;;WAGD,gBAAOU,GAAP,EAAuD;AACrD,UAAI,KAAKC,YAAL,EAAJ,EAAyB;AACvB;AACD;;AACD,WAAKH,WAAL;AACA,WAAKI,QAAL,GAAgB,KAAKA,QAAL,IAAiB,KAAKC,cAAL,CAAoBH,GAApB,CAAjC;AACA,WAAKD,UAAL,IAAmB,KAAKG,QAAxB;;AACA,UAAIE,KAAK,CAACC,OAAN,CAAcL,GAAd,CAAJ,EAAwB;AACtB,aAAKM,WAAL,CAAiBN,GAAjB;AACD,OAFD,MAEO;AACL,aAAKO,YAAL,CAAkBP,GAAlB;AACD;AACF;;;WAGD,qBAAsBA,GAAtB,EAAkC;AAChC,UAAI,CAAC,KAAKQ,UAAV,EAAsB;AACpB,YAAMC,cAAc,GAAG,KAAKC,kBAAL,EAAvB;;AACA,aAAKF,UAAL,GAAkB,IAAIC,cAAJ,CAAmB,KAAKhB,MAAxB,EAAgC,KAAKC,OAArC,CAAlB;AACD;;AACD,WAAKc,UAAL,CAAgBF,WAAhB,CAA4BN,GAA5B;AACD;;;WAGD,sBAAuBA,GAAvB,EAA+D;AAC7D,UAAI,CAAC,KAAKQ,UAAV,EAAsB;AACpB,YAAMC,cAAc,GAAG,KAAKC,kBAAL,EAAvB;;AACA,aAAKF,UAAL,GAAkB,IAAIC,cAAJ,CAAmB,KAAKhB,MAAxB,EAAgC,KAAKC,OAArC,CAAlB;AACD;;AACD,WAAKc,UAAL,CAAgBD,YAAhB,CAA6BP,GAA7B;AACD;;;WAGD,uBAAcW,KAAd,EAAiD;AAC/C,UAAIA,KAAK,YAAYC,WAArB,EAAkC;AAChC,aAAKC,SAAL,IAAkBF,KAAK,CAACG,UAAxB;AACD;;AACD,UAAI,OAAOH,KAAP,KAAiB,QAArB,EAA+B;AAC7B,aAAKE,SAAL,IAAkBF,KAAK,CAACI,MAAxB;AACD;;AACD,WAAKC,eAAL,GAAuB,IAAvB;AACD;;;WAED,sBAAatB,OAAb,EAA2D;AACzD,aAAO,KAAKuB,OAAL,KAAiB,KAAKC,SAAL,CAAexB,OAAf,CAAjB,GAA2C,IAAlD;AACD;;;WAED,uBAAcA,OAAd,EAA4D;AAC1D,aAAO,KAAKwB,SAAL,CAAexB,OAAf,CAAP;AACD;;;WAID,wBAAeM,GAAf,EAAoB;AAClB,aAAOI,KAAK,CAACC,OAAN,CAAcL,GAAd,IAAqBA,GAAG,CAACe,MAAJ,GAAa,CAAlC,GAAsCI,MAAM,CAACC,IAAP,CAAYpB,GAAZ,EAAiBe,MAAjB,GAA0B,CAAvE;AACD;;;WAED,mBAA2B;AAEzB,UAAI,CAAC,KAAKP,UAAN,IAAoB,KAAKA,UAAL,CAAgBa,QAAhB,OAA+B,CAAvD,EAA0D;AACxD,eAAO,KAAP;AACD;;AAID,UAAI,KAAK3B,OAAL,CAAaP,SAAb,KAA2B,MAA/B,EAAuC;AACrC,YAAI,CAAC,KAAK6B,eAAV,EAA2B;AACzB,iBAAO,KAAP;AACD;AACF,OAJD,MAIO,IAAI,KAAKtB,OAAL,CAAaP,SAAb,GAAyB,KAAKqB,UAAL,CAAgBa,QAAhB,EAA7B,EAAyD;AAC9D,eAAO,KAAP;AACD;;AAGD,UAAI,KAAK3B,OAAL,CAAaN,eAAb,GAA+BO,IAAI,CAACC,GAAL,KAAa,KAAK0B,kBAArD,EAAyE;AACvE,eAAO,KAAP;AACD;;AAGD,WAAKN,eAAL,GAAuB,KAAvB;AACA,WAAKM,kBAAL,GAA0B3B,IAAI,CAACC,GAAL,EAA1B;AACA,aAAO,IAAP;AACD;;;WAKD,mBAAkBF,OAAlB,EAAgE;AAC9D,UAAI,CAAC,KAAKc,UAAV,EAAsB;AACpB,eAAO,IAAP;AACD;;AAGD,UAAId,OAAJ,aAAIA,OAAJ,eAAIA,OAAO,CAAEmB,SAAb,EAAwB;AACtB,aAAKA,SAAL,GAAiBnB,OAAO,CAACmB,SAAzB;AACD;;AACD,UAAMU,eAAe,GAAG,KAAKf,UAAL,CAAgBgB,QAAhB,EAAxB;AACAD,MAAAA,eAAe,CAACE,KAAhB,GAAwB,KAAKC,UAA7B;AACAH,MAAAA,eAAe,CAACV,SAAhB,GAA4B,KAAKA,SAAjC;AACAM,MAAAA,MAAM,CAACQ,MAAP,CAAcJ,eAAd,EAA+B7B,OAA/B;AAEA,WAAKgC,UAAL;AACA,WAAKlB,UAAL,GAAkB,IAAlB;AACA,aAAOe,eAAP;AACD;;;WAED,8BAAoD;AAClD,cAAQ,KAAK7B,OAAL,CAAaR,KAArB;AACE,aAAK,WAAL;AACE,iBAAO0C,iCAAP;;AACF,aAAK,iBAAL;AACA,aAAK,kBAAL;AACE,iBAAOC,gCAAP;;AACF,aAAK,gBAAL;AACE,iBAAOC,qCAAP;;AACF,aAAK,aAAL;AACE,cAAI,CAACtC,iBAAiB,CAACuC,UAAvB,EAAmC;AACjC,kBAAM,IAAIC,KAAJ,CAAUzC,WAAV,CAAN;AACD;;AACD,iBAAOC,iBAAiB,CAACuC,UAAzB;;AACF;AACE,gBAAM,IAAIC,KAAJ,CAAUzC,WAAV,CAAN;AAdJ;AAgBD;;;;;;8BA1JkBC,iB","sourcesContent":["import type {Schema} from '../schema';\nimport type {TableBatch} from '../../category/table';\nimport type {TableBatchAggregator, TableBatchConstructor} from './table-batch-aggregator';\nimport BaseTableBatchAggregator from './base-table-batch-aggregator';\nimport RowTableBatchAggregator from './row-table-batch-aggregator';\nimport ColumnarTableBatchAggregator from './columnar-table-batch-aggregator';\n\n// TODO define interface instead\ntype TableBatchBuilderOptions = {\n shape: 'row-table' | 'array-row-table' | 'object-row-table' | 'columnar-table' | 'arrow-table';\n batchSize?: number | 'auto';\n batchDebounceMs?: number;\n limit: number;\n _limitMB: number;\n};\n\ntype GetBatchOptions = {\n bytesUsed?: number;\n [key: string]: any;\n};\n\nconst DEFAULT_OPTIONS: Required<TableBatchBuilderOptions> = {\n shape: 'array-row-table',\n batchSize: 'auto',\n batchDebounceMs: 0,\n limit: 0,\n _limitMB: 0\n};\n\nconst ERR_MESSAGE = 'TableBatchBuilder';\n\n/** Incrementally builds batches from a stream of rows */\nexport default class TableBatchBuilder {\n schema: Schema;\n options: Required<TableBatchBuilderOptions>;\n\n private aggregator: TableBatchAggregator | null = null;\n private batchCount: number = 0;\n private bytesUsed: number = 0;\n private isChunkComplete: boolean = false;\n private lastBatchEmittedMs: number = Date.now();\n private totalLength: number = 0;\n private totalBytes: number = 0;\n private rowBytes: number = 0;\n\n static ArrowBatch?: TableBatchConstructor;\n\n constructor(schema: Schema, options?: TableBatchBuilderOptions) {\n this.schema = schema;\n this.options = {...DEFAULT_OPTIONS, ...options};\n }\n\n limitReached(): boolean {\n if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {\n return true;\n }\n if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {\n return true;\n }\n return false;\n }\n\n /** @deprecated Use addArrayRow or addObjectRow */\n addRow(row: any[] | {[columnName: string]: any}): void {\n if (this.limitReached()) {\n return;\n }\n this.totalLength++;\n this.rowBytes = this.rowBytes || this._estimateRowMB(row);\n this.totalBytes += this.rowBytes;\n if (Array.isArray(row)) {\n this.addArrayRow(row);\n } else {\n this.addObjectRow(row);\n }\n }\n\n /** Add one row to the batch */\n protected addArrayRow(row: any[]) {\n if (!this.aggregator) {\n const TableBatchType = this._getTableBatchType();\n this.aggregator = new TableBatchType(this.schema, this.options);\n }\n this.aggregator.addArrayRow(row);\n }\n\n /** Add one row to the batch */\n protected addObjectRow(row: {[columnName: string]: any}): void {\n if (!this.aggregator) {\n const TableBatchType = this._getTableBatchType();\n this.aggregator = new TableBatchType(this.schema, this.options);\n }\n this.aggregator.addObjectRow(row);\n }\n\n /** Mark an incoming raw memory chunk has completed */\n chunkComplete(chunk: ArrayBuffer | string): void {\n if (chunk instanceof ArrayBuffer) {\n this.bytesUsed += chunk.byteLength;\n }\n if (typeof chunk === 'string') {\n this.bytesUsed += chunk.length;\n }\n this.isChunkComplete = true;\n }\n\n getFullBatch(options?: GetBatchOptions): TableBatch | null {\n return this._isFull() ? this._getBatch(options) : null;\n }\n\n getFinalBatch(options?: GetBatchOptions): TableBatch | null {\n return this._getBatch(options);\n }\n\n // INTERNAL\n\n _estimateRowMB(row) {\n return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;\n }\n\n private _isFull(): boolean {\n // No batch, not ready\n if (!this.aggregator || this.aggregator.rowCount() === 0) {\n return false;\n }\n\n // if batchSize === 'auto' we wait for chunk to complete\n // if batchSize === number, ensure we have enough rows\n if (this.options.batchSize === 'auto') {\n if (!this.isChunkComplete) {\n return false;\n }\n } else if (this.options.batchSize > this.aggregator.rowCount()) {\n return false;\n }\n\n // Debounce batches\n if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {\n return false;\n }\n\n // Emit batch\n this.isChunkComplete = false;\n this.lastBatchEmittedMs = Date.now();\n return true;\n }\n\n /**\n * bytesUsed can be set via chunkComplete or via getBatch*\n */\n private _getBatch(options?: GetBatchOptions): TableBatch | null {\n if (!this.aggregator) {\n return null;\n }\n\n // TODO - this can overly increment bytes used?\n if (options?.bytesUsed) {\n this.bytesUsed = options.bytesUsed;\n }\n const normalizedBatch = this.aggregator.getBatch() as TableBatch;\n normalizedBatch.count = this.batchCount;\n normalizedBatch.bytesUsed = this.bytesUsed;\n Object.assign(normalizedBatch, options);\n\n this.batchCount++;\n this.aggregator = null;\n return normalizedBatch;\n }\n\n private _getTableBatchType(): TableBatchConstructor {\n switch (this.options.shape) {\n case 'row-table':\n return BaseTableBatchAggregator;\n case 'array-row-table':\n case 'object-row-table':\n return RowTableBatchAggregator;\n case 'columnar-table':\n return ColumnarTableBatchAggregator;\n case 'arrow-table':\n if (!TableBatchBuilder.ArrowBatch) {\n throw new Error(ERR_MESSAGE);\n }\n return TableBatchBuilder.ArrowBatch;\n default:\n throw new Error(ERR_MESSAGE);\n }\n }\n}\n"],"file":"table-batch-builder.js"}
|
|
@@ -8,25 +8,59 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
8
8
|
exports.takeAsync = takeAsync;
|
|
9
9
|
exports.default = void 0;
|
|
10
10
|
|
|
11
|
+
var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
|
|
12
|
+
|
|
13
|
+
var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
|
|
14
|
+
|
|
11
15
|
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
12
16
|
|
|
13
|
-
|
|
17
|
+
var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
|
|
14
18
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
+
var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
|
|
20
|
+
|
|
21
|
+
var _inherits2 = _interopRequireDefault(require("@babel/runtime/helpers/inherits"));
|
|
22
|
+
|
|
23
|
+
var _possibleConstructorReturn2 = _interopRequireDefault(require("@babel/runtime/helpers/possibleConstructorReturn"));
|
|
24
|
+
|
|
25
|
+
var _getPrototypeOf2 = _interopRequireDefault(require("@babel/runtime/helpers/getPrototypeOf"));
|
|
26
|
+
|
|
27
|
+
var _wrapNativeSuper2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapNativeSuper"));
|
|
19
28
|
|
|
20
|
-
|
|
21
|
-
|
|
29
|
+
var _Symbol$asyncIterator;
|
|
30
|
+
|
|
31
|
+
function _createSuper(Derived) { var hasNativeReflectConstruct = _isNativeReflectConstruct(); return function _createSuperInternal() { var Super = (0, _getPrototypeOf2.default)(Derived), result; if (hasNativeReflectConstruct) { var NewTarget = (0, _getPrototypeOf2.default)(this).constructor; result = Reflect.construct(Super, arguments, NewTarget); } else { result = Super.apply(this, arguments); } return (0, _possibleConstructorReturn2.default)(this, result); }; }
|
|
32
|
+
|
|
33
|
+
function _isNativeReflectConstruct() { if (typeof Reflect === "undefined" || !Reflect.construct) return false; if (Reflect.construct.sham) return false; if (typeof Proxy === "function") return true; try { Boolean.prototype.valueOf.call(Reflect.construct(Boolean, [], function () {})); return true; } catch (e) { return false; } }
|
|
34
|
+
|
|
35
|
+
var ArrayQueue = function (_Array) {
|
|
36
|
+
(0, _inherits2.default)(ArrayQueue, _Array);
|
|
37
|
+
|
|
38
|
+
var _super = _createSuper(ArrayQueue);
|
|
39
|
+
|
|
40
|
+
function ArrayQueue() {
|
|
41
|
+
(0, _classCallCheck2.default)(this, ArrayQueue);
|
|
42
|
+
return _super.apply(this, arguments);
|
|
22
43
|
}
|
|
23
44
|
|
|
24
|
-
|
|
45
|
+
(0, _createClass2.default)(ArrayQueue, [{
|
|
46
|
+
key: "enqueue",
|
|
47
|
+
value: function enqueue(value) {
|
|
48
|
+
return this.push(value);
|
|
49
|
+
}
|
|
50
|
+
}, {
|
|
51
|
+
key: "dequeue",
|
|
52
|
+
value: function dequeue() {
|
|
53
|
+
return this.shift();
|
|
54
|
+
}
|
|
55
|
+
}]);
|
|
56
|
+
return ArrayQueue;
|
|
57
|
+
}((0, _wrapNativeSuper2.default)(Array));
|
|
25
58
|
|
|
26
59
|
_Symbol$asyncIterator = Symbol.asyncIterator;
|
|
27
60
|
|
|
28
|
-
|
|
29
|
-
|
|
61
|
+
var AsyncQueue = function () {
|
|
62
|
+
function AsyncQueue() {
|
|
63
|
+
(0, _classCallCheck2.default)(this, AsyncQueue);
|
|
30
64
|
(0, _defineProperty2.default)(this, "_values", void 0);
|
|
31
65
|
(0, _defineProperty2.default)(this, "_settlers", void 0);
|
|
32
66
|
(0, _defineProperty2.default)(this, "_closed", void 0);
|
|
@@ -35,96 +69,145 @@ class AsyncQueue {
|
|
|
35
69
|
this._closed = false;
|
|
36
70
|
}
|
|
37
71
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
[_Symbol$asyncIterator]() {
|
|
49
|
-
return this;
|
|
50
|
-
}
|
|
72
|
+
(0, _createClass2.default)(AsyncQueue, [{
|
|
73
|
+
key: "close",
|
|
74
|
+
value: function close() {
|
|
75
|
+
while (this._settlers.length > 0) {
|
|
76
|
+
this._settlers.dequeue().resolve({
|
|
77
|
+
done: true
|
|
78
|
+
});
|
|
79
|
+
}
|
|
51
80
|
|
|
52
|
-
|
|
53
|
-
if (this._closed) {
|
|
54
|
-
throw new Error('Closed');
|
|
81
|
+
this._closed = true;
|
|
55
82
|
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
83
|
+
}, {
|
|
84
|
+
key: _Symbol$asyncIterator,
|
|
85
|
+
value: function value() {
|
|
86
|
+
return this;
|
|
87
|
+
}
|
|
88
|
+
}, {
|
|
89
|
+
key: "enqueue",
|
|
90
|
+
value: function enqueue(value) {
|
|
91
|
+
if (this._closed) {
|
|
92
|
+
throw new Error('Closed');
|
|
60
93
|
}
|
|
61
94
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
95
|
+
if (this._settlers.length > 0) {
|
|
96
|
+
if (this._values.length > 0) {
|
|
97
|
+
throw new Error('Illegal internal state');
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
var settler = this._settlers.dequeue();
|
|
101
|
+
|
|
102
|
+
if (value instanceof Error) {
|
|
103
|
+
settler.reject(value);
|
|
104
|
+
} else {
|
|
105
|
+
settler.resolve({
|
|
106
|
+
value: value
|
|
107
|
+
});
|
|
108
|
+
}
|
|
66
109
|
} else {
|
|
67
|
-
|
|
68
|
-
value
|
|
69
|
-
});
|
|
110
|
+
this._values.enqueue(value);
|
|
70
111
|
}
|
|
71
|
-
} else {
|
|
72
|
-
this._values.enqueue(value);
|
|
73
112
|
}
|
|
74
|
-
}
|
|
113
|
+
}, {
|
|
114
|
+
key: "next",
|
|
115
|
+
value: function next() {
|
|
116
|
+
var _this = this;
|
|
117
|
+
|
|
118
|
+
if (this._values.length > 0) {
|
|
119
|
+
var value = this._values.dequeue();
|
|
75
120
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
121
|
+
if (value instanceof Error) {
|
|
122
|
+
return Promise.reject(value);
|
|
123
|
+
}
|
|
79
124
|
|
|
80
|
-
|
|
81
|
-
|
|
125
|
+
return Promise.resolve({
|
|
126
|
+
value: value
|
|
127
|
+
});
|
|
82
128
|
}
|
|
83
129
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
130
|
+
if (this._closed) {
|
|
131
|
+
if (this._settlers.length > 0) {
|
|
132
|
+
throw new Error('Illegal internal state');
|
|
133
|
+
}
|
|
88
134
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
135
|
+
return Promise.resolve({
|
|
136
|
+
done: true
|
|
137
|
+
});
|
|
92
138
|
}
|
|
93
139
|
|
|
94
|
-
return Promise
|
|
95
|
-
|
|
140
|
+
return new Promise(function (resolve, reject) {
|
|
141
|
+
_this._settlers.enqueue({
|
|
142
|
+
resolve: resolve,
|
|
143
|
+
reject: reject
|
|
144
|
+
});
|
|
96
145
|
});
|
|
97
146
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
resolve,
|
|
102
|
-
reject
|
|
103
|
-
});
|
|
104
|
-
});
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
}
|
|
147
|
+
}]);
|
|
148
|
+
return AsyncQueue;
|
|
149
|
+
}();
|
|
108
150
|
|
|
109
151
|
exports.default = AsyncQueue;
|
|
110
152
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
while (result.length < count) {
|
|
116
|
-
const {
|
|
117
|
-
value,
|
|
118
|
-
done
|
|
119
|
-
} = await iterator.next();
|
|
120
|
-
|
|
121
|
-
if (done) {
|
|
122
|
-
break;
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
result.push(value);
|
|
126
|
-
}
|
|
153
|
+
function takeAsync(_x) {
|
|
154
|
+
return _takeAsync.apply(this, arguments);
|
|
155
|
+
}
|
|
127
156
|
|
|
128
|
-
|
|
157
|
+
function _takeAsync() {
|
|
158
|
+
_takeAsync = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(asyncIterable) {
|
|
159
|
+
var count,
|
|
160
|
+
result,
|
|
161
|
+
iterator,
|
|
162
|
+
_yield$iterator$next,
|
|
163
|
+
value,
|
|
164
|
+
done,
|
|
165
|
+
_args = arguments;
|
|
166
|
+
|
|
167
|
+
return _regenerator.default.wrap(function _callee$(_context) {
|
|
168
|
+
while (1) {
|
|
169
|
+
switch (_context.prev = _context.next) {
|
|
170
|
+
case 0:
|
|
171
|
+
count = _args.length > 1 && _args[1] !== undefined ? _args[1] : Infinity;
|
|
172
|
+
result = [];
|
|
173
|
+
iterator = asyncIterable[Symbol.asyncIterator]();
|
|
174
|
+
|
|
175
|
+
case 3:
|
|
176
|
+
if (!(result.length < count)) {
|
|
177
|
+
_context.next = 14;
|
|
178
|
+
break;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
_context.next = 6;
|
|
182
|
+
return iterator.next();
|
|
183
|
+
|
|
184
|
+
case 6:
|
|
185
|
+
_yield$iterator$next = _context.sent;
|
|
186
|
+
value = _yield$iterator$next.value;
|
|
187
|
+
done = _yield$iterator$next.done;
|
|
188
|
+
|
|
189
|
+
if (!done) {
|
|
190
|
+
_context.next = 11;
|
|
191
|
+
break;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
return _context.abrupt("break", 14);
|
|
195
|
+
|
|
196
|
+
case 11:
|
|
197
|
+
result.push(value);
|
|
198
|
+
_context.next = 3;
|
|
199
|
+
break;
|
|
200
|
+
|
|
201
|
+
case 14:
|
|
202
|
+
return _context.abrupt("return", result);
|
|
203
|
+
|
|
204
|
+
case 15:
|
|
205
|
+
case "end":
|
|
206
|
+
return _context.stop();
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}, _callee);
|
|
210
|
+
}));
|
|
211
|
+
return _takeAsync.apply(this, arguments);
|
|
129
212
|
}
|
|
130
213
|
//# sourceMappingURL=async-queue.js.map
|