@loaders.gl/schema 3.1.0-alpha.3 → 3.1.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (250) hide show
  1. package/dist/bundle.d.ts +2 -0
  2. package/dist/bundle.d.ts.map +1 -0
  3. package/dist/bundle.js +1288 -4
  4. package/dist/category/common.d.ts +20 -0
  5. package/dist/category/common.d.ts.map +1 -0
  6. package/dist/category/common.js +10 -2
  7. package/dist/category/gis.d.ts +50 -0
  8. package/dist/category/gis.d.ts.map +1 -0
  9. package/dist/category/gis.js +2 -2
  10. package/dist/category/mesh/convert-mesh.d.ts +15 -0
  11. package/dist/category/mesh/convert-mesh.d.ts.map +1 -0
  12. package/dist/category/mesh/convert-mesh.js +37 -31
  13. package/dist/category/mesh/deduce-mesh-schema.d.ts +24 -0
  14. package/dist/category/mesh/deduce-mesh-schema.d.ts.map +1 -0
  15. package/dist/category/mesh/deduce-mesh-schema.js +57 -37
  16. package/dist/category/mesh/mesh-to-arrow-table.d.ts +1 -0
  17. package/dist/category/mesh/mesh-to-arrow-table.d.ts.map +1 -0
  18. package/dist/category/mesh/mesh-to-arrow-table.js +28 -15
  19. package/dist/category/mesh/mesh-types.d.ts +50 -0
  20. package/dist/category/mesh/mesh-types.d.ts.map +1 -0
  21. package/dist/category/mesh/mesh-types.js +2 -2
  22. package/dist/category/mesh/mesh-utils.d.ts +23 -0
  23. package/dist/category/mesh/mesh-utils.d.ts.map +1 -0
  24. package/dist/category/mesh/mesh-utils.js +48 -35
  25. package/dist/category/table/deduce-table-schema.d.ts +9 -0
  26. package/dist/category/table/deduce-table-schema.d.ts.map +1 -0
  27. package/dist/category/table/deduce-table-schema.js +94 -39
  28. package/dist/category/table/table-types.d.ts +78 -0
  29. package/dist/category/table/table-types.d.ts.map +1 -0
  30. package/dist/category/table/table-types.js +2 -2
  31. package/dist/es5/bundle.js +7 -0
  32. package/dist/es5/bundle.js.map +1 -0
  33. package/dist/es5/category/common.js +2 -0
  34. package/dist/{category → es5/category}/common.js.map +0 -0
  35. package/dist/es5/category/gis.js +2 -0
  36. package/dist/{category → es5/category}/gis.js.map +0 -0
  37. package/dist/es5/category/mesh/convert-mesh.js +35 -0
  38. package/dist/es5/category/mesh/convert-mesh.js.map +1 -0
  39. package/dist/es5/category/mesh/deduce-mesh-schema.js +54 -0
  40. package/dist/es5/category/mesh/deduce-mesh-schema.js.map +1 -0
  41. package/dist/es5/category/mesh/mesh-to-arrow-table.js +2 -0
  42. package/dist/es5/category/mesh/mesh-to-arrow-table.js.map +1 -0
  43. package/dist/es5/category/mesh/mesh-types.js +2 -0
  44. package/dist/{category → es5/category}/mesh/mesh-types.js.map +0 -0
  45. package/dist/es5/category/mesh/mesh-utils.js +47 -0
  46. package/dist/es5/category/mesh/mesh-utils.js.map +1 -0
  47. package/dist/es5/category/table/deduce-table-schema.js +58 -0
  48. package/dist/es5/category/table/deduce-table-schema.js.map +1 -0
  49. package/dist/es5/category/table/table-types.js +2 -0
  50. package/dist/{category → es5/category}/table/table-types.js.map +0 -0
  51. package/dist/es5/index.js +344 -0
  52. package/dist/es5/index.js.map +1 -0
  53. package/dist/es5/lib/arrow/arrow-like-type-utils.js +40 -0
  54. package/dist/es5/lib/arrow/arrow-like-type-utils.js.map +1 -0
  55. package/dist/es5/lib/arrow/arrow-type-utils.js +2 -0
  56. package/dist/es5/lib/arrow/arrow-type-utils.js.map +1 -0
  57. package/dist/es5/lib/arrow/get-type-info.js +33 -0
  58. package/dist/es5/lib/arrow/get-type-info.js.map +1 -0
  59. package/dist/es5/lib/batches/base-table-batch-aggregator.js +81 -0
  60. package/dist/es5/lib/batches/base-table-batch-aggregator.js.map +1 -0
  61. package/dist/es5/lib/batches/columnar-table-batch-aggregator.js +109 -0
  62. package/dist/es5/lib/batches/columnar-table-batch-aggregator.js.map +1 -0
  63. package/dist/es5/lib/batches/row-table-batch-aggregator.js +102 -0
  64. package/dist/es5/lib/batches/row-table-batch-aggregator.js.map +1 -0
  65. package/dist/es5/lib/batches/table-batch-aggregator.js +2 -0
  66. package/dist/{lib → es5/lib}/batches/table-batch-aggregator.js.map +0 -0
  67. package/dist/es5/lib/batches/table-batch-builder.js +187 -0
  68. package/dist/es5/lib/batches/table-batch-builder.js.map +1 -0
  69. package/dist/es5/lib/schema/impl/enum.js +56 -0
  70. package/dist/es5/lib/schema/impl/enum.js.map +1 -0
  71. package/dist/es5/lib/schema/impl/field.js +43 -0
  72. package/dist/es5/lib/schema/impl/field.js.map +1 -0
  73. package/dist/es5/lib/schema/impl/schema.js +103 -0
  74. package/dist/es5/lib/schema/impl/schema.js.map +1 -0
  75. package/dist/es5/lib/schema/impl/type.js +594 -0
  76. package/dist/es5/lib/schema/impl/type.js.map +1 -0
  77. package/dist/es5/lib/schema/schema.js +236 -0
  78. package/dist/es5/lib/schema/schema.js.map +1 -0
  79. package/dist/es5/lib/schema-utils/deduce-column-type.js +32 -0
  80. package/dist/es5/lib/schema-utils/deduce-column-type.js.map +1 -0
  81. package/dist/es5/lib/utils/assert.js +13 -0
  82. package/dist/es5/lib/utils/assert.js.map +1 -0
  83. package/dist/es5/lib/utils/async-queue.js +130 -0
  84. package/dist/es5/lib/utils/async-queue.js.map +1 -0
  85. package/dist/es5/lib/utils/row-utils.js +44 -0
  86. package/dist/es5/lib/utils/row-utils.js.map +1 -0
  87. package/dist/es5/types.js +2 -0
  88. package/dist/{types.js.map → es5/types.js.map} +0 -0
  89. package/dist/esm/bundle.js +5 -0
  90. package/dist/esm/bundle.js.map +1 -0
  91. package/dist/esm/category/common.js +2 -0
  92. package/dist/esm/category/common.js.map +1 -0
  93. package/dist/esm/category/gis.js +2 -0
  94. package/dist/esm/category/gis.js.map +1 -0
  95. package/dist/esm/category/mesh/convert-mesh.js +26 -0
  96. package/dist/esm/category/mesh/convert-mesh.js.map +1 -0
  97. package/dist/esm/category/mesh/deduce-mesh-schema.js +42 -0
  98. package/dist/esm/category/mesh/deduce-mesh-schema.js.map +1 -0
  99. package/dist/esm/category/mesh/mesh-to-arrow-table.js +2 -0
  100. package/dist/esm/category/mesh/mesh-to-arrow-table.js.map +1 -0
  101. package/dist/esm/category/mesh/mesh-types.js +2 -0
  102. package/dist/esm/category/mesh/mesh-types.js.map +1 -0
  103. package/dist/esm/category/mesh/mesh-utils.js +38 -0
  104. package/dist/esm/category/mesh/mesh-utils.js.map +1 -0
  105. package/dist/esm/category/table/deduce-table-schema.js +51 -0
  106. package/dist/esm/category/table/deduce-table-schema.js.map +1 -0
  107. package/dist/esm/category/table/table-types.js +2 -0
  108. package/dist/esm/category/table/table-types.js.map +1 -0
  109. package/dist/esm/index.js +13 -0
  110. package/dist/esm/index.js.map +1 -0
  111. package/dist/esm/lib/arrow/arrow-like-type-utils.js +32 -0
  112. package/dist/esm/lib/arrow/arrow-like-type-utils.js.map +1 -0
  113. package/dist/esm/lib/arrow/arrow-type-utils.js +2 -0
  114. package/dist/esm/lib/arrow/arrow-type-utils.js.map +1 -0
  115. package/dist/esm/lib/arrow/get-type-info.js +24 -0
  116. package/dist/esm/lib/arrow/get-type-info.js.map +1 -0
  117. package/dist/esm/lib/batches/base-table-batch-aggregator.js +74 -0
  118. package/dist/esm/lib/batches/base-table-batch-aggregator.js.map +1 -0
  119. package/dist/esm/lib/batches/columnar-table-batch-aggregator.js +100 -0
  120. package/dist/esm/lib/batches/columnar-table-batch-aggregator.js.map +1 -0
  121. package/dist/esm/lib/batches/row-table-batch-aggregator.js +95 -0
  122. package/dist/esm/lib/batches/row-table-batch-aggregator.js.map +1 -0
  123. package/dist/esm/lib/batches/table-batch-aggregator.js +2 -0
  124. package/dist/esm/lib/batches/table-batch-aggregator.js.map +1 -0
  125. package/dist/esm/lib/batches/table-batch-builder.js +182 -0
  126. package/dist/esm/lib/batches/table-batch-builder.js.map +1 -0
  127. package/dist/esm/lib/schema/impl/enum.js +49 -0
  128. package/dist/esm/lib/schema/impl/enum.js.map +1 -0
  129. package/dist/esm/lib/schema/impl/field.js +35 -0
  130. package/dist/esm/lib/schema/impl/field.js.map +1 -0
  131. package/dist/esm/lib/schema/impl/schema.js +92 -0
  132. package/dist/esm/lib/schema/impl/schema.js.map +1 -0
  133. package/dist/esm/lib/schema/impl/type.js +498 -0
  134. package/dist/esm/lib/schema/impl/type.js.map +1 -0
  135. package/dist/{lib/schema/index.js → esm/lib/schema/schema.js} +1 -1
  136. package/dist/esm/lib/schema/schema.js.map +1 -0
  137. package/dist/esm/lib/schema-utils/deduce-column-type.js +23 -0
  138. package/dist/esm/lib/schema-utils/deduce-column-type.js.map +1 -0
  139. package/dist/esm/lib/utils/assert.js +6 -0
  140. package/dist/esm/lib/utils/assert.js.map +1 -0
  141. package/dist/esm/lib/utils/async-queue.js +119 -0
  142. package/dist/esm/lib/utils/async-queue.js.map +1 -0
  143. package/dist/esm/lib/utils/row-utils.js +35 -0
  144. package/dist/esm/lib/utils/row-utils.js.map +1 -0
  145. package/dist/esm/types.js +2 -0
  146. package/dist/esm/types.js.map +1 -0
  147. package/dist/index.d.ts +23 -0
  148. package/dist/index.d.ts.map +1 -0
  149. package/dist/index.js +75 -13
  150. package/dist/lib/arrow/arrow-like-type-utils.d.ts +4 -0
  151. package/dist/lib/arrow/arrow-like-type-utils.d.ts.map +1 -0
  152. package/dist/lib/arrow/arrow-like-type-utils.js +26 -31
  153. package/dist/lib/arrow/arrow-type-utils.d.ts +1 -0
  154. package/dist/lib/arrow/arrow-type-utils.d.ts.map +1 -0
  155. package/dist/lib/arrow/arrow-type-utils.js +28 -20
  156. package/dist/lib/arrow/get-type-info.d.ts +14 -0
  157. package/dist/lib/arrow/get-type-info.d.ts.map +1 -0
  158. package/dist/lib/arrow/get-type-info.js +23 -19
  159. package/dist/lib/batches/base-table-batch-aggregator.d.ts +19 -0
  160. package/dist/lib/batches/base-table-batch-aggregator.d.ts.map +1 -0
  161. package/dist/lib/batches/base-table-batch-aggregator.js +52 -68
  162. package/dist/lib/batches/columnar-table-batch-aggregator.d.ts +23 -0
  163. package/dist/lib/batches/columnar-table-batch-aggregator.d.ts.map +1 -0
  164. package/dist/lib/batches/columnar-table-batch-aggregator.js +82 -92
  165. package/dist/lib/batches/row-table-batch-aggregator.d.ts +22 -0
  166. package/dist/lib/batches/row-table-batch-aggregator.d.ts.map +1 -0
  167. package/dist/lib/batches/row-table-batch-aggregator.js +72 -88
  168. package/dist/lib/batches/table-batch-aggregator.d.ts +25 -0
  169. package/dist/lib/batches/table-batch-aggregator.d.ts.map +1 -0
  170. package/dist/lib/batches/table-batch-aggregator.js +2 -2
  171. package/dist/lib/batches/table-batch-builder.d.ts +53 -0
  172. package/dist/lib/batches/table-batch-builder.d.ts.map +1 -0
  173. package/dist/lib/batches/table-batch-builder.js +137 -166
  174. package/dist/lib/schema/impl/enum.d.ts +92 -0
  175. package/dist/lib/schema/impl/enum.d.ts.map +1 -0
  176. package/dist/lib/schema/impl/enum.js +96 -48
  177. package/dist/lib/schema/impl/field.d.ts +19 -0
  178. package/dist/lib/schema/impl/field.d.ts.map +1 -0
  179. package/dist/lib/schema/impl/field.js +31 -34
  180. package/dist/lib/schema/impl/schema.d.ts +16 -0
  181. package/dist/lib/schema/impl/schema.d.ts.map +1 -0
  182. package/dist/lib/schema/impl/schema.js +72 -81
  183. package/dist/lib/schema/impl/type.d.ts +176 -0
  184. package/dist/lib/schema/impl/type.d.ts.map +1 -0
  185. package/dist/lib/schema/impl/type.js +454 -490
  186. package/dist/lib/schema/schema.d.ts +5 -0
  187. package/dist/lib/schema/schema.d.ts.map +1 -0
  188. package/dist/lib/schema/schema.js +90 -0
  189. package/dist/lib/schema-utils/deduce-column-type.d.ts +3 -0
  190. package/dist/lib/schema-utils/deduce-column-type.d.ts.map +1 -0
  191. package/dist/lib/schema-utils/deduce-column-type.js +89 -20
  192. package/dist/lib/utils/assert.d.ts +2 -0
  193. package/dist/lib/utils/assert.d.ts.map +1 -0
  194. package/dist/lib/utils/assert.js +11 -5
  195. package/dist/lib/utils/async-queue.d.ts +18 -0
  196. package/dist/lib/utils/async-queue.d.ts.map +1 -0
  197. package/dist/lib/utils/async-queue.js +82 -109
  198. package/dist/lib/utils/row-utils.d.ts +9 -0
  199. package/dist/lib/utils/row-utils.d.ts.map +1 -0
  200. package/dist/lib/utils/row-utils.js +31 -33
  201. package/dist/types.d.ts +8 -0
  202. package/dist/types.d.ts.map +1 -0
  203. package/dist/types.js +2 -2
  204. package/package.json +7 -9
  205. package/src/category/common.ts +2 -1
  206. package/src/category/mesh/convert-mesh.ts +6 -6
  207. package/src/category/mesh/deduce-mesh-schema.ts +1 -1
  208. package/src/category/mesh/mesh-to-arrow-table.ts +4 -2
  209. package/src/category/mesh/mesh-types.ts +1 -1
  210. package/src/category/table/deduce-table-schema.ts +1 -1
  211. package/src/category/table/table-types.ts +5 -2
  212. package/src/index.ts +1 -1
  213. package/src/lib/arrow/arrow-like-type-utils.ts +11 -1
  214. package/src/lib/arrow/arrow-type-utils.ts +3 -1
  215. package/src/lib/arrow/get-type-info.ts +1 -1
  216. package/src/lib/batches/base-table-batch-aggregator.ts +1 -1
  217. package/src/lib/batches/columnar-table-batch-aggregator.ts +1 -1
  218. package/src/lib/batches/row-table-batch-aggregator.ts +1 -1
  219. package/src/lib/batches/table-batch-aggregator.ts +1 -1
  220. package/src/lib/batches/table-batch-builder.ts +2 -2
  221. package/src/lib/schema/impl/schema.ts +2 -2
  222. package/src/lib/schema/impl/type.ts +2 -2
  223. package/src/lib/schema/{index.ts → schema.ts} +0 -0
  224. package/src/lib/schema-utils/deduce-column-type.ts +6 -2
  225. package/src/lib/utils/async-queue.ts +7 -4
  226. package/dist/bundle.js.map +0 -1
  227. package/dist/category/mesh/convert-mesh.js.map +0 -1
  228. package/dist/category/mesh/deduce-mesh-schema.js.map +0 -1
  229. package/dist/category/mesh/mesh-to-arrow-table.js.map +0 -1
  230. package/dist/category/mesh/mesh-utils.js.map +0 -1
  231. package/dist/category/table/deduce-table-schema.js.map +0 -1
  232. package/dist/dist.min.js +0 -2
  233. package/dist/dist.min.js.map +0 -1
  234. package/dist/index.js.map +0 -1
  235. package/dist/lib/arrow/arrow-like-type-utils.js.map +0 -1
  236. package/dist/lib/arrow/arrow-type-utils.js.map +0 -1
  237. package/dist/lib/arrow/get-type-info.js.map +0 -1
  238. package/dist/lib/batches/base-table-batch-aggregator.js.map +0 -1
  239. package/dist/lib/batches/columnar-table-batch-aggregator.js.map +0 -1
  240. package/dist/lib/batches/row-table-batch-aggregator.js.map +0 -1
  241. package/dist/lib/batches/table-batch-builder.js.map +0 -1
  242. package/dist/lib/schema/impl/enum.js.map +0 -1
  243. package/dist/lib/schema/impl/field.js.map +0 -1
  244. package/dist/lib/schema/impl/schema.js.map +0 -1
  245. package/dist/lib/schema/impl/type.js.map +0 -1
  246. package/dist/lib/schema/index.js.map +0 -1
  247. package/dist/lib/schema-utils/deduce-column-type.js.map +0 -1
  248. package/dist/lib/utils/assert.js.map +0 -1
  249. package/dist/lib/utils/async-queue.js.map +0 -1
  250. package/dist/lib/utils/row-utils.js.map +0 -1
@@ -1,74 +1,58 @@
1
- import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
2
3
  const DEFAULT_ROW_COUNT = 100;
3
- export default class RowTableBatchAggregator {
4
- constructor(schema, options) {
5
- _defineProperty(this, "schema", void 0);
6
-
7
- _defineProperty(this, "options", void 0);
8
-
9
- _defineProperty(this, "length", 0);
10
-
11
- _defineProperty(this, "rows", null);
12
-
13
- _defineProperty(this, "cursor", 0);
14
-
15
- _defineProperty(this, "_headers", []);
16
-
17
- this.options = options;
18
- this.schema = schema;
19
-
20
- if (!Array.isArray(schema)) {
21
- this._headers = [];
22
-
23
- for (const key in schema) {
24
- this._headers[schema[key].index] = schema[key].name;
25
- }
4
+ class RowTableBatchAggregator {
5
+ constructor(schema, options) {
6
+ this.length = 0;
7
+ this.rows = null;
8
+ this.cursor = 0;
9
+ this._headers = [];
10
+ this.options = options;
11
+ this.schema = schema;
12
+ // schema is an array if there're no headers
13
+ // object if there are headers
14
+ if (!Array.isArray(schema)) {
15
+ this._headers = [];
16
+ for (const key in schema) {
17
+ this._headers[schema[key].index] = schema[key].name;
18
+ }
19
+ }
26
20
  }
27
- }
28
-
29
- rowCount() {
30
- return this.length;
31
- }
32
-
33
- addArrayRow(row, cursor) {
34
- if (Number.isFinite(cursor)) {
35
- this.cursor = cursor;
21
+ rowCount() {
22
+ return this.length;
36
23
  }
37
-
38
- this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
39
- this.rows[this.length] = row;
40
- this.length++;
41
- }
42
-
43
- addObjectRow(row, cursor) {
44
- if (Number.isFinite(cursor)) {
45
- this.cursor = cursor;
24
+ addArrayRow(row, cursor) {
25
+ if (Number.isFinite(cursor)) {
26
+ this.cursor = cursor;
27
+ }
28
+ this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
29
+ this.rows[this.length] = row;
30
+ this.length++;
46
31
  }
47
-
48
- this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
49
- this.rows[this.length] = row;
50
- this.length++;
51
- }
52
-
53
- getBatch() {
54
- let rows = this.rows;
55
-
56
- if (!rows) {
57
- return null;
32
+ addObjectRow(row, cursor) {
33
+ if (Number.isFinite(cursor)) {
34
+ this.cursor = cursor;
35
+ }
36
+ this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
37
+ this.rows[this.length] = row;
38
+ this.length++;
39
+ }
40
+ getBatch() {
41
+ let rows = this.rows;
42
+ if (!rows) {
43
+ return null;
44
+ }
45
+ rows = rows.slice(0, this.length);
46
+ this.rows = null;
47
+ const batch = {
48
+ shape: this.options.shape,
49
+ batchType: 'data',
50
+ data: rows,
51
+ length: this.length,
52
+ schema: this.schema,
53
+ cursor: this.cursor
54
+ };
55
+ return batch;
58
56
  }
59
-
60
- rows = rows.slice(0, this.length);
61
- this.rows = null;
62
- const batch = {
63
- shape: this.options.shape,
64
- batchType: 'data',
65
- data: rows,
66
- length: this.length,
67
- schema: this.schema,
68
- cursor: this.cursor
69
- };
70
- return batch;
71
- }
72
-
73
57
  }
74
- //# sourceMappingURL=base-table-batch-aggregator.js.map
58
+ exports.default = RowTableBatchAggregator;
@@ -0,0 +1,23 @@
1
+ import type { Schema } from '../schema/schema';
2
+ import type { ColumnarTableBatch, ArrowTableBatch } from '../../category/table/table-types';
3
+ import { TableBatchAggregator } from './table-batch-aggregator';
4
+ declare type ColumnarTableBatchOptions = {};
5
+ export default class ColumnarTableBatchAggregator implements TableBatchAggregator {
6
+ schema: Schema;
7
+ length: number;
8
+ allocated: number;
9
+ columns: {
10
+ [columnName: string]: any[];
11
+ };
12
+ constructor(schema: Schema, options: ColumnarTableBatchOptions);
13
+ rowCount(): number;
14
+ addArrayRow(row: any[]): void;
15
+ addObjectRow(row: {
16
+ [columnName: string]: any;
17
+ }): void;
18
+ getBatch(): ColumnarTableBatch | ArrowTableBatch | null;
19
+ _reallocateColumns(): void;
20
+ _pruneColumns(): void;
21
+ }
22
+ export {};
23
+ //# sourceMappingURL=columnar-table-batch-aggregator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"columnar-table-batch-aggregator.d.ts","sourceRoot":"","sources":["../../../src/lib/batches/columnar-table-batch-aggregator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAC,kBAAkB,EAAE,eAAe,EAAC,MAAM,kCAAkC,CAAC;AAC1F,OAAO,EAAC,oBAAoB,EAAC,MAAM,0BAA0B,CAAC;AAE9D,aAAK,yBAAyB,GAAG,EAAE,CAAC;AAIpC,MAAM,CAAC,OAAO,OAAO,4BAA6B,YAAW,oBAAoB;IAC/E,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAK;IACnB,SAAS,EAAE,MAAM,CAAK;IACtB,OAAO,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,EAAE,CAAA;KAAC,CAAM;gBAEhC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,yBAAyB;IAK9D,QAAQ,IAAI,MAAM;IAIlB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE;IAWtB,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,GAAG,IAAI;IASpD,QAAQ,IAAI,kBAAkB,GAAG,eAAe,GAAG,IAAI;IA6BvD,kBAAkB;IA8BlB,aAAa;CAKd"}
@@ -1,100 +1,90 @@
1
- import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
2
3
  const DEFAULT_ROW_COUNT = 100;
3
- export default class ColumnarTableBatchAggregator {
4
- constructor(schema, options) {
5
- _defineProperty(this, "schema", void 0);
6
-
7
- _defineProperty(this, "length", 0);
8
-
9
- _defineProperty(this, "allocated", 0);
10
-
11
- _defineProperty(this, "columns", {});
12
-
13
- this.schema = schema;
14
-
15
- this._reallocateColumns();
16
- }
17
-
18
- rowCount() {
19
- return this.length;
20
- }
21
-
22
- addArrayRow(row) {
23
- this._reallocateColumns();
24
-
25
- let i = 0;
26
-
27
- for (const fieldName in this.columns) {
28
- this.columns[fieldName][this.length] = row[i++];
4
+ class ColumnarTableBatchAggregator {
5
+ constructor(schema, options) {
6
+ this.length = 0;
7
+ this.allocated = 0;
8
+ this.columns = {};
9
+ this.schema = schema;
10
+ this._reallocateColumns();
29
11
  }
30
-
31
- this.length++;
32
- }
33
-
34
- addObjectRow(row) {
35
- this._reallocateColumns();
36
-
37
- for (const fieldName in row) {
38
- this.columns[fieldName][this.length] = row[fieldName];
12
+ rowCount() {
13
+ return this.length;
39
14
  }
40
-
41
- this.length++;
42
- }
43
-
44
- getBatch() {
45
- this._pruneColumns();
46
-
47
- const columns = Array.isArray(this.schema) ? this.columns : {};
48
-
49
- if (!Array.isArray(this.schema)) {
50
- for (const fieldName in this.schema) {
51
- const field = this.schema[fieldName];
52
- columns[field.name] = this.columns[field.index];
53
- }
15
+ addArrayRow(row) {
16
+ // If user keeps pushing rows beyond batch size, reallocate
17
+ this._reallocateColumns();
18
+ let i = 0;
19
+ // TODO what if no csv header, columns not populated?
20
+ for (const fieldName in this.columns) {
21
+ this.columns[fieldName][this.length] = row[i++];
22
+ }
23
+ this.length++;
54
24
  }
55
-
56
- this.columns = {};
57
- const batch = {
58
- shape: 'columnar-table',
59
- batchType: 'data',
60
- data: columns,
61
- schema: this.schema,
62
- length: this.length
63
- };
64
- return batch;
65
- }
66
-
67
- _reallocateColumns() {
68
- if (this.length < this.allocated) {
69
- return;
25
+ addObjectRow(row) {
26
+ // If user keeps pushing rows beyond batch size, reallocate
27
+ this._reallocateColumns();
28
+ for (const fieldName in row) {
29
+ this.columns[fieldName][this.length] = row[fieldName];
30
+ }
31
+ this.length++;
70
32
  }
71
-
72
- this.allocated = this.allocated > 0 ? this.allocated *= 2 : DEFAULT_ROW_COUNT;
73
- this.columns = {};
74
-
75
- for (const fieldName in this.schema) {
76
- const field = this.schema[fieldName];
77
- const ArrayType = field.type || Float32Array;
78
- const oldColumn = this.columns[field.index];
79
-
80
- if (oldColumn && ArrayBuffer.isView(oldColumn)) {
81
- const typedArray = new ArrayType(this.allocated);
82
- typedArray.set(oldColumn);
83
- this.columns[field.index] = typedArray;
84
- } else if (oldColumn) {
85
- oldColumn.length = this.allocated;
86
- this.columns[field.index] = oldColumn;
87
- } else {
88
- this.columns[field.index] = new ArrayType(this.allocated);
89
- }
33
+ getBatch() {
34
+ this._pruneColumns();
35
+ const columns = Array.isArray(this.schema) ? this.columns : {};
36
+ // schema is an array if there're no headers
37
+ // object if there are headers
38
+ // columns should match schema format
39
+ if (!Array.isArray(this.schema)) {
40
+ for (const fieldName in this.schema) {
41
+ const field = this.schema[fieldName];
42
+ columns[field.name] = this.columns[field.index];
43
+ }
44
+ }
45
+ this.columns = {};
46
+ const batch = {
47
+ shape: 'columnar-table',
48
+ batchType: 'data',
49
+ data: columns,
50
+ schema: this.schema,
51
+ length: this.length
52
+ };
53
+ return batch;
90
54
  }
91
- }
92
-
93
- _pruneColumns() {
94
- for (const [columnName, column] of Object.entries(this.columns)) {
95
- this.columns[columnName] = column.slice(0, this.length);
55
+ // HELPERS
56
+ _reallocateColumns() {
57
+ if (this.length < this.allocated) {
58
+ return;
59
+ }
60
+ // @ts-ignore TODO
61
+ this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;
62
+ this.columns = {};
63
+ for (const fieldName in this.schema) {
64
+ const field = this.schema[fieldName];
65
+ const ArrayType = field.type || Float32Array;
66
+ const oldColumn = this.columns[field.index];
67
+ if (oldColumn && ArrayBuffer.isView(oldColumn)) {
68
+ // Copy the old data to the new array
69
+ const typedArray = new ArrayType(this.allocated);
70
+ typedArray.set(oldColumn);
71
+ this.columns[field.index] = typedArray;
72
+ }
73
+ else if (oldColumn) {
74
+ // Plain array
75
+ oldColumn.length = this.allocated;
76
+ this.columns[field.index] = oldColumn;
77
+ }
78
+ else {
79
+ // Create new
80
+ this.columns[field.index] = new ArrayType(this.allocated);
81
+ }
82
+ }
83
+ }
84
+ _pruneColumns() {
85
+ for (const [columnName, column] of Object.entries(this.columns)) {
86
+ this.columns[columnName] = column.slice(0, this.length);
87
+ }
96
88
  }
97
- }
98
-
99
89
  }
100
- //# sourceMappingURL=columnar-table-batch-aggregator.js.map
90
+ exports.default = ColumnarTableBatchAggregator;
@@ -0,0 +1,22 @@
1
+ import type { Schema } from '../schema/schema';
2
+ import type { TableBatch } from '../../category/table/table-types';
3
+ import { TableBatchAggregator, TableBatchOptions } from './table-batch-aggregator';
4
+ export default class RowTableBatchAggregator implements TableBatchAggregator {
5
+ schema: Schema;
6
+ options: TableBatchOptions;
7
+ length: number;
8
+ objectRows: {
9
+ [columnName: string]: any;
10
+ } | null;
11
+ arrayRows: any[] | null;
12
+ cursor: number;
13
+ private _headers;
14
+ constructor(schema: Schema, options: TableBatchOptions);
15
+ rowCount(): number;
16
+ addArrayRow(row: any[], cursor?: number): void;
17
+ addObjectRow(row: {
18
+ [columnName: string]: any;
19
+ }, cursor?: number): void;
20
+ getBatch(): TableBatch | null;
21
+ }
22
+ //# sourceMappingURL=row-table-batch-aggregator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"row-table-batch-aggregator.d.ts","sourceRoot":"","sources":["../../../src/lib/batches/row-table-batch-aggregator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kCAAkC,CAAC;AAGjE,OAAO,EAAC,oBAAoB,EAAE,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAIjF,MAAM,CAAC,OAAO,OAAO,uBAAwB,YAAW,oBAAoB;IAC1E,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,iBAAiB,CAAC;IAE3B,MAAM,EAAE,MAAM,CAAK;IACnB,UAAU,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,GAAG,IAAI,CAAQ;IACtD,SAAS,EAAE,GAAG,EAAE,GAAG,IAAI,CAAQ;IAC/B,MAAM,EAAE,MAAM,CAAK;IACnB,OAAO,CAAC,QAAQ,CAAgB;gBAEpB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB;IActD,QAAQ,IAAI,MAAM;IAIlB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAmB9C,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAmBrE,QAAQ,IAAI,UAAU,GAAG,IAAI;CAmB9B"}
@@ -1,95 +1,79 @@
1
- import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
2
- import { convertToArrayRow, convertToObjectRow } from '../utils/row-utils';
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ // import type {ArrayRowTableBatch, ObjectRowTableBatch} from '../../category/table.js';
4
+ const row_utils_1 = require("../utils/row-utils");
3
5
  const DEFAULT_ROW_COUNT = 100;
4
- export default class RowTableBatchAggregator {
5
- constructor(schema, options) {
6
- _defineProperty(this, "schema", void 0);
7
-
8
- _defineProperty(this, "options", void 0);
9
-
10
- _defineProperty(this, "length", 0);
11
-
12
- _defineProperty(this, "objectRows", null);
13
-
14
- _defineProperty(this, "arrayRows", null);
15
-
16
- _defineProperty(this, "cursor", 0);
17
-
18
- _defineProperty(this, "_headers", []);
19
-
20
- this.options = options;
21
- this.schema = schema;
22
-
23
- if (!Array.isArray(schema)) {
24
- this._headers = [];
25
-
26
- for (const key in schema) {
27
- this._headers[schema[key].index] = schema[key].name;
28
- }
6
+ class RowTableBatchAggregator {
7
+ constructor(schema, options) {
8
+ this.length = 0;
9
+ this.objectRows = null;
10
+ this.arrayRows = null;
11
+ this.cursor = 0;
12
+ this._headers = [];
13
+ this.options = options;
14
+ this.schema = schema;
15
+ // schema is an array if there're no headers
16
+ // object if there are headers
17
+ if (!Array.isArray(schema)) {
18
+ this._headers = [];
19
+ for (const key in schema) {
20
+ this._headers[schema[key].index] = schema[key].name;
21
+ }
22
+ }
29
23
  }
30
- }
31
-
32
- rowCount() {
33
- return this.length;
34
- }
35
-
36
- addArrayRow(row, cursor) {
37
- if (Number.isFinite(cursor)) {
38
- this.cursor = cursor;
24
+ rowCount() {
25
+ return this.length;
39
26
  }
40
-
41
- switch (this.options.shape) {
42
- case 'object-row-table':
43
- const rowObject = convertToObjectRow(row, this._headers);
44
- this.addObjectRow(rowObject, cursor);
45
- break;
46
-
47
- case 'array-row-table':
48
- this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);
49
- this.arrayRows[this.length] = row;
50
- this.length++;
51
- break;
27
+ addArrayRow(row, cursor) {
28
+ if (Number.isFinite(cursor)) {
29
+ this.cursor = cursor;
30
+ }
31
+ // eslint-disable-next-line default-case
32
+ switch (this.options.shape) {
33
+ case 'object-row-table':
34
+ const rowObject = (0, row_utils_1.convertToObjectRow)(row, this._headers);
35
+ this.addObjectRow(rowObject, cursor);
36
+ break;
37
+ case 'array-row-table':
38
+ this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);
39
+ this.arrayRows[this.length] = row;
40
+ this.length++;
41
+ break;
42
+ }
52
43
  }
53
- }
54
-
55
- addObjectRow(row, cursor) {
56
- if (Number.isFinite(cursor)) {
57
- this.cursor = cursor;
44
+ addObjectRow(row, cursor) {
45
+ if (Number.isFinite(cursor)) {
46
+ this.cursor = cursor;
47
+ }
48
+ // eslint-disable-next-line default-case
49
+ switch (this.options.shape) {
50
+ case 'array-row-table':
51
+ const rowArray = (0, row_utils_1.convertToArrayRow)(row, this._headers);
52
+ this.addArrayRow(rowArray, cursor);
53
+ break;
54
+ case 'object-row-table':
55
+ this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);
56
+ this.objectRows[this.length] = row;
57
+ this.length++;
58
+ break;
59
+ }
58
60
  }
59
-
60
- switch (this.options.shape) {
61
- case 'array-row-table':
62
- const rowArray = convertToArrayRow(row, this._headers);
63
- this.addArrayRow(rowArray, cursor);
64
- break;
65
-
66
- case 'object-row-table':
67
- this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);
68
- this.objectRows[this.length] = row;
69
- this.length++;
70
- break;
61
+ getBatch() {
62
+ let rows = this.arrayRows || this.objectRows;
63
+ if (!rows) {
64
+ return null;
65
+ }
66
+ rows = rows.slice(0, this.length);
67
+ this.arrayRows = null;
68
+ this.objectRows = null;
69
+ return {
70
+ shape: this.options.shape,
71
+ batchType: 'data',
72
+ data: rows,
73
+ length: this.length,
74
+ schema: this.schema,
75
+ cursor: this.cursor
76
+ };
71
77
  }
72
- }
73
-
74
- getBatch() {
75
- let rows = this.arrayRows || this.objectRows;
76
-
77
- if (!rows) {
78
- return null;
79
- }
80
-
81
- rows = rows.slice(0, this.length);
82
- this.arrayRows = null;
83
- this.objectRows = null;
84
- return {
85
- shape: this.options.shape,
86
- batchType: 'data',
87
- data: rows,
88
- length: this.length,
89
- schema: this.schema,
90
- cursor: this.cursor
91
- };
92
- }
93
-
94
78
  }
95
- //# sourceMappingURL=row-table-batch-aggregator.js.map
79
+ exports.default = RowTableBatchAggregator;
@@ -0,0 +1,25 @@
1
+ import type { Schema } from '../schema/schema';
2
+ import type { TableBatch } from '../../category/table/table-types';
3
+ export interface TableBatchOptions {
4
+ batchSize: number | string;
5
+ [key: string]: any;
6
+ }
7
+ export interface TableBatchConstructor {
8
+ new (schema: Schema, options: TableBatchOptions): TableBatchAggregator;
9
+ }
10
+ /**
11
+ * TableBatchBuilder delegates batch building to this interface
12
+ */
13
+ export interface TableBatchAggregator {
14
+ /** Number of rows */
15
+ rowCount(): number;
16
+ /** Add one row */
17
+ addArrayRow(row: any[]): void;
18
+ /** Add one row */
19
+ addObjectRow(row: {
20
+ [columnName: string]: any;
21
+ }): void;
22
+ /** return a batch object */
23
+ getBatch(): TableBatch | null;
24
+ }
25
+ //# sourceMappingURL=table-batch-aggregator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"table-batch-aggregator.d.ts","sourceRoot":"","sources":["../../../src/lib/batches/table-batch-aggregator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kCAAkC,CAAC;AAEjE,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;IAC3B,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,oBAAoB,CAAC;CACxE;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,qBAAqB;IACrB,QAAQ,IAAI,MAAM,CAAC;IACnB,kBAAkB;IAClB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC9B,kBAAkB;IAClB,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,GAAG,IAAI,CAAC;IACrD,4BAA4B;IAC5B,QAAQ,IAAI,UAAU,GAAG,IAAI,CAAC;CAC/B"}
@@ -1,2 +1,2 @@
1
- export {};
2
- //# sourceMappingURL=table-batch-aggregator.js.map
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
@@ -0,0 +1,53 @@
1
+ import type { Schema } from '../schema/schema';
2
+ import type { TableBatch } from '../../category/table/table-types';
3
+ import type { TableBatchConstructor } from './table-batch-aggregator';
4
+ declare type TableBatchBuilderOptions = {
5
+ shape: 'row-table' | 'array-row-table' | 'object-row-table' | 'columnar-table' | 'arrow-table';
6
+ batchSize?: number | 'auto';
7
+ batchDebounceMs?: number;
8
+ limit: number;
9
+ _limitMB: number;
10
+ };
11
+ declare type GetBatchOptions = {
12
+ bytesUsed?: number;
13
+ [key: string]: any;
14
+ };
15
+ /** Incrementally builds batches from a stream of rows */
16
+ export default class TableBatchBuilder {
17
+ schema: Schema;
18
+ options: Required<TableBatchBuilderOptions>;
19
+ private aggregator;
20
+ private batchCount;
21
+ private bytesUsed;
22
+ private isChunkComplete;
23
+ private lastBatchEmittedMs;
24
+ private totalLength;
25
+ private totalBytes;
26
+ private rowBytes;
27
+ static ArrowBatch?: TableBatchConstructor;
28
+ constructor(schema: Schema, options?: TableBatchBuilderOptions);
29
+ limitReached(): boolean;
30
+ /** @deprecated Use addArrayRow or addObjectRow */
31
+ addRow(row: any[] | {
32
+ [columnName: string]: any;
33
+ }): void;
34
+ /** Add one row to the batch */
35
+ protected addArrayRow(row: any[]): void;
36
+ /** Add one row to the batch */
37
+ protected addObjectRow(row: {
38
+ [columnName: string]: any;
39
+ }): void;
40
+ /** Mark an incoming raw memory chunk has completed */
41
+ chunkComplete(chunk: ArrayBuffer | string): void;
42
+ getFullBatch(options?: GetBatchOptions): TableBatch | null;
43
+ getFinalBatch(options?: GetBatchOptions): TableBatch | null;
44
+ _estimateRowMB(row: any[] | object): number;
45
+ private _isFull;
46
+ /**
47
+ * bytesUsed can be set via chunkComplete or via getBatch*
48
+ */
49
+ private _getBatch;
50
+ private _getTableBatchType;
51
+ }
52
+ export {};
53
+ //# sourceMappingURL=table-batch-builder.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"table-batch-builder.d.ts","sourceRoot":"","sources":["../../../src/lib/batches/table-batch-builder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kCAAkC,CAAC;AACjE,OAAO,KAAK,EAAuB,qBAAqB,EAAC,MAAM,0BAA0B,CAAC;AAM1F,aAAK,wBAAwB,GAAG;IAC9B,KAAK,EAAE,WAAW,GAAG,iBAAiB,GAAG,kBAAkB,GAAG,gBAAgB,GAAG,aAAa,CAAC;IAC/F,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,aAAK,eAAe,GAAG;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB,CAAC;AAYF,yDAAyD;AACzD,MAAM,CAAC,OAAO,OAAO,iBAAiB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,QAAQ,CAAC,wBAAwB,CAAC,CAAC;IAE5C,OAAO,CAAC,UAAU,CAAqC;IACvD,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,SAAS,CAAa;IAC9B,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,kBAAkB,CAAsB;IAChD,OAAO,CAAC,WAAW,CAAa;IAChC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,QAAQ,CAAa;IAE7B,MAAM,CAAC,UAAU,CAAC,EAAE,qBAAqB,CAAC;gBAE9B,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,wBAAwB;IAK9D,YAAY,IAAI,OAAO;IAUvB,kDAAkD;IAClD,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,GAAG,IAAI;IActD,+BAA+B;IAC/B,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE;IAQhC,+BAA+B;IAC/B,SAAS,CAAC,YAAY,CAAC,GAAG,EAAE;QAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,GAAG,IAAI;IAQ9D,sDAAsD;IACtD,aAAa,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,GAAG,IAAI;IAUhD,YAAY,CAAC,OAAO,CAAC,EAAE,eAAe,GAAG,UAAU,GAAG,IAAI;IAI1D,aAAa,CAAC,OAAO,CAAC,EAAE,eAAe,GAAG,UAAU,GAAG,IAAI;IAM3D,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,MAAM,GAAG,MAAM;IAI3C,OAAO,CAAC,OAAO;IA2Bf;;OAEG;IACH,OAAO,CAAC,SAAS;IAmBjB,OAAO,CAAC,kBAAkB;CAkB3B"}