@loaders.gl/schema 3.1.0-alpha.2 → 3.1.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (248) hide show
  1. package/dist/bundle.d.ts +2 -0
  2. package/dist/bundle.d.ts.map +1 -0
  3. package/dist/bundle.js +16157 -5
  4. package/dist/category/common.d.ts +19 -0
  5. package/dist/category/common.d.ts.map +1 -0
  6. package/dist/category/common.js +10 -2
  7. package/dist/category/gis.d.ts +50 -0
  8. package/dist/category/gis.d.ts.map +1 -0
  9. package/dist/category/gis.js +2 -2
  10. package/dist/category/mesh/convert-mesh.d.ts +15 -0
  11. package/dist/category/mesh/convert-mesh.d.ts.map +1 -0
  12. package/dist/category/mesh/convert-mesh.js +38 -31
  13. package/dist/category/mesh/deduce-mesh-schema.d.ts +24 -0
  14. package/dist/category/mesh/deduce-mesh-schema.d.ts.map +1 -0
  15. package/dist/category/mesh/deduce-mesh-schema.js +57 -37
  16. package/dist/category/mesh/mesh-to-arrow-table.d.ts +11 -0
  17. package/dist/category/mesh/mesh-to-arrow-table.d.ts.map +1 -0
  18. package/dist/category/mesh/mesh-to-arrow-table.js +33 -30
  19. package/dist/category/mesh/mesh-types.d.ts +50 -0
  20. package/dist/category/mesh/mesh-types.d.ts.map +1 -0
  21. package/dist/category/mesh/mesh-types.js +2 -2
  22. package/dist/category/mesh/mesh-utils.d.ts +23 -0
  23. package/dist/category/mesh/mesh-utils.d.ts.map +1 -0
  24. package/dist/category/mesh/mesh-utils.js +48 -35
  25. package/dist/category/table/deduce-table-schema.d.ts +9 -0
  26. package/dist/category/table/deduce-table-schema.d.ts.map +1 -0
  27. package/dist/category/table/deduce-table-schema.js +94 -39
  28. package/dist/category/table/table-types.d.ts +76 -0
  29. package/dist/category/table/table-types.d.ts.map +1 -0
  30. package/dist/category/table/table-types.js +2 -2
  31. package/dist/es5/bundle.js +7 -0
  32. package/dist/es5/bundle.js.map +1 -0
  33. package/dist/es5/category/common.js +2 -0
  34. package/dist/{category → es5/category}/common.js.map +0 -0
  35. package/dist/es5/category/gis.js +2 -0
  36. package/dist/{category → es5/category}/gis.js.map +0 -0
  37. package/dist/es5/category/mesh/convert-mesh.js +43 -0
  38. package/dist/{category → es5/category}/mesh/convert-mesh.js.map +1 -1
  39. package/dist/es5/category/mesh/deduce-mesh-schema.js +54 -0
  40. package/dist/es5/category/mesh/deduce-mesh-schema.js.map +1 -0
  41. package/dist/es5/category/mesh/mesh-to-arrow-table.js +41 -0
  42. package/dist/es5/category/mesh/mesh-to-arrow-table.js.map +1 -0
  43. package/dist/es5/category/mesh/mesh-types.js +2 -0
  44. package/dist/{category → es5/category}/mesh/mesh-types.js.map +0 -0
  45. package/dist/es5/category/mesh/mesh-utils.js +47 -0
  46. package/dist/es5/category/mesh/mesh-utils.js.map +1 -0
  47. package/dist/es5/category/table/deduce-table-schema.js +58 -0
  48. package/dist/es5/category/table/deduce-table-schema.js.map +1 -0
  49. package/dist/es5/category/table/table-types.js +2 -0
  50. package/dist/{category → es5/category}/table/table-types.js.map +0 -0
  51. package/dist/es5/index.js +344 -0
  52. package/dist/es5/index.js.map +1 -0
  53. package/dist/es5/lib/arrow/arrow-like-type-utils.js +40 -0
  54. package/dist/es5/lib/arrow/arrow-like-type-utils.js.map +1 -0
  55. package/dist/es5/lib/arrow/arrow-type-utils.js +72 -0
  56. package/dist/es5/lib/arrow/arrow-type-utils.js.map +1 -0
  57. package/dist/es5/lib/arrow/get-type-info.js +33 -0
  58. package/dist/es5/lib/arrow/get-type-info.js.map +1 -0
  59. package/dist/es5/lib/batches/base-table-batch-aggregator.js +81 -0
  60. package/dist/es5/lib/batches/base-table-batch-aggregator.js.map +1 -0
  61. package/dist/es5/lib/batches/columnar-table-batch-aggregator.js +109 -0
  62. package/dist/es5/lib/batches/columnar-table-batch-aggregator.js.map +1 -0
  63. package/dist/es5/lib/batches/row-table-batch-aggregator.js +102 -0
  64. package/dist/es5/lib/batches/row-table-batch-aggregator.js.map +1 -0
  65. package/dist/es5/lib/batches/table-batch-aggregator.js +2 -0
  66. package/dist/{lib → es5/lib}/batches/table-batch-aggregator.js.map +0 -0
  67. package/dist/es5/lib/batches/table-batch-builder.js +187 -0
  68. package/dist/es5/lib/batches/table-batch-builder.js.map +1 -0
  69. package/dist/es5/lib/schema/impl/enum.js +56 -0
  70. package/dist/es5/lib/schema/impl/enum.js.map +1 -0
  71. package/dist/es5/lib/schema/impl/field.js +43 -0
  72. package/dist/es5/lib/schema/impl/field.js.map +1 -0
  73. package/dist/es5/lib/schema/impl/schema.js +103 -0
  74. package/dist/es5/lib/schema/impl/schema.js.map +1 -0
  75. package/dist/es5/lib/schema/impl/type.js +594 -0
  76. package/dist/es5/lib/schema/impl/type.js.map +1 -0
  77. package/dist/es5/lib/schema/schema.js +236 -0
  78. package/dist/es5/lib/schema/schema.js.map +1 -0
  79. package/dist/es5/lib/schema-utils/deduce-column-type.js +32 -0
  80. package/dist/es5/lib/schema-utils/deduce-column-type.js.map +1 -0
  81. package/dist/es5/lib/utils/assert.js +13 -0
  82. package/dist/es5/lib/utils/assert.js.map +1 -0
  83. package/dist/es5/lib/utils/async-queue.js +130 -0
  84. package/dist/es5/lib/utils/async-queue.js.map +1 -0
  85. package/dist/es5/lib/utils/row-utils.js +44 -0
  86. package/dist/es5/lib/utils/row-utils.js.map +1 -0
  87. package/dist/es5/types.js +2 -0
  88. package/dist/{types.js.map → es5/types.js.map} +0 -0
  89. package/dist/esm/bundle.js +5 -0
  90. package/dist/esm/bundle.js.map +1 -0
  91. package/dist/esm/category/common.js +2 -0
  92. package/dist/esm/category/common.js.map +1 -0
  93. package/dist/esm/category/gis.js +2 -0
  94. package/dist/esm/category/gis.js.map +1 -0
  95. package/dist/esm/category/mesh/convert-mesh.js +33 -0
  96. package/dist/esm/category/mesh/convert-mesh.js.map +1 -0
  97. package/dist/esm/category/mesh/deduce-mesh-schema.js +42 -0
  98. package/dist/esm/category/mesh/deduce-mesh-schema.js.map +1 -0
  99. package/dist/esm/category/mesh/mesh-to-arrow-table.js +31 -0
  100. package/dist/esm/category/mesh/mesh-to-arrow-table.js.map +1 -0
  101. package/dist/esm/category/mesh/mesh-types.js +2 -0
  102. package/dist/esm/category/mesh/mesh-types.js.map +1 -0
  103. package/dist/esm/category/mesh/mesh-utils.js +38 -0
  104. package/dist/esm/category/mesh/mesh-utils.js.map +1 -0
  105. package/dist/esm/category/table/deduce-table-schema.js +51 -0
  106. package/dist/esm/category/table/deduce-table-schema.js.map +1 -0
  107. package/dist/esm/category/table/table-types.js +2 -0
  108. package/dist/esm/category/table/table-types.js.map +1 -0
  109. package/dist/esm/index.js +13 -0
  110. package/dist/esm/index.js.map +1 -0
  111. package/dist/esm/lib/arrow/arrow-like-type-utils.js +32 -0
  112. package/dist/esm/lib/arrow/arrow-like-type-utils.js.map +1 -0
  113. package/dist/esm/lib/arrow/arrow-type-utils.js +62 -0
  114. package/dist/esm/lib/arrow/arrow-type-utils.js.map +1 -0
  115. package/dist/esm/lib/arrow/get-type-info.js +24 -0
  116. package/dist/esm/lib/arrow/get-type-info.js.map +1 -0
  117. package/dist/esm/lib/batches/base-table-batch-aggregator.js +74 -0
  118. package/dist/esm/lib/batches/base-table-batch-aggregator.js.map +1 -0
  119. package/dist/esm/lib/batches/columnar-table-batch-aggregator.js +100 -0
  120. package/dist/esm/lib/batches/columnar-table-batch-aggregator.js.map +1 -0
  121. package/dist/esm/lib/batches/row-table-batch-aggregator.js +95 -0
  122. package/dist/esm/lib/batches/row-table-batch-aggregator.js.map +1 -0
  123. package/dist/esm/lib/batches/table-batch-aggregator.js +2 -0
  124. package/dist/esm/lib/batches/table-batch-aggregator.js.map +1 -0
  125. package/dist/esm/lib/batches/table-batch-builder.js +182 -0
  126. package/dist/esm/lib/batches/table-batch-builder.js.map +1 -0
  127. package/dist/esm/lib/schema/impl/enum.js +49 -0
  128. package/dist/esm/lib/schema/impl/enum.js.map +1 -0
  129. package/dist/esm/lib/schema/impl/field.js +35 -0
  130. package/dist/esm/lib/schema/impl/field.js.map +1 -0
  131. package/dist/esm/lib/schema/impl/schema.js +92 -0
  132. package/dist/esm/lib/schema/impl/schema.js.map +1 -0
  133. package/dist/esm/lib/schema/impl/type.js +498 -0
  134. package/dist/esm/lib/schema/impl/type.js.map +1 -0
  135. package/dist/{lib/schema/index.js → esm/lib/schema/schema.js} +1 -1
  136. package/dist/esm/lib/schema/schema.js.map +1 -0
  137. package/dist/esm/lib/schema-utils/deduce-column-type.js +23 -0
  138. package/dist/esm/lib/schema-utils/deduce-column-type.js.map +1 -0
  139. package/dist/esm/lib/utils/assert.js +6 -0
  140. package/dist/esm/lib/utils/assert.js.map +1 -0
  141. package/dist/esm/lib/utils/async-queue.js +119 -0
  142. package/dist/esm/lib/utils/async-queue.js.map +1 -0
  143. package/dist/esm/lib/utils/row-utils.js +35 -0
  144. package/dist/esm/lib/utils/row-utils.js.map +1 -0
  145. package/dist/esm/types.js +2 -0
  146. package/dist/esm/types.js.map +1 -0
  147. package/dist/index.d.ts +23 -0
  148. package/dist/index.d.ts.map +1 -0
  149. package/dist/index.js +75 -13
  150. package/dist/lib/arrow/arrow-like-type-utils.d.ts +4 -0
  151. package/dist/lib/arrow/arrow-like-type-utils.d.ts.map +1 -0
  152. package/dist/lib/arrow/arrow-like-type-utils.js +26 -31
  153. package/dist/lib/arrow/arrow-type-utils.d.ts +6 -0
  154. package/dist/lib/arrow/arrow-type-utils.d.ts.map +1 -0
  155. package/dist/lib/arrow/arrow-type-utils.js +48 -60
  156. package/dist/lib/arrow/get-type-info.d.ts +14 -0
  157. package/dist/lib/arrow/get-type-info.d.ts.map +1 -0
  158. package/dist/lib/arrow/get-type-info.js +23 -19
  159. package/dist/lib/batches/base-table-batch-aggregator.d.ts +19 -0
  160. package/dist/lib/batches/base-table-batch-aggregator.d.ts.map +1 -0
  161. package/dist/lib/batches/base-table-batch-aggregator.js +52 -68
  162. package/dist/lib/batches/columnar-table-batch-aggregator.d.ts +23 -0
  163. package/dist/lib/batches/columnar-table-batch-aggregator.d.ts.map +1 -0
  164. package/dist/lib/batches/columnar-table-batch-aggregator.js +82 -92
  165. package/dist/lib/batches/row-table-batch-aggregator.d.ts +22 -0
  166. package/dist/lib/batches/row-table-batch-aggregator.d.ts.map +1 -0
  167. package/dist/lib/batches/row-table-batch-aggregator.js +72 -88
  168. package/dist/lib/batches/table-batch-aggregator.d.ts +25 -0
  169. package/dist/lib/batches/table-batch-aggregator.d.ts.map +1 -0
  170. package/dist/lib/batches/table-batch-aggregator.js +2 -2
  171. package/dist/lib/batches/table-batch-builder.d.ts +53 -0
  172. package/dist/lib/batches/table-batch-builder.d.ts.map +1 -0
  173. package/dist/lib/batches/table-batch-builder.js +137 -166
  174. package/dist/lib/schema/impl/enum.d.ts +92 -0
  175. package/dist/lib/schema/impl/enum.d.ts.map +1 -0
  176. package/dist/lib/schema/impl/enum.js +96 -48
  177. package/dist/lib/schema/impl/field.d.ts +19 -0
  178. package/dist/lib/schema/impl/field.d.ts.map +1 -0
  179. package/dist/lib/schema/impl/field.js +31 -34
  180. package/dist/lib/schema/impl/schema.d.ts +16 -0
  181. package/dist/lib/schema/impl/schema.d.ts.map +1 -0
  182. package/dist/lib/schema/impl/schema.js +72 -81
  183. package/dist/lib/schema/impl/type.d.ts +176 -0
  184. package/dist/lib/schema/impl/type.d.ts.map +1 -0
  185. package/dist/lib/schema/impl/type.js +454 -490
  186. package/dist/lib/schema/schema.d.ts +5 -0
  187. package/dist/lib/schema/schema.d.ts.map +1 -0
  188. package/dist/lib/schema/schema.js +90 -0
  189. package/dist/lib/schema-utils/deduce-column-type.d.ts +3 -0
  190. package/dist/lib/schema-utils/deduce-column-type.d.ts.map +1 -0
  191. package/dist/lib/schema-utils/deduce-column-type.js +89 -20
  192. package/dist/lib/utils/assert.d.ts +2 -0
  193. package/dist/lib/utils/assert.d.ts.map +1 -0
  194. package/dist/lib/utils/assert.js +11 -5
  195. package/dist/lib/utils/async-queue.d.ts +18 -0
  196. package/dist/lib/utils/async-queue.d.ts.map +1 -0
  197. package/dist/lib/utils/async-queue.js +82 -109
  198. package/dist/lib/utils/row-utils.d.ts +9 -0
  199. package/dist/lib/utils/row-utils.d.ts.map +1 -0
  200. package/dist/lib/utils/row-utils.js +31 -33
  201. package/dist/types.d.ts +8 -0
  202. package/dist/types.d.ts.map +1 -0
  203. package/dist/types.js +2 -2
  204. package/package.json +7 -8
  205. package/src/bundle.ts +2 -3
  206. package/src/category/mesh/deduce-mesh-schema.ts +1 -1
  207. package/src/category/mesh/mesh-to-arrow-table.ts +1 -1
  208. package/src/category/mesh/mesh-types.ts +1 -1
  209. package/src/category/table/deduce-table-schema.ts +1 -1
  210. package/src/category/table/table-types.ts +2 -2
  211. package/src/index.ts +1 -1
  212. package/src/lib/arrow/arrow-like-type-utils.ts +11 -1
  213. package/src/lib/arrow/arrow-type-utils.ts +1 -1
  214. package/src/lib/arrow/get-type-info.ts +1 -1
  215. package/src/lib/batches/base-table-batch-aggregator.ts +1 -1
  216. package/src/lib/batches/columnar-table-batch-aggregator.ts +1 -1
  217. package/src/lib/batches/row-table-batch-aggregator.ts +1 -1
  218. package/src/lib/batches/table-batch-aggregator.ts +1 -1
  219. package/src/lib/batches/table-batch-builder.ts +2 -2
  220. package/src/lib/schema/impl/schema.ts +2 -2
  221. package/src/lib/schema/impl/type.ts +2 -2
  222. package/src/lib/schema/{index.ts → schema.ts} +0 -0
  223. package/src/lib/schema-utils/deduce-column-type.ts +6 -2
  224. package/src/lib/utils/async-queue.ts +7 -4
  225. package/dist/bundle.js.map +0 -1
  226. package/dist/category/mesh/deduce-mesh-schema.js.map +0 -1
  227. package/dist/category/mesh/mesh-to-arrow-table.js.map +0 -1
  228. package/dist/category/mesh/mesh-utils.js.map +0 -1
  229. package/dist/category/table/deduce-table-schema.js.map +0 -1
  230. package/dist/dist.min.js +0 -2
  231. package/dist/dist.min.js.map +0 -1
  232. package/dist/index.js.map +0 -1
  233. package/dist/lib/arrow/arrow-like-type-utils.js.map +0 -1
  234. package/dist/lib/arrow/arrow-type-utils.js.map +0 -1
  235. package/dist/lib/arrow/get-type-info.js.map +0 -1
  236. package/dist/lib/batches/base-table-batch-aggregator.js.map +0 -1
  237. package/dist/lib/batches/columnar-table-batch-aggregator.js.map +0 -1
  238. package/dist/lib/batches/row-table-batch-aggregator.js.map +0 -1
  239. package/dist/lib/batches/table-batch-builder.js.map +0 -1
  240. package/dist/lib/schema/impl/enum.js.map +0 -1
  241. package/dist/lib/schema/impl/field.js.map +0 -1
  242. package/dist/lib/schema/impl/schema.js.map +0 -1
  243. package/dist/lib/schema/impl/type.js.map +0 -1
  244. package/dist/lib/schema/index.js.map +0 -1
  245. package/dist/lib/schema-utils/deduce-column-type.js.map +0 -1
  246. package/dist/lib/utils/assert.js.map +0 -1
  247. package/dist/lib/utils/async-queue.js.map +0 -1
  248. package/dist/lib/utils/row-utils.js.map +0 -1
@@ -1,182 +1,153 @@
1
- import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
2
- import BaseTableBatchAggregator from './base-table-batch-aggregator';
3
- import RowTableBatchAggregator from './row-table-batch-aggregator';
4
- import ColumnarTableBatchAggregator from './columnar-table-batch-aggregator';
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ const base_table_batch_aggregator_1 = __importDefault(require("./base-table-batch-aggregator"));
7
+ const row_table_batch_aggregator_1 = __importDefault(require("./row-table-batch-aggregator"));
8
+ const columnar_table_batch_aggregator_1 = __importDefault(require("./columnar-table-batch-aggregator"));
5
9
  const DEFAULT_OPTIONS = {
6
- shape: 'array-row-table',
7
- batchSize: 'auto',
8
- batchDebounceMs: 0,
9
- limit: 0,
10
- _limitMB: 0
10
+ shape: 'array-row-table',
11
+ batchSize: 'auto',
12
+ batchDebounceMs: 0,
13
+ limit: 0,
14
+ _limitMB: 0
11
15
  };
12
16
  const ERR_MESSAGE = 'TableBatchBuilder';
13
- export default class TableBatchBuilder {
14
- constructor(schema, options) {
15
- _defineProperty(this, "schema", void 0);
16
-
17
- _defineProperty(this, "options", void 0);
18
-
19
- _defineProperty(this, "aggregator", null);
20
-
21
- _defineProperty(this, "batchCount", 0);
22
-
23
- _defineProperty(this, "bytesUsed", 0);
24
-
25
- _defineProperty(this, "isChunkComplete", false);
26
-
27
- _defineProperty(this, "lastBatchEmittedMs", Date.now());
28
-
29
- _defineProperty(this, "totalLength", 0);
30
-
31
- _defineProperty(this, "totalBytes", 0);
32
-
33
- _defineProperty(this, "rowBytes", 0);
34
-
35
- this.schema = schema;
36
- this.options = { ...DEFAULT_OPTIONS,
37
- ...options
38
- };
39
- }
40
-
41
- limitReached() {
42
- var _this$options, _this$options2;
43
-
44
- if (Boolean((_this$options = this.options) === null || _this$options === void 0 ? void 0 : _this$options.limit) && this.totalLength >= this.options.limit) {
45
- return true;
46
- }
47
-
48
- if (Boolean((_this$options2 = this.options) === null || _this$options2 === void 0 ? void 0 : _this$options2._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
49
- return true;
50
- }
51
-
52
- return false;
53
- }
54
-
55
- addRow(row) {
56
- if (this.limitReached()) {
57
- return;
17
+ /** Incrementally builds batches from a stream of rows */
18
+ class TableBatchBuilder {
19
+ constructor(schema, options) {
20
+ this.aggregator = null;
21
+ this.batchCount = 0;
22
+ this.bytesUsed = 0;
23
+ this.isChunkComplete = false;
24
+ this.lastBatchEmittedMs = Date.now();
25
+ this.totalLength = 0;
26
+ this.totalBytes = 0;
27
+ this.rowBytes = 0;
28
+ this.schema = schema;
29
+ this.options = { ...DEFAULT_OPTIONS, ...options };
58
30
  }
59
-
60
- this.totalLength++;
61
- this.rowBytes = this.rowBytes || this._estimateRowMB(row);
62
- this.totalBytes += this.rowBytes;
63
-
64
- if (Array.isArray(row)) {
65
- this.addArrayRow(row);
66
- } else {
67
- this.addObjectRow(row);
31
+ limitReached() {
32
+ if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {
33
+ return true;
34
+ }
35
+ if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
36
+ return true;
37
+ }
38
+ return false;
68
39
  }
69
- }
70
-
71
- addArrayRow(row) {
72
- if (!this.aggregator) {
73
- const TableBatchType = this._getTableBatchType();
74
-
75
- this.aggregator = new TableBatchType(this.schema, this.options);
40
+ /** @deprecated Use addArrayRow or addObjectRow */
41
+ addRow(row) {
42
+ if (this.limitReached()) {
43
+ return;
44
+ }
45
+ this.totalLength++;
46
+ this.rowBytes = this.rowBytes || this._estimateRowMB(row);
47
+ this.totalBytes += this.rowBytes;
48
+ if (Array.isArray(row)) {
49
+ this.addArrayRow(row);
50
+ }
51
+ else {
52
+ this.addObjectRow(row);
53
+ }
76
54
  }
77
-
78
- this.aggregator.addArrayRow(row);
79
- }
80
-
81
- addObjectRow(row) {
82
- if (!this.aggregator) {
83
- const TableBatchType = this._getTableBatchType();
84
-
85
- this.aggregator = new TableBatchType(this.schema, this.options);
55
+ /** Add one row to the batch */
56
+ addArrayRow(row) {
57
+ if (!this.aggregator) {
58
+ const TableBatchType = this._getTableBatchType();
59
+ this.aggregator = new TableBatchType(this.schema, this.options);
60
+ }
61
+ this.aggregator.addArrayRow(row);
86
62
  }
87
-
88
- this.aggregator.addObjectRow(row);
89
- }
90
-
91
- chunkComplete(chunk) {
92
- if (chunk instanceof ArrayBuffer) {
93
- this.bytesUsed += chunk.byteLength;
63
+ /** Add one row to the batch */
64
+ addObjectRow(row) {
65
+ if (!this.aggregator) {
66
+ const TableBatchType = this._getTableBatchType();
67
+ this.aggregator = new TableBatchType(this.schema, this.options);
68
+ }
69
+ this.aggregator.addObjectRow(row);
94
70
  }
95
-
96
- if (typeof chunk === 'string') {
97
- this.bytesUsed += chunk.length;
71
+ /** Mark an incoming raw memory chunk has completed */
72
+ chunkComplete(chunk) {
73
+ if (chunk instanceof ArrayBuffer) {
74
+ this.bytesUsed += chunk.byteLength;
75
+ }
76
+ if (typeof chunk === 'string') {
77
+ this.bytesUsed += chunk.length;
78
+ }
79
+ this.isChunkComplete = true;
98
80
  }
99
-
100
- this.isChunkComplete = true;
101
- }
102
-
103
- getFullBatch(options) {
104
- return this._isFull() ? this._getBatch(options) : null;
105
- }
106
-
107
- getFinalBatch(options) {
108
- return this._getBatch(options);
109
- }
110
-
111
- _estimateRowMB(row) {
112
- return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
113
- }
114
-
115
- _isFull() {
116
- if (!this.aggregator || this.aggregator.rowCount() === 0) {
117
- return false;
81
+ getFullBatch(options) {
82
+ return this._isFull() ? this._getBatch(options) : null;
118
83
  }
119
-
120
- if (this.options.batchSize === 'auto') {
121
- if (!this.isChunkComplete) {
122
- return false;
123
- }
124
- } else if (this.options.batchSize > this.aggregator.rowCount()) {
125
- return false;
84
+ getFinalBatch(options) {
85
+ return this._getBatch(options);
126
86
  }
127
-
128
- if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
129
- return false;
87
+ // INTERNAL
88
+ _estimateRowMB(row) {
89
+ return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
130
90
  }
131
-
132
- this.isChunkComplete = false;
133
- this.lastBatchEmittedMs = Date.now();
134
- return true;
135
- }
136
-
137
- _getBatch(options) {
138
- if (!this.aggregator) {
139
- return null;
91
+ _isFull() {
92
+ // No batch, not ready
93
+ if (!this.aggregator || this.aggregator.rowCount() === 0) {
94
+ return false;
95
+ }
96
+ // if batchSize === 'auto' we wait for chunk to complete
97
+ // if batchSize === number, ensure we have enough rows
98
+ if (this.options.batchSize === 'auto') {
99
+ if (!this.isChunkComplete) {
100
+ return false;
101
+ }
102
+ }
103
+ else if (this.options.batchSize > this.aggregator.rowCount()) {
104
+ return false;
105
+ }
106
+ // Debounce batches
107
+ if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
108
+ return false;
109
+ }
110
+ // Emit batch
111
+ this.isChunkComplete = false;
112
+ this.lastBatchEmittedMs = Date.now();
113
+ return true;
140
114
  }
141
-
142
- if (options !== null && options !== void 0 && options.bytesUsed) {
143
- this.bytesUsed = options.bytesUsed;
115
+ /**
116
+ * bytesUsed can be set via chunkComplete or via getBatch*
117
+ */
118
+ _getBatch(options) {
119
+ if (!this.aggregator) {
120
+ return null;
121
+ }
122
+ // TODO - this can overly increment bytes used?
123
+ if (options?.bytesUsed) {
124
+ this.bytesUsed = options.bytesUsed;
125
+ }
126
+ const normalizedBatch = this.aggregator.getBatch();
127
+ normalizedBatch.count = this.batchCount;
128
+ normalizedBatch.bytesUsed = this.bytesUsed;
129
+ Object.assign(normalizedBatch, options);
130
+ this.batchCount++;
131
+ this.aggregator = null;
132
+ return normalizedBatch;
144
133
  }
145
-
146
- const normalizedBatch = this.aggregator.getBatch();
147
- normalizedBatch.count = this.batchCount;
148
- normalizedBatch.bytesUsed = this.bytesUsed;
149
- Object.assign(normalizedBatch, options);
150
- this.batchCount++;
151
- this.aggregator = null;
152
- return normalizedBatch;
153
- }
154
-
155
- _getTableBatchType() {
156
- switch (this.options.shape) {
157
- case 'row-table':
158
- return BaseTableBatchAggregator;
159
-
160
- case 'array-row-table':
161
- case 'object-row-table':
162
- return RowTableBatchAggregator;
163
-
164
- case 'columnar-table':
165
- return ColumnarTableBatchAggregator;
166
-
167
- case 'arrow-table':
168
- if (!TableBatchBuilder.ArrowBatch) {
169
- throw new Error(ERR_MESSAGE);
170
- }
171
-
172
- return TableBatchBuilder.ArrowBatch;
173
-
174
- default:
175
- throw new Error(ERR_MESSAGE);
134
+ _getTableBatchType() {
135
+ switch (this.options.shape) {
136
+ case 'row-table':
137
+ return base_table_batch_aggregator_1.default;
138
+ case 'array-row-table':
139
+ case 'object-row-table':
140
+ return row_table_batch_aggregator_1.default;
141
+ case 'columnar-table':
142
+ return columnar_table_batch_aggregator_1.default;
143
+ case 'arrow-table':
144
+ if (!TableBatchBuilder.ArrowBatch) {
145
+ throw new Error(ERR_MESSAGE);
146
+ }
147
+ return TableBatchBuilder.ArrowBatch;
148
+ default:
149
+ throw new Error(ERR_MESSAGE);
150
+ }
176
151
  }
177
- }
178
-
179
152
  }
180
-
181
- _defineProperty(TableBatchBuilder, "ArrowBatch", void 0);
182
- //# sourceMappingURL=table-batch-builder.js.map
153
+ exports.default = TableBatchBuilder;
@@ -0,0 +1,92 @@
1
+ /**
2
+ * Main data type enumeration.
3
+ *
4
+ * Data types in this library are all *logical*. They can be expressed as
5
+ * either a primitive physical type (bytes or bits of some fixed size), a
6
+ * nested type consisting of other data types, or another data type (e.g. a
7
+ * timestamp encoded as an int64).
8
+ *
9
+ * **Note**: Only enum values 0-17 (NONE through Map) are written to an Arrow
10
+ * IPC payload.
11
+ *
12
+ * The rest of the values are specified here so TypeScript can narrow the type
13
+ * signatures further beyond the base Arrow Types. The Arrow DataTypes include
14
+ * metadata like `bitWidth` that impact the type signatures of the values we
15
+ * accept and return.
16
+ *
17
+ * For example, the `Int8Vector` reads 1-byte numbers from an `Int8Array`, an
18
+ * `Int32Vector` reads a 4-byte number from an `Int32Array`, and an `Int64Vector`
19
+ * reads a pair of 4-byte lo, hi 32-bit integers as a zero-copy slice from the
20
+ * underlying `Int32Array`.
21
+ *
22
+ * Library consumers benefit by knowing the narrowest type, since we can ensure
23
+ * the types across all public methods are propagated, and never bail to `any`.
24
+ * These values are _never_ used at runtime, and they will _never_ be written
25
+ * to the flatbuffers metadata of serialized Arrow IPC payloads.
26
+ */
27
+ export declare enum Type {
28
+ /** The default placeholder type */
29
+ NONE = 0,
30
+ /** A NULL type having no physical storage */
31
+ Null = 1,
32
+ /** Signed or unsigned 8, 16, 32, or 64-bit little-endian integer */
33
+ Int = 2,
34
+ /** 2, 4, or 8-byte floating point value */
35
+ Float = 3,
36
+ /** Variable-length bytes (no guarantee of UTF8-ness) */
37
+ Binary = 4,
38
+ /** UTF8 variable-length string as List<Char> */
39
+ Utf8 = 5,
40
+ /** Boolean as 1 bit, LSB bit-packed ordering */
41
+ Bool = 6,
42
+ /** Precision-and-scale-based decimal type. Storage type depends on the parameters. */
43
+ Decimal = 7,
44
+ /** int32_t days or int64_t milliseconds since the UNIX epoch */
45
+ Date = 8,
46
+ /** Time as signed 32 or 64-bit integer, representing either seconds, milliseconds, microseconds, or nanoseconds since midnight since midnight */
47
+ Time = 9,
48
+ /** Exact timestamp encoded with int64 since UNIX epoch (Default unit millisecond) */
49
+ Timestamp = 10,
50
+ /** YEAR_MONTH or DAY_TIME interval in SQL style */
51
+ Interval = 11,
52
+ /** A list of some logical data type */
53
+ List = 12,
54
+ /** Struct of logical types */
55
+ Struct = 13,
56
+ /** Union of logical types */
57
+ Union = 14,
58
+ /** Fixed-size binary. Each value occupies the same number of bytes */
59
+ FixedSizeBinary = 15,
60
+ /** Fixed-size list. Each value occupies the same number of bytes */
61
+ FixedSizeList = 16,
62
+ /** Map of named logical types */
63
+ Map = 17,
64
+ /** Dictionary aka Category type */
65
+ Dictionary = -1,
66
+ Int8 = -2,
67
+ Int16 = -3,
68
+ Int32 = -4,
69
+ Int64 = -5,
70
+ Uint8 = -6,
71
+ Uint16 = -7,
72
+ Uint32 = -8,
73
+ Uint64 = -9,
74
+ Float16 = -10,
75
+ Float32 = -11,
76
+ Float64 = -12,
77
+ DateDay = -13,
78
+ DateMillisecond = -14,
79
+ TimestampSecond = -15,
80
+ TimestampMillisecond = -16,
81
+ TimestampMicrosecond = -17,
82
+ TimestampNanosecond = -18,
83
+ TimeSecond = -19,
84
+ TimeMillisecond = -20,
85
+ TimeMicrosecond = -21,
86
+ TimeNanosecond = -22,
87
+ DenseUnion = -23,
88
+ SparseUnion = -24,
89
+ IntervalDayTime = -25,
90
+ IntervalYearMonth = -26
91
+ }
92
+ //# sourceMappingURL=enum.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"enum.d.ts","sourceRoot":"","sources":["../../../../src/lib/schema/impl/enum.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,oBAAY,IAAI;IACd,mCAAmC;IACnC,IAAI,IAAI;IACR,6CAA6C;IAC7C,IAAI,IAAI;IACR,oEAAoE;IACpE,GAAG,IAAI;IACP,2CAA2C;IAC3C,KAAK,IAAI;IACT,wDAAwD;IACxD,MAAM,IAAI;IACV,gDAAgD;IAChD,IAAI,IAAI;IACR,gDAAgD;IAChD,IAAI,IAAI;IACR,sFAAsF;IACtF,OAAO,IAAI;IACX,gEAAgE;IAChE,IAAI,IAAI;IACR,iJAAiJ;IACjJ,IAAI,IAAI;IACR,qFAAqF;IACrF,SAAS,KAAK;IACd,mDAAmD;IACnD,QAAQ,KAAK;IACb,uCAAuC;IACvC,IAAI,KAAK;IACT,8BAA8B;IAC9B,MAAM,KAAK;IACX,6BAA6B;IAC7B,KAAK,KAAK;IACV,sEAAsE;IACtE,eAAe,KAAK;IACpB,oEAAoE;IACpE,aAAa,KAAK;IAClB,iCAAiC;IACjC,GAAG,KAAK;IAER,mCAAmC;IACnC,UAAU,KAAK;IACf,IAAI,KAAK;IACT,KAAK,KAAK;IACV,KAAK,KAAK;IACV,KAAK,KAAK;IACV,KAAK,KAAK;IACV,MAAM,KAAK;IACX,MAAM,KAAK;IACX,MAAM,KAAK;IACX,OAAO,MAAM;IACb,OAAO,MAAM;IACb,OAAO,MAAM;IACb,OAAO,MAAM;IACb,eAAe,MAAM;IACrB,eAAe,MAAM;IACrB,oBAAoB,MAAM;IAC1B,oBAAoB,MAAM;IAC1B,mBAAmB,MAAM;IACzB,UAAU,MAAM;IAChB,eAAe,MAAM;IACrB,eAAe,MAAM;IACrB,cAAc,MAAM;IACpB,UAAU,MAAM;IAChB,WAAW,MAAM;IACjB,eAAe,MAAM;IACrB,iBAAiB,MAAM;CACxB"}
@@ -1,49 +1,97 @@
1
- export let Type;
2
-
1
+ "use strict";
2
+ // This code is adapted from ArrowJS https://github.com/apache/arrow
3
+ // under Apache license http://www.apache.org/licenses/LICENSE-2.0
4
+ Object.defineProperty(exports, "__esModule", { value: true });
5
+ exports.Type = void 0;
6
+ /**
7
+ * Main data type enumeration.
8
+ *
9
+ * Data types in this library are all *logical*. They can be expressed as
10
+ * either a primitive physical type (bytes or bits of some fixed size), a
11
+ * nested type consisting of other data types, or another data type (e.g. a
12
+ * timestamp encoded as an int64).
13
+ *
14
+ * **Note**: Only enum values 0-17 (NONE through Map) are written to an Arrow
15
+ * IPC payload.
16
+ *
17
+ * The rest of the values are specified here so TypeScript can narrow the type
18
+ * signatures further beyond the base Arrow Types. The Arrow DataTypes include
19
+ * metadata like `bitWidth` that impact the type signatures of the values we
20
+ * accept and return.
21
+ *
22
+ * For example, the `Int8Vector` reads 1-byte numbers from an `Int8Array`, an
23
+ * `Int32Vector` reads a 4-byte number from an `Int32Array`, and an `Int64Vector`
24
+ * reads a pair of 4-byte lo, hi 32-bit integers as a zero-copy slice from the
25
+ * underlying `Int32Array`.
26
+ *
27
+ * Library consumers benefit by knowing the narrowest type, since we can ensure
28
+ * the types across all public methods are propagated, and never bail to `any`.
29
+ * These values are _never_ used at runtime, and they will _never_ be written
30
+ * to the flatbuffers metadata of serialized Arrow IPC payloads.
31
+ */
32
+ var Type;
3
33
  (function (Type) {
4
- Type[Type["NONE"] = 0] = "NONE";
5
- Type[Type["Null"] = 1] = "Null";
6
- Type[Type["Int"] = 2] = "Int";
7
- Type[Type["Float"] = 3] = "Float";
8
- Type[Type["Binary"] = 4] = "Binary";
9
- Type[Type["Utf8"] = 5] = "Utf8";
10
- Type[Type["Bool"] = 6] = "Bool";
11
- Type[Type["Decimal"] = 7] = "Decimal";
12
- Type[Type["Date"] = 8] = "Date";
13
- Type[Type["Time"] = 9] = "Time";
14
- Type[Type["Timestamp"] = 10] = "Timestamp";
15
- Type[Type["Interval"] = 11] = "Interval";
16
- Type[Type["List"] = 12] = "List";
17
- Type[Type["Struct"] = 13] = "Struct";
18
- Type[Type["Union"] = 14] = "Union";
19
- Type[Type["FixedSizeBinary"] = 15] = "FixedSizeBinary";
20
- Type[Type["FixedSizeList"] = 16] = "FixedSizeList";
21
- Type[Type["Map"] = 17] = "Map";
22
- Type[Type["Dictionary"] = -1] = "Dictionary";
23
- Type[Type["Int8"] = -2] = "Int8";
24
- Type[Type["Int16"] = -3] = "Int16";
25
- Type[Type["Int32"] = -4] = "Int32";
26
- Type[Type["Int64"] = -5] = "Int64";
27
- Type[Type["Uint8"] = -6] = "Uint8";
28
- Type[Type["Uint16"] = -7] = "Uint16";
29
- Type[Type["Uint32"] = -8] = "Uint32";
30
- Type[Type["Uint64"] = -9] = "Uint64";
31
- Type[Type["Float16"] = -10] = "Float16";
32
- Type[Type["Float32"] = -11] = "Float32";
33
- Type[Type["Float64"] = -12] = "Float64";
34
- Type[Type["DateDay"] = -13] = "DateDay";
35
- Type[Type["DateMillisecond"] = -14] = "DateMillisecond";
36
- Type[Type["TimestampSecond"] = -15] = "TimestampSecond";
37
- Type[Type["TimestampMillisecond"] = -16] = "TimestampMillisecond";
38
- Type[Type["TimestampMicrosecond"] = -17] = "TimestampMicrosecond";
39
- Type[Type["TimestampNanosecond"] = -18] = "TimestampNanosecond";
40
- Type[Type["TimeSecond"] = -19] = "TimeSecond";
41
- Type[Type["TimeMillisecond"] = -20] = "TimeMillisecond";
42
- Type[Type["TimeMicrosecond"] = -21] = "TimeMicrosecond";
43
- Type[Type["TimeNanosecond"] = -22] = "TimeNanosecond";
44
- Type[Type["DenseUnion"] = -23] = "DenseUnion";
45
- Type[Type["SparseUnion"] = -24] = "SparseUnion";
46
- Type[Type["IntervalDayTime"] = -25] = "IntervalDayTime";
47
- Type[Type["IntervalYearMonth"] = -26] = "IntervalYearMonth";
48
- })(Type || (Type = {}));
49
- //# sourceMappingURL=enum.js.map
34
+ /** The default placeholder type */
35
+ Type[Type["NONE"] = 0] = "NONE";
36
+ /** A NULL type having no physical storage */
37
+ Type[Type["Null"] = 1] = "Null";
38
+ /** Signed or unsigned 8, 16, 32, or 64-bit little-endian integer */
39
+ Type[Type["Int"] = 2] = "Int";
40
+ /** 2, 4, or 8-byte floating point value */
41
+ Type[Type["Float"] = 3] = "Float";
42
+ /** Variable-length bytes (no guarantee of UTF8-ness) */
43
+ Type[Type["Binary"] = 4] = "Binary";
44
+ /** UTF8 variable-length string as List<Char> */
45
+ Type[Type["Utf8"] = 5] = "Utf8";
46
+ /** Boolean as 1 bit, LSB bit-packed ordering */
47
+ Type[Type["Bool"] = 6] = "Bool";
48
+ /** Precision-and-scale-based decimal type. Storage type depends on the parameters. */
49
+ Type[Type["Decimal"] = 7] = "Decimal";
50
+ /** int32_t days or int64_t milliseconds since the UNIX epoch */
51
+ Type[Type["Date"] = 8] = "Date";
52
+ /** Time as signed 32 or 64-bit integer, representing either seconds, milliseconds, microseconds, or nanoseconds since midnight since midnight */
53
+ Type[Type["Time"] = 9] = "Time";
54
+ /** Exact timestamp encoded with int64 since UNIX epoch (Default unit millisecond) */
55
+ Type[Type["Timestamp"] = 10] = "Timestamp";
56
+ /** YEAR_MONTH or DAY_TIME interval in SQL style */
57
+ Type[Type["Interval"] = 11] = "Interval";
58
+ /** A list of some logical data type */
59
+ Type[Type["List"] = 12] = "List";
60
+ /** Struct of logical types */
61
+ Type[Type["Struct"] = 13] = "Struct";
62
+ /** Union of logical types */
63
+ Type[Type["Union"] = 14] = "Union";
64
+ /** Fixed-size binary. Each value occupies the same number of bytes */
65
+ Type[Type["FixedSizeBinary"] = 15] = "FixedSizeBinary";
66
+ /** Fixed-size list. Each value occupies the same number of bytes */
67
+ Type[Type["FixedSizeList"] = 16] = "FixedSizeList";
68
+ /** Map of named logical types */
69
+ Type[Type["Map"] = 17] = "Map";
70
+ /** Dictionary aka Category type */
71
+ Type[Type["Dictionary"] = -1] = "Dictionary";
72
+ Type[Type["Int8"] = -2] = "Int8";
73
+ Type[Type["Int16"] = -3] = "Int16";
74
+ Type[Type["Int32"] = -4] = "Int32";
75
+ Type[Type["Int64"] = -5] = "Int64";
76
+ Type[Type["Uint8"] = -6] = "Uint8";
77
+ Type[Type["Uint16"] = -7] = "Uint16";
78
+ Type[Type["Uint32"] = -8] = "Uint32";
79
+ Type[Type["Uint64"] = -9] = "Uint64";
80
+ Type[Type["Float16"] = -10] = "Float16";
81
+ Type[Type["Float32"] = -11] = "Float32";
82
+ Type[Type["Float64"] = -12] = "Float64";
83
+ Type[Type["DateDay"] = -13] = "DateDay";
84
+ Type[Type["DateMillisecond"] = -14] = "DateMillisecond";
85
+ Type[Type["TimestampSecond"] = -15] = "TimestampSecond";
86
+ Type[Type["TimestampMillisecond"] = -16] = "TimestampMillisecond";
87
+ Type[Type["TimestampMicrosecond"] = -17] = "TimestampMicrosecond";
88
+ Type[Type["TimestampNanosecond"] = -18] = "TimestampNanosecond";
89
+ Type[Type["TimeSecond"] = -19] = "TimeSecond";
90
+ Type[Type["TimeMillisecond"] = -20] = "TimeMillisecond";
91
+ Type[Type["TimeMicrosecond"] = -21] = "TimeMicrosecond";
92
+ Type[Type["TimeNanosecond"] = -22] = "TimeNanosecond";
93
+ Type[Type["DenseUnion"] = -23] = "DenseUnion";
94
+ Type[Type["SparseUnion"] = -24] = "SparseUnion";
95
+ Type[Type["IntervalDayTime"] = -25] = "IntervalDayTime";
96
+ Type[Type["IntervalYearMonth"] = -26] = "IntervalYearMonth";
97
+ })(Type = exports.Type || (exports.Type = {}));
@@ -0,0 +1,19 @@
1
+ import { DataType } from './type';
2
+ /**
3
+ * ArrowJS `Field` API-compatible class for row-based tables
4
+ * https://loaders.gl/arrowjs/docs/api-reference/field
5
+ * A field holds name, nullable, and metadata information about a table "column"
6
+ * A Schema is essentially a list of fields
7
+ */
8
+ export default class Field {
9
+ name: string;
10
+ type: DataType;
11
+ nullable: boolean;
12
+ metadata: Map<string, string>;
13
+ constructor(name: string, type: DataType, nullable?: boolean, metadata?: Map<string, string>);
14
+ get typeId(): number;
15
+ clone(): Field;
16
+ compareTo(other: this): boolean;
17
+ toString(): string;
18
+ }
19
+ //# sourceMappingURL=field.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"field.d.ts","sourceRoot":"","sources":["../../../../src/lib/schema/impl/field.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,QAAQ,EAAC,MAAM,QAAQ,CAAC;AAEhC;;;;;GAKG;AACH,MAAM,CAAC,OAAO,OAAO,KAAK;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,CAAC;IACf,QAAQ,EAAE,OAAO,CAAC;IAClB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAG5B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,QAAQ,EACd,QAAQ,UAAQ,EAChB,QAAQ,GAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAa;IAQ3C,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,KAAK,IAAI,KAAK;IAId,SAAS,CAAC,KAAK,EAAE,IAAI,GAAG,OAAO;IAS/B,QAAQ,IAAI,MAAM;CAKnB"}