@loaders.gl/schema 3.1.3 → 4.0.0-alpha.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. package/dist/bundle.js +2 -2
  2. package/dist/bundle.js.map +1 -0
  3. package/dist/category/common.d.ts +1 -2
  4. package/dist/category/common.d.ts.map +1 -1
  5. package/dist/category/common.js +2 -10
  6. package/dist/{es5/category → category}/common.js.map +0 -0
  7. package/dist/category/gis.js +2 -2
  8. package/dist/{es5/category → category}/gis.js.map +0 -0
  9. package/dist/category/image/image.d.ts +18 -0
  10. package/dist/category/image/image.d.ts.map +1 -0
  11. package/dist/category/image/image.js +2 -0
  12. package/dist/{esm/category/gis.js.map → category/image/image.js.map} +1 -1
  13. package/dist/category/mesh/convert-mesh.js +31 -37
  14. package/dist/category/mesh/convert-mesh.js.map +1 -0
  15. package/dist/category/mesh/deduce-mesh-schema.js +37 -57
  16. package/dist/category/mesh/deduce-mesh-schema.js.map +1 -0
  17. package/dist/category/mesh/mesh-to-arrow-table.d.ts +10 -0
  18. package/dist/category/mesh/mesh-to-arrow-table.d.ts.map +1 -1
  19. package/dist/category/mesh/mesh-to-arrow-table.js +15 -28
  20. package/dist/category/mesh/mesh-to-arrow-table.js.map +1 -0
  21. package/dist/category/mesh/mesh-types.js +2 -2
  22. package/dist/{es5/category → category}/mesh/mesh-types.js.map +0 -0
  23. package/dist/category/mesh/mesh-utils.js +35 -48
  24. package/dist/category/mesh/mesh-utils.js.map +1 -0
  25. package/dist/category/table/deduce-table-schema.js +39 -94
  26. package/dist/category/table/deduce-table-schema.js.map +1 -0
  27. package/dist/category/table/table-types.d.ts +1 -3
  28. package/dist/category/table/table-types.d.ts.map +1 -1
  29. package/dist/category/table/table-types.js +2 -2
  30. package/dist/{es5/category → category}/table/table-types.js.map +0 -0
  31. package/dist/category/texture/texture.d.ts +18 -0
  32. package/dist/category/texture/texture.d.ts.map +1 -0
  33. package/dist/category/texture/texture.js +2 -0
  34. package/dist/{esm/category/common.js.map → category/texture/texture.js.map} +1 -1
  35. package/dist/dist.min.js +15078 -208
  36. package/dist/index.d.ts +2 -0
  37. package/dist/index.d.ts.map +1 -1
  38. package/dist/index.js +13 -75
  39. package/dist/index.js.map +1 -0
  40. package/dist/lib/arrow/arrow-like-type-utils.js +31 -26
  41. package/dist/lib/arrow/arrow-like-type-utils.js.map +1 -0
  42. package/dist/lib/arrow/arrow-type-utils.d.ts +5 -0
  43. package/dist/lib/arrow/arrow-type-utils.d.ts.map +1 -1
  44. package/dist/lib/arrow/arrow-type-utils.js +20 -28
  45. package/dist/lib/arrow/arrow-type-utils.js.map +1 -0
  46. package/dist/lib/arrow/get-type-info.js +19 -23
  47. package/dist/lib/arrow/get-type-info.js.map +1 -0
  48. package/dist/lib/batches/base-table-batch-aggregator.js +68 -52
  49. package/dist/lib/batches/base-table-batch-aggregator.js.map +1 -0
  50. package/dist/lib/batches/columnar-table-batch-aggregator.js +92 -82
  51. package/dist/lib/batches/columnar-table-batch-aggregator.js.map +1 -0
  52. package/dist/lib/batches/row-table-batch-aggregator.js +88 -72
  53. package/dist/lib/batches/row-table-batch-aggregator.js.map +1 -0
  54. package/dist/lib/batches/table-batch-aggregator.js +2 -2
  55. package/dist/{es5/lib → lib}/batches/table-batch-aggregator.js.map +0 -0
  56. package/dist/lib/batches/table-batch-builder.js +166 -137
  57. package/dist/lib/batches/table-batch-builder.js.map +1 -0
  58. package/dist/lib/schema/impl/enum.js +48 -96
  59. package/dist/lib/schema/impl/enum.js.map +1 -0
  60. package/dist/lib/schema/impl/field.js +34 -31
  61. package/dist/lib/schema/impl/field.js.map +1 -0
  62. package/dist/lib/schema/impl/schema.js +81 -72
  63. package/dist/lib/schema/impl/schema.js.map +1 -0
  64. package/dist/lib/schema/impl/type.js +490 -454
  65. package/dist/lib/schema/impl/type.js.map +1 -0
  66. package/dist/lib/schema/schema.js +5 -90
  67. package/dist/lib/schema/schema.js.map +1 -0
  68. package/dist/lib/schema-utils/deduce-column-type.js +20 -89
  69. package/dist/lib/schema-utils/deduce-column-type.js.map +1 -0
  70. package/dist/lib/utils/assert.js +5 -11
  71. package/dist/lib/utils/assert.js.map +1 -0
  72. package/dist/lib/utils/async-queue.js +109 -82
  73. package/dist/lib/utils/async-queue.js.map +1 -0
  74. package/dist/lib/utils/row-utils.js +33 -31
  75. package/dist/lib/utils/row-utils.js.map +1 -0
  76. package/dist/types.js +2 -2
  77. package/dist/{es5/types.js.map → types.js.map} +0 -0
  78. package/package.json +6 -5
  79. package/src/category/common.ts +1 -2
  80. package/src/category/image/image.ts +19 -0
  81. package/src/category/mesh/convert-mesh.ts +6 -6
  82. package/src/category/mesh/mesh-to-arrow-table.ts +2 -4
  83. package/src/category/table/table-types.ts +1 -4
  84. package/src/category/texture/texture.ts +28 -0
  85. package/src/index.ts +6 -1
  86. package/src/lib/arrow/arrow-type-utils.ts +0 -2
  87. package/dist/es5/bundle.js +0 -7
  88. package/dist/es5/bundle.js.map +0 -1
  89. package/dist/es5/category/common.js +0 -2
  90. package/dist/es5/category/gis.js +0 -2
  91. package/dist/es5/category/mesh/convert-mesh.js +0 -43
  92. package/dist/es5/category/mesh/convert-mesh.js.map +0 -1
  93. package/dist/es5/category/mesh/deduce-mesh-schema.js +0 -54
  94. package/dist/es5/category/mesh/deduce-mesh-schema.js.map +0 -1
  95. package/dist/es5/category/mesh/mesh-to-arrow-table.js +0 -2
  96. package/dist/es5/category/mesh/mesh-to-arrow-table.js.map +0 -1
  97. package/dist/es5/category/mesh/mesh-types.js +0 -2
  98. package/dist/es5/category/mesh/mesh-utils.js +0 -47
  99. package/dist/es5/category/mesh/mesh-utils.js.map +0 -1
  100. package/dist/es5/category/table/deduce-table-schema.js +0 -58
  101. package/dist/es5/category/table/deduce-table-schema.js.map +0 -1
  102. package/dist/es5/category/table/table-types.js +0 -2
  103. package/dist/es5/index.js +0 -344
  104. package/dist/es5/index.js.map +0 -1
  105. package/dist/es5/lib/arrow/arrow-like-type-utils.js +0 -40
  106. package/dist/es5/lib/arrow/arrow-like-type-utils.js.map +0 -1
  107. package/dist/es5/lib/arrow/arrow-type-utils.js +0 -2
  108. package/dist/es5/lib/arrow/arrow-type-utils.js.map +0 -1
  109. package/dist/es5/lib/arrow/get-type-info.js +0 -33
  110. package/dist/es5/lib/arrow/get-type-info.js.map +0 -1
  111. package/dist/es5/lib/batches/base-table-batch-aggregator.js +0 -92
  112. package/dist/es5/lib/batches/base-table-batch-aggregator.js.map +0 -1
  113. package/dist/es5/lib/batches/columnar-table-batch-aggregator.js +0 -128
  114. package/dist/es5/lib/batches/columnar-table-batch-aggregator.js.map +0 -1
  115. package/dist/es5/lib/batches/row-table-batch-aggregator.js +0 -113
  116. package/dist/es5/lib/batches/row-table-batch-aggregator.js.map +0 -1
  117. package/dist/es5/lib/batches/table-batch-aggregator.js +0 -2
  118. package/dist/es5/lib/batches/table-batch-builder.js +0 -207
  119. package/dist/es5/lib/batches/table-batch-builder.js.map +0 -1
  120. package/dist/es5/lib/schema/impl/enum.js +0 -56
  121. package/dist/es5/lib/schema/impl/enum.js.map +0 -1
  122. package/dist/es5/lib/schema/impl/field.js +0 -56
  123. package/dist/es5/lib/schema/impl/field.js.map +0 -1
  124. package/dist/es5/lib/schema/impl/schema.js +0 -168
  125. package/dist/es5/lib/schema/impl/schema.js.map +0 -1
  126. package/dist/es5/lib/schema/impl/type.js +0 -927
  127. package/dist/es5/lib/schema/impl/type.js.map +0 -1
  128. package/dist/es5/lib/schema/schema.js +0 -236
  129. package/dist/es5/lib/schema/schema.js.map +0 -1
  130. package/dist/es5/lib/schema-utils/deduce-column-type.js +0 -32
  131. package/dist/es5/lib/schema-utils/deduce-column-type.js.map +0 -1
  132. package/dist/es5/lib/utils/assert.js +0 -13
  133. package/dist/es5/lib/utils/assert.js.map +0 -1
  134. package/dist/es5/lib/utils/async-queue.js +0 -213
  135. package/dist/es5/lib/utils/async-queue.js.map +0 -1
  136. package/dist/es5/lib/utils/row-utils.js +0 -44
  137. package/dist/es5/lib/utils/row-utils.js.map +0 -1
  138. package/dist/es5/types.js +0 -2
  139. package/dist/esm/bundle.js +0 -5
  140. package/dist/esm/bundle.js.map +0 -1
  141. package/dist/esm/category/common.js +0 -2
  142. package/dist/esm/category/gis.js +0 -2
  143. package/dist/esm/category/mesh/convert-mesh.js +0 -26
  144. package/dist/esm/category/mesh/convert-mesh.js.map +0 -1
  145. package/dist/esm/category/mesh/deduce-mesh-schema.js +0 -42
  146. package/dist/esm/category/mesh/deduce-mesh-schema.js.map +0 -1
  147. package/dist/esm/category/mesh/mesh-to-arrow-table.js +0 -2
  148. package/dist/esm/category/mesh/mesh-to-arrow-table.js.map +0 -1
  149. package/dist/esm/category/mesh/mesh-types.js +0 -2
  150. package/dist/esm/category/mesh/mesh-types.js.map +0 -1
  151. package/dist/esm/category/mesh/mesh-utils.js +0 -38
  152. package/dist/esm/category/mesh/mesh-utils.js.map +0 -1
  153. package/dist/esm/category/table/deduce-table-schema.js +0 -51
  154. package/dist/esm/category/table/deduce-table-schema.js.map +0 -1
  155. package/dist/esm/category/table/table-types.js +0 -2
  156. package/dist/esm/category/table/table-types.js.map +0 -1
  157. package/dist/esm/index.js +0 -13
  158. package/dist/esm/index.js.map +0 -1
  159. package/dist/esm/lib/arrow/arrow-like-type-utils.js +0 -32
  160. package/dist/esm/lib/arrow/arrow-like-type-utils.js.map +0 -1
  161. package/dist/esm/lib/arrow/arrow-type-utils.js +0 -2
  162. package/dist/esm/lib/arrow/arrow-type-utils.js.map +0 -1
  163. package/dist/esm/lib/arrow/get-type-info.js +0 -24
  164. package/dist/esm/lib/arrow/get-type-info.js.map +0 -1
  165. package/dist/esm/lib/batches/base-table-batch-aggregator.js +0 -74
  166. package/dist/esm/lib/batches/base-table-batch-aggregator.js.map +0 -1
  167. package/dist/esm/lib/batches/columnar-table-batch-aggregator.js +0 -100
  168. package/dist/esm/lib/batches/columnar-table-batch-aggregator.js.map +0 -1
  169. package/dist/esm/lib/batches/row-table-batch-aggregator.js +0 -95
  170. package/dist/esm/lib/batches/row-table-batch-aggregator.js.map +0 -1
  171. package/dist/esm/lib/batches/table-batch-aggregator.js +0 -2
  172. package/dist/esm/lib/batches/table-batch-aggregator.js.map +0 -1
  173. package/dist/esm/lib/batches/table-batch-builder.js +0 -182
  174. package/dist/esm/lib/batches/table-batch-builder.js.map +0 -1
  175. package/dist/esm/lib/schema/impl/enum.js +0 -49
  176. package/dist/esm/lib/schema/impl/enum.js.map +0 -1
  177. package/dist/esm/lib/schema/impl/field.js +0 -35
  178. package/dist/esm/lib/schema/impl/field.js.map +0 -1
  179. package/dist/esm/lib/schema/impl/schema.js +0 -92
  180. package/dist/esm/lib/schema/impl/schema.js.map +0 -1
  181. package/dist/esm/lib/schema/impl/type.js +0 -498
  182. package/dist/esm/lib/schema/impl/type.js.map +0 -1
  183. package/dist/esm/lib/schema/schema.js +0 -5
  184. package/dist/esm/lib/schema/schema.js.map +0 -1
  185. package/dist/esm/lib/schema-utils/deduce-column-type.js +0 -23
  186. package/dist/esm/lib/schema-utils/deduce-column-type.js.map +0 -1
  187. package/dist/esm/lib/utils/assert.js +0 -6
  188. package/dist/esm/lib/utils/assert.js.map +0 -1
  189. package/dist/esm/lib/utils/async-queue.js +0 -119
  190. package/dist/esm/lib/utils/async-queue.js.map +0 -1
  191. package/dist/esm/lib/utils/row-utils.js +0 -35
  192. package/dist/esm/lib/utils/row-utils.js.map +0 -1
  193. package/dist/esm/types.js +0 -2
  194. package/dist/esm/types.js.map +0 -1
@@ -1,90 +1,100 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
1
+ import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
3
2
  const DEFAULT_ROW_COUNT = 100;
4
- class ColumnarTableBatchAggregator {
5
- constructor(schema, options) {
6
- this.length = 0;
7
- this.allocated = 0;
8
- this.columns = {};
9
- this.schema = schema;
10
- this._reallocateColumns();
3
+ export default class ColumnarTableBatchAggregator {
4
+ constructor(schema, options) {
5
+ _defineProperty(this, "schema", void 0);
6
+
7
+ _defineProperty(this, "length", 0);
8
+
9
+ _defineProperty(this, "allocated", 0);
10
+
11
+ _defineProperty(this, "columns", {});
12
+
13
+ this.schema = schema;
14
+
15
+ this._reallocateColumns();
16
+ }
17
+
18
+ rowCount() {
19
+ return this.length;
20
+ }
21
+
22
+ addArrayRow(row) {
23
+ this._reallocateColumns();
24
+
25
+ let i = 0;
26
+
27
+ for (const fieldName in this.columns) {
28
+ this.columns[fieldName][this.length] = row[i++];
11
29
  }
12
- rowCount() {
13
- return this.length;
30
+
31
+ this.length++;
32
+ }
33
+
34
+ addObjectRow(row) {
35
+ this._reallocateColumns();
36
+
37
+ for (const fieldName in row) {
38
+ this.columns[fieldName][this.length] = row[fieldName];
14
39
  }
15
- addArrayRow(row) {
16
- // If user keeps pushing rows beyond batch size, reallocate
17
- this._reallocateColumns();
18
- let i = 0;
19
- // TODO what if no csv header, columns not populated?
20
- for (const fieldName in this.columns) {
21
- this.columns[fieldName][this.length] = row[i++];
22
- }
23
- this.length++;
40
+
41
+ this.length++;
42
+ }
43
+
44
+ getBatch() {
45
+ this._pruneColumns();
46
+
47
+ const columns = Array.isArray(this.schema) ? this.columns : {};
48
+
49
+ if (!Array.isArray(this.schema)) {
50
+ for (const fieldName in this.schema) {
51
+ const field = this.schema[fieldName];
52
+ columns[field.name] = this.columns[field.index];
53
+ }
24
54
  }
25
- addObjectRow(row) {
26
- // If user keeps pushing rows beyond batch size, reallocate
27
- this._reallocateColumns();
28
- for (const fieldName in row) {
29
- this.columns[fieldName][this.length] = row[fieldName];
30
- }
31
- this.length++;
55
+
56
+ this.columns = {};
57
+ const batch = {
58
+ shape: 'columnar-table',
59
+ batchType: 'data',
60
+ data: columns,
61
+ schema: this.schema,
62
+ length: this.length
63
+ };
64
+ return batch;
65
+ }
66
+
67
+ _reallocateColumns() {
68
+ if (this.length < this.allocated) {
69
+ return;
32
70
  }
33
- getBatch() {
34
- this._pruneColumns();
35
- const columns = Array.isArray(this.schema) ? this.columns : {};
36
- // schema is an array if there're no headers
37
- // object if there are headers
38
- // columns should match schema format
39
- if (!Array.isArray(this.schema)) {
40
- for (const fieldName in this.schema) {
41
- const field = this.schema[fieldName];
42
- columns[field.name] = this.columns[field.index];
43
- }
44
- }
45
- this.columns = {};
46
- const batch = {
47
- shape: 'columnar-table',
48
- batchType: 'data',
49
- data: columns,
50
- schema: this.schema,
51
- length: this.length
52
- };
53
- return batch;
71
+
72
+ this.allocated = this.allocated > 0 ? this.allocated *= 2 : DEFAULT_ROW_COUNT;
73
+ this.columns = {};
74
+
75
+ for (const fieldName in this.schema) {
76
+ const field = this.schema[fieldName];
77
+ const ArrayType = field.type || Float32Array;
78
+ const oldColumn = this.columns[field.index];
79
+
80
+ if (oldColumn && ArrayBuffer.isView(oldColumn)) {
81
+ const typedArray = new ArrayType(this.allocated);
82
+ typedArray.set(oldColumn);
83
+ this.columns[field.index] = typedArray;
84
+ } else if (oldColumn) {
85
+ oldColumn.length = this.allocated;
86
+ this.columns[field.index] = oldColumn;
87
+ } else {
88
+ this.columns[field.index] = new ArrayType(this.allocated);
89
+ }
54
90
  }
55
- // HELPERS
56
- _reallocateColumns() {
57
- if (this.length < this.allocated) {
58
- return;
59
- }
60
- // @ts-ignore TODO
61
- this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;
62
- this.columns = {};
63
- for (const fieldName in this.schema) {
64
- const field = this.schema[fieldName];
65
- const ArrayType = field.type || Float32Array;
66
- const oldColumn = this.columns[field.index];
67
- if (oldColumn && ArrayBuffer.isView(oldColumn)) {
68
- // Copy the old data to the new array
69
- const typedArray = new ArrayType(this.allocated);
70
- typedArray.set(oldColumn);
71
- this.columns[field.index] = typedArray;
72
- }
73
- else if (oldColumn) {
74
- // Plain array
75
- oldColumn.length = this.allocated;
76
- this.columns[field.index] = oldColumn;
77
- }
78
- else {
79
- // Create new
80
- this.columns[field.index] = new ArrayType(this.allocated);
81
- }
82
- }
83
- }
84
- _pruneColumns() {
85
- for (const [columnName, column] of Object.entries(this.columns)) {
86
- this.columns[columnName] = column.slice(0, this.length);
87
- }
91
+ }
92
+
93
+ _pruneColumns() {
94
+ for (const [columnName, column] of Object.entries(this.columns)) {
95
+ this.columns[columnName] = column.slice(0, this.length);
88
96
  }
97
+ }
98
+
89
99
  }
90
- exports.default = ColumnarTableBatchAggregator;
100
+ //# sourceMappingURL=columnar-table-batch-aggregator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../../src/lib/batches/columnar-table-batch-aggregator.ts"],"names":["DEFAULT_ROW_COUNT","ColumnarTableBatchAggregator","constructor","schema","options","_reallocateColumns","rowCount","length","addArrayRow","row","i","fieldName","columns","addObjectRow","getBatch","_pruneColumns","Array","isArray","field","name","index","batch","shape","batchType","data","allocated","ArrayType","type","Float32Array","oldColumn","ArrayBuffer","isView","typedArray","set","columnName","column","Object","entries","slice"],"mappings":";AAMA,MAAMA,iBAAiB,GAAG,GAA1B;AAEA,eAAe,MAAMC,4BAAN,CAAmE;AAMhFC,EAAAA,WAAW,CAACC,MAAD,EAAiBC,OAAjB,EAAqD;AAAA;;AAAA,oCAJ/C,CAI+C;;AAAA,uCAH5C,CAG4C;;AAAA,qCAFvB,EAEuB;;AAC9D,SAAKD,MAAL,GAAcA,MAAd;;AACA,SAAKE,kBAAL;AACD;;AAEDC,EAAAA,QAAQ,GAAW;AACjB,WAAO,KAAKC,MAAZ;AACD;;AAEDC,EAAAA,WAAW,CAACC,GAAD,EAAa;AAEtB,SAAKJ,kBAAL;;AACA,QAAIK,CAAC,GAAG,CAAR;;AAEA,SAAK,MAAMC,SAAX,IAAwB,KAAKC,OAA7B,EAAsC;AACpC,WAAKA,OAAL,CAAaD,SAAb,EAAwB,KAAKJ,MAA7B,IAAuCE,GAAG,CAACC,CAAC,EAAF,CAA1C;AACD;;AACD,SAAKH,MAAL;AACD;;AAEDM,EAAAA,YAAY,CAACJ,GAAD,EAAyC;AAEnD,SAAKJ,kBAAL;;AACA,SAAK,MAAMM,SAAX,IAAwBF,GAAxB,EAA6B;AAC3B,WAAKG,OAAL,CAAaD,SAAb,EAAwB,KAAKJ,MAA7B,IAAuCE,GAAG,CAACE,SAAD,CAA1C;AACD;;AACD,SAAKJ,MAAL;AACD;;AAEDO,EAAAA,QAAQ,GAAgD;AACtD,SAAKC,aAAL;;AACA,UAAMH,OAAO,GAAGI,KAAK,CAACC,OAAN,CAAc,KAAKd,MAAnB,IAA6B,KAAKS,OAAlC,GAA4C,EAA5D;;AAKA,QAAI,CAACI,KAAK,CAACC,OAAN,CAAc,KAAKd,MAAnB,CAAL,EAAiC;AAC/B,WAAK,MAAMQ,SAAX,IAAwB,KAAKR,MAA7B,EAAqC;AACnC,cAAMe,KAAK,GAAG,KAAKf,MAAL,CAAYQ,SAAZ,CAAd;AACAC,QAAAA,OAAO,CAACM,KAAK,CAACC,IAAP,CAAP,GAAsB,KAAKP,OAAL,CAAaM,KAAK,CAACE,KAAnB,CAAtB;AACD;AACF;;AAED,SAAKR,OAAL,GAAe,EAAf;AAEA,UAAMS,KAAyB,GAAG;AAChCC,MAAAA,KAAK,EAAE,gBADyB;AAEhCC,MAAAA,SAAS,EAAE,MAFqB;AAGhCC,MAAAA,IAAI,EAAEZ,OAH0B;AAIhCT,MAAAA,MAAM,EAAE,KAAKA,MAJmB;AAKhCI,MAAAA,MAAM,EAAE,KAAKA;AALmB,KAAlC;AAQA,WAAOc,KAAP;AACD;;AAIDhB,EAAAA,kBAAkB,GAAG;AACnB,QAAI,KAAKE,MAAL,GAAc,KAAKkB,SAAvB,EAAkC;AAChC;AACD;;AAGD,SAAKA,SAAL,GAAiB,KAAKA,SAAL,GAAiB,CAAjB,GAAsB,KAAKA,SAAL,IAAkB,CAAxC,GAA6CzB,iBAA9D;AACA,SAAKY,OAAL,GAAe,EAAf;;AAEA,SAAK,MAAMD,SAAX,IAAwB,KAAKR,MAA7B,EAAqC;AACnC,YAAMe,KAAK,GAAG,KAAKf,MAAL,CAAYQ,SAAZ,CAAd;AACA,YAAMe,SAAS,GAAGR,KAAK,CAACS,IAAN,IAAcC,YAAhC;AACA,YAAMC,SAAS,GAAG,KAAKjB,OAAL,CAAaM,KAAK,CAACE,KAAnB,CAAlB;;AAEA,UAAIS,SAAS,IAAIC,WAAW,CAACC,MAAZ,CAAmBF,SAAnB,CAAjB,EAAgD;AAE9C,cAAMG,UAAU,GAAG,IAAIN,SAAJ,CAAc,KAAKD,SAAnB,CAAnB;AACAO,QAAAA,UAAU,CAACC,GAAX,CAAeJ,SAAf;AACA,aAAKjB,OAAL,CAAaM,KAAK,CAACE,KAAnB,IAA4BY,UAA5B;AACD,OALD,MAKO,IAAIH,SAAJ,EAAe;AAEpBA,QAAAA,SAAS,CAACtB,MAAV,GAAmB,KAAKkB,SAAxB;AACA,aAAKb,OAAL,CAAaM,KAAK,CAACE,KAAnB,IAA4BS,SAA5B;AACD,OAJM,MAIA;AAEL,aAAKjB,OAAL,CAAaM,KAAK,CAACE,KAAnB,IAA4B,IAAIM,SAAJ,CAAc,KAAKD,SAAnB,CAA5B;AACD;AACF;AACF;;AAEDV,EAAAA,aAAa,GAAG;AACd,SAAK,MAAM,CAACmB,UAAD,EAAaC,MAAb,CAAX,IAAmCC,MAAM,CAACC,OAAP,CAAe,KAAKzB,OAApB,CAAnC,EAAiE;AAC/D,WAAKA,OAAL,CAAasB,UAAb,IAA2BC,MAAM,CAACG,KAAP,CAAa,CAAb,EAAgB,KAAK/B,MAArB,CAA3B;AACD;AACF;;AAlG+E","sourcesContent":["import type {Schema} from '../schema/schema';\nimport type {ColumnarTableBatch, ArrowTableBatch} from '../../category/table/table-types';\nimport {TableBatchAggregator} from './table-batch-aggregator';\n\ntype ColumnarTableBatchOptions = {};\n\nconst DEFAULT_ROW_COUNT = 100;\n\nexport default class ColumnarTableBatchAggregator implements TableBatchAggregator {\n schema: Schema;\n length: number = 0;\n allocated: number = 0;\n columns: {[columnName: string]: any[]} = {};\n\n constructor(schema: Schema, options: ColumnarTableBatchOptions) {\n this.schema = schema;\n this._reallocateColumns();\n }\n\n rowCount(): number {\n return this.length;\n }\n\n addArrayRow(row: any[]) {\n // If user keeps pushing rows beyond batch size, reallocate\n this._reallocateColumns();\n let i = 0;\n // TODO what if no csv header, columns not populated?\n for (const fieldName in this.columns) {\n this.columns[fieldName][this.length] = row[i++];\n }\n this.length++;\n }\n\n addObjectRow(row: {[columnName: string]: any}): void {\n // If user keeps pushing rows beyond batch size, reallocate\n this._reallocateColumns();\n for (const fieldName in row) {\n this.columns[fieldName][this.length] = row[fieldName];\n }\n this.length++;\n }\n\n getBatch(): ColumnarTableBatch | ArrowTableBatch | null {\n this._pruneColumns();\n const columns = Array.isArray(this.schema) ? this.columns : {};\n\n // schema is an array if there're no headers\n // object if there are headers\n // columns should match schema format\n if (!Array.isArray(this.schema)) {\n for (const fieldName in this.schema) {\n const field = this.schema[fieldName];\n columns[field.name] = this.columns[field.index];\n }\n }\n\n this.columns = {};\n\n const batch: ColumnarTableBatch = {\n shape: 'columnar-table',\n batchType: 'data',\n data: columns,\n schema: this.schema,\n length: this.length\n };\n\n return batch;\n }\n\n // HELPERS\n\n _reallocateColumns() {\n if (this.length < this.allocated) {\n return;\n }\n\n // @ts-ignore TODO\n this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;\n this.columns = {};\n\n for (const fieldName in this.schema) {\n const field = this.schema[fieldName];\n const ArrayType = field.type || Float32Array;\n const oldColumn = this.columns[field.index];\n\n if (oldColumn && ArrayBuffer.isView(oldColumn)) {\n // Copy the old data to the new array\n const typedArray = new ArrayType(this.allocated);\n typedArray.set(oldColumn);\n this.columns[field.index] = typedArray;\n } else if (oldColumn) {\n // Plain array\n oldColumn.length = this.allocated;\n this.columns[field.index] = oldColumn;\n } else {\n // Create new\n this.columns[field.index] = new ArrayType(this.allocated);\n }\n }\n }\n\n _pruneColumns() {\n for (const [columnName, column] of Object.entries(this.columns)) {\n this.columns[columnName] = column.slice(0, this.length);\n }\n }\n}\n"],"file":"columnar-table-batch-aggregator.js"}
@@ -1,79 +1,95 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- // import type {ArrayRowTableBatch, ObjectRowTableBatch} from '../../category/table.js';
4
- const row_utils_1 = require("../utils/row-utils");
1
+ import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
2
+ import { convertToArrayRow, convertToObjectRow } from '../utils/row-utils';
5
3
  const DEFAULT_ROW_COUNT = 100;
6
- class RowTableBatchAggregator {
7
- constructor(schema, options) {
8
- this.length = 0;
9
- this.objectRows = null;
10
- this.arrayRows = null;
11
- this.cursor = 0;
12
- this._headers = [];
13
- this.options = options;
14
- this.schema = schema;
15
- // schema is an array if there're no headers
16
- // object if there are headers
17
- if (!Array.isArray(schema)) {
18
- this._headers = [];
19
- for (const key in schema) {
20
- this._headers[schema[key].index] = schema[key].name;
21
- }
22
- }
4
+ export default class RowTableBatchAggregator {
5
+ constructor(schema, options) {
6
+ _defineProperty(this, "schema", void 0);
7
+
8
+ _defineProperty(this, "options", void 0);
9
+
10
+ _defineProperty(this, "length", 0);
11
+
12
+ _defineProperty(this, "objectRows", null);
13
+
14
+ _defineProperty(this, "arrayRows", null);
15
+
16
+ _defineProperty(this, "cursor", 0);
17
+
18
+ _defineProperty(this, "_headers", []);
19
+
20
+ this.options = options;
21
+ this.schema = schema;
22
+
23
+ if (!Array.isArray(schema)) {
24
+ this._headers = [];
25
+
26
+ for (const key in schema) {
27
+ this._headers[schema[key].index] = schema[key].name;
28
+ }
23
29
  }
24
- rowCount() {
25
- return this.length;
30
+ }
31
+
32
+ rowCount() {
33
+ return this.length;
34
+ }
35
+
36
+ addArrayRow(row, cursor) {
37
+ if (Number.isFinite(cursor)) {
38
+ this.cursor = cursor;
26
39
  }
27
- addArrayRow(row, cursor) {
28
- if (Number.isFinite(cursor)) {
29
- this.cursor = cursor;
30
- }
31
- // eslint-disable-next-line default-case
32
- switch (this.options.shape) {
33
- case 'object-row-table':
34
- const rowObject = (0, row_utils_1.convertToObjectRow)(row, this._headers);
35
- this.addObjectRow(rowObject, cursor);
36
- break;
37
- case 'array-row-table':
38
- this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);
39
- this.arrayRows[this.length] = row;
40
- this.length++;
41
- break;
42
- }
40
+
41
+ switch (this.options.shape) {
42
+ case 'object-row-table':
43
+ const rowObject = convertToObjectRow(row, this._headers);
44
+ this.addObjectRow(rowObject, cursor);
45
+ break;
46
+
47
+ case 'array-row-table':
48
+ this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);
49
+ this.arrayRows[this.length] = row;
50
+ this.length++;
51
+ break;
43
52
  }
44
- addObjectRow(row, cursor) {
45
- if (Number.isFinite(cursor)) {
46
- this.cursor = cursor;
47
- }
48
- // eslint-disable-next-line default-case
49
- switch (this.options.shape) {
50
- case 'array-row-table':
51
- const rowArray = (0, row_utils_1.convertToArrayRow)(row, this._headers);
52
- this.addArrayRow(rowArray, cursor);
53
- break;
54
- case 'object-row-table':
55
- this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);
56
- this.objectRows[this.length] = row;
57
- this.length++;
58
- break;
59
- }
53
+ }
54
+
55
+ addObjectRow(row, cursor) {
56
+ if (Number.isFinite(cursor)) {
57
+ this.cursor = cursor;
60
58
  }
61
- getBatch() {
62
- let rows = this.arrayRows || this.objectRows;
63
- if (!rows) {
64
- return null;
65
- }
66
- rows = rows.slice(0, this.length);
67
- this.arrayRows = null;
68
- this.objectRows = null;
69
- return {
70
- shape: this.options.shape,
71
- batchType: 'data',
72
- data: rows,
73
- length: this.length,
74
- schema: this.schema,
75
- cursor: this.cursor
76
- };
59
+
60
+ switch (this.options.shape) {
61
+ case 'array-row-table':
62
+ const rowArray = convertToArrayRow(row, this._headers);
63
+ this.addArrayRow(rowArray, cursor);
64
+ break;
65
+
66
+ case 'object-row-table':
67
+ this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);
68
+ this.objectRows[this.length] = row;
69
+ this.length++;
70
+ break;
77
71
  }
72
+ }
73
+
74
+ getBatch() {
75
+ let rows = this.arrayRows || this.objectRows;
76
+
77
+ if (!rows) {
78
+ return null;
79
+ }
80
+
81
+ rows = rows.slice(0, this.length);
82
+ this.arrayRows = null;
83
+ this.objectRows = null;
84
+ return {
85
+ shape: this.options.shape,
86
+ batchType: 'data',
87
+ data: rows,
88
+ length: this.length,
89
+ schema: this.schema,
90
+ cursor: this.cursor
91
+ };
92
+ }
93
+
78
94
  }
79
- exports.default = RowTableBatchAggregator;
95
+ //# sourceMappingURL=row-table-batch-aggregator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../../src/lib/batches/row-table-batch-aggregator.ts"],"names":["convertToArrayRow","convertToObjectRow","DEFAULT_ROW_COUNT","RowTableBatchAggregator","constructor","schema","options","Array","isArray","_headers","key","index","name","rowCount","length","addArrayRow","row","cursor","Number","isFinite","shape","rowObject","addObjectRow","arrayRows","rowArray","objectRows","getBatch","rows","slice","batchType","data"],"mappings":";AAGA,SAAQA,iBAAR,EAA2BC,kBAA3B,QAAoD,oBAApD;AAGA,MAAMC,iBAAiB,GAAG,GAA1B;AAEA,eAAe,MAAMC,uBAAN,CAA8D;AAU3EC,EAAAA,WAAW,CAACC,MAAD,EAAiBC,OAAjB,EAA6C;AAAA;;AAAA;;AAAA,oCANvC,CAMuC;;AAAA,wCALP,IAKO;;AAAA,uCAJ9B,IAI8B;;AAAA,oCAHvC,CAGuC;;AAAA,sCAF3B,EAE2B;;AACtD,SAAKA,OAAL,GAAeA,OAAf;AACA,SAAKD,MAAL,GAAcA,MAAd;;AAIA,QAAI,CAACE,KAAK,CAACC,OAAN,CAAcH,MAAd,CAAL,EAA4B;AAC1B,WAAKI,QAAL,GAAgB,EAAhB;;AACA,WAAK,MAAMC,GAAX,IAAkBL,MAAlB,EAA0B;AACxB,aAAKI,QAAL,CAAcJ,MAAM,CAACK,GAAD,CAAN,CAAYC,KAA1B,IAAmCN,MAAM,CAACK,GAAD,CAAN,CAAYE,IAA/C;AACD;AACF;AACF;;AAEDC,EAAAA,QAAQ,GAAW;AACjB,WAAO,KAAKC,MAAZ;AACD;;AAEDC,EAAAA,WAAW,CAACC,GAAD,EAAaC,MAAb,EAAoC;AAC7C,QAAIC,MAAM,CAACC,QAAP,CAAgBF,MAAhB,CAAJ,EAA6B;AAC3B,WAAKA,MAAL,GAAcA,MAAd;AACD;;AAGD,YAAQ,KAAKX,OAAL,CAAac,KAArB;AACE,WAAK,kBAAL;AACE,cAAMC,SAAS,GAAGpB,kBAAkB,CAACe,GAAD,EAAM,KAAKP,QAAX,CAApC;AACA,aAAKa,YAAL,CAAkBD,SAAlB,EAA6BJ,MAA7B;AACA;;AACF,WAAK,iBAAL;AACE,aAAKM,SAAL,GAAiB,KAAKA,SAAL,IAAkB,IAAIhB,KAAJ,CAAUL,iBAAV,CAAnC;AACA,aAAKqB,SAAL,CAAe,KAAKT,MAApB,IAA8BE,GAA9B;AACA,aAAKF,MAAL;AACA;AATJ;AAWD;;AAEDQ,EAAAA,YAAY,CAACN,GAAD,EAAmCC,MAAnC,EAA0D;AACpE,QAAIC,MAAM,CAACC,QAAP,CAAgBF,MAAhB,CAAJ,EAA6B;AAC3B,WAAKA,MAAL,GAAcA,MAAd;AACD;;AAGD,YAAQ,KAAKX,OAAL,CAAac,KAArB;AACE,WAAK,iBAAL;AACE,cAAMI,QAAQ,GAAGxB,iBAAiB,CAACgB,GAAD,EAAM,KAAKP,QAAX,CAAlC;AACA,aAAKM,WAAL,CAAiBS,QAAjB,EAA2BP,MAA3B;AACA;;AACF,WAAK,kBAAL;AACE,aAAKQ,UAAL,GAAkB,KAAKA,UAAL,IAAmB,IAAIlB,KAAJ,CAAUL,iBAAV,CAArC;AACA,aAAKuB,UAAL,CAAgB,KAAKX,MAArB,IAA+BE,GAA/B;AACA,aAAKF,MAAL;AACA;AATJ;AAWD;;AAEDY,EAAAA,QAAQ,GAAsB;AAC5B,QAAIC,IAAI,GAAG,KAAKJ,SAAL,IAAkB,KAAKE,UAAlC;;AACA,QAAI,CAACE,IAAL,EAAW;AACT,aAAO,IAAP;AACD;;AAEDA,IAAAA,IAAI,GAAGA,IAAI,CAACC,KAAL,CAAW,CAAX,EAAc,KAAKd,MAAnB,CAAP;AACA,SAAKS,SAAL,GAAiB,IAAjB;AACA,SAAKE,UAAL,GAAkB,IAAlB;AAEA,WAAO;AACLL,MAAAA,KAAK,EAAE,KAAKd,OAAL,CAAac,KADf;AAELS,MAAAA,SAAS,EAAE,MAFN;AAGLC,MAAAA,IAAI,EAAEH,IAHD;AAILb,MAAAA,MAAM,EAAE,KAAKA,MAJR;AAKLT,MAAAA,MAAM,EAAE,KAAKA,MALR;AAMLY,MAAAA,MAAM,EAAE,KAAKA;AANR,KAAP;AAQD;;AApF0E","sourcesContent":["import type {Schema} from '../schema/schema';\nimport type {TableBatch} from '../../category/table/table-types';\n// import type {ArrayRowTableBatch, ObjectRowTableBatch} from '../../category/table';\nimport {convertToArrayRow, convertToObjectRow} from '../utils/row-utils';\nimport {TableBatchAggregator, TableBatchOptions} from './table-batch-aggregator';\n\nconst DEFAULT_ROW_COUNT = 100;\n\nexport default class RowTableBatchAggregator implements TableBatchAggregator {\n schema: Schema;\n options: TableBatchOptions;\n\n length: number = 0;\n objectRows: {[columnName: string]: any} | null = null;\n arrayRows: any[] | null = null;\n cursor: number = 0;\n private _headers: string[] = [];\n\n constructor(schema: Schema, options: TableBatchOptions) {\n this.options = options;\n this.schema = schema;\n\n // schema is an array if there're no headers\n // object if there are headers\n if (!Array.isArray(schema)) {\n this._headers = [];\n for (const key in schema) {\n this._headers[schema[key].index] = schema[key].name;\n }\n }\n }\n\n rowCount(): number {\n return this.length;\n }\n\n addArrayRow(row: any[], cursor?: number): void {\n if (Number.isFinite(cursor)) {\n this.cursor = cursor as number;\n }\n\n // eslint-disable-next-line default-case\n switch (this.options.shape) {\n case 'object-row-table':\n const rowObject = convertToObjectRow(row, this._headers);\n this.addObjectRow(rowObject, cursor);\n break;\n case 'array-row-table':\n this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT);\n this.arrayRows[this.length] = row;\n this.length++;\n break;\n }\n }\n\n addObjectRow(row: {[columnName: string]: any}, cursor?: number): void {\n if (Number.isFinite(cursor)) {\n this.cursor = cursor as number;\n }\n\n // eslint-disable-next-line default-case\n switch (this.options.shape) {\n case 'array-row-table':\n const rowArray = convertToArrayRow(row, this._headers);\n this.addArrayRow(rowArray, cursor);\n break;\n case 'object-row-table':\n this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT);\n this.objectRows[this.length] = row;\n this.length++;\n break;\n }\n }\n\n getBatch(): TableBatch | null {\n let rows = this.arrayRows || this.objectRows;\n if (!rows) {\n return null;\n }\n\n rows = rows.slice(0, this.length);\n this.arrayRows = null;\n this.objectRows = null;\n\n return {\n shape: this.options.shape,\n batchType: 'data',\n data: rows,\n length: this.length,\n schema: this.schema,\n cursor: this.cursor\n };\n }\n}\n"],"file":"row-table-batch-aggregator.js"}
@@ -1,2 +1,2 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
1
+ export {};
2
+ //# sourceMappingURL=table-batch-aggregator.js.map