@loaders.gl/schema-utils 4.4.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/LICENSE +41 -0
  2. package/README.md +9 -0
  3. package/dist/dist.dev.js +14269 -0
  4. package/dist/dist.min.js +18 -0
  5. package/dist/index.cjs +2436 -0
  6. package/dist/index.cjs.map +7 -0
  7. package/dist/index.d.ts +28 -0
  8. package/dist/index.d.ts.map +1 -0
  9. package/dist/index.js +73 -0
  10. package/dist/lib/arrow-utils/arrow-fixed-size-list-utils.d.ts +11 -0
  11. package/dist/lib/arrow-utils/arrow-fixed-size-list-utils.d.ts.map +1 -0
  12. package/dist/lib/arrow-utils/arrow-fixed-size-list-utils.js +40 -0
  13. package/dist/lib/arrow-utils/arrow-list-of-fixed-size-list-utils.d.ts +11 -0
  14. package/dist/lib/arrow-utils/arrow-list-of-fixed-size-list-utils.d.ts.map +1 -0
  15. package/dist/lib/arrow-utils/arrow-list-of-fixed-size-list-utils.js +32 -0
  16. package/dist/lib/mesh/convert-mesh-to-table.d.ts +18 -0
  17. package/dist/lib/mesh/convert-mesh-to-table.d.ts.map +1 -0
  18. package/dist/lib/mesh/convert-mesh-to-table.js +60 -0
  19. package/dist/lib/mesh/convert-table-to-mesh.d.ts +7 -0
  20. package/dist/lib/mesh/convert-table-to-mesh.d.ts.map +1 -0
  21. package/dist/lib/mesh/convert-table-to-mesh.js +40 -0
  22. package/dist/lib/mesh/deduce-mesh-schema.d.ts +23 -0
  23. package/dist/lib/mesh/deduce-mesh-schema.d.ts.map +1 -0
  24. package/dist/lib/mesh/deduce-mesh-schema.js +62 -0
  25. package/dist/lib/mesh/mesh-utils.d.ts +23 -0
  26. package/dist/lib/mesh/mesh-utils.d.ts.map +1 -0
  27. package/dist/lib/mesh/mesh-utils.js +46 -0
  28. package/dist/lib/schema/convert-arrow-schema.d.ts +23 -0
  29. package/dist/lib/schema/convert-arrow-schema.d.ts.map +1 -0
  30. package/dist/lib/schema/convert-arrow-schema.js +280 -0
  31. package/dist/lib/schema/data-type.d.ts +16 -0
  32. package/dist/lib/schema/data-type.d.ts.map +1 -0
  33. package/dist/lib/schema/data-type.js +88 -0
  34. package/dist/lib/schema/deduce-table-schema.d.ts +14 -0
  35. package/dist/lib/schema/deduce-table-schema.d.ts.map +1 -0
  36. package/dist/lib/schema/deduce-table-schema.js +100 -0
  37. package/dist/lib/table/arrow-api/arrow-like-field.d.ts +19 -0
  38. package/dist/lib/table/arrow-api/arrow-like-field.d.ts.map +1 -0
  39. package/dist/lib/table/arrow-api/arrow-like-field.js +36 -0
  40. package/dist/lib/table/arrow-api/arrow-like-interface.d.ts +43 -0
  41. package/dist/lib/table/arrow-api/arrow-like-interface.d.ts.map +1 -0
  42. package/dist/lib/table/arrow-api/arrow-like-interface.js +4 -0
  43. package/dist/lib/table/arrow-api/arrow-like-schema.d.ts +12 -0
  44. package/dist/lib/table/arrow-api/arrow-like-schema.d.ts.map +1 -0
  45. package/dist/lib/table/arrow-api/arrow-like-schema.js +81 -0
  46. package/dist/lib/table/arrow-api/arrow-like-table.d.ts +28 -0
  47. package/dist/lib/table/arrow-api/arrow-like-table.d.ts.map +1 -0
  48. package/dist/lib/table/arrow-api/arrow-like-table.js +63 -0
  49. package/dist/lib/table/arrow-api/arrow-like-type.d.ts +177 -0
  50. package/dist/lib/table/arrow-api/arrow-like-type.d.ts.map +1 -0
  51. package/dist/lib/table/arrow-api/arrow-like-type.js +442 -0
  52. package/dist/lib/table/arrow-api/enum.d.ts +92 -0
  53. package/dist/lib/table/arrow-api/enum.d.ts.map +1 -0
  54. package/dist/lib/table/arrow-api/enum.js +97 -0
  55. package/dist/lib/table/arrow-api/get-type-info.d.ts +13 -0
  56. package/dist/lib/table/arrow-api/get-type-info.d.ts.map +1 -0
  57. package/dist/lib/table/arrow-api/get-type-info.js +27 -0
  58. package/dist/lib/table/arrow-api/index.d.ts +5 -0
  59. package/dist/lib/table/arrow-api/index.d.ts.map +1 -0
  60. package/dist/lib/table/arrow-api/index.js +7 -0
  61. package/dist/lib/table/batch-builder/arrow-table-batch-aggregator.d.ts +10 -0
  62. package/dist/lib/table/batch-builder/arrow-table-batch-aggregator.d.ts.map +1 -0
  63. package/dist/lib/table/batch-builder/arrow-table-batch-aggregator.js +67 -0
  64. package/dist/lib/table/batch-builder/arrow-table-builder.d.ts +25 -0
  65. package/dist/lib/table/batch-builder/arrow-table-builder.d.ts.map +1 -0
  66. package/dist/lib/table/batch-builder/arrow-table-builder.js +99 -0
  67. package/dist/lib/table/batch-builder/base-table-batch-aggregator.d.ts +19 -0
  68. package/dist/lib/table/batch-builder/base-table-batch-aggregator.d.ts.map +1 -0
  69. package/dist/lib/table/batch-builder/base-table-batch-aggregator.js +63 -0
  70. package/dist/lib/table/batch-builder/columnar-table-batch-aggregator.d.ts +20 -0
  71. package/dist/lib/table/batch-builder/columnar-table-batch-aggregator.d.ts.map +1 -0
  72. package/dist/lib/table/batch-builder/columnar-table-batch-aggregator.js +83 -0
  73. package/dist/lib/table/batch-builder/row-table-batch-aggregator.d.ts +21 -0
  74. package/dist/lib/table/batch-builder/row-table-batch-aggregator.d.ts.map +1 -0
  75. package/dist/lib/table/batch-builder/row-table-batch-aggregator.js +85 -0
  76. package/dist/lib/table/batch-builder/table-batch-aggregator.d.ts +24 -0
  77. package/dist/lib/table/batch-builder/table-batch-aggregator.d.ts.map +1 -0
  78. package/dist/lib/table/batch-builder/table-batch-aggregator.js +4 -0
  79. package/dist/lib/table/batch-builder/table-batch-builder.d.ts +52 -0
  80. package/dist/lib/table/batch-builder/table-batch-builder.d.ts.map +1 -0
  81. package/dist/lib/table/batch-builder/table-batch-builder.js +148 -0
  82. package/dist/lib/table/batches/convert-batches.d.ts +10 -0
  83. package/dist/lib/table/batches/convert-batches.d.ts.map +1 -0
  84. package/dist/lib/table/batches/convert-batches.js +45 -0
  85. package/dist/lib/table/batches/make-arrow-batch-iterator.d.ts +17 -0
  86. package/dist/lib/table/batches/make-arrow-batch-iterator.d.ts.map +1 -0
  87. package/dist/lib/table/batches/make-arrow-batch-iterator.js +57 -0
  88. package/dist/lib/table/batches/make-table-batch-iterator.d.ts +15 -0
  89. package/dist/lib/table/batches/make-table-batch-iterator.d.ts.map +1 -0
  90. package/dist/lib/table/batches/make-table-batch-iterator.js +21 -0
  91. package/dist/lib/table/batches/make-table-from-batches.d.ts +9 -0
  92. package/dist/lib/table/batches/make-table-from-batches.d.ts.map +1 -0
  93. package/dist/lib/table/batches/make-table-from-batches.js +62 -0
  94. package/dist/lib/table/tables/convert-arrow-table.d.ts +24 -0
  95. package/dist/lib/table/tables/convert-arrow-table.d.ts.map +1 -0
  96. package/dist/lib/table/tables/convert-arrow-table.js +150 -0
  97. package/dist/lib/table/tables/convert-table.d.ts +43 -0
  98. package/dist/lib/table/tables/convert-table.d.ts.map +1 -0
  99. package/dist/lib/table/tables/convert-table.js +120 -0
  100. package/dist/lib/table/tables/make-table.d.ts +67 -0
  101. package/dist/lib/table/tables/make-table.d.ts.map +1 -0
  102. package/dist/lib/table/tables/make-table.js +101 -0
  103. package/dist/lib/table/tables/row-utils.d.ts +15 -0
  104. package/dist/lib/table/tables/row-utils.d.ts.map +1 -0
  105. package/dist/lib/table/tables/row-utils.js +49 -0
  106. package/dist/lib/table/tables/table-accessors.d.ts +68 -0
  107. package/dist/lib/table/tables/table-accessors.d.ts.map +1 -0
  108. package/dist/lib/table/tables/table-accessors.js +350 -0
  109. package/dist/lib/table/tables/table-column.d.ts +7 -0
  110. package/dist/lib/table/tables/table-column.d.ts.map +1 -0
  111. package/dist/lib/table/tables/table-column.js +39 -0
  112. package/dist/lib/table/tables/table-types.d.ts +12 -0
  113. package/dist/lib/table/tables/table-types.d.ts.map +1 -0
  114. package/dist/lib/table/tables/table-types.js +23 -0
  115. package/dist/lib/utils/assert.d.ts +2 -0
  116. package/dist/lib/utils/assert.d.ts.map +1 -0
  117. package/dist/lib/utils/assert.js +8 -0
  118. package/dist/lib/utils/async-queue.d.ts +18 -0
  119. package/dist/lib/utils/async-queue.d.ts.map +1 -0
  120. package/dist/lib/utils/async-queue.js +90 -0
  121. package/dist/types/types.d.ts +15 -0
  122. package/dist/types/types.d.ts.map +1 -0
  123. package/dist/types/types.js +4 -0
  124. package/package.json +52 -0
  125. package/src/index.ts +134 -0
  126. package/src/lib/arrow-utils/arrow-fixed-size-list-utils.ts +63 -0
  127. package/src/lib/arrow-utils/arrow-list-of-fixed-size-list-utils.ts +48 -0
  128. package/src/lib/mesh/convert-mesh-to-table.ts +77 -0
  129. package/src/lib/mesh/convert-table-to-mesh.ts +49 -0
  130. package/src/lib/mesh/deduce-mesh-schema.ts +75 -0
  131. package/src/lib/mesh/mesh-utils.ts +64 -0
  132. package/src/lib/schema/convert-arrow-schema.ts +301 -0
  133. package/src/lib/schema/data-type.ts +103 -0
  134. package/src/lib/schema/deduce-table-schema.ts +119 -0
  135. package/src/lib/table/arrow-api/arrow-like-field.ts +53 -0
  136. package/src/lib/table/arrow-api/arrow-like-interface.ts +84 -0
  137. package/src/lib/table/arrow-api/arrow-like-schema.ts +101 -0
  138. package/src/lib/table/arrow-api/arrow-like-table.ts +85 -0
  139. package/src/lib/table/arrow-api/arrow-like-type.ts +502 -0
  140. package/src/lib/table/arrow-api/enum.ts +99 -0
  141. package/src/lib/table/arrow-api/get-type-info.ts +38 -0
  142. package/src/lib/table/arrow-api/index.ts +8 -0
  143. package/src/lib/table/batch-builder/arrow-table-batch-aggregator-builders.ts.disabled +190 -0
  144. package/src/lib/table/batch-builder/arrow-table-batch-aggregator.ts +82 -0
  145. package/src/lib/table/batch-builder/arrow-table-builder.ts +112 -0
  146. package/src/lib/table/batch-builder/base-table-batch-aggregator.ts +82 -0
  147. package/src/lib/table/batch-builder/columnar-table-batch-aggregator.ts +99 -0
  148. package/src/lib/table/batch-builder/row-table-batch-aggregator.ts +108 -0
  149. package/src/lib/table/batch-builder/table-batch-aggregator.ts +28 -0
  150. package/src/lib/table/batch-builder/table-batch-builder.ts +185 -0
  151. package/src/lib/table/batches/convert-batches.ts +83 -0
  152. package/src/lib/table/batches/make-arrow-batch-iterator.ts +75 -0
  153. package/src/lib/table/batches/make-table-batch-iterator.ts +25 -0
  154. package/src/lib/table/batches/make-table-from-batches.ts +84 -0
  155. package/src/lib/table/tables/convert-arrow-table.ts +198 -0
  156. package/src/lib/table/tables/convert-table.ts +151 -0
  157. package/src/lib/table/tables/make-table.ts +127 -0
  158. package/src/lib/table/tables/row-utils.ts +59 -0
  159. package/src/lib/table/tables/table-accessors.ts +417 -0
  160. package/src/lib/table/tables/table-column.ts +49 -0
  161. package/src/lib/table/tables/table-types.ts +37 -0
  162. package/src/lib/utils/assert.ts +8 -0
  163. package/src/lib/utils/async-queue.ts +104 -0
  164. package/src/types/types.ts +45 -0
@@ -0,0 +1,150 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+ import * as arrow from 'apache-arrow';
5
+ import { convertTable } from "./convert-table.js";
6
+ import { convertArrowToSchema } from "../../schema/convert-arrow-schema.js";
7
+ import { makeArrowRecordBatchIterator } from "../batches/make-arrow-batch-iterator.js";
8
+ /**
9
+ * * Convert a loaders.gl Table to an Apache Arrow Table
10
+ * @param mesh
11
+ * @param metadata
12
+ * @param batchSize
13
+ * @returns
14
+ */
15
+ export function convertTableToArrow(table, options) {
16
+ switch (table.shape) {
17
+ case 'arrow-table':
18
+ return table.data;
19
+ case 'columnar-table':
20
+ // TODO - optimized implementation is possible
21
+ // return convertColumnarTableToArrow(table, options);
22
+ // fall through
23
+ default:
24
+ const arrowBatchIterator = makeArrowRecordBatchIterator(table, options);
25
+ return new arrow.Table(arrowBatchIterator);
26
+ }
27
+ }
28
+ /**
29
+ * Convert an Apache Arrow table to a loaders.gl Table
30
+ * @note Currently does not convert schema
31
+ */
32
+ export function convertArrowToTable(arrowTable, shape) {
33
+ switch (shape) {
34
+ case 'arrow-table':
35
+ return convertArrowToArrowTable(arrowTable);
36
+ case 'array-row-table':
37
+ return convertArrowToArrayRowTable(arrowTable);
38
+ case 'object-row-table':
39
+ return convertArrowToObjectRowTable(arrowTable);
40
+ case 'columnar-table':
41
+ return convertArrowToColumnarTable(arrowTable);
42
+ case 'geojson-table':
43
+ return convertArrowToGeoJSONTable(arrowTable);
44
+ default:
45
+ throw new Error(shape);
46
+ }
47
+ }
48
+ /**
49
+ * Wrap an apache arrow table in a loaders.gl table wrapper.
50
+ * From this additional conversions are available.
51
+ * @param arrowTable
52
+ * @returns
53
+ */
54
+ function convertArrowToArrowTable(arrowTable) {
55
+ return {
56
+ shape: 'arrow-table',
57
+ schema: convertArrowToSchema(arrowTable.schema),
58
+ data: arrowTable
59
+ };
60
+ }
61
+ function convertArrowToArrayRowTable(arrowTable) {
62
+ const columnarTable = convertArrowToColumnarTable(arrowTable);
63
+ return convertTable(columnarTable, 'array-row-table');
64
+ }
65
+ function convertArrowToObjectRowTable(arrowTable) {
66
+ const columnarTable = convertArrowToColumnarTable(arrowTable);
67
+ return convertTable(columnarTable, 'object-row-table');
68
+ }
69
+ /**
70
+ * Convert an Apache Arrow table to a ColumnarTable
71
+ * @note Currently does not convert schema
72
+ */
73
+ function convertArrowToColumnarTable(arrowTable) {
74
+ // TODO - avoid calling `getColumn` on columns we are not interested in?
75
+ // Add options object?
76
+ const columns = {};
77
+ for (const field of arrowTable.schema.fields) {
78
+ // This (is intended to) coalesce all record batches into a single typed array
79
+ const arrowColumn = arrowTable.getChild(field.name);
80
+ const values = arrowColumn?.toArray();
81
+ columns[field.name] = values;
82
+ }
83
+ const schema = convertArrowToSchema(arrowTable.schema);
84
+ return {
85
+ shape: 'columnar-table',
86
+ schema,
87
+ data: columns
88
+ };
89
+ }
90
+ /**
91
+ * Convert an Apache Arrow table to a GeoJSONTable
92
+ * @note Currently does not convert schema
93
+ */
94
+ function convertArrowToGeoJSONTable(arrowTable) {
95
+ const schema = convertArrowToSchema(arrowTable.schema);
96
+ const features = [];
97
+ for (let row = 0; row < arrowTable.numRows; row++) {
98
+ // parse arrow geometry to geojson feature
99
+ const properties = arrowTable.get(row)?.toJSON() || {};
100
+ features.push({ type: 'Feature', geometry: null, properties });
101
+ }
102
+ return {
103
+ shape: 'geojson-table',
104
+ type: 'FeatureCollection',
105
+ schema,
106
+ features
107
+ };
108
+ }
109
+ // /**
110
+ // * Wrap an apache arrow table in a loaders.gl table wrapper.
111
+ // * From this additional conversions are available.
112
+ // * @param arrowTable
113
+ // * @returns
114
+ // */
115
+ // function convertArrowToArrowTable(arrowTable: arrow.Table): ArrowTable {
116
+ // return {
117
+ // shape: 'arrow-table',
118
+ // schema: convertArrowToSchema(arrowTable.schema),
119
+ // data: arrowTable
120
+ // };
121
+ // }
122
+ // function convertArrowToArrayRowTable(arrowTable: arrow.Table): Table {
123
+ // const columnarTable = convertArrowToColumnarTable(arrowTable);
124
+ // return convertTable(columnarTable, 'array-row-table');
125
+ // }
126
+ // function convertArrowToObjectRowTable(arrowTable: arrow.Table): Table {
127
+ // const columnarTable = convertArrowToColumnarTable(arrowTable);
128
+ // return convertTable(columnarTable, 'object-row-table');
129
+ // }
130
+ // /**
131
+ // * Convert an Apache Arrow table to a ColumnarTable
132
+ // * @note Currently does not convert schema
133
+ // */
134
+ // function convertArrowToColumnarTable(arrowTable: arrow.Table): ColumnarTable {
135
+ // // TODO - avoid calling `getColumn` on columns we are not interested in?
136
+ // // Add options object?
137
+ // const columns: ColumnarTable['data'] = {};
138
+ // for (const field of arrowTable.schema.fields) {
139
+ // // This (is intended to) coalesce all record batches into a single typed array
140
+ // const arrowColumn = arrowTable.getChild(field.name);
141
+ // const values = arrowColumn?.toArray();
142
+ // columns[field.name] = values;
143
+ // }
144
+ // const schema = convertArrowToSchema(arrowTable.schema);
145
+ // return {
146
+ // shape: 'columnar-table',
147
+ // schema,
148
+ // data: columns
149
+ // };
150
+ // }
@@ -0,0 +1,43 @@
1
+ import type { Table, ArrayRowTable, ObjectRowTable, ColumnarTable, ArrowTable } from '@loaders.gl/schema';
2
+ export declare function convertTable(table: Table, shape: 'object-row-table'): ObjectRowTable;
3
+ export declare function convertTable(table: Table, shape: 'array-row-table'): ArrayRowTable;
4
+ export declare function convertTable(table: Table, shape: 'columnar-table'): ColumnarTable;
5
+ export declare function convertTable(table: Table, shape: 'arrow-table'): ArrowTable;
6
+ /** Convert any simple table into columnar format */
7
+ export declare function convertToColumnarTable(table: Table): ColumnarTable;
8
+ /** Convert any table into array row format */
9
+ export declare function convertToArrayRowTable(table: Table): ArrayRowTable;
10
+ /** Convert any table into object row format */
11
+ export declare function convertToObjectRowTable(table: Table): ObjectRowTable;
12
+ /**
13
+ * Convert a table to apache arrow format
14
+ * @note this depends on the `@loaders.gl/arrow module being imported
15
+ */
16
+ export declare function convertToArrowTable(table: Table): ArrowTable;
17
+ /**
18
+ *
19
+ * @note - should be part of schema module
20
+ *
21
+ export function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {
22
+ const tableKeys = Object.keys(columnarTable);
23
+ const tableRowsCount = columnarTable[tableKeys[0]].length;
24
+
25
+ const rowFormatTable: {}[] = [];
26
+
27
+ for (let index = 0; index < tableRowsCount; index++) {
28
+ const tableItem = {};
29
+ for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
30
+ const fieldName = tableKeys[keyIndex];
31
+ tableItem[fieldName] = columnarTable[fieldName][index];
32
+ }
33
+ rowFormatTable.push(tableItem);
34
+ }
35
+
36
+ return {
37
+ shape: 'object-row-table',
38
+ schema: columnarTable.schema,
39
+ data: rowFormatTable
40
+ };
41
+ }
42
+ */
43
+ //# sourceMappingURL=convert-table.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"convert-table.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/tables/convert-table.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,KAAK,EACL,aAAa,EACb,cAAc,EACd,aAAa,EACb,UAAU,EACX,MAAM,oBAAoB,CAAC;AAW5B,wBAAgB,YAAY,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,kBAAkB,GAAG,cAAc,CAAC;AACtF,wBAAgB,YAAY,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,iBAAiB,GAAG,aAAa,CAAC;AACpF,wBAAgB,YAAY,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,gBAAgB,GAAG,aAAa,CAAC;AACnF,wBAAgB,YAAY,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,GAAG,UAAU,CAAC;AA0B7E,oDAAoD;AACpD,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,KAAK,GAAG,aAAa,CAyBlE;AAED,8CAA8C;AAC9C,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,KAAK,GAAG,aAAa,CAclE;AAED,+CAA+C;AAC/C,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,KAAK,GAAG,cAAc,CAcpE;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,KAAK,GAAG,UAAU,CAO5D;AAED;;;;;;;;;;;;;;;;;;;;;;;;;EAyBE"}
@@ -0,0 +1,120 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+ import { getTableCell, getTableLength, getTableRowAsArray, getTableRowAsObject } from "./table-accessors.js";
5
+ import { deduceTableSchema } from "../../schema/deduce-table-schema.js";
6
+ import { makeColumnFromField } from "./table-column.js";
7
+ import { convertTableToArrow } from "./convert-arrow-table.js";
8
+ /**
9
+ * Convert a table to a different shape
10
+ * @param table
11
+ * @param shape
12
+ * @returns
13
+ */
14
+ export function convertTable(table, shape) {
15
+ switch (shape) {
16
+ case 'object-row-table':
17
+ return convertToObjectRowTable(table);
18
+ case 'array-row-table':
19
+ return convertToArrayRowTable(table);
20
+ case 'columnar-table':
21
+ return convertToColumnarTable(table);
22
+ case 'arrow-table':
23
+ return convertToArrowTable(table);
24
+ default:
25
+ throw new Error(shape);
26
+ }
27
+ }
28
+ /** Convert any simple table into columnar format */
29
+ export function convertToColumnarTable(table) {
30
+ // TODO - should schema really be optional?
31
+ const schema = table.schema || deduceTableSchema(table);
32
+ const fields = table.schema?.fields || [];
33
+ if (table.shape === 'columnar-table') {
34
+ return { ...table, schema };
35
+ }
36
+ const length = getTableLength(table);
37
+ const columns = {};
38
+ for (const field of fields) {
39
+ const column = makeColumnFromField(field, length);
40
+ columns[field.name] = column;
41
+ for (let rowIndex = 0; rowIndex < length; rowIndex++) {
42
+ column[rowIndex] = getTableCell(table, rowIndex, field.name);
43
+ }
44
+ }
45
+ return {
46
+ shape: 'columnar-table',
47
+ schema,
48
+ data: columns
49
+ };
50
+ }
51
+ /** Convert any table into array row format */
52
+ export function convertToArrayRowTable(table) {
53
+ if (table.shape === 'array-row-table') {
54
+ return table;
55
+ }
56
+ const length = getTableLength(table);
57
+ const data = new Array(length);
58
+ for (let rowIndex = 0; rowIndex < length; rowIndex++) {
59
+ data[rowIndex] = getTableRowAsArray(table, rowIndex);
60
+ }
61
+ return {
62
+ shape: 'array-row-table',
63
+ schema: table.schema,
64
+ data
65
+ };
66
+ }
67
+ /** Convert any table into object row format */
68
+ export function convertToObjectRowTable(table) {
69
+ if (table.shape === 'object-row-table') {
70
+ return table;
71
+ }
72
+ const length = getTableLength(table);
73
+ const data = new Array(length);
74
+ for (let rowIndex = 0; rowIndex < length; rowIndex++) {
75
+ data[rowIndex] = getTableRowAsObject(table, rowIndex);
76
+ }
77
+ return {
78
+ shape: 'object-row-table',
79
+ schema: table.schema,
80
+ data
81
+ };
82
+ }
83
+ /**
84
+ * Convert a table to apache arrow format
85
+ * @note this depends on the `@loaders.gl/arrow module being imported
86
+ */
87
+ export function convertToArrowTable(table) {
88
+ const arrowTable = convertTableToArrow(table);
89
+ return {
90
+ shape: 'arrow-table',
91
+ schema: table.schema,
92
+ data: arrowTable
93
+ };
94
+ }
95
+ /**
96
+ *
97
+ * @note - should be part of schema module
98
+ *
99
+ export function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {
100
+ const tableKeys = Object.keys(columnarTable);
101
+ const tableRowsCount = columnarTable[tableKeys[0]].length;
102
+
103
+ const rowFormatTable: {}[] = [];
104
+
105
+ for (let index = 0; index < tableRowsCount; index++) {
106
+ const tableItem = {};
107
+ for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
108
+ const fieldName = tableKeys[keyIndex];
109
+ tableItem[fieldName] = columnarTable[fieldName][index];
110
+ }
111
+ rowFormatTable.push(tableItem);
112
+ }
113
+
114
+ return {
115
+ shape: 'object-row-table',
116
+ schema: columnarTable.schema,
117
+ data: rowFormatTable
118
+ };
119
+ }
120
+ */
@@ -0,0 +1,67 @@
1
+ import * as arrow from 'apache-arrow';
2
+ import type { ArrayRowTable, ObjectRowTable, ColumnarTable, ArrowTable } from '@loaders.gl/schema';
3
+ /**
4
+ * Makes a typed table from data.
5
+ * @throws Row tables must contain at least one row. Columnar tables must contain empty arrays
6
+ */
7
+ export declare function makeTableFromData(data: unknown[][]): ArrayRowTable;
8
+ export declare function makeTableFromData(data: {
9
+ [column: string]: unknown;
10
+ }[]): ObjectRowTable;
11
+ export declare function makeTableFromData(data: {
12
+ [column: string]: ArrayLike<unknown>;
13
+ }): ColumnarTable;
14
+ export declare function makeTableFromData(data: arrow.Table): ArrowTable;
15
+ /** Convert any table into object row format *
16
+ export function makeColumnarTable(table: Table): ColumnarTable {
17
+ if (table.shape === 'columnar-table') {
18
+ return table;
19
+ }
20
+ const length = getTableLength(table);
21
+ const data = new Array<{[key: string]: unknown}>(length);
22
+ for (let rowIndex = 0; rowIndex < length; rowIndex++) {
23
+ data[rowIndex] = getTableRowAsObject(table, rowIndex);
24
+ }
25
+ return {
26
+ shape: 'columnar-table',
27
+ schema: table.schema,
28
+ data
29
+ };
30
+ }
31
+
32
+
33
+ /** Convert any table into array row format *
34
+ export function makeArrayRowTable(table: TableLike): ArrayRowTable {
35
+ if (table.shape === 'array-row-table') {
36
+ return table;
37
+ }
38
+ const length = getTableLength(table);
39
+ const data = new Array<unknown[]>(length);
40
+ for (let rowIndex = 0; rowIndex < length; rowIndex++) {
41
+ data[rowIndex] = getTableRowAsArray(table, rowIndex);
42
+ }
43
+ return {
44
+ shape: 'array-row-table',
45
+ schema: table.schema,
46
+ data
47
+ };
48
+ }
49
+
50
+ /** Convert any table into object row format *
51
+ export function makeObjectRowTable(table: Table): ObjectRowTable {
52
+ if (table.shape === 'object-row-table') {
53
+ return table;
54
+ }
55
+ const length = getTableLength(table);
56
+ const data = new Array<{[key: string]: unknown}>(length);
57
+ for (let rowIndex = 0; rowIndex < length; rowIndex++) {
58
+ data[rowIndex] = getTableRowAsObject(table, rowIndex);
59
+ }
60
+ return {
61
+ shape: 'object-row-table',
62
+ schema: table.schema,
63
+ data
64
+ };
65
+ }
66
+ */
67
+ //# sourceMappingURL=make-table.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"make-table.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/tables/make-table.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AACtC,OAAO,KAAK,EAEV,aAAa,EACb,cAAc,EACd,aAAa,EACb,UAAU,EACX,MAAM,oBAAoB,CAAC;AAG5B;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,GAAG,aAAa,CAAC;AACpE,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IAAC,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAA;CAAC,EAAE,GAAG,cAAc,CAAC;AACvF,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IAAC,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAA;CAAC,GAAG,aAAa,CAAC;AAC/F,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,GAAG,UAAU,CAAC;AAsDjE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAmDE"}
@@ -0,0 +1,101 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+ import * as arrow from 'apache-arrow';
5
+ import { deduceTableSchema } from "../../schema/deduce-table-schema.js";
6
+ export function makeTableFromData(data) {
7
+ let table;
8
+ switch (getTableShapeFromData(data)) {
9
+ case 'array-row-table':
10
+ table = { shape: 'array-row-table', data: data };
11
+ break;
12
+ case 'object-row-table':
13
+ table = { shape: 'object-row-table', data: data };
14
+ break;
15
+ case 'columnar-table':
16
+ table = { shape: 'columnar-table', data: data };
17
+ break;
18
+ case 'arrow-table':
19
+ table = { shape: 'arrow-table', data: data };
20
+ break;
21
+ default:
22
+ throw new Error('table');
23
+ }
24
+ const schema = deduceTableSchema(table);
25
+ return { ...table, schema };
26
+ }
27
+ /** Helper function to get shape of data */
28
+ function getTableShapeFromData(data) {
29
+ if (data instanceof arrow.Table) {
30
+ return 'arrow-table';
31
+ }
32
+ if (Array.isArray(data)) {
33
+ if (data.length === 0) {
34
+ throw new Error('cannot deduce type of empty table');
35
+ }
36
+ // Deduce the table shape from the first row
37
+ const firstRow = data[0];
38
+ if (Array.isArray(firstRow)) {
39
+ return 'array-row-table';
40
+ }
41
+ if (firstRow && typeof firstRow === 'object') {
42
+ return 'object-row-table';
43
+ }
44
+ }
45
+ if (data && typeof data === 'object') {
46
+ return 'columnar-table';
47
+ }
48
+ throw new Error('invalid table');
49
+ }
50
+ /** Convert any table into object row format *
51
+ export function makeColumnarTable(table: Table): ColumnarTable {
52
+ if (table.shape === 'columnar-table') {
53
+ return table;
54
+ }
55
+ const length = getTableLength(table);
56
+ const data = new Array<{[key: string]: unknown}>(length);
57
+ for (let rowIndex = 0; rowIndex < length; rowIndex++) {
58
+ data[rowIndex] = getTableRowAsObject(table, rowIndex);
59
+ }
60
+ return {
61
+ shape: 'columnar-table',
62
+ schema: table.schema,
63
+ data
64
+ };
65
+ }
66
+
67
+
68
+ /** Convert any table into array row format *
69
+ export function makeArrayRowTable(table: TableLike): ArrayRowTable {
70
+ if (table.shape === 'array-row-table') {
71
+ return table;
72
+ }
73
+ const length = getTableLength(table);
74
+ const data = new Array<unknown[]>(length);
75
+ for (let rowIndex = 0; rowIndex < length; rowIndex++) {
76
+ data[rowIndex] = getTableRowAsArray(table, rowIndex);
77
+ }
78
+ return {
79
+ shape: 'array-row-table',
80
+ schema: table.schema,
81
+ data
82
+ };
83
+ }
84
+
85
+ /** Convert any table into object row format *
86
+ export function makeObjectRowTable(table: Table): ObjectRowTable {
87
+ if (table.shape === 'object-row-table') {
88
+ return table;
89
+ }
90
+ const length = getTableLength(table);
91
+ const data = new Array<{[key: string]: unknown}>(length);
92
+ for (let rowIndex = 0; rowIndex < length; rowIndex++) {
93
+ data[rowIndex] = getTableRowAsObject(table, rowIndex);
94
+ }
95
+ return {
96
+ shape: 'object-row-table',
97
+ schema: table.schema,
98
+ data
99
+ };
100
+ }
101
+ */
@@ -0,0 +1,15 @@
1
+ /** Convert an object row to an array row */
2
+ export declare function convertToObjectRow(arrayRow: unknown[], headers: string[] | null): {
3
+ [columnName: string]: unknown;
4
+ };
5
+ /** Convert an object row to an array row */
6
+ export declare function convertToArrayRow(objectRow: {
7
+ [columnName: string]: unknown;
8
+ }, headers: string[] | null): unknown[];
9
+ /** Get headers from a sample array row */
10
+ export declare function inferHeadersFromArrayRow(arrayRow: unknown[]): string[];
11
+ /** Get headers from a smaple object row */
12
+ export declare function inferHeadersFromObjectRow(row: {
13
+ [columnName: string]: unknown;
14
+ }): string[];
15
+ //# sourceMappingURL=row-utils.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"row-utils.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/tables/row-utils.ts"],"names":[],"mappings":"AAIA,4CAA4C;AAC5C,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,OAAO,EAAE,EACnB,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,GACvB;IAAC,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAA;CAAC,CAgBjC;AAED,4CAA4C;AAC5C,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE;IAAC,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAA;CAAC,EAC1C,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,GACvB,OAAO,EAAE,CAaX;AAED,0CAA0C;AAC1C,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,OAAO,EAAE,YAO3D;AAED,2CAA2C;AAC3C,wBAAgB,yBAAyB,CAAC,GAAG,EAAE;IAAC,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAA;CAAC,YAE7E"}
@@ -0,0 +1,49 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+ /** Convert an object row to an array row */
5
+ export function convertToObjectRow(arrayRow, headers) {
6
+ if (!arrayRow) {
7
+ throw new Error('null row');
8
+ }
9
+ const objectRow = {};
10
+ if (headers) {
11
+ for (let i = 0; i < headers.length; i++) {
12
+ objectRow[headers[i]] = arrayRow[i];
13
+ }
14
+ }
15
+ else {
16
+ for (let i = 0; i < arrayRow.length; i++) {
17
+ const columnName = `column-${i}`;
18
+ objectRow[columnName] = arrayRow[i];
19
+ }
20
+ }
21
+ return objectRow;
22
+ }
23
+ /** Convert an object row to an array row */
24
+ export function convertToArrayRow(objectRow, headers) {
25
+ if (!objectRow) {
26
+ throw new Error('null row');
27
+ }
28
+ if (headers) {
29
+ const arrayRow = new Array(headers.length);
30
+ for (let i = 0; i < headers.length; i++) {
31
+ arrayRow[i] = objectRow[headers[i]];
32
+ }
33
+ return arrayRow;
34
+ }
35
+ return Object.values(objectRow);
36
+ }
37
+ /** Get headers from a sample array row */
38
+ export function inferHeadersFromArrayRow(arrayRow) {
39
+ const headers = [];
40
+ for (let i = 0; i < arrayRow.length; i++) {
41
+ const columnName = `column-${i}`;
42
+ headers.push(columnName);
43
+ }
44
+ return headers;
45
+ }
46
+ /** Get headers from a smaple object row */
47
+ export function inferHeadersFromObjectRow(row) {
48
+ return Object.keys(row);
49
+ }
@@ -0,0 +1,68 @@
1
+ import type { Table, ArrayRowTable, ObjectRowTable } from '@loaders.gl/schema';
2
+ export declare function isTable(table: any): table is Table;
3
+ /**
4
+ * Returns the length of the table (i.e. the number of rows)
5
+ */
6
+ export declare function getTableLength(table: Table): number;
7
+ /**
8
+ * Returns the number of columns in the table
9
+ * @throws Fails to deduce number of columns if the table has no schema and is empty
10
+ */
11
+ export declare function getTableNumCols(table: Table): number;
12
+ /** Get a table cell value at row index and column name */
13
+ export declare function getTableCell(table: Table, rowIndex: number, columnName: string): unknown;
14
+ /** Get a table cell value at row index and column name */
15
+ export declare function getTableCellAt(table: Table, rowIndex: number, columnIndex: number): unknown;
16
+ /** Deduce the table row shape */
17
+ export declare function getTableRowShape(table: Table): 'array-row-table' | 'object-row-table';
18
+ /** Get the index of a named table column. Requires the table to have a schema */
19
+ export declare function getTableColumnIndex(table: Table, columnName: string): number;
20
+ /** Get the name of a table column by index. Requires the table to have a schema */
21
+ export declare function getTableColumnName(table: Table, columnIndex: number): string;
22
+ /**
23
+ * Returns one row of the table in object format.
24
+ * @param target Optional parameter will be used if needed to store the row. Can be reused between calls to improve performance
25
+ * @returns an array representing the row. May be the original array in the row, a new object, or the target parameter
26
+ */
27
+ export declare function getTableRowAsObject(table: Table, rowIndex: number, target?: {
28
+ [columnName: string]: unknown;
29
+ }, copy?: 'copy'): {
30
+ [columnName: string]: unknown;
31
+ };
32
+ /**
33
+ * Returns one row of the table in array format.
34
+ * @param target Optional parameter will be used if needed to store the row. Can be reused between calls to improve performance.
35
+ * @returns an array representing the row. May be the original array in the row, a new object, or the target parameter
36
+ */
37
+ export declare function getTableRowAsArray(table: Table, rowIndex: number, target?: unknown[], copy?: 'copy'): unknown[];
38
+ /** Convert any table into array row format */
39
+ export declare function makeArrayRowTable(table: Table): ArrayRowTable;
40
+ /** Convert any table into object row format */
41
+ export declare function makeObjectRowTable(table: Table): ObjectRowTable;
42
+ /** Convert any table into object row format */
43
+ export declare function makeColumnarTable(table: Table): ObjectRowTable;
44
+ /**
45
+ * Iterate over table rows
46
+ * @param table
47
+ * @param shape
48
+ */
49
+ export declare function makeRowIterator(table: Table, shape: 'object-row-table' | 'array-row-table'): Iterable<unknown[] | {
50
+ [key: string]: unknown;
51
+ }>;
52
+ /**
53
+ * Streaming processing: Iterate over table, yielding array rows
54
+ * @param table
55
+ * @param shape
56
+ */
57
+ export declare function makeArrayRowIterator(table: Table, target?: unknown[]): Iterable<unknown[]>;
58
+ /**
59
+ * Streaming processing: Iterate over table, yielding object rows
60
+ * @param table
61
+ * @param shape
62
+ */
63
+ export declare function makeObjectRowIterator(table: Table, target?: {
64
+ [key: string]: unknown;
65
+ }): Iterable<{
66
+ [key: string]: unknown;
67
+ }>;
68
+ //# sourceMappingURL=table-accessors.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"table-accessors.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/tables/table-accessors.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAC,KAAK,EAAE,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AAE7E,wBAAgB,OAAO,CAAC,KAAK,EAAE,GAAG,GAAG,KAAK,IAAI,KAAK,CAmBlD;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAqBnD;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAsBpD;AAED,0DAA0D;AAC1D,wBAAgB,YAAY,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CA0BxF;AAED,0DAA0D;AAC1D,wBAAgB,cAAc,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAyB3F;AAED,iCAAiC;AACjC,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,KAAK,GAAG,iBAAiB,GAAG,kBAAkB,CAcrF;AAED,iFAAiF;AACjF,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAM5E;AAED,mFAAmF;AACnF,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAM5E;AAED;;;;GAIG;AAEH,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,KAAK,EACZ,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE;IAAC,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAA;CAAC,EACxC,IAAI,CAAC,EAAE,MAAM,GACZ;IAAC,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAA;CAAC,CAwDjC;AAED;;;;GAIG;AAEH,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,KAAK,EACZ,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,OAAO,EAAE,EAClB,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,EAAE,CA2DX;AAED,8CAA8C;AAC9C,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,KAAK,GAAG,aAAa,CAc7D;AAED,+CAA+C;AAC/C,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,KAAK,GAAG,cAAc,CAc/D;AAED,+CAA+C;AAC/C,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,KAAK,GAAG,cAAc,CAc9D;AAID;;;;GAIG;AACH,wBAAiB,eAAe,CAC9B,KAAK,EAAE,KAAK,EACZ,KAAK,EAAE,kBAAkB,GAAG,iBAAiB,GAC5C,QAAQ,CAAC,OAAO,EAAE,GAAG;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAC,CAAC,CAYhD;AAED;;;;GAIG;AACH,wBAAiB,oBAAoB,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,GAAE,OAAO,EAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAK/F;AAED;;;;GAIG;AACH,wBAAiB,qBAAqB,CACpC,KAAK,EAAE,KAAK,EACZ,MAAM,GAAE;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAM,GACpC,QAAQ,CAAC;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAC,CAAC,CAKpC"}