@loaders.gl/schema-utils 4.4.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/LICENSE +41 -0
  2. package/README.md +9 -0
  3. package/dist/dist.dev.js +14269 -0
  4. package/dist/dist.min.js +18 -0
  5. package/dist/index.cjs +2436 -0
  6. package/dist/index.cjs.map +7 -0
  7. package/dist/index.d.ts +28 -0
  8. package/dist/index.d.ts.map +1 -0
  9. package/dist/index.js +73 -0
  10. package/dist/lib/arrow-utils/arrow-fixed-size-list-utils.d.ts +11 -0
  11. package/dist/lib/arrow-utils/arrow-fixed-size-list-utils.d.ts.map +1 -0
  12. package/dist/lib/arrow-utils/arrow-fixed-size-list-utils.js +40 -0
  13. package/dist/lib/arrow-utils/arrow-list-of-fixed-size-list-utils.d.ts +11 -0
  14. package/dist/lib/arrow-utils/arrow-list-of-fixed-size-list-utils.d.ts.map +1 -0
  15. package/dist/lib/arrow-utils/arrow-list-of-fixed-size-list-utils.js +32 -0
  16. package/dist/lib/mesh/convert-mesh-to-table.d.ts +18 -0
  17. package/dist/lib/mesh/convert-mesh-to-table.d.ts.map +1 -0
  18. package/dist/lib/mesh/convert-mesh-to-table.js +60 -0
  19. package/dist/lib/mesh/convert-table-to-mesh.d.ts +7 -0
  20. package/dist/lib/mesh/convert-table-to-mesh.d.ts.map +1 -0
  21. package/dist/lib/mesh/convert-table-to-mesh.js +40 -0
  22. package/dist/lib/mesh/deduce-mesh-schema.d.ts +23 -0
  23. package/dist/lib/mesh/deduce-mesh-schema.d.ts.map +1 -0
  24. package/dist/lib/mesh/deduce-mesh-schema.js +62 -0
  25. package/dist/lib/mesh/mesh-utils.d.ts +23 -0
  26. package/dist/lib/mesh/mesh-utils.d.ts.map +1 -0
  27. package/dist/lib/mesh/mesh-utils.js +46 -0
  28. package/dist/lib/schema/convert-arrow-schema.d.ts +23 -0
  29. package/dist/lib/schema/convert-arrow-schema.d.ts.map +1 -0
  30. package/dist/lib/schema/convert-arrow-schema.js +280 -0
  31. package/dist/lib/schema/data-type.d.ts +16 -0
  32. package/dist/lib/schema/data-type.d.ts.map +1 -0
  33. package/dist/lib/schema/data-type.js +88 -0
  34. package/dist/lib/schema/deduce-table-schema.d.ts +14 -0
  35. package/dist/lib/schema/deduce-table-schema.d.ts.map +1 -0
  36. package/dist/lib/schema/deduce-table-schema.js +100 -0
  37. package/dist/lib/table/arrow-api/arrow-like-field.d.ts +19 -0
  38. package/dist/lib/table/arrow-api/arrow-like-field.d.ts.map +1 -0
  39. package/dist/lib/table/arrow-api/arrow-like-field.js +36 -0
  40. package/dist/lib/table/arrow-api/arrow-like-interface.d.ts +43 -0
  41. package/dist/lib/table/arrow-api/arrow-like-interface.d.ts.map +1 -0
  42. package/dist/lib/table/arrow-api/arrow-like-interface.js +4 -0
  43. package/dist/lib/table/arrow-api/arrow-like-schema.d.ts +12 -0
  44. package/dist/lib/table/arrow-api/arrow-like-schema.d.ts.map +1 -0
  45. package/dist/lib/table/arrow-api/arrow-like-schema.js +81 -0
  46. package/dist/lib/table/arrow-api/arrow-like-table.d.ts +28 -0
  47. package/dist/lib/table/arrow-api/arrow-like-table.d.ts.map +1 -0
  48. package/dist/lib/table/arrow-api/arrow-like-table.js +63 -0
  49. package/dist/lib/table/arrow-api/arrow-like-type.d.ts +177 -0
  50. package/dist/lib/table/arrow-api/arrow-like-type.d.ts.map +1 -0
  51. package/dist/lib/table/arrow-api/arrow-like-type.js +442 -0
  52. package/dist/lib/table/arrow-api/enum.d.ts +92 -0
  53. package/dist/lib/table/arrow-api/enum.d.ts.map +1 -0
  54. package/dist/lib/table/arrow-api/enum.js +97 -0
  55. package/dist/lib/table/arrow-api/get-type-info.d.ts +13 -0
  56. package/dist/lib/table/arrow-api/get-type-info.d.ts.map +1 -0
  57. package/dist/lib/table/arrow-api/get-type-info.js +27 -0
  58. package/dist/lib/table/arrow-api/index.d.ts +5 -0
  59. package/dist/lib/table/arrow-api/index.d.ts.map +1 -0
  60. package/dist/lib/table/arrow-api/index.js +7 -0
  61. package/dist/lib/table/batch-builder/arrow-table-batch-aggregator.d.ts +10 -0
  62. package/dist/lib/table/batch-builder/arrow-table-batch-aggregator.d.ts.map +1 -0
  63. package/dist/lib/table/batch-builder/arrow-table-batch-aggregator.js +67 -0
  64. package/dist/lib/table/batch-builder/arrow-table-builder.d.ts +25 -0
  65. package/dist/lib/table/batch-builder/arrow-table-builder.d.ts.map +1 -0
  66. package/dist/lib/table/batch-builder/arrow-table-builder.js +99 -0
  67. package/dist/lib/table/batch-builder/base-table-batch-aggregator.d.ts +19 -0
  68. package/dist/lib/table/batch-builder/base-table-batch-aggregator.d.ts.map +1 -0
  69. package/dist/lib/table/batch-builder/base-table-batch-aggregator.js +63 -0
  70. package/dist/lib/table/batch-builder/columnar-table-batch-aggregator.d.ts +20 -0
  71. package/dist/lib/table/batch-builder/columnar-table-batch-aggregator.d.ts.map +1 -0
  72. package/dist/lib/table/batch-builder/columnar-table-batch-aggregator.js +83 -0
  73. package/dist/lib/table/batch-builder/row-table-batch-aggregator.d.ts +21 -0
  74. package/dist/lib/table/batch-builder/row-table-batch-aggregator.d.ts.map +1 -0
  75. package/dist/lib/table/batch-builder/row-table-batch-aggregator.js +85 -0
  76. package/dist/lib/table/batch-builder/table-batch-aggregator.d.ts +24 -0
  77. package/dist/lib/table/batch-builder/table-batch-aggregator.d.ts.map +1 -0
  78. package/dist/lib/table/batch-builder/table-batch-aggregator.js +4 -0
  79. package/dist/lib/table/batch-builder/table-batch-builder.d.ts +52 -0
  80. package/dist/lib/table/batch-builder/table-batch-builder.d.ts.map +1 -0
  81. package/dist/lib/table/batch-builder/table-batch-builder.js +148 -0
  82. package/dist/lib/table/batches/convert-batches.d.ts +10 -0
  83. package/dist/lib/table/batches/convert-batches.d.ts.map +1 -0
  84. package/dist/lib/table/batches/convert-batches.js +45 -0
  85. package/dist/lib/table/batches/make-arrow-batch-iterator.d.ts +17 -0
  86. package/dist/lib/table/batches/make-arrow-batch-iterator.d.ts.map +1 -0
  87. package/dist/lib/table/batches/make-arrow-batch-iterator.js +57 -0
  88. package/dist/lib/table/batches/make-table-batch-iterator.d.ts +15 -0
  89. package/dist/lib/table/batches/make-table-batch-iterator.d.ts.map +1 -0
  90. package/dist/lib/table/batches/make-table-batch-iterator.js +21 -0
  91. package/dist/lib/table/batches/make-table-from-batches.d.ts +9 -0
  92. package/dist/lib/table/batches/make-table-from-batches.d.ts.map +1 -0
  93. package/dist/lib/table/batches/make-table-from-batches.js +62 -0
  94. package/dist/lib/table/tables/convert-arrow-table.d.ts +24 -0
  95. package/dist/lib/table/tables/convert-arrow-table.d.ts.map +1 -0
  96. package/dist/lib/table/tables/convert-arrow-table.js +150 -0
  97. package/dist/lib/table/tables/convert-table.d.ts +43 -0
  98. package/dist/lib/table/tables/convert-table.d.ts.map +1 -0
  99. package/dist/lib/table/tables/convert-table.js +120 -0
  100. package/dist/lib/table/tables/make-table.d.ts +67 -0
  101. package/dist/lib/table/tables/make-table.d.ts.map +1 -0
  102. package/dist/lib/table/tables/make-table.js +101 -0
  103. package/dist/lib/table/tables/row-utils.d.ts +15 -0
  104. package/dist/lib/table/tables/row-utils.d.ts.map +1 -0
  105. package/dist/lib/table/tables/row-utils.js +49 -0
  106. package/dist/lib/table/tables/table-accessors.d.ts +68 -0
  107. package/dist/lib/table/tables/table-accessors.d.ts.map +1 -0
  108. package/dist/lib/table/tables/table-accessors.js +350 -0
  109. package/dist/lib/table/tables/table-column.d.ts +7 -0
  110. package/dist/lib/table/tables/table-column.d.ts.map +1 -0
  111. package/dist/lib/table/tables/table-column.js +39 -0
  112. package/dist/lib/table/tables/table-types.d.ts +12 -0
  113. package/dist/lib/table/tables/table-types.d.ts.map +1 -0
  114. package/dist/lib/table/tables/table-types.js +23 -0
  115. package/dist/lib/utils/assert.d.ts +2 -0
  116. package/dist/lib/utils/assert.d.ts.map +1 -0
  117. package/dist/lib/utils/assert.js +8 -0
  118. package/dist/lib/utils/async-queue.d.ts +18 -0
  119. package/dist/lib/utils/async-queue.d.ts.map +1 -0
  120. package/dist/lib/utils/async-queue.js +90 -0
  121. package/dist/types/types.d.ts +15 -0
  122. package/dist/types/types.d.ts.map +1 -0
  123. package/dist/types/types.js +4 -0
  124. package/package.json +52 -0
  125. package/src/index.ts +134 -0
  126. package/src/lib/arrow-utils/arrow-fixed-size-list-utils.ts +63 -0
  127. package/src/lib/arrow-utils/arrow-list-of-fixed-size-list-utils.ts +48 -0
  128. package/src/lib/mesh/convert-mesh-to-table.ts +77 -0
  129. package/src/lib/mesh/convert-table-to-mesh.ts +49 -0
  130. package/src/lib/mesh/deduce-mesh-schema.ts +75 -0
  131. package/src/lib/mesh/mesh-utils.ts +64 -0
  132. package/src/lib/schema/convert-arrow-schema.ts +301 -0
  133. package/src/lib/schema/data-type.ts +103 -0
  134. package/src/lib/schema/deduce-table-schema.ts +119 -0
  135. package/src/lib/table/arrow-api/arrow-like-field.ts +53 -0
  136. package/src/lib/table/arrow-api/arrow-like-interface.ts +84 -0
  137. package/src/lib/table/arrow-api/arrow-like-schema.ts +101 -0
  138. package/src/lib/table/arrow-api/arrow-like-table.ts +85 -0
  139. package/src/lib/table/arrow-api/arrow-like-type.ts +502 -0
  140. package/src/lib/table/arrow-api/enum.ts +99 -0
  141. package/src/lib/table/arrow-api/get-type-info.ts +38 -0
  142. package/src/lib/table/arrow-api/index.ts +8 -0
  143. package/src/lib/table/batch-builder/arrow-table-batch-aggregator-builders.ts.disabled +190 -0
  144. package/src/lib/table/batch-builder/arrow-table-batch-aggregator.ts +82 -0
  145. package/src/lib/table/batch-builder/arrow-table-builder.ts +112 -0
  146. package/src/lib/table/batch-builder/base-table-batch-aggregator.ts +82 -0
  147. package/src/lib/table/batch-builder/columnar-table-batch-aggregator.ts +99 -0
  148. package/src/lib/table/batch-builder/row-table-batch-aggregator.ts +108 -0
  149. package/src/lib/table/batch-builder/table-batch-aggregator.ts +28 -0
  150. package/src/lib/table/batch-builder/table-batch-builder.ts +185 -0
  151. package/src/lib/table/batches/convert-batches.ts +83 -0
  152. package/src/lib/table/batches/make-arrow-batch-iterator.ts +75 -0
  153. package/src/lib/table/batches/make-table-batch-iterator.ts +25 -0
  154. package/src/lib/table/batches/make-table-from-batches.ts +84 -0
  155. package/src/lib/table/tables/convert-arrow-table.ts +198 -0
  156. package/src/lib/table/tables/convert-table.ts +151 -0
  157. package/src/lib/table/tables/make-table.ts +127 -0
  158. package/src/lib/table/tables/row-utils.ts +59 -0
  159. package/src/lib/table/tables/table-accessors.ts +417 -0
  160. package/src/lib/table/tables/table-column.ts +49 -0
  161. package/src/lib/table/tables/table-types.ts +37 -0
  162. package/src/lib/utils/assert.ts +8 -0
  163. package/src/lib/utils/async-queue.ts +104 -0
  164. package/src/types/types.ts +45 -0
@@ -0,0 +1,88 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+ /** Deduce column types from values */
5
+ export function getDataTypeFromValue(value, defaultNumberType = 'float32') {
6
+ if (value instanceof Date) {
7
+ return 'date-millisecond';
8
+ }
9
+ if (value instanceof Number) {
10
+ return defaultNumberType;
11
+ }
12
+ if (typeof value === 'string') {
13
+ return 'utf8';
14
+ }
15
+ if (value === null || value === 'undefined') {
16
+ return 'null';
17
+ }
18
+ return 'null';
19
+ }
20
+ /**
21
+ * Deduces a simple data type "descriptor from a typed array instance
22
+ */
23
+ export function getDataTypeFromArray(array) {
24
+ let type = getDataTypeFromTypedArray(array);
25
+ if (type !== 'null') {
26
+ return { type, nullable: false };
27
+ }
28
+ if (array.length > 0) {
29
+ type = getDataTypeFromValue(array[0]);
30
+ return { type, nullable: true };
31
+ }
32
+ return { type: 'null', nullable: true };
33
+ }
34
+ /**
35
+ * Deduces a simple data type "descriptor from a typed array instance
36
+ */
37
+ export function getDataTypeFromTypedArray(array) {
38
+ switch (array.constructor) {
39
+ case Int8Array:
40
+ return 'int8';
41
+ case Uint8Array:
42
+ case Uint8ClampedArray:
43
+ return 'uint8';
44
+ case Int16Array:
45
+ return 'int16';
46
+ case Uint16Array:
47
+ return 'uint16';
48
+ case Int32Array:
49
+ return 'int32';
50
+ case Uint32Array:
51
+ return 'uint32';
52
+ case Float32Array:
53
+ return 'float32';
54
+ case Float64Array:
55
+ return 'float64';
56
+ default:
57
+ return 'null';
58
+ }
59
+ }
60
+ export function getArrayTypeFromDataType(type, nullable) {
61
+ if (!nullable) {
62
+ switch (type) {
63
+ case 'int8':
64
+ return Int8Array;
65
+ case 'uint8':
66
+ return Uint8Array;
67
+ case 'int16':
68
+ return Int16Array;
69
+ case 'uint16':
70
+ return Uint16Array;
71
+ case 'int32':
72
+ return Int32Array;
73
+ case 'uint32':
74
+ return Uint32Array;
75
+ case 'float32':
76
+ return Float32Array;
77
+ case 'float64':
78
+ return Float64Array;
79
+ default:
80
+ break;
81
+ }
82
+ }
83
+ // if (typeof BigInt64Array !== 'undefined') {
84
+ // TYPED_ARRAY_TO_TYPE.BigInt64Array = new Int64();
85
+ // TYPED_ARRAY_TO_TYPE.BigUint64Array = new Uint64();
86
+ // }
87
+ return Array;
88
+ }
@@ -0,0 +1,14 @@
1
+ import type { Schema, Table } from '@loaders.gl/schema';
2
+ /**
3
+ * SCHEMA SUPPORT - AUTODEDUCTION
4
+ * @param {*} table
5
+ * @param {*} schema
6
+ * @returns
7
+ */
8
+ export declare function deduceTableSchema(table: Table): Schema;
9
+ export declare function deduceSchema(data: unknown[][] | {
10
+ [key: string]: unknown;
11
+ }[] | {
12
+ [key: string]: unknown[];
13
+ }): Schema;
14
+ //# sourceMappingURL=deduce-table-schema.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"deduce-table-schema.d.ts","sourceRoot":"","sources":["../../../src/lib/schema/deduce-table-schema.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,MAAM,EAAoB,KAAK,EAAC,MAAM,oBAAoB,CAAC;AAGxE;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAgBtD;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,OAAO,EAAE,EAAE,GAAG;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAC,EAAE,GAAG;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,EAAE,CAAA;CAAC,GAC1E,MAAM,CAER"}
@@ -0,0 +1,100 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+ import { getDataTypeFromArray, getDataTypeFromValue } from "./data-type.js";
5
+ /**
6
+ * SCHEMA SUPPORT - AUTODEDUCTION
7
+ * @param {*} table
8
+ * @param {*} schema
9
+ * @returns
10
+ */
11
+ export function deduceTableSchema(table) {
12
+ switch (table.shape) {
13
+ case 'array-row-table':
14
+ case 'object-row-table':
15
+ return deduceSchemaFromRows(table.data);
16
+ case 'geojson-table':
17
+ return deduceSchemaFromGeoJSON(table.features);
18
+ case 'columnar-table':
19
+ return deduceSchemaFromColumns(table.data);
20
+ case 'arrow-table':
21
+ default:
22
+ throw new Error('Deduce schema');
23
+ }
24
+ }
25
+ export function deduceSchema(data) {
26
+ return Array.isArray(data) ? deduceSchemaFromRows(data) : deduceSchemaFromColumns(data);
27
+ }
28
+ /** Given an object with columnar arrays, try to deduce a schema */
29
+ function deduceSchemaFromColumns(columnarTable) {
30
+ const fields = [];
31
+ for (const [columnName, column] of Object.entries(columnarTable)) {
32
+ const field = deduceFieldFromColumn(column, columnName);
33
+ fields.push(field);
34
+ }
35
+ return { fields, metadata: {} };
36
+ }
37
+ /** Given an array of rows, try to deduce a schema */
38
+ function deduceSchemaFromRows(rowTable) {
39
+ if (!rowTable.length) {
40
+ throw new Error('deduce from empty table');
41
+ }
42
+ const fields = [];
43
+ const row0 = rowTable[0];
44
+ // TODO - fields can be nullable, false detection...
45
+ // Could look at additional rows if nulls in first row
46
+ // TODO - if array, column names will be numbers
47
+ for (const [columnName, value] of Object.entries(row0)) {
48
+ fields.push(deduceFieldFromValue(value, columnName));
49
+ }
50
+ return { fields, metadata: {} };
51
+ }
52
+ /** Given a GeoJSON, try to deduce a schema */
53
+ function deduceSchemaFromGeoJSON(features) {
54
+ if (!features.length) {
55
+ throw new Error('deduce from empty table');
56
+ }
57
+ const fields = [];
58
+ const row0 = features[0].properties || {};
59
+ // TODO - fields can be nullable, false detection...
60
+ // Could look at additional rows if nulls in first row
61
+ // TODO - if array, column names will be numbers
62
+ for (const [columnName, value] of Object.entries(row0)) {
63
+ fields.push(deduceFieldFromValue(value, columnName));
64
+ }
65
+ return { fields, metadata: {} };
66
+ }
67
+ /** Given a column (i.e. array), attempt to deduce an appropriate `Field` */
68
+ function deduceFieldFromColumn(column, name) {
69
+ if (ArrayBuffer.isView(column)) {
70
+ const type = getDataTypeFromArray(column);
71
+ return {
72
+ name,
73
+ type: type.type || 'null',
74
+ nullable: type.nullable
75
+ // metadata: {}
76
+ };
77
+ }
78
+ if (Array.isArray(column) && column.length > 0) {
79
+ const value = column[0];
80
+ const type = getDataTypeFromValue(value);
81
+ // TODO - support nested schemas?
82
+ return {
83
+ name,
84
+ type,
85
+ nullable: true
86
+ // metadata: {},
87
+ };
88
+ }
89
+ throw new Error('empty table');
90
+ }
91
+ /** Given a value, attempt to deduce an appropriate `Field` */
92
+ function deduceFieldFromValue(value, name) {
93
+ const type = getDataTypeFromValue(value);
94
+ return {
95
+ name,
96
+ type,
97
+ nullable: true
98
+ // metadata: {}
99
+ };
100
+ }
@@ -0,0 +1,19 @@
1
+ import { DataType } from "./arrow-like-type.js";
2
+ /**
3
+ * ArrowJS `Field` API-compatible class for row-based tables
4
+ * https://loaders.gl/arrowjs/docs/api-reference/field
5
+ * A field holds name, nullable, and metadata information about a table "column"
6
+ * A Schema is essentially a list of fields
7
+ */
8
+ export declare class ArrowLikeField {
9
+ name: string;
10
+ type: DataType;
11
+ nullable: boolean;
12
+ metadata: Map<string, string>;
13
+ constructor(name: string, type: DataType, nullable?: boolean, metadata?: Map<string, string>);
14
+ get typeId(): number;
15
+ clone(): ArrowLikeField;
16
+ compareTo(other: this): boolean;
17
+ toString(): string;
18
+ }
19
+ //# sourceMappingURL=arrow-like-field.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"arrow-like-field.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/arrow-api/arrow-like-field.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,QAAQ,EAAC,6BAA0B;AAE3C;;;;;GAKG;AACH,qBAAa,cAAc;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,CAAC;IACf,QAAQ,EAAE,OAAO,CAAC;IAClB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAG5B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,QAAQ,EACd,QAAQ,UAAQ,EAChB,QAAQ,GAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAa;IAQ3C,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,KAAK,IAAI,cAAc;IAIvB,SAAS,CAAC,KAAK,EAAE,IAAI,GAAG,OAAO;IAS/B,QAAQ,IAAI,MAAM;CAKnB"}
@@ -0,0 +1,36 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+ /**
5
+ * ArrowJS `Field` API-compatible class for row-based tables
6
+ * https://loaders.gl/arrowjs/docs/api-reference/field
7
+ * A field holds name, nullable, and metadata information about a table "column"
8
+ * A Schema is essentially a list of fields
9
+ */
10
+ export class ArrowLikeField {
11
+ name;
12
+ type;
13
+ nullable;
14
+ metadata;
15
+ constructor(name, type, nullable = false, metadata = new Map()) {
16
+ this.name = name;
17
+ this.type = type;
18
+ this.nullable = nullable;
19
+ this.metadata = metadata;
20
+ }
21
+ get typeId() {
22
+ return this.type && this.type.typeId;
23
+ }
24
+ clone() {
25
+ return new ArrowLikeField(this.name, this.type, this.nullable, this.metadata);
26
+ }
27
+ compareTo(other) {
28
+ return (this.name === other.name &&
29
+ this.type === other.type &&
30
+ this.nullable === other.nullable &&
31
+ this.metadata === other.metadata);
32
+ }
33
+ toString() {
34
+ return `${JSON.stringify(this.type)}${this.nullable ? ', nullable' : ''}${this.metadata ? `, metadata: ${JSON.stringify(this.metadata)}` : ''}`;
35
+ }
36
+ }
@@ -0,0 +1,43 @@
1
+ import { DataType } from "./arrow-like-type.js";
2
+ /**
3
+ * ArrowJS `Field` API-compatible class for row-based tables
4
+ * https://loaders.gl/arrowjs/docs/api-reference/field
5
+ * A field holds name, nullable, and metadata information about a table "column"
6
+ * A Schema is essentially a list of fields
7
+ */
8
+ export interface Field {
9
+ name: string;
10
+ type: DataType;
11
+ nullable: boolean;
12
+ metadata: Map<string, string>;
13
+ typeId(): number;
14
+ clone(): Field;
15
+ compareTo(other: this): boolean;
16
+ toString(): any;
17
+ }
18
+ export interface Schema {
19
+ fields: Field[];
20
+ metadata: Map<string, string>;
21
+ compareTo(other: Schema): boolean;
22
+ select(...columnNames: string[]): Schema;
23
+ selectAt(...columnIndices: number[]): Schema;
24
+ assign(schemaOrFields: Schema | Field[]): Schema;
25
+ }
26
+ export interface ArrowLikeVector {
27
+ table: Table;
28
+ columnName: string;
29
+ get(rowIndex: number): unknown;
30
+ toArray(): ArrayLike<unknown>;
31
+ }
32
+ /**
33
+ * Class that provides an API similar to Apache Arrow Table class
34
+ * Forwards methods directly if the underlying table is Arrow, otherwise calls accessor functions
35
+ */
36
+ export interface Table {
37
+ schema: Schema;
38
+ data: any;
39
+ numCols: number;
40
+ length(): number;
41
+ getChild(columnName: string): ArrowLikeVector;
42
+ }
43
+ //# sourceMappingURL=arrow-like-interface.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"arrow-like-interface.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/arrow-api/arrow-like-interface.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,QAAQ,EAAC,6BAA0B;AAE3C;;;;;GAKG;AACH,MAAM,WAAW,KAAK;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,CAAC;IACf,QAAQ,EAAE,OAAO,CAAC;IAClB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAS9B,MAAM,IAAI,MAAM,CAAC;IAEjB,KAAK,IAAI,KAAK,CAAC;IAEf,SAAS,CAAC,KAAK,EAAE,IAAI,GAAG,OAAO,CAAC;IAEhC,QAAQ,QAAG;CACZ;AAED,MAAM,WAAW,MAAM;IACrB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAQ9B,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IAElC,MAAM,CAAC,GAAG,WAAW,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAEzC,QAAQ,CAAC,GAAG,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAE7C,MAAM,CAAC,cAAc,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,MAAM,CAAC;CAClD;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,KAAK,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IAInB,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IAC/B,OAAO,IAAI,SAAS,CAAC,OAAO,CAAC,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,WAAW,KAAK;IACpB,MAAM,EAAE,MAAM,CAAC;IAIf,IAAI,EAAE,GAAG,CAAC;IAEV,OAAO,EAAE,MAAM,CAAC;IAEhB,MAAM,IAAI,MAAM,CAAC;IAEjB,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,eAAe,CAAC;CAK/C"}
@@ -0,0 +1,4 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+ export {};
@@ -0,0 +1,12 @@
1
+ import type { SchemaMetadata, Field } from '@loaders.gl/schema';
2
+ import { ArrowLikeField } from "./arrow-like-field.js";
3
+ export declare class ArrowLikeSchema {
4
+ fields: ArrowLikeField[];
5
+ metadata: Map<string, string>;
6
+ constructor(fields: ArrowLikeField[] | Field[], metadata?: SchemaMetadata | Map<string, string>);
7
+ compareTo(other: ArrowLikeSchema): boolean;
8
+ select(...columnNames: string[]): ArrowLikeSchema;
9
+ selectAt(...columnIndices: number[]): ArrowLikeSchema;
10
+ assign(schemaOrFields: ArrowLikeSchema | ArrowLikeField[]): ArrowLikeSchema;
11
+ }
12
+ //# sourceMappingURL=arrow-like-schema.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"arrow-like-schema.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/arrow-api/arrow-like-schema.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,cAAc,EAAE,KAAK,EAAC,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAC,cAAc,EAAC,8BAA2B;AAElD,qBAAa,eAAe;IAC1B,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAG5B,MAAM,EAAE,cAAc,EAAE,GAAG,KAAK,EAAE,EAClC,QAAQ,GAAE,cAAc,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAA6B;IAY5E,SAAS,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO;IAe1C,MAAM,CAAC,GAAG,WAAW,EAAE,MAAM,EAAE,GAAG,eAAe;IAUjD,QAAQ,CAAC,GAAG,aAAa,EAAE,MAAM,EAAE,GAAG,eAAe;IAMrD,MAAM,CAAC,cAAc,EAAE,eAAe,GAAG,cAAc,EAAE,GAAG,eAAe;CA2B5E"}
@@ -0,0 +1,81 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+ import { ArrowLikeField } from "./arrow-like-field.js";
5
+ export class ArrowLikeSchema {
6
+ fields;
7
+ metadata;
8
+ constructor(fields, metadata = new Map()) {
9
+ // checkNames(fields);
10
+ // For kepler fields, create arrow compatible `Fields` that have kepler fields as `metadata`
11
+ this.fields = fields.map((field) => new ArrowLikeField(field.name, field.type, field.nullable, field.metadata));
12
+ this.metadata =
13
+ metadata instanceof Map ? metadata : new Map(Object.entries(metadata));
14
+ }
15
+ // TODO - arrow only seems to compare fields, not metadata
16
+ compareTo(other) {
17
+ if (this.metadata !== other.metadata) {
18
+ return false;
19
+ }
20
+ if (this.fields.length !== other.fields.length) {
21
+ return false;
22
+ }
23
+ for (let i = 0; i < this.fields.length; ++i) {
24
+ if (!this.fields[i].compareTo(other.fields[i])) {
25
+ return false;
26
+ }
27
+ }
28
+ return true;
29
+ }
30
+ select(...columnNames) {
31
+ // Ensure column names reference valid fields
32
+ const nameMap = Object.create(null);
33
+ for (const name of columnNames) {
34
+ nameMap[name] = true;
35
+ }
36
+ const selectedFields = this.fields.filter((field) => nameMap[field.name]);
37
+ return new ArrowLikeSchema(selectedFields, this.metadata);
38
+ }
39
+ selectAt(...columnIndices) {
40
+ // Ensure column indices reference valid fields
41
+ const selectedFields = columnIndices.map((index) => this.fields[index]).filter(Boolean);
42
+ return new ArrowLikeSchema(selectedFields, this.metadata);
43
+ }
44
+ assign(schemaOrFields) {
45
+ let fields;
46
+ let metadata = this.metadata;
47
+ if (schemaOrFields instanceof ArrowLikeSchema) {
48
+ const otherArrowLikeSchema = schemaOrFields;
49
+ fields = otherArrowLikeSchema.fields;
50
+ metadata = mergeMaps(mergeMaps(new Map(), this.metadata), otherArrowLikeSchema.metadata);
51
+ }
52
+ else {
53
+ fields = schemaOrFields;
54
+ }
55
+ // Create a merged list of fields, overwrite fields in place, new fields at end
56
+ const fieldMap = Object.create(null);
57
+ for (const field of this.fields) {
58
+ fieldMap[field.name] = field;
59
+ }
60
+ for (const field of fields) {
61
+ fieldMap[field.name] = field;
62
+ }
63
+ const mergedFields = Object.values(fieldMap);
64
+ return new ArrowLikeSchema(mergedFields, metadata);
65
+ }
66
+ }
67
+ // Warn if any duplicated field names
68
+ // function checkNames(fields: Field[]): void {
69
+ // const usedNames: Record<string, boolean> = {};
70
+ // for (const field of fields) {
71
+ // if (usedNames[field.name]) {
72
+ // // eslint-disable-next-line
73
+ // console.warn('ArrowLikeSchema: duplicated field name', field.name, field);
74
+ // }
75
+ // usedNames[field.name] = true;
76
+ // }
77
+ // }
78
+ function mergeMaps(m1, m2) {
79
+ // @ts-ignore
80
+ return new Map([...(m1 || new Map()), ...(m2 || new Map())]);
81
+ }
@@ -0,0 +1,28 @@
1
+ import type { Table } from '@loaders.gl/schema';
2
+ import { ArrowLikeSchema } from "./arrow-like-schema.js";
3
+ declare class ArrowLikeVector {
4
+ table: Table;
5
+ columnName: string;
6
+ constructor(table: Table, columnName: string);
7
+ get(rowIndex: number): unknown;
8
+ toArray(): ArrayLike<unknown>;
9
+ }
10
+ /**
11
+ * Class that provides an API similar to Apache Arrow Table class
12
+ * Forwards methods directly if the underlying table is Arrow, otherwise calls accessor functions
13
+ */
14
+ export declare class ArrowLikeTable {
15
+ schema: ArrowLikeSchema;
16
+ table: Table;
17
+ constructor(table: Table);
18
+ get data(): any[][] | {
19
+ [columnName: string]: any;
20
+ }[] | import("geojson").Feature<import("geojson").Geometry, import("geojson").GeoJsonProperties>[] | {
21
+ [columnName: string]: ArrayLike<unknown>;
22
+ } | import("apache-arrow").Table<any>;
23
+ get numCols(): number;
24
+ get length(): number;
25
+ getChild(columnName: string): ArrowLikeVector;
26
+ }
27
+ export {};
28
+ //# sourceMappingURL=arrow-like-table.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"arrow-like-table.d.ts","sourceRoot":"","sources":["../../../../src/lib/table/arrow-api/arrow-like-table.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,oBAAoB,CAAC;AAE9C,OAAO,EAAC,eAAe,EAAC,+BAA4B;AAepD,cAAM,eAAe;IACnB,KAAK,EAAE,KAAK,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;gBAEP,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM;IAK5C,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAI9B,OAAO,IAAI,SAAS,CAAC,OAAO,CAAC;CAW9B;AAED;;;GAGG;AACH,qBAAa,cAAc;IACzB,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,KAAK,CAAC;gBAED,KAAK,EAAE,KAAK;IAUxB,IAAI,IAAI;;;;0CAEP;IAED,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,eAAe;CAO9C"}
@@ -0,0 +1,63 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+ import { ArrowLikeSchema } from "./arrow-like-schema.js";
5
+ import { deduceTableSchema } from "../../schema/deduce-table-schema.js";
6
+ import { getTableCell, getTableLength, getTableNumCols
7
+ // getTableCell,
8
+ // getTableRowShape,
9
+ // getTableColumnIndex,
10
+ // getTableColumnName,
11
+ // getTableRowAsObject,
12
+ // getTableRowAsArray
13
+ } from "../tables/table-accessors.js";
14
+ class ArrowLikeVector {
15
+ table;
16
+ columnName;
17
+ constructor(table, columnName) {
18
+ this.table = table;
19
+ this.columnName = columnName;
20
+ }
21
+ get(rowIndex) {
22
+ return getTableCell(this.table, rowIndex, this.columnName);
23
+ }
24
+ toArray() {
25
+ switch (this.table.shape) {
26
+ case 'arrow-table':
27
+ const arrowTable = this.table.data;
28
+ return arrowTable.getChild(this.columnName)?.toArray();
29
+ case 'columnar-table':
30
+ return this.table.data[this.columnName];
31
+ default:
32
+ throw new Error(this.table.shape);
33
+ }
34
+ }
35
+ }
36
+ /**
37
+ * Class that provides an API similar to Apache Arrow Table class
38
+ * Forwards methods directly if the underlying table is Arrow, otherwise calls accessor functions
39
+ */
40
+ export class ArrowLikeTable {
41
+ schema;
42
+ table;
43
+ constructor(table) {
44
+ const schema = table.schema || deduceTableSchema(table);
45
+ this.schema = new ArrowLikeSchema(schema.fields, schema.metadata);
46
+ this.table = { ...table, schema };
47
+ }
48
+ // get schema() {
49
+ // return this.table.schema;
50
+ // }
51
+ get data() {
52
+ return this.table.shape === 'geojson-table' ? this.table.features : this.table.data;
53
+ }
54
+ get numCols() {
55
+ return getTableNumCols(this.table);
56
+ }
57
+ get length() {
58
+ return getTableLength(this.table);
59
+ }
60
+ getChild(columnName) {
61
+ return new ArrowLikeVector(this.table, columnName);
62
+ }
63
+ }