@loaders.gl/sql 5.0.0-alpha.9 → 5.0.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/LICENSE +2 -30
  2. package/README.md +21 -12
  3. package/THIRD-PARTY-NOTICES.md +17 -0
  4. package/dist/arrow-table-source.d.ts +9 -9
  5. package/dist/arrow-table-source.d.ts.map +1 -1
  6. package/dist/arrow-table-source.js +11 -7
  7. package/dist/arrow-table-source.js.map +1 -1
  8. package/dist/compile-table-query.d.ts +3 -3
  9. package/dist/compile-table-query.d.ts.map +1 -1
  10. package/dist/compile-table-query.js +2 -2
  11. package/dist/compile-table-query.js.map +1 -1
  12. package/dist/duckdb-sql-source.d.ts +36 -32
  13. package/dist/duckdb-sql-source.d.ts.map +1 -1
  14. package/dist/duckdb-sql-source.js +61 -17
  15. package/dist/duckdb-sql-source.js.map +1 -1
  16. package/dist/duckdb-table-query.d.ts +9 -0
  17. package/dist/duckdb-table-query.d.ts.map +1 -0
  18. package/dist/duckdb-table-query.js +242 -0
  19. package/dist/duckdb-table-query.js.map +1 -0
  20. package/dist/index.cjs +1843 -1473
  21. package/dist/index.cjs.map +3 -3
  22. package/dist/index.d.ts +12 -12
  23. package/dist/index.d.ts.map +1 -1
  24. package/dist/index.js +7 -7
  25. package/dist/index.js.map +1 -1
  26. package/dist/snowflake-sql-source.d.ts +18 -31
  27. package/dist/snowflake-sql-source.d.ts.map +1 -1
  28. package/dist/snowflake-sql-source.js +2 -2
  29. package/dist/snowflake-sql-source.js.map +1 -1
  30. package/dist/sql-source.d.ts +24 -11
  31. package/dist/sql-source.d.ts.map +1 -1
  32. package/dist/sql-source.js +35 -3
  33. package/dist/sql-source.js.map +1 -1
  34. package/dist/sql-types.d.ts +15 -3
  35. package/dist/sql-types.d.ts.map +1 -1
  36. package/dist/sql-utils.d.ts +2 -3
  37. package/dist/sql-utils.d.ts.map +1 -1
  38. package/dist/sql-utils.js +1 -34
  39. package/dist/sql-utils.js.map +1 -1
  40. package/package.json +8 -28
  41. package/src/arrow-table-source.ts +17 -16
  42. package/src/compile-table-query.ts +4 -4
  43. package/src/duckdb-sql-source.ts +81 -20
  44. package/src/duckdb-table-query.ts +277 -0
  45. package/src/index.ts +19 -11
  46. package/src/snowflake-sql-source.ts +3 -3
  47. package/src/sql-source.ts +46 -7
  48. package/src/sql-types.ts +16 -3
  49. package/src/sql-utils.ts +2 -40
  50. package/dist/arrow-query.cjs +0 -1404
  51. package/dist/arrow-query.cjs.map +0 -6
  52. package/dist/arrow-query.d.ts +0 -5
  53. package/dist/arrow-query.d.ts.map +0 -1
  54. package/dist/arrow-query.js +0 -6
  55. package/dist/arrow-query.js.map +0 -1
  56. package/dist/bind-sql-predicate.d.ts +0 -4
  57. package/dist/bind-sql-predicate.d.ts.map +0 -1
  58. package/dist/bind-sql-predicate.js +0 -11
  59. package/dist/bind-sql-predicate.js.map +0 -1
  60. package/dist/parse-sql-predicate.d.ts +0 -10
  61. package/dist/parse-sql-predicate.d.ts.map +0 -1
  62. package/dist/parse-sql-predicate.js +0 -322
  63. package/dist/parse-sql-predicate.js.map +0 -1
  64. package/dist/query-arrow-table.d.ts +0 -83
  65. package/dist/query-arrow-table.d.ts.map +0 -1
  66. package/dist/query-arrow-table.js +0 -663
  67. package/dist/query-arrow-table.js.map +0 -1
  68. package/dist/sql-predicate-schema.d.ts +0 -156
  69. package/dist/sql-predicate-schema.d.ts.map +0 -1
  70. package/dist/sql-predicate-schema.js +0 -226
  71. package/dist/sql-predicate-schema.js.map +0 -1
  72. package/dist/sql-predicate-types.d.ts +0 -45
  73. package/dist/sql-predicate-types.d.ts.map +0 -1
  74. package/dist/sql-predicate-types.js +0 -9
  75. package/dist/sql-predicate-types.js.map +0 -1
  76. package/dist/sql-predicate-zod.cjs +0 -54
  77. package/dist/sql-predicate-zod.cjs.map +0 -6
  78. package/dist/sql-predicate-zod.d.ts +0 -11
  79. package/dist/sql-predicate-zod.d.ts.map +0 -1
  80. package/dist/sql-predicate-zod.js +0 -45
  81. package/dist/sql-predicate-zod.js.map +0 -1
  82. package/dist/sql-predicate.cjs +0 -607
  83. package/dist/sql-predicate.cjs.map +0 -6
  84. package/dist/sql-predicate.d.ts +0 -6
  85. package/dist/sql-predicate.d.ts.map +0 -1
  86. package/dist/sql-predicate.js +0 -8
  87. package/dist/sql-predicate.js.map +0 -1
  88. package/dist/table-query-capabilities.d.ts +0 -6
  89. package/dist/table-query-capabilities.d.ts.map +0 -1
  90. package/dist/table-query-capabilities.js +0 -26
  91. package/dist/table-query-capabilities.js.map +0 -1
  92. package/dist/table-query.cjs +0 -286
  93. package/dist/table-query.cjs.map +0 -6
  94. package/dist/table-query.d.ts +0 -29
  95. package/dist/table-query.d.ts.map +0 -1
  96. package/dist/table-query.js +0 -30
  97. package/dist/table-query.js.map +0 -1
  98. package/src/arrow-query.ts +0 -17
  99. package/src/bind-sql-predicate.ts +0 -21
  100. package/src/parse-sql-predicate.ts +0 -381
  101. package/src/query-arrow-table.ts +0 -846
  102. package/src/sql-predicate-schema.ts +0 -251
  103. package/src/sql-predicate-types.ts +0 -87
  104. package/src/sql-predicate-zod.ts +0 -52
  105. package/src/sql-predicate.ts +0 -28
  106. package/src/table-query-capabilities.ts +0 -29
  107. package/src/table-query.ts +0 -78
package/src/sql-source.ts CHANGED
@@ -2,9 +2,10 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import type {ArrowTable, Schema} from '@loaders.gl/schema';
6
- import {DataSource} from '@loaders.gl/loader-utils';
7
- import type {CoreAPI, RequiredOptions} from '@loaders.gl/loader-utils';
5
+ import type {Table} from 'apache-arrow';
6
+ import type {Schema} from '@loaders.gl/core/schema';
7
+ import {DataSource} from '@loaders.gl/core/loader-utils';
8
+ import type {CoreAPI, RequiredOptions, TableMetadata} from '@loaders.gl/core/loader-utils';
8
9
 
9
10
  import type {
10
11
  SQLAdapter,
@@ -18,9 +19,9 @@ import type {
18
19
  SQLTableInfo
19
20
  } from './sql-types';
20
21
  import {compileSQLTableQuery, type SQLTableQueryDialect} from './compile-table-query';
21
- import type {SQLPredicateParameterValues} from './sql-predicate-types';
22
+ import type {SQLPredicateParameterValues} from '@loaders.gl/core/scan/sql-predicate-types';
22
23
  import {convertRowsToArrowTable, isNodeRuntime} from './sql-utils';
23
- import {SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES} from './table-query-capabilities';
24
+ import {SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES} from '@loaders.gl/core/scan/table-query-capabilities';
24
25
 
25
26
  const SQL_ADAPTER_FACTORIES = new Map<string, SQLAdapterFactory>();
26
27
 
@@ -108,6 +109,36 @@ export class SQLDataSource extends DataSource<string, SQLSourceOptions> {
108
109
  return await adapter.listTables(options);
109
110
  }
110
111
 
112
+ /** Lists stable native table identifiers without fetching any column schemas or data rows. */
113
+ async getTablesMetadata(options: {signal?: AbortSignal} = {}): Promise<readonly TableMetadata[]> {
114
+ options.signal?.throwIfAborted();
115
+ const tables = await this.listTables();
116
+ options.signal?.throwIfAborted();
117
+ return tables.map(table => ({name: getTableMetadataName(table), queryable: false}));
118
+ }
119
+
120
+ /** Discovers one native table schema; ambiguous unqualified names are rejected. */
121
+ async getTableMetadata(
122
+ name: string,
123
+ options: {signal?: AbortSignal} = {}
124
+ ): Promise<TableMetadata> {
125
+ const table = await this.resolveTable(name, options.signal);
126
+ const schema = await this.getTableSchema(table);
127
+ options.signal?.throwIfAborted();
128
+ return {name: getTableMetadataName(table), schema, queryable: false};
129
+ }
130
+
131
+ /** Resolves a qualified or unique unqualified table name. */
132
+ protected async resolveTable(name: string, signal?: AbortSignal): Promise<SQLTableInfo> {
133
+ signal?.throwIfAborted();
134
+ const tables = await this.listTables();
135
+ signal?.throwIfAborted();
136
+ const exact = tables.filter(table => getTableMetadataName(table) === name);
137
+ const matches = exact.length ? exact : tables.filter(table => table.tableName === name);
138
+ if (matches.length !== 1) throw new Error(`Unknown or ambiguous SQL table: ${name}`);
139
+ return matches[0];
140
+ }
141
+
111
142
  /** Returns the schema for a specific SQL table. */
112
143
  async getTableSchema(options: {
113
144
  catalogName?: string;
@@ -128,8 +159,8 @@ export class SQLDataSource extends DataSource<string, SQLSourceOptions> {
128
159
  return await adapter.executeRows(compiledQuery.sql, compiledQuery.options);
129
160
  }
130
161
 
131
- /** Executes a SQL query and returns a loaders.gl Arrow table. */
132
- async queryArrow(query: SQLQuery, options: SQLQueryOptions = {}): Promise<ArrowTable> {
162
+ /** Executes a SQL query and returns a native Arrow table. */
163
+ async queryArrow(query: SQLQuery, options: SQLQueryOptions = {}): Promise<Table> {
133
164
  const adapter = await this.getAdapter();
134
165
  const compiledQuery = this.compileQuery(query, options);
135
166
  if (adapter.executeArrow) {
@@ -226,3 +257,11 @@ function getSQLTableQueryDialect(sourceType: string): SQLTableQueryDialect {
226
257
  if (sourceType === 'snowflake-sql') return 'snowflake';
227
258
  throw new Error(`Portable table queries are not supported for SQL source type "${sourceType}".`);
228
259
  }
260
+
261
+ /** Preserves native namespace components as one unambiguous provider-defined loaded name. */
262
+ function getTableMetadataName(table: SQLTableInfo): string {
263
+ return [table.catalogName, table.schemaName, table.tableName]
264
+ .filter((name): name is string => typeof name === 'string' && name.length > 0)
265
+ .map(name => `"${name.replaceAll('"', '""')}"`)
266
+ .join('.');
267
+ }
package/src/sql-types.ts CHANGED
@@ -2,8 +2,9 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import type {ArrowTable, Schema} from '@loaders.gl/schema';
6
- import type {CoreAPI, DataSourceOptions} from '@loaders.gl/loader-utils';
5
+ import type {Table} from 'apache-arrow';
6
+ import type {Schema} from '@loaders.gl/core/schema';
7
+ import type {CoreAPI, DataSourceOptions} from '@loaders.gl/core/loader-utils';
7
8
  import type {SQLTableQuery} from './compile-table-query';
8
9
 
9
10
  export type SQLParameterValues = unknown[] | Record<string, unknown>;
@@ -64,6 +65,16 @@ export type SQLAdapterCapabilities = {
64
65
  isDynamic?: boolean;
65
66
  };
66
67
 
68
+ /** Destination for an Arrow IPC upload. */
69
+ export type SQLArrowInsertOptions = {
70
+ /** Destination table name. */
71
+ tableName: string;
72
+ /** Destination schema, defaulting to DuckDB's default schema. */
73
+ schemaName?: string;
74
+ /** Create a table when true (default); append to an existing table when false. */
75
+ create?: boolean;
76
+ };
77
+
67
78
  /** Runtime adapter interface used by SQL-backed data sources. */
68
79
  export interface SQLAdapter {
69
80
  readonly capabilities: SQLAdapterCapabilities;
@@ -78,7 +89,9 @@ export interface SQLAdapter {
78
89
  tableName: string;
79
90
  }): Promise<Schema>;
80
91
  executeRows(sql: string, options?: SQLQueryOptions): Promise<Record<string, unknown>[]>;
81
- executeArrow?(sql: string, options?: SQLQueryOptions): Promise<ArrowTable>;
92
+ /** Uploads an Arrow IPC stream without passing Arrow runtime objects. */
93
+ insertArrowIPC?(bytes: Uint8Array, options: SQLArrowInsertOptions): Promise<void>;
94
+ executeArrow?(sql: string, options?: SQLQueryOptions): Promise<Table>;
82
95
  }
83
96
 
84
97
  /** Context passed to SQL adapter factories during source initialization. */
package/src/sql-utils.ts CHANGED
@@ -2,9 +2,7 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import * as arrow from 'apache-arrow';
6
- import type {ArrowTable, DataType, Field, ObjectRowTable, Schema} from '@loaders.gl/schema';
7
- import {makeTableFromData, getDataTypeFromArray} from '@loaders.gl/schema-utils';
5
+ import type {DataType, Schema} from '@loaders.gl/core/schema';
8
6
 
9
7
  import type {SQLColumnInfo} from './sql-types';
10
8
 
@@ -23,28 +21,7 @@ export function quoteSqlIdentifier(identifier: string): string {
23
21
  return `"${identifier.replace(/"/g, '""')}"`;
24
22
  }
25
23
 
26
- /** Converts object rows into a loaders.gl Arrow table. */
27
- export function convertRowsToArrowTable(rows: Record<string, unknown>[]): ArrowTable {
28
- const table: ObjectRowTable =
29
- rows.length > 0
30
- ? makeTableFromData(rows)
31
- : {shape: 'object-row-table', schema: {fields: [], metadata: {}}, data: rows};
32
- const schema = deducePrimitiveAwareSchema(table);
33
-
34
- if (table.data.length === 0 || schema.fields.length === 0) {
35
- return {
36
- shape: 'arrow-table',
37
- schema,
38
- data: new arrow.Table(new arrow.Schema([]))
39
- };
40
- }
41
-
42
- return {
43
- shape: 'arrow-table',
44
- schema,
45
- data: arrow.tableFromJSON(table.data)
46
- };
47
- }
24
+ export {convertRowsToArrowTable} from '@loaders.gl/core/scan/rows-to-arrow';
48
25
 
49
26
  /** Converts normalized SQL columns into a loaders.gl schema. */
50
27
  export function convertSQLColumnsToSchema(columns: SQLColumnInfo[]): Schema {
@@ -109,21 +86,6 @@ function mapSQLTypeToDataType(sqlType: string): DataType {
109
86
  return 'utf8';
110
87
  }
111
88
 
112
- function deducePrimitiveAwareSchema(table: ObjectRowTable): Schema {
113
- return {
114
- ...table.schema!,
115
- fields: table.schema!.fields.map((field): Field => {
116
- if (field.type !== 'null') {
117
- return field;
118
- }
119
-
120
- const columnValues = table.data.map(row => row[field.name]);
121
- const inferredType = getDataTypeFromArray(columnValues);
122
- return {...field, type: inferredType.type, nullable: inferredType.nullable};
123
- })
124
- };
125
- }
126
-
127
89
  /** Returns a fully qualified SQL object name. */
128
90
  export function getQualifiedTableName(options: {
129
91
  catalogName?: string;