@loaders.gl/sql 5.0.0-alpha.9 → 5.0.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +2 -30
- package/README.md +21 -12
- package/THIRD-PARTY-NOTICES.md +17 -0
- package/dist/arrow-table-source.d.ts +9 -9
- package/dist/arrow-table-source.d.ts.map +1 -1
- package/dist/arrow-table-source.js +11 -7
- package/dist/arrow-table-source.js.map +1 -1
- package/dist/compile-table-query.d.ts +3 -3
- package/dist/compile-table-query.d.ts.map +1 -1
- package/dist/compile-table-query.js +2 -2
- package/dist/compile-table-query.js.map +1 -1
- package/dist/duckdb-sql-source.d.ts +36 -32
- package/dist/duckdb-sql-source.d.ts.map +1 -1
- package/dist/duckdb-sql-source.js +61 -17
- package/dist/duckdb-sql-source.js.map +1 -1
- package/dist/duckdb-table-query.d.ts +9 -0
- package/dist/duckdb-table-query.d.ts.map +1 -0
- package/dist/duckdb-table-query.js +242 -0
- package/dist/duckdb-table-query.js.map +1 -0
- package/dist/index.cjs +1843 -1473
- package/dist/index.cjs.map +3 -3
- package/dist/index.d.ts +12 -12
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -7
- package/dist/index.js.map +1 -1
- package/dist/snowflake-sql-source.d.ts +18 -31
- package/dist/snowflake-sql-source.d.ts.map +1 -1
- package/dist/snowflake-sql-source.js +2 -2
- package/dist/snowflake-sql-source.js.map +1 -1
- package/dist/sql-source.d.ts +24 -11
- package/dist/sql-source.d.ts.map +1 -1
- package/dist/sql-source.js +35 -3
- package/dist/sql-source.js.map +1 -1
- package/dist/sql-types.d.ts +15 -3
- package/dist/sql-types.d.ts.map +1 -1
- package/dist/sql-utils.d.ts +2 -3
- package/dist/sql-utils.d.ts.map +1 -1
- package/dist/sql-utils.js +1 -34
- package/dist/sql-utils.js.map +1 -1
- package/package.json +8 -28
- package/src/arrow-table-source.ts +17 -16
- package/src/compile-table-query.ts +4 -4
- package/src/duckdb-sql-source.ts +81 -20
- package/src/duckdb-table-query.ts +277 -0
- package/src/index.ts +19 -11
- package/src/snowflake-sql-source.ts +3 -3
- package/src/sql-source.ts +46 -7
- package/src/sql-types.ts +16 -3
- package/src/sql-utils.ts +2 -40
- package/dist/arrow-query.cjs +0 -1404
- package/dist/arrow-query.cjs.map +0 -6
- package/dist/arrow-query.d.ts +0 -5
- package/dist/arrow-query.d.ts.map +0 -1
- package/dist/arrow-query.js +0 -6
- package/dist/arrow-query.js.map +0 -1
- package/dist/bind-sql-predicate.d.ts +0 -4
- package/dist/bind-sql-predicate.d.ts.map +0 -1
- package/dist/bind-sql-predicate.js +0 -11
- package/dist/bind-sql-predicate.js.map +0 -1
- package/dist/parse-sql-predicate.d.ts +0 -10
- package/dist/parse-sql-predicate.d.ts.map +0 -1
- package/dist/parse-sql-predicate.js +0 -322
- package/dist/parse-sql-predicate.js.map +0 -1
- package/dist/query-arrow-table.d.ts +0 -83
- package/dist/query-arrow-table.d.ts.map +0 -1
- package/dist/query-arrow-table.js +0 -663
- package/dist/query-arrow-table.js.map +0 -1
- package/dist/sql-predicate-schema.d.ts +0 -156
- package/dist/sql-predicate-schema.d.ts.map +0 -1
- package/dist/sql-predicate-schema.js +0 -226
- package/dist/sql-predicate-schema.js.map +0 -1
- package/dist/sql-predicate-types.d.ts +0 -45
- package/dist/sql-predicate-types.d.ts.map +0 -1
- package/dist/sql-predicate-types.js +0 -9
- package/dist/sql-predicate-types.js.map +0 -1
- package/dist/sql-predicate-zod.cjs +0 -54
- package/dist/sql-predicate-zod.cjs.map +0 -6
- package/dist/sql-predicate-zod.d.ts +0 -11
- package/dist/sql-predicate-zod.d.ts.map +0 -1
- package/dist/sql-predicate-zod.js +0 -45
- package/dist/sql-predicate-zod.js.map +0 -1
- package/dist/sql-predicate.cjs +0 -607
- package/dist/sql-predicate.cjs.map +0 -6
- package/dist/sql-predicate.d.ts +0 -6
- package/dist/sql-predicate.d.ts.map +0 -1
- package/dist/sql-predicate.js +0 -8
- package/dist/sql-predicate.js.map +0 -1
- package/dist/table-query-capabilities.d.ts +0 -6
- package/dist/table-query-capabilities.d.ts.map +0 -1
- package/dist/table-query-capabilities.js +0 -26
- package/dist/table-query-capabilities.js.map +0 -1
- package/dist/table-query.cjs +0 -286
- package/dist/table-query.cjs.map +0 -6
- package/dist/table-query.d.ts +0 -29
- package/dist/table-query.d.ts.map +0 -1
- package/dist/table-query.js +0 -30
- package/dist/table-query.js.map +0 -1
- package/src/arrow-query.ts +0 -17
- package/src/bind-sql-predicate.ts +0 -21
- package/src/parse-sql-predicate.ts +0 -381
- package/src/query-arrow-table.ts +0 -846
- package/src/sql-predicate-schema.ts +0 -251
- package/src/sql-predicate-types.ts +0 -87
- package/src/sql-predicate-zod.ts +0 -52
- package/src/sql-predicate.ts +0 -28
- package/src/table-query-capabilities.ts +0 -29
- package/src/table-query.ts +0 -78
package/src/sql-source.ts
CHANGED
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import type {
|
|
6
|
-
import {
|
|
7
|
-
import
|
|
5
|
+
import type {Table} from 'apache-arrow';
|
|
6
|
+
import type {Schema} from '@loaders.gl/core/schema';
|
|
7
|
+
import {DataSource} from '@loaders.gl/core/loader-utils';
|
|
8
|
+
import type {CoreAPI, RequiredOptions, TableMetadata} from '@loaders.gl/core/loader-utils';
|
|
8
9
|
|
|
9
10
|
import type {
|
|
10
11
|
SQLAdapter,
|
|
@@ -18,9 +19,9 @@ import type {
|
|
|
18
19
|
SQLTableInfo
|
|
19
20
|
} from './sql-types';
|
|
20
21
|
import {compileSQLTableQuery, type SQLTableQueryDialect} from './compile-table-query';
|
|
21
|
-
import type {SQLPredicateParameterValues} from '
|
|
22
|
+
import type {SQLPredicateParameterValues} from '@loaders.gl/core/scan/sql-predicate-types';
|
|
22
23
|
import {convertRowsToArrowTable, isNodeRuntime} from './sql-utils';
|
|
23
|
-
import {SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES} from '
|
|
24
|
+
import {SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES} from '@loaders.gl/core/scan/table-query-capabilities';
|
|
24
25
|
|
|
25
26
|
const SQL_ADAPTER_FACTORIES = new Map<string, SQLAdapterFactory>();
|
|
26
27
|
|
|
@@ -108,6 +109,36 @@ export class SQLDataSource extends DataSource<string, SQLSourceOptions> {
|
|
|
108
109
|
return await adapter.listTables(options);
|
|
109
110
|
}
|
|
110
111
|
|
|
112
|
+
/** Lists stable native table identifiers without fetching any column schemas or data rows. */
|
|
113
|
+
async getTablesMetadata(options: {signal?: AbortSignal} = {}): Promise<readonly TableMetadata[]> {
|
|
114
|
+
options.signal?.throwIfAborted();
|
|
115
|
+
const tables = await this.listTables();
|
|
116
|
+
options.signal?.throwIfAborted();
|
|
117
|
+
return tables.map(table => ({name: getTableMetadataName(table), queryable: false}));
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Discovers one native table schema; ambiguous unqualified names are rejected. */
|
|
121
|
+
async getTableMetadata(
|
|
122
|
+
name: string,
|
|
123
|
+
options: {signal?: AbortSignal} = {}
|
|
124
|
+
): Promise<TableMetadata> {
|
|
125
|
+
const table = await this.resolveTable(name, options.signal);
|
|
126
|
+
const schema = await this.getTableSchema(table);
|
|
127
|
+
options.signal?.throwIfAborted();
|
|
128
|
+
return {name: getTableMetadataName(table), schema, queryable: false};
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Resolves a qualified or unique unqualified table name. */
|
|
132
|
+
protected async resolveTable(name: string, signal?: AbortSignal): Promise<SQLTableInfo> {
|
|
133
|
+
signal?.throwIfAborted();
|
|
134
|
+
const tables = await this.listTables();
|
|
135
|
+
signal?.throwIfAborted();
|
|
136
|
+
const exact = tables.filter(table => getTableMetadataName(table) === name);
|
|
137
|
+
const matches = exact.length ? exact : tables.filter(table => table.tableName === name);
|
|
138
|
+
if (matches.length !== 1) throw new Error(`Unknown or ambiguous SQL table: ${name}`);
|
|
139
|
+
return matches[0];
|
|
140
|
+
}
|
|
141
|
+
|
|
111
142
|
/** Returns the schema for a specific SQL table. */
|
|
112
143
|
async getTableSchema(options: {
|
|
113
144
|
catalogName?: string;
|
|
@@ -128,8 +159,8 @@ export class SQLDataSource extends DataSource<string, SQLSourceOptions> {
|
|
|
128
159
|
return await adapter.executeRows(compiledQuery.sql, compiledQuery.options);
|
|
129
160
|
}
|
|
130
161
|
|
|
131
|
-
/** Executes a SQL query and returns a
|
|
132
|
-
async queryArrow(query: SQLQuery, options: SQLQueryOptions = {}): Promise<
|
|
162
|
+
/** Executes a SQL query and returns a native Arrow table. */
|
|
163
|
+
async queryArrow(query: SQLQuery, options: SQLQueryOptions = {}): Promise<Table> {
|
|
133
164
|
const adapter = await this.getAdapter();
|
|
134
165
|
const compiledQuery = this.compileQuery(query, options);
|
|
135
166
|
if (adapter.executeArrow) {
|
|
@@ -226,3 +257,11 @@ function getSQLTableQueryDialect(sourceType: string): SQLTableQueryDialect {
|
|
|
226
257
|
if (sourceType === 'snowflake-sql') return 'snowflake';
|
|
227
258
|
throw new Error(`Portable table queries are not supported for SQL source type "${sourceType}".`);
|
|
228
259
|
}
|
|
260
|
+
|
|
261
|
+
/** Preserves native namespace components as one unambiguous provider-defined loaded name. */
|
|
262
|
+
function getTableMetadataName(table: SQLTableInfo): string {
|
|
263
|
+
return [table.catalogName, table.schemaName, table.tableName]
|
|
264
|
+
.filter((name): name is string => typeof name === 'string' && name.length > 0)
|
|
265
|
+
.map(name => `"${name.replaceAll('"', '""')}"`)
|
|
266
|
+
.join('.');
|
|
267
|
+
}
|
package/src/sql-types.ts
CHANGED
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import type {
|
|
6
|
-
import type {
|
|
5
|
+
import type {Table} from 'apache-arrow';
|
|
6
|
+
import type {Schema} from '@loaders.gl/core/schema';
|
|
7
|
+
import type {CoreAPI, DataSourceOptions} from '@loaders.gl/core/loader-utils';
|
|
7
8
|
import type {SQLTableQuery} from './compile-table-query';
|
|
8
9
|
|
|
9
10
|
export type SQLParameterValues = unknown[] | Record<string, unknown>;
|
|
@@ -64,6 +65,16 @@ export type SQLAdapterCapabilities = {
|
|
|
64
65
|
isDynamic?: boolean;
|
|
65
66
|
};
|
|
66
67
|
|
|
68
|
+
/** Destination for an Arrow IPC upload. */
|
|
69
|
+
export type SQLArrowInsertOptions = {
|
|
70
|
+
/** Destination table name. */
|
|
71
|
+
tableName: string;
|
|
72
|
+
/** Destination schema, defaulting to DuckDB's default schema. */
|
|
73
|
+
schemaName?: string;
|
|
74
|
+
/** Create a table when true (default); append to an existing table when false. */
|
|
75
|
+
create?: boolean;
|
|
76
|
+
};
|
|
77
|
+
|
|
67
78
|
/** Runtime adapter interface used by SQL-backed data sources. */
|
|
68
79
|
export interface SQLAdapter {
|
|
69
80
|
readonly capabilities: SQLAdapterCapabilities;
|
|
@@ -78,7 +89,9 @@ export interface SQLAdapter {
|
|
|
78
89
|
tableName: string;
|
|
79
90
|
}): Promise<Schema>;
|
|
80
91
|
executeRows(sql: string, options?: SQLQueryOptions): Promise<Record<string, unknown>[]>;
|
|
81
|
-
|
|
92
|
+
/** Uploads an Arrow IPC stream without passing Arrow runtime objects. */
|
|
93
|
+
insertArrowIPC?(bytes: Uint8Array, options: SQLArrowInsertOptions): Promise<void>;
|
|
94
|
+
executeArrow?(sql: string, options?: SQLQueryOptions): Promise<Table>;
|
|
82
95
|
}
|
|
83
96
|
|
|
84
97
|
/** Context passed to SQL adapter factories during source initialization. */
|
package/src/sql-utils.ts
CHANGED
|
@@ -2,9 +2,7 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import
|
|
6
|
-
import type {ArrowTable, DataType, Field, ObjectRowTable, Schema} from '@loaders.gl/schema';
|
|
7
|
-
import {makeTableFromData, getDataTypeFromArray} from '@loaders.gl/schema-utils';
|
|
5
|
+
import type {DataType, Schema} from '@loaders.gl/core/schema';
|
|
8
6
|
|
|
9
7
|
import type {SQLColumnInfo} from './sql-types';
|
|
10
8
|
|
|
@@ -23,28 +21,7 @@ export function quoteSqlIdentifier(identifier: string): string {
|
|
|
23
21
|
return `"${identifier.replace(/"/g, '""')}"`;
|
|
24
22
|
}
|
|
25
23
|
|
|
26
|
-
|
|
27
|
-
export function convertRowsToArrowTable(rows: Record<string, unknown>[]): ArrowTable {
|
|
28
|
-
const table: ObjectRowTable =
|
|
29
|
-
rows.length > 0
|
|
30
|
-
? makeTableFromData(rows)
|
|
31
|
-
: {shape: 'object-row-table', schema: {fields: [], metadata: {}}, data: rows};
|
|
32
|
-
const schema = deducePrimitiveAwareSchema(table);
|
|
33
|
-
|
|
34
|
-
if (table.data.length === 0 || schema.fields.length === 0) {
|
|
35
|
-
return {
|
|
36
|
-
shape: 'arrow-table',
|
|
37
|
-
schema,
|
|
38
|
-
data: new arrow.Table(new arrow.Schema([]))
|
|
39
|
-
};
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
return {
|
|
43
|
-
shape: 'arrow-table',
|
|
44
|
-
schema,
|
|
45
|
-
data: arrow.tableFromJSON(table.data)
|
|
46
|
-
};
|
|
47
|
-
}
|
|
24
|
+
export {convertRowsToArrowTable} from '@loaders.gl/core/scan/rows-to-arrow';
|
|
48
25
|
|
|
49
26
|
/** Converts normalized SQL columns into a loaders.gl schema. */
|
|
50
27
|
export function convertSQLColumnsToSchema(columns: SQLColumnInfo[]): Schema {
|
|
@@ -109,21 +86,6 @@ function mapSQLTypeToDataType(sqlType: string): DataType {
|
|
|
109
86
|
return 'utf8';
|
|
110
87
|
}
|
|
111
88
|
|
|
112
|
-
function deducePrimitiveAwareSchema(table: ObjectRowTable): Schema {
|
|
113
|
-
return {
|
|
114
|
-
...table.schema!,
|
|
115
|
-
fields: table.schema!.fields.map((field): Field => {
|
|
116
|
-
if (field.type !== 'null') {
|
|
117
|
-
return field;
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
const columnValues = table.data.map(row => row[field.name]);
|
|
121
|
-
const inferredType = getDataTypeFromArray(columnValues);
|
|
122
|
-
return {...field, type: inferredType.type, nullable: inferredType.nullable};
|
|
123
|
-
})
|
|
124
|
-
};
|
|
125
|
-
}
|
|
126
|
-
|
|
127
89
|
/** Returns a fully qualified SQL object name. */
|
|
128
90
|
export function getQualifiedTableName(options: {
|
|
129
91
|
catalogName?: string;
|