@loaders.gl/sql 5.0.0-beta.10 → 5.0.0-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow-table-source.js +1 -1
- package/dist/arrow-table-source.js.map +1 -1
- package/dist/duckdb-sql-source.js +1 -1
- package/dist/duckdb-table-query.d.ts.map +1 -1
- package/dist/duckdb-table-query.js +14 -23
- package/dist/duckdb-table-query.js.map +1 -1
- package/dist/index.cjs +206 -251
- package/dist/index.cjs.map +2 -2
- package/dist/snowflake-sql-source.js +1 -1
- package/package.json +2 -2
- package/src/arrow-table-source.ts +1 -1
- package/src/duckdb-table-query.ts +22 -28
|
@@ -6,14 +6,13 @@ import {RecordBatch, Struct, Table, makeData, vectorFromArray} from 'apache-arro
|
|
|
6
6
|
import {
|
|
7
7
|
createQueryMetadata,
|
|
8
8
|
executeTableScanBatches,
|
|
9
|
-
makeTableScanBatch,
|
|
10
9
|
validateTableQueryLimit,
|
|
11
10
|
type TableQuerySource,
|
|
12
11
|
type TableQueryPushdownRequest,
|
|
13
12
|
type TableQueryReadOptions
|
|
14
13
|
} from '@loaders.gl/core/loader-utils';
|
|
15
14
|
import {filterArrowTable} from '@loaders.gl/core/query/arrow-execution';
|
|
16
|
-
import type {
|
|
15
|
+
import type {Field, Schema} from '@loaders.gl/core/schema';
|
|
17
16
|
import {convertSchemaToArrow, makeArrowMetadataBatch} from '@loaders.gl/core/schema-utils';
|
|
18
17
|
import type {DuckDBSQLDataSource} from './duckdb-sql-source';
|
|
19
18
|
import type {SQLTableInfo} from './sql-types';
|
|
@@ -108,20 +107,14 @@ export function makeDuckDBTableQuerySource(
|
|
|
108
107
|
field.type
|
|
109
108
|
)
|
|
110
109
|
);
|
|
111
|
-
yield
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
length: rows.length,
|
|
120
|
-
children: vectors.map(vector => vector.data[0])
|
|
121
|
-
})
|
|
122
|
-
)
|
|
123
|
-
])
|
|
124
|
-
});
|
|
110
|
+
yield new RecordBatch(
|
|
111
|
+
arrowSchema,
|
|
112
|
+
makeData({
|
|
113
|
+
type: new Struct(arrowSchema.fields),
|
|
114
|
+
length: rows.length,
|
|
115
|
+
children: vectors.map(vector => vector.data[0])
|
|
116
|
+
})
|
|
117
|
+
);
|
|
125
118
|
}
|
|
126
119
|
|
|
127
120
|
/** Compiles the exact subset without evaluating data or changing the connection. */
|
|
@@ -191,9 +184,7 @@ export function makeDuckDBTableQuerySource(
|
|
|
191
184
|
schema: outputSchema,
|
|
192
185
|
/** Executes the planned SQL when the consumer starts reading. */
|
|
193
186
|
async *read(options?: {signal?: AbortSignal}) {
|
|
194
|
-
|
|
195
|
-
yield* data.batches.length ? data.batches : [makeArrowMetadataBatch(data.schema)];
|
|
196
|
-
}
|
|
187
|
+
yield* readStatement(sql, outputSchema, options?.signal);
|
|
197
188
|
}
|
|
198
189
|
};
|
|
199
190
|
}
|
|
@@ -220,22 +211,25 @@ export function makeDuckDBTableQuerySource(
|
|
|
220
211
|
yield* accepted.read({signal});
|
|
221
212
|
return;
|
|
222
213
|
}
|
|
223
|
-
for await (const batch of executeTableScanBatches(
|
|
224
|
-
|
|
214
|
+
for await (const batch of executeTableScanBatches<Table>(
|
|
215
|
+
async function* (readSignal) {
|
|
216
|
+
for await (const batch of readStatement(
|
|
217
|
+
`SELECT * FROM ${tableName}${order}`,
|
|
218
|
+
schema,
|
|
219
|
+
readSignal
|
|
220
|
+
)) {
|
|
221
|
+
yield new Table(batch.schema, [batch]);
|
|
222
|
+
}
|
|
223
|
+
},
|
|
225
224
|
options,
|
|
226
225
|
{
|
|
227
226
|
filter: (batch, predicate, filterSignal) => {
|
|
228
227
|
if (!predicate) return batch;
|
|
229
|
-
|
|
230
|
-
return makeTableScanBatch({
|
|
231
|
-
...arrowBatch,
|
|
232
|
-
data: filterArrowTable(arrowBatch.data, predicate, {signal: filterSignal})
|
|
233
|
-
});
|
|
228
|
+
return filterArrowTable(batch, predicate, {signal: filterSignal});
|
|
234
229
|
}
|
|
235
230
|
}
|
|
236
231
|
)) {
|
|
237
|
-
|
|
238
|
-
yield* data.batches.length ? data.batches : [makeArrowMetadataBatch(data.schema)];
|
|
232
|
+
yield* batch.batches.length ? batch.batches : [makeArrowMetadataBatch(batch.schema)];
|
|
239
233
|
}
|
|
240
234
|
}
|
|
241
235
|
};
|