@walkthru-earth/objex 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/components/browser/FileBrowser.svelte +25 -14
- package/dist/components/browser/FileTreeSidebar.svelte +42 -6
- package/dist/components/layout/ConnectionDialog.svelte +100 -1
- package/dist/components/layout/Sidebar.svelte +43 -25
- package/dist/components/viewers/CodeViewer.svelte +23 -0
- package/dist/components/viewers/CogControls.svelte +208 -0
- package/dist/components/viewers/CogControls.svelte.d.ts +12 -0
- package/dist/components/viewers/CogViewer.svelte +353 -1160
- package/dist/components/viewers/CogViewer.svelte.d.ts +1 -1
- package/dist/components/viewers/DatabaseViewer.svelte +345 -37
- package/dist/components/viewers/MarkdownViewer.svelte +1 -1
- package/dist/components/viewers/TableViewer.svelte +123 -41
- package/dist/components/viewers/ZarrMapViewer.svelte +29 -0
- package/dist/components/viewers/ZarrViewer.svelte +1 -4
- package/dist/constants.d.ts +6 -2
- package/dist/constants.js +6 -2
- package/dist/file-icons/index.d.ts +1 -1
- package/dist/file-icons/index.js +12 -2
- package/dist/i18n/ar.js +24 -0
- package/dist/i18n/en.js +24 -0
- package/dist/i18n/index.svelte.d.ts +0 -1
- package/dist/i18n/index.svelte.js +0 -3
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1 -0
- package/dist/query/engine.d.ts +20 -4
- package/dist/query/index.d.ts +2 -1
- package/dist/query/index.js +1 -0
- package/dist/query/source.d.ts +30 -0
- package/dist/query/source.js +37 -0
- package/dist/query/wasm.d.ts +7 -5
- package/dist/query/wasm.js +138 -85
- package/dist/storage/providers.d.ts +47 -0
- package/dist/storage/providers.js +160 -0
- package/dist/stores/files.svelte.d.ts +1 -2
- package/dist/stores/files.svelte.js +1 -2
- package/dist/stores/tabs.svelte.d.ts +9 -2
- package/dist/stores/tabs.svelte.js +11 -2
- package/dist/types.d.ts +11 -0
- package/dist/utils/cog.d.ts +244 -0
- package/dist/utils/cog.js +1039 -0
- package/dist/utils/deck.d.ts +0 -18
- package/dist/utils/deck.js +0 -36
- package/dist/utils/geometry-type.d.ts +52 -0
- package/dist/utils/geometry-type.js +76 -0
- package/dist/utils/markdown-sql.d.ts +1 -1
- package/dist/utils/markdown-sql.js +3 -4
- package/dist/utils/pmtiles-tile.d.ts +0 -2
- package/dist/utils/pmtiles-tile.js +0 -8
- package/dist/utils/url-state.d.ts +6 -0
- package/dist/utils/url-state.js +34 -26
- package/dist/utils/url.d.ts +13 -9
- package/dist/utils/url.js +16 -25
- package/dist/utils/zarr-tab.d.ts +22 -0
- package/dist/utils/zarr-tab.js +30 -0
- package/dist/utils/zarr.d.ts +0 -2
- package/dist/utils/zarr.js +73 -44
- package/package.json +47 -43
- package/dist/components/ui/tabs/index.d.ts +0 -5
- package/dist/components/ui/tabs/index.js +0 -7
- package/dist/components/ui/tabs/tabs-content.svelte +0 -17
- package/dist/components/ui/tabs/tabs-content.svelte.d.ts +0 -4
- package/dist/components/ui/tabs/tabs-list.svelte +0 -16
- package/dist/components/ui/tabs/tabs-list.svelte.d.ts +0 -4
- package/dist/components/ui/tabs/tabs-trigger.svelte +0 -20
- package/dist/components/ui/tabs/tabs-trigger.svelte.d.ts +0 -4
- package/dist/components/ui/tabs/tabs.svelte +0 -19
- package/dist/components/ui/tabs/tabs.svelte.d.ts +0 -4
- package/dist/components/viewers/MapViewer.svelte +0 -234
- package/dist/components/viewers/MapViewer.svelte.d.ts +0 -7
- package/dist/components/viewers/StyleEditorOverlay.svelte +0 -27
- package/dist/components/viewers/StyleEditorOverlay.svelte.d.ts +0 -7
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve a Tab to a QuerySource — the abstraction every engine helper
|
|
3
|
+
* consumes. A tab is either file-backed (path → `read_parquet('url')`) or
|
|
4
|
+
* SQL-backed (`tab.sourceRef` is a pre-built FROM-clause target, such as an
|
|
5
|
+
* attached DuckLake table).
|
|
6
|
+
*
|
|
7
|
+
* This module is the only place that knows how to map a Tab to both a SQL
|
|
8
|
+
* FROM target AND a resolved file URL, so TableViewer and DatabaseViewer
|
|
9
|
+
* stay free of ad-hoc branching.
|
|
10
|
+
*/
|
|
11
|
+
import type { Tab } from '../types.js';
|
|
12
|
+
import type { QuerySource } from './engine.js';
|
|
13
|
+
export interface ResolvedTableSource extends QuerySource {
|
|
14
|
+
/**
|
|
15
|
+
* True when the tab is file-backed and hyparquet / parquet metadata
|
|
16
|
+
* shortcuts apply. False for SQL-backed sources like attached DuckLake
|
|
17
|
+
* tables.
|
|
18
|
+
*/
|
|
19
|
+
isFileSource: boolean;
|
|
20
|
+
/** File URL used for hyparquet metadata fetches. Null for SQL-backed sources. */
|
|
21
|
+
fileUrl: string | null;
|
|
22
|
+
/** Display label, typically the tab name. */
|
|
23
|
+
label: string;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Resolve a tab to its QuerySource. Must be called lazily (inside reactive
|
|
27
|
+
* expressions or functions) because `tab.sourceRef` and `tab.path` can change
|
|
28
|
+
* over a tab's lifetime.
|
|
29
|
+
*/
|
|
30
|
+
export declare function resolveTableSource(tab: Tab): ResolvedTableSource;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve a Tab to a QuerySource — the abstraction every engine helper
|
|
3
|
+
* consumes. A tab is either file-backed (path → `read_parquet('url')`) or
|
|
4
|
+
* SQL-backed (`tab.sourceRef` is a pre-built FROM-clause target, such as an
|
|
5
|
+
* attached DuckLake table).
|
|
6
|
+
*
|
|
7
|
+
* This module is the only place that knows how to map a Tab to both a SQL
|
|
8
|
+
* FROM target AND a resolved file URL, so TableViewer and DatabaseViewer
|
|
9
|
+
* stay free of ad-hoc branching.
|
|
10
|
+
*/
|
|
11
|
+
import { buildDuckDbSource } from '../file-icons/index.js';
|
|
12
|
+
import { buildDuckDbUrl } from '../utils/url.js';
|
|
13
|
+
/**
|
|
14
|
+
* Resolve a tab to its QuerySource. Must be called lazily (inside reactive
|
|
15
|
+
* expressions or functions) because `tab.sourceRef` and `tab.path` can change
|
|
16
|
+
* over a tab's lifetime.
|
|
17
|
+
*/
|
|
18
|
+
export function resolveTableSource(tab) {
|
|
19
|
+
if (tab.sourceRef) {
|
|
20
|
+
return {
|
|
21
|
+
ref: tab.sourceRef,
|
|
22
|
+
filePath: undefined,
|
|
23
|
+
isFileSource: false,
|
|
24
|
+
fileUrl: null,
|
|
25
|
+
label: tab.name
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
const fileUrl = buildDuckDbUrl(tab);
|
|
29
|
+
const ref = buildDuckDbSource(tab.path, fileUrl);
|
|
30
|
+
return {
|
|
31
|
+
ref,
|
|
32
|
+
filePath: tab.path,
|
|
33
|
+
isFileSource: true,
|
|
34
|
+
fileUrl,
|
|
35
|
+
label: tab.name
|
|
36
|
+
};
|
|
37
|
+
}
|
package/dist/query/wasm.d.ts
CHANGED
|
@@ -1,20 +1,22 @@
|
|
|
1
|
-
import { type MapQueryHandle, type MapQueryResult, type QueryEngine, type QueryHandle, type QueryResult, type SchemaField } from './engine';
|
|
1
|
+
import { type MapQueryHandle, type MapQueryResult, type QueryEngine, type QueryHandle, type QueryResult, type QuerySource, type SchemaField } from './engine';
|
|
2
2
|
export declare class WasmQueryEngine implements QueryEngine {
|
|
3
3
|
query(connId: string, sql: string): Promise<QueryResult>;
|
|
4
4
|
queryForMap(connId: string, sql: string, geomCol: string, geomColType: string, sourceCrs?: string | null): Promise<MapQueryResult>;
|
|
5
|
-
getSchema(connId: string,
|
|
6
|
-
getRowCount(connId: string,
|
|
7
|
-
getSchemaAndCrs(connId: string,
|
|
5
|
+
getSchema(connId: string, source: QuerySource): Promise<SchemaField[]>;
|
|
6
|
+
getRowCount(connId: string, source: QuerySource): Promise<number>;
|
|
7
|
+
getSchemaAndCrs(connId: string, source: QuerySource, findGeoCol: (schema: SchemaField[]) => string | null): Promise<{
|
|
8
8
|
schema: SchemaField[];
|
|
9
9
|
geomCol: string | null;
|
|
10
10
|
crs: string | null;
|
|
11
11
|
}>;
|
|
12
12
|
private configureStorage;
|
|
13
|
-
detectCrs(connId: string,
|
|
13
|
+
detectCrs(connId: string, source: QuerySource, geomCol: string): Promise<string | null>;
|
|
14
14
|
private detectCrsWithConn;
|
|
15
15
|
queryCancellable(connId: string, sql: string): QueryHandle;
|
|
16
16
|
queryForMapCancellable(connId: string, sql: string, geomCol: string, geomColType: string, sourceCrs?: string | null): MapQueryHandle;
|
|
17
17
|
forceCancel(): Promise<void>;
|
|
18
|
+
registerFileBuffer(name: string, buffer: Uint8Array): Promise<void>;
|
|
19
|
+
dropFile(name: string): Promise<void>;
|
|
18
20
|
releaseMemory(): Promise<void>;
|
|
19
21
|
dispose(): Promise<void>;
|
|
20
22
|
}
|
package/dist/query/wasm.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { DEFAULT_TARGET_CRS, DUCKDB_INIT_TIMEOUT_MS, WGS84_CODES } from '../constants.js';
|
|
2
|
-
import {
|
|
2
|
+
import { getAccessMode } from '../storage/providers.js';
|
|
3
3
|
import { credentialStore } from '../stores/credentials.svelte.js';
|
|
4
|
+
import { buildTransformExpr, wrapWkbWithCrs } from '../utils/geometry-type.js';
|
|
4
5
|
import { QueryCancelledError } from './engine';
|
|
5
6
|
const DUCKDB_VERSION = __DUCKDB_WASM_VERSION__;
|
|
6
7
|
const CDN_BASE = `https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@${DUCKDB_VERSION}/dist`;
|
|
@@ -66,21 +67,29 @@ async function getDB() {
|
|
|
66
67
|
const conn = await db.connect();
|
|
67
68
|
try {
|
|
68
69
|
const tExt = performance.now();
|
|
69
|
-
|
|
70
|
-
//
|
|
71
|
-
//
|
|
72
|
-
// "
|
|
73
|
-
|
|
74
|
-
//
|
|
75
|
-
//
|
|
70
|
+
// Workaround for duckdb/duckdb-wasm#2199: calling duckdb_coordinate_systems()
|
|
71
|
+
// FIRST autoloads spatial and registers default CRS properly. Running it
|
|
72
|
+
// AFTER an explicit LOAD spatial triggers a timing bug in PROJ (WASM-only)
|
|
73
|
+
// that causes "stoi: no conversion" on any GeoParquet with CRS metadata.
|
|
74
|
+
await withTimeout(conn.query('SELECT * FROM duckdb_coordinate_systems(); INSTALL httpfs; LOAD httpfs; INSTALL spatial; LOAD spatial;'), INIT_TIMEOUT_MS, 'extension install (httpfs + spatial)');
|
|
75
|
+
// DuckDB v1.5: GEOMETRY is a core type with optional CRS parameter.
|
|
76
|
+
// SET geometry_always_xy = true forces lon/lat (x/y) axis order globally,
|
|
77
|
+
// matching GeoJSON/GeoParquet convention. Without this, DuckDB v1.5 emits
|
|
78
|
+
// warnings on ST_Transform and other coordinate-sensitive functions.
|
|
76
79
|
// SET GLOBAL applies to all future connections (no per-connection overhead).
|
|
77
80
|
try {
|
|
78
|
-
await conn.query('SET GLOBAL
|
|
79
|
-
geoConversionGlobal = true;
|
|
81
|
+
await conn.query('SET GLOBAL geometry_always_xy = true');
|
|
80
82
|
}
|
|
81
83
|
catch {
|
|
82
|
-
|
|
83
|
-
|
|
84
|
+
logWarn('geometry_always_xy not available — ST_Transform calls may warn');
|
|
85
|
+
}
|
|
86
|
+
// Raise httpfs force_download threshold so moderately sized remote files
|
|
87
|
+
// are fetched in one shot instead of many small range requests.
|
|
88
|
+
try {
|
|
89
|
+
await conn.query('SET GLOBAL force_download_threshold = 2000000');
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
logWarn('force_download_threshold not available');
|
|
84
93
|
}
|
|
85
94
|
log(`getDB → extensions loaded in ${elapsed(tExt)}`);
|
|
86
95
|
}
|
|
@@ -96,19 +105,41 @@ async function getDB() {
|
|
|
96
105
|
});
|
|
97
106
|
return dbPromise;
|
|
98
107
|
}
|
|
99
|
-
|
|
108
|
+
// ─── Geometry type helpers ────────────────────────────────────────────
|
|
100
109
|
/**
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
* Otherwise falls back to per-connection SET.
|
|
110
|
+
* Check if a DuckDB column type string is a spatial type that ST_AsWKB accepts directly.
|
|
111
|
+
* Handles DuckDB v1.5 parameterized types like GEOMETRY('EPSG:4326').
|
|
104
112
|
*/
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
113
|
+
function isSpatialColumnType(typeUpper) {
|
|
114
|
+
return (typeUpper.startsWith('GEOMETRY') ||
|
|
115
|
+
typeUpper.startsWith('GEOGRAPHY') ||
|
|
116
|
+
typeUpper === 'WKB_BLOB' ||
|
|
117
|
+
typeUpper.includes('POINT') ||
|
|
118
|
+
typeUpper.includes('LINESTRING') ||
|
|
119
|
+
typeUpper.includes('POLYGON') ||
|
|
120
|
+
typeUpper.includes('BINARY') // Arrow serialization of DuckDB GEOMETRY
|
|
121
|
+
);
|
|
109
122
|
}
|
|
110
123
|
// ─── CRS detection helpers ───────────────────────────────────────────
|
|
111
124
|
// WGS84_CODES imported from constants.ts
|
|
125
|
+
/**
|
|
126
|
+
* Extract CRS from DuckDB v1.5 parameterized GEOMETRY type string.
|
|
127
|
+
* e.g., "GEOMETRY('EPSG:4326')" → { found: true, crs: null } (WGS84)
|
|
128
|
+
* e.g., "GEOMETRY('EPSG:27700')" → { found: true, crs: 'EPSG:27700' }
|
|
129
|
+
* e.g., "GEOMETRY" → { found: false, crs: null } (no CRS in type)
|
|
130
|
+
*/
|
|
131
|
+
function extractCrsFromTypeString(typeStr) {
|
|
132
|
+
const match = typeStr.match(/^GEOMETRY\('([^']+)'\)/i);
|
|
133
|
+
if (!match)
|
|
134
|
+
return { found: false, crs: null };
|
|
135
|
+
const crs = match[1];
|
|
136
|
+
if (crs === 'EPSG:4326' || crs === 'OGC:CRS84')
|
|
137
|
+
return { found: true, crs: null };
|
|
138
|
+
const epsgMatch = crs.match(/^EPSG:(\d+)$/);
|
|
139
|
+
if (epsgMatch && WGS84_CODES.has(Number(epsgMatch[1])))
|
|
140
|
+
return { found: true, crs: null };
|
|
141
|
+
return { found: true, crs };
|
|
142
|
+
}
|
|
112
143
|
/** Extract EPSG code from a PROJJSON object. Returns null for WGS84/CRS84. */
|
|
113
144
|
function extractEpsgFromProjjson(crs) {
|
|
114
145
|
if (!crs)
|
|
@@ -264,7 +295,6 @@ export class WasmQueryEngine {
|
|
|
264
295
|
log(`query → ${sqlPreview}`);
|
|
265
296
|
const db = await getDB();
|
|
266
297
|
const conn = await db.connect();
|
|
267
|
-
await ensureGeoConversionDisabled(conn);
|
|
268
298
|
const tConn = performance.now();
|
|
269
299
|
log(`query → connected in ${elapsed(t0)}`);
|
|
270
300
|
try {
|
|
@@ -316,50 +346,39 @@ export class WasmQueryEngine {
|
|
|
316
346
|
log(`queryForMap → geomCol: ${geomCol}, type: ${geomColType}, crs: ${sourceCrs ?? 'WGS84'}`);
|
|
317
347
|
const db = await getDB();
|
|
318
348
|
const conn = await db.connect();
|
|
319
|
-
await ensureGeoConversionDisabled(conn);
|
|
320
349
|
try {
|
|
321
350
|
if (connId) {
|
|
322
351
|
await this.configureStorage(conn, connId);
|
|
323
352
|
}
|
|
324
353
|
// Build geometry expression based on column type:
|
|
325
|
-
// - Native spatial types (GEOMETRY,
|
|
326
|
-
// - BLOB/BINARY →
|
|
354
|
+
// - Native spatial types (GEOMETRY, GEOMETRY('EPSG:...'), WKB_BLOB, etc.) → use directly
|
|
355
|
+
// - BLOB/BINARY → need explicit ST_GeomFromWKB
|
|
327
356
|
// - Everything else (VARCHAR, JSON, STRUCT, ...) → GeoJSON text
|
|
328
357
|
const quoted = `"${geomCol}"`;
|
|
329
358
|
const upper = geomColType.toUpperCase();
|
|
330
|
-
|
|
331
|
-
// Includes Arrow "Binary"/"LargeBinary" — DuckDB GEOMETRY columns from
|
|
332
|
-
// ST_ReadSHP/ST_Read appear as Arrow Binary but are NOT WKB blobs.
|
|
333
|
-
const isSpatialType = upper === 'GEOMETRY' ||
|
|
334
|
-
upper === 'GEOGRAPHY' ||
|
|
335
|
-
upper === 'WKB_BLOB' ||
|
|
336
|
-
upper.includes('POINT') ||
|
|
337
|
-
upper.includes('LINESTRING') ||
|
|
338
|
-
upper.includes('POLYGON') ||
|
|
339
|
-
upper.includes('BINARY'); // Arrow serialization of DuckDB GEOMETRY
|
|
340
|
-
// Actual WKB BLOB columns (e.g. GeoParquet) need explicit ST_GeomFromWKB
|
|
341
|
-
// because DuckDB has no implicit BLOB→GEOMETRY cast.
|
|
359
|
+
const spatialType = isSpatialColumnType(upper);
|
|
342
360
|
const isWkbBlob = upper === 'BLOB' || upper === 'BYTEA';
|
|
343
361
|
let wkbExpr;
|
|
344
362
|
let geomExpr;
|
|
345
363
|
if (isWkbBlob && !sourceCrs) {
|
|
346
364
|
// Already WKB — use directly, no spatial function calls needed.
|
|
347
|
-
// Avoids ST_GeomFromWKB which can fail if DuckDB auto-converted
|
|
348
|
-
// the column to GEOMETRY despite enable_geoparquet_conversion=false.
|
|
349
365
|
wkbExpr = quoted;
|
|
350
366
|
geomExpr = null; // geometry type detected client-side from WKB headers
|
|
351
367
|
}
|
|
352
368
|
else {
|
|
353
|
-
|
|
369
|
+
// For BLOB inputs with a known source CRS, attach it via ST_SetCRS so
|
|
370
|
+
// downstream ST_Transform can use the 2-arg form and the CRS propagates
|
|
371
|
+
// through any subsequent spatial ops (DuckDB v1.5+).
|
|
372
|
+
geomExpr = spatialType
|
|
354
373
|
? quoted
|
|
355
374
|
: isWkbBlob
|
|
356
|
-
?
|
|
375
|
+
? wrapWkbWithCrs(quoted, sourceCrs)
|
|
357
376
|
: `ST_GeomFromGeoJSON(${quoted})`;
|
|
358
377
|
// Re-project to WGS84 if the source CRS is not EPSG:4326/CRS84.
|
|
359
|
-
//
|
|
360
|
-
// target, matching the GeoParquet convention regardless of CRS authority.
|
|
378
|
+
// geometry_always_xy is set globally at DB init, so no per-call always_xy needed.
|
|
361
379
|
if (sourceCrs) {
|
|
362
|
-
|
|
380
|
+
const effectiveType = isWkbBlob ? `GEOMETRY('${sourceCrs}')` : geomColType;
|
|
381
|
+
geomExpr = buildTransformExpr(geomExpr, effectiveType, sourceCrs, DEFAULT_TARGET_CRS);
|
|
363
382
|
}
|
|
364
383
|
// ST_AsWKB needed — DuckDB GEOMETRY columns (from ST_ReadSHP, ST_Read)
|
|
365
384
|
// use an internal binary format, not WKB, even though Arrow reports Binary type.
|
|
@@ -416,18 +435,16 @@ export class WasmQueryEngine {
|
|
|
416
435
|
await conn.close();
|
|
417
436
|
}
|
|
418
437
|
}
|
|
419
|
-
async getSchema(connId,
|
|
438
|
+
async getSchema(connId, source) {
|
|
420
439
|
const t0 = performance.now();
|
|
421
|
-
log('getSchema →',
|
|
440
|
+
log('getSchema →', source.ref);
|
|
422
441
|
const db = await getDB();
|
|
423
442
|
const conn = await db.connect();
|
|
424
|
-
await ensureGeoConversionDisabled(conn);
|
|
425
443
|
try {
|
|
426
444
|
if (connId) {
|
|
427
445
|
await this.configureStorage(conn, connId);
|
|
428
446
|
}
|
|
429
|
-
const
|
|
430
|
-
const result = await conn.query(`DESCRIBE SELECT * FROM ${source}`);
|
|
447
|
+
const result = await conn.query(`DESCRIBE SELECT * FROM ${source.ref}`);
|
|
431
448
|
const rows = result.toArray();
|
|
432
449
|
const schema = rows.map((row) => ({
|
|
433
450
|
name: row.column_name,
|
|
@@ -445,12 +462,11 @@ export class WasmQueryEngine {
|
|
|
445
462
|
await conn.close();
|
|
446
463
|
}
|
|
447
464
|
}
|
|
448
|
-
async getRowCount(connId,
|
|
465
|
+
async getRowCount(connId, source) {
|
|
449
466
|
const t0 = performance.now();
|
|
450
|
-
log('getRowCount →',
|
|
467
|
+
log('getRowCount →', source.ref);
|
|
451
468
|
const db = await getDB();
|
|
452
469
|
const conn = await db.connect();
|
|
453
|
-
await ensureGeoConversionDisabled(conn);
|
|
454
470
|
try {
|
|
455
471
|
if (connId) {
|
|
456
472
|
await this.configureStorage(conn, connId);
|
|
@@ -458,10 +474,12 @@ export class WasmQueryEngine {
|
|
|
458
474
|
// For Parquet files, try reading row count from file footer metadata first.
|
|
459
475
|
// This avoids parsing column types (which can fail on exotic geometry types)
|
|
460
476
|
// and is faster than SELECT COUNT(*) since it reads only footer bytes.
|
|
461
|
-
|
|
462
|
-
|
|
477
|
+
// Only applicable when we have a concrete file path — SQL-backed sources
|
|
478
|
+
// (attached DuckLake/DuckDB/SQLite tables) fall through to COUNT(*).
|
|
479
|
+
const isParquet = source.filePath ? /\.parquet$/i.test(source.filePath) : false;
|
|
480
|
+
if (isParquet && source.filePath) {
|
|
463
481
|
try {
|
|
464
|
-
const metaResult = await conn.query(`SELECT SUM(num_rows)::BIGINT as cnt FROM parquet_file_metadata('${
|
|
482
|
+
const metaResult = await conn.query(`SELECT SUM(num_rows)::BIGINT as cnt FROM parquet_file_metadata('${source.filePath}')`);
|
|
465
483
|
const metaRows = metaResult.toArray();
|
|
466
484
|
const count = Number(metaRows[0].cnt);
|
|
467
485
|
log(`getRowCount → ${count} via parquet_file_metadata in ${elapsed(t0)}`);
|
|
@@ -471,8 +489,7 @@ export class WasmQueryEngine {
|
|
|
471
489
|
logWarn('getRowCount → parquet_file_metadata failed, falling back to COUNT(*):', metaErr?.message ?? metaErr);
|
|
472
490
|
}
|
|
473
491
|
}
|
|
474
|
-
const
|
|
475
|
-
const result = await conn.query(`SELECT COUNT(*) as cnt FROM ${source}`);
|
|
492
|
+
const result = await conn.query(`SELECT COUNT(*) as cnt FROM ${source.ref}`);
|
|
476
493
|
const rows = result.toArray();
|
|
477
494
|
const count = Number(rows[0].cnt);
|
|
478
495
|
log(`getRowCount → ${count} via COUNT(*) in ${elapsed(t0)}`);
|
|
@@ -486,20 +503,18 @@ export class WasmQueryEngine {
|
|
|
486
503
|
await conn.close();
|
|
487
504
|
}
|
|
488
505
|
}
|
|
489
|
-
async getSchemaAndCrs(connId,
|
|
506
|
+
async getSchemaAndCrs(connId, source, findGeoCol) {
|
|
490
507
|
const t0 = performance.now();
|
|
491
|
-
log('getSchemaAndCrs →',
|
|
508
|
+
log('getSchemaAndCrs →', source.ref);
|
|
492
509
|
const db = await getDB();
|
|
493
510
|
const conn = await db.connect();
|
|
494
|
-
await ensureGeoConversionDisabled(conn);
|
|
495
511
|
try {
|
|
496
512
|
if (connId) {
|
|
497
513
|
await this.configureStorage(conn, connId);
|
|
498
514
|
}
|
|
499
515
|
// Schema detection
|
|
500
516
|
const tSchema = performance.now();
|
|
501
|
-
const
|
|
502
|
-
const result = await conn.query(`DESCRIBE SELECT * FROM ${source}`);
|
|
517
|
+
const result = await conn.query(`DESCRIBE SELECT * FROM ${source.ref}`);
|
|
503
518
|
const schemaRows = result.toArray();
|
|
504
519
|
const schema = schemaRows.map((row) => ({
|
|
505
520
|
name: row.column_name,
|
|
@@ -516,7 +531,7 @@ export class WasmQueryEngine {
|
|
|
516
531
|
// CRS detection reusing the same connection
|
|
517
532
|
log(`getSchemaAndCrs → geo column: ${geomCol}, detecting CRS...`);
|
|
518
533
|
const tCrs = performance.now();
|
|
519
|
-
const crs = await this.detectCrsWithConn(conn,
|
|
534
|
+
const crs = await this.detectCrsWithConn(conn, source, geomCol);
|
|
520
535
|
log(`getSchemaAndCrs → CRS: ${crs ?? 'WGS84/null'} in ${elapsed(tCrs)}, total ${elapsed(t0)}`);
|
|
521
536
|
return { schema, geomCol, crs };
|
|
522
537
|
}
|
|
@@ -542,9 +557,13 @@ export class WasmQueryEngine {
|
|
|
542
557
|
logWarn(`configureStorage → connection "${connId}" not found`);
|
|
543
558
|
return;
|
|
544
559
|
}
|
|
545
|
-
//
|
|
546
|
-
|
|
547
|
-
|
|
560
|
+
// For public and SAS-signed connections DuckDB hits the HTTPS URL
|
|
561
|
+
// directly — no S3 signing config needed. Saves a worker round-trip
|
|
562
|
+
// on every query for anonymous/public buckets (AWS, GCS, R2, etc.)
|
|
563
|
+
// and Azure Blob (SAS token embedded in the URL).
|
|
564
|
+
const mode = getAccessMode(connection);
|
|
565
|
+
if (mode !== 'signed-s3') {
|
|
566
|
+
log(`configureStorage → ${mode}, skipping S3 config`);
|
|
548
567
|
return;
|
|
549
568
|
}
|
|
550
569
|
// Batch all SET commands into a single query to minimize web worker round-trips
|
|
@@ -577,17 +596,16 @@ export class WasmQueryEngine {
|
|
|
577
596
|
console.error(LOG_PREFIX, 'configureStorage error:', err);
|
|
578
597
|
}
|
|
579
598
|
}
|
|
580
|
-
async detectCrs(connId,
|
|
599
|
+
async detectCrs(connId, source, geomCol) {
|
|
581
600
|
const t0 = performance.now();
|
|
582
|
-
log(`detectCrs → standalone call for "${geomCol}"`,
|
|
601
|
+
log(`detectCrs → standalone call for "${geomCol}"`, source.ref);
|
|
583
602
|
const db = await getDB();
|
|
584
603
|
const conn = await db.connect();
|
|
585
|
-
await ensureGeoConversionDisabled(conn);
|
|
586
604
|
try {
|
|
587
605
|
if (connId) {
|
|
588
606
|
await this.configureStorage(conn, connId);
|
|
589
607
|
}
|
|
590
|
-
const crs = await this.detectCrsWithConn(conn,
|
|
608
|
+
const crs = await this.detectCrsWithConn(conn, source, geomCol);
|
|
591
609
|
log(`detectCrs → ${crs ?? 'WGS84/null'} in ${elapsed(t0)}`);
|
|
592
610
|
return crs;
|
|
593
611
|
}
|
|
@@ -599,7 +617,36 @@ export class WasmQueryEngine {
|
|
|
599
617
|
await conn.close();
|
|
600
618
|
}
|
|
601
619
|
}
|
|
602
|
-
async detectCrsWithConn(conn,
|
|
620
|
+
async detectCrsWithConn(conn, source, geomCol) {
|
|
621
|
+
// Strategy 0: DuckDB v1.5 — CRS embedded in GEOMETRY column type
|
|
622
|
+
// e.g., GEOMETRY('EPSG:4326') → extract CRS directly from type string.
|
|
623
|
+
// Works uniformly for file-backed and SQL-backed (attached) sources.
|
|
624
|
+
try {
|
|
625
|
+
const t0 = performance.now();
|
|
626
|
+
const descResult = await conn.query(`SELECT column_type FROM (DESCRIBE SELECT * FROM ${source.ref}) WHERE column_name = '${geomCol}'`);
|
|
627
|
+
const descRows = descResult.toArray();
|
|
628
|
+
log(`detectCrs strategy 0 (column type) → ${descRows.length} rows in ${elapsed(t0)}`);
|
|
629
|
+
if (descRows.length > 0) {
|
|
630
|
+
const colType = String(descRows[0].column_type);
|
|
631
|
+
log(`detectCrs strategy 0 → type: "${colType}"`);
|
|
632
|
+
const result = extractCrsFromTypeString(colType);
|
|
633
|
+
if (result.found) {
|
|
634
|
+
log(`detectCrs strategy 0 → authoritative: ${result.crs ?? 'WGS84/null'} (skipping strategies 1-2)`);
|
|
635
|
+
return result.crs;
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
catch (err) {
|
|
640
|
+
log('detectCrs strategy 0 → skipped:', err?.message ?? err);
|
|
641
|
+
}
|
|
642
|
+
// Strategies 1 and 2 rely on file-level Parquet metadata functions.
|
|
643
|
+
// SQL-backed sources (attached DuckLake/DuckDB/SQLite tables) don't have
|
|
644
|
+
// a Parquet file path, so strategy 0 is authoritative for them.
|
|
645
|
+
if (!source.filePath) {
|
|
646
|
+
log('detectCrs → no filePath, skipping Parquet metadata strategies');
|
|
647
|
+
return null;
|
|
648
|
+
}
|
|
649
|
+
const path = source.filePath;
|
|
603
650
|
// Strategy 1: GeoParquet file-level metadata (geo key in KV metadata)
|
|
604
651
|
try {
|
|
605
652
|
const t1 = performance.now();
|
|
@@ -656,7 +703,6 @@ export class WasmQueryEngine {
|
|
|
656
703
|
log(`queryCancellable → ${sqlPreview}`);
|
|
657
704
|
const db = await getDB();
|
|
658
705
|
conn = await db.connect();
|
|
659
|
-
await ensureGeoConversionDisabled(conn);
|
|
660
706
|
log(`queryCancellable → connected in ${elapsed(t0)}`);
|
|
661
707
|
try {
|
|
662
708
|
if (connId) {
|
|
@@ -732,7 +778,6 @@ export class WasmQueryEngine {
|
|
|
732
778
|
log(`queryForMapCancellable → geomCol: ${geomCol}, type: ${geomColType}, crs: ${sourceCrs ?? 'WGS84'}`);
|
|
733
779
|
const db = await getDB();
|
|
734
780
|
conn = await db.connect();
|
|
735
|
-
await ensureGeoConversionDisabled(conn);
|
|
736
781
|
try {
|
|
737
782
|
if (connId) {
|
|
738
783
|
await this.configureStorage(conn, connId);
|
|
@@ -740,29 +785,23 @@ export class WasmQueryEngine {
|
|
|
740
785
|
// Build geometry expression (same logic as queryForMap)
|
|
741
786
|
const quoted = `"${geomCol}"`;
|
|
742
787
|
const upper = geomColType.toUpperCase();
|
|
743
|
-
const
|
|
744
|
-
upper === 'GEOGRAPHY' ||
|
|
745
|
-
upper === 'WKB_BLOB' ||
|
|
746
|
-
upper.includes('POINT') ||
|
|
747
|
-
upper.includes('LINESTRING') ||
|
|
748
|
-
upper.includes('POLYGON') ||
|
|
749
|
-
upper.includes('BINARY');
|
|
788
|
+
const spatialType = isSpatialColumnType(upper);
|
|
750
789
|
const isWkbBlob = upper === 'BLOB' || upper === 'BYTEA';
|
|
751
790
|
let wkbExpr;
|
|
752
791
|
let geomExpr;
|
|
753
792
|
if (isWkbBlob && !sourceCrs) {
|
|
754
|
-
// Already WKB — use directly, no spatial function calls needed.
|
|
755
793
|
wkbExpr = quoted;
|
|
756
|
-
geomExpr = null;
|
|
794
|
+
geomExpr = null;
|
|
757
795
|
}
|
|
758
796
|
else {
|
|
759
|
-
geomExpr =
|
|
797
|
+
geomExpr = spatialType
|
|
760
798
|
? quoted
|
|
761
799
|
: isWkbBlob
|
|
762
|
-
?
|
|
800
|
+
? wrapWkbWithCrs(quoted, sourceCrs)
|
|
763
801
|
: `ST_GeomFromGeoJSON(${quoted})`;
|
|
764
802
|
if (sourceCrs) {
|
|
765
|
-
|
|
803
|
+
const effectiveType = isWkbBlob ? `GEOMETRY('${sourceCrs}')` : geomColType;
|
|
804
|
+
geomExpr = buildTransformExpr(geomExpr, effectiveType, sourceCrs, DEFAULT_TARGET_CRS);
|
|
766
805
|
}
|
|
767
806
|
wkbExpr = `ST_AsWKB(${geomExpr})`;
|
|
768
807
|
}
|
|
@@ -867,10 +906,24 @@ export class WasmQueryEngine {
|
|
|
867
906
|
}
|
|
868
907
|
finally {
|
|
869
908
|
dbPromise = null;
|
|
870
|
-
geoConversionGlobal = false;
|
|
871
909
|
log('forceCancel → done, next getDB() will reinitialize');
|
|
872
910
|
}
|
|
873
911
|
}
|
|
912
|
+
async registerFileBuffer(name, buffer) {
|
|
913
|
+
const db = await getDB();
|
|
914
|
+
await db.registerFileBuffer(name, buffer);
|
|
915
|
+
log(`registerFileBuffer → "${name}" (${buffer.byteLength} bytes)`);
|
|
916
|
+
}
|
|
917
|
+
async dropFile(name) {
|
|
918
|
+
const db = await getDB();
|
|
919
|
+
try {
|
|
920
|
+
await db.dropFile(name);
|
|
921
|
+
log(`dropFile → "${name}"`);
|
|
922
|
+
}
|
|
923
|
+
catch {
|
|
924
|
+
// Ignore — file may not exist
|
|
925
|
+
}
|
|
926
|
+
}
|
|
874
927
|
async releaseMemory() {
|
|
875
928
|
const db = await getDB();
|
|
876
929
|
const conn = await db.connect();
|
|
@@ -38,6 +38,26 @@ export interface ProviderDef {
|
|
|
38
38
|
schemes: string[];
|
|
39
39
|
}
|
|
40
40
|
export declare const PROVIDERS: Record<ProviderId, ProviderDef>;
|
|
41
|
+
export interface CorsHelp {
|
|
42
|
+
/** True if the provider returns CORS headers by default. */
|
|
43
|
+
defaultEnabled: boolean;
|
|
44
|
+
/** Official CORS configuration docs URL. */
|
|
45
|
+
docsUrl?: string;
|
|
46
|
+
/** Brief note shown in the UI. */
|
|
47
|
+
note?: string;
|
|
48
|
+
/** CLI steps when no console UI or docs are insufficient. */
|
|
49
|
+
cliSteps?: string[];
|
|
50
|
+
}
|
|
51
|
+
export declare const CORS_HELP: Record<ProviderId, CorsHelp>;
|
|
52
|
+
export interface ReadOnlyHelp {
|
|
53
|
+
/** Brief note shown in the UI. */
|
|
54
|
+
note: string;
|
|
55
|
+
/** Official docs URL. */
|
|
56
|
+
docsUrl?: string;
|
|
57
|
+
/** CLI steps to apply a read-only bucket policy. */
|
|
58
|
+
cliSteps?: string[];
|
|
59
|
+
}
|
|
60
|
+
export declare const READ_ONLY_HELP: Partial<Record<ProviderId, ReadOnlyHelp>>;
|
|
41
61
|
/** All provider IDs, ordered for the UI. */
|
|
42
62
|
export declare const PROVIDER_IDS: ProviderId[];
|
|
43
63
|
/** Get provider def, falling back to S3 for unknown. */
|
|
@@ -51,3 +71,30 @@ export declare function buildEndpointFromTemplate(id: ProviderId, region: string
|
|
|
51
71
|
export declare function buildProviderBaseUrl(provider: ProviderId, endpoint: string, bucket: string, region: string): string;
|
|
52
72
|
/** Check if a provider uses the GCS JSON API (not S3 XML). */
|
|
53
73
|
export declare function isGcsProvider(provider: string, endpoint: string): boolean;
|
|
74
|
+
/**
|
|
75
|
+
* Minimal connection shape needed to decide access mode.
|
|
76
|
+
* Kept loose so callers don't need to import the full Connection type.
|
|
77
|
+
*/
|
|
78
|
+
export interface AccessModeInput {
|
|
79
|
+
provider: string;
|
|
80
|
+
anonymous?: boolean;
|
|
81
|
+
endpoint?: string;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* How a connection's files can be read by the browser:
|
|
85
|
+
*
|
|
86
|
+
* - `public-https`: plain HTTPS via any HTTP client. No auth, no signing.
|
|
87
|
+
* Covers anonymous AWS/GCS/R2/Storj/Wasabi/etc.
|
|
88
|
+
* - `sas-https`: HTTPS with SAS token embedded in the URL. Still works with
|
|
89
|
+
* any HTTP client. Azure only.
|
|
90
|
+
* - `signed-s3`: requires SigV4 signing. DuckDB uses the `s3://` URI and
|
|
91
|
+
* signs it via its S3 config; other viewers must go through the storage
|
|
92
|
+
* adapter (which returns a blob) instead of streaming the HTTPS URL.
|
|
93
|
+
*/
|
|
94
|
+
export type AccessMode = 'public-https' | 'sas-https' | 'signed-s3';
|
|
95
|
+
export declare function getAccessMode(conn: AccessModeInput): AccessMode;
|
|
96
|
+
/**
|
|
97
|
+
* True when the connection's files can be fetched by any HTTP client
|
|
98
|
+
* (fetch/img/video/DuckDB httpfs/COG/Zarr/etc.) without the storage adapter.
|
|
99
|
+
*/
|
|
100
|
+
export declare function isPubliclyStreamable(conn: AccessModeInput): boolean;
|