@walkthru-earth/objex 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -3
- package/dist/components/browser/FileTreeSidebar.svelte +1 -1
- package/dist/components/layout/ConnectionDialog.svelte +35 -3
- package/dist/components/layout/Sidebar.svelte +28 -2
- package/dist/components/viewers/ArchiveViewer.svelte +4 -4
- package/dist/components/viewers/CodeViewer.svelte +72 -19
- package/dist/components/viewers/CodeViewer.svelte.d.ts +11 -1
- package/dist/components/viewers/CogControls.svelte +151 -22
- package/dist/components/viewers/CogControls.svelte.d.ts +5 -1
- package/dist/components/viewers/CogViewer.svelte +45 -10
- package/dist/components/viewers/CopcViewer.svelte +20 -2
- package/dist/components/viewers/FlatGeobufViewer.svelte +15 -9
- package/dist/components/viewers/MultiCogViewer.svelte +416 -0
- package/dist/components/viewers/MultiCogViewer.svelte.d.ts +9 -0
- package/dist/components/viewers/PmtilesViewer.svelte +2 -2
- package/dist/components/viewers/StacMapViewer.svelte +34 -12
- package/dist/components/viewers/StacMapViewer.svelte.d.ts +1 -0
- package/dist/components/viewers/StacMosaicViewer.svelte +699 -0
- package/dist/components/viewers/StacMosaicViewer.svelte.d.ts +9 -0
- package/dist/components/viewers/StacTabViewer.svelte +254 -0
- package/dist/components/viewers/StacTabViewer.svelte.d.ts +13 -0
- package/dist/components/viewers/TableViewer.svelte +50 -21
- package/dist/components/viewers/ViewerRouter.svelte +155 -2
- package/dist/components/viewers/ViewerRouter.svelte.d.ts +1 -1
- package/dist/components/viewers/ZarrMapViewer.svelte +147 -8
- package/dist/components/viewers/ZarrMapViewer.svelte.d.ts +8 -2
- package/dist/components/viewers/ZarrViewer.svelte +3 -2
- package/dist/components/viewers/pmtiles/PmtilesMapView.svelte +0 -1
- package/dist/i18n/ar.js +28 -0
- package/dist/i18n/en.js +28 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +2 -0
- package/dist/query/index.d.ts +1 -1
- package/dist/query/index.js +1 -1
- package/dist/query/source.d.ts +12 -0
- package/dist/query/source.js +25 -8
- package/dist/query/stac-geoparquet.d.ts +31 -0
- package/dist/query/stac-geoparquet.js +136 -0
- package/dist/query/wasm.js +130 -23
- package/dist/storage/adapter.d.ts +9 -0
- package/dist/storage/adapter.js +13 -1
- package/dist/storage/browser-azure.d.ts +1 -1
- package/dist/storage/browser-azure.js +4 -0
- package/dist/storage/browser-cloud.d.ts +1 -1
- package/dist/storage/browser-cloud.js +7 -0
- package/dist/storage/presign.d.ts +13 -0
- package/dist/storage/presign.js +55 -0
- package/dist/storage/providers.d.ts +6 -0
- package/dist/storage/providers.js +13 -2
- package/dist/stores/browser.svelte.d.ts +2 -0
- package/dist/stores/browser.svelte.js +17 -1
- package/dist/stores/connections.svelte.d.ts +38 -23
- package/dist/stores/connections.svelte.js +105 -114
- package/dist/utils/cog.d.ts +80 -18
- package/dist/utils/cog.js +187 -125
- package/dist/utils/colormap-sprite.d.ts +39 -0
- package/dist/utils/colormap-sprite.js +77 -0
- package/dist/utils/connection-identity.d.ts +51 -0
- package/dist/utils/connection-identity.js +97 -0
- package/dist/utils/host-detection.js +48 -302
- package/dist/utils/parquet-metadata.d.ts +7 -1
- package/dist/utils/parquet-metadata.js +35 -1
- package/dist/utils/stac-geoparquet.d.ts +90 -0
- package/dist/utils/stac-geoparquet.js +223 -0
- package/dist/utils/stac-hydrate.d.ts +38 -0
- package/dist/utils/stac-hydrate.js +243 -0
- package/dist/utils/stac.d.ts +136 -0
- package/dist/utils/stac.js +176 -0
- package/dist/utils/storage-url.d.ts +26 -0
- package/dist/utils/storage-url.js +164 -28
- package/dist/utils/url.d.ts +13 -0
- package/dist/utils/url.js +36 -0
- package/dist/utils/wkb.js +22 -8
- package/dist/utils/zarr.d.ts +34 -0
- package/dist/utils/zarr.js +94 -0
- package/package.json +14 -13
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read a stac-geoparquet file through the existing DuckDB-WASM engine and
|
|
3
|
+
* materialize a standard STAC FeatureCollection in memory.
|
|
4
|
+
*
|
|
5
|
+
* Reuses:
|
|
6
|
+
* - `getQueryEngine()` + `queryCancellable`/`query` for the single worker
|
|
7
|
+
* - `resolveTableSourceAsync(tab)` for presigned `signed-s3` URL handling
|
|
8
|
+
* - `stacRowToItem` from `utils/stac-geoparquet.js` for the pure transform
|
|
9
|
+
* - `parseWKB` from `utils/wkb.js` for geometry decoding
|
|
10
|
+
*
|
|
11
|
+
* The returned `FeatureCollection` is the same shape `classifyStac()` returns
|
|
12
|
+
* as `{ kind: 'item-collection', fc }`, so downstream viewers
|
|
13
|
+
* (`StacMosaicViewer`, `MultiCogViewer`) consume it unchanged.
|
|
14
|
+
*/
|
|
15
|
+
import { stacRowToItem } from '../utils/stac-geoparquet.js';
|
|
16
|
+
import { parseWKB } from '../utils/wkb.js';
|
|
17
|
+
import { QueryCancelledError } from './engine.js';
|
|
18
|
+
import { getQueryEngine } from './index.js';
|
|
19
|
+
import { resolveTableSourceAsync } from './source.js';
|
|
20
|
+
const DEFAULT_LIMIT = 2000;
|
|
21
|
+
/**
|
|
22
|
+
* Build the SELECT list. All columns are optional in the stac-geoparquet
|
|
23
|
+
* spec, so we only project what we know we'll use and the spec requires.
|
|
24
|
+
* The optional `proj:*` / `raster:*` / `bands` columns are sniffed from the
|
|
25
|
+
* schema so missing columns don't trigger a DuckDB binder error.
|
|
26
|
+
*/
|
|
27
|
+
function buildSelectList(availableColumns) {
|
|
28
|
+
const required = [
|
|
29
|
+
'id',
|
|
30
|
+
'collection',
|
|
31
|
+
'type',
|
|
32
|
+
'stac_version',
|
|
33
|
+
'stac_extensions',
|
|
34
|
+
'assets',
|
|
35
|
+
'bbox',
|
|
36
|
+
'links'
|
|
37
|
+
];
|
|
38
|
+
const optional = [
|
|
39
|
+
'datetime',
|
|
40
|
+
'proj:code',
|
|
41
|
+
'proj:bbox',
|
|
42
|
+
'proj:transform',
|
|
43
|
+
'proj:shape',
|
|
44
|
+
'raster:spatial_resolution',
|
|
45
|
+
'bands'
|
|
46
|
+
];
|
|
47
|
+
const cols = [];
|
|
48
|
+
for (const name of required) {
|
|
49
|
+
if (availableColumns.has(name))
|
|
50
|
+
cols.push(quoteIdent(name));
|
|
51
|
+
}
|
|
52
|
+
for (const name of optional) {
|
|
53
|
+
if (availableColumns.has(name))
|
|
54
|
+
cols.push(quoteIdent(name));
|
|
55
|
+
}
|
|
56
|
+
// Always project geometry as WKB so parseWKB can decode it regardless of
|
|
57
|
+
// whether DuckDB presents it as the v1.5 GEOMETRY type or a plain BLOB.
|
|
58
|
+
if (availableColumns.has('geometry')) {
|
|
59
|
+
cols.push('ST_AsWKB(geometry) AS geom_wkb');
|
|
60
|
+
}
|
|
61
|
+
return cols.join(', ');
|
|
62
|
+
}
|
|
63
|
+
function quoteIdent(name) {
|
|
64
|
+
return `"${name.replace(/"/g, '""')}"`;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Query a stac-geoparquet tab and return a STAC FeatureCollection whose
|
|
68
|
+
* features are proper STAC Items (assets absolutized, WKB decoded, bbox
|
|
69
|
+
* flattened).
|
|
70
|
+
*
|
|
71
|
+
* @param tab - the tab pointing at the `.parquet` file
|
|
72
|
+
* @param connId - connection id used for DuckDB's httpfs S3 config; pass
|
|
73
|
+
* an empty string for URL-source tabs (DuckDB will use anonymous httpfs)
|
|
74
|
+
*/
|
|
75
|
+
export async function queryStacGeoparquetFeatureCollection(tab, connId, opts = {}) {
|
|
76
|
+
const { signal, limit = DEFAULT_LIMIT } = opts;
|
|
77
|
+
if (signal?.aborted)
|
|
78
|
+
throw new QueryCancelledError();
|
|
79
|
+
const engine = await getQueryEngine();
|
|
80
|
+
const resolved = await resolveTableSourceAsync(tab);
|
|
81
|
+
if (signal?.aborted)
|
|
82
|
+
throw new QueryCancelledError();
|
|
83
|
+
// Discover which optional columns are present so the SELECT list doesn't
|
|
84
|
+
// reference missing columns.
|
|
85
|
+
const schema = await engine.getSchema(connId, resolved);
|
|
86
|
+
if (signal?.aborted)
|
|
87
|
+
throw new QueryCancelledError();
|
|
88
|
+
const available = new Set(schema.map((f) => f.name));
|
|
89
|
+
const selectList = buildSelectList(available);
|
|
90
|
+
if (!available.has('geometry') || !available.has('assets')) {
|
|
91
|
+
throw new Error('Not a stac-geoparquet file (missing geometry or assets column)');
|
|
92
|
+
}
|
|
93
|
+
const sql = `SELECT ${selectList} FROM ${resolved.ref} LIMIT ${limit}`;
|
|
94
|
+
// Prefer cancellable path when the engine exposes it.
|
|
95
|
+
let resultPromise;
|
|
96
|
+
let cancel = null;
|
|
97
|
+
if (engine.queryCancellable) {
|
|
98
|
+
const handle = engine.queryCancellable(connId, sql);
|
|
99
|
+
cancel = handle.cancel;
|
|
100
|
+
resultPromise = handle.result;
|
|
101
|
+
}
|
|
102
|
+
else {
|
|
103
|
+
resultPromise = engine.query(connId, sql);
|
|
104
|
+
}
|
|
105
|
+
const onAbort = () => {
|
|
106
|
+
cancel?.().catch(() => { });
|
|
107
|
+
};
|
|
108
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
109
|
+
let rows;
|
|
110
|
+
try {
|
|
111
|
+
const result = await resultPromise;
|
|
112
|
+
rows = result.rows ?? [];
|
|
113
|
+
}
|
|
114
|
+
finally {
|
|
115
|
+
signal?.removeEventListener('abort', onAbort);
|
|
116
|
+
}
|
|
117
|
+
if (signal?.aborted)
|
|
118
|
+
throw new QueryCancelledError();
|
|
119
|
+
// Asset hrefs in stac-geoparquet are typically written relative to each
|
|
120
|
+
// item's original `self` URL, not the parquet URL. The stactools default
|
|
121
|
+
// layout places each item JSON at `{catalog_dir}/{item.id}/{item.id}.json`,
|
|
122
|
+
// so a per-row base of `{parquet_dir}/{item.id}/` resolves `./foo.tif` to
|
|
123
|
+
// `{parquet_dir}/{item.id}/foo.tif`. Absolute hrefs pass through unchanged
|
|
124
|
+
// via `resolveStacAssetHref`.
|
|
125
|
+
const parquetUrl = resolved.fileUrl ?? tab.path;
|
|
126
|
+
const parquetDir = parquetUrl.replace(/[^/]*(?:\?.*)?$/, '');
|
|
127
|
+
const features = rows.map((row) => {
|
|
128
|
+
const id = typeof row.id === 'string' ? row.id : String(row.id ?? '');
|
|
129
|
+
const itemBase = id ? `${parquetDir}${id}/` : parquetUrl;
|
|
130
|
+
return stacRowToItem(row, itemBase, { wkbParser: parseWKB });
|
|
131
|
+
});
|
|
132
|
+
return {
|
|
133
|
+
type: 'FeatureCollection',
|
|
134
|
+
features
|
|
135
|
+
};
|
|
136
|
+
}
|
package/dist/query/wasm.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { DEFAULT_TARGET_CRS, DUCKDB_INIT_TIMEOUT_MS, WGS84_CODES } from '../constants.js';
|
|
2
|
-
import { getAccessMode } from '../storage/providers.js';
|
|
2
|
+
import { getAccessMode, resolveProviderEndpoint } from '../storage/providers.js';
|
|
3
3
|
import { credentialStore } from '../stores/credentials.svelte.js';
|
|
4
4
|
import { buildTransformExpr, wrapWkbWithCrs } from '../utils/geometry-type.js';
|
|
5
5
|
import { QueryCancelledError } from './engine';
|
|
6
|
+
import { isHttpsSourceRef } from './source.js';
|
|
6
7
|
const DUCKDB_VERSION = __DUCKDB_WASM_VERSION__;
|
|
7
8
|
const CDN_BASE = `https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@${DUCKDB_VERSION}/dist`;
|
|
8
9
|
const duckdb_wasm = `${CDN_BASE}/duckdb-mvp.wasm`;
|
|
@@ -234,13 +235,18 @@ async function extractCrsFromLogicalType(logicalType, conn, path) {
|
|
|
234
235
|
// DuckDB Arrow type strings that represent binary/blob data — not useful
|
|
235
236
|
// for map tooltips and expensive to extract row-by-row.
|
|
236
237
|
const BINARY_TYPES = new Set(['BLOB', 'BYTEA', 'BINARY', 'LARGEBINARY', 'WKB_BLOB']);
|
|
237
|
-
/**
|
|
238
|
+
/**
|
|
239
|
+
* True if the Arrow type string represents a numeric primitive whose `.toArray()`
|
|
240
|
+
* returns a plain typed array (zero-copy fast path). DECIMAL is excluded: Arrow
|
|
241
|
+
* emits decimals as multi-word BigInt buffers that need scale-aware formatting.
|
|
242
|
+
*/
|
|
238
243
|
function isNumericArrowType(typeStr) {
|
|
239
244
|
const t = typeStr.toUpperCase();
|
|
245
|
+
if (t.startsWith('DECIMAL'))
|
|
246
|
+
return false;
|
|
240
247
|
return (t.includes('INT') ||
|
|
241
248
|
t.includes('FLOAT') ||
|
|
242
249
|
t.includes('DOUBLE') ||
|
|
243
|
-
t.includes('DECIMAL') ||
|
|
244
250
|
t === 'TINYINT' ||
|
|
245
251
|
t === 'SMALLINT' ||
|
|
246
252
|
t === 'BIGINT' ||
|
|
@@ -250,6 +256,53 @@ function isNumericArrowType(typeStr) {
|
|
|
250
256
|
t === 'USMALLINT' ||
|
|
251
257
|
t === 'UTINYINT');
|
|
252
258
|
}
|
|
259
|
+
/**
|
|
260
|
+
* Parse DuckDB/Arrow DECIMAL type string → scale. Returns -1 if not a decimal.
|
|
261
|
+
*
|
|
262
|
+
* DuckDB DESCRIBE emits `DECIMAL(10,2)`. Arrow's `Decimal.toString()` emits
|
|
263
|
+
* `Decimal[10e+2]` (precision `e` signed-scale). Accept both.
|
|
264
|
+
*/
|
|
265
|
+
function decimalScale(typeStr) {
|
|
266
|
+
const m = /^(?:DECIMAL\(\s*\d+\s*,\s*(-?\d+)\s*\)|Decimal\[\s*\d+e([+-]?\d+)\s*\])/i.exec(typeStr);
|
|
267
|
+
if (!m)
|
|
268
|
+
return -1;
|
|
269
|
+
return Number(m[1] ?? m[2]);
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Format an Arrow Decimal value (BigInt or Uint32Array of little-endian words)
|
|
273
|
+
* into a human-readable decimal string, applying the column scale.
|
|
274
|
+
*/
|
|
275
|
+
function formatDecimal(raw, scale) {
|
|
276
|
+
if (raw == null)
|
|
277
|
+
return null;
|
|
278
|
+
let bn;
|
|
279
|
+
if (typeof raw === 'bigint') {
|
|
280
|
+
bn = raw;
|
|
281
|
+
}
|
|
282
|
+
else if (raw instanceof Uint32Array || raw instanceof Int32Array) {
|
|
283
|
+
const hi = BigInt(raw[raw.length - 1] >>> 0);
|
|
284
|
+
const signed = hi >= 0x80000000n;
|
|
285
|
+
let acc = 0n;
|
|
286
|
+
for (let w = raw.length - 1; w >= 0; w--) {
|
|
287
|
+
acc = (acc << 32n) | BigInt(raw[w] >>> 0);
|
|
288
|
+
}
|
|
289
|
+
bn = signed ? acc - (1n << BigInt(raw.length * 32)) : acc;
|
|
290
|
+
}
|
|
291
|
+
else if (typeof raw === 'number') {
|
|
292
|
+
bn = BigInt(raw);
|
|
293
|
+
}
|
|
294
|
+
else {
|
|
295
|
+
return String(raw);
|
|
296
|
+
}
|
|
297
|
+
const neg = bn < 0n;
|
|
298
|
+
const abs = neg ? -bn : bn;
|
|
299
|
+
if (scale <= 0)
|
|
300
|
+
return (neg ? '-' : '') + abs.toString();
|
|
301
|
+
const divisor = 10n ** BigInt(scale);
|
|
302
|
+
const intPart = abs / divisor;
|
|
303
|
+
const fracPart = (abs % divisor).toString().padStart(scale, '0');
|
|
304
|
+
return `${neg ? '-' : ''}${intPart}.${fracPart}`;
|
|
305
|
+
}
|
|
253
306
|
/**
|
|
254
307
|
* Extract column values using the fastest available method:
|
|
255
308
|
* - Numeric primitives → .toArray() returns a typed array view (zero-copy),
|
|
@@ -257,6 +310,14 @@ function isNumericArrowType(typeStr) {
|
|
|
257
310
|
* - Other types → per-element .get(i) for correctness (strings, structs, etc.)
|
|
258
311
|
*/
|
|
259
312
|
function extractColumnBulk(col, numRows, typeStr) {
|
|
313
|
+
const scale = decimalScale(typeStr);
|
|
314
|
+
if (scale >= 0) {
|
|
315
|
+
const values = new Array(numRows);
|
|
316
|
+
for (let i = 0; i < numRows; i++) {
|
|
317
|
+
values[i] = formatDecimal(col.get(i), scale);
|
|
318
|
+
}
|
|
319
|
+
return values;
|
|
320
|
+
}
|
|
260
321
|
if (isNumericArrowType(typeStr)) {
|
|
261
322
|
// .toArray() returns a TypedArray (Float64Array, Int32Array, etc.)
|
|
262
323
|
// which is a zero-copy view over the Arrow buffer.
|
|
@@ -273,6 +334,13 @@ function extractColumnBulk(col, numRows, typeStr) {
|
|
|
273
334
|
* Same optimisation as extractColumnBulk but appends instead of creating new.
|
|
274
335
|
*/
|
|
275
336
|
function appendColumnBulk(target, col, numRows, typeStr) {
|
|
337
|
+
const scale = decimalScale(typeStr);
|
|
338
|
+
if (scale >= 0) {
|
|
339
|
+
for (let i = 0; i < numRows; i++) {
|
|
340
|
+
target.push(formatDecimal(col.get(i), scale));
|
|
341
|
+
}
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
276
344
|
if (isNumericArrowType(typeStr)) {
|
|
277
345
|
const arr = col.toArray();
|
|
278
346
|
for (let i = 0; i < arr.length; i++) {
|
|
@@ -299,7 +367,7 @@ export class WasmQueryEngine {
|
|
|
299
367
|
log(`query → connected in ${elapsed(t0)}`);
|
|
300
368
|
try {
|
|
301
369
|
if (connId) {
|
|
302
|
-
await this.configureStorage(conn, connId);
|
|
370
|
+
await this.configureStorage(conn, connId, sql);
|
|
303
371
|
log(`query → storage configured in ${elapsed(tConn)}`);
|
|
304
372
|
}
|
|
305
373
|
const tQuery = performance.now();
|
|
@@ -320,14 +388,25 @@ export class WasmQueryEngine {
|
|
|
320
388
|
rows: []
|
|
321
389
|
};
|
|
322
390
|
}
|
|
391
|
+
// Arrow emits DECIMAL columns as multi-word BigInt / Uint32Array buffers.
|
|
392
|
+
// `String(rawDecimal)` yields the unscaled integer (or "0,0,0,0"),
|
|
393
|
+
// so rewrite each decimal cell through formatDecimal with the column scale.
|
|
394
|
+
const decimalCols = [];
|
|
395
|
+
for (let i = 0; i < cols.length; i++) {
|
|
396
|
+
const s = decimalScale(types[i]);
|
|
397
|
+
if (s >= 0)
|
|
398
|
+
decimalCols.push({ name: cols[i], scale: s });
|
|
399
|
+
}
|
|
323
400
|
// Extract rows directly — avoids Arrow version mismatch
|
|
324
401
|
const rows = result.toArray().map((row) => {
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
402
|
+
const obj = typeof row.toJSON === 'function' ? row.toJSON() : {};
|
|
403
|
+
if (typeof row.toJSON !== 'function') {
|
|
404
|
+
for (const col of cols)
|
|
405
|
+
obj[col] = row[col];
|
|
406
|
+
}
|
|
407
|
+
for (const { name, scale } of decimalCols) {
|
|
408
|
+
obj[name] = formatDecimal(obj[name], scale);
|
|
409
|
+
}
|
|
331
410
|
return obj;
|
|
332
411
|
});
|
|
333
412
|
log(`query → done in ${elapsed(t0)}, ${numRows} rows, ${cols.length} cols`);
|
|
@@ -348,7 +427,7 @@ export class WasmQueryEngine {
|
|
|
348
427
|
const conn = await db.connect();
|
|
349
428
|
try {
|
|
350
429
|
if (connId) {
|
|
351
|
-
await this.configureStorage(conn, connId);
|
|
430
|
+
await this.configureStorage(conn, connId, sql);
|
|
352
431
|
}
|
|
353
432
|
// Build geometry expression based on column type:
|
|
354
433
|
// - Native spatial types (GEOMETRY, GEOMETRY('EPSG:...'), WKB_BLOB, etc.) → use directly
|
|
@@ -442,7 +521,7 @@ export class WasmQueryEngine {
|
|
|
442
521
|
const conn = await db.connect();
|
|
443
522
|
try {
|
|
444
523
|
if (connId) {
|
|
445
|
-
await this.configureStorage(conn, connId);
|
|
524
|
+
await this.configureStorage(conn, connId, source.ref);
|
|
446
525
|
}
|
|
447
526
|
const result = await conn.query(`DESCRIBE SELECT * FROM ${source.ref}`);
|
|
448
527
|
const rows = result.toArray();
|
|
@@ -469,7 +548,7 @@ export class WasmQueryEngine {
|
|
|
469
548
|
const conn = await db.connect();
|
|
470
549
|
try {
|
|
471
550
|
if (connId) {
|
|
472
|
-
await this.configureStorage(conn, connId);
|
|
551
|
+
await this.configureStorage(conn, connId, source.ref);
|
|
473
552
|
}
|
|
474
553
|
// For Parquet files, try reading row count from file footer metadata first.
|
|
475
554
|
// This avoids parsing column types (which can fail on exotic geometry types)
|
|
@@ -510,7 +589,7 @@ export class WasmQueryEngine {
|
|
|
510
589
|
const conn = await db.connect();
|
|
511
590
|
try {
|
|
512
591
|
if (connId) {
|
|
513
|
-
await this.configureStorage(conn, connId);
|
|
592
|
+
await this.configureStorage(conn, connId, source.ref);
|
|
514
593
|
}
|
|
515
594
|
// Schema detection
|
|
516
595
|
const tSchema = performance.now();
|
|
@@ -543,9 +622,18 @@ export class WasmQueryEngine {
|
|
|
543
622
|
await conn.close();
|
|
544
623
|
}
|
|
545
624
|
}
|
|
546
|
-
async configureStorage(conn, connId) {
|
|
625
|
+
async configureStorage(conn, connId, sourceRef) {
|
|
547
626
|
try {
|
|
548
|
-
//
|
|
627
|
+
// Defensive: callers may pass a destroyed Svelte $derived (returns a
|
|
628
|
+
// Symbol sentinel) across async boundaries. Template literals below
|
|
629
|
+
// would throw "can't convert symbol to string" and pollute logs.
|
|
630
|
+
if (typeof connId !== 'string' || !connId)
|
|
631
|
+
return;
|
|
632
|
+
// Presigned HTTPS refs are self-authenticating; no S3 SETs needed.
|
|
633
|
+
if (sourceRef && isHttpsSourceRef(sourceRef)) {
|
|
634
|
+
log('configureStorage → presigned HTTPS source, skipping S3 config');
|
|
635
|
+
return;
|
|
636
|
+
}
|
|
549
637
|
const stored = localStorage.getItem('obstore-explore-connections');
|
|
550
638
|
if (!stored) {
|
|
551
639
|
log('configureStorage → no connections in localStorage');
|
|
@@ -577,10 +665,16 @@ export class WasmQueryEngine {
|
|
|
577
665
|
if (connection.region) {
|
|
578
666
|
sets.push(`SET s3_region = '${connection.region}'`);
|
|
579
667
|
}
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
668
|
+
// Non-AWS providers with an empty `endpoint` field fall back to the
|
|
669
|
+
// provider registry's template, otherwise DuckDB routes them to AWS.
|
|
670
|
+
let endpoint = connection.endpoint;
|
|
671
|
+
if (!endpoint && connection.provider && connection.provider !== 's3') {
|
|
672
|
+
endpoint = resolveProviderEndpoint(connection.provider, connection.region);
|
|
673
|
+
}
|
|
674
|
+
if (endpoint) {
|
|
675
|
+
const endpointHost = endpoint.replace(/^https?:\/\//, '');
|
|
676
|
+
sets.push(`SET s3_endpoint = '${endpointHost}'`);
|
|
677
|
+
if (endpoint.startsWith('http://')) {
|
|
584
678
|
sets.push(`SET s3_use_ssl = false`);
|
|
585
679
|
}
|
|
586
680
|
}
|
|
@@ -603,7 +697,7 @@ export class WasmQueryEngine {
|
|
|
603
697
|
const conn = await db.connect();
|
|
604
698
|
try {
|
|
605
699
|
if (connId) {
|
|
606
|
-
await this.configureStorage(conn, connId);
|
|
700
|
+
await this.configureStorage(conn, connId, source.ref);
|
|
607
701
|
}
|
|
608
702
|
const crs = await this.detectCrsWithConn(conn, source, geomCol);
|
|
609
703
|
log(`detectCrs → ${crs ?? 'WGS84/null'} in ${elapsed(t0)}`);
|
|
@@ -706,7 +800,7 @@ export class WasmQueryEngine {
|
|
|
706
800
|
log(`queryCancellable → connected in ${elapsed(t0)}`);
|
|
707
801
|
try {
|
|
708
802
|
if (connId) {
|
|
709
|
-
await this.configureStorage(conn, connId);
|
|
803
|
+
await this.configureStorage(conn, connId, sql);
|
|
710
804
|
}
|
|
711
805
|
const tQuery = performance.now();
|
|
712
806
|
const reader = await conn.send(sql);
|
|
@@ -714,6 +808,7 @@ export class WasmQueryEngine {
|
|
|
714
808
|
const rows = [];
|
|
715
809
|
let cols = [];
|
|
716
810
|
let types = [];
|
|
811
|
+
let decimalCols = [];
|
|
717
812
|
const batches = reader[Symbol.asyncIterator]();
|
|
718
813
|
let first = true;
|
|
719
814
|
while (true) {
|
|
@@ -725,6 +820,12 @@ export class WasmQueryEngine {
|
|
|
725
820
|
if (first && batch.schema) {
|
|
726
821
|
cols = batch.schema.fields.map((f) => f.name);
|
|
727
822
|
types = batch.schema.fields.map((f) => String(f.type));
|
|
823
|
+
decimalCols = [];
|
|
824
|
+
for (let i = 0; i < cols.length; i++) {
|
|
825
|
+
const s = decimalScale(types[i]);
|
|
826
|
+
if (s >= 0)
|
|
827
|
+
decimalCols.push({ name: cols[i], scale: s });
|
|
828
|
+
}
|
|
728
829
|
first = false;
|
|
729
830
|
}
|
|
730
831
|
for (const row of batch.toArray()) {
|
|
@@ -738,6 +839,12 @@ export class WasmQueryEngine {
|
|
|
738
839
|
json[key] = json[key].slice();
|
|
739
840
|
}
|
|
740
841
|
}
|
|
842
|
+
// DECIMAL raw values are BigInt / Uint32Array (unscaled). Convert
|
|
843
|
+
// to a human-readable string via formatDecimal — also drops the
|
|
844
|
+
// Uint32Array view, so stale-buffer reuse across batches is moot.
|
|
845
|
+
for (const { name, scale } of decimalCols) {
|
|
846
|
+
json[name] = formatDecimal(json[name], scale);
|
|
847
|
+
}
|
|
741
848
|
rows.push(json);
|
|
742
849
|
}
|
|
743
850
|
}
|
|
@@ -780,7 +887,7 @@ export class WasmQueryEngine {
|
|
|
780
887
|
conn = await db.connect();
|
|
781
888
|
try {
|
|
782
889
|
if (connId) {
|
|
783
|
-
await this.configureStorage(conn, connId);
|
|
890
|
+
await this.configureStorage(conn, connId, sql);
|
|
784
891
|
}
|
|
785
892
|
// Build geometry expression (same logic as queryForMap)
|
|
786
893
|
const quoted = `"${geomCol}"`;
|
|
@@ -5,6 +5,15 @@ export interface ListPage {
|
|
|
5
5
|
continuationToken?: string;
|
|
6
6
|
hasMore: boolean;
|
|
7
7
|
}
|
|
8
|
+
/**
|
|
9
|
+
* Thrown by adapters when the server returns 401 or 403 on an anonymous
|
|
10
|
+
* request. The browser store catches this to trigger a credential prompt
|
|
11
|
+
* for auto-detected `?url=` connections that turned out to be private.
|
|
12
|
+
*/
|
|
13
|
+
export declare class AuthRequiredError extends Error {
|
|
14
|
+
readonly status: number;
|
|
15
|
+
constructor(status: number, message: string);
|
|
16
|
+
}
|
|
8
17
|
export interface StorageAdapter {
|
|
9
18
|
list(path: string, signal?: AbortSignal): Promise<FileEntry[]>;
|
|
10
19
|
read(path: string, offset?: number, length?: number, signal?: AbortSignal): Promise<Uint8Array>;
|
package/dist/storage/adapter.js
CHANGED
|
@@ -1 +1,13 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Thrown by adapters when the server returns 401 or 403 on an anonymous
|
|
3
|
+
* request. The browser store catches this to trigger a credential prompt
|
|
4
|
+
* for auto-detected `?url=` connections that turned out to be private.
|
|
5
|
+
*/
|
|
6
|
+
export class AuthRequiredError extends Error {
|
|
7
|
+
status;
|
|
8
|
+
constructor(status, message) {
|
|
9
|
+
super(message);
|
|
10
|
+
this.name = 'AuthRequiredError';
|
|
11
|
+
this.status = status;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { connectionStore } from '../stores/connections.svelte.js';
|
|
2
2
|
import { credentialStore } from '../stores/credentials.svelte.js';
|
|
3
|
+
import { AuthRequiredError } from './adapter.js';
|
|
3
4
|
// --- Helpers ---
|
|
4
5
|
function nameFromKey(key) {
|
|
5
6
|
const trimmed = key.replace(/\/$/, '');
|
|
@@ -92,6 +93,9 @@ export class BrowserAzureAdapter {
|
|
|
92
93
|
const res = await fetch(url, { signal });
|
|
93
94
|
if (!res.ok) {
|
|
94
95
|
const body = await res.text().catch(() => '');
|
|
96
|
+
if (res.status === 401 || res.status === 403) {
|
|
97
|
+
throw new AuthRequiredError(res.status, `Azure list failed (${res.status}): ${body || res.statusText}`);
|
|
98
|
+
}
|
|
95
99
|
throw new Error(`Azure list failed (${res.status}): ${body || res.statusText}`);
|
|
96
100
|
}
|
|
97
101
|
const xml = await res.text();
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { AwsClient } from 'aws4fetch';
|
|
2
2
|
import { connectionStore } from '../stores/connections.svelte.js';
|
|
3
3
|
import { credentialStore } from '../stores/credentials.svelte.js';
|
|
4
|
+
import { AuthRequiredError } from './adapter.js';
|
|
4
5
|
import { buildProviderBaseUrl, isGcsProvider } from './providers.js';
|
|
5
6
|
// --- Helpers ---
|
|
6
7
|
/** Extract the last path segment from an object key. */
|
|
@@ -128,6 +129,9 @@ export class BrowserCloudAdapter {
|
|
|
128
129
|
const res = await fetch(`${url}?${params}`, { signal });
|
|
129
130
|
if (!res.ok) {
|
|
130
131
|
const body = await res.text().catch(() => '');
|
|
132
|
+
if (res.status === 401 || res.status === 403) {
|
|
133
|
+
throw new AuthRequiredError(res.status, `GCS list failed (${res.status}): ${body || res.statusText}`);
|
|
134
|
+
}
|
|
131
135
|
throw new Error(`GCS list failed (${res.status}): ${body || res.statusText}`);
|
|
132
136
|
}
|
|
133
137
|
const json = await res.json();
|
|
@@ -185,6 +189,9 @@ export class BrowserCloudAdapter {
|
|
|
185
189
|
const res = await cloudFetch(`${baseUrl}?${params}`, { signal });
|
|
186
190
|
if (!res.ok) {
|
|
187
191
|
const body = await res.text().catch(() => '');
|
|
192
|
+
if (res.status === 401 || res.status === 403) {
|
|
193
|
+
throw new AuthRequiredError(res.status, `List failed (${res.status}): ${body || res.statusText}`);
|
|
194
|
+
}
|
|
188
195
|
throw new Error(`List failed (${res.status}): ${body || res.statusText}`);
|
|
189
196
|
}
|
|
190
197
|
const xml = await res.text();
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Connection } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Presign an HTTPS URL using SigV4 query-string authentication (`X-Amz-*` params).
|
|
4
|
+
*
|
|
5
|
+
* Consumers like DuckDB's httpfs can fetch the returned URL directly with just a
|
|
6
|
+
* `Range` header, which avoids the `Authorization` header preflight that breaks
|
|
7
|
+
* on GCS's S3-compatible endpoint (cached preflight mismatches, `responseHeader`
|
|
8
|
+
* list not matching the browser's requested headers, etc.).
|
|
9
|
+
*
|
|
10
|
+
* Returns null when the connection is anonymous, Azure, or has no SigV4 creds.
|
|
11
|
+
* Callers should fall back to the `s3://` + SigV4 header path in that case.
|
|
12
|
+
*/
|
|
13
|
+
export declare function presignHttpsUrl(conn: Connection, key: string, expiresIn?: number): Promise<string | null>;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { credentialStore } from '../stores/credentials.svelte.js';
|
|
2
|
+
import { safeDecodeURIComponent } from '../utils/cloud-url.js';
|
|
3
|
+
import { buildProviderBaseUrl, getAccessMode } from './providers.js';
|
|
4
|
+
// 7 days is the SigV4 protocol maximum and is the hard cap on every
|
|
5
|
+
// S3-compatible provider we support (AWS, GCS, R2, B2, DO, Wasabi, Storj,
|
|
6
|
+
// Hetzner, Contabo, Linode, OVHcloud, MinIO). SDK defaults are lower
|
|
7
|
+
// (GCS ships 3600s) but that's a default, not a limit.
|
|
8
|
+
const MAX_EXPIRES_IN_SECONDS = 7 * 24 * 3600;
|
|
9
|
+
const DEFAULT_EXPIRES_IN_SECONDS = MAX_EXPIRES_IN_SECONDS;
|
|
10
|
+
/**
|
|
11
|
+
* Presign an HTTPS URL using SigV4 query-string authentication (`X-Amz-*` params).
|
|
12
|
+
*
|
|
13
|
+
* Consumers like DuckDB's httpfs can fetch the returned URL directly with just a
|
|
14
|
+
* `Range` header, which avoids the `Authorization` header preflight that breaks
|
|
15
|
+
* on GCS's S3-compatible endpoint (cached preflight mismatches, `responseHeader`
|
|
16
|
+
* list not matching the browser's requested headers, etc.).
|
|
17
|
+
*
|
|
18
|
+
* Returns null when the connection is anonymous, Azure, or has no SigV4 creds.
|
|
19
|
+
* Callers should fall back to the `s3://` + SigV4 header path in that case.
|
|
20
|
+
*/
|
|
21
|
+
export async function presignHttpsUrl(conn, key, expiresIn = DEFAULT_EXPIRES_IN_SECONDS) {
|
|
22
|
+
if (getAccessMode(conn) !== 'signed-s3')
|
|
23
|
+
return null;
|
|
24
|
+
const creds = credentialStore.get(conn.id);
|
|
25
|
+
if (!creds || creds.type !== 'sigv4')
|
|
26
|
+
return null;
|
|
27
|
+
const cleanKey = safeDecodeURIComponent(key.replace(/^\//, ''));
|
|
28
|
+
const baseUrl = buildProviderBaseUrl(conn.provider, conn.endpoint, conn.bucket, conn.region);
|
|
29
|
+
const url = new URL(`${baseUrl}/${encodeKey(cleanKey)}`);
|
|
30
|
+
// Clamp to the protocol max so callers asking for longer don't silently
|
|
31
|
+
// produce URLs every provider rejects.
|
|
32
|
+
const effectiveExpiry = Math.min(Math.max(1, expiresIn), MAX_EXPIRES_IN_SECONDS);
|
|
33
|
+
url.searchParams.set('X-Amz-Expires', String(effectiveExpiry));
|
|
34
|
+
// Lazy-load aws4fetch so public-only sessions don't pull it into the
|
|
35
|
+
// shared viewer chunk (utils/url.ts is imported widely).
|
|
36
|
+
const { AwsClient } = await import('aws4fetch');
|
|
37
|
+
const client = new AwsClient({
|
|
38
|
+
accessKeyId: creds.accessKey,
|
|
39
|
+
secretAccessKey: creds.secretKey,
|
|
40
|
+
service: 's3',
|
|
41
|
+
region: conn.region || 'us-east-1'
|
|
42
|
+
});
|
|
43
|
+
const signed = await client.sign(url.toString(), {
|
|
44
|
+
method: 'GET',
|
|
45
|
+
aws: { signQuery: true, allHeaders: false }
|
|
46
|
+
});
|
|
47
|
+
return signed.url;
|
|
48
|
+
}
|
|
49
|
+
/** Encode an object key for URL path, preserving `/` separators. */
|
|
50
|
+
function encodeKey(key) {
|
|
51
|
+
return key
|
|
52
|
+
.split('/')
|
|
53
|
+
.map((s) => encodeURIComponent(s))
|
|
54
|
+
.join('/');
|
|
55
|
+
}
|
|
@@ -64,6 +64,12 @@ export declare const PROVIDER_IDS: ProviderId[];
|
|
|
64
64
|
export declare function getProvider(id: string): ProviderDef;
|
|
65
65
|
/** Build endpoint URL from template + region. */
|
|
66
66
|
export declare function buildEndpointFromTemplate(id: ProviderId, region: string): string;
|
|
67
|
+
/**
|
|
68
|
+
* Resolve an endpoint URL for a provider using its registered template,
|
|
69
|
+
* falling back to the provider's default region when none is supplied.
|
|
70
|
+
* Returns '' when the provider has no template (e.g. plain S3 or MinIO).
|
|
71
|
+
*/
|
|
72
|
+
export declare function resolveProviderEndpoint(provider: string, region?: string): string;
|
|
67
73
|
/**
|
|
68
74
|
* Build the base URL for API requests (endpoint + bucket).
|
|
69
75
|
* Used by browser-cloud adapter and url-state.
|
|
@@ -275,9 +275,9 @@ export const CORS_HELP = {
|
|
|
275
275
|
gcs: {
|
|
276
276
|
defaultEnabled: false,
|
|
277
277
|
docsUrl: 'https://cloud.google.com/storage/docs/using-cors',
|
|
278
|
-
note: '
|
|
278
|
+
note: 'Use the gcloud CLI. GCS `responseHeader` is dual-purpose (Access-Control-Expose-Headers AND Access-Control-Allow-Headers), so every request header the browser sends must be listed or the preflight fails silently. For private buckets signed with HMAC, include the AWS SigV4 headers (Authorization, x-amz-date, x-amz-content-sha256). For DuckDB httpfs partial reads, also include Range and the conditional If-* headers.',
|
|
279
279
|
cliSteps: [
|
|
280
|
-
'Create a cors.json file:\n[\n {\n "origin": ["*"],\n "method": ["GET", "HEAD"],\n "responseHeader": [\n "Content-Type",\n "Content-Length",\n "Content-Range",\n "Accept-Ranges",\n "ETag"\n ],\n "maxAgeSeconds": 3600\n }\n]',
|
|
280
|
+
'Create a cors.json file:\n[\n {\n "origin": ["*"],\n "method": ["GET", "HEAD"],\n "responseHeader": [\n "Content-Type",\n "Content-Length",\n "Content-Range",\n "Accept-Ranges",\n "Range",\n "If-Match",\n "If-Modified-Since",\n "If-None-Match",\n "If-Unmodified-Since",\n "ETag",\n "Authorization",\n "x-amz-content-sha256",\n "x-amz-date",\n "x-amz-*",\n "x-goog-*"\n ],\n "maxAgeSeconds": 3600\n }\n]',
|
|
281
281
|
'gcloud storage buckets update gs://BUCKET --cors-file=cors.json'
|
|
282
282
|
]
|
|
283
283
|
},
|
|
@@ -438,6 +438,17 @@ export function buildEndpointFromTemplate(id, region) {
|
|
|
438
438
|
return '';
|
|
439
439
|
return def.endpointTemplate.replace('{region}', region);
|
|
440
440
|
}
|
|
441
|
+
/**
|
|
442
|
+
* Resolve an endpoint URL for a provider using its registered template,
|
|
443
|
+
* falling back to the provider's default region when none is supplied.
|
|
444
|
+
* Returns '' when the provider has no template (e.g. plain S3 or MinIO).
|
|
445
|
+
*/
|
|
446
|
+
export function resolveProviderEndpoint(provider, region) {
|
|
447
|
+
const def = PROVIDERS[provider];
|
|
448
|
+
if (!def?.endpointTemplate)
|
|
449
|
+
return '';
|
|
450
|
+
return buildEndpointFromTemplate(provider, region || def.defaultRegion);
|
|
451
|
+
}
|
|
441
452
|
/**
|
|
442
453
|
* Build the base URL for API requests (endpoint + bucket).
|
|
443
454
|
* Used by browser-cloud adapter and url-state.
|
|
@@ -11,6 +11,8 @@ export declare const browser: {
|
|
|
11
11
|
total: number;
|
|
12
12
|
};
|
|
13
13
|
readonly canWrite: boolean;
|
|
14
|
+
readonly authRequired: Connection | null;
|
|
15
|
+
clearAuthRequired: () => void;
|
|
14
16
|
browse: (connection: Connection, prefix?: string) => Promise<void>;
|
|
15
17
|
navigateTo: (prefix: string) => Promise<void>;
|
|
16
18
|
navigateUp: () => Promise<void>;
|