@walkthru-earth/objex 1.2.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/components/browser/FileTreeSidebar.svelte +1 -1
- package/dist/components/layout/Sidebar.svelte +27 -0
- package/dist/components/viewers/ArchiveViewer.svelte +4 -4
- package/dist/components/viewers/CodeViewer.svelte +21 -5
- package/dist/components/viewers/CogViewer.svelte +21 -3
- package/dist/components/viewers/CopcViewer.svelte +20 -2
- package/dist/components/viewers/FlatGeobufViewer.svelte +15 -9
- package/dist/components/viewers/PmtilesViewer.svelte +2 -2
- package/dist/components/viewers/StacMapViewer.svelte +25 -9
- package/dist/components/viewers/TableViewer.svelte +50 -21
- package/dist/components/viewers/ZarrMapViewer.svelte +4 -4
- package/dist/components/viewers/ZarrViewer.svelte +2 -2
- package/dist/components/viewers/pmtiles/PmtilesMapView.svelte +0 -1
- package/dist/i18n/ar.js +1 -0
- package/dist/i18n/en.js +1 -0
- package/dist/query/index.d.ts +1 -1
- package/dist/query/index.js +1 -1
- package/dist/query/source.d.ts +12 -0
- package/dist/query/source.js +25 -8
- package/dist/query/wasm.js +130 -23
- package/dist/storage/adapter.d.ts +9 -0
- package/dist/storage/adapter.js +13 -1
- package/dist/storage/browser-azure.d.ts +1 -1
- package/dist/storage/browser-azure.js +4 -0
- package/dist/storage/browser-cloud.d.ts +1 -1
- package/dist/storage/browser-cloud.js +7 -0
- package/dist/storage/presign.d.ts +13 -0
- package/dist/storage/presign.js +55 -0
- package/dist/storage/providers.d.ts +6 -0
- package/dist/storage/providers.js +13 -2
- package/dist/stores/browser.svelte.d.ts +2 -0
- package/dist/stores/browser.svelte.js +17 -1
- package/dist/utils/url.d.ts +13 -0
- package/dist/utils/url.js +36 -0
- package/dist/utils/wkb.js +22 -8
- package/package.json +1 -1
package/dist/i18n/en.js
CHANGED
|
@@ -342,6 +342,7 @@ export const en = {
|
|
|
342
342
|
'map.flatgeobufInfo': 'FlatGeobuf Info',
|
|
343
343
|
'map.cogInfo': 'COG Info',
|
|
344
344
|
'map.cogCorsError': 'Cannot load COG: the server does not allow cross-origin requests (CORS). The file must be hosted with CORS headers enabled.',
|
|
345
|
+
'map.cogInvalidTiff': 'This file is not a valid TIFF. The server advertises image/tiff but the bytes do not match the TIFF signature, the file may be corrupt, encrypted, or mislabeled.',
|
|
345
346
|
'map.cogUnsupportedFormat': 'This COG uses {{type}} format which is not supported for map rendering. Only RGB COGs can be displayed.',
|
|
346
347
|
'map.noGeoColumn': 'No geometry column detected in schema',
|
|
347
348
|
'map.noData': 'No data available for map view',
|
package/dist/query/index.d.ts
CHANGED
|
@@ -2,4 +2,4 @@ import type { QueryEngine } from './engine';
|
|
|
2
2
|
export declare function getQueryEngine(): Promise<QueryEngine>;
|
|
3
3
|
export type { MapQueryHandle, MapQueryResult, QueryEngine, QueryHandle, QueryResult, QuerySource, SchemaField } from './engine';
|
|
4
4
|
export { QueryCancelledError } from './engine';
|
|
5
|
-
export { type ResolvedTableSource, resolveTableSource } from './source.js';
|
|
5
|
+
export { type ResolvedTableSource, resolveTableSource, resolveTableSourceAsync } from './source.js';
|
package/dist/query/index.js
CHANGED
package/dist/query/source.d.ts
CHANGED
|
@@ -10,6 +10,12 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import type { Tab } from '../types.js';
|
|
12
12
|
import type { QuerySource } from './engine.js';
|
|
13
|
+
/**
|
|
14
|
+
* True when a source ref points at a self-authenticating HTTPS URL (e.g. a
|
|
15
|
+
* presigned `read_parquet('https://...?X-Amz-Signature=...')`). Used to decide
|
|
16
|
+
* whether DuckDB needs S3 credential config — presigned URLs don't.
|
|
17
|
+
*/
|
|
18
|
+
export declare function isHttpsSourceRef(ref: string): boolean;
|
|
13
19
|
export interface ResolvedTableSource extends QuerySource {
|
|
14
20
|
/**
|
|
15
21
|
* True when the tab is file-backed and hyparquet / parquet metadata
|
|
@@ -28,3 +34,9 @@ export interface ResolvedTableSource extends QuerySource {
|
|
|
28
34
|
* over a tab's lifetime.
|
|
29
35
|
*/
|
|
30
36
|
export declare function resolveTableSource(tab: Tab): ResolvedTableSource;
|
|
37
|
+
/**
|
|
38
|
+
* Async counterpart of `resolveTableSource`. Returns a presigned HTTPS URL
|
|
39
|
+
* for `signed-s3` connections so DuckDB httpfs can fetch without the
|
|
40
|
+
* `Authorization` header preflight.
|
|
41
|
+
*/
|
|
42
|
+
export declare function resolveTableSourceAsync(tab: Tab): Promise<ResolvedTableSource>;
|
package/dist/query/source.js
CHANGED
|
@@ -9,13 +9,16 @@
|
|
|
9
9
|
* stay free of ad-hoc branching.
|
|
10
10
|
*/
|
|
11
11
|
import { buildDuckDbSource } from '../file-icons/index.js';
|
|
12
|
-
import { buildDuckDbUrl } from '../utils/url.js';
|
|
12
|
+
import { buildDuckDbUrl, buildDuckDbUrlAsync } from '../utils/url.js';
|
|
13
13
|
/**
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
14
|
+
* True when a source ref points at a self-authenticating HTTPS URL (e.g. a
|
|
15
|
+
* presigned `read_parquet('https://...?X-Amz-Signature=...')`). Used to decide
|
|
16
|
+
* whether DuckDB needs S3 credential config — presigned URLs don't.
|
|
17
17
|
*/
|
|
18
|
-
export function
|
|
18
|
+
export function isHttpsSourceRef(ref) {
|
|
19
|
+
return /(?:^|\(\s*['"])https:\/\//.test(ref);
|
|
20
|
+
}
|
|
21
|
+
function toResolved(tab, fileUrl) {
|
|
19
22
|
if (tab.sourceRef) {
|
|
20
23
|
return {
|
|
21
24
|
ref: tab.sourceRef,
|
|
@@ -25,13 +28,27 @@ export function resolveTableSource(tab) {
|
|
|
25
28
|
label: tab.name
|
|
26
29
|
};
|
|
27
30
|
}
|
|
28
|
-
const fileUrl = buildDuckDbUrl(tab);
|
|
29
|
-
const ref = buildDuckDbSource(tab.path, fileUrl);
|
|
30
31
|
return {
|
|
31
|
-
ref,
|
|
32
|
+
ref: buildDuckDbSource(tab.path, fileUrl ?? ''),
|
|
32
33
|
filePath: tab.path,
|
|
33
34
|
isFileSource: true,
|
|
34
35
|
fileUrl,
|
|
35
36
|
label: tab.name
|
|
36
37
|
};
|
|
37
38
|
}
|
|
39
|
+
/**
|
|
40
|
+
* Resolve a tab to its QuerySource. Must be called lazily (inside reactive
|
|
41
|
+
* expressions or functions) because `tab.sourceRef` and `tab.path` can change
|
|
42
|
+
* over a tab's lifetime.
|
|
43
|
+
*/
|
|
44
|
+
export function resolveTableSource(tab) {
|
|
45
|
+
return toResolved(tab, tab.sourceRef ? null : buildDuckDbUrl(tab));
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Async counterpart of `resolveTableSource`. Returns a presigned HTTPS URL
|
|
49
|
+
* for `signed-s3` connections so DuckDB httpfs can fetch without the
|
|
50
|
+
* `Authorization` header preflight.
|
|
51
|
+
*/
|
|
52
|
+
export async function resolveTableSourceAsync(tab) {
|
|
53
|
+
return toResolved(tab, tab.sourceRef ? null : await buildDuckDbUrlAsync(tab));
|
|
54
|
+
}
|
package/dist/query/wasm.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { DEFAULT_TARGET_CRS, DUCKDB_INIT_TIMEOUT_MS, WGS84_CODES } from '../constants.js';
|
|
2
|
-
import { getAccessMode } from '../storage/providers.js';
|
|
2
|
+
import { getAccessMode, resolveProviderEndpoint } from '../storage/providers.js';
|
|
3
3
|
import { credentialStore } from '../stores/credentials.svelte.js';
|
|
4
4
|
import { buildTransformExpr, wrapWkbWithCrs } from '../utils/geometry-type.js';
|
|
5
5
|
import { QueryCancelledError } from './engine';
|
|
6
|
+
import { isHttpsSourceRef } from './source.js';
|
|
6
7
|
const DUCKDB_VERSION = __DUCKDB_WASM_VERSION__;
|
|
7
8
|
const CDN_BASE = `https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@${DUCKDB_VERSION}/dist`;
|
|
8
9
|
const duckdb_wasm = `${CDN_BASE}/duckdb-mvp.wasm`;
|
|
@@ -234,13 +235,18 @@ async function extractCrsFromLogicalType(logicalType, conn, path) {
|
|
|
234
235
|
// DuckDB Arrow type strings that represent binary/blob data — not useful
|
|
235
236
|
// for map tooltips and expensive to extract row-by-row.
|
|
236
237
|
const BINARY_TYPES = new Set(['BLOB', 'BYTEA', 'BINARY', 'LARGEBINARY', 'WKB_BLOB']);
|
|
237
|
-
/**
|
|
238
|
+
/**
|
|
239
|
+
* True if the Arrow type string represents a numeric primitive whose `.toArray()`
|
|
240
|
+
* returns a plain typed array (zero-copy fast path). DECIMAL is excluded: Arrow
|
|
241
|
+
* emits decimals as multi-word BigInt buffers that need scale-aware formatting.
|
|
242
|
+
*/
|
|
238
243
|
function isNumericArrowType(typeStr) {
|
|
239
244
|
const t = typeStr.toUpperCase();
|
|
245
|
+
if (t.startsWith('DECIMAL'))
|
|
246
|
+
return false;
|
|
240
247
|
return (t.includes('INT') ||
|
|
241
248
|
t.includes('FLOAT') ||
|
|
242
249
|
t.includes('DOUBLE') ||
|
|
243
|
-
t.includes('DECIMAL') ||
|
|
244
250
|
t === 'TINYINT' ||
|
|
245
251
|
t === 'SMALLINT' ||
|
|
246
252
|
t === 'BIGINT' ||
|
|
@@ -250,6 +256,53 @@ function isNumericArrowType(typeStr) {
|
|
|
250
256
|
t === 'USMALLINT' ||
|
|
251
257
|
t === 'UTINYINT');
|
|
252
258
|
}
|
|
259
|
+
/**
|
|
260
|
+
* Parse DuckDB/Arrow DECIMAL type string → scale. Returns -1 if not a decimal.
|
|
261
|
+
*
|
|
262
|
+
* DuckDB DESCRIBE emits `DECIMAL(10,2)`. Arrow's `Decimal.toString()` emits
|
|
263
|
+
* `Decimal[10e+2]` (precision `e` signed-scale). Accept both.
|
|
264
|
+
*/
|
|
265
|
+
function decimalScale(typeStr) {
|
|
266
|
+
const m = /^(?:DECIMAL\(\s*\d+\s*,\s*(-?\d+)\s*\)|Decimal\[\s*\d+e([+-]?\d+)\s*\])/i.exec(typeStr);
|
|
267
|
+
if (!m)
|
|
268
|
+
return -1;
|
|
269
|
+
return Number(m[1] ?? m[2]);
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Format an Arrow Decimal value (BigInt or Uint32Array of little-endian words)
|
|
273
|
+
* into a human-readable decimal string, applying the column scale.
|
|
274
|
+
*/
|
|
275
|
+
function formatDecimal(raw, scale) {
|
|
276
|
+
if (raw == null)
|
|
277
|
+
return null;
|
|
278
|
+
let bn;
|
|
279
|
+
if (typeof raw === 'bigint') {
|
|
280
|
+
bn = raw;
|
|
281
|
+
}
|
|
282
|
+
else if (raw instanceof Uint32Array || raw instanceof Int32Array) {
|
|
283
|
+
const hi = BigInt(raw[raw.length - 1] >>> 0);
|
|
284
|
+
const signed = hi >= 0x80000000n;
|
|
285
|
+
let acc = 0n;
|
|
286
|
+
for (let w = raw.length - 1; w >= 0; w--) {
|
|
287
|
+
acc = (acc << 32n) | BigInt(raw[w] >>> 0);
|
|
288
|
+
}
|
|
289
|
+
bn = signed ? acc - (1n << BigInt(raw.length * 32)) : acc;
|
|
290
|
+
}
|
|
291
|
+
else if (typeof raw === 'number') {
|
|
292
|
+
bn = BigInt(raw);
|
|
293
|
+
}
|
|
294
|
+
else {
|
|
295
|
+
return String(raw);
|
|
296
|
+
}
|
|
297
|
+
const neg = bn < 0n;
|
|
298
|
+
const abs = neg ? -bn : bn;
|
|
299
|
+
if (scale <= 0)
|
|
300
|
+
return (neg ? '-' : '') + abs.toString();
|
|
301
|
+
const divisor = 10n ** BigInt(scale);
|
|
302
|
+
const intPart = abs / divisor;
|
|
303
|
+
const fracPart = (abs % divisor).toString().padStart(scale, '0');
|
|
304
|
+
return `${neg ? '-' : ''}${intPart}.${fracPart}`;
|
|
305
|
+
}
|
|
253
306
|
/**
|
|
254
307
|
* Extract column values using the fastest available method:
|
|
255
308
|
* - Numeric primitives → .toArray() returns a typed array view (zero-copy),
|
|
@@ -257,6 +310,14 @@ function isNumericArrowType(typeStr) {
|
|
|
257
310
|
* - Other types → per-element .get(i) for correctness (strings, structs, etc.)
|
|
258
311
|
*/
|
|
259
312
|
function extractColumnBulk(col, numRows, typeStr) {
|
|
313
|
+
const scale = decimalScale(typeStr);
|
|
314
|
+
if (scale >= 0) {
|
|
315
|
+
const values = new Array(numRows);
|
|
316
|
+
for (let i = 0; i < numRows; i++) {
|
|
317
|
+
values[i] = formatDecimal(col.get(i), scale);
|
|
318
|
+
}
|
|
319
|
+
return values;
|
|
320
|
+
}
|
|
260
321
|
if (isNumericArrowType(typeStr)) {
|
|
261
322
|
// .toArray() returns a TypedArray (Float64Array, Int32Array, etc.)
|
|
262
323
|
// which is a zero-copy view over the Arrow buffer.
|
|
@@ -273,6 +334,13 @@ function extractColumnBulk(col, numRows, typeStr) {
|
|
|
273
334
|
* Same optimisation as extractColumnBulk but appends instead of creating new.
|
|
274
335
|
*/
|
|
275
336
|
function appendColumnBulk(target, col, numRows, typeStr) {
|
|
337
|
+
const scale = decimalScale(typeStr);
|
|
338
|
+
if (scale >= 0) {
|
|
339
|
+
for (let i = 0; i < numRows; i++) {
|
|
340
|
+
target.push(formatDecimal(col.get(i), scale));
|
|
341
|
+
}
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
276
344
|
if (isNumericArrowType(typeStr)) {
|
|
277
345
|
const arr = col.toArray();
|
|
278
346
|
for (let i = 0; i < arr.length; i++) {
|
|
@@ -299,7 +367,7 @@ export class WasmQueryEngine {
|
|
|
299
367
|
log(`query → connected in ${elapsed(t0)}`);
|
|
300
368
|
try {
|
|
301
369
|
if (connId) {
|
|
302
|
-
await this.configureStorage(conn, connId);
|
|
370
|
+
await this.configureStorage(conn, connId, sql);
|
|
303
371
|
log(`query → storage configured in ${elapsed(tConn)}`);
|
|
304
372
|
}
|
|
305
373
|
const tQuery = performance.now();
|
|
@@ -320,14 +388,25 @@ export class WasmQueryEngine {
|
|
|
320
388
|
rows: []
|
|
321
389
|
};
|
|
322
390
|
}
|
|
391
|
+
// Arrow emits DECIMAL columns as multi-word BigInt / Uint32Array buffers.
|
|
392
|
+
// `String(rawDecimal)` yields the unscaled integer (or "0,0,0,0"),
|
|
393
|
+
// so rewrite each decimal cell through formatDecimal with the column scale.
|
|
394
|
+
const decimalCols = [];
|
|
395
|
+
for (let i = 0; i < cols.length; i++) {
|
|
396
|
+
const s = decimalScale(types[i]);
|
|
397
|
+
if (s >= 0)
|
|
398
|
+
decimalCols.push({ name: cols[i], scale: s });
|
|
399
|
+
}
|
|
323
400
|
// Extract rows directly — avoids Arrow version mismatch
|
|
324
401
|
const rows = result.toArray().map((row) => {
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
402
|
+
const obj = typeof row.toJSON === 'function' ? row.toJSON() : {};
|
|
403
|
+
if (typeof row.toJSON !== 'function') {
|
|
404
|
+
for (const col of cols)
|
|
405
|
+
obj[col] = row[col];
|
|
406
|
+
}
|
|
407
|
+
for (const { name, scale } of decimalCols) {
|
|
408
|
+
obj[name] = formatDecimal(obj[name], scale);
|
|
409
|
+
}
|
|
331
410
|
return obj;
|
|
332
411
|
});
|
|
333
412
|
log(`query → done in ${elapsed(t0)}, ${numRows} rows, ${cols.length} cols`);
|
|
@@ -348,7 +427,7 @@ export class WasmQueryEngine {
|
|
|
348
427
|
const conn = await db.connect();
|
|
349
428
|
try {
|
|
350
429
|
if (connId) {
|
|
351
|
-
await this.configureStorage(conn, connId);
|
|
430
|
+
await this.configureStorage(conn, connId, sql);
|
|
352
431
|
}
|
|
353
432
|
// Build geometry expression based on column type:
|
|
354
433
|
// - Native spatial types (GEOMETRY, GEOMETRY('EPSG:...'), WKB_BLOB, etc.) → use directly
|
|
@@ -442,7 +521,7 @@ export class WasmQueryEngine {
|
|
|
442
521
|
const conn = await db.connect();
|
|
443
522
|
try {
|
|
444
523
|
if (connId) {
|
|
445
|
-
await this.configureStorage(conn, connId);
|
|
524
|
+
await this.configureStorage(conn, connId, source.ref);
|
|
446
525
|
}
|
|
447
526
|
const result = await conn.query(`DESCRIBE SELECT * FROM ${source.ref}`);
|
|
448
527
|
const rows = result.toArray();
|
|
@@ -469,7 +548,7 @@ export class WasmQueryEngine {
|
|
|
469
548
|
const conn = await db.connect();
|
|
470
549
|
try {
|
|
471
550
|
if (connId) {
|
|
472
|
-
await this.configureStorage(conn, connId);
|
|
551
|
+
await this.configureStorage(conn, connId, source.ref);
|
|
473
552
|
}
|
|
474
553
|
// For Parquet files, try reading row count from file footer metadata first.
|
|
475
554
|
// This avoids parsing column types (which can fail on exotic geometry types)
|
|
@@ -510,7 +589,7 @@ export class WasmQueryEngine {
|
|
|
510
589
|
const conn = await db.connect();
|
|
511
590
|
try {
|
|
512
591
|
if (connId) {
|
|
513
|
-
await this.configureStorage(conn, connId);
|
|
592
|
+
await this.configureStorage(conn, connId, source.ref);
|
|
514
593
|
}
|
|
515
594
|
// Schema detection
|
|
516
595
|
const tSchema = performance.now();
|
|
@@ -543,9 +622,18 @@ export class WasmQueryEngine {
|
|
|
543
622
|
await conn.close();
|
|
544
623
|
}
|
|
545
624
|
}
|
|
546
|
-
async configureStorage(conn, connId) {
|
|
625
|
+
async configureStorage(conn, connId, sourceRef) {
|
|
547
626
|
try {
|
|
548
|
-
//
|
|
627
|
+
// Defensive: callers may pass a destroyed Svelte $derived (returns a
|
|
628
|
+
// Symbol sentinel) across async boundaries. Template literals below
|
|
629
|
+
// would throw "can't convert symbol to string" and pollute logs.
|
|
630
|
+
if (typeof connId !== 'string' || !connId)
|
|
631
|
+
return;
|
|
632
|
+
// Presigned HTTPS refs are self-authenticating; no S3 SETs needed.
|
|
633
|
+
if (sourceRef && isHttpsSourceRef(sourceRef)) {
|
|
634
|
+
log('configureStorage → presigned HTTPS source, skipping S3 config');
|
|
635
|
+
return;
|
|
636
|
+
}
|
|
549
637
|
const stored = localStorage.getItem('obstore-explore-connections');
|
|
550
638
|
if (!stored) {
|
|
551
639
|
log('configureStorage → no connections in localStorage');
|
|
@@ -577,10 +665,16 @@ export class WasmQueryEngine {
|
|
|
577
665
|
if (connection.region) {
|
|
578
666
|
sets.push(`SET s3_region = '${connection.region}'`);
|
|
579
667
|
}
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
668
|
+
// Non-AWS providers with an empty `endpoint` field fall back to the
|
|
669
|
+
// provider registry's template, otherwise DuckDB routes them to AWS.
|
|
670
|
+
let endpoint = connection.endpoint;
|
|
671
|
+
if (!endpoint && connection.provider && connection.provider !== 's3') {
|
|
672
|
+
endpoint = resolveProviderEndpoint(connection.provider, connection.region);
|
|
673
|
+
}
|
|
674
|
+
if (endpoint) {
|
|
675
|
+
const endpointHost = endpoint.replace(/^https?:\/\//, '');
|
|
676
|
+
sets.push(`SET s3_endpoint = '${endpointHost}'`);
|
|
677
|
+
if (endpoint.startsWith('http://')) {
|
|
584
678
|
sets.push(`SET s3_use_ssl = false`);
|
|
585
679
|
}
|
|
586
680
|
}
|
|
@@ -603,7 +697,7 @@ export class WasmQueryEngine {
|
|
|
603
697
|
const conn = await db.connect();
|
|
604
698
|
try {
|
|
605
699
|
if (connId) {
|
|
606
|
-
await this.configureStorage(conn, connId);
|
|
700
|
+
await this.configureStorage(conn, connId, source.ref);
|
|
607
701
|
}
|
|
608
702
|
const crs = await this.detectCrsWithConn(conn, source, geomCol);
|
|
609
703
|
log(`detectCrs → ${crs ?? 'WGS84/null'} in ${elapsed(t0)}`);
|
|
@@ -706,7 +800,7 @@ export class WasmQueryEngine {
|
|
|
706
800
|
log(`queryCancellable → connected in ${elapsed(t0)}`);
|
|
707
801
|
try {
|
|
708
802
|
if (connId) {
|
|
709
|
-
await this.configureStorage(conn, connId);
|
|
803
|
+
await this.configureStorage(conn, connId, sql);
|
|
710
804
|
}
|
|
711
805
|
const tQuery = performance.now();
|
|
712
806
|
const reader = await conn.send(sql);
|
|
@@ -714,6 +808,7 @@ export class WasmQueryEngine {
|
|
|
714
808
|
const rows = [];
|
|
715
809
|
let cols = [];
|
|
716
810
|
let types = [];
|
|
811
|
+
let decimalCols = [];
|
|
717
812
|
const batches = reader[Symbol.asyncIterator]();
|
|
718
813
|
let first = true;
|
|
719
814
|
while (true) {
|
|
@@ -725,6 +820,12 @@ export class WasmQueryEngine {
|
|
|
725
820
|
if (first && batch.schema) {
|
|
726
821
|
cols = batch.schema.fields.map((f) => f.name);
|
|
727
822
|
types = batch.schema.fields.map((f) => String(f.type));
|
|
823
|
+
decimalCols = [];
|
|
824
|
+
for (let i = 0; i < cols.length; i++) {
|
|
825
|
+
const s = decimalScale(types[i]);
|
|
826
|
+
if (s >= 0)
|
|
827
|
+
decimalCols.push({ name: cols[i], scale: s });
|
|
828
|
+
}
|
|
728
829
|
first = false;
|
|
729
830
|
}
|
|
730
831
|
for (const row of batch.toArray()) {
|
|
@@ -738,6 +839,12 @@ export class WasmQueryEngine {
|
|
|
738
839
|
json[key] = json[key].slice();
|
|
739
840
|
}
|
|
740
841
|
}
|
|
842
|
+
// DECIMAL raw values are BigInt / Uint32Array (unscaled). Convert
|
|
843
|
+
// to a human-readable string via formatDecimal — also drops the
|
|
844
|
+
// Uint32Array view, so stale-buffer reuse across batches is moot.
|
|
845
|
+
for (const { name, scale } of decimalCols) {
|
|
846
|
+
json[name] = formatDecimal(json[name], scale);
|
|
847
|
+
}
|
|
741
848
|
rows.push(json);
|
|
742
849
|
}
|
|
743
850
|
}
|
|
@@ -780,7 +887,7 @@ export class WasmQueryEngine {
|
|
|
780
887
|
conn = await db.connect();
|
|
781
888
|
try {
|
|
782
889
|
if (connId) {
|
|
783
|
-
await this.configureStorage(conn, connId);
|
|
890
|
+
await this.configureStorage(conn, connId, sql);
|
|
784
891
|
}
|
|
785
892
|
// Build geometry expression (same logic as queryForMap)
|
|
786
893
|
const quoted = `"${geomCol}"`;
|
|
@@ -5,6 +5,15 @@ export interface ListPage {
|
|
|
5
5
|
continuationToken?: string;
|
|
6
6
|
hasMore: boolean;
|
|
7
7
|
}
|
|
8
|
+
/**
|
|
9
|
+
* Thrown by adapters when the server returns 401 or 403 on an anonymous
|
|
10
|
+
* request. The browser store catches this to trigger a credential prompt
|
|
11
|
+
* for auto-detected `?url=` connections that turned out to be private.
|
|
12
|
+
*/
|
|
13
|
+
export declare class AuthRequiredError extends Error {
|
|
14
|
+
readonly status: number;
|
|
15
|
+
constructor(status: number, message: string);
|
|
16
|
+
}
|
|
8
17
|
export interface StorageAdapter {
|
|
9
18
|
list(path: string, signal?: AbortSignal): Promise<FileEntry[]>;
|
|
10
19
|
read(path: string, offset?: number, length?: number, signal?: AbortSignal): Promise<Uint8Array>;
|
package/dist/storage/adapter.js
CHANGED
|
@@ -1 +1,13 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Thrown by adapters when the server returns 401 or 403 on an anonymous
|
|
3
|
+
* request. The browser store catches this to trigger a credential prompt
|
|
4
|
+
* for auto-detected `?url=` connections that turned out to be private.
|
|
5
|
+
*/
|
|
6
|
+
export class AuthRequiredError extends Error {
|
|
7
|
+
status;
|
|
8
|
+
constructor(status, message) {
|
|
9
|
+
super(message);
|
|
10
|
+
this.name = 'AuthRequiredError';
|
|
11
|
+
this.status = status;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { connectionStore } from '../stores/connections.svelte.js';
|
|
2
2
|
import { credentialStore } from '../stores/credentials.svelte.js';
|
|
3
|
+
import { AuthRequiredError } from './adapter.js';
|
|
3
4
|
// --- Helpers ---
|
|
4
5
|
function nameFromKey(key) {
|
|
5
6
|
const trimmed = key.replace(/\/$/, '');
|
|
@@ -92,6 +93,9 @@ export class BrowserAzureAdapter {
|
|
|
92
93
|
const res = await fetch(url, { signal });
|
|
93
94
|
if (!res.ok) {
|
|
94
95
|
const body = await res.text().catch(() => '');
|
|
96
|
+
if (res.status === 401 || res.status === 403) {
|
|
97
|
+
throw new AuthRequiredError(res.status, `Azure list failed (${res.status}): ${body || res.statusText}`);
|
|
98
|
+
}
|
|
95
99
|
throw new Error(`Azure list failed (${res.status}): ${body || res.statusText}`);
|
|
96
100
|
}
|
|
97
101
|
const xml = await res.text();
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { AwsClient } from 'aws4fetch';
|
|
2
2
|
import { connectionStore } from '../stores/connections.svelte.js';
|
|
3
3
|
import { credentialStore } from '../stores/credentials.svelte.js';
|
|
4
|
+
import { AuthRequiredError } from './adapter.js';
|
|
4
5
|
import { buildProviderBaseUrl, isGcsProvider } from './providers.js';
|
|
5
6
|
// --- Helpers ---
|
|
6
7
|
/** Extract the last path segment from an object key. */
|
|
@@ -128,6 +129,9 @@ export class BrowserCloudAdapter {
|
|
|
128
129
|
const res = await fetch(`${url}?${params}`, { signal });
|
|
129
130
|
if (!res.ok) {
|
|
130
131
|
const body = await res.text().catch(() => '');
|
|
132
|
+
if (res.status === 401 || res.status === 403) {
|
|
133
|
+
throw new AuthRequiredError(res.status, `GCS list failed (${res.status}): ${body || res.statusText}`);
|
|
134
|
+
}
|
|
131
135
|
throw new Error(`GCS list failed (${res.status}): ${body || res.statusText}`);
|
|
132
136
|
}
|
|
133
137
|
const json = await res.json();
|
|
@@ -185,6 +189,9 @@ export class BrowserCloudAdapter {
|
|
|
185
189
|
const res = await cloudFetch(`${baseUrl}?${params}`, { signal });
|
|
186
190
|
if (!res.ok) {
|
|
187
191
|
const body = await res.text().catch(() => '');
|
|
192
|
+
if (res.status === 401 || res.status === 403) {
|
|
193
|
+
throw new AuthRequiredError(res.status, `List failed (${res.status}): ${body || res.statusText}`);
|
|
194
|
+
}
|
|
188
195
|
throw new Error(`List failed (${res.status}): ${body || res.statusText}`);
|
|
189
196
|
}
|
|
190
197
|
const xml = await res.text();
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Connection } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Presign an HTTPS URL using SigV4 query-string authentication (`X-Amz-*` params).
|
|
4
|
+
*
|
|
5
|
+
* Consumers like DuckDB's httpfs can fetch the returned URL directly with just a
|
|
6
|
+
* `Range` header, which avoids the `Authorization` header preflight that breaks
|
|
7
|
+
* on GCS's S3-compatible endpoint (cached preflight mismatches, `responseHeader`
|
|
8
|
+
* list not matching the browser's requested headers, etc.).
|
|
9
|
+
*
|
|
10
|
+
* Returns null when the connection is anonymous, Azure, or has no SigV4 creds.
|
|
11
|
+
* Callers should fall back to the `s3://` + SigV4 header path in that case.
|
|
12
|
+
*/
|
|
13
|
+
export declare function presignHttpsUrl(conn: Connection, key: string, expiresIn?: number): Promise<string | null>;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { credentialStore } from '../stores/credentials.svelte.js';
|
|
2
|
+
import { safeDecodeURIComponent } from '../utils/cloud-url.js';
|
|
3
|
+
import { buildProviderBaseUrl, getAccessMode } from './providers.js';
|
|
4
|
+
// 7 days is the SigV4 protocol maximum and is the hard cap on every
|
|
5
|
+
// S3-compatible provider we support (AWS, GCS, R2, B2, DO, Wasabi, Storj,
|
|
6
|
+
// Hetzner, Contabo, Linode, OVHcloud, MinIO). SDK defaults are lower
|
|
7
|
+
// (GCS ships 3600s) but that's a default, not a limit.
|
|
8
|
+
const MAX_EXPIRES_IN_SECONDS = 7 * 24 * 3600;
|
|
9
|
+
const DEFAULT_EXPIRES_IN_SECONDS = MAX_EXPIRES_IN_SECONDS;
|
|
10
|
+
/**
|
|
11
|
+
* Presign an HTTPS URL using SigV4 query-string authentication (`X-Amz-*` params).
|
|
12
|
+
*
|
|
13
|
+
* Consumers like DuckDB's httpfs can fetch the returned URL directly with just a
|
|
14
|
+
* `Range` header, which avoids the `Authorization` header preflight that breaks
|
|
15
|
+
* on GCS's S3-compatible endpoint (cached preflight mismatches, `responseHeader`
|
|
16
|
+
* list not matching the browser's requested headers, etc.).
|
|
17
|
+
*
|
|
18
|
+
* Returns null when the connection is anonymous, Azure, or has no SigV4 creds.
|
|
19
|
+
* Callers should fall back to the `s3://` + SigV4 header path in that case.
|
|
20
|
+
*/
|
|
21
|
+
export async function presignHttpsUrl(conn, key, expiresIn = DEFAULT_EXPIRES_IN_SECONDS) {
|
|
22
|
+
if (getAccessMode(conn) !== 'signed-s3')
|
|
23
|
+
return null;
|
|
24
|
+
const creds = credentialStore.get(conn.id);
|
|
25
|
+
if (!creds || creds.type !== 'sigv4')
|
|
26
|
+
return null;
|
|
27
|
+
const cleanKey = safeDecodeURIComponent(key.replace(/^\//, ''));
|
|
28
|
+
const baseUrl = buildProviderBaseUrl(conn.provider, conn.endpoint, conn.bucket, conn.region);
|
|
29
|
+
const url = new URL(`${baseUrl}/${encodeKey(cleanKey)}`);
|
|
30
|
+
// Clamp to the protocol max so callers asking for longer don't silently
|
|
31
|
+
// produce URLs every provider rejects.
|
|
32
|
+
const effectiveExpiry = Math.min(Math.max(1, expiresIn), MAX_EXPIRES_IN_SECONDS);
|
|
33
|
+
url.searchParams.set('X-Amz-Expires', String(effectiveExpiry));
|
|
34
|
+
// Lazy-load aws4fetch so public-only sessions don't pull it into the
|
|
35
|
+
// shared viewer chunk (utils/url.ts is imported widely).
|
|
36
|
+
const { AwsClient } = await import('aws4fetch');
|
|
37
|
+
const client = new AwsClient({
|
|
38
|
+
accessKeyId: creds.accessKey,
|
|
39
|
+
secretAccessKey: creds.secretKey,
|
|
40
|
+
service: 's3',
|
|
41
|
+
region: conn.region || 'us-east-1'
|
|
42
|
+
});
|
|
43
|
+
const signed = await client.sign(url.toString(), {
|
|
44
|
+
method: 'GET',
|
|
45
|
+
aws: { signQuery: true, allHeaders: false }
|
|
46
|
+
});
|
|
47
|
+
return signed.url;
|
|
48
|
+
}
|
|
49
|
+
/** Encode an object key for URL path, preserving `/` separators. */
|
|
50
|
+
function encodeKey(key) {
|
|
51
|
+
return key
|
|
52
|
+
.split('/')
|
|
53
|
+
.map((s) => encodeURIComponent(s))
|
|
54
|
+
.join('/');
|
|
55
|
+
}
|
|
@@ -64,6 +64,12 @@ export declare const PROVIDER_IDS: ProviderId[];
|
|
|
64
64
|
export declare function getProvider(id: string): ProviderDef;
|
|
65
65
|
/** Build endpoint URL from template + region. */
|
|
66
66
|
export declare function buildEndpointFromTemplate(id: ProviderId, region: string): string;
|
|
67
|
+
/**
|
|
68
|
+
* Resolve an endpoint URL for a provider using its registered template,
|
|
69
|
+
* falling back to the provider's default region when none is supplied.
|
|
70
|
+
* Returns '' when the provider has no template (e.g. plain S3 or MinIO).
|
|
71
|
+
*/
|
|
72
|
+
export declare function resolveProviderEndpoint(provider: string, region?: string): string;
|
|
67
73
|
/**
|
|
68
74
|
* Build the base URL for API requests (endpoint + bucket).
|
|
69
75
|
* Used by browser-cloud adapter and url-state.
|
|
@@ -275,9 +275,9 @@ export const CORS_HELP = {
|
|
|
275
275
|
gcs: {
|
|
276
276
|
defaultEnabled: false,
|
|
277
277
|
docsUrl: 'https://cloud.google.com/storage/docs/using-cors',
|
|
278
|
-
note: '
|
|
278
|
+
note: 'Use the gcloud CLI. GCS `responseHeader` is dual-purpose (Access-Control-Expose-Headers AND Access-Control-Allow-Headers), so every request header the browser sends must be listed or the preflight fails silently. For private buckets signed with HMAC, include the AWS SigV4 headers (Authorization, x-amz-date, x-amz-content-sha256). For DuckDB httpfs partial reads, also include Range and the conditional If-* headers.',
|
|
279
279
|
cliSteps: [
|
|
280
|
-
'Create a cors.json file:\n[\n {\n "origin": ["*"],\n "method": ["GET", "HEAD"],\n "responseHeader": [\n "Content-Type",\n "Content-Length",\n "Content-Range",\n "Accept-Ranges",\n "ETag"\n ],\n "maxAgeSeconds": 3600\n }\n]',
|
|
280
|
+
'Create a cors.json file:\n[\n {\n "origin": ["*"],\n "method": ["GET", "HEAD"],\n "responseHeader": [\n "Content-Type",\n "Content-Length",\n "Content-Range",\n "Accept-Ranges",\n "Range",\n "If-Match",\n "If-Modified-Since",\n "If-None-Match",\n "If-Unmodified-Since",\n "ETag",\n "Authorization",\n "x-amz-content-sha256",\n "x-amz-date",\n "x-amz-*",\n "x-goog-*"\n ],\n "maxAgeSeconds": 3600\n }\n]',
|
|
281
281
|
'gcloud storage buckets update gs://BUCKET --cors-file=cors.json'
|
|
282
282
|
]
|
|
283
283
|
},
|
|
@@ -438,6 +438,17 @@ export function buildEndpointFromTemplate(id, region) {
|
|
|
438
438
|
return '';
|
|
439
439
|
return def.endpointTemplate.replace('{region}', region);
|
|
440
440
|
}
|
|
441
|
+
/**
|
|
442
|
+
* Resolve an endpoint URL for a provider using its registered template,
|
|
443
|
+
* falling back to the provider's default region when none is supplied.
|
|
444
|
+
* Returns '' when the provider has no template (e.g. plain S3 or MinIO).
|
|
445
|
+
*/
|
|
446
|
+
export function resolveProviderEndpoint(provider, region) {
|
|
447
|
+
const def = PROVIDERS[provider];
|
|
448
|
+
if (!def?.endpointTemplate)
|
|
449
|
+
return '';
|
|
450
|
+
return buildEndpointFromTemplate(provider, region || def.defaultRegion);
|
|
451
|
+
}
|
|
441
452
|
/**
|
|
442
453
|
* Build the base URL for API requests (endpoint + bucket).
|
|
443
454
|
* Used by browser-cloud adapter and url-state.
|
|
@@ -11,6 +11,8 @@ export declare const browser: {
|
|
|
11
11
|
total: number;
|
|
12
12
|
};
|
|
13
13
|
readonly canWrite: boolean;
|
|
14
|
+
readonly authRequired: Connection | null;
|
|
15
|
+
clearAuthRequired: () => void;
|
|
14
16
|
browse: (connection: Connection, prefix?: string) => Promise<void>;
|
|
15
17
|
navigateTo: (prefix: string) => Promise<void>;
|
|
16
18
|
navigateUp: () => Promise<void>;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { AuthRequiredError } from '../storage/adapter.js';
|
|
1
2
|
import { getAdapter } from '../storage/index.js';
|
|
2
3
|
import { credentialStore } from './credentials.svelte.js';
|
|
3
4
|
import { safeLock } from './safelock.svelte.js';
|
|
@@ -7,6 +8,7 @@ function createBrowserStore() {
|
|
|
7
8
|
let entries = $state([]);
|
|
8
9
|
let loading = $state(false);
|
|
9
10
|
let error = $state(null);
|
|
11
|
+
let authRequired = $state(null);
|
|
10
12
|
let uploading = $state(false);
|
|
11
13
|
let uploadProgress = $state({ current: 0, total: 0 });
|
|
12
14
|
async function browse(connection, prefix) {
|
|
@@ -22,12 +24,22 @@ function createBrowserStore() {
|
|
|
22
24
|
entries = result;
|
|
23
25
|
}
|
|
24
26
|
catch (e) {
|
|
25
|
-
|
|
27
|
+
if (e instanceof AuthRequiredError && connection.anonymous) {
|
|
28
|
+
// Auto-detected a private bucket. Surface it for the Sidebar to
|
|
29
|
+
// flip the connection to non-anonymous and prompt for credentials.
|
|
30
|
+
authRequired = connection;
|
|
31
|
+
}
|
|
32
|
+
else {
|
|
33
|
+
error = e instanceof Error ? e.message : String(e);
|
|
34
|
+
}
|
|
26
35
|
}
|
|
27
36
|
finally {
|
|
28
37
|
loading = false;
|
|
29
38
|
}
|
|
30
39
|
}
|
|
40
|
+
function clearAuthRequired() {
|
|
41
|
+
authRequired = null;
|
|
42
|
+
}
|
|
31
43
|
async function navigateTo(prefix) {
|
|
32
44
|
if (!activeConnection)
|
|
33
45
|
return;
|
|
@@ -145,6 +157,10 @@ function createBrowserStore() {
|
|
|
145
157
|
return false;
|
|
146
158
|
return credentialStore.has(activeConnection.id);
|
|
147
159
|
},
|
|
160
|
+
get authRequired() {
|
|
161
|
+
return authRequired;
|
|
162
|
+
},
|
|
163
|
+
clearAuthRequired,
|
|
148
164
|
browse,
|
|
149
165
|
navigateTo,
|
|
150
166
|
navigateUp,
|