@walkthru-earth/objex 1.3.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +5 -0
- package/README.md +20 -12
- package/dist/components/browser/FileTreeSidebar.svelte +32 -17
- package/dist/components/layout/AboutSheet.svelte +5 -2
- package/dist/components/layout/ConnectionDialog.svelte +1 -1
- package/dist/components/layout/SettingsSheet.svelte +237 -0
- package/dist/components/layout/SettingsSheet.svelte.d.ts +6 -0
- package/dist/components/layout/Sidebar.svelte +73 -6
- package/dist/components/layout/Sidebar.svelte.d.ts +4 -1
- package/dist/components/layout/StatusBar.svelte +1 -1
- package/dist/components/layout/TabBar.svelte +2 -2
- package/dist/components/ui/context-menu/context-menu-radio-group.svelte.d.ts +1 -1
- package/dist/components/ui/dropdown-menu/dropdown-menu-checkbox-group.svelte.d.ts +1 -1
- package/dist/components/ui/dropdown-menu/dropdown-menu-radio-group.svelte.d.ts +1 -1
- package/dist/components/ui/input/input.svelte.d.ts +1 -1
- package/dist/components/ui/resizable/index.d.ts +1 -1
- package/dist/components/ui/resizable/index.js +2 -2
- package/dist/components/ui/slider/index.d.ts +3 -0
- package/dist/components/ui/slider/index.js +5 -0
- package/dist/components/ui/slider/range-slider.svelte +94 -0
- package/dist/components/ui/slider/range-slider.svelte.d.ts +21 -0
- package/dist/components/ui/slider/slider.svelte +83 -0
- package/dist/components/ui/slider/slider.svelte.d.ts +7 -0
- package/dist/components/viewers/ArchiveViewer.svelte +2 -2
- package/dist/components/viewers/CodeViewer.svelte +31 -22
- package/dist/components/viewers/CogControls.svelte +338 -184
- package/dist/components/viewers/CogControls.svelte.d.ts +33 -10
- package/dist/components/viewers/CogViewer.svelte +263 -112
- package/dist/components/viewers/CopcViewer.svelte +1 -1
- package/dist/components/viewers/FlatGeobufViewer.svelte +1 -1
- package/dist/components/viewers/GeoParquetMapViewer.svelte +6 -6
- package/dist/components/viewers/GeoParquetMapViewer.svelte.d.ts +1 -1
- package/dist/components/viewers/ImageViewer.svelte +2 -2
- package/dist/components/viewers/MarkdownViewer.svelte +12 -9
- package/dist/components/viewers/MediaViewer.svelte +2 -2
- package/dist/components/viewers/ModelViewer.svelte +1 -1
- package/dist/components/viewers/MultiCogViewer.svelte +467 -102
- package/dist/components/viewers/MultiCogViewer.svelte.d.ts +1 -1
- package/dist/components/viewers/NotebookViewer.svelte +6 -3
- package/dist/components/viewers/PdfViewer.svelte +2 -2
- package/dist/components/viewers/PmtilesViewer.svelte +3 -6
- package/dist/components/viewers/RawViewer.svelte +6 -3
- package/dist/components/viewers/StacMapViewer.svelte +1 -1
- package/dist/components/viewers/StacMosaicViewer.svelte +1760 -408
- package/dist/components/viewers/StacMosaicViewer.svelte.d.ts +1 -1
- package/dist/components/viewers/StacTabViewer.svelte +24 -13
- package/dist/components/viewers/StacTabViewer.svelte.d.ts +1 -1
- package/dist/components/viewers/TableGrid.svelte +4 -4
- package/dist/components/viewers/TableStatusBar.svelte +1 -1
- package/dist/components/viewers/TableToolbar.svelte +1 -1
- package/dist/components/viewers/TableViewer.svelte +25 -17
- package/dist/components/viewers/TableViewer.svelte.d.ts +1 -0
- package/dist/components/viewers/ViewerRouter.svelte +16 -8
- package/dist/components/viewers/ZarrMapViewer.svelte +11 -9
- package/dist/components/viewers/ZarrViewer.svelte +4 -4
- package/dist/components/viewers/cog/ChannelPicker.svelte +83 -0
- package/dist/components/viewers/cog/ChannelPicker.svelte.d.ts +13 -0
- package/dist/components/viewers/cog/PixelInspectorPanel.svelte +87 -0
- package/dist/components/viewers/cog/PixelInspectorPanel.svelte.d.ts +17 -0
- package/dist/components/viewers/cog/buildRgbLayer.d.ts +78 -0
- package/dist/components/viewers/cog/buildRgbLayer.js +176 -0
- package/dist/components/viewers/map/AttributeTable.svelte +1 -1
- package/dist/components/viewers/map/MapContainer.svelte +37 -11
- package/dist/components/viewers/pmtiles/PmtilesArchiveView.svelte +1 -1
- package/dist/components/viewers/pmtiles/PmtilesTileInspector.svelte +1 -1
- package/dist/components/viewers/stac/StacDatetimeBar.svelte +175 -0
- package/dist/components/viewers/stac/StacDatetimeBar.svelte.d.ts +10 -0
- package/dist/components/viewers/stac/StacFilterPanel.svelte +243 -0
- package/dist/components/viewers/stac/StacFilterPanel.svelte.d.ts +14 -0
- package/dist/components/viewers/stac/StacItemInspector.svelte +223 -0
- package/dist/components/viewers/stac/StacItemInspector.svelte.d.ts +10 -0
- package/dist/components/viewers/stac/StacItemStrip.svelte +228 -0
- package/dist/components/viewers/stac/StacItemStrip.svelte.d.ts +12 -0
- package/dist/file-icons/index.d.ts +1 -1
- package/dist/file-icons/index.js +1 -1
- package/dist/i18n/ar.js +110 -2
- package/dist/i18n/en.js +110 -2
- package/dist/index.d.ts +2 -28
- package/dist/index.js +7 -23
- package/dist/query/engine.d.ts +10 -0
- package/dist/query/source.js +1 -1
- package/dist/query/stac-source-factory.d.ts +65 -0
- package/dist/query/stac-source-factory.js +77 -0
- package/dist/query/stac-source-parquet.d.ts +135 -0
- package/dist/query/stac-source-parquet.js +465 -0
- package/dist/query/wasm.d.ts +8 -0
- package/dist/query/wasm.js +304 -2
- package/dist/storage/presign.js +1 -1
- package/dist/storage/providers.js +5 -5
- package/dist/stores/config.svelte.d.ts +15 -0
- package/dist/stores/config.svelte.js +46 -0
- package/dist/stores/connections.svelte.d.ts +2 -2
- package/dist/stores/connections.svelte.js +1 -2
- package/dist/stores/files.svelte.d.ts +1 -1
- package/dist/stores/files.svelte.js +1 -1
- package/dist/stores/query-history.svelte.js +1 -1
- package/dist/stores/settings.svelte.d.ts +16 -1
- package/dist/stores/settings.svelte.js +104 -48
- package/dist/stores/tabs.svelte.d.ts +3 -0
- package/dist/stores/tabs.svelte.js +17 -0
- package/dist/utils/cog-histogram.d.ts +121 -0
- package/dist/utils/cog-histogram.js +424 -0
- package/dist/utils/cog.d.ts +177 -20
- package/dist/utils/cog.js +361 -76
- package/dist/utils/colormap-sprite.d.ts +0 -9
- package/dist/utils/colormap-sprite.js +0 -21
- package/dist/utils/deck.d.ts +16 -12
- package/dist/utils/deck.js +10 -4
- package/dist/utils/pmtiles-tile.js +2 -2
- package/dist/utils/{url.d.ts → signed-url.d.ts} +15 -1
- package/dist/utils/{url.js → signed-url.js} +32 -10
- package/dist/utils/url-state.d.ts +36 -0
- package/dist/utils/url-state.js +72 -2
- package/dist/utils/zarr-tab.d.ts +1 -2
- package/dist/utils/zarr-tab.js +1 -2
- package/dist/utils/zarr.d.ts +0 -17
- package/dist/utils/zarr.js +1 -45
- package/package.json +55 -84
- package/dist/components/browser/Breadcrumb.svelte +0 -50
- package/dist/components/browser/Breadcrumb.svelte.d.ts +0 -7
- package/dist/components/browser/CreateFolderDialog.svelte +0 -98
- package/dist/components/browser/CreateFolderDialog.svelte.d.ts +0 -6
- package/dist/components/browser/DeleteConfirmDialog.svelte +0 -90
- package/dist/components/browser/DeleteConfirmDialog.svelte.d.ts +0 -8
- package/dist/components/browser/DropZone.svelte +0 -83
- package/dist/components/browser/DropZone.svelte.d.ts +0 -7
- package/dist/components/browser/FileBrowser.svelte +0 -252
- package/dist/components/browser/FileBrowser.svelte.d.ts +0 -3
- package/dist/components/browser/FileRow.svelte +0 -117
- package/dist/components/browser/FileRow.svelte.d.ts +0 -9
- package/dist/components/browser/RenameDialog.svelte +0 -101
- package/dist/components/browser/RenameDialog.svelte.d.ts +0 -8
- package/dist/components/browser/SearchBar.svelte +0 -40
- package/dist/components/browser/SearchBar.svelte.d.ts +0 -6
- package/dist/components/browser/UploadButton.svelte +0 -65
- package/dist/components/browser/UploadButton.svelte.d.ts +0 -3
- package/dist/query/stac-geoparquet.d.ts +0 -31
- package/dist/query/stac-geoparquet.js +0 -136
- package/dist/utils/clipboard.d.ts +0 -13
- package/dist/utils/clipboard.js +0 -38
- package/dist/utils/cloud-url.d.ts +0 -27
- package/dist/utils/cloud-url.js +0 -61
- package/dist/utils/cog-pure.d.ts +0 -25
- package/dist/utils/cog-pure.js +0 -35
- package/dist/utils/column-types.d.ts +0 -5
- package/dist/utils/column-types.js +0 -137
- package/dist/utils/connection-identity.d.ts +0 -51
- package/dist/utils/connection-identity.js +0 -97
- package/dist/utils/error.d.ts +0 -8
- package/dist/utils/error.js +0 -12
- package/dist/utils/evidence-context.d.ts +0 -22
- package/dist/utils/evidence-context.js +0 -56
- package/dist/utils/export.d.ts +0 -22
- package/dist/utils/export.js +0 -76
- package/dist/utils/file-sort.d.ts +0 -20
- package/dist/utils/file-sort.js +0 -41
- package/dist/utils/format.d.ts +0 -24
- package/dist/utils/format.js +0 -78
- package/dist/utils/geoarrow.d.ts +0 -32
- package/dist/utils/geoarrow.js +0 -672
- package/dist/utils/geometry-type.d.ts +0 -52
- package/dist/utils/geometry-type.js +0 -76
- package/dist/utils/hex.d.ts +0 -10
- package/dist/utils/hex.js +0 -27
- package/dist/utils/host-detection.d.ts +0 -23
- package/dist/utils/host-detection.js +0 -95
- package/dist/utils/local-storage.d.ts +0 -16
- package/dist/utils/local-storage.js +0 -37
- package/dist/utils/markdown-sql.d.ts +0 -30
- package/dist/utils/markdown-sql.js +0 -72
- package/dist/utils/notebook.d.ts +0 -59
- package/dist/utils/notebook.js +0 -211
- package/dist/utils/parquet-metadata.d.ts +0 -64
- package/dist/utils/parquet-metadata.js +0 -262
- package/dist/utils/stac-geoparquet.d.ts +0 -90
- package/dist/utils/stac-geoparquet.js +0 -223
- package/dist/utils/stac-hydrate.d.ts +0 -38
- package/dist/utils/stac-hydrate.js +0 -243
- package/dist/utils/stac.d.ts +0 -136
- package/dist/utils/stac.js +0 -176
- package/dist/utils/storage-url.d.ts +0 -90
- package/dist/utils/storage-url.js +0 -568
- package/dist/utils/wkb.d.ts +0 -43
- package/dist/utils/wkb.js +0 -359
package/dist/query/wasm.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
+
import { buildTransformExpr, wrapWkbWithCrs } from '@walkthru-earth/objex-utils';
|
|
1
2
|
import { DEFAULT_TARGET_CRS, DUCKDB_INIT_TIMEOUT_MS, WGS84_CODES } from '../constants.js';
|
|
2
3
|
import { getAccessMode, resolveProviderEndpoint } from '../storage/providers.js';
|
|
3
4
|
import { credentialStore } from '../stores/credentials.svelte.js';
|
|
4
|
-
import { buildTransformExpr, wrapWkbWithCrs } from '../utils/geometry-type.js';
|
|
5
5
|
import { QueryCancelledError } from './engine';
|
|
6
6
|
import { isHttpsSourceRef } from './source.js';
|
|
7
7
|
const DUCKDB_VERSION = __DUCKDB_WASM_VERSION__;
|
|
@@ -23,6 +23,22 @@ function elapsed(start) {
|
|
|
23
23
|
return `${(performance.now() - start).toFixed(1)}ms`;
|
|
24
24
|
}
|
|
25
25
|
let dbPromise = null;
|
|
26
|
+
const opfsState = { active: false, reasons: [] };
|
|
27
|
+
let initSummaryLogged = false;
|
|
28
|
+
function logInitSummary() {
|
|
29
|
+
if (initSummaryLogged)
|
|
30
|
+
return;
|
|
31
|
+
initSummaryLogged = true;
|
|
32
|
+
const coi = typeof globalThis !== 'undefined' && globalThis.crossOriginIsolated === true;
|
|
33
|
+
if (opfsState.active) {
|
|
34
|
+
log(`▶ DB mode: OPFS-backed (spill enabled), crossOriginIsolated=${coi}`);
|
|
35
|
+
}
|
|
36
|
+
else {
|
|
37
|
+
logWarn(`▶ DB mode: IN-MEMORY (no spill — OOMs at WASM heap ceiling), crossOriginIsolated=${coi}`);
|
|
38
|
+
for (const r of opfsState.reasons)
|
|
39
|
+
logWarn(` ↳ ${r}`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
26
42
|
function withTimeout(promise, ms, label) {
|
|
27
43
|
return new Promise((resolve, reject) => {
|
|
28
44
|
const timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
|
|
@@ -37,7 +53,7 @@ function withTimeout(promise, ms, label) {
|
|
|
37
53
|
}
|
|
38
54
|
async function getDB() {
|
|
39
55
|
if (dbPromise) {
|
|
40
|
-
|
|
56
|
+
logInitSummary();
|
|
41
57
|
return dbPromise;
|
|
42
58
|
}
|
|
43
59
|
dbPromise = (async () => {
|
|
@@ -64,6 +80,139 @@ async function getDB() {
|
|
|
64
80
|
const db = new duckdb.AsyncDuckDB(logger, worker);
|
|
65
81
|
await withTimeout(db.instantiate(bundle.mainModule, bundle.pthreadWorker), INIT_TIMEOUT_MS, 'DuckDB WASM instantiation');
|
|
66
82
|
log(`getDB → instantiated in ${elapsed(t0)}`);
|
|
83
|
+
// Open an OPFS-backed database so DuckDB can spill blocks to disk under
|
|
84
|
+
// memory pressure. In-memory mode caps at the WASM heap (~1.3-3.1 GiB
|
|
85
|
+
// depending on browser), with no temp_directory backing — large
|
|
86
|
+
// aggregations / sorts then OOM. OPFS gives DuckDB a real filesystem
|
|
87
|
+
// for paged storage, lifting the budget to the user's disk quota.
|
|
88
|
+
//
|
|
89
|
+
// Why register a handle instead of just `path: 'opfs://...'`: on the
|
|
90
|
+
// pinned 1.33.1-dev53.0 build, the URL-scheme path opens silently as a
|
|
91
|
+
// virtual MEMFS file, so the DB stays in-memory. Registering the handle
|
|
92
|
+
// via DuckDBDataProtocol.BROWSER_FSACCESS (OPFS) and opening it by the
|
|
93
|
+
// registered name forces DuckDB onto OPFS and lets it spill blocks.
|
|
94
|
+
// Falls back to in-memory if OPFS is unavailable (private mode, older
|
|
95
|
+
// browsers, cross-origin isolation issues).
|
|
96
|
+
const hasOpfs = typeof navigator !== 'undefined' &&
|
|
97
|
+
typeof navigator.storage !== 'undefined' &&
|
|
98
|
+
typeof navigator.storage.getDirectory === 'function';
|
|
99
|
+
const isCOI = typeof globalThis !== 'undefined' && globalThis.crossOriginIsolated === true;
|
|
100
|
+
if (!isCOI) {
|
|
101
|
+
opfsState.reasons.push('crossOriginIsolated=false — page is missing COOP/COEP headers. OPFS sync handles will throw inside the worker. Restart the dev server so vite.config.ts headers take effect, then hard-reload.');
|
|
102
|
+
}
|
|
103
|
+
let opfsActive = false;
|
|
104
|
+
if (hasOpfs) {
|
|
105
|
+
// Pin the OPFS filename to the duckdb-wasm version. Bumping
|
|
106
|
+
// duckdb-wasm can change the on-disk storage version, and reusing a
|
|
107
|
+
// stale file then throws "not a valid DuckDB database file" inside
|
|
108
|
+
// the worker. Worse, on Chrome the failed open keeps the
|
|
109
|
+
// FileSystemSyncAccessHandle alive, so even a same-tab retry
|
|
110
|
+
// (`removeEntry` + recreate) reads back the old state and trips
|
|
111
|
+
// "Access Handles cannot be created if there is another open Access
|
|
112
|
+
// Handle" on every fallback path. A version-pinned filename sidesteps
|
|
113
|
+
// the whole recovery problem: a new wasm build always gets a fresh
|
|
114
|
+
// file, and the old file just becomes an orphan we can reap below.
|
|
115
|
+
// Sanitize the version into a filename-safe slug.
|
|
116
|
+
const dbVersionSlug = DUCKDB_VERSION.replace(/[^a-zA-Z0-9._-]/g, '-');
|
|
117
|
+
const dbName = `objex-duckdb-v${dbVersionSlug}.db`;
|
|
118
|
+
const accessMode = duckdb.DuckDBAccessMode?.READ_WRITE ?? 1;
|
|
119
|
+
const protocol = duckdb.DuckDBDataProtocol?.BROWSER_FSACCESS;
|
|
120
|
+
// Firefox exposes `navigator.storage.getDirectory` in Private Browsing
|
|
121
|
+
// but throws `SecurityError: Security error when calling GetDirectory`
|
|
122
|
+
// when invoked. The `hasOpfs` check above only verifies the function
|
|
123
|
+
// exists, so we must guard the actual call. Other "exists-but-throws"
|
|
124
|
+
// environments (older Safari WKWebView, restricted enterprise policies,
|
|
125
|
+
// non-secure contexts that still expose the API) hit the same path.
|
|
126
|
+
let root = null;
|
|
127
|
+
try {
|
|
128
|
+
root = await navigator.storage.getDirectory();
|
|
129
|
+
}
|
|
130
|
+
catch (err) {
|
|
131
|
+
const msg = err?.message ?? String(err);
|
|
132
|
+
logWarn('getDB → navigator.storage.getDirectory() threw:', msg);
|
|
133
|
+
opfsState.reasons.push(`navigator.storage.getDirectory() threw: ${msg} — likely Firefox Private Browsing, non-secure context, or storage permission denied`);
|
|
134
|
+
}
|
|
135
|
+
if (root) {
|
|
136
|
+
// Reap orphan DBs from prior wasm versions. They are unreachable
|
|
137
|
+
// once the version slug bumps, so leaving them around silently
|
|
138
|
+
// burns OPFS quota. removeEntry throws NoModificationAllowedError
|
|
139
|
+
// while a sync access handle is still open against the file —
|
|
140
|
+
// that only matters for the file we're actively using, and we
|
|
141
|
+
// skip that one.
|
|
142
|
+
try {
|
|
143
|
+
const orphans = [];
|
|
144
|
+
for await (const [name, handle] of root.entries()) {
|
|
145
|
+
if (handle.kind === 'file' &&
|
|
146
|
+
/^objex-duckdb(-.*)?\.db$/.test(name) &&
|
|
147
|
+
name !== dbName) {
|
|
148
|
+
orphans.push(name);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
for (const name of orphans) {
|
|
152
|
+
try {
|
|
153
|
+
await root.removeEntry(name);
|
|
154
|
+
log(`getDB → reaped orphan OPFS db "${name}"`);
|
|
155
|
+
}
|
|
156
|
+
catch (rmErr) {
|
|
157
|
+
logWarn(`getDB → could not reap orphan OPFS db "${name}":`, rmErr?.message ?? rmErr);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
catch (enumErr) {
|
|
162
|
+
logWarn('getDB → could not enumerate OPFS root for orphan cleanup:', enumErr?.message ?? enumErr);
|
|
163
|
+
}
|
|
164
|
+
// Path A — registered FileSystemFileHandle. This is the fully wired
|
|
165
|
+
// OPFS path on duckdb-wasm 1.29+. It needs the real handle (passing
|
|
166
|
+
// null silently registers a non-OPFS file and the open() reverts to
|
|
167
|
+
// MEMFS, which is why earlier attempts kept reporting "in-memory
|
|
168
|
+
// mode" in OOM errors).
|
|
169
|
+
if (protocol !== undefined) {
|
|
170
|
+
const tOpenA = performance.now();
|
|
171
|
+
try {
|
|
172
|
+
const fileHandle = await root.getFileHandle(dbName, { create: true });
|
|
173
|
+
await db.registerFileHandle(dbName, fileHandle, protocol, true);
|
|
174
|
+
await withTimeout(db.open({ path: dbName, accessMode }), INIT_TIMEOUT_MS, 'DuckDB OPFS open (registered handle)');
|
|
175
|
+
opfsActive = true;
|
|
176
|
+
log(`getDB → OPFS via registered handle in ${elapsed(tOpenA)}`);
|
|
177
|
+
}
|
|
178
|
+
catch (err) {
|
|
179
|
+
const msg = err?.message ?? String(err);
|
|
180
|
+
logWarn('getDB → OPFS via registered handle failed:', msg);
|
|
181
|
+
opfsState.reasons.push(`registered-handle path: ${msg}`);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
else {
|
|
185
|
+
logWarn('getDB → DuckDBDataProtocol.BROWSER_FSACCESS missing in this build');
|
|
186
|
+
opfsState.reasons.push('DuckDBDataProtocol.BROWSER_FSACCESS missing in this WASM build');
|
|
187
|
+
}
|
|
188
|
+
// Path B — URL-scheme fallback. Some duckdb-wasm builds wire OPFS
|
|
189
|
+
// internally on `opfs://` paths without needing manual registration.
|
|
190
|
+
if (!opfsActive) {
|
|
191
|
+
const tOpenB = performance.now();
|
|
192
|
+
try {
|
|
193
|
+
await withTimeout(db.open({ path: `opfs://${dbName}`, accessMode }), INIT_TIMEOUT_MS, 'DuckDB OPFS open (url scheme)');
|
|
194
|
+
opfsActive = true;
|
|
195
|
+
log(`getDB → OPFS via opfs:// scheme in ${elapsed(tOpenB)}`);
|
|
196
|
+
}
|
|
197
|
+
catch (err) {
|
|
198
|
+
const msg = err?.message ?? String(err);
|
|
199
|
+
logWarn('getDB → OPFS via opfs:// scheme failed:', msg);
|
|
200
|
+
opfsState.reasons.push(`opfs:// scheme path: ${msg}`);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
if (!opfsActive) {
|
|
204
|
+
logWarn('getDB → all OPFS paths failed, falling back to in-memory');
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
else {
|
|
208
|
+
logWarn('getDB → OPFS root unavailable, falling back to in-memory');
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
else {
|
|
212
|
+
log('getDB → OPFS unavailable (no navigator.storage), using in-memory DB');
|
|
213
|
+
opfsState.reasons.push('navigator.storage.getDirectory is unavailable (private mode, older browser, or non-secure context)');
|
|
214
|
+
}
|
|
215
|
+
opfsState.active = opfsActive;
|
|
67
216
|
// Load httpfs for remote file access and spatial for ST_ReadSHP
|
|
68
217
|
const conn = await db.connect();
|
|
69
218
|
try {
|
|
@@ -92,12 +241,84 @@ async function getDB() {
|
|
|
92
241
|
catch {
|
|
93
242
|
logWarn('force_download_threshold not available');
|
|
94
243
|
}
|
|
244
|
+
// Memory tuning is only applied when OPFS spill is actually wired.
|
|
245
|
+
// Without OPFS, capping memory_limit below the WASM heap ceiling
|
|
246
|
+
// just makes queries OOM sooner with no upside (DuckDB still cannot
|
|
247
|
+
// spill, and TableViewer's existing workloads need the full ~3 GiB
|
|
248
|
+
// heap). preserve_insertion_order = false is the one tuning that
|
|
249
|
+
// helps both modes — it skips buffering full result sets to re-sort
|
|
250
|
+
// on output, a real OOM source on big scans.
|
|
251
|
+
// Mobile WebKit / Android Chrome run in a much smaller WASM heap
|
|
252
|
+
// (~1.8 GiB on iOS Safari, ~2 GiB on Android) than desktop
|
|
253
|
+
// (~3.1 GiB), AND mobile Safari < 17.6 has no `credentialless`
|
|
254
|
+
// COEP, so OPFS engages on far fewer devices. Detect with the UA
|
|
255
|
+
// hint + screen-size heuristic so we can ratchet down the limits
|
|
256
|
+
// on mobile-in-memory before the first OOM and so the user sees
|
|
257
|
+
// a usable session instead of a hard failure on the first
|
|
258
|
+
// pan/scan. Heuristic, not load-bearing for correctness.
|
|
259
|
+
const isMobileLike = typeof navigator !== 'undefined' &&
|
|
260
|
+
(/Mobi|Android|iPhone|iPad|iPod/i.test(navigator.userAgent) ||
|
|
261
|
+
(typeof window !== 'undefined' &&
|
|
262
|
+
Math.min(window.innerWidth, window.innerHeight) <= 820));
|
|
263
|
+
try {
|
|
264
|
+
const sets = [`SET preserve_insertion_order = false`];
|
|
265
|
+
const cores = navigator.hardwareConcurrency || 4;
|
|
266
|
+
if (opfsActive) {
|
|
267
|
+
// OPFS spill works: room to use multiple threads + a real
|
|
268
|
+
// memory_limit. On mobile the OPFS quota is also smaller
|
|
269
|
+
// (typically a few hundred MB by default), so still
|
|
270
|
+
// half-cap threads + memory_limit to reduce concurrent
|
|
271
|
+
// thread-local buffers.
|
|
272
|
+
const threads = Math.max(1, Math.min(isMobileLike ? 2 : 4, Math.floor(cores / 2)));
|
|
273
|
+
sets.push(`SET memory_limit = '${isMobileLike ? '900MB' : '2GB'}'`);
|
|
274
|
+
sets.push(`SET threads = ${threads}`);
|
|
275
|
+
sets.push(`SET temp_directory = '.tmp'`);
|
|
276
|
+
}
|
|
277
|
+
else if (isMobileLike) {
|
|
278
|
+
// Mobile + no OPFS spill is the worst case: the heap caps
|
|
279
|
+
// at ~1.8 GiB on iOS Safari, queries cannot offload, and
|
|
280
|
+
// the previous "leave memory_limit at defaults" path
|
|
281
|
+
// reproduces the OOM the user is seeing. Cap memory_limit
|
|
282
|
+
// well under the heap ceiling so DuckDB's planner picks
|
|
283
|
+
// streaming / out-of-core operator variants and surfaces
|
|
284
|
+
// `Out of Memory` from inside the planner (recoverable,
|
|
285
|
+
// the worker keeps running) instead of from the WASM heap
|
|
286
|
+
// allocator (kills the worker and forces a tab reload).
|
|
287
|
+
// Threads are pinned at 1 for the same reason — every
|
|
288
|
+
// extra thread doubles the per-pipeline scratch buffers.
|
|
289
|
+
sets.push(`SET memory_limit = '900MB'`);
|
|
290
|
+
sets.push(`SET threads = 1`);
|
|
291
|
+
sets.push(`SET temp_directory = '.tmp'`);
|
|
292
|
+
}
|
|
293
|
+
else {
|
|
294
|
+
// Desktop + no OPFS spill: leave memory_limit at defaults
|
|
295
|
+
// so the full WASM heap (~3.1 GiB) is available for big
|
|
296
|
+
// scans. Cap threads conservatively because each thread
|
|
297
|
+
// keeps its own hash-aggregate / sort buffer alive.
|
|
298
|
+
const threads = Math.max(1, Math.min(2, Math.floor(cores / 2)));
|
|
299
|
+
sets.push(`SET threads = ${threads}`);
|
|
300
|
+
sets.push(`SET temp_directory = '.tmp'`);
|
|
301
|
+
}
|
|
302
|
+
await conn.query(`${sets.join('; ')};`);
|
|
303
|
+
log(opfsActive
|
|
304
|
+
? `getDB → OPFS, memory_limit=${isMobileLike ? '900MB' : '2GB'}, mobile=${isMobileLike}`
|
|
305
|
+
: isMobileLike
|
|
306
|
+
? `getDB → mobile in-memory, memory_limit=900MB, threads=1 (no OPFS spill)`
|
|
307
|
+
: `getDB → desktop in-memory, threads capped, memory_limit at defaults`);
|
|
308
|
+
if (isMobileLike && !opfsActive) {
|
|
309
|
+
opfsState.reasons.push('mobile session with no OPFS spill: memory_limit clamped to 900MB to surface OOMs from the planner instead of the WASM heap allocator. Some large-catalog stac-geoparquet queries may still fail; reduce mosaicItemLimit or open on desktop.');
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
catch (err) {
|
|
313
|
+
logWarn('memory tuning not available:', err?.message ?? err);
|
|
314
|
+
}
|
|
95
315
|
log(`getDB → extensions loaded in ${elapsed(tExt)}`);
|
|
96
316
|
}
|
|
97
317
|
finally {
|
|
98
318
|
await conn.close();
|
|
99
319
|
}
|
|
100
320
|
log(`getDB → ready (total ${elapsed(t0)})`);
|
|
321
|
+
logInitSummary();
|
|
101
322
|
return db;
|
|
102
323
|
})();
|
|
103
324
|
// If init fails, clear the promise so it can be retried
|
|
@@ -877,6 +1098,87 @@ export class WasmQueryEngine {
|
|
|
877
1098
|
};
|
|
878
1099
|
return { result, cancel };
|
|
879
1100
|
}
|
|
1101
|
+
/**
|
|
1102
|
+
* Streaming variant of `queryCancellable`. Yields one chunk per Arrow
|
|
1103
|
+
* RecordBatch so peak memory tracks one batch instead of the full result.
|
|
1104
|
+
* Used by `stac-source-parquet` to ingest large catalogs progressively
|
|
1105
|
+
* without OOMing the WASM heap. Cancellation routes through `conn.cancelSent`
|
|
1106
|
+
* and `signal.aborted`; the connection is always closed in `finally`.
|
|
1107
|
+
*/
|
|
1108
|
+
async *queryStream(connId, sql, signal) {
|
|
1109
|
+
const t0 = performance.now();
|
|
1110
|
+
const sqlPreview = sql.length > 120 ? `${sql.slice(0, 120)}…` : sql;
|
|
1111
|
+
log(`queryStream → ${sqlPreview}`);
|
|
1112
|
+
const db = await getDB();
|
|
1113
|
+
const conn = await db.connect();
|
|
1114
|
+
const onAbort = () => {
|
|
1115
|
+
try {
|
|
1116
|
+
conn?.cancelSent?.();
|
|
1117
|
+
}
|
|
1118
|
+
catch {
|
|
1119
|
+
/* noop */
|
|
1120
|
+
}
|
|
1121
|
+
};
|
|
1122
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
1123
|
+
try {
|
|
1124
|
+
if (connId) {
|
|
1125
|
+
await this.configureStorage(conn, connId, sql);
|
|
1126
|
+
}
|
|
1127
|
+
const reader = await conn.send(sql);
|
|
1128
|
+
let cols = [];
|
|
1129
|
+
let types = [];
|
|
1130
|
+
let decimalCols = [];
|
|
1131
|
+
let first = true;
|
|
1132
|
+
let total = 0;
|
|
1133
|
+
const batches = reader[Symbol.asyncIterator]();
|
|
1134
|
+
while (true) {
|
|
1135
|
+
if (signal?.aborted)
|
|
1136
|
+
throw new QueryCancelledError();
|
|
1137
|
+
const { value: batch, done } = await batches.next();
|
|
1138
|
+
if (done)
|
|
1139
|
+
break;
|
|
1140
|
+
if (first && batch.schema) {
|
|
1141
|
+
cols = batch.schema.fields.map((f) => f.name);
|
|
1142
|
+
types = batch.schema.fields.map((f) => String(f.type));
|
|
1143
|
+
decimalCols = [];
|
|
1144
|
+
for (let i = 0; i < cols.length; i++) {
|
|
1145
|
+
const s = decimalScale(types[i]);
|
|
1146
|
+
if (s >= 0)
|
|
1147
|
+
decimalCols.push({ name: cols[i], scale: s });
|
|
1148
|
+
}
|
|
1149
|
+
first = false;
|
|
1150
|
+
}
|
|
1151
|
+
const rows = [];
|
|
1152
|
+
for (const row of batch.toArray()) {
|
|
1153
|
+
const json = typeof row.toJSON === 'function' ? row.toJSON() : { ...row };
|
|
1154
|
+
for (const key in json) {
|
|
1155
|
+
if (json[key] instanceof Uint8Array) {
|
|
1156
|
+
json[key] = json[key].slice();
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
for (const { name, scale } of decimalCols) {
|
|
1160
|
+
json[name] = formatDecimal(json[name], scale);
|
|
1161
|
+
}
|
|
1162
|
+
rows.push(json);
|
|
1163
|
+
}
|
|
1164
|
+
total += rows.length;
|
|
1165
|
+
yield { columns: cols, types, rowCount: rows.length, rows };
|
|
1166
|
+
}
|
|
1167
|
+
log(`queryStream → done in ${elapsed(t0)}, ${total} rows total`);
|
|
1168
|
+
}
|
|
1169
|
+
catch (err) {
|
|
1170
|
+
if (signal?.aborted || err instanceof QueryCancelledError) {
|
|
1171
|
+
log(`queryStream → cancelled after ${elapsed(t0)}`);
|
|
1172
|
+
throw new QueryCancelledError();
|
|
1173
|
+
}
|
|
1174
|
+
logWarn(`queryStream → failed after ${elapsed(t0)}:`, err?.message ?? err);
|
|
1175
|
+
throw err;
|
|
1176
|
+
}
|
|
1177
|
+
finally {
|
|
1178
|
+
signal?.removeEventListener('abort', onAbort);
|
|
1179
|
+
await conn?.close?.();
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
880
1182
|
queryForMapCancellable(connId, sql, geomCol, geomColType, sourceCrs) {
|
|
881
1183
|
let cancelled = false;
|
|
882
1184
|
let conn = null;
|
package/dist/storage/presign.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import { safeDecodeURIComponent } from '@walkthru-earth/objex-utils';
|
|
1
2
|
import { credentialStore } from '../stores/credentials.svelte.js';
|
|
2
|
-
import { safeDecodeURIComponent } from '../utils/cloud-url.js';
|
|
3
3
|
import { buildProviderBaseUrl, getAccessMode } from './providers.js';
|
|
4
4
|
// 7 days is the SigV4 protocol maximum and is the hard cap on every
|
|
5
5
|
// S3-compatible provider we support (AWS, GCS, R2, B2, DO, Wasabi, Storj,
|
|
@@ -87,15 +87,15 @@ export const PROVIDERS = {
|
|
|
87
87
|
schemes: ['azure', 'az', 'abfs', 'abfss', 'wasbs', 'adl']
|
|
88
88
|
},
|
|
89
89
|
minio: {
|
|
90
|
-
label: 'MinIO',
|
|
91
|
-
description: '
|
|
90
|
+
label: 'MinIO / RustFS / Custom',
|
|
91
|
+
description: 'MinIO, RustFS, or any custom S3-compatible endpoint',
|
|
92
92
|
authMethod: 'sigv4',
|
|
93
93
|
needsRegion: false,
|
|
94
94
|
needsEndpoint: true,
|
|
95
95
|
defaultRegion: 'us-east-1',
|
|
96
96
|
endpointTemplate: null,
|
|
97
97
|
regions: [],
|
|
98
|
-
endpointPlaceholder: 'https://
|
|
98
|
+
endpointPlaceholder: 'https://s3.example.com or http://localhost:9000',
|
|
99
99
|
schemes: []
|
|
100
100
|
},
|
|
101
101
|
storj: {
|
|
@@ -297,7 +297,7 @@ export const CORS_HELP = {
|
|
|
297
297
|
minio: {
|
|
298
298
|
defaultEnabled: true,
|
|
299
299
|
docsUrl: 'https://docs.min.io/enterprise/aistor-object-store/reference/cli/mc-cors/',
|
|
300
|
-
note: 'MinIO allows all origins by default
|
|
300
|
+
note: 'MinIO allows all origins by default (for custom rules use mc cors set). For RustFS or any other custom S3 service, set a CORS policy on the server that allows this origin.'
|
|
301
301
|
},
|
|
302
302
|
storj: {
|
|
303
303
|
defaultEnabled: true,
|
|
@@ -389,7 +389,7 @@ export const READ_ONLY_HELP = {
|
|
|
389
389
|
]
|
|
390
390
|
},
|
|
391
391
|
minio: {
|
|
392
|
-
note: '
|
|
392
|
+
note: 'For MinIO, create a read-only policy with mc admin policy or use the built-in readonly canned policy. For RustFS or any other custom S3 service, attach a read-only bucket policy on the server.',
|
|
393
393
|
docsUrl: 'https://docs.min.io/enterprise/aistor-object-store/administration/iam/access/'
|
|
394
394
|
},
|
|
395
395
|
digitalocean: {
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { type AppConfig } from '@walkthru-earth/objex-utils';
|
|
2
|
+
export type ConfigStatus = 'pending' | 'bundled' | 'custom' | 'error';
|
|
3
|
+
/** Reactive accessor for the loaded config and its load status. */
|
|
4
|
+
export declare const appConfig: {
|
|
5
|
+
readonly value: AppConfig;
|
|
6
|
+
readonly status: ConfigStatus;
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Fetch and merge the runtime config. Awaited in +layout.ts `load` so the
|
|
10
|
+
* config is ready before any component mounts. A `?config=<url>` param loads a
|
|
11
|
+
* remote file (status `custom`), otherwise the bundled `static/config.json`
|
|
12
|
+
* (status `bundled`). Any failure falls back to defaults (status `error`) and
|
|
13
|
+
* the app still boots.
|
|
14
|
+
*/
|
|
15
|
+
export declare function loadConfig(basePath: string): Promise<void>;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { DEFAULT_APP_CONFIG, mergeAppConfig } from '@walkthru-earth/objex-utils';
|
|
2
|
+
let config = $state.raw(DEFAULT_APP_CONFIG);
|
|
3
|
+
let status = $state('pending');
|
|
4
|
+
/** Reactive accessor for the loaded config and its load status. */
|
|
5
|
+
export const appConfig = {
|
|
6
|
+
get value() {
|
|
7
|
+
return config;
|
|
8
|
+
},
|
|
9
|
+
get status() {
|
|
10
|
+
return status;
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
function readConfigParam() {
|
|
14
|
+
if (typeof window === 'undefined')
|
|
15
|
+
return null;
|
|
16
|
+
try {
|
|
17
|
+
return new URL(window.location.href).searchParams.get('config');
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
return null;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Fetch and merge the runtime config. Awaited in +layout.ts `load` so the
|
|
25
|
+
* config is ready before any component mounts. A `?config=<url>` param loads a
|
|
26
|
+
* remote file (status `custom`), otherwise the bundled `static/config.json`
|
|
27
|
+
* (status `bundled`). Any failure falls back to defaults (status `error`) and
|
|
28
|
+
* the app still boots.
|
|
29
|
+
*/
|
|
30
|
+
export async function loadConfig(basePath) {
|
|
31
|
+
const customUrl = readConfigParam();
|
|
32
|
+
const url = customUrl ?? `${basePath}/config.json`;
|
|
33
|
+
try {
|
|
34
|
+
const res = await fetch(url, { headers: { accept: 'application/json' } });
|
|
35
|
+
if (!res.ok)
|
|
36
|
+
throw new Error(`HTTP ${res.status}`);
|
|
37
|
+
const json = await res.json();
|
|
38
|
+
config = mergeAppConfig(DEFAULT_APP_CONFIG, json);
|
|
39
|
+
status = customUrl ? 'custom' : 'bundled';
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
console.warn('[objex] config load failed, using defaults', err);
|
|
43
|
+
config = DEFAULT_APP_CONFIG;
|
|
44
|
+
status = 'error';
|
|
45
|
+
}
|
|
46
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
+
import type { DetectedHost } from '@walkthru-earth/objex-utils';
|
|
2
|
+
import { type ConnectionIdentityInput } from '@walkthru-earth/objex-utils';
|
|
1
3
|
import type { Connection, ConnectionConfig } from '../types.js';
|
|
2
|
-
import { type ConnectionIdentityInput } from '../utils/connection-identity.js';
|
|
3
|
-
import type { DetectedHost } from '../utils/host-detection.js';
|
|
4
4
|
/**
|
|
5
5
|
* Outcome of a write. `existed` is true when dedup reused an already-saved
|
|
6
6
|
* connection, false when a new row was persisted. Callers that present UI
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
+
import { connectionIdentityKey, loadFromStorage, persistToStorage } from '@walkthru-earth/objex-utils';
|
|
1
2
|
import { STORAGE_KEYS } from '../constants.js';
|
|
2
|
-
import { connectionIdentityKey } from '../utils/connection-identity.js';
|
|
3
|
-
import { loadFromStorage, persistToStorage } from '../utils/local-storage.js';
|
|
4
3
|
import { credentialStore, storeToNative } from './credentials.svelte.js';
|
|
5
4
|
function toConnection(id, config) {
|
|
6
5
|
return {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import { type SortConfig, type SortField } from '@walkthru-earth/objex-utils';
|
|
1
2
|
import type { FileEntry } from '../types.js';
|
|
2
|
-
import { type SortConfig, type SortField } from '../utils/file-sort.js';
|
|
3
3
|
export declare const files: {
|
|
4
4
|
readonly entries: FileEntry[];
|
|
5
5
|
readonly currentPath: string;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import { loadFromStorage, persistToStorage } from '@walkthru-earth/objex-utils';
|
|
1
2
|
import { MAX_QUERY_HISTORY_ENTRIES, STORAGE_KEYS } from '../constants.js';
|
|
2
|
-
import { loadFromStorage, persistToStorage } from '../utils/local-storage.js';
|
|
3
3
|
function createQueryHistoryStore() {
|
|
4
4
|
let entries = $state(loadFromStorage(STORAGE_KEYS.QUERY_HISTORY, []));
|
|
5
5
|
function save() {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { Locale } from '../i18n/index.svelte.js';
|
|
2
2
|
import type { Theme } from '../types.js';
|
|
3
3
|
export declare function resolveTheme(theme: Theme): 'light' | 'dark';
|
|
4
4
|
export declare const settings: {
|
|
@@ -6,7 +6,22 @@ export declare const settings: {
|
|
|
6
6
|
readonly resolved: "light" | "dark";
|
|
7
7
|
readonly locale: Locale;
|
|
8
8
|
readonly featureLimit: number;
|
|
9
|
+
readonly mosaicItemLimit: number;
|
|
10
|
+
readonly showConnectionRail: boolean;
|
|
11
|
+
readonly showFileTree: boolean;
|
|
12
|
+
/** True when a link param is forcing the connection-rail visibility. */
|
|
13
|
+
readonly railLockedByParam: boolean;
|
|
14
|
+
/** True when a link param is forcing the file-tree visibility. */
|
|
15
|
+
readonly treeLockedByParam: boolean;
|
|
16
|
+
/** The user-picked basemap id, or undefined to follow config/theme defaults. */
|
|
17
|
+
readonly basemapId: string | undefined;
|
|
9
18
|
setTheme(t: Theme): void;
|
|
10
19
|
setLocale(l: Locale): void;
|
|
11
20
|
setFeatureLimit(n: number): void;
|
|
21
|
+
setMosaicItemLimit(n: number): void;
|
|
22
|
+
setShowConnectionRail(v: boolean): void;
|
|
23
|
+
setShowFileTree(v: boolean): void;
|
|
24
|
+
setBasemap(id: string | undefined): void;
|
|
25
|
+
/** Clear all user overrides, reverting every value to config or hardcoded fallback. */
|
|
26
|
+
reset(): void;
|
|
12
27
|
};
|