@walkthru-earth/objex 1.3.1 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/LICENSE +5 -0
  2. package/README.md +28 -20
  3. package/dist/components/browser/FileTreeSidebar.svelte +32 -17
  4. package/dist/components/layout/AboutSheet.svelte +5 -2
  5. package/dist/components/layout/ConnectionDialog.svelte +7 -2
  6. package/dist/components/layout/SettingsSheet.svelte +238 -0
  7. package/dist/components/layout/SettingsSheet.svelte.d.ts +6 -0
  8. package/dist/components/layout/Sidebar.svelte +73 -6
  9. package/dist/components/layout/Sidebar.svelte.d.ts +4 -1
  10. package/dist/components/layout/StatusBar.svelte +17 -14
  11. package/dist/components/layout/TabBar.svelte +4 -4
  12. package/dist/components/ui/context-menu/context-menu-radio-group.svelte.d.ts +1 -1
  13. package/dist/components/ui/dropdown-menu/dropdown-menu-checkbox-group.svelte.d.ts +1 -1
  14. package/dist/components/ui/dropdown-menu/dropdown-menu-radio-group.svelte.d.ts +1 -1
  15. package/dist/components/ui/input/input.svelte.d.ts +1 -1
  16. package/dist/components/ui/resizable/index.d.ts +1 -1
  17. package/dist/components/ui/resizable/index.js +2 -2
  18. package/dist/components/ui/slider/index.d.ts +3 -0
  19. package/dist/components/ui/slider/index.js +5 -0
  20. package/dist/components/ui/slider/range-slider.svelte +94 -0
  21. package/dist/components/ui/slider/range-slider.svelte.d.ts +21 -0
  22. package/dist/components/ui/slider/slider.svelte +83 -0
  23. package/dist/components/ui/slider/slider.svelte.d.ts +7 -0
  24. package/dist/components/viewers/ArchiveViewer.svelte +140 -113
  25. package/dist/components/viewers/CodeViewer.svelte +45 -48
  26. package/dist/components/viewers/CodeViewer.svelte.d.ts +1 -1
  27. package/dist/components/viewers/CogControls.svelte +338 -184
  28. package/dist/components/viewers/CogControls.svelte.d.ts +33 -10
  29. package/dist/components/viewers/CogViewer.svelte +269 -116
  30. package/dist/components/viewers/CopcViewer.svelte +8 -15
  31. package/dist/components/viewers/DatabaseViewer.svelte +22 -21
  32. package/dist/components/viewers/FileInfo.svelte +16 -16
  33. package/dist/components/viewers/FlatGeobufViewer.svelte +16 -46
  34. package/dist/components/viewers/GeoParquetMapViewer.svelte +11 -9
  35. package/dist/components/viewers/GeoParquetMapViewer.svelte.d.ts +1 -1
  36. package/dist/components/viewers/ImageViewer.svelte +12 -14
  37. package/dist/components/viewers/LoadProgress.svelte +6 -6
  38. package/dist/components/viewers/MarkdownViewer.svelte +29 -30
  39. package/dist/components/viewers/MediaViewer.svelte +13 -14
  40. package/dist/components/viewers/ModelViewer.svelte +18 -21
  41. package/dist/components/viewers/MultiCogViewer.svelte +474 -106
  42. package/dist/components/viewers/MultiCogViewer.svelte.d.ts +1 -1
  43. package/dist/components/viewers/NotebookViewer.svelte +28 -29
  44. package/dist/components/viewers/PdfViewer.svelte +24 -33
  45. package/dist/components/viewers/PmtilesViewer.svelte +13 -15
  46. package/dist/components/viewers/QueryHistoryPanel.svelte +18 -18
  47. package/dist/components/viewers/RawViewer.svelte +27 -21
  48. package/dist/components/viewers/StacMapViewer.svelte +6 -13
  49. package/dist/components/viewers/StacMosaicViewer.svelte +1764 -410
  50. package/dist/components/viewers/StacMosaicViewer.svelte.d.ts +1 -1
  51. package/dist/components/viewers/StacTabViewer.svelte +26 -15
  52. package/dist/components/viewers/StacTabViewer.svelte.d.ts +1 -1
  53. package/dist/components/viewers/TableGrid.svelte +38 -34
  54. package/dist/components/viewers/TableStatusBar.svelte +7 -7
  55. package/dist/components/viewers/TableToolbar.svelte +10 -9
  56. package/dist/components/viewers/TableViewer.svelte +47 -30
  57. package/dist/components/viewers/TableViewer.svelte.d.ts +1 -0
  58. package/dist/components/viewers/ViewerHeader.svelte +18 -0
  59. package/dist/components/viewers/ViewerHeader.svelte.d.ts +10 -0
  60. package/dist/components/viewers/ViewerRouter.svelte +16 -8
  61. package/dist/components/viewers/ViewerStatus.svelte +19 -0
  62. package/dist/components/viewers/ViewerStatus.svelte.d.ts +7 -0
  63. package/dist/components/viewers/ZarrMapViewer.svelte +24 -21
  64. package/dist/components/viewers/ZarrViewer.svelte +98 -65
  65. package/dist/components/viewers/cog/ChannelPicker.svelte +83 -0
  66. package/dist/components/viewers/cog/ChannelPicker.svelte.d.ts +13 -0
  67. package/dist/components/viewers/cog/PixelInspectorPanel.svelte +87 -0
  68. package/dist/components/viewers/cog/PixelInspectorPanel.svelte.d.ts +17 -0
  69. package/dist/components/viewers/cog/buildRgbLayer.d.ts +78 -0
  70. package/dist/components/viewers/cog/buildRgbLayer.js +176 -0
  71. package/dist/components/viewers/map/AttributeTable.svelte +7 -7
  72. package/dist/components/viewers/map/MapContainer.svelte +38 -12
  73. package/dist/components/viewers/pmtiles/PmtilesArchiveView.svelte +109 -83
  74. package/dist/components/viewers/pmtiles/PmtilesTileInspector.svelte +16 -16
  75. package/dist/components/viewers/stac/StacDatetimeBar.svelte +175 -0
  76. package/dist/components/viewers/stac/StacDatetimeBar.svelte.d.ts +10 -0
  77. package/dist/components/viewers/stac/StacFilterPanel.svelte +243 -0
  78. package/dist/components/viewers/stac/StacFilterPanel.svelte.d.ts +14 -0
  79. package/dist/components/viewers/stac/StacItemInspector.svelte +223 -0
  80. package/dist/components/viewers/stac/StacItemInspector.svelte.d.ts +10 -0
  81. package/dist/components/viewers/stac/StacItemStrip.svelte +228 -0
  82. package/dist/components/viewers/stac/StacItemStrip.svelte.d.ts +12 -0
  83. package/dist/constants.d.ts +6 -0
  84. package/dist/constants.js +8 -0
  85. package/dist/file-icons/index.d.ts +1 -1
  86. package/dist/file-icons/index.js +1 -1
  87. package/dist/i18n/ar.js +113 -2
  88. package/dist/i18n/en.js +113 -2
  89. package/dist/index.d.ts +2 -28
  90. package/dist/index.js +7 -23
  91. package/dist/query/engine.d.ts +10 -0
  92. package/dist/query/source.js +1 -1
  93. package/dist/query/stac-source-factory.d.ts +65 -0
  94. package/dist/query/stac-source-factory.js +77 -0
  95. package/dist/query/stac-source-parquet.d.ts +135 -0
  96. package/dist/query/stac-source-parquet.js +468 -0
  97. package/dist/query/wasm.d.ts +8 -0
  98. package/dist/query/wasm.js +310 -65
  99. package/dist/storage/presign.js +3 -2
  100. package/dist/storage/providers.js +7 -6
  101. package/dist/stores/config.svelte.d.ts +15 -0
  102. package/dist/stores/config.svelte.js +46 -0
  103. package/dist/stores/connections.svelte.d.ts +2 -2
  104. package/dist/stores/connections.svelte.js +1 -2
  105. package/dist/stores/files.svelte.d.ts +1 -1
  106. package/dist/stores/files.svelte.js +1 -1
  107. package/dist/stores/query-history.svelte.js +1 -1
  108. package/dist/stores/settings.svelte.d.ts +16 -1
  109. package/dist/stores/settings.svelte.js +104 -48
  110. package/dist/stores/tabs.svelte.d.ts +3 -0
  111. package/dist/stores/tabs.svelte.js +17 -0
  112. package/dist/utils/cog-histogram.d.ts +121 -0
  113. package/dist/utils/cog-histogram.js +424 -0
  114. package/dist/utils/cog.d.ts +177 -20
  115. package/dist/utils/cog.js +361 -76
  116. package/dist/utils/colormap-sprite.d.ts +0 -9
  117. package/dist/utils/colormap-sprite.js +0 -21
  118. package/dist/utils/deck.d.ts +18 -12
  119. package/dist/utils/deck.js +15 -7
  120. package/dist/utils/media-query.svelte.d.ts +14 -0
  121. package/dist/utils/media-query.svelte.js +29 -0
  122. package/dist/utils/pmtiles-tile.js +2 -2
  123. package/dist/utils/signed-url-effect.d.ts +7 -0
  124. package/dist/utils/signed-url-effect.js +19 -0
  125. package/dist/utils/{url.d.ts → signed-url.d.ts} +15 -1
  126. package/dist/utils/{url.js → signed-url.js} +32 -10
  127. package/dist/utils/url-state.d.ts +36 -0
  128. package/dist/utils/url-state.js +72 -2
  129. package/dist/utils/zarr-tab.d.ts +1 -2
  130. package/dist/utils/zarr-tab.js +1 -2
  131. package/dist/utils/zarr.d.ts +0 -17
  132. package/dist/utils/zarr.js +1 -45
  133. package/package.json +55 -84
  134. package/dist/components/browser/Breadcrumb.svelte +0 -50
  135. package/dist/components/browser/Breadcrumb.svelte.d.ts +0 -7
  136. package/dist/components/browser/CreateFolderDialog.svelte +0 -98
  137. package/dist/components/browser/CreateFolderDialog.svelte.d.ts +0 -6
  138. package/dist/components/browser/DeleteConfirmDialog.svelte +0 -90
  139. package/dist/components/browser/DeleteConfirmDialog.svelte.d.ts +0 -8
  140. package/dist/components/browser/DropZone.svelte +0 -83
  141. package/dist/components/browser/DropZone.svelte.d.ts +0 -7
  142. package/dist/components/browser/FileBrowser.svelte +0 -252
  143. package/dist/components/browser/FileBrowser.svelte.d.ts +0 -3
  144. package/dist/components/browser/FileRow.svelte +0 -117
  145. package/dist/components/browser/FileRow.svelte.d.ts +0 -9
  146. package/dist/components/browser/RenameDialog.svelte +0 -101
  147. package/dist/components/browser/RenameDialog.svelte.d.ts +0 -8
  148. package/dist/components/browser/SearchBar.svelte +0 -40
  149. package/dist/components/browser/SearchBar.svelte.d.ts +0 -6
  150. package/dist/components/browser/UploadButton.svelte +0 -65
  151. package/dist/components/browser/UploadButton.svelte.d.ts +0 -3
  152. package/dist/query/stac-geoparquet.d.ts +0 -31
  153. package/dist/query/stac-geoparquet.js +0 -136
  154. package/dist/utils/clipboard.d.ts +0 -13
  155. package/dist/utils/clipboard.js +0 -38
  156. package/dist/utils/cloud-url.d.ts +0 -27
  157. package/dist/utils/cloud-url.js +0 -61
  158. package/dist/utils/cog-pure.d.ts +0 -25
  159. package/dist/utils/cog-pure.js +0 -35
  160. package/dist/utils/column-types.d.ts +0 -5
  161. package/dist/utils/column-types.js +0 -137
  162. package/dist/utils/connection-identity.d.ts +0 -51
  163. package/dist/utils/connection-identity.js +0 -97
  164. package/dist/utils/error.d.ts +0 -8
  165. package/dist/utils/error.js +0 -12
  166. package/dist/utils/evidence-context.d.ts +0 -22
  167. package/dist/utils/evidence-context.js +0 -56
  168. package/dist/utils/export.d.ts +0 -22
  169. package/dist/utils/export.js +0 -76
  170. package/dist/utils/file-sort.d.ts +0 -20
  171. package/dist/utils/file-sort.js +0 -41
  172. package/dist/utils/format.d.ts +0 -24
  173. package/dist/utils/format.js +0 -78
  174. package/dist/utils/geoarrow.d.ts +0 -32
  175. package/dist/utils/geoarrow.js +0 -672
  176. package/dist/utils/geometry-type.d.ts +0 -52
  177. package/dist/utils/geometry-type.js +0 -76
  178. package/dist/utils/hex.d.ts +0 -10
  179. package/dist/utils/hex.js +0 -27
  180. package/dist/utils/host-detection.d.ts +0 -23
  181. package/dist/utils/host-detection.js +0 -95
  182. package/dist/utils/local-storage.d.ts +0 -16
  183. package/dist/utils/local-storage.js +0 -37
  184. package/dist/utils/markdown-sql.d.ts +0 -30
  185. package/dist/utils/markdown-sql.js +0 -72
  186. package/dist/utils/notebook.d.ts +0 -59
  187. package/dist/utils/notebook.js +0 -211
  188. package/dist/utils/parquet-metadata.d.ts +0 -64
  189. package/dist/utils/parquet-metadata.js +0 -262
  190. package/dist/utils/stac-geoparquet.d.ts +0 -90
  191. package/dist/utils/stac-geoparquet.js +0 -223
  192. package/dist/utils/stac-hydrate.d.ts +0 -38
  193. package/dist/utils/stac-hydrate.js +0 -243
  194. package/dist/utils/stac.d.ts +0 -136
  195. package/dist/utils/stac.js +0 -176
  196. package/dist/utils/storage-url.d.ts +0 -90
  197. package/dist/utils/storage-url.js +0 -568
  198. package/dist/utils/wkb.d.ts +0 -43
  199. package/dist/utils/wkb.js +0 -359
@@ -1,7 +1,7 @@
1
- import { DEFAULT_TARGET_CRS, DUCKDB_INIT_TIMEOUT_MS, WGS84_CODES } from '../constants.js';
1
+ import { buildTransformExpr, wrapWkbWithCrs } from '@walkthru-earth/objex-utils';
2
+ import { DEFAULT_TARGET_CRS, DUCKDB_INIT_TIMEOUT_MS, STORAGE_KEYS, WGS84_CODES } from '../constants.js';
2
3
  import { getAccessMode, resolveProviderEndpoint } from '../storage/providers.js';
3
4
  import { credentialStore } from '../stores/credentials.svelte.js';
4
- import { buildTransformExpr, wrapWkbWithCrs } from '../utils/geometry-type.js';
5
5
  import { QueryCancelledError } from './engine';
6
6
  import { isHttpsSourceRef } from './source.js';
7
7
  const DUCKDB_VERSION = __DUCKDB_WASM_VERSION__;
@@ -23,6 +23,22 @@ function elapsed(start) {
23
23
  return `${(performance.now() - start).toFixed(1)}ms`;
24
24
  }
25
25
  let dbPromise = null;
26
+ const opfsState = { active: false, reasons: [] };
27
+ let initSummaryLogged = false;
28
+ function logInitSummary() {
29
+ if (initSummaryLogged)
30
+ return;
31
+ initSummaryLogged = true;
32
+ const coi = typeof globalThis !== 'undefined' && globalThis.crossOriginIsolated === true;
33
+ if (opfsState.active) {
34
+ log(`▶ DB mode: OPFS-backed (spill enabled), crossOriginIsolated=${coi}`);
35
+ }
36
+ else {
37
+ logWarn(`▶ DB mode: IN-MEMORY (no spill — OOMs at WASM heap ceiling), crossOriginIsolated=${coi}`);
38
+ for (const r of opfsState.reasons)
39
+ logWarn(` ↳ ${r}`);
40
+ }
41
+ }
26
42
  function withTimeout(promise, ms, label) {
27
43
  return new Promise((resolve, reject) => {
28
44
  const timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
@@ -37,7 +53,7 @@ function withTimeout(promise, ms, label) {
37
53
  }
38
54
  async function getDB() {
39
55
  if (dbPromise) {
40
- log('getDB → cached');
56
+ logInitSummary();
41
57
  return dbPromise;
42
58
  }
43
59
  dbPromise = (async () => {
@@ -64,6 +80,139 @@ async function getDB() {
64
80
  const db = new duckdb.AsyncDuckDB(logger, worker);
65
81
  await withTimeout(db.instantiate(bundle.mainModule, bundle.pthreadWorker), INIT_TIMEOUT_MS, 'DuckDB WASM instantiation');
66
82
  log(`getDB → instantiated in ${elapsed(t0)}`);
83
+ // Open an OPFS-backed database so DuckDB can spill blocks to disk under
84
+ // memory pressure. In-memory mode caps at the WASM heap (~1.3-3.1 GiB
85
+ // depending on browser), with no temp_directory backing — large
86
+ // aggregations / sorts then OOM. OPFS gives DuckDB a real filesystem
87
+ // for paged storage, lifting the budget to the user's disk quota.
88
+ //
89
+ // Why register a handle instead of just `path: 'opfs://...'`: on the
90
+ // pinned 1.33.1-dev53.0 build, the URL-scheme path opens silently as a
91
+ // virtual MEMFS file, so the DB stays in-memory. Registering the handle
92
+ // via DuckDBDataProtocol.BROWSER_FSACCESS (OPFS) and opening it by the
93
+ // registered name forces DuckDB onto OPFS and lets it spill blocks.
94
+ // Falls back to in-memory if OPFS is unavailable (private mode, older
95
+ // browsers, cross-origin isolation issues).
96
+ const hasOpfs = typeof navigator !== 'undefined' &&
97
+ typeof navigator.storage !== 'undefined' &&
98
+ typeof navigator.storage.getDirectory === 'function';
99
+ const isCOI = typeof globalThis !== 'undefined' && globalThis.crossOriginIsolated === true;
100
+ if (!isCOI) {
101
+ opfsState.reasons.push('crossOriginIsolated=false — page is missing COOP/COEP headers. OPFS sync handles will throw inside the worker. Restart the dev server so vite.config.ts headers take effect, then hard-reload.');
102
+ }
103
+ let opfsActive = false;
104
+ if (hasOpfs) {
105
+ // Pin the OPFS filename to the duckdb-wasm version. Bumping
106
+ // duckdb-wasm can change the on-disk storage version, and reusing a
107
+ // stale file then throws "not a valid DuckDB database file" inside
108
+ // the worker. Worse, on Chrome the failed open keeps the
109
+ // FileSystemSyncAccessHandle alive, so even a same-tab retry
110
+ // (`removeEntry` + recreate) reads back the old state and trips
111
+ // "Access Handles cannot be created if there is another open Access
112
+ // Handle" on every fallback path. A version-pinned filename sidesteps
113
+ // the whole recovery problem: a new wasm build always gets a fresh
114
+ // file, and the old file just becomes an orphan we can reap below.
115
+ // Sanitize the version into a filename-safe slug.
116
+ const dbVersionSlug = DUCKDB_VERSION.replace(/[^a-zA-Z0-9._-]/g, '-');
117
+ const dbName = `objex-duckdb-v${dbVersionSlug}.db`;
118
+ const accessMode = duckdb.DuckDBAccessMode?.READ_WRITE ?? 1;
119
+ const protocol = duckdb.DuckDBDataProtocol?.BROWSER_FSACCESS;
120
+ // Firefox exposes `navigator.storage.getDirectory` in Private Browsing
121
+ // but throws `SecurityError: Security error when calling GetDirectory`
122
+ // when invoked. The `hasOpfs` check above only verifies the function
123
+ // exists, so we must guard the actual call. Other "exists-but-throws"
124
+ // environments (older Safari WKWebView, restricted enterprise policies,
125
+ // non-secure contexts that still expose the API) hit the same path.
126
+ let root = null;
127
+ try {
128
+ root = await navigator.storage.getDirectory();
129
+ }
130
+ catch (err) {
131
+ const msg = err?.message ?? String(err);
132
+ logWarn('getDB → navigator.storage.getDirectory() threw:', msg);
133
+ opfsState.reasons.push(`navigator.storage.getDirectory() threw: ${msg} — likely Firefox Private Browsing, non-secure context, or storage permission denied`);
134
+ }
135
+ if (root) {
136
+ // Reap orphan DBs from prior wasm versions. They are unreachable
137
+ // once the version slug bumps, so leaving them around silently
138
+ // burns OPFS quota. removeEntry throws NoModificationAllowedError
139
+ // while a sync access handle is still open against the file —
140
+ // that only matters for the file we're actively using, and we
141
+ // skip that one.
142
+ try {
143
+ const orphans = [];
144
+ for await (const [name, handle] of root.entries()) {
145
+ if (handle.kind === 'file' &&
146
+ /^objex-duckdb(-.*)?\.db$/.test(name) &&
147
+ name !== dbName) {
148
+ orphans.push(name);
149
+ }
150
+ }
151
+ for (const name of orphans) {
152
+ try {
153
+ await root.removeEntry(name);
154
+ log(`getDB → reaped orphan OPFS db "${name}"`);
155
+ }
156
+ catch (rmErr) {
157
+ logWarn(`getDB → could not reap orphan OPFS db "${name}":`, rmErr?.message ?? rmErr);
158
+ }
159
+ }
160
+ }
161
+ catch (enumErr) {
162
+ logWarn('getDB → could not enumerate OPFS root for orphan cleanup:', enumErr?.message ?? enumErr);
163
+ }
164
+ // Path A — registered FileSystemFileHandle. This is the fully wired
165
+ // OPFS path on duckdb-wasm 1.29+. It needs the real handle (passing
166
+ // null silently registers a non-OPFS file and the open() reverts to
167
+ // MEMFS, which is why earlier attempts kept reporting "in-memory
168
+ // mode" in OOM errors).
169
+ if (protocol !== undefined) {
170
+ const tOpenA = performance.now();
171
+ try {
172
+ const fileHandle = await root.getFileHandle(dbName, { create: true });
173
+ await db.registerFileHandle(dbName, fileHandle, protocol, true);
174
+ await withTimeout(db.open({ path: dbName, accessMode }), INIT_TIMEOUT_MS, 'DuckDB OPFS open (registered handle)');
175
+ opfsActive = true;
176
+ log(`getDB → OPFS via registered handle in ${elapsed(tOpenA)}`);
177
+ }
178
+ catch (err) {
179
+ const msg = err?.message ?? String(err);
180
+ logWarn('getDB → OPFS via registered handle failed:', msg);
181
+ opfsState.reasons.push(`registered-handle path: ${msg}`);
182
+ }
183
+ }
184
+ else {
185
+ logWarn('getDB → DuckDBDataProtocol.BROWSER_FSACCESS missing in this build');
186
+ opfsState.reasons.push('DuckDBDataProtocol.BROWSER_FSACCESS missing in this WASM build');
187
+ }
188
+ // Path B — URL-scheme fallback. Some duckdb-wasm builds wire OPFS
189
+ // internally on `opfs://` paths without needing manual registration.
190
+ if (!opfsActive) {
191
+ const tOpenB = performance.now();
192
+ try {
193
+ await withTimeout(db.open({ path: `opfs://${dbName}`, accessMode }), INIT_TIMEOUT_MS, 'DuckDB OPFS open (url scheme)');
194
+ opfsActive = true;
195
+ log(`getDB → OPFS via opfs:// scheme in ${elapsed(tOpenB)}`);
196
+ }
197
+ catch (err) {
198
+ const msg = err?.message ?? String(err);
199
+ logWarn('getDB → OPFS via opfs:// scheme failed:', msg);
200
+ opfsState.reasons.push(`opfs:// scheme path: ${msg}`);
201
+ }
202
+ }
203
+ if (!opfsActive) {
204
+ logWarn('getDB → all OPFS paths failed, falling back to in-memory');
205
+ }
206
+ }
207
+ else {
208
+ logWarn('getDB → OPFS root unavailable, falling back to in-memory');
209
+ }
210
+ }
211
+ else {
212
+ log('getDB → OPFS unavailable (no navigator.storage), using in-memory DB');
213
+ opfsState.reasons.push('navigator.storage.getDirectory is unavailable (private mode, older browser, or non-secure context)');
214
+ }
215
+ opfsState.active = opfsActive;
67
216
  // Load httpfs for remote file access and spatial for ST_ReadSHP
68
217
  const conn = await db.connect();
69
218
  try {
@@ -92,12 +241,84 @@ async function getDB() {
92
241
  catch {
93
242
  logWarn('force_download_threshold not available');
94
243
  }
244
+ // Memory tuning is only applied when OPFS spill is actually wired.
245
+ // Without OPFS, capping memory_limit below the WASM heap ceiling
246
+ // just makes queries OOM sooner with no upside (DuckDB still cannot
247
+ // spill, and TableViewer's existing workloads need the full ~3 GiB
248
+ // heap). preserve_insertion_order = false is the one tuning that
249
+ // helps both modes — it skips buffering full result sets to re-sort
250
+ // on output, a real OOM source on big scans.
251
+ // Mobile WebKit / Android Chrome run in a much smaller WASM heap
252
+ // (~1.8 GiB on iOS Safari, ~2 GiB on Android) than desktop
253
+ // (~3.1 GiB), AND mobile Safari < 17.6 has no `credentialless`
254
+ // COEP, so OPFS engages on far fewer devices. Detect with the UA
255
+ // hint + screen-size heuristic so we can ratchet down the limits
256
+ // on mobile-in-memory before the first OOM and so the user sees
257
+ // a usable session instead of a hard failure on the first
258
+ // pan/scan. Heuristic, not load-bearing for correctness.
259
+ const isMobileLike = typeof navigator !== 'undefined' &&
260
+ (/Mobi|Android|iPhone|iPad|iPod/i.test(navigator.userAgent) ||
261
+ (typeof window !== 'undefined' &&
262
+ Math.min(window.innerWidth, window.innerHeight) <= 820));
263
+ try {
264
+ const sets = [`SET preserve_insertion_order = false`];
265
+ const cores = navigator.hardwareConcurrency || 4;
266
+ if (opfsActive) {
267
+ // OPFS spill works: room to use multiple threads + a real
268
+ // memory_limit. On mobile the OPFS quota is also smaller
269
+ // (typically a few hundred MB by default), so still
270
+ // half-cap threads + memory_limit to reduce concurrent
271
+ // thread-local buffers.
272
+ const threads = Math.max(1, Math.min(isMobileLike ? 2 : 4, Math.floor(cores / 2)));
273
+ sets.push(`SET memory_limit = '${isMobileLike ? '900MB' : '2GB'}'`);
274
+ sets.push(`SET threads = ${threads}`);
275
+ sets.push(`SET temp_directory = '.tmp'`);
276
+ }
277
+ else if (isMobileLike) {
278
+ // Mobile + no OPFS spill is the worst case: the heap caps
279
+ // at ~1.8 GiB on iOS Safari, queries cannot offload, and
280
+ // the previous "leave memory_limit at defaults" path
281
+ // reproduces the OOM the user is seeing. Cap memory_limit
282
+ // well under the heap ceiling so DuckDB's planner picks
283
+ // streaming / out-of-core operator variants and surfaces
284
+ // `Out of Memory` from inside the planner (recoverable,
285
+ // the worker keeps running) instead of from the WASM heap
286
+ // allocator (kills the worker and forces a tab reload).
287
+ // Threads are pinned at 1 for the same reason — every
288
+ // extra thread doubles the per-pipeline scratch buffers.
289
+ sets.push(`SET memory_limit = '900MB'`);
290
+ sets.push(`SET threads = 1`);
291
+ sets.push(`SET temp_directory = '.tmp'`);
292
+ }
293
+ else {
294
+ // Desktop + no OPFS spill: leave memory_limit at defaults
295
+ // so the full WASM heap (~3.1 GiB) is available for big
296
+ // scans. Cap threads conservatively because each thread
297
+ // keeps its own hash-aggregate / sort buffer alive.
298
+ const threads = Math.max(1, Math.min(2, Math.floor(cores / 2)));
299
+ sets.push(`SET threads = ${threads}`);
300
+ sets.push(`SET temp_directory = '.tmp'`);
301
+ }
302
+ await conn.query(`${sets.join('; ')};`);
303
+ log(opfsActive
304
+ ? `getDB → OPFS, memory_limit=${isMobileLike ? '900MB' : '2GB'}, mobile=${isMobileLike}`
305
+ : isMobileLike
306
+ ? `getDB → mobile in-memory, memory_limit=900MB, threads=1 (no OPFS spill)`
307
+ : `getDB → desktop in-memory, threads capped, memory_limit at defaults`);
308
+ if (isMobileLike && !opfsActive) {
309
+ opfsState.reasons.push('mobile session with no OPFS spill: memory_limit clamped to 900MB to surface OOMs from the planner instead of the WASM heap allocator. Some large-catalog stac-geoparquet queries may still fail; reduce mosaicItemLimit or open on desktop.');
310
+ }
311
+ }
312
+ catch (err) {
313
+ logWarn('memory tuning not available:', err?.message ?? err);
314
+ }
95
315
  log(`getDB → extensions loaded in ${elapsed(tExt)}`);
96
316
  }
97
317
  finally {
98
318
  await conn.close();
99
319
  }
100
320
  log(`getDB → ready (total ${elapsed(t0)})`);
321
+ logInitSummary();
101
322
  return db;
102
323
  })();
103
324
  // If init fails, clear the promise so it can be retried
@@ -358,67 +579,10 @@ function isBinaryType(typeStr) {
358
579
  }
359
580
  export class WasmQueryEngine {
360
581
  async query(connId, sql) {
361
- const t0 = performance.now();
362
- const sqlPreview = sql.length > 120 ? `${sql.slice(0, 120)}…` : sql;
363
- log(`query ${sqlPreview}`);
364
- const db = await getDB();
365
- const conn = await db.connect();
366
- const tConn = performance.now();
367
- log(`query → connected in ${elapsed(t0)}`);
368
- try {
369
- if (connId) {
370
- await this.configureStorage(conn, connId, sql);
371
- log(`query → storage configured in ${elapsed(tConn)}`);
372
- }
373
- const tQuery = performance.now();
374
- const result = await conn.query(sql);
375
- log(`query → executed in ${elapsed(tQuery)}, rows: ${result.numRows}`);
376
- // DuckDB WASM returns an Arrow Table (bundled apache-arrow@17).
377
- // Our project uses apache-arrow@21 — cross-version tableToIPC/tableFromIPC
378
- // loses data rows. Extract rows directly from DuckDB's own Arrow Table.
379
- const numRows = result.numRows;
380
- const cols = result.schema.fields.map((f) => f.name);
381
- const types = result.schema.fields.map((f) => String(f.type));
382
- if (numRows === 0) {
383
- log(`query → done (empty) in ${elapsed(t0)}`);
384
- return {
385
- columns: cols,
386
- types,
387
- rowCount: 0,
388
- rows: []
389
- };
390
- }
391
- // Arrow emits DECIMAL columns as multi-word BigInt / Uint32Array buffers.
392
- // `String(rawDecimal)` yields the unscaled integer (or "0,0,0,0"),
393
- // so rewrite each decimal cell through formatDecimal with the column scale.
394
- const decimalCols = [];
395
- for (let i = 0; i < cols.length; i++) {
396
- const s = decimalScale(types[i]);
397
- if (s >= 0)
398
- decimalCols.push({ name: cols[i], scale: s });
399
- }
400
- // Extract rows directly — avoids Arrow version mismatch
401
- const rows = result.toArray().map((row) => {
402
- const obj = typeof row.toJSON === 'function' ? row.toJSON() : {};
403
- if (typeof row.toJSON !== 'function') {
404
- for (const col of cols)
405
- obj[col] = row[col];
406
- }
407
- for (const { name, scale } of decimalCols) {
408
- obj[name] = formatDecimal(obj[name], scale);
409
- }
410
- return obj;
411
- });
412
- log(`query → done in ${elapsed(t0)}, ${numRows} rows, ${cols.length} cols`);
413
- return { columns: cols, types, rowCount: numRows, rows };
414
- }
415
- catch (err) {
416
- logWarn(`query → failed after ${elapsed(t0)}:`, err?.message ?? err);
417
- throw err;
418
- }
419
- finally {
420
- await conn.close();
421
- }
582
+ // Delegate to the send()-based path so a data query never blocks the
583
+ // single DuckDB worker (conn.query() is blocking). Same return shape.
584
+ const { result } = this.queryCancellable(connId, sql);
585
+ return result;
422
586
  }
423
587
  async queryForMap(connId, sql, geomCol, geomColType, sourceCrs) {
424
588
  const t0 = performance.now();
@@ -634,7 +798,7 @@ export class WasmQueryEngine {
634
798
  log('configureStorage → presigned HTTPS source, skipping S3 config');
635
799
  return;
636
800
  }
637
- const stored = localStorage.getItem('obstore-explore-connections');
801
+ const stored = localStorage.getItem(STORAGE_KEYS.CONNECTIONS);
638
802
  if (!stored) {
639
803
  log('configureStorage → no connections in localStorage');
640
804
  return;
@@ -877,6 +1041,87 @@ export class WasmQueryEngine {
877
1041
  };
878
1042
  return { result, cancel };
879
1043
  }
1044
+ /**
1045
+ * Streaming variant of `queryCancellable`. Yields one chunk per Arrow
1046
+ * RecordBatch so peak memory tracks one batch instead of the full result.
1047
+ * Used by `stac-source-parquet` to ingest large catalogs progressively
1048
+ * without OOMing the WASM heap. Cancellation routes through `conn.cancelSent`
1049
+ * and `signal.aborted`; the connection is always closed in `finally`.
1050
+ */
1051
+ async *queryStream(connId, sql, signal) {
1052
+ const t0 = performance.now();
1053
+ const sqlPreview = sql.length > 120 ? `${sql.slice(0, 120)}…` : sql;
1054
+ log(`queryStream → ${sqlPreview}`);
1055
+ const db = await getDB();
1056
+ const conn = await db.connect();
1057
+ const onAbort = () => {
1058
+ try {
1059
+ conn?.cancelSent?.();
1060
+ }
1061
+ catch {
1062
+ /* noop */
1063
+ }
1064
+ };
1065
+ signal?.addEventListener('abort', onAbort, { once: true });
1066
+ try {
1067
+ if (connId) {
1068
+ await this.configureStorage(conn, connId, sql);
1069
+ }
1070
+ const reader = await conn.send(sql);
1071
+ let cols = [];
1072
+ let types = [];
1073
+ let decimalCols = [];
1074
+ let first = true;
1075
+ let total = 0;
1076
+ const batches = reader[Symbol.asyncIterator]();
1077
+ while (true) {
1078
+ if (signal?.aborted)
1079
+ throw new QueryCancelledError();
1080
+ const { value: batch, done } = await batches.next();
1081
+ if (done)
1082
+ break;
1083
+ if (first && batch.schema) {
1084
+ cols = batch.schema.fields.map((f) => f.name);
1085
+ types = batch.schema.fields.map((f) => String(f.type));
1086
+ decimalCols = [];
1087
+ for (let i = 0; i < cols.length; i++) {
1088
+ const s = decimalScale(types[i]);
1089
+ if (s >= 0)
1090
+ decimalCols.push({ name: cols[i], scale: s });
1091
+ }
1092
+ first = false;
1093
+ }
1094
+ const rows = [];
1095
+ for (const row of batch.toArray()) {
1096
+ const json = typeof row.toJSON === 'function' ? row.toJSON() : { ...row };
1097
+ for (const key in json) {
1098
+ if (json[key] instanceof Uint8Array) {
1099
+ json[key] = json[key].slice();
1100
+ }
1101
+ }
1102
+ for (const { name, scale } of decimalCols) {
1103
+ json[name] = formatDecimal(json[name], scale);
1104
+ }
1105
+ rows.push(json);
1106
+ }
1107
+ total += rows.length;
1108
+ yield { columns: cols, types, rowCount: rows.length, rows };
1109
+ }
1110
+ log(`queryStream → done in ${elapsed(t0)}, ${total} rows total`);
1111
+ }
1112
+ catch (err) {
1113
+ if (signal?.aborted || err instanceof QueryCancelledError) {
1114
+ log(`queryStream → cancelled after ${elapsed(t0)}`);
1115
+ throw new QueryCancelledError();
1116
+ }
1117
+ logWarn(`queryStream → failed after ${elapsed(t0)}:`, err?.message ?? err);
1118
+ throw err;
1119
+ }
1120
+ finally {
1121
+ signal?.removeEventListener('abort', onAbort);
1122
+ await conn?.close?.();
1123
+ }
1124
+ }
880
1125
  queryForMapCancellable(connId, sql, geomCol, geomColType, sourceCrs) {
881
1126
  let cancelled = false;
882
1127
  let conn = null;
@@ -1,5 +1,6 @@
1
+ import { safeDecodeURIComponent } from '@walkthru-earth/objex-utils';
2
+ import { DEFAULT_AWS_REGION } from '../constants.js';
1
3
  import { credentialStore } from '../stores/credentials.svelte.js';
2
- import { safeDecodeURIComponent } from '../utils/cloud-url.js';
3
4
  import { buildProviderBaseUrl, getAccessMode } from './providers.js';
4
5
  // 7 days is the SigV4 protocol maximum and is the hard cap on every
5
6
  // S3-compatible provider we support (AWS, GCS, R2, B2, DO, Wasabi, Storj,
@@ -38,7 +39,7 @@ export async function presignHttpsUrl(conn, key, expiresIn = DEFAULT_EXPIRES_IN_
38
39
  accessKeyId: creds.accessKey,
39
40
  secretAccessKey: creds.secretKey,
40
41
  service: 's3',
41
- region: conn.region || 'us-east-1'
42
+ region: conn.region || DEFAULT_AWS_REGION
42
43
  });
43
44
  const signed = await client.sign(url.toString(), {
44
45
  method: 'GET',
@@ -4,6 +4,7 @@
4
4
  * Centralizes endpoint patterns, regions, auth methods, and UI metadata.
5
5
  * Used by ConnectionDialog, browser-cloud adapter, host-detection, url-state, etc.
6
6
  */
7
+ import { DEFAULT_AWS_REGION } from '../constants.js';
7
8
  // ---------------------------------------------------------------------------
8
9
  // Registry
9
10
  // ---------------------------------------------------------------------------
@@ -87,15 +88,15 @@ export const PROVIDERS = {
87
88
  schemes: ['azure', 'az', 'abfs', 'abfss', 'wasbs', 'adl']
88
89
  },
89
90
  minio: {
90
- label: 'MinIO',
91
- description: 'Self-hosted MinIO or S3-compatible',
91
+ label: 'MinIO / RustFS / Custom',
92
+ description: 'MinIO, RustFS, or any custom S3-compatible endpoint',
92
93
  authMethod: 'sigv4',
93
94
  needsRegion: false,
94
95
  needsEndpoint: true,
95
96
  defaultRegion: 'us-east-1',
96
97
  endpointTemplate: null,
97
98
  regions: [],
98
- endpointPlaceholder: 'https://minio.example.com or http://localhost:9000',
99
+ endpointPlaceholder: 'https://s3.example.com or http://localhost:9000',
99
100
  schemes: []
100
101
  },
101
102
  storj: {
@@ -297,7 +298,7 @@ export const CORS_HELP = {
297
298
  minio: {
298
299
  defaultEnabled: true,
299
300
  docsUrl: 'https://docs.min.io/enterprise/aistor-object-store/reference/cli/mc-cors/',
300
- note: 'MinIO allows all origins by default. For custom rules, use mc cors set.'
301
+ note: 'MinIO allows all origins by default (for custom rules use mc cors set). For RustFS or any other custom S3 service, set a CORS policy on the server that allows this origin.'
301
302
  },
302
303
  storj: {
303
304
  defaultEnabled: true,
@@ -389,7 +390,7 @@ export const READ_ONLY_HELP = {
389
390
  ]
390
391
  },
391
392
  minio: {
392
- note: 'Create a read-only policy with mc admin policy, or use the built-in readonly canned policy.',
393
+ note: 'For MinIO, create a read-only policy with mc admin policy or use the built-in readonly canned policy. For RustFS or any other custom S3 service, attach a read-only bucket policy on the server.',
393
394
  docsUrl: 'https://docs.min.io/enterprise/aistor-object-store/administration/iam/access/'
394
395
  },
395
396
  digitalocean: {
@@ -463,7 +464,7 @@ export function buildProviderBaseUrl(provider, endpoint, bucket, region) {
463
464
  return `${resolved}/${bucket}`;
464
465
  }
465
466
  // Fallback: AWS S3 path-style
466
- return `https://s3.${region || 'us-east-1'}.amazonaws.com/${bucket}`;
467
+ return `https://s3.${region || DEFAULT_AWS_REGION}.amazonaws.com/${bucket}`;
467
468
  }
468
469
  /** Check if a provider uses the GCS JSON API (not S3 XML). */
469
470
  export function isGcsProvider(provider, endpoint) {
@@ -0,0 +1,15 @@
1
+ import { type AppConfig } from '@walkthru-earth/objex-utils';
2
+ export type ConfigStatus = 'pending' | 'bundled' | 'custom' | 'error';
3
+ /** Reactive accessor for the loaded config and its load status. */
4
+ export declare const appConfig: {
5
+ readonly value: AppConfig;
6
+ readonly status: ConfigStatus;
7
+ };
8
+ /**
9
+ * Fetch and merge the runtime config. Awaited in +layout.ts `load` so the
10
+ * config is ready before any component mounts. A `?config=<url>` param loads a
11
+ * remote file (status `custom`), otherwise the bundled `static/config.json`
12
+ * (status `bundled`). Any failure falls back to defaults (status `error`) and
13
+ * the app still boots.
14
+ */
15
+ export declare function loadConfig(basePath: string): Promise<void>;
@@ -0,0 +1,46 @@
1
+ import { DEFAULT_APP_CONFIG, mergeAppConfig } from '@walkthru-earth/objex-utils';
2
+ let config = $state.raw(DEFAULT_APP_CONFIG);
3
+ let status = $state('pending');
4
+ /** Reactive accessor for the loaded config and its load status. */
5
+ export const appConfig = {
6
+ get value() {
7
+ return config;
8
+ },
9
+ get status() {
10
+ return status;
11
+ }
12
+ };
13
+ function readConfigParam() {
14
+ if (typeof window === 'undefined')
15
+ return null;
16
+ try {
17
+ return new URL(window.location.href).searchParams.get('config');
18
+ }
19
+ catch {
20
+ return null;
21
+ }
22
+ }
23
+ /**
24
+ * Fetch and merge the runtime config. Awaited in +layout.ts `load` so the
25
+ * config is ready before any component mounts. A `?config=<url>` param loads a
26
+ * remote file (status `custom`), otherwise the bundled `static/config.json`
27
+ * (status `bundled`). Any failure falls back to defaults (status `error`) and
28
+ * the app still boots.
29
+ */
30
+ export async function loadConfig(basePath) {
31
+ const customUrl = readConfigParam();
32
+ const url = customUrl ?? `${basePath}/config.json`;
33
+ try {
34
+ const res = await fetch(url, { headers: { accept: 'application/json' } });
35
+ if (!res.ok)
36
+ throw new Error(`HTTP ${res.status}`);
37
+ const json = await res.json();
38
+ config = mergeAppConfig(DEFAULT_APP_CONFIG, json);
39
+ status = customUrl ? 'custom' : 'bundled';
40
+ }
41
+ catch (err) {
42
+ console.warn('[objex] config load failed, using defaults', err);
43
+ config = DEFAULT_APP_CONFIG;
44
+ status = 'error';
45
+ }
46
+ }
@@ -1,6 +1,6 @@
1
+ import type { DetectedHost } from '@walkthru-earth/objex-utils';
2
+ import { type ConnectionIdentityInput } from '@walkthru-earth/objex-utils';
1
3
  import type { Connection, ConnectionConfig } from '../types.js';
2
- import { type ConnectionIdentityInput } from '../utils/connection-identity.js';
3
- import type { DetectedHost } from '../utils/host-detection.js';
4
4
  /**
5
5
  * Outcome of a write. `existed` is true when dedup reused an already-saved
6
6
  * connection, false when a new row was persisted. Callers that present UI
@@ -1,6 +1,5 @@
1
+ import { connectionIdentityKey, loadFromStorage, persistToStorage } from '@walkthru-earth/objex-utils';
1
2
  import { STORAGE_KEYS } from '../constants.js';
2
- import { connectionIdentityKey } from '../utils/connection-identity.js';
3
- import { loadFromStorage, persistToStorage } from '../utils/local-storage.js';
4
3
  import { credentialStore, storeToNative } from './credentials.svelte.js';
5
4
  function toConnection(id, config) {
6
5
  return {
@@ -1,5 +1,5 @@
1
+ import { type SortConfig, type SortField } from '@walkthru-earth/objex-utils';
1
2
  import type { FileEntry } from '../types.js';
2
- import { type SortConfig, type SortField } from '../utils/file-sort.js';
3
3
  export declare const files: {
4
4
  readonly entries: FileEntry[];
5
5
  readonly currentPath: string;
@@ -1,4 +1,4 @@
1
- import { sortFileEntries, toggleSortField } from '../utils/file-sort.js';
1
+ import { sortFileEntries, toggleSortField } from '@walkthru-earth/objex-utils';
2
2
  function createFilesStore() {
3
3
  let files = $state([]);
4
4
  let currentPath = $state('');
@@ -1,5 +1,5 @@
1
+ import { loadFromStorage, persistToStorage } from '@walkthru-earth/objex-utils';
1
2
  import { MAX_QUERY_HISTORY_ENTRIES, STORAGE_KEYS } from '../constants.js';
2
- import { loadFromStorage, persistToStorage } from '../utils/local-storage.js';
3
3
  function createQueryHistoryStore() {
4
4
  let entries = $state(loadFromStorage(STORAGE_KEYS.QUERY_HISTORY, []));
5
5
  function save() {
@@ -1,4 +1,4 @@
1
- import { type Locale } from '../i18n/index.svelte.js';
1
+ import type { Locale } from '../i18n/index.svelte.js';
2
2
  import type { Theme } from '../types.js';
3
3
  export declare function resolveTheme(theme: Theme): 'light' | 'dark';
4
4
  export declare const settings: {
@@ -6,7 +6,22 @@ export declare const settings: {
6
6
  readonly resolved: "light" | "dark";
7
7
  readonly locale: Locale;
8
8
  readonly featureLimit: number;
9
+ readonly mosaicItemLimit: number;
10
+ readonly showConnectionRail: boolean;
11
+ readonly showFileTree: boolean;
12
+ /** True when a link param is forcing the connection-rail visibility. */
13
+ readonly railLockedByParam: boolean;
14
+ /** True when a link param is forcing the file-tree visibility. */
15
+ readonly treeLockedByParam: boolean;
16
+ /** The user-picked basemap id, or undefined to follow config/theme defaults. */
17
+ readonly basemapId: string | undefined;
9
18
  setTheme(t: Theme): void;
10
19
  setLocale(l: Locale): void;
11
20
  setFeatureLimit(n: number): void;
21
+ setMosaicItemLimit(n: number): void;
22
+ setShowConnectionRail(v: boolean): void;
23
+ setShowFileTree(v: boolean): void;
24
+ setBasemap(id: string | undefined): void;
25
+ /** Clear all user overrides, reverting every value to config or hardcoded fallback. */
26
+ reset(): void;
12
27
  };