@walkthru-earth/objex 1.3.1 → 1.4.0

package/dist/index.js

@@ -1,4 +1,11 @@
 // Core types
+// Clipboard
+// Cloud URL resolution
+// Error handling
+// Data export / serialization
+// localStorage helpers
+// Utilities
+export { allChannelsBand0, applyFacets, applyPreset, applyStorageHintsToConnection, availablePresets, buildDataTypeLabel, buildFacets, buildGeoArrowTables, clampBounds, classifyType, compositeFromUrl, compositeToUrl, connectionIdentityKey, copyToClipboard, DATETIME_HISTOGRAM_BINS, describeParseResult, detectStorageExtensionVersion, emptyFacetState, emptyPushdown, emptyStorageHints, escapeCsvField, extractBounds, extractCogAssets, extractEpsgFromGeoMeta, extractGeometryTypes, extractItemView, extractStorageHints, findGeoColumn, findGeoColumnFromRows, flattenStacBbox, formatDate, formatFileSize, formatValue, generateHexDump, getFileExtension, getNativeScheme, handleLoadError, hasActiveFilters, interpolateTemplates, isAbortError, isSameConnectionIdentity, isSingleAssetComposite, isStacGeoparquetSchema, jsonReplacerBigInt, loadFromStorage, looksLikeUrl, markSqlBlocks, normalizeEndpoint, normalizeGeomType, normalizeProvider, PRESETS, parseMarkdownDocument, parseStorageUrl, parseWKB, persistToStorage, pickNaturalColorComposite, pickStacPrimaryAsset, presetMatchesComposite, readParquetMetadata, residualState, resolveCloudUrl, resolveStacAssetHref, SF_LABELS, STAC_GEOPARQUET_REQUIRED_COLUMNS, safeClamp, safeDecodeURIComponent, serializeToCsv, serializeToJson, sniffApiCapabilities, sortFileEntries, sortViews, stacRowToItem, syntheticSelfAsset, toBinary, toCql2Filter, toggleSortField, toNativeQuery, typeBadgeClass, typeColor, typeLabel, wireCodeCopyButtons } from '@walkthru-earth/objex-utils';
 // Constants
 export { COPY_FEEDBACK_MS, DEFAULT_TARGET_CRS, DUCKDB_INIT_TIMEOUT_MS, LAYER_HUE_MULTIPLIER, MAX_QUERY_HISTORY_ENTRIES, SQL_PREVIEW_LENGTH, STORAGE_KEYS, VIEWER_DIR_EXTENSIONS, WGS84_CODES } from './constants.js';
 // File icons registry
@@ -6,26 +13,3 @@ export { buildDuckDbSource, getDuckDbReadFn, getFileTypeInfo, getMimeType, getVi
 export { QueryCancelledError } from './query/engine.js';
 export { buildEndpointFromTemplate, buildProviderBaseUrl, getProvider, isGcsProvider, PROVIDER_IDS, PROVIDERS } from './storage/providers.js';
 export { UrlAdapter } from './storage/url-adapter.js';
-// Clipboard
-export { copyToClipboard, wireCodeCopyButtons } from './utils/clipboard.js';
-// Cloud URL resolution
-export { getNativeScheme, resolveCloudUrl, safeDecodeURIComponent } from './utils/cloud-url.js';
-export { buildDataTypeLabel, clampBounds, SF_LABELS, safeClamp } from './utils/cog.js';
-export { classifyType, typeBadgeClass, typeColor, typeLabel } from './utils/column-types.js';
-export { connectionIdentityKey, isSameConnectionIdentity, normalizeEndpoint, normalizeProvider } from './utils/connection-identity.js';
-// Error handling
-export { handleLoadError } from './utils/error.js';
-// Data export / serialization
-export { escapeCsvField, serializeToCsv, serializeToJson } from './utils/export.js';
-export { sortFileEntries, toggleSortField } from './utils/file-sort.js';
-export { formatDate, formatFileSize, formatValue, getFileExtension, jsonReplacerBigInt } from './utils/format.js';
-export { buildGeoArrowTables, normalizeGeomType } from './utils/geoarrow.js';
-export { generateHexDump } from './utils/hex.js';
-// localStorage helpers
-export { loadFromStorage, persistToStorage } from './utils/local-storage.js';
-export { interpolateTemplates, markSqlBlocks, parseMarkdownDocument } from './utils/markdown-sql.js';
-export { extractBounds, extractEpsgFromGeoMeta, extractGeometryTypes, readParquetMetadata } from './utils/parquet-metadata.js';
-export { flattenStacBbox, isStacGeoparquetSchema, pickStacPrimaryAsset, resolveStacAssetHref, STAC_GEOPARQUET_REQUIRED_COLUMNS, stacRowToItem } from './utils/stac-geoparquet.js';
-export { describeParseResult, looksLikeUrl, parseStorageUrl } from './utils/storage-url.js';
-// Utilities
-export { findGeoColumn, findGeoColumnFromRows, parseWKB, toBinary } from './utils/wkb.js';

package/dist/query/engine.d.ts

@@ -61,6 +61,16 @@ export interface QueryEngine {
         crs: string | null;
     }>;
     queryCancellable?(connId: string, sql: string): QueryHandle;
+    /**
+     * Streaming variant of `queryCancellable`. Yields one `QueryResult`-shaped
+     * chunk per Arrow RecordBatch instead of materializing the full row set
+     * before returning. Peak memory tracks one batch (~64 KiB rows) rather
+     * than the full result, which avoids DuckDB-WASM OOMs on large parquet
+     * scans. The first chunk carries the schema (`columns` / `types`); later
+     * chunks repeat them so consumers don't need to track first-batch state.
+     * Cancellation aborts the inner DuckDB cursor.
+     */
+    queryStream?(connId: string, sql: string, signal?: AbortSignal): AsyncIterable<QueryResult>;
     queryForMapCancellable?(connId: string, sql: string, geomCol: string, geomColType: string, sourceCrs?: string | null): MapQueryHandle;
     forceCancel?(): Promise<void>;
     /** Register a file buffer in DuckDB-WASM's virtual filesystem for ATTACH. */

package/dist/query/source.js

@@ -9,7 +9,7 @@
  * stay free of ad-hoc branching.
  */
 import { buildDuckDbSource } from '../file-icons/index.js';
-import { buildDuckDbUrl, buildDuckDbUrlAsync } from '../utils/url.js';
+import { buildDuckDbUrl, buildDuckDbUrlAsync } from '../utils/signed-url.js';
 /**
  * True when a source ref points at a self-authenticating HTTPS URL (e.g. a
  * presigned `read_parquet('https://...?X-Amz-Signature=...')`). Used to decide

package/dist/query/stac-source-factory.d.ts

@@ -0,0 +1,65 @@
+/**
+ * Dispatch a `Tab` + classified STAC payload to the appropriate StacSource
+ * implementation. The only module allowed to import both `utils/stac-source-*`
+ * (API / static) and `query/stac-source-parquet` (DuckDB-bound) together.
+ *
+ * Keeping this in `query/` rather than `utils/` is deliberate: it imports the
+ * parquet source, which transitively pulls in DuckDB-WASM. `utils/` modules
+ * must stay DuckDB-free so the objex-utils sub-package doesn't accidentally
+ * pull DuckDB into its bundle (slice 6 promotion).
+ */
+import type { StacRoutableKind, StacSource } from '@walkthru-earth/objex-utils';
+import type { StorageAdapter } from '../storage/adapter.js';
+import type { Tab } from '../types.js';
+export interface CreateStacSourceDeps {
+    adapter: StorageAdapter;
+    urlToKey?: (absoluteUrl: string) => string | null;
+    baseHref: string;
+    /**
+     * Connection id resolved at the dispatch site. Threaded into every source
+     * so the parquet impl does not have to read `tab.connectionId` itself
+     * (asymmetric vs. api/static, would silently break for callers passing a
+     * tab whose connectionId is missing).
+     */
+    connectionId: string;
+    /**
+     * SDK opt-in: force the parquet source to treat `tab.path` as a
+     * hive-partitioned parquet directory and query it with
+     * `read_parquet('.../**\/*.parquet', hive_partitioning=true,
+     * union_by_name=true)`. Mirrors lazycogs'
+     * `DuckdbClient(use_hive_partitioning=True)`. When undefined, the factory
+     * also auto-detects directory layouts (see below).
+     */
+    useHivePartitioning?: boolean;
+    /**
+     * Debug flag. When true the parquet source runs `EXPLAIN <query>` once per
+     * `runQuery()` and logs the plan to the console so we can verify partition
+     * stats are pruning files. OFF by default — never enable in shipped UI.
+     */
+    debugExplain?: boolean;
+}
+/**
+ * True when the tab looks like a hive-partitioned parquet directory rather
+ * than a single parquet file. Conservative: requires either an explicit SDK
+ * opt-in (`deps.useHivePartitioning`) OR a `tab.path` ending in `/` AND a
+ * `.parquet`/`.geoparquet` extension. We do NOT auto-promote arbitrary
+ * extensionless paths to parquet — that would silently mis-dispatch JSON
+ * catalogs whose URL happens to have no extension. The factory defers the
+ * actual `adapter.list()` probe to the parquet source so the dispatch stays
+ * synchronous.
+ */
+export declare function looksLikeHivePartitionedParquet(tab: Tab, deps: CreateStacSourceDeps): boolean;
+/**
+ * True when a tab.path looks like a STAC API endpoint (search / items /
+ * collections URL pattern). Used to pick the API source for FeatureCollection
+ * responses served from API URLs, where the response itself has no
+ * `rel="items"` link.
+ */
+export declare function tabLooksLikeStacApi(path: string): boolean;
+/**
+ * Pick a `StacSource` for the tab. Synchronous: the orchestrator branches on
+ * `source.capabilities` immediately, before any I/O. The factory does NOT
+ * read the tab payload; it relies on the caller's `classified` argument
+ * (from `classifyStac`) plus extension and URL heuristics.
+ */
+export declare function createStacSourceForTab(tab: Tab, classified: StacRoutableKind, deps: CreateStacSourceDeps): StacSource;

package/dist/query/stac-source-factory.js

@@ -0,0 +1,77 @@
+/**
+ * Dispatch a `Tab` + classified STAC payload to the appropriate StacSource
+ * implementation. The only module allowed to import both `utils/stac-source-*`
+ * (API / static) and `query/stac-source-parquet` (DuckDB-bound) together.
+ *
+ * Keeping this in `query/` rather than `utils/` is deliberate: it imports the
+ * parquet source, which transitively pulls in DuckDB-WASM. `utils/` modules
+ * must stay DuckDB-free so the objex-utils sub-package doesn't accidentally
+ * pull DuckDB into its bundle (slice 6 promotion).
+ */
+import { createApiSource, createStaticSource, hasStacItemsEndpoint, STAC_API_PATH_RE } from '@walkthru-earth/objex-utils';
+import { createParquetSource } from './stac-source-parquet.js';
+/**
+ * True when the tab looks like a hive-partitioned parquet directory rather
+ * than a single parquet file. Conservative: requires either an explicit SDK
+ * opt-in (`deps.useHivePartitioning`) OR a `tab.path` ending in `/` AND a
+ * `.parquet`/`.geoparquet` extension. We do NOT auto-promote arbitrary
+ * extensionless paths to parquet — that would silently mis-dispatch JSON
+ * catalogs whose URL happens to have no extension. The factory defers the
+ * actual `adapter.list()` probe to the parquet source so the dispatch stays
+ * synchronous.
+ */
+export function looksLikeHivePartitionedParquet(tab, deps) {
+    if (deps.useHivePartitioning === true)
+        return true;
+    const path = tab.path ?? '';
+    const ext = (tab.extension ?? '').toLowerCase();
+    if (path.endsWith('/') && (ext === 'parquet' || ext === 'geoparquet'))
+        return true;
+    return false;
+}
+/**
+ * True when a tab.path looks like a STAC API endpoint (search / items /
+ * collections URL pattern). Used to pick the API source for FeatureCollection
+ * responses served from API URLs, where the response itself has no
+ * `rel="items"` link.
+ */
+export function tabLooksLikeStacApi(path) {
+    try {
+        return STAC_API_PATH_RE.test(new URL(path).pathname);
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Pick a `StacSource` for the tab. Synchronous: the orchestrator branches on
+ * `source.capabilities` immediately, before any I/O. The factory does NOT
+ * read the tab payload; it relies on the caller's `classified` argument
+ * (from `classifyStac`) plus extension and URL heuristics.
+ */
+export function createStacSourceForTab(tab, classified, deps) {
+    const ext = (tab.extension ?? '').toLowerCase();
+    const hive = looksLikeHivePartitionedParquet(tab, deps);
+    if (ext === 'parquet' || ext === 'geoparquet' || hive) {
+        return createParquetSource(tab, deps.connectionId, {
+            adapter: deps.adapter,
+            useHivePartitioning: hive,
+            debugExplain: deps.debugExplain
+        });
+    }
+    if (classified.kind === 'collection' || classified.kind === 'catalog') {
+        if (hasStacItemsEndpoint(classified.payload)) {
+            return createApiSource(classified, deps);
+        }
+        return createStaticSource(classified, deps);
+    }
+    if (classified.kind === 'item-collection') {
+        if (tabLooksLikeStacApi(tab.path))
+            return createApiSource(classified, deps);
+        return createStaticSource(classified, deps);
+    }
+    if (classified.kind === 'item') {
+        return createStaticSource(classified, deps);
+    }
+    throw new Error(`Unsupported STAC kind: ${classified.kind}`);
+}

package/dist/query/stac-source-parquet.d.ts

@@ -0,0 +1,135 @@
+/**
+ * stac-geoparquet implementation of the StacSource contract.
+ *
+ * Reuses:
+ *   - `getQueryEngine()` + `queryCancellable`/`query` for the single worker
+ *   - `resolveTableSourceAsync(tab)` for presigned `signed-s3` URL handling
+ *   - `stacRowToItem` from `@walkthru-earth/objex-utils` for the pure transform
+ *   - `parseWKB` from `@walkthru-earth/objex-utils` for geometry decoding
+ *
+ * Push-down: `bbox` (`ST_Intersects` + `ST_MakeEnvelope`) and `datetime`
+ * (`datetime BETWEEN TIMESTAMPTZ ...`). Without the datetime push-down,
+ * `LIMIT + ORDER BY datetime DESC` silently drops older rows before the
+ * client-side filter ever runs, so any window outside the freshest N items
+ * returned zero matches. Cloud cover / GSD / platform / etc. still ride
+ * along on the residual until slice 3 plumbs them through DuckDB SQL.
+ *
+ * Hive partitioning: when the factory (or an SDK caller) sets
+ * `useHivePartitioning: true`, the FROM target switches to
+ * `read_parquet('.../**\/*.parquet', hive_partitioning=true,
+ * union_by_name=true)`. Mirrors lazycogs'
+ * `DuckdbClient(use_hive_partitioning=True)`. Partition columns appear as
+ * virtual columns on the schema, but `buildSelectList` only projects known
+ * STAC columns so they never leak into the rendered Items. `union_by_name`
+ * is required because partitioned writes can drift schemas across
+ * partitions (extra `proj:*` columns added later, etc.).
+ *
+ * Yields a single batch with `done: true`. Slice 3 turns this into a real
+ * stream via `conn.send()` so large catalogs can render progressively.
+ */
+import { type StacSource } from '@walkthru-earth/objex-utils';
+import type { StorageAdapter } from '../storage/adapter.js';
+import type { Tab } from '../types.js';
+/**
+ * Options for `createParquetSource`. The factory threads adapter +
+ * `useHivePartitioning` + `debugExplain` from `CreateStacSourceDeps`, but
+ * library consumers can construct a `StacSource` directly.
+ */
+export interface CreateParquetSourceOptions {
+    /**
+     * Storage adapter for the connection backing `tab`. Used solely to probe
+     * `tab.path` for `.parquet` children when `useHivePartitioning` is set —
+     * never consulted on the per-row read path (DuckDB httpfs handles I/O
+     * directly via the presigned / signed URL).
+     */
+    adapter?: StorageAdapter;
+    /**
+     * When true, treat `tab.path` as a hive-partitioned parquet directory
+     * (e.g. `s3://bucket/year=2023/month=01/...`) and build SQL with
+     * `read_parquet('.../**\/*.parquet', hive_partitioning=true,
+     * union_by_name=true)` so DuckDB prunes partitions per `bbox` /
+     * `datetime` predicate. Mirrors lazycogs'
+     * `DuckdbClient(use_hive_partitioning=True)`.
+     */
+    useHivePartitioning?: boolean;
+    /**
+     * When true, run `EXPLAIN <query>` once per `runQuery()` and log the plan
+     * to the console. Used to verify partition pruning hits parquet stats.
+     * Off by default — never enable in shipped UI.
+     */
+    debugExplain?: boolean;
+    /**
+     * Mobile / low-memory mode. When true the source caps the effective
+     * LIMIT (regardless of the caller's request) and skips
+     * `ORDER BY datetime DESC` so DuckDB can stop reading after the first
+     * N parquet rows instead of fully materializing every row's STRUCT
+     * `assets` column to compute a Top-N. The trade-off: items are
+     * returned in file order, not freshness order. Defaults to a
+     * mobile-UA detection at module load when undefined.
+     */
+    lowMemoryMode?: boolean;
+    /** Hard cap on `req.limit` when `lowMemoryMode` is on. Default 200. */
+    lowMemoryLimit?: number;
+}
+export interface QueryStacGeoparquetOptions {
+    signal?: AbortSignal;
+    /** Hard cap on rows. Matches `hydrateStacItems` default. */
+    limit?: number;
+    /**
+     * When true, omit `ORDER BY datetime DESC` from the SQL. Sorting forces
+     * DuckDB's Top-N operator to read every row's heavy STRUCT `assets`
+     * column before the LIMIT engages, which OOMs the WASM heap on mobile
+     * Safari (~1.8 GiB cap, no OPFS spill). Without the sort the parquet
+     * scan can stop after the first N rows in file order. The trade-off:
+     * the user sees the first N items by file order, not the freshest N.
+     */
+    skipOrderBy?: boolean;
+    /**
+     * Hive-partitioning context resolved at source construction. When
+     * `enabled === true` the runtime FROM target is a `read_parquet` glob
+     * over `tab.path` and `union_by_name=true` is applied. The probe lives
+     * on the source (`createParquetSource` awaits it once before the first
+     * runQuery), not in `runQuery` itself, so the per-pan path stays cheap.
+     */
+    hive?: {
+        enabled: boolean;
+    };
+    /**
+     * Debug flag, propagated from source options. See
+     * `CreateParquetSourceOptions.debugExplain`.
+     */
+    debugExplain?: boolean;
+    /**
+     * Optional WGS84 viewport bbox `[west, south, east, north]`. When set the
+     * query is filtered with `ST_Intersects(geometry, ST_MakeEnvelope(...))` so
+     * the parquet path mirrors the API's viewport-scoped behavior. Skipped when
+     * the geometry column is missing (already guarded above).
+     */
+    bbox?: [number, number, number, number];
+    /**
+     * Optional ISO 8601 datetime range. When set and the parquet has a
+     * `datetime` column, the predicate is pushed into the WHERE clause as
+     * `datetime BETWEEN TIMESTAMPTZ ... AND TIMESTAMPTZ ...`. Without this
+     * push-down the LIMIT + `ORDER BY datetime DESC` would drop older rows
+     * before the client-side filter ever ran, so picking an older window
+     * returned zero items.
+     */
+    datetime?: {
+        min?: string;
+        max?: string;
+    };
+}
+/**
+ * stac-geoparquet `StacSource`. Slice 1: bbox is the only push-down,
+ * single yield with `done: true`. Slice 3 widens push-down (cloud cover /
+ * gsd / platform via DuckDB SQL) and turns this into a streaming
+ * `conn.send()` cursor.
+ *
+ * `options.useHivePartitioning` switches the FROM target to a recursive
+ * `read_parquet` glob over `tab.path` so DuckDB prunes partitions per
+ * `bbox` / `datetime` predicate. The first `query()` call awaits a
+ * best-effort `adapter.list()` probe to confirm at least one `.parquet`
+ * child exists; if listing fails (e.g. UrlAdapter, AccessDenied) we still
+ * attempt the hive query and let DuckDB surface the real error.
+ */
+export declare function createParquetSource(tab: Tab, connectionId: string, options?: CreateParquetSourceOptions): StacSource;