@walkthru-earth/objex-utils 0.1.0

package/dist/index.d.cts

@@ -0,0 +1,433 @@
+import { Table } from 'apache-arrow';
+
+/**
+ * Centralized file type detection module.
+ *
+ * Single source of truth for extension → icon, color, label, category,
+ * viewer, DuckDB read function, MIME type, and queryability.
+ *
+ * Used by: FileRow, FileTreeSidebar, ViewerRouter, TableViewer, WasmQueryEngine, etc.
+ */
+type FileCategory = 'data' | 'geo' | 'code' | 'document' | 'config' | 'image' | 'video' | 'audio' | 'archive' | 'database' | '3d' | 'other';
+type ViewerKind = 'table' | 'image' | 'video' | 'audio' | 'markdown' | 'code' | 'cog' | 'pmtiles' | 'flatgeobuf' | 'pdf' | '3d' | 'archive' | 'database' | 'zarr' | 'copc' | 'notebook' | 'raw';
+type DuckDbReadFn = 'read_parquet' | 'read_csv' | 'read_json' | 'ST_Read';
+interface FileTypeInfo {
+    /** Lucide icon name */
+    icon: string;
+    /** Tailwind color classes (light + dark) */
+    color: string;
+    /** Human-readable type label, e.g. "Apache Parquet" */
+    label: string;
+    /** High-level category */
+    category: FileCategory;
+    /** Which viewer component to use */
+    viewer: ViewerKind;
+    /** Whether this file can be queried with DuckDB */
+    queryable: boolean;
+    /** DuckDB read function (null if not queryable) */
+    duckdbReadFn: DuckDbReadFn | null;
+    /** MIME type */
+    mimeType: string;
+}
+/**
+ * Returns full file type information for a given extension.
+ * Extension may be with or without leading dot: ".parquet" or "parquet".
+ */
+declare function getFileTypeInfo(extension: string, isDir?: boolean): FileTypeInfo;
+/** Returns the DuckDB read function name for a file path. Falls back to read_parquet. */
+declare function getDuckDbReadFn(pathOrExt: string): string;
+/**
+ * Returns a DuckDB FROM-clause expression for a file.
+ *
+ * For GeoJSON files the raw `read_json_auto` returns a single row with the
+ * FeatureCollection structure (`type`, `features`, `bbox`). This helper
+ * unnests the features array so each feature becomes its own row with
+ * property columns + a geometry column.
+ */
+declare function buildDuckDbSource(pathOrExt: string, url: string): string;
+declare function isCloudNativeFormat(pathOrExt: string): boolean;
+/** Returns the viewer kind for a given extension. */
+declare function getViewerKind(extension: string): ViewerKind;
+/** Returns whether a file can be queried with DuckDB. */
+declare function isQueryable(extension: string): boolean;
+/** Returns the MIME type for a file extension. */
+declare function getMimeType(extension: string): string;
+
+declare class QueryCancelledError extends Error {
+    constructor();
+}
+interface QueryHandle {
+    result: Promise<QueryResult>;
+    cancel: () => Promise<boolean>;
+}
+interface MapQueryHandle {
+    result: Promise<MapQueryResult>;
+    cancel: () => Promise<boolean>;
+}
+interface QueryResult {
+    columns: string[];
+    types: string[];
+    rowCount: number;
+    /** Pre-parsed rows — avoids Arrow version mismatch in WASM engine */
+    rows: Record<string, any>[];
+}
+/** Raw column data for map rendering (bypasses toJSON serialization). */
+interface MapQueryResult {
+    /** Geometry column as raw WKB binary arrays */
+    wkbArrays: Uint8Array[];
+    /** Geometry type from ST_GeometryType (e.g., 'POLYGON') */
+    geometryType: string;
+    /** Non-geometry attribute columns: name → JS values */
+    attributes: Map<string, {
+        values: any[];
+        type: string;
+    }>;
+    /** Number of rows */
+    rowCount: number;
+}
+interface SchemaField {
+    name: string;
+    type: string;
+    nullable: boolean;
+}
+interface QueryEngine {
+    query(connId: string, sql: string): Promise<QueryResult>;
+    queryForMap(connId: string, sql: string, geomCol: string, geomColType: string, sourceCrs?: string | null): Promise<MapQueryResult>;
+    getSchema(connId: string, path: string): Promise<SchemaField[]>;
+    getRowCount(connId: string, path: string): Promise<number>;
+    /** Detect CRS from GeoParquet metadata. Returns e.g. 'EPSG:27700' or null if WGS84/unknown. */
+    detectCrs(connId: string, path: string, geomCol: string): Promise<string | null>;
+    /** Combined schema + CRS detection in a single connection (fewer web worker round-trips). */
+    getSchemaAndCrs?(connId: string, path: string, findGeoCol: (schema: SchemaField[]) => string | null): Promise<{
+        schema: SchemaField[];
+        geomCol: string | null;
+        crs: string | null;
+    }>;
+    queryCancellable?(connId: string, sql: string): QueryHandle;
+    queryForMapCancellable?(connId: string, sql: string, geomCol: string, geomColType: string, sourceCrs?: string | null): MapQueryHandle;
+    forceCancel?(): Promise<void>;
+    releaseMemory(): Promise<void>;
+    dispose(): Promise<void>;
+}
+
+interface FileEntry {
+    name: string;
+    path: string;
+    is_dir: boolean;
+    size: number;
+    modified: number;
+    extension: string;
+}
+interface Connection {
+    id: string;
+    name: string;
+    provider: 's3' | 'gcs' | 'r2' | 'minio' | 'azure' | 'storj';
+    endpoint: string;
+    bucket: string;
+    region: string;
+    anonymous: boolean;
+    authMethod?: 'sigv4' | 'sas-token';
+    rootPrefix?: string;
+}
+interface ConnectionConfig {
+    name: string;
+    provider: 's3' | 'gcs' | 'r2' | 'minio' | 'azure' | 'storj';
+    endpoint: string;
+    bucket: string;
+    region: string;
+    access_key?: string;
+    secret_key?: string;
+    sas_token?: string;
+    anonymous: boolean;
+    authMethod?: 'sigv4' | 'sas-token';
+    rootPrefix?: string;
+}
+interface Tab {
+    id: string;
+    name: string;
+    path: string;
+    source: 'remote' | 'url';
+    connectionId?: string;
+    extension: string;
+    size?: number;
+}
+interface WriteResult {
+    key: string;
+    size: number;
+    e_tag?: string;
+}
+type Theme = 'light' | 'dark' | 'system';
+
+/** A single page of listing results with optional continuation. */
+interface ListPage {
+    entries: FileEntry[];
+    continuationToken?: string;
+    hasMore: boolean;
+}
+interface StorageAdapter {
+    list(path: string, signal?: AbortSignal): Promise<FileEntry[]>;
+    read(path: string, offset?: number, length?: number, signal?: AbortSignal): Promise<Uint8Array>;
+    head(path: string, signal?: AbortSignal): Promise<FileEntry>;
+    /** Fetch a single page of listing results. Supports progressive rendering. */
+    listPage?(path: string, continuationToken?: string, pageSize?: number, signal?: AbortSignal): Promise<ListPage>;
+    put(key: string, data: Uint8Array, contentType?: string): Promise<WriteResult>;
+    delete(key: string): Promise<void>;
+    deletePrefix(prefix: string): Promise<{
+        deleted: number;
+    }>;
+    copy(srcKey: string, destKey: string): Promise<WriteResult>;
+    readonly supportsWrite: boolean;
+}
+
+/**
+ * Minimal adapter for direct HTTPS URLs (source: 'url').
+ * Only supports read/head — no listing or writing.
+ * The `path` parameter is the full HTTPS URL.
+ */
+declare class UrlAdapter implements StorageAdapter {
+    readonly supportsWrite = false;
+    read(url: string, offset?: number, length?: number, signal?: AbortSignal): Promise<Uint8Array>;
+    head(url: string, signal?: AbortSignal): Promise<FileEntry>;
+    list(): Promise<FileEntry[]>;
+    put(): Promise<WriteResult>;
+    delete(): Promise<void>;
+    deletePrefix(): Promise<{
+        deleted: number;
+    }>;
+    copy(): Promise<WriteResult>;
+}
+
+type TypeCategory = 'number' | 'string' | 'date' | 'boolean' | 'geo' | 'binary' | 'json' | 'other';
+declare function classifyType(duckdbType: string): TypeCategory;
+declare function typeColor(category: TypeCategory): string;
+declare function typeBadgeClass(category: TypeCategory): string;
+declare function typeLabel(category: TypeCategory): string;
+
+/**
+ * Formats a byte count into a human-readable string.
+ */
+declare function formatFileSize(bytes: number): string;
+/**
+ * Formats a unix timestamp (milliseconds) to a human-readable date string.
+ * Shows relative time for recent dates, absolute for older ones.
+ */
+declare function formatDate(timestamp: number): string;
+/**
+ * Extracts the file extension from a filename, including the leading dot.
+ * Returns an empty string if no extension is found.
+ */
+declare function getFileExtension(filename: string): string;
+
+/**
+ * Direct WKB → GeoArrow bridge (zero-copy).
+ *
+ * Reads raw WKB binary directly into pre-allocated Arrow typed arrays
+ * without any intermediate JS object allocation. No parseWKB(), no GeoJSON.
+ *
+ * Data flow: WKB Uint8Array[] → DataView binary reads → Float64Array/Int32Array
+ *            → arrow.makeData() → arrow.Table with ARROW:extension:name metadata
+ */
+
+type GeoArrowGeomType = 'point' | 'linestring' | 'polygon' | 'multipoint' | 'multilinestring' | 'multipolygon';
+interface GeoArrowResult {
+    table: Table;
+    geometryType: GeoArrowGeomType;
+    bounds: [number, number, number, number];
+    /** Maps local table row index → original WKB array index (for selection lookup). */
+    sourceIndices: number[];
+}
+/** Map DuckDB ST_GeometryType output to our normalized type. */
+declare function normalizeGeomType(raw: string): GeoArrowGeomType;
+/**
+ * Build GeoArrow tables from raw WKB arrays, automatically splitting by geometry type.
+ * Returns one GeoArrowResult per non-empty type group, with shared merged bounds.
+ *
+ * @param wkbArrays Raw WKB binary arrays from DuckDB
+ * @param attributes Attribute columns (non-geometry)
+ * @param knownGeomType If provided (e.g. from GeoParquet metadata), skip classification
+ */
+declare function buildGeoArrowTables(wkbArrays: Uint8Array[], attributes: Map<string, {
+    values: any[];
+    type: string;
+}>, knownGeomType?: GeoArrowGeomType): GeoArrowResult[];
+
+interface HexRow {
+    offset: string;
+    hex: string[];
+    ascii: string;
+}
+/**
+ * Generates a hex dump from binary data.
+ * Returns rows of offset | hex bytes | ASCII representation.
+ */
+declare function generateHexDump(data: Uint8Array, bytesPerRow?: number): HexRow[];
+
+/**
+ * Lightweight Parquet metadata reader using hyparquet.
+ *
+ * Reads schema, row count, CRS, and geometry types from the Parquet footer
+ * via a single HTTP range request (~512KB). No DuckDB boot needed.
+ *
+ * This provides instant metadata display before DuckDB-WASM finishes loading.
+ */
+
+interface GeoColumnMeta {
+    encoding: string;
+    geometryTypes: string[];
+    crs: any | null;
+    bbox?: number[];
+}
+interface GeoParquetMeta {
+    primaryColumn: string;
+    columns: Record<string, GeoColumnMeta>;
+}
+interface ParquetFileMetadata {
+    /** Total number of rows across all row groups. */
+    rowCount: number;
+    /** Column schema (name + type). */
+    schema: {
+        name: string;
+        type: string;
+    }[];
+    /** GeoParquet "geo" metadata if present. */
+    geo: GeoParquetMeta | null;
+    /** True when file has legacy GeoParquet metadata (schema_version 0.x without "version" field). */
+    legacyGeoParquet: boolean;
+    /** Tool that created the file (e.g. "pyarrow 15.0.0"). */
+    createdBy: string | null;
+    /** Number of row groups. */
+    numRowGroups: number;
+    /** Primary compression codec (e.g. "SNAPPY", "ZSTD"). */
+    compression: string | null;
+}
+/**
+ * Read Parquet footer metadata from a remote URL using hyparquet.
+ * Uses HTTP range requests — typically a single ~512KB fetch.
+ */
+declare function readParquetMetadata(url: string): Promise<ParquetFileMetadata>;
+/**
+ * Extract EPSG code from GeoParquet CRS metadata.
+ * Returns null for WGS84/CRS84 (no reprojection needed).
+ */
+declare function extractEpsgFromGeoMeta(geo: GeoParquetMeta): string | null;
+/**
+ * Extract normalized geometry types from GeoParquet metadata.
+ * Returns the set of geometry types for the primary column.
+ */
+declare function extractGeometryTypes(geo: GeoParquetMeta): GeoArrowGeomType[];
+/**
+ * Extract bounds from GeoParquet metadata (bbox field).
+ * Returns [minX, minY, maxX, maxY] or null.
+ */
+declare function extractBounds(geo: GeoParquetMeta): [number, number, number, number] | null;
+
+/**
+ * Universal cloud storage URL / bucket parser.
+ *
+ * Accepts the many URI/URL formats that users commonly paste and extracts
+ * the correct bucket, region, endpoint, and provider.
+ *
+ * Supported URI schemes:
+ *   s3://   s3a://   s3n://   aws://   — Amazon S3 / S3-compatible
+ *   r2://                              — Cloudflare R2
+ *   gs://   gcs://                     — Google Cloud Storage
+ *   azure://  az://                    — Azure Blob Storage
+ *   abfs://   abfss://                — Azure Data Lake (ADLS Gen2)
+ *   wasbs://                           — Azure Blob (Hadoop WASB driver)
+ *   swift://                           — OpenStack Swift
+ *   file://  filesystem://             — Local filesystem
+ *
+ * Supported HTTPS URL patterns:
+ *   https://<bucket>.s3.<region>.amazonaws.com[/prefix]     — AWS virtual-hosted
+ *   https://s3.<region>.amazonaws.com/<bucket>[/prefix]     — AWS path-style
+ *   https://s3.amazonaws.com/<bucket>                       — AWS global
+ *   https://<account>.r2.cloudflarestorage.com/<bucket>     — Cloudflare R2
+ *   https://storage.googleapis.com/<bucket>                 — Google Cloud Storage
+ *   https://<bucket>.storage.googleapis.com[/prefix]        — GCS virtual-hosted
+ *   https://<bucket>.<region>.digitaloceanspaces.com        — DigitalOcean Spaces
+ *   https://<region>.digitaloceanspaces.com/<bucket>        — DO Spaces path-style
+ *   https://s3.<region>.wasabisys.com/<bucket>              — Wasabi
+ *   https://f<id>.backblazeb2.com/file/<bucket>             — Backblaze B2
+ *   https://<bucket>.s3.<region>.backblazeb2.com            — B2 S3-compatible
+ *   https://<bucket>.oss-<region>.aliyuncs.com              — Alibaba Cloud OSS
+ *   https://<bucket>.cos.<region>.myqcloud.com              — Tencent COS
+ *   https://storage.yandexcloud.net/<bucket>                — Yandex Cloud
+ *   https://gateway.storjshare.io/<bucket>                   — Storj S3 gateway
+ *   https://link.storjshare.io/raw/<access>/<bucket>        — Storj linksharing
+ *   https://<custom-endpoint>/<bucket>                      — Generic S3-compatible
+ *
+ * Also handles plain bucket names (no protocol).
+ */
+type StorageProvider = 's3' | 'gcs' | 'r2' | 'minio' | 'azure' | 'storj' | 'unknown';
+interface ParsedStorageUrl {
+    bucket: string;
+    region: string;
+    endpoint: string;
+    provider: StorageProvider;
+    /** Original prefix/path after bucket, if any */
+    prefix: string;
+}
+interface Defaults {
+    region?: string;
+    endpoint?: string;
+    provider?: StorageProvider;
+}
+/**
+ * Parse a user-provided bucket/URL string into structured storage connection parts.
+ */
+declare function parseStorageUrl(input: string, defaults?: Defaults): ParsedStorageUrl;
+/**
+ * Returns true if the input looks like a URL/URI rather than a plain bucket name.
+ * Covers all recognized cloud storage URI schemes.
+ */
+declare function looksLikeUrl(input: string): boolean;
+/**
+ * Given a parsed URL result, build a human-readable summary of what was detected.
+ */
+declare function describeParseResult(parsed: ParsedStorageUrl): string;
+
+/**
+ * Lightweight WKB (Well-Known Binary) parser for extracting coordinates.
+ * Supports Point, LineString, Polygon, and Multi* variants.
+ * Handles standard WKB, ISO WKB (with Z/M), and EWKB (PostGIS SRID).
+ */
+type GeoType = 'Point' | 'LineString' | 'Polygon' | 'MultiPoint' | 'MultiLineString' | 'MultiPolygon' | 'Unknown';
+interface ParsedGeometry {
+    type: GeoType;
+    coordinates: number[] | number[][] | number[][][] | number[][][][];
+}
+/**
+ * Normalize binary data from various formats to Uint8Array.
+ * Handles Uint8Array, ArrayBuffer, number arrays, hex strings,
+ * and DuckDB toJSON() blob objects ({0: byte, 1: byte, ...}).
+ */
+declare function toBinary(value: unknown): Uint8Array | null;
+/**
+ * Parse a WKB binary blob into coordinates and geometry type.
+ */
+declare function parseWKB(data: Uint8Array): ParsedGeometry | null;
+/**
+ * Find the geometry column name from a schema.
+ *
+ * Detection priority:
+ * 1. Column type contains a geometry keyword (GEOMETRY, POINT, WKB, etc.)
+ * 2. Exact well-known column name with BLOB/BINARY type
+ * 3. Exact well-known column name regardless of type
+ * 4. Column name contains a geo-related substring with BLOB/BINARY type
+ * 5. Column name contains a geo-related substring regardless of type
+ */
+declare function findGeoColumn(schema: {
+    name: string;
+    type: string;
+}[]): string | null;
+/**
+ * Fallback: probe actual row data to find a column containing WKB geometry.
+ * Use this when schema-only detection via `findGeoColumn()` returns null.
+ * Samples the first row and checks each BLOB/binary column for WKB magic bytes.
+ */
+declare function findGeoColumnFromRows(rows: Record<string, unknown>[], schema: {
+    name: string;
+    type: string;
+}[]): string | null;
+
+export { type Connection, type ConnectionConfig, type Defaults, type DuckDbReadFn, type FileCategory, type FileEntry, type FileTypeInfo, type GeoArrowGeomType, type GeoArrowResult, type GeoColumnMeta, type GeoParquetMeta, type GeoType, type HexRow, type ListPage, type MapQueryHandle, type MapQueryResult, type ParquetFileMetadata, type ParsedGeometry, type ParsedStorageUrl, QueryCancelledError, type QueryEngine, type QueryHandle, type QueryResult, type SchemaField, type StorageAdapter, type StorageProvider, type Tab, type Theme, type TypeCategory, UrlAdapter, type ViewerKind, type WriteResult, buildDuckDbSource, buildGeoArrowTables, classifyType, describeParseResult, extractBounds, extractEpsgFromGeoMeta, extractGeometryTypes, findGeoColumn, findGeoColumnFromRows, formatDate, formatFileSize, generateHexDump, getDuckDbReadFn, getFileExtension, getFileTypeInfo, getMimeType, getViewerKind, isCloudNativeFormat, isQueryable, looksLikeUrl, normalizeGeomType, parseStorageUrl, parseWKB, readParquetMetadata, toBinary, typeBadgeClass, typeColor, typeLabel };