@loaders.gl/core 5.0.0-alpha.11 → 5.0.0-alpha.13

package/dist/loader-utils/lib/scan-utils/table-scan-batch.d.ts

@@ -0,0 +1,30 @@
+import type { ArrowTable, ArrowTableBatch, TableBatch } from '@loaders.gl/core/schema';
+import type { TableScanReadOptions } from "./scan-query-metadata.js";
+/** Reads source batches while allowing the shared executor to observe fetched bytes. */
+export type TableScanBatchReader = (signal?: AbortSignal, onByteLength?: (byteLength: number) => void) => AsyncIterable<TableBatch>;
+/** Optional batch kernels for formats with a native vectorized predicate implementation. */
+export type TableScanBatchOperators = Readonly<{
+    /** Applies a residual predicate while preserving the source batch shape. */
+    filter?: (batch: TableBatch, predicate: TableScanReadOptions['predicate']) => TableBatch;
+    /** Applies a projection while preserving the source batch shape. */
+    project?: (batch: TableBatch, columns?: readonly string[]) => TableBatch;
+}>;
+/** Wraps an Arrow table in the standard single-result scan batch envelope. */
+export declare function makeTableScanBatch(table: ArrowTable): ArrowTableBatch;
+/**
+ * Executes the common residual table-scan stages for row and columnar batches.
+ *
+ * Format adapters provide only their parser-specific batch reader. Keeping filtering, projection,
+ * limits, cancellation, and telemetry here avoids copying the same executor into every format
+ * package while preserving each format's native batch shape.
+ */
+export declare function executeTableScanBatches(readBatches: TableScanBatchReader, options?: TableScanReadOptions, operators?: TableScanBatchOperators): AsyncIterable<TableBatch>;
+/** Truncates a batch without changing its native representation. */
+export declare function truncateTableBatch(batch: TableBatch, length: number): TableBatch;
+/** Projects a batch in the caller-requested order while preserving its shape. */
+export declare function projectTableBatch(batch: TableBatch, columns?: readonly string[]): TableBatch;
+/** Retains portable schema fields in the caller-requested projection order. */
+export declare function projectTableSchema(batch: TableBatch, columns: readonly string[]): TableBatch['schema'];
+/** Applies a portable residual predicate to object-row batches. */
+export declare function filterTableBatch(batch: TableBatch, predicate: TableScanReadOptions['predicate']): TableBatch;
+//# sourceMappingURL=table-scan-batch.d.ts.map

package/dist/loader-utils/lib/scan-utils/table-scan-batch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"table-scan-batch.d.ts","sourceRoot":"","sources":["../../../../src/loader-utils/lib/scan-utils/table-scan-batch.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAE,eAAe,EAAE,UAAU,EAAC,MAAM,yBAAyB,CAAC;AACrF,OAAO,KAAK,EAAC,oBAAoB,EAAyB,iCAA8B;AAKxF,wFAAwF;AACxF,MAAM,MAAM,oBAAoB,GAAG,CACjC,MAAM,CAAC,EAAE,WAAW,EACpB,YAAY,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,IAAI,KACxC,aAAa,CAAC,UAAU,CAAC,CAAC;AAE/B,4FAA4F;AAC5F,MAAM,MAAM,uBAAuB,GAAG,QAAQ,CAAC;IAC7C,4EAA4E;IAC5E,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,oBAAoB,CAAC,WAAW,CAAC,KAAK,UAAU,CAAC;IACzF,oEAAoE;IACpE,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,KAAK,UAAU,CAAC;CAC1E,CAAC,CAAC;AAEH,8EAA8E;AAC9E,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,UAAU,GAAG,eAAe,CAErE;AAED;;;;;;GAMG;AACH,wBAAuB,uBAAuB,CAC5C,WAAW,EAAE,oBAAoB,EACjC,OAAO,GAAE,oBAAyB,EAClC,SAAS,GAAE,uBAA4B,GACtC,aAAa,CAAC,UAAU,CAAC,CAmF3B;AAED,oEAAoE;AACpE,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,UAAU,CAkBhF;AAED,iFAAiF;AACjF,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,UAAU,CAoB5F;AAED,+EAA+E;AAC/E,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE,SAAS,MAAM,EAAE,GACzB,UAAU,CAAC,QAAQ,CAAC,CAUtB;AAED,mEAAmE;AACnE,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,UAAU,EACjB,SAAS,EAAE,oBAAoB,CAAC,WAAW,CAAC,GAC3C,UAAU,CAUZ"}

package/dist/loader-utils/lib/scan-utils/table-scan-batch.js

@@ -0,0 +1,161 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { emitScanExecutionTelemetry } from "./scan-query-metadata.js";
+import { filterColumnarRowIndices, validateColumnarPredicate } from "./columnar-predicate.js";
+import { validateTableQueryLimit } from "./table-query.js";
+/** Wraps an Arrow table in the standard single-result scan batch envelope. */
+export function makeTableScanBatch(table) {
+    return { ...table, batchType: 'data', length: table.data.numRows };
+}
+/**
+ * Executes the common residual table-scan stages for row and columnar batches.
+ *
+ * Format adapters provide only their parser-specific batch reader. Keeping filtering, projection,
+ * limits, cancellation, and telemetry here avoids copying the same executor into every format
+ * package while preserving each format's native batch shape.
+ */
+export async function* executeTableScanBatches(readBatches, options = {}, operators = {}) {
+    validateTableQueryLimit(options.limit);
+    const startedAt = Date.now();
+    let sourcesRead = 0;
+    let batchesRead = 0;
+    let rowsRead = 0;
+    let rowsTested = 0;
+    let rowsRetained = 0;
+    let rowsReturned = 0;
+    let bytesFetched = 0;
+    let status = 'early-terminated';
+    let earlyTerminationReason;
+    let executionError;
+    let remaining = options.limit === undefined ? Number.POSITIVE_INFINITY : Math.max(0, options.limit);
+    try {
+        if (remaining <= 0) {
+            earlyTerminationReason = 'limit';
+            return;
+        }
+        sourcesRead = 1;
+        for await (const batch of readBatches(options.signal, byteLength => (bytesFetched += byteLength))) {
+            if (remaining <= 0) {
+                earlyTerminationReason = 'limit';
+                return;
+            }
+            batchesRead++;
+            rowsRead += batch.length;
+            const filteredBatch = (operators.filter || filterTableBatch)(batch, options.predicate);
+            if (options.predicate) {
+                rowsTested += batch.length;
+                rowsRetained += filteredBatch.length;
+            }
+            const projectedBatch = (operators.project || projectTableBatch)(filteredBatch, options.columns);
+            if (projectedBatch.length <= remaining) {
+                remaining -= projectedBatch.length;
+                rowsReturned += projectedBatch.length;
+                yield projectedBatch;
+            }
+            else {
+                const outputBatch = truncateTableBatch(projectedBatch, remaining);
+                rowsReturned += outputBatch.length;
+                earlyTerminationReason = 'limit';
+                yield outputBatch;
+                return;
+            }
+        }
+        status = 'completed';
+    }
+    catch (error) {
+        status = options.signal?.aborted ? 'cancelled' : 'failed';
+        executionError = error;
+        throw error;
+    }
+    finally {
+        if (status === 'early-terminated' && !earlyTerminationReason) {
+            earlyTerminationReason = 'consumer-return';
+        }
+        emitScanExecutionTelemetry(options.onTelemetry, Object.freeze({
+            status,
+            sourcesPlanned: 1,
+            sourcesRead,
+            batchesRead,
+            batchesDecoded: batchesRead,
+            rowsRead,
+            rowsTested: rowsTested || undefined,
+            rowsRetained: rowsRetained || undefined,
+            rowsReturned,
+            bytesRead: bytesFetched,
+            bytesFetched,
+            filesOpened: sourcesRead,
+            tasksOpened: sourcesRead,
+            durationMilliseconds: Date.now() - startedAt,
+            earlyTerminationReason,
+            ...(executionError === undefined ? {} : { error: executionError })
+        }));
+    }
+}
+/** Truncates a batch without changing its native representation. */
+export function truncateTableBatch(batch, length) {
+    if (batch.shape === 'object-row-table' || batch.shape === 'array-row-table') {
+        return { ...batch, data: batch.data.slice(0, length), length };
+    }
+    if (batch.shape === 'columnar-table') {
+        return {
+            ...batch,
+            data: Object.fromEntries(Object.entries(batch.data).map(([name, values]) => [
+                name,
+                Array.from(values).slice(0, length)
+            ])),
+            length
+        };
+    }
+    if (batch.shape === 'arrow-table')
+        return { ...batch, data: batch.data.slice(0, length), length };
+    return { ...batch, features: batch.features.slice(0, length), length };
+}
+/** Projects a batch in the caller-requested order while preserving its shape. */
+export function projectTableBatch(batch, columns) {
+    if (columns === undefined)
+        return batch;
+    const schema = projectTableSchema(batch, columns);
+    if (batch.shape === 'object-row-table') {
+        return {
+            ...batch,
+            schema,
+            data: batch.data.map(row => Object.fromEntries(columns.map(column => [column, row[column]])))
+        };
+    }
+    if (batch.shape === 'columnar-table') {
+        return {
+            ...batch,
+            schema,
+            data: Object.fromEntries(columns.map(column => [column, batch.data[column]]))
+        };
+    }
+    if (batch.shape === 'arrow-table')
+        return { ...batch, schema, data: batch.data.select([...columns]) };
+    return batch;
+}
+/** Retains portable schema fields in the caller-requested projection order. */
+export function projectTableSchema(batch, columns) {
+    if (!batch.schema)
+        return undefined;
+    const fieldsByName = new Map(batch.schema.fields.map(field => [field.name, field]));
+    return {
+        ...batch.schema,
+        fields: columns.flatMap(column => {
+            const field = fieldsByName.get(column);
+            return field ? [field] : [];
+        })
+    };
+}
+/** Applies a portable residual predicate to object-row batches. */
+export function filterTableBatch(batch, predicate) {
+    if (!predicate || batch.shape !== 'object-row-table')
+        return batch;
+    const rows = batch.data;
+    const columnNames = new Set(batch.schema?.fields.map(field => field.name) || []);
+    validateColumnarPredicate(predicate, columnNames);
+    const columns = Object.fromEntries([...columnNames].map(name => [name, rows.map(row => row[name])]));
+    const rowIndices = filterColumnarRowIndices(predicate, columns, rows.length);
+    return { ...batch, data: rowIndices.map(rowIndex => rows[rowIndex]), length: rowIndices.length };
+}
+//# sourceMappingURL=table-scan-batch.js.map

package/dist/loader-utils/lib/scan-utils/table-scan-batch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"table-scan-batch.js","sourceRoot":"","sources":["../../../../src/loader-utils/lib/scan-utils/table-scan-batch.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,OAAO,EAAC,0BAA0B,EAAC,iCAA8B;AACjE,OAAO,EAAC,wBAAwB,EAAE,yBAAyB,EAAC,gCAA6B;AACzF,OAAO,EAAC,uBAAuB,EAAC,yBAAsB;AAgBtD,8EAA8E;AAC9E,MAAM,UAAU,kBAAkB,CAAC,KAAiB;IAClD,OAAO,EAAC,GAAG,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,EAAoB,CAAC;AACtF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,uBAAuB,CAC5C,WAAiC,EACjC,UAAgC,EAAE,EAClC,YAAqC,EAAE;IAEvC,uBAAuB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACvC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,MAAM,GAAqC,kBAAkB,CAAC;IAClE,IAAI,sBAAwE,CAAC;IAC7E,IAAI,cAAuB,CAAC;IAC5B,IAAI,SAAS,GACX,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACtF,IAAI,CAAC;QACH,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;YACnB,sBAAsB,GAAG,OAAO,CAAC;YACjC,OAAO;QACT,CAAC;QACD,WAAW,GAAG,CAAC,CAAC;QAChB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,WAAW,CACnC,OAAO,CAAC,MAAM,EACd,UAAU,CAAC,EAAE,CAAC,CAAC,YAAY,IAAI,UAAU,CAAC,CAC3C,EAAE,CAAC;YACF,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;gBACnB,sBAAsB,GAAG,OAAO,CAAC;gBACjC,OAAO;YACT,CAAC;YACD,WAAW,EAAE,CAAC;YACd,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC;YACzB,MAAM,aAAa,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,gBAAgB,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;YACvF,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBACtB,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC;gBAC3B,YAAY,IAAI,aAAa,CAAC,MAAM,CAAC;YACvC,CAAC;YACD,MAAM,cAAc,GAAG,CAAC,SAAS,CAAC,OAAO,IAAI,iBAAiB,CAAC,CAC7D,aAAa,EACb,OAAO,CAAC,OAAO,CAChB,CAAC;YACF,IAAI,cAAc,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;gBACvC,SAAS,IAAI,cAAc,CAAC,MAAM,CAAC;gBACnC,YAAY,IAAI,cAAc,CAAC,MAAM,CAAC;gBACtC,MAAM,cAAc,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,MAAM,WAAW,GAAG,kBAAkB,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;gBAClE,YAAY,IAAI,WAAW,CAAC,MAAM,CAAC;gBACnC,sBAAsB,GAAG,OAAO,CAAC;gBACjC,MAAM,WAAW,CAAC;gBAClB,OAAO;YACT,CAAC;QACH,CAAC;QACD,MAAM,GAAG,WAAW,CAAC;IACvB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC1D,cAAc,GAAG,KAAK,CAAC;QACvB,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,IAAI,MAAM,KAAK,kBAAkB,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC7D,sBAAsB,GAAG,iBAAiB,CAAC;QAC7C,CAAC;QACD,0BAA0B,CACxB,OAAO,CAAC,WAAW,EACnB,MAAM,CAAC,MAAM,CAAC;YACZ,MAAM;YACN,cAAc,EAAE,CAAC;YACjB,WAAW;YACX,WAAW;YACX,cAAc,EAAE,WAAW;YAC3B,QAAQ;YACR,UAAU,EAAE,UAAU,IAAI,SAAS;YACnC,YAAY,EAAE,YAAY,IAAI,SAAS;YACvC,YAAY;YACZ,SAAS,EAAE,YAAY;YACvB,YAAY;YACZ,WAAW,EAAE,WAAW;YACxB,WAAW,EAAE,WAAW;YACxB,oBAAoB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAC5C,sBAAsB;YACtB,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAC,KAAK,EAAE,cAAc,EAAC,CAAC;SACjE,CAAC,CACH,CAAC;IACJ,CAAC;AACH,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,kBAAkB,CAAC,KAAiB,EAAE,MAAc;IAClE,IAAI,KAAK,CAAC,KAAK,KAAK,kBAAkB,IAAI,KAAK,CAAC,KAAK,KAAK,iBAAiB,EAAE,CAAC;QAC5E,OAAO,EAAC,GAAG,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,MAAM,EAAe,CAAC;IAC7E,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,KAAK,gBAAgB,EAAE,CAAC;QACrC,OAAO;YACL,GAAG,KAAK;YACR,IAAI,EAAE,MAAM,CAAC,WAAW,CACtB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC;gBACjD,IAAI;gBACJ,KAAK,CAAC,IAAI,CAAC,MAA4B,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC;aAC1D,CAAC,CACH;YACD,MAAM;SACP,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,KAAK,aAAa;QAAE,OAAO,EAAC,GAAG,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,MAAM,EAAC,CAAC;IAChG,OAAO,EAAC,GAAG,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,MAAM,EAAC,CAAC;AACvE,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,iBAAiB,CAAC,KAAiB,EAAE,OAA2B;IAC9E,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACxC,MAAM,MAAM,GAAG,kBAAkB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAClD,IAAI,KAAK,CAAC,KAAK,KAAK,kBAAkB,EAAE,CAAC;QACvC,OAAO;YACL,GAAG,KAAK;YACR,MAAM;YACN,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;SAChF,CAAC;IAClB,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,KAAK,gBAAgB,EAAE,CAAC;QACrC,OAAO;YACL,GAAG,KAAK;YACR,MAAM;YACN,IAAI,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;SAC9E,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,KAAK,aAAa;QAC/B,OAAO,EAAC,GAAG,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,EAAC,CAAC;IACnE,OAAO,KAAK,CAAC;AACf,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,kBAAkB,CAChC,KAAiB,EACjB,OAA0B;IAE1B,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IACpC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACpF,OAAO;QACL,GAAG,KAAK,CAAC,MAAM;QACf,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YAC/B,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvC,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9B,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,mEAAmE;AACnE,MAAM,UAAU,gBAAgB,CAC9B,KAAiB,EACjB,SAA4C;IAE5C,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,kBAAkB;QAAE,OAAO,KAAK,CAAC;IACnE,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IACxB,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACjF,yBAAyB,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAChC,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CACjE,CAAC;IACF,MAAM,UAAU,GAAG,wBAAwB,CAAC,SAAkB,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACtF,OAAO,EAAC,GAAG,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAC,CAAC;AACjG,CAAC"}

package/dist/loader-utils/lib/sources/catalog-source.d.ts

@@ -0,0 +1,36 @@
+/** Common feature declarations for discoverable resource catalogs. */
+export type CatalogSourceCapabilities = Readonly<{
+    /** Catalog records can be queried. */
+    search: boolean;
+    /** Search results can be consumed across multiple result pages. */
+    pagination: boolean;
+    /** The catalog exposes parent/child traversal. */
+    hierarchy: boolean;
+    /** Search accepts a spatial constraint. */
+    spatialFilter: boolean;
+    /** Search accepts a temporal constraint. */
+    temporalFilter: boolean;
+    /** Search accepts a free-text constraint. */
+    textFilter: boolean;
+    /** Search accepts a CQL2 filter. */
+    cql2Filter: boolean;
+    /** The catalog exposes named collections. */
+    collections: boolean;
+    /** Records can describe downloadable or streamable assets. */
+    assets: boolean;
+}>;
+/**
+ * Minimal shared interface for catalog protocols such as STAC and OGC CSW.
+ *
+ * Protocol-specific sources retain their richer methods and record models. This interface only
+ * standardizes capability discovery, catalog metadata, and asynchronous record search.
+ */
+export interface CatalogSource<RecordT, QueryT = undefined, MetadataT = unknown> {
+    /** Features implemented by this catalog source. */
+    readonly capabilities: CatalogSourceCapabilities;
+    /** Returns protocol-specific catalog metadata. */
+    getMetadata(): Promise<MetadataT>;
+    /** Searches catalog records, following protocol pagination when supported. */
+    search(query?: QueryT): AsyncIterable<RecordT>;
+}
+//# sourceMappingURL=catalog-source.d.ts.map

package/dist/loader-utils/lib/sources/catalog-source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"catalog-source.d.ts","sourceRoot":"","sources":["../../../../src/loader-utils/lib/sources/catalog-source.ts"],"names":[],"mappings":"AAIA,sEAAsE;AACtE,MAAM,MAAM,yBAAyB,GAAG,QAAQ,CAAC;IAC/C,sCAAsC;IACtC,MAAM,EAAE,OAAO,CAAC;IAChB,mEAAmE;IACnE,UAAU,EAAE,OAAO,CAAC;IACpB,kDAAkD;IAClD,SAAS,EAAE,OAAO,CAAC;IACnB,2CAA2C;IAC3C,aAAa,EAAE,OAAO,CAAC;IACvB,4CAA4C;IAC5C,cAAc,EAAE,OAAO,CAAC;IACxB,6CAA6C;IAC7C,UAAU,EAAE,OAAO,CAAC;IACpB,oCAAoC;IACpC,UAAU,EAAE,OAAO,CAAC;IACpB,6CAA6C;IAC7C,WAAW,EAAE,OAAO,CAAC;IACrB,8DAA8D;IAC9D,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC,CAAC;AAEH;;;;;GAKG;AACH,MAAM,WAAW,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,GAAG,OAAO;IAC7E,mDAAmD;IACnD,QAAQ,CAAC,YAAY,EAAE,yBAAyB,CAAC;IAEjD,kDAAkD;IAClD,WAAW,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC;IAElC,8EAA8E;IAC9E,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;CAChD"}

package/dist/loader-utils/lib/sources/catalog-source.js

@@ -0,0 +1,5 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+export {};
+//# sourceMappingURL=catalog-source.js.map

package/dist/loader-utils/lib/sources/catalog-source.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"catalog-source.js","sourceRoot":"","sources":["../../../../src/loader-utils/lib/sources/catalog-source.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC"}

package/dist/loader-utils/lib/sources/data-source-manager.d.ts

@@ -0,0 +1,136 @@
+import type { DataSource, DataSourceOptions } from "./data-source.js";
+import type { ScanQueryMetadata } from "../scan-utils/scan-query-metadata.js";
+/** DataSource instances that can be owned by a DataSourceManager. */
+export type ManageableDataSource = DataSource<unknown, DataSourceOptions> & {
+    /** Releases underlying resources, such as database connections. */
+    close?: () => Promise<void> | void;
+    /** Releases source-owned caches or child resources. */
+    finalize?: () => Promise<void> | void;
+    /** Releases source-owned resources. */
+    destroy?: () => Promise<void> | void;
+};
+/** Lifecycle state reported for one managed DataSource registration. */
+export type DataSourceManagerEntryStatus = 'ready' | 'pending' | 'placeholder' | 'error';
+/** Read-only discovery information for one managed DataSource registration. */
+export type DataSourceManagerEntryInfo = Readonly<{
+    /** Stable id used to retrieve or subscribe to the DataSource. */
+    dataSourceId: string;
+    /** Current lifecycle state without exposing the owned DataSource instance. */
+    status: DataSourceManagerEntryStatus;
+    /** Whether the manager keeps the entry after its last consumer and retain are released. */
+    persistent: boolean;
+    /** Number of consumers currently subscribed to this entry. */
+    subscriberCount: number;
+    /** Number of callers currently retaining this entry through getOrCreate(). */
+    retainCount: number;
+    /** Resolution failure when status is `error`. */
+    error?: Error;
+}>;
+/** Filters and cancellation accepted by read-only DataSource discovery. */
+export type DataSourceManagerDiscoveryOptions = Readonly<{
+    /** Limits compatible results to one common scan family. */
+    queryType?: ScanQueryMetadata['queryType'];
+    /** Aborts metadata discovery without modifying manager registrations. */
+    signal?: AbortSignal;
+}>;
+/** Picker-ready metadata for one managed registration. */
+export type DataSourceManagerDiscoveryInfo = DataSourceManagerEntryInfo & Readonly<{
+    /** Whether the ready source implements the requested executable scan contract. */
+    compatible: boolean;
+    /** Common scan metadata returned by a compatible or metadata-only source. */
+    queryMetadata?: ScanQueryMetadata;
+    /** Metadata discovery failure without changing the registration lifecycle state. */
+    discoveryError?: unknown;
+}>;
+/** Callback notified whenever a managed DataSource is replaced or resolves. */
+export type DataSourceSubscriber<DataSourceT extends ManageableDataSource = ManageableDataSource> = {
+    /** Receives the current DataSource, a pending DataSource promise, or null for placeholders. */
+    onChange: (dataSource: DataSourceT | Promise<DataSourceT> | null) => void;
+};
+/** Parameters for adding or updating a managed DataSource. */
+export type DataSourceManagerAddParameters<DataSourceT extends ManageableDataSource = ManageableDataSource> = {
+    /** Stable id used to register and retrieve the DataSource. */
+    dataSourceId: string;
+    /** DataSource instance, placeholder, or promise resolving to a DataSource. */
+    dataSource: DataSourceT | Promise<DataSourceT> | null;
+    /** Notify subscribers even if the DataSource object is unchanged. */
+    forceUpdate?: boolean;
+    /** If false, the DataSource may be pruned once it has no subscribers. */
+    persistent?: boolean;
+};
+/** Parameters for subscribing a consumer to a managed DataSource. */
+export type DataSourceManagerSubscribeParameters<DataSourceT extends ManageableDataSource = ManageableDataSource> = {
+    /** DataSource id, optionally prefixed with the manager protocol for deferred lookup. */
+    dataSourceId: string;
+    /** Callback invoked when the DataSource changes. */
+    onChange: (dataSource: DataSourceT | Promise<DataSourceT> | null) => void;
+    /** Stable consumer id used to replace or remove all requests by a consumer. */
+    consumerId: string;
+    /** Stable request id scoped to the consumer id. */
+    requestId?: string;
+};
+/** Parameters for deduplicated DataSource creation. */
+export type DataSourceManagerGetOrCreateParameters<DataSourceT extends ManageableDataSource = ManageableDataSource> = {
+    /** Stable id used to deduplicate and later release the DataSource. */
+    dataSourceId?: string;
+    /** Input data used to derive a dedupe id when dataSourceId is not provided. */
+    data?: string | Blob | object;
+    /** Factory called only when the manager does not already have a matching DataSource. */
+    createDataSource: (data?: string | Blob | object) => DataSourceT | Promise<DataSourceT>;
+    /** If false, the DataSource may be pruned once it has no subscribers or retains. */
+    persistent?: boolean;
+};
+/**
+ * Manages shared DataSource instances and notifies consumers when they become available or change.
+ */
+export declare class DataSourceManager {
+    /** Protocol used to refer to deferred DataSource ids. */
+    readonly protocol: string;
+    /** Managed DataSource entries keyed by normalized DataSource id. */
+    private dataSources;
+    /** Consumer subscriptions keyed by consumer id. */
+    private consumers;
+    /** Generated DataSource ids keyed by object identity for non-string inputs. */
+    private dataSourceIds;
+    /** Monotonic counter used to create stable object-identity DataSource ids. */
+    private nextDataSourceId;
+    /** Pending prune timer used to batch unsubscribe cleanup. */
+    private pruneRequest;
+    /** Creates a DataSourceManager with an optional deferred DataSource URL protocol. */
+    constructor(props?: {
+        protocol?: string;
+    });
+    /** Returns true if the id is registered or uses the deferred DataSource protocol. */
+    contains(dataSourceId: string): boolean;
+    /** Lists managed registrations in insertion order for source discovery and diagnostics. */
+    listDataSources(): readonly DataSourceManagerEntryInfo[];
+    /** Discovers picker-ready scan metadata without exposing or retaining owned source instances. */
+    discoverDataSources(options?: DataSourceManagerDiscoveryOptions): Promise<readonly DataSourceManagerDiscoveryInfo[]>;
+    /** Adds or replaces a managed DataSource. */
+    add<DataSourceT extends ManageableDataSource = ManageableDataSource>({ dataSourceId, dataSource, forceUpdate, persistent }: DataSourceManagerAddParameters<DataSourceT>): void;
+    /** Returns an existing DataSource for the same key, or creates and retains a new one. */
+    getOrCreate<DataSourceT extends ManageableDataSource = ManageableDataSource>({ dataSourceId, data, createDataSource, persistent }: DataSourceManagerGetOrCreateParameters<DataSourceT>): DataSourceT | Promise<DataSourceT>;
+    /** Removes a managed DataSource and releases its lifecycle resources. */
+    remove(dataSourceId: string): Promise<void>;
+    /** Releases one retained DataSource reference created by getOrCreate(). */
+    release(dataSourceId: string): Promise<void>;
+    /** Unsubscribes all requests made by one consumer. */
+    unsubscribe({ consumerId }: {
+        consumerId: string;
+    }): void;
+    /** Subscribes a consumer to a managed DataSource. */
+    subscribe<DataSourceT extends ManageableDataSource = ManageableDataSource>({ dataSourceId, onChange, consumerId, requestId }: DataSourceManagerSubscribeParameters<DataSourceT>): DataSourceT | Promise<DataSourceT> | null | undefined;
+    /** Schedules pruning of unused non-persistent DataSources. */
+    prune(): void;
+    /** Releases all managed DataSources and clears all subscriptions. */
+    finalize(): Promise<void>;
+    /** Removes the deferred DataSource protocol prefix from a DataSource id. */
+    private normalizeDataSourceId;
+    /** Resolves an explicit or data-derived DataSource id for deduplicated creation. */
+    private resolveDataSourceId;
+    /** Tracks one consumer request against one managed DataSource entry. */
+    private track;
+    /** Prunes unused non-persistent DataSources immediately. */
+    private pruneNow;
+}
+//# sourceMappingURL=data-source-manager.d.ts.map

package/dist/loader-utils/lib/sources/data-source-manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"data-source-manager.d.ts","sourceRoot":"","sources":["../../../../src/loader-utils/lib/sources/data-source-manager.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAE,iBAAiB,EAAC,yBAAsB;AACjE,OAAO,KAAK,EAAC,iBAAiB,EAA4B,6CAA0C;AAEpG,qEAAqE;AACrE,MAAM,MAAM,oBAAoB,GAAG,UAAU,CAAC,OAAO,EAAE,iBAAiB,CAAC,GAAG;IAC1E,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACnC,uDAAuD;IACvD,QAAQ,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACtC,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CACtC,CAAC;AAEF,wEAAwE;AACxE,MAAM,MAAM,4BAA4B,GAAG,OAAO,GAAG,SAAS,GAAG,aAAa,GAAG,OAAO,CAAC;AAEzF,+EAA+E;AAC/E,MAAM,MAAM,0BAA0B,GAAG,QAAQ,CAAC;IAChD,iEAAiE;IACjE,YAAY,EAAE,MAAM,CAAC;IACrB,8EAA8E;IAC9E,MAAM,EAAE,4BAA4B,CAAC;IACrC,2FAA2F;IAC3F,UAAU,EAAE,OAAO,CAAC;IACpB,8DAA8D;IAC9D,eAAe,EAAE,MAAM,CAAC;IACxB,8EAA8E;IAC9E,WAAW,EAAE,MAAM,CAAC;IACpB,iDAAiD;IACjD,KAAK,CAAC,EAAE,KAAK,CAAC;CACf,CAAC,CAAC;AAEH,2EAA2E;AAC3E,MAAM,MAAM,iCAAiC,GAAG,QAAQ,CAAC;IACvD,2DAA2D;IAC3D,SAAS,CAAC,EAAE,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAC3C,yEAAyE;IACzE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,CAAC,CAAC;AAEH,0DAA0D;AAC1D,MAAM,MAAM,8BAA8B,GAAG,0BAA0B,GACrE,QAAQ,CAAC;IACP,kFAAkF;IAClF,UAAU,EAAE,OAAO,CAAC;IACpB,6EAA6E;IAC7E,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC,oFAAoF;IACpF,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,CAAC,CAAC;AAEL,+EAA+E;AAC/E,MAAM,MAAM,oBAAoB,CAAC,WAAW,SAAS,oBAAoB,GAAG,oBAAoB,IAC9F;IACE,+FAA+F;IAC/F,QAAQ,EAAE,CAAC,UAAU,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,IAAI,KAAK,IAAI,CAAC;CAC3E,CAAC;AAEJ,8DAA8D;AAC9D,MAAM,MAAM,8BAA8B,CACxC,WAAW,SAAS,oBAAoB,GAAG,oBAAoB,IAC7D;IACF,8DAA8D;IAC9D,YAAY,EAAE,MAAM,CAAC;IACrB,8EAA8E;IAC9E,UAAU,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC;IACtD,qEAAqE;IACrE,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,yEAAyE;IACzE,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB,CAAC;AAEF,qEAAqE;AACrE,MAAM,MAAM,oCAAoC,CAC9C,WAAW,SAAS,oBAAoB,GAAG,oBAAoB,IAC7D;IACF,wFAAwF;IACxF,YAAY,EAAE,MAAM,CAAC;IACrB,oDAAoD;IACpD,QAAQ,EAAE,CAAC,UAAU,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,IAAI,KAAK,IAAI,CAAC;IAC1E,+EAA+E;IAC/E,UAAU,EAAE,MAAM,CAAC;IACnB,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,uDAAuD;AACvD,MAAM,MAAM,sCAAsC,CAChD,WAAW,SAAS,oBAAoB,GAAG,oBAAoB,IAC7D;IACF,sEAAsE;IACtE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,+EAA+E;IAC/E,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAAC;IAC9B,wFAAwF;IACxF,gBAAgB,EAAE,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,KAAK,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACxF,oFAAoF;IACpF,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB,CAAC;AAKF;;GAEG;AACH,qBAAa,iBAAiB;IAC5B,yDAAyD;IACzD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,oEAAoE;IACpE,OAAO,CAAC,WAAW,CAA+D;IAClF,mDAAmD;IACnD,OAAO,CAAC,SAAS,CAAiD;IAClE,+EAA+E;IAC/E,OAAO,CAAC,aAAa,CAAiC;IACtD,8EAA8E;IAC9E,OAAO,CAAC,gBAAgB,CAAa;IACrC,6DAA6D;IAC7D,OAAO,CAAC,YAAY,CAA8C;IAElE,qFAAqF;gBACzE,KAAK,GAAE;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAM;IAI3C,qFAAqF;IACrF,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAOvC,2FAA2F;IAC3F,eAAe,IAAI,SAAS,0BAA0B,EAAE;IAMxD,iGAAiG;IAC3F,mBAAmB,CACvB,OAAO,GAAE,iCAAsC,GAC9C,OAAO,CAAC,SAAS,8BAA8B,EAAE,CAAC;IAmCrD,6CAA6C;IAC7C,GAAG,CAAC,WAAW,SAAS,oBAAoB,GAAG,oBAAoB,EAAE,EACnE,YAAY,EACZ,UAAU,EACV,WAAmB,EACnB,UAAiB,EAClB,EAAE,8BAA8B,CAAC,WAAW,CAAC,GAAG,IAAI;IAcrD,yFAAyF;IACzF,WAAW,CAAC,WAAW,SAAS,oBAAoB,GAAG,oBAAoB,EAAE,EAC3E,YAAY,EACZ,IAAI,EACJ,gBAAgB,EAChB,UAAiB,EAClB,EAAE,sCAAsC,CAAC,WAAW,CAAC,GAAG,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAiB3F,yEAAyE;IACnE,MAAM,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAUjD,2EAA2E;IACrE,OAAO,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAalD,sDAAsD;IACtD,WAAW,CAAC,EAAC,UAAU,EAAC,EAAE;QAAC,UAAU,EAAE,MAAM,CAAA;KAAC,GAAG,IAAI;IAarD,qDAAqD;IACrD,SAAS,CAAC,WAAW,SAAS,oBAAoB,GAAG,oBAAoB,EAAE,EACzE,YAAY,EACZ,QAAQ,EACR,UAAU,EACV,SAAqB,EACtB,EAAE,oCAAoC,CAAC,WAAW,CAAC,GAChD,WAAW,GACX,OAAO,CAAC,WAAW,CAAC,GACpB,IAAI,GACJ,SAAS;IAsBb,8DAA8D;IAC9D,KAAK,IAAI,IAAI;IAMb,qEAAqE;IAC/D,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAY/B,4EAA4E;IAC5E,OAAO,CAAC,qBAAqB;IAM7B,oFAAoF;IACpF,OAAO,CAAC,mBAAmB;IAkB3B,wEAAwE;IACxE,OAAO,CAAC,KAAK;IA4Bb,4DAA4D;YAC9C,QAAQ;CAkBvB"}