@loaders.gl/scan 5.0.0-alpha.3

package/dist/federated-table-scan-source.d.ts

@@ -0,0 +1,116 @@
+import { type DataSourceManager, type ScanQueryMetadata, type ScanQueryMetadataOptions, type TableQueryCapabilities, type TableQueryExplain, type TableScanReadOptions, type TableScanSource } from '@loaders.gl/loader-utils';
+import type { ArrowTableBatch, DataType, Schema } from '@loaders.gl/schema';
+import { type SQLPredicate } from '@loaders.gl/sql';
+/** Schema reconciliation modes supported by ordered append federation. */
+export type FederatedTableSchemaPolicy = 'strict' | 'union';
+/** One named table source participating in an ordered federated append. */
+export type FederatedTableSourceEntry = Readonly<{
+    /** Stable id resolved through the shared {@link DataSourceManager}. */
+    dataSourceId: string;
+    /** Source-local predicate, projection, and limit applied before schema reconciliation. */
+    query?: Omit<TableScanReadOptions<SQLPredicate>, 'signal' | 'onTelemetry'>;
+    /** Source-column to federated-column renames applied before compatibility checks. */
+    columnMapping?: Readonly<Record<string, string>>;
+}>;
+/** Options for an ordered federated table source. */
+export type FederatedTableScanSourceOptions = Readonly<{
+    /** Named sources appended in caller-specified order. */
+    sources: readonly FederatedTableSourceEntry[];
+    /** Strict requires matching columns; union null-fills missing columns. Defaults to strict. */
+    schemaPolicy?: FederatedTableSchemaPolicy;
+    /** Explicit federated schema used to validate, order, and safely normalize physical columns. */
+    outputSchema?: Schema;
+    /** Optional display name reported through scan metadata. */
+    name?: string;
+    /** Optional description reported through scan metadata. */
+    description?: string;
+}>;
+/** Provenance attached to every federated Arrow batch. */
+export type FederatedTableBatchProvenance = Readonly<{
+    /** Manager id of the source that emitted this batch. */
+    sourceId: string;
+    /** Zero-based source position in append order. */
+    sourceIndex: number;
+    /** Zero-based data-batch position within the source. */
+    sourceBatchIndex: number;
+    /** Original batch metadata supplied by the physical source. */
+    sourceMetadata?: unknown;
+}>;
+/** Arrow batch emitted by an ordered federated scan. */
+export type FederatedTableBatch = ArrowTableBatch<FederatedTableBatchProvenance> & Readonly<{
+    /** Manager id of the source that emitted this batch. */
+    sourceId: string;
+    /** Zero-based source position in append order. */
+    sourceIndex: number;
+    /** Zero-based data-batch position within the source. */
+    sourceBatchIndex: number;
+}>;
+/** Per-source diagnostics included in a federated explanation. */
+export type FederatedTableSourceExplain = Readonly<{
+    /** Manager id resolved for this source. */
+    sourceId: string;
+    /** Zero-based source position in append order. */
+    sourceIndex: number;
+    /** Physical source type reported by metadata discovery. */
+    sourceType: string;
+    /** Source columns visible after the source-local projection. */
+    sourceColumns: readonly string[];
+    /** Federated column names produced after explicit mappings. */
+    outputColumns: readonly string[];
+    /** Explicit source-column to federated-column mappings. */
+    columnMapping: Readonly<Record<string, string>>;
+    /** Output types that require an explicit lossless normalization for this source. */
+    normalizedTypes: Readonly<Record<string, DataType>>;
+}>;
+/** Serializable logical explanation for an ordered federated table scan. */
+export type FederatedTableScanExplain = TableQueryExplain<SQLPredicate> & Readonly<{
+    /** Schema policy used to reconcile child sources. */
+    schemaPolicy: FederatedTableSchemaPolicy;
+    /** Sources in physical append order. */
+    sources: readonly FederatedTableSourceExplain[];
+}>;
+/** Correctness capabilities of the federated append executor. */
+export declare const FEDERATED_TABLE_QUERY_CAPABILITIES: TableQueryCapabilities;
+/**
+ * Resolves managed table sources and appends their results as one ordered Arrow stream.
+ *
+ * `DataSourceManager` remains the single registry and lifecycle owner. This adapter subscribes for
+ * the duration of metadata discovery or iteration, so deferred and non-persistent sources use the
+ * same replacement, retention, and cleanup system as the rest of loaders.gl.
+ */
+export declare class FederatedTableScanSource implements TableScanSource<FederatedTableBatch, SQLPredicate> {
+    /** Shared manager used as the authoritative named-source registry. */
+    readonly dataSourceManager: DataSourceManager;
+    /** Immutable sources in physical append order. */
+    readonly sources: readonly FederatedTableSourceEntry[];
+    /** Schema reconciliation policy. */
+    readonly schemaPolicy: FederatedTableSchemaPolicy;
+    /** Optional caller-declared normalized output schema. */
+    readonly outputSchema?: Schema;
+    /** Optional display name reported through scan metadata. */
+    readonly name?: string;
+    /** Human-readable description reported through scan metadata. */
+    readonly description: string;
+    /** Stable id used to isolate this adapter's manager subscriptions. */
+    private readonly instanceId;
+    /** Monotonic id used to isolate overlapping operations on this adapter. */
+    private nextOperationId;
+    /** Creates an ordered federated source backed by one shared DataSourceManager. */
+    constructor(dataSourceManager: DataSourceManager, options: FederatedTableScanSourceOptions);
+    /** Discovers and reconciles every child schema without reading federated result batches. */
+    getQueryMetadata(options?: ScanQueryMetadataOptions): Promise<ScanQueryMetadata>;
+    /** Explains source resolution, schema reconciliation, and the global table query. */
+    explain(options?: TableScanReadOptions<SQLPredicate>): Promise<FederatedTableScanExplain>;
+    /**
+     * Emits source and batch ordered Arrow results with one global limit and batch provenance.
+     *
+     * Returning early from the iterator closes the active child iterator and releases every manager
+     * subscription. Later sources are never read once the global limit has been satisfied.
+     */
+    read(options?: TableScanReadOptions<SQLPredicate>): AsyncIterableIterator<FederatedTableBatch>;
+    /** Resolves managed sources, discovers their schemas, and creates one compatibility plan. */
+    private resolvePlan;
+    /** Returns a collision-free manager consumer id for one operation. */
+    private createConsumerId;
+}
+//# sourceMappingURL=federated-table-scan-source.d.ts.map

package/dist/federated-table-scan-source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"federated-table-scan-source.d.ts","sourceRoot":"","sources":["../src/federated-table-scan-source.ts"],"names":[],"mappings":"AAKA,OAAO,EAML,KAAK,iBAAiB,EAEtB,KAAK,iBAAiB,EACtB,KAAK,wBAAwB,EAI7B,KAAK,sBAAsB,EAC3B,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EACzB,KAAK,eAAe,EACrB,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAEV,eAAe,EACf,QAAQ,EAER,MAAM,EAEP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAkB,KAAK,YAAY,EAAC,MAAM,iBAAiB,CAAC;AAEnE,0EAA0E;AAC1E,MAAM,MAAM,0BAA0B,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE5D,2EAA2E;AAC3E,MAAM,MAAM,yBAAyB,GAAG,QAAQ,CAAC;IAC/C,uEAAuE;IACvE,YAAY,EAAE,MAAM,CAAC;IACrB,0FAA0F;IAC1F,KAAK,CAAC,EAAE,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,EAAE,QAAQ,GAAG,aAAa,CAAC,CAAC;IAC3E,qFAAqF;IACrF,aAAa,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAClD,CAAC,CAAC;AAEH,qDAAqD;AACrD,MAAM,MAAM,+BAA+B,GAAG,QAAQ,CAAC;IACrD,wDAAwD;IACxD,OAAO,EAAE,SAAS,yBAAyB,EAAE,CAAC;IAC9C,8FAA8F;IAC9F,YAAY,CAAC,EAAE,0BAA0B,CAAC;IAC1C,gGAAgG;IAChG,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,4DAA4D;IAC5D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC,CAAC;AAEH,0DAA0D;AAC1D,MAAM,MAAM,6BAA6B,GAAG,QAAQ,CAAC;IACnD,wDAAwD;IACxD,QAAQ,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,WAAW,EAAE,MAAM,CAAC;IACpB,wDAAwD;IACxD,gBAAgB,EAAE,MAAM,CAAC;IACzB,+DAA+D;IAC/D,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,CAAC,CAAC;AAEH,wDAAwD;AACxD,MAAM,MAAM,mBAAmB,GAAG,eAAe,CAAC,6BAA6B,CAAC,GAC9E,QAAQ,CAAC;IACP,wDAAwD;IACxD,QAAQ,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,WAAW,EAAE,MAAM,CAAC;IACpB,wDAAwD;IACxD,gBAAgB,EAAE,MAAM,CAAC;CAC1B,CAAC,CAAC;AAEL,kEAAkE;AAClE,MAAM,MAAM,2BAA2B,GAAG,QAAQ,CAAC;IACjD,2CAA2C;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,WAAW,EAAE,MAAM,CAAC;IACpB,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAC;IACnB,gEAAgE;IAChE,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC,+DAA+D;IAC/D,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC,2DAA2D;IAC3D,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAChD,oFAAoF;IACpF,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;CACrD,CAAC,CAAC;AAEH,4EAA4E;AAC5E,MAAM,MAAM,yBAAyB,GAAG,iBAAiB,CAAC,YAAY,CAAC,GACrE,QAAQ,CAAC;IACP,qDAAqD;IACrD,YAAY,EAAE,0BAA0B,CAAC;IACzC,wCAAwC;IACxC,OAAO,EAAE,SAAS,2BAA2B,EAAE,CAAC;CACjD,CAAC,CAAC;AAEL,iEAAiE;AACjE,eAAO,MAAM,kCAAkC,EAAE,sBAM/C,CAAC;AA6CH;;;;;;GAMG;AACH,qBAAa,wBACX,YAAW,eAAe,CAAC,mBAAmB,EAAE,YAAY,CAAC;IAE7D,sEAAsE;IACtE,QAAQ,CAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAC9C,kDAAkD;IAClD,QAAQ,CAAC,OAAO,EAAE,SAAS,yBAAyB,EAAE,CAAC;IACvD,oCAAoC;IACpC,QAAQ,CAAC,YAAY,EAAE,0BAA0B,CAAC;IAClD,yDAAyD;IACzD,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,4DAA4D;IAC5D,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,iEAAiE;IACjE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAE7B,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA2B;IACtD,2EAA2E;IAC3E,OAAO,CAAC,eAAe,CAAK;IAE5B,kFAAkF;gBACtE,iBAAiB,EAAE,iBAAiB,EAAE,OAAO,EAAE,+BAA+B;IAmB1F,4FAA4F;IACtF,gBAAgB,CAAC,OAAO,GAAE,wBAA6B,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAmB1F,qFAAqF;IAC/E,OAAO,CACX,OAAO,GAAE,oBAAoB,CAAC,YAAY,CAAM,GAC/C,OAAO,CAAC,yBAAyB,CAAC;IAqBrC;;;;;OAKG;IACI,IAAI,CACT,OAAO,GAAE,oBAAoB,CAAC,YAAY,CAAM,GAC/C,qBAAqB,CAAC,mBAAmB,CAAC;IA8K7C,6FAA6F;YAC/E,WAAW;IAsCzB,sEAAsE;IACtE,OAAO,CAAC,gBAAgB;CAGzB"}