lakeql 0.1.0 → 0.1.7

package/dist/index.d.ts

@@ -1,18 +1,23 @@
-import * as hyparquet_src_types_js from 'hyparquet/src/types.js';
-import { parquetMetadataAsync, RowGroup } from 'hyparquet';
 import { ParquetWriteOptions, ColumnSource, BasicType } from 'hyparquet-writer';
+import { parquetMetadataAsync, RowGroup } from 'hyparquet';
 
-declare const ERROR_CODES: readonly ["LAQL_PARSE_ERROR", "LAQL_TYPE_ERROR", "LAQL_UNKNOWN_TABLE", "LAQL_UNKNOWN_COLUMN", "LAQL_UNSUPPORTED_PUSHDOWN", "LAQL_BUDGET_EXCEEDED", "LAQL_GROUP_LIMIT_EXCEEDED", "LAQL_OBJECT_NOT_FOUND", "LAQL_CATALOG_ERROR", "LAQL_UNSUPPORTED_ICEBERG_FEATURE", "LAQL_UNSUPPORTED_PARQUET_FEATURE", "LAQL_ICEBERG_COMMIT_CONFLICT", "LAQL_UNSUPPORTED_DELETE_FILES", "LAQL_PARQUET_READ_ERROR", "LAQL_PARQUET_WRITE_ERROR", "LAQL_VALIDATION_ERROR", "LAQL_BOOKMARK_STALE", "LAQL_BOOKMARK_INVALID", "LAQL_ABORTED"];
-type LaQLErrorCode = (typeof ERROR_CODES)[number];
-type ErrorDetails = Record<string, unknown>;
-declare class LaQLError extends Error {
-    readonly code: LaQLErrorCode;
-    readonly details: ErrorDetails;
-    constructor(code: LaQLErrorCode, message: string, details?: ErrorDetails);
-}
-declare function isLaQLError(value: unknown): value is LaQLError;
+type TimestampUnit = "millis" | "micros" | "nanos";
+declare class TimestampValue {
+    readonly epochNanoseconds: bigint;
+    readonly unit: TimestampUnit;
+    readonly isAdjustedToUTC: boolean;
+    constructor(epochNanoseconds: bigint, unit: TimestampUnit, isAdjustedToUTC?: boolean);
+    toJSON(): string;
+    toString(): string;
+}
+declare function timestampFromEpoch(value: bigint, unit: TimestampUnit, isAdjustedToUTC?: boolean): TimestampValue;
+declare function timestampValueFromIso(value: string): TimestampValue | undefined;
+declare function isTimestampValue(value: unknown): value is TimestampValue;
+declare function timestampToIsoString(value: TimestampValue): string;
+declare function timestampEpochForUnit(value: TimestampValue, unit: TimestampUnit): bigint;
+declare function compareTimestampValues(left: TimestampValue, right: TimestampValue): number;
 
-type Scalar = string | number | boolean | bigint | null;
+type Scalar = string | number | boolean | bigint | TimestampValue | null;
 type CompareOp = "eq" | "ne" | "lt" | "lte" | "gt" | "gte";
 interface LiteralExpr {
     kind: "literal";
@@ -65,7 +70,22 @@ interface CallExpr {
     fn: string;
     args: Expr[];
 }
-type Expr = LiteralExpr | ColumnExpr | CompareExpr | InExpr | BetweenExpr | NullCheckExpr | LogicalExpr | NotExpr | LikeExpr | CallExpr;
+interface ArithmeticExpr {
+    kind: "arithmetic";
+    op: "add" | "sub" | "mul" | "div" | "mod";
+    left: Expr;
+    right: Expr;
+}
+interface CaseWhenExpr {
+    when: Expr;
+    value: Expr;
+}
+interface CaseExpr {
+    kind: "case";
+    whens: CaseWhenExpr[];
+    else?: Expr;
+}
+type Expr = LiteralExpr | ColumnExpr | CompareExpr | InExpr | BetweenExpr | NullCheckExpr | LogicalExpr | NotExpr | LikeExpr | CallExpr | ArithmeticExpr | CaseExpr;
 /** A value accepted where a column is expected: a column name or any expression. */
 type ColumnInput = string | Expr;
 /** A value accepted where a literal is expected: a scalar or any expression. */
@@ -90,12 +110,19 @@ declare function like(column: ColumnInput, pattern: string): LikeExpr;
 declare function ilike(column: ColumnInput, pattern: string): LikeExpr;
 /** Generic function-call expression; named functions (h3_*, st_*) build on this. */
 declare function fn(name: string, ...args: ValueInput[]): CallExpr;
+declare function add(left: ValueInput, right: ValueInput): ArithmeticExpr;
+declare function sub(left: ValueInput, right: ValueInput): ArithmeticExpr;
+declare function mul(left: ValueInput, right: ValueInput): ArithmeticExpr;
+declare function div(left: ValueInput, right: ValueInput): ArithmeticExpr;
+declare function mod(left: ValueInput, right: ValueInput): ArithmeticExpr;
 
 type Row = Record<string, unknown>;
 interface BookmarkQuery {
     source: string;
     select?: string[];
+    projections?: Record<string, Expr>;
     where?: Expr;
+    distinct?: boolean;
     orderBy?: {
         column: string;
         direction?: "asc" | "desc";
@@ -175,31 +202,302 @@ interface QueryStats {
     cacheMisses: number;
 }
 
+type Vector = {
+    type: "null";
+    length: number;
+} | {
+    type: "f64";
+    values: Float64Array;
+    valid?: Uint8Array;
+} | {
+    type: "i64";
+    values: BigInt64Array;
+    valid?: Uint8Array;
+} | {
+    type: "timestamp";
+    values: BigInt64Array;
+    unit: TimestampUnit;
+    isAdjustedToUTC: boolean;
+    valid?: Uint8Array;
+} | {
+    type: "bool";
+    values: Uint8Array;
+    valid?: Uint8Array;
+} | {
+    type: "utf8";
+    values: string[];
+    valid?: Uint8Array;
+} | {
+    type: "dict";
+    indices: Uint32Array;
+    dictionary: Vector;
+    valid?: Uint8Array;
+} | {
+    type: "list";
+    offsets: Int32Array;
+    child: Vector;
+    valid?: Uint8Array;
+} | {
+    type: "struct";
+    fields: Record<string, Vector>;
+    length: number;
+    valid?: Uint8Array;
+} | {
+    type: "map";
+    offsets: Int32Array;
+    keys: Vector;
+    values: Vector;
+    valid?: Uint8Array;
+};
+interface Batch {
+    rowCount: number;
+    columns: Record<string, Vector>;
+}
+type Selection = Uint8Array;
+type VectorValue = unknown;
+declare function batchFromColumns(columns: Record<string, ArrayLike<VectorValue>>): Batch;
+declare function batchFromVectors(columns: Record<string, Vector>): Batch;
+declare function vectorFromValues(values: ArrayLike<VectorValue>): Vector;
+declare function materializeBatchRows(batch: Batch): Row[];
+declare function materializeSelectedBatchRows(batch: Batch, selection?: Selection): Row[];
+declare function selectedRowCount(rowCount: number, selection?: Selection): number;
+declare function selectedRowIndices(rowCount: number, selection?: Selection): Iterable<number>;
+declare function predicateSelection(batch: Batch, expr: Expr | undefined): Selection;
+declare function tryPredicateSelection(batch: Batch, expr: Expr | undefined): Selection | undefined;
+declare function vectorValue(vector: Vector, index: number): string | number | bigint | boolean | TimestampValue | unknown[] | Record<string, unknown> | null;
+declare function vectorLength(vector: Vector): number;
+interface BatchExprValues {
+    rowCount: number;
+    vector?: Vector;
+    literal?: Scalar;
+    valueAt(index: number): Scalar;
+}
+declare function batchExprValues(batch: Batch, expr: Expr): BatchExprValues;
+declare function scalarVectorValue(vector: Vector, index: number): Scalar;
+
+declare const ERROR_CODES: readonly ["LAKEQL_PARSE_ERROR", "LAKEQL_SQL_UNSUPPORTED", "LAKEQL_TYPE_ERROR", "LAKEQL_UNKNOWN_TABLE", "LAKEQL_UNKNOWN_COLUMN", "LAKEQL_UNSUPPORTED_PUSHDOWN", "LAKEQL_BUDGET_EXCEEDED", "LAKEQL_GROUP_LIMIT_EXCEEDED", "LAKEQL_OBJECT_NOT_FOUND", "LAKEQL_CATALOG_ERROR", "LAKEQL_UNSUPPORTED_ICEBERG_FEATURE", "LAKEQL_UNSUPPORTED_PARQUET_FEATURE", "LAKEQL_ICEBERG_COMMIT_CONFLICT", "LAKEQL_UNSUPPORTED_DELETE_FILES", "LAKEQL_PARQUET_READ_ERROR", "LAKEQL_PARQUET_WRITE_ERROR", "LAKEQL_VALIDATION_ERROR", "LAKEQL_BOOKMARK_STALE", "LAKEQL_BOOKMARK_INVALID", "LAKEQL_ABORTED", "LAKEQL_GEO_BACKEND_MISSING"];
+type LakeqlErrorCode = (typeof ERROR_CODES)[number];
+type ErrorDetails = Record<string, unknown>;
+declare class LakeqlError extends Error {
+    readonly code: LakeqlErrorCode;
+    readonly details: ErrorDetails;
+    constructor(code: LakeqlErrorCode, message: string, details?: ErrorDetails);
+}
+declare function isLakeqlError(value: unknown): value is LakeqlError;
+
 type SqlBoolean = boolean | null;
+/** External geometry/H3 primitives the evaluator can't compute from bbox math alone. */
+interface GeoBackend {
+    booleanContains(a: GeoJsonGeometry, b: GeoJsonGeometry): boolean;
+    booleanIntersects(a: GeoJsonGeometry, b: GeoJsonGeometry): boolean;
+    cellToParent(cell: string, res: number): string;
+    gridDisk(origin: string, k: number): string[];
+    isValidCell(cell: string): boolean;
+    latLngToCell(lat: number, lon: number, res: number): string;
+}
+/** Install the backend. Called by `./geo-backend.ts` once its libraries load. */
+declare function setGeoBackend(backend: GeoBackend): void;
+declare function requireGeoBackend(): GeoBackend;
+/** Force-load the geo backend (idempotent). Exposed for direct evaluate() callers. */
+declare function loadGeoBackend(): Promise<void>;
+/**
+ * Load the geo backend iff any of `exprs` uses a backend-requiring spatial
+ * function. Cheap no-op once the backend is installed or when no geo is present,
+ * so query execution can call it unconditionally.
+ */
+declare function ensureGeoBackendForExprs(exprs: Iterable<Expr | undefined>): Promise<void>;
 type EvalValue = Scalar;
+interface BBox {
+    minx: number;
+    miny: number;
+    maxx: number;
+    maxy: number;
+}
 declare function evaluate(expr: Expr, row: Row): EvalValue;
 declare function matches(expr: Expr | undefined, row: Row): boolean;
 declare function jsonSafeValue(value: unknown): unknown;
 declare function encodeJsonLine(row: Row): Uint8Array;
+declare function envelopeOf(geometry: GeoJsonGeometry): BBox;
+type GeoJsonGeometry = {
+    type: "Point";
+    coordinates: [number, number];
+} | {
+    type: "LineString";
+    coordinates: [number, number][];
+} | {
+    type: "Polygon";
+    coordinates: [number, number][][];
+};
+declare function toGeometry(parsed: Record<string, unknown>, name: string): GeoJsonGeometry;
+declare function parseGeometry(value: EvalValue, name: string): Record<string, unknown>;
+declare function envelopeFromGeometry(parsed: Record<string, unknown>, name: string): BBox;
+declare function bboxIntersects(left: BBox, right: BBox): boolean;
 
-type JoinType = "inner" | "left" | "semi" | "anti";
-interface BroadcastJoinOptions {
-    leftKey: string;
-    rightKey: string;
-    maxRightRows: number;
-    type?: JoinType;
-    rightPrefix?: string;
+interface PutOptions {
+    contentType?: string;
+    metadata?: Record<string, string>;
 }
-interface LookupJoinOptions {
-    leftKey: string;
-    rightKey: string;
-    maxRightRows: number;
-    type?: JoinType;
-    rightPrefix?: string;
+interface ConditionalPutOptions extends PutOptions {
+    /**
+     * ETag that must still identify the existing object. Use null to require
+     * the object to be absent.
+     */
+    expectedEtag: string | null;
 }
-type LookupJoinFunction = (key: string | number | boolean | bigint | null, leftRow: Row) => AsyncIterable<Row> | Iterable<Row> | Promise<AsyncIterable<Row> | Iterable<Row>>;
-declare function broadcastJoin(left: AsyncIterable<Row> | Iterable<Row>, right: AsyncIterable<Row> | Iterable<Row>, options: BroadcastJoinOptions): Promise<Row[]>;
-declare function lookupJoin(left: AsyncIterable<Row> | Iterable<Row>, lookup: LookupJoinFunction, options: LookupJoinOptions): Promise<Row[]>;
+interface ListOptions {
+    /** Stop listing after this many objects. */
+    limit?: number;
+    /** List "directories" by stopping at this delimiter. */
+    delimiter?: string;
+}
+interface ObjectInfo {
+    path: string;
+    size: number;
+    etag?: string;
+    lastModified?: Date;
+}
+interface ObjectHead {
+    size: number;
+    etag?: string;
+    lastModified?: Date;
+    contentType?: string;
+}
+/**
+ * The single environmental contract for storage. Every runtime driver
+ * (R2, S3, HTTP, filesystem) supplies one of these; core never touches
+ * a runtime API directly.
+ */
+interface ObjectStore {
+    get(path: string): Promise<Uint8Array | null>;
+    getRange(path: string, range: {
+        offset: number;
+        length: number;
+    }): Promise<Uint8Array>;
+    put(path: string, body: Uint8Array | ReadableStream<Uint8Array>, options?: PutOptions): Promise<void>;
+    delete(path: string): Promise<void>;
+    list(prefix: string, options?: ListOptions): AsyncIterable<ObjectInfo>;
+    head(path: string): Promise<ObjectHead | null>;
+}
+interface ConditionalObjectStore extends ObjectStore {
+    conditionalPut(path: string, body: Uint8Array | ReadableStream<Uint8Array>, options: ConditionalPutOptions): Promise<boolean>;
+}
+interface ObjectStoreReadControls {
+    maxConcurrentReads?: number;
+    signal?: AbortSignal;
+    maxElapsedMs?: number;
+}
+declare function withObjectStoreReadControls(store: ObjectStore, controls: ObjectStoreReadControls): ObjectStore;
+declare function readControlSignal(controls: ObjectStoreReadControls): AbortSignal | undefined;
+declare function throwIfAborted(signal: AbortSignal | undefined): void;
+
+interface TaskManifestTask {
+    id: string;
+    input: TaskInput;
+    outputRole: "rows" | "data-file" | "manifest";
+}
+interface TaskManifest {
+    version: 1;
+    jobId: string;
+    planFingerprint: string;
+    snapshot: string;
+    tasks: TaskManifestTask[];
+}
+interface OutputManifestEntry {
+    taskId: string;
+    outputPath: string;
+    partitionValues: Record<string, string>;
+    rowCount: number;
+    byteSize: number;
+    contentHash?: string;
+    etag?: string;
+    iceberg?: {
+        recordCount: number;
+        fileSizeInBytes: number;
+        partitionValues: Record<string, string>;
+    };
+}
+interface OutputManifest {
+    version: 1;
+    jobId: string;
+    planFingerprint: string;
+    entries: OutputManifestEntry[];
+}
+type TaskState = "planned" | "running" | "output-written" | "manifest-recorded" | "complete";
+interface TaskCheckpoint {
+    taskId: string;
+    state: TaskState;
+    idempotencyKey: string;
+    updatedAtMs: number;
+    output?: OutputManifestEntry;
+    outputs?: OutputManifestEntry[];
+}
+interface CheckpointAdapter {
+    get(taskId: string): Promise<TaskCheckpoint | undefined>;
+    put(checkpoint: TaskCheckpoint): Promise<void>;
+    list(jobId?: string): AsyncIterable<TaskCheckpoint>;
+}
+interface TaskTransitionInput {
+    taskId: string;
+    nextState: TaskState;
+    idempotencyKey: string;
+    nowMs: number;
+    staleTimeoutMs?: number;
+    output?: OutputManifestEntry;
+    outputs?: OutputManifestEntry[];
+}
+interface BookmarkPosition {
+    fileIndex: number;
+    rowGroup: number;
+    rowOffset: number;
+    taskId?: string;
+    outputManifestCursor?: number;
+}
+interface BookmarkInit {
+    planFingerprint: string;
+    snapshot: string;
+    query?: BookmarkQuery;
+    position: BookmarkPosition;
+    writeState?: Bookmark["writeState"];
+    operatorState?: Bookmark["operatorState"];
+}
+declare function createTaskManifest(input: {
+    jobId: string;
+    snapshot?: string;
+    tasks: TaskInput[];
+    outputRole?: TaskManifestTask["outputRole"];
+}): TaskManifest;
+declare function createOutputManifest(input: {
+    jobId: string;
+    planFingerprint: string;
+    entries: OutputManifestEntry[];
+}): OutputManifest;
+declare function createOutputManifestFromCheckpoints(input: {
+    jobId: string;
+    planFingerprint: string;
+    checkpoints: CheckpointAdapter;
+}): Promise<OutputManifest>;
+declare function writeOutputManifest(store: ObjectStore, path: string, manifest: OutputManifest): Promise<{
+    path: string;
+    byteSize: number;
+    etag?: string;
+}>;
+declare function readOutputManifest(store: ObjectStore, path: string): Promise<OutputManifest>;
+declare function createBookmark(init: BookmarkInit): Bookmark;
+declare function assertBookmarkMatches(bookmark: Bookmark, planFingerprint: string): void;
+declare function signPaginationToken(bookmark: Bookmark, secret: string | Uint8Array): Promise<string>;
+declare function verifyPaginationToken(token: string, secret: string | Uint8Array): Promise<Bookmark>;
+declare class MemoryCheckpointAdapter implements CheckpointAdapter {
+    private readonly checkpoints;
+    private readonly taskJobs;
+    get(taskId: string): Promise<TaskCheckpoint | undefined>;
+    put(checkpoint: TaskCheckpoint): Promise<void>;
+    list(jobId?: string): AsyncIterable<TaskCheckpoint>;
+}
+declare function memoryCheckpointAdapter(): MemoryCheckpointAdapter;
+declare function advanceTaskCheckpoint(checkpoints: CheckpointAdapter, input: TaskTransitionInput): Promise<TaskCheckpoint>;
+declare function transitionTaskCheckpoint(existing: TaskCheckpoint | undefined, input: TaskTransitionInput): TaskCheckpoint;
+declare function stableStringify(value: unknown): string;
+declare function fingerprint(value: unknown): string;
 
 interface PredicatePlan {
     partition: Expr[];
@@ -345,63 +643,6 @@ declare function buildBBoxIndex<T extends Record<string, unknown>>(rows: T[], co
     maxy: string;
 }): BBoxIndex;
 
-interface PutOptions {
-    contentType?: string;
-    metadata?: Record<string, string>;
-}
-interface ConditionalPutOptions extends PutOptions {
-    /**
-     * ETag that must still identify the existing object. Use null to require
-     * the object to be absent.
-     */
-    expectedEtag: string | null;
-}
-interface ListOptions {
-    /** Stop listing after this many objects. */
-    limit?: number;
-    /** List "directories" by stopping at this delimiter. */
-    delimiter?: string;
-}
-interface ObjectInfo {
-    path: string;
-    size: number;
-    etag?: string;
-    lastModified?: Date;
-}
-interface ObjectHead {
-    size: number;
-    etag?: string;
-    lastModified?: Date;
-    contentType?: string;
-}
-/**
- * The single environmental contract for storage. Every runtime driver
- * (R2, S3, HTTP, filesystem) supplies one of these; core never touches
- * a runtime API directly.
- */
-interface ObjectStore {
-    get(path: string): Promise<Uint8Array | null>;
-    getRange(path: string, range: {
-        offset: number;
-        length: number;
-    }): Promise<Uint8Array>;
-    put(path: string, body: Uint8Array | ReadableStream<Uint8Array>, options?: PutOptions): Promise<void>;
-    delete(path: string): Promise<void>;
-    list(prefix: string, options?: ListOptions): AsyncIterable<ObjectInfo>;
-    head(path: string): Promise<ObjectHead | null>;
-}
-interface ConditionalObjectStore extends ObjectStore {
-    conditionalPut(path: string, body: Uint8Array | ReadableStream<Uint8Array>, options: ConditionalPutOptions): Promise<boolean>;
-}
-interface ObjectStoreReadControls {
-    maxConcurrentReads?: number;
-    signal?: AbortSignal;
-    maxElapsedMs?: number;
-}
-declare function withObjectStoreReadControls(store: ObjectStore, controls: ObjectStoreReadControls): ObjectStore;
-declare function readControlSignal(controls: ObjectStoreReadControls): AbortSignal | undefined;
-declare function throwIfAborted(signal: AbortSignal | undefined): void;
-
 interface QueryBudget {
     maxFiles?: number;
     maxBytes?: number;
@@ -432,6 +673,7 @@ interface LakeConfig {
     store: ObjectStore;
     scanner: ScanAdapter;
     sidecarIndex?: SidecarFileIndex[];
+    planningCache?: CacheAdapter<ObjectInfo[]>;
     budget?: QueryBudget;
     policy?: QueryPolicy;
     substrate?: RuntimeSubstrate;
@@ -441,6 +683,8 @@ interface LakeConfig {
 interface ScanOptions {
     columns?: string[];
     where?: Expr;
+    rowStart?: number;
+    rowEnd?: number;
     batchSize: number;
     stats: QueryStats;
     budget: QueryBudget;
@@ -449,23 +693,38 @@ interface ScanOptions {
 }
 interface ScanAdapter {
     scan(path: string, options: ScanOptions): AsyncIterable<Row[]>;
+    scanColumns?(path: string, options: ScanOptions): AsyncIterable<Batch>;
+    scanVectorBatches?(path: string, options: ScanOptions): AsyncIterable<ScanVectorBatch>;
+    scanColumnBatches?(path: string, options: ScanOptions): AsyncIterable<ScanColumnBatch>;
     planTask?(path: string, options: ScanTaskPlanOptions): Promise<ScanTaskPlan>;
 }
+interface ScanVectorBatch {
+    rowOffset: number;
+    batch: Batch;
+}
+interface ScanColumnBatch {
+    rowOffset: number;
+    batch: Batch;
+}
 interface ScanTaskPlanOptions {
     columns?: string[];
     where?: Expr;
     partitionValues: Record<string, string>;
+    object?: ObjectInfo;
 }
 interface ScanTaskPlan {
     rowGroupRanges: {
         start: number;
         end: number;
     }[];
+    rowGroupCount?: number;
 }
 interface PathQueryInit {
     source: string;
     select?: string[];
+    projections?: Record<string, Expr>;
     where?: Expr;
+    distinct?: boolean;
     orderBy?: OrderByTerm[];
     limit?: number;
     offset?: number;
@@ -494,14 +753,20 @@ interface CsvStreamOptions {
     /** Column order for CSV output. Defaults to select columns or first-row keys. */
     columns?: string[];
 }
-type AggregateOp = "count" | "sum" | "avg" | "min" | "max" | "count_distinct" | "approx_count_distinct" | "first" | "last" | "any";
+type AggregateOp = "count" | "sum" | "avg" | "var_samp" | "var_pop" | "stddev_samp" | "stddev_pop" | "median" | "quantile" | "min" | "max" | "count_distinct" | "approx_count_distinct" | "mode" | "first" | "last" | "any";
 interface AggregateExpr {
     op: AggregateOp;
     column?: string;
+    expr?: Expr;
+    quantile?: number;
 }
 type AggregateSpec = Record<string, AggregateExpr>;
 interface AggregateOptions {
     maxGroups?: number;
+    having?: Expr;
+    orderBy?: OrderByTerm[];
+    limit?: number;
+    offset?: number;
     operatorState?: Uint8Array | AggregateOperatorState | {
         spillRef: string;
     };
@@ -579,12 +844,31 @@ type AggregateStateSnapshot = {
     op: "avg";
     sum: number;
     count: number;
+} | {
+    op: "var_samp" | "var_pop" | "stddev_samp" | "stddev_pop";
+    count: number;
+    mean: number;
+    m2: number;
+} | {
+    op: "median";
+    values: (number | string)[];
+} | {
+    op: "quantile";
+    quantile: number;
+    values: number[];
 } | {
     op: "min" | "max";
     value: AggregateSnapshotValue;
 } | {
     op: "count_distinct" | "approx_count_distinct";
     values: string[];
+} | {
+    op: "mode";
+    values: {
+        key: string;
+        value: AggregateSnapshotValue;
+        count: number;
+    }[];
 } | {
     op: "first" | "last" | "any";
     seen: boolean;
@@ -594,6 +878,7 @@ interface TaskInput {
     path: string;
     etag?: string;
     size?: number;
+    rowGroupCount?: number;
     rowGroupRanges: {
         start: number;
         end: number;
@@ -619,6 +904,7 @@ interface JsonQueryV1 {
     from: string;
     select?: string[];
     where?: JsonExpr;
+    distinct?: boolean;
     orderBy?: JsonOrderByTerm[];
     limit?: number;
     offset?: number;
@@ -670,6 +956,7 @@ declare class Lake {
     private readonly queryId;
     private readonly substrate;
     private readonly sidecarIndex;
+    private readonly planningCache;
     constructor(config: LakeConfig);
     path(source: string): QueryBuilder;
     hive(source: string): QueryBuilder;
@@ -688,7 +975,9 @@ declare class QueryBuilder {
     private readonly init;
     constructor(lake: Lake, init: PathQueryInit);
     select(columns: string[]): QueryBuilder;
+    project(projections: Record<string, Expr>): QueryBuilder;
     where(expr: Expr): QueryBuilder;
+    distinct(enabled?: boolean): QueryBuilder;
     orderBy(terms: OrderByTerm[]): QueryBuilder;
     limit(limit: number): QueryBuilder;
     offset(offset: number): QueryBuilder;
@@ -727,6 +1016,7 @@ interface QueryResultConfig extends PathQueryInit {
     queryId: string;
     metrics?: MetricsHook;
     sidecarIndex?: SidecarFileIndex[];
+    planningCache?: CacheAdapter<ObjectInfo[]>;
 }
 declare class QueryResult {
     readonly stats: QueryStats;
@@ -735,6 +1025,8 @@ declare class QueryResult {
     rows(): AsyncIterable<Row>;
     private matchedRows;
     batches(): AsyncIterable<Row[]>;
+    private tryColumnarProjectedBatches;
+    private columnarProjectedBatches;
     toArray(): Promise<Row[]>;
     topKWithState(options?: TopKOptions): Promise<TopKResult>;
     sortWithState(options?: SortOptions): Promise<SortResult>;
@@ -745,8 +1037,15 @@ declare class QueryResult {
     slice(options: SliceOptions): Promise<SliceResult>;
     resumableBatches(options: ResumableBatchOptions): AsyncIterable<SliceResult>;
     aggregate(groupColumns: string[], spec: AggregateSpec, options?: AggregateOptions): Promise<Row[]>;
+    private tryColumnarAggregateRows;
+    private tryLateMaterializedAggregateRows;
+    private aggregateMaterializedRows;
     aggregateWithState(groupColumns: string[], spec: AggregateSpec, options?: AggregateOptions): Promise<AggregateResult>;
     private orderedBatches;
+    private tryColumnarTopKRows;
+    private tryLateMaterializedTopKRows;
+    private materializeRowRefs;
+    private columnBatches;
     private collectOrderedMatches;
     private collectSortRuns;
     explain(): Promise<ExplainResult>;
@@ -764,115 +1063,59 @@ declare function deserializeTopKOperatorState(bytes: Uint8Array | TopKOperatorSt
 declare function deserializeSortOperatorState(bytes: Uint8Array | SortOperatorState): SortOperatorState;
 declare function parseJsonQuery(input: unknown): PathQueryInit;
 declare function parseHivePartitions(path: string): Record<string, string>;
-
-interface TaskManifestTask {
-    id: string;
-    input: TaskInput;
-    outputRole: "rows" | "data-file" | "manifest";
-}
-interface TaskManifest {
-    version: 1;
-    jobId: string;
-    planFingerprint: string;
-    snapshot: string;
-    tasks: TaskManifestTask[];
-}
-interface OutputManifestEntry {
-    taskId: string;
-    outputPath: string;
-    partitionValues: Record<string, string>;
-    rowCount: number;
-    byteSize: number;
-    contentHash?: string;
-    etag?: string;
-    iceberg?: {
-        recordCount: number;
-        fileSizeInBytes: number;
-        partitionValues: Record<string, string>;
-    };
-}
-interface OutputManifest {
-    version: 1;
-    jobId: string;
-    planFingerprint: string;
-    entries: OutputManifestEntry[];
-}
-type TaskState = "planned" | "running" | "output-written" | "manifest-recorded" | "complete";
-interface TaskCheckpoint {
-    taskId: string;
-    state: TaskState;
-    idempotencyKey: string;
-    updatedAtMs: number;
-    output?: OutputManifestEntry;
-    outputs?: OutputManifestEntry[];
+
+interface InMemoryTableOptions {
+    maxRows?: number;
+    maxBytes?: number;
 }
-interface CheckpointAdapter {
-    get(taskId: string): Promise<TaskCheckpoint | undefined>;
-    put(checkpoint: TaskCheckpoint): Promise<void>;
-    list(jobId?: string): AsyncIterable<TaskCheckpoint>;
+interface InMemoryLakeOptions extends Omit<LakeConfig, "store" | "scanner" | "sidecarIndex">, InMemoryTableOptions {
 }
-interface TaskTransitionInput {
-    taskId: string;
-    nextState: TaskState;
-    idempotencyKey: string;
-    nowMs: number;
-    staleTimeoutMs?: number;
-    output?: OutputManifestEntry;
-    outputs?: OutputManifestEntry[];
+declare class InMemoryRowsScanner implements ScanAdapter {
+    private readonly tables;
+    constructor(tables: Record<string, readonly Row[]>, options?: InMemoryTableOptions);
+    scan(path: string, options: ScanOptions): AsyncIterable<Row[]>;
+    planTask(path: string): Promise<{
+        rowGroupRanges: {
+            start: number;
+            end: number;
+        }[];
+    }>;
+    tableInfo(path: string): ObjectInfo;
+    listTables(prefix: string, options?: ListOptions): ObjectInfo[];
+    private table;
 }
-interface BookmarkPosition {
-    fileIndex: number;
-    rowGroup: number;
-    rowOffset: number;
-    taskId?: string;
-    outputManifestCursor?: number;
+declare class InMemoryRowsStore implements ObjectStore {
+    private readonly scanner;
+    constructor(scanner: InMemoryRowsScanner);
+    get(path: string): Promise<Uint8Array | null>;
+    getRange(path: string): Promise<Uint8Array>;
+    put(): Promise<void>;
+    delete(): Promise<void>;
+    list(prefix: string, options?: ListOptions): AsyncIterable<ObjectInfo>;
+    head(path: string): Promise<ObjectHead | null>;
 }
-interface BookmarkInit {
-    planFingerprint: string;
-    snapshot: string;
-    query?: BookmarkQuery;
-    position: BookmarkPosition;
-    writeState?: Bookmark["writeState"];
-    operatorState?: Bookmark["operatorState"];
+declare function inMemoryRowsScanner(tables: Record<string, readonly Row[]>, options?: InMemoryTableOptions): InMemoryRowsScanner;
+declare function createInMemoryLake(tables: Record<string, readonly Row[]>, options?: InMemoryLakeOptions): Lake;
+
+type JoinType = "inner" | "left" | "semi" | "anti";
+type JoinKey = string | string[];
+interface BroadcastJoinOptions {
+    leftKey: JoinKey;
+    rightKey: JoinKey;
+    maxRightRows: number;
+    type?: JoinType;
+    rightPrefix?: string;
 }
-declare function createTaskManifest(input: {
-    jobId: string;
-    snapshot?: string;
-    tasks: TaskInput[];
-    outputRole?: TaskManifestTask["outputRole"];
-}): TaskManifest;
-declare function createOutputManifest(input: {
-    jobId: string;
-    planFingerprint: string;
-    entries: OutputManifestEntry[];
-}): OutputManifest;
-declare function createOutputManifestFromCheckpoints(input: {
-    jobId: string;
-    planFingerprint: string;
-    checkpoints: CheckpointAdapter;
-}): Promise<OutputManifest>;
-declare function writeOutputManifest(store: ObjectStore, path: string, manifest: OutputManifest): Promise<{
-    path: string;
-    byteSize: number;
-    etag?: string;
-}>;
-declare function readOutputManifest(store: ObjectStore, path: string): Promise<OutputManifest>;
-declare function createBookmark(init: BookmarkInit): Bookmark;
-declare function assertBookmarkMatches(bookmark: Bookmark, planFingerprint: string): void;
-declare function signPaginationToken(bookmark: Bookmark, secret: string | Uint8Array): Promise<string>;
-declare function verifyPaginationToken(token: string, secret: string | Uint8Array): Promise<Bookmark>;
-declare class MemoryCheckpointAdapter implements CheckpointAdapter {
-    private readonly checkpoints;
-    private readonly taskJobs;
-    get(taskId: string): Promise<TaskCheckpoint | undefined>;
-    put(checkpoint: TaskCheckpoint): Promise<void>;
-    list(jobId?: string): AsyncIterable<TaskCheckpoint>;
+interface LookupJoinOptions {
+    leftKey: JoinKey;
+    rightKey: JoinKey;
+    maxRightRows: number;
+    type?: JoinType;
+    rightPrefix?: string;
 }
-declare function memoryCheckpointAdapter(): MemoryCheckpointAdapter;
-declare function advanceTaskCheckpoint(checkpoints: CheckpointAdapter, input: TaskTransitionInput): Promise<TaskCheckpoint>;
-declare function transitionTaskCheckpoint(existing: TaskCheckpoint | undefined, input: TaskTransitionInput): TaskCheckpoint;
-declare function stableStringify(value: unknown): string;
-declare function fingerprint(value: unknown): string;
+type LookupJoinFunction = (key: string | number | boolean | bigint | null | (string | number | boolean | bigint | null)[], leftRow: Row) => AsyncIterable<Row> | Iterable<Row> | Promise<AsyncIterable<Row> | Iterable<Row>>;
+declare function broadcastJoin(left: AsyncIterable<Row> | Iterable<Row>, right: AsyncIterable<Row> | Iterable<Row>, options: BroadcastJoinOptions): Promise<Row[]>;
+declare function lookupJoin(left: AsyncIterable<Row> | Iterable<Row>, lookup: LookupJoinFunction, options: LookupJoinOptions): Promise<Row[]>;
 
 /**
  * In-memory ObjectStore. The reference implementation of the store
@@ -895,7 +1138,244 @@ declare class MemoryObjectStore implements ObjectStore {
 }
 declare function memoryStore(): MemoryObjectStore;
 
-declare const PACKAGE: "@laql/iceberg";
+interface ObjectStoreCacheOptions {
+    /** Maximum cached object/range bytes retained in memory. Defaults to 64 MiB. */
+    maxBytes?: number;
+    /** Optional time-to-live for cache entries. Entries do not expire by default. */
+    ttlMs?: number;
+    /** How LakeQL should spend the cache budget. Defaults to balanced. */
+    policy?: CachePolicy;
+}
+type CachePolicy = "balanced" | "io" | "latency";
+interface SharedCacheEntry<T> {
+    value: T;
+    bytes: number;
+}
+interface SharedCacheSetOptions {
+    priority?: number;
+}
+declare class SharedMemoryCache {
+    private readonly maxBytes;
+    private readonly ttlMs;
+    private readonly entries;
+    private cachedBytes;
+    constructor(options?: ObjectStoreCacheOptions);
+    get<T>(key: string): SharedCacheEntry<T> | undefined;
+    set<T>(key: string, value: T, bytes: number, options?: SharedCacheSetOptions): void;
+    delete(key: string): void;
+    deleteWhere(predicate: (value: unknown) => boolean): void;
+    private expiresAt;
+    private evict;
+    private evictKey;
+}
+declare function cachedObjectStore(inner: ObjectStore, options?: ObjectStoreCacheOptions, sharedCache?: SharedMemoryCache): ObjectStore;
+
+type VectorDistinctAggregateState = {
+    op: "count_distinct" | "approx_count_distinct";
+    values: Set<string>;
+    sortedValues?: string[];
+    sortedRuns?: string[][];
+    memoryBytes: number;
+};
+
+type VectorAggregateValue = string | number | boolean | bigint | TimestampValue | null;
+type VectorAggregateState = {
+    op: "count";
+    count: number;
+} | {
+    op: "sum";
+    sum: number;
+} | {
+    op: "avg";
+    sum: number;
+    count: number;
+} | {
+    op: "var_samp" | "var_pop" | "stddev_samp" | "stddev_pop";
+    count: number;
+    mean: number;
+    m2: number;
+} | {
+    op: "median";
+    values: (number | string)[];
+    memoryBytes: number;
+} | {
+    op: "quantile";
+    quantile: number;
+    values: number[];
+    memoryBytes: number;
+} | {
+    op: "min" | "max";
+    value: VectorAggregateValue;
+} | VectorDistinctAggregateState | {
+    op: "mode";
+    values: Map<string, {
+        value: Exclude<VectorAggregateValue, null>;
+        count: number;
+    }>;
+    memoryBytes: number;
+} | {
+    op: "first";
+    seen: boolean;
+    value: VectorAggregateValue;
+} | {
+    op: "last";
+    seen: boolean;
+    value: VectorAggregateValue;
+} | {
+    op: "any";
+    seen: boolean;
+    value: VectorAggregateValue;
+};
+type VectorAggregateStates = Record<string, VectorAggregateState>;
+type VectorAggregateSnapshotValue = string | number | boolean | null | {
+    type: "bigint";
+    value: string;
+} | {
+    type: "timestamp";
+    epochNanoseconds: string;
+    unit: TimestampValue["unit"];
+    isAdjustedToUTC: boolean;
+};
+type VectorAggregateStateSnapshot = {
+    op: "count";
+    count: number;
+} | {
+    op: "sum";
+    sum: number;
+} | {
+    op: "avg";
+    sum: number;
+    count: number;
+} | {
+    op: "var_samp" | "var_pop" | "stddev_samp" | "stddev_pop";
+    count: number;
+    mean: number;
+    m2: number;
+} | {
+    op: "median";
+    values: (number | string)[];
+} | {
+    op: "quantile";
+    quantile: number;
+    values: number[];
+} | {
+    op: "min" | "max";
+    value: VectorAggregateSnapshotValue;
+} | {
+    op: "count_distinct" | "approx_count_distinct";
+    values: string[];
+} | {
+    op: "mode";
+    values: {
+        key: string;
+        value: VectorAggregateSnapshotValue;
+        count: number;
+    }[];
+} | {
+    op: "first";
+    seen: boolean;
+    value: VectorAggregateSnapshotValue;
+} | {
+    op: "last";
+    seen: boolean;
+    value: VectorAggregateSnapshotValue;
+} | {
+    op: "any";
+    seen: boolean;
+    value: VectorAggregateSnapshotValue;
+};
+type VectorAggregateStateSnapshots = Record<string, VectorAggregateStateSnapshot>;
+interface VectorAggregateOptions {
+    budget?: QueryBudget;
+}
+declare function createVectorAggregateStates(spec: AggregateSpec, options?: VectorAggregateOptions): VectorAggregateStates;
+declare function updateVectorAggregateStates(states: VectorAggregateStates, spec: AggregateSpec, batch: Batch, selection?: Selection, options?: VectorAggregateOptions): void;
+declare function mergeVectorAggregateStates(target: VectorAggregateStates, source: VectorAggregateStates, options?: VectorAggregateOptions): void;
+declare function mergeVectorAggregateStateSnapshots(target: VectorAggregateStates, snapshots: VectorAggregateStateSnapshots, options?: VectorAggregateOptions): void;
+declare function finalizeVectorAggregateStates(states: VectorAggregateStates): Record<string, unknown>;
+declare function snapshotVectorAggregateStates(states: VectorAggregateStates): VectorAggregateStateSnapshots;
+declare function restoreVectorAggregateStates(snapshots: VectorAggregateStateSnapshots, options?: VectorAggregateOptions): VectorAggregateStates;
+declare function vectorAggregateBatch(spec: AggregateSpec, batch: Batch, selection?: Selection, options?: VectorAggregateOptions): Record<string, unknown>;
+declare function updateVectorAggregateStateValue(state: VectorAggregateState, value: VectorAggregateValue, options?: VectorAggregateOptions): void;
+
+interface VectorGroupByOptions extends VectorAggregateOptions {
+    maxGroups?: number;
+}
+interface VectorGroupByState {
+    readonly keys: readonly string[];
+    readonly spec: AggregateSpec;
+    readonly groups: Map<string, VectorGroup>;
+}
+interface VectorGroup {
+    keyValues: Scalar[];
+    states: VectorAggregateStates;
+}
+type VectorGroupBySnapshotValue = string | number | boolean | null | {
+    type: "bigint";
+    value: string;
+} | {
+    type: "timestamp";
+    epochNanoseconds: string;
+    unit: TimestampValue["unit"];
+    isAdjustedToUTC: boolean;
+};
+interface VectorGroupByGroupSnapshot {
+    keyValues: VectorGroupBySnapshotValue[];
+    states: VectorAggregateStateSnapshots;
+}
+interface VectorGroupByStateSnapshot {
+    groups: VectorGroupByGroupSnapshot[];
+}
+declare function createVectorGroupByState(keys: readonly string[], spec: AggregateSpec): VectorGroupByState;
+declare function updateVectorGroupByState(state: VectorGroupByState, batch: Batch, selection?: Selection, options?: VectorGroupByOptions): number;
+declare function getOrCreateVectorGroup(state: VectorGroupByState, batch: Batch, index: number, options?: VectorGroupByOptions): VectorGroup;
+declare function updateVectorGroupAggregateValue(group: VectorGroup, alias: string, value: VectorAggregateValue, options?: VectorAggregateOptions): void;
+declare function enforceVectorGroupByBudget(state: VectorGroupByState, budget?: QueryBudget): void;
+declare function finalizeVectorGroupByRows(state: VectorGroupByState): Row[];
+declare function finalizeVectorGroupByBatch(state: VectorGroupByState): Batch;
+declare function snapshotVectorGroupByState(state: VectorGroupByState): VectorGroupByStateSnapshot;
+declare function restoreVectorGroupByState(keys: readonly string[], spec: AggregateSpec, snapshot: VectorGroupByStateSnapshot, options?: VectorGroupByOptions): VectorGroupByState;
+declare function mergeVectorGroupByStates(target: VectorGroupByState, source: VectorGroupByState, options?: VectorGroupByOptions): void;
+declare function vectorGroupByBatch(keys: readonly string[], spec: AggregateSpec, batch: Batch, selection?: Selection, options?: VectorGroupByOptions): Batch;
+
+interface VectorHashJoinOptions {
+    leftKey: JoinKey;
+    rightKey: JoinKey;
+    maxRightRows: number;
+    type?: JoinType;
+    rightPrefix?: string;
+    leftSelection?: Selection;
+    rightSelection?: Selection;
+}
+declare function vectorHashJoin(left: Batch, right: Batch, options: VectorHashJoinOptions): Batch;
+
+type VectorProjectionSpec = Record<string, Expr>;
+declare function vectorProjectBatch(batch: Batch, select?: readonly string[], projections?: VectorProjectionSpec): Batch;
+
+interface VectorTopKOptions {
+    offset?: number;
+    limit: number;
+}
+declare function vectorOrderByBatch(batch: Batch, orderBy: readonly OrderByTerm[], selection?: Selection): Batch;
+declare function vectorTopKBatch(batch: Batch, orderBy: readonly OrderByTerm[], options: VectorTopKOptions, selection?: Selection): Batch;
+declare function vectorSortIndices(batch: Batch, orderBy: readonly OrderByTerm[], selection?: Selection): number[];
+declare function vectorTopKIndices(batch: Batch, orderBy: readonly OrderByTerm[], topK: number, selection?: Selection): number[];
+declare function gatherBatch(batch: Batch, indices: readonly number[]): Batch;
+declare function concatBatches(batches: readonly Batch[]): Batch;
+
+interface WorkUnitFanInOptions<Input, Partial, Accumulator> {
+    inputs: readonly Input[];
+    initial: Accumulator;
+    maxConcurrentTasks?: number;
+    maxBufferedPartials?: number;
+    run(input: Input, index: number): Promise<Partial>;
+    reduce(accumulator: Accumulator, partial: Partial, input: Input, index: number): void | Promise<void>;
+    boundary?(partial: Partial, input: Input, index: number): Partial | Promise<Partial>;
+}
+declare function fanInWorkUnits<Input, Partial, Accumulator>(options: WorkUnitFanInOptions<Input, Partial, Accumulator>): Promise<Accumulator>;
+declare function jsonWorkUnitBoundary<T>(value: T): T;
+
+declare const PACKAGE: "lakeql-iceberg";
 type IcebergReadMode = "strict" | "ignore-deletes" | "ignore-unsupported-deletes";
 interface LoadIcebergTableOptions extends ObjectStoreReadControls {
     store: ObjectStore;
@@ -1166,6 +1646,20 @@ declare function loadIcebergTableFromRest(options: LoadIcebergTableFromRestOptio
 declare function applyIcebergDeletes(options: ApplyIcebergDeletesOptions): Row[];
 declare function scanPlannedIcebergRows(options: ScanPlannedIcebergRowsOptions): AsyncIterable<Row[]>;
 
+declare class DecodedColumnCache {
+    private readonly cache;
+    private readonly options;
+    constructor(cache: SharedMemoryCache, options: ObjectStoreCacheOptions);
+    get(key: string): Batch | undefined;
+    set(key: string, batch: Batch): void;
+    getVector(key: string): Vector | undefined;
+    setVector(key: string, vector: Vector): boolean;
+    getValue<T>(key: string): T | undefined;
+    setValue<T>(key: string, value: T, bytes: number): void;
+    private shouldAdmitDecodedVector;
+    private shouldAdmitDecodedWorkProduct;
+}
+
 interface ReadParquetOptions {
     /** Columns to project; all columns when omitted. */
     columns?: string[];
@@ -1175,14 +1669,33 @@ interface ReadParquetOptions {
 interface ReadParquetBatchOptions extends ReadParquetOptions {
     batchSize?: number;
     where?: Expr;
+    stats?: QueryStats;
+    decodedColumnCache?: DecodedColumnCache;
+    decodedColumnCacheKey?: string;
 }
 interface ParquetRowBatch {
     rowOffset: number;
     rows: Row[];
 }
-interface PlanParquetRowGroupsOptions {
-    where?: Expr;
+interface ParquetColumnBatch {
+    rowOffset: number;
+    batch: Batch;
+    residualPredicateSatisfied?: boolean;
+}
+type ParquetMetadata = Awaited<ReturnType<typeof parquetMetadataAsync>>;
+interface StoreAsyncBuffer {
+    byteLength: number;
+    etag?: string;
+    slice(start: number, end?: number): Promise<ArrayBuffer>;
+}
+
+interface RangeCacheOptions {
+    maxBytes: number;
+    maxEntryBytes?: number;
+    sharedCache?: SharedMemoryCache;
+    cacheOptions?: ObjectStoreCacheOptions;
 }
+
 interface PlannedParquetRowGroup {
     index: number;
     rowStart: number;
@@ -1199,6 +1712,88 @@ interface ParquetRowGroupPlan {
         end: number;
     }[];
 }
+declare function planRowGroupsFromMetadata(metadata: ParquetMetadata, where: Expr | undefined): ParquetRowGroupPlan;
+
+interface ScanParquetTaskOptions {
+    batchSize?: number;
+    stats?: QueryStats;
+    metadataCache?: CacheAdapter<ParquetMetadata>;
+}
+interface PlanParquetTaskWorkUnitsOptions {
+    maxRowGroupsPerTask?: number;
+    maxRowsPerTask?: number;
+    metadataCache?: CacheAdapter<ParquetMetadata>;
+}
+interface AggregateParquetTaskOptions extends ScanParquetTaskOptions {
+    budget?: QueryBudget;
+}
+interface AggregateParquetTasksOptions extends AggregateParquetTaskOptions {
+    maxConcurrentTasks?: number;
+    maxBufferedPartials?: number;
+    preserveTaskBoundaries?: boolean;
+    partialBoundary?(partial: VectorAggregateStateSnapshots, task: TaskInput, index: number): VectorAggregateStateSnapshots | Promise<VectorAggregateStateSnapshots>;
+}
+interface AggregateParquetGroupTaskOptions extends AggregateParquetTaskOptions {
+    maxGroups?: number;
+}
+interface AggregateParquetGroupTasksOptions extends AggregateParquetGroupTaskOptions {
+    maxConcurrentTasks?: number;
+    maxBufferedPartials?: number;
+    preserveTaskBoundaries?: boolean;
+    orderBy?: OrderByTerm[];
+    limit?: number;
+    offset?: number;
+    partialBoundary?(partial: VectorGroupByStateSnapshot, task: TaskInput, index: number): VectorGroupByStateSnapshot | Promise<VectorGroupByStateSnapshot>;
+}
+
+declare function aggregateParquetTask(store: ObjectStore, task: TaskInput, spec: AggregateSpec, options?: AggregateParquetTaskOptions): Promise<VectorAggregateStateSnapshots>;
+declare function aggregateParquetTasks(store: ObjectStore, tasks: TaskInput[], spec: AggregateSpec, options?: AggregateParquetTasksOptions): Promise<Record<string, unknown>>;
+declare function aggregateParquetGroupTask(store: ObjectStore, task: TaskInput, groupColumns: readonly string[], spec: AggregateSpec, options?: AggregateParquetGroupTaskOptions): Promise<VectorGroupByStateSnapshot>;
+declare function aggregateParquetGroupTasks(store: ObjectStore, tasks: TaskInput[], groupColumns: readonly string[], spec: AggregateSpec, options?: AggregateParquetGroupTasksOptions): Promise<Row[]>;
+declare function aggregateParquetGroupTasksBatch(store: ObjectStore, tasks: TaskInput[], groupColumns: readonly string[], spec: AggregateSpec, options?: AggregateParquetGroupTasksOptions): Promise<Batch>;
+
+/** @internal Exposed for pruning tests; not part of the stable public API. */
+declare function rowGroupMayMatch(rowGroup: RowGroup, expr: Expr | undefined): boolean;
+/** @internal Exposed for scan/aggregate tests; not part of the stable public API. */
+declare function rowGroupMustMatch(rowGroup: RowGroup, expr: Expr | undefined): boolean;
+
+declare class ParquetScanAdapter implements ScanAdapter {
+    private readonly store;
+    private readonly defaultBatchSize;
+    private readonly metadataCache;
+    private readonly decodedColumnCache;
+    private readonly scanRangeCache;
+    constructor(store: ObjectStore, options?: {
+        batchSize?: number;
+        metadataCache?: CacheAdapter<ParquetMetadata>;
+        decodedColumnCache?: DecodedColumnCache;
+        scanRangeCache?: RangeCacheOptions;
+    });
+    scan(path: string, options: ScanOptions): AsyncIterable<Row[]>;
+    scanColumns(path: string, options: ScanOptions): AsyncIterable<Batch>;
+    scanColumnBatches(path: string, options: ScanOptions): AsyncIterable<ScanColumnBatch>;
+    scanVectorBatches(path: string, options: ScanOptions): AsyncIterable<ScanVectorBatch>;
+    planTask(path: string, options: ScanTaskPlanOptions): Promise<ScanTaskPlan>;
+    private metadata;
+    private scanBuffer;
+}
+
+declare function rejectUnsupportedParquetSchema(metadata: ParquetMetadata): void;
+
+/**
+ * Bridge an ObjectStore path to hyparquet's AsyncBuffer (length + ranged slice).
+ */
+declare function asyncBufferFromStore(store: ObjectStore, path: string, options?: ScanOptions | undefined): Promise<StoreAsyncBuffer>;
+
+declare function scanParquetTaskBatches(store: ObjectStore, task: TaskInput, options?: ScanParquetTaskOptions): AsyncIterable<Row[]>;
+declare function scanParquetTaskColumnBatches(store: ObjectStore, task: TaskInput, options?: ScanParquetTaskOptions): AsyncIterable<ParquetColumnBatch>;
+declare function planParquetTaskWorkUnits(store: ObjectStore, task: TaskInput, options: PlanParquetTaskWorkUnitsOptions): Promise<TaskInput[]>;
+/** Read Parquet footer metadata (row groups, schema, stats). */
+declare function readParquetMetadata(store: ObjectStore, path: string, metadataCache?: ScanParquetTaskOptions["metadataCache"]): Promise<ParquetMetadata>;
+
+interface PlanParquetRowGroupsOptions {
+    where?: Expr;
+}
 type IcebergParquetDeleteFileContent = "position-delete" | "equality-delete" | "deletion-vector";
 interface IcebergParquetDeleteFile {
     content: IcebergParquetDeleteFileContent;
@@ -1282,26 +1877,13 @@ interface PartitionedParquetOutputEntryOptions {
 }
 interface ParquetLakeConfig extends Omit<LakeConfig, "scanner"> {
     batchSize?: number;
+    cache?: ObjectStoreCacheOptions;
     metadataCache?: CacheAdapter<ParquetMetadata>;
+    scanRangeCache?: RangeCacheOptions;
 }
-type ParquetMetadata = Awaited<ReturnType<typeof parquetMetadataAsync>>;
-interface StoreAsyncBuffer {
-    byteLength: number;
-    etag?: string;
-    slice(start: number, end?: number): Promise<ArrayBuffer>;
-}
-/**
- * Bridge an ObjectStore path to hyparquet's AsyncBuffer (length + ranged slice).
- */
-declare function asyncBufferFromStore(store: ObjectStore, path: string, options?: ScanOptions | undefined): Promise<StoreAsyncBuffer>;
-/**
- * Read rows from a Parquet object. Early scaffold: full planner-driven
- * row-group pruning and batch streaming land in phase 1-2 (see BUILD_PLAN.md).
- */
 declare function readParquetObjects(store: ObjectStore, path: string, options?: ReadParquetOptions): Promise<Record<string, unknown>[]>;
 declare function readParquetObjectBatches(store: ObjectStore, path: string, options?: ReadParquetBatchOptions): AsyncIterable<ParquetRowBatch>;
-/** Read Parquet footer metadata (row groups, schema, stats). */
-declare function readParquetMetadata(store: ObjectStore, path: string): Promise<hyparquet_src_types_js.FileMetaData>;
+declare function readParquetColumnBatches(store: ObjectStore, path: string, options?: ReadParquetBatchOptions): AsyncIterable<ParquetColumnBatch>;
 declare function planRowGroups(store: ObjectStore, path: string, options?: PlanParquetRowGroupsOptions): Promise<ParquetRowGroupPlan>;
 declare function readIcebergParquetDeletes(store: ObjectStore, deleteFile: IcebergParquetDeleteFile): Promise<DecodedIcebergParquetDeletes>;
 declare function writeParquet(store: ObjectStore, path: string, options: WriteParquetOptions): Promise<{
@@ -1313,27 +1895,12 @@ declare function writePartitionedParquet(store: ObjectStore, prefix: string, opt
 declare function writePartitionedParquetTask(store: ObjectStore, prefix: string, options: WritePartitionedParquetTaskOptions): Promise<WritePartitionedParquetTaskResult>;
 declare function createParquetTableAs(store: ObjectStore, prefix: string, options: CreateParquetTableAsOptions): Promise<CreateParquetTableAsResult>;
 declare function partitionedParquetOutputEntries(result: WritePartitionedParquetResult, options: PartitionedParquetOutputEntryOptions): OutputManifestEntry[];
-declare class ParquetScanAdapter implements ScanAdapter {
-    private readonly store;
-    private readonly defaultBatchSize;
-    private readonly metadataCache;
-    constructor(store: ObjectStore, options?: {
-        batchSize?: number;
-        metadataCache?: CacheAdapter<ParquetMetadata>;
-    });
-    scan(path: string, options: ScanOptions): AsyncIterable<Row[]>;
-    planTask(path: string, options: ScanTaskPlanOptions): Promise<ScanTaskPlan>;
-    private metadata;
-}
-declare function planRowGroupsFromMetadata(metadata: ParquetMetadata, where: Expr | undefined): ParquetRowGroupPlan;
-declare function rejectUnsupportedParquetSchema(metadata: ParquetMetadata): void;
 type InsertValue = string | number | boolean | bigint | null;
 type ComparableInsertValue = string | number | bigint;
-/** @internal Exposed for pruning tests; not part of the stable public API. */
-declare function rowGroupMayMatch(rowGroup: RowGroup, expr: Expr | undefined): boolean;
 declare function parquetScanner(store: ObjectStore, options?: {
     batchSize?: number;
     metadataCache?: CacheAdapter<ParquetMetadata>;
+    decodedColumnCache?: DecodedColumnCache;
 }): ScanAdapter;
 declare function createParquetLake(config: ParquetLakeConfig): Lake;
 
@@ -1383,4 +1950,4 @@ declare function planFiles(table: EngineTable, options?: PlanIcebergFilesOptions
 declare function scanBatches(plan: EngineFilePlan, options?: ScanEngineOptions): AsyncIterable<ScanBatch>;
 declare function scanRows(plan: EngineFilePlan, options?: ScanEngineOptions): AsyncIterable<Row>;
 
-export { type AggregateExpr, type AggregateGroupSnapshot, type AggregateOp, type AggregateOperatorState, type AggregateOptions, type AggregateResult, type AggregateSnapshotValue, type AggregateSpec, type AggregateStateSnapshot, AggregationBuilder, type ApplyIcebergDeletesOptions, type BBoxIndex, type BetweenExpr, type Bookmark, type BookmarkInit, type BookmarkPosition, type BookmarkQuery, type BroadcastJoinOptions, type CacheAdapter, CacheApiCache, type CacheApiCacheOptions, type CacheEntry, type CallExpr, type CheckpointAdapter, type CheckpointStore, type Clock, type ColumnExpr, type ColumnInput, type CompareExpr, type CompareOp, type ConditionalObjectStore, type ConditionalPutOptions, type CreateParquetTableAsOptions, type CreateParquetTableAsQuery, type CreateParquetTableAsResult, type CsvStreamOptions, type DecodedIcebergDeletes, type DecodedIcebergParquetDeletes, ERROR_CODES, type EngineFilePlan, type EngineTable, type ErrorDetails, type ExplainJson, type ExplainResult, type Expr, type IcebergAppendFile, type IcebergAppendOptions, type IcebergAppendOutputManifestOptions, type IcebergAppendResult, type IcebergCatalog, type IcebergCommitCatalog, type IcebergCommitInput, type IcebergCommitResult, type IcebergDeleteFile, type IcebergDeletionVector, type IcebergEnginePlan, type IcebergEngineTable, type IcebergEqualityDelete, type IcebergField, type IcebergGlueCatalogOptions, type IcebergNessieCatalogOptions, type IcebergParquetDeleteFile, type IcebergParquetDeleteFileContent, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPlan, type IcebergPositionDelete, type IcebergReadMode, IcebergRestCatalog, type IcebergRestCatalogOptions, type IcebergRowBatch, type IcebergSortOrder, IcebergTable, type IcebergTableIdentifier, IcebergUnsupportedCatalog, type IdGenerator, type InExpr, type IndexPruneResult, type IndexValue, type InsertValidationRules, type JoinType, type JsonExpr, type JsonOrderByTerm, type JsonQueryV1, LaQLError, type LaQLErrorCode, Lake, type LakeConfig, type LikeExpr, type ListOptions, type LiteralExpr, type LoadIcebergEngineTableOptions, type LoadIcebergTableFromObjectStoreOptions, type LoadIcebergTableFromRestOptions, type LoadIcebergTableOptions, type LoadParquetEngineTableOptions, type LoadTableOptions, type LockAdapter, type LogHook, type LogicalExpr, type LookupJoinFunction, type LookupJoinOptions, type Manifest, type ManifestDeleteFile, type ManifestFile, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, type MemorySpillAdapterOptions, type MetadataFile, type MetricsHook, type MinMaxColumnIndex, type NotExpr, type NullCheckExpr, type ObjectHead, type ObjectInfo, type ObjectStore, ObjectStoreIcebergCommitCatalog, type ObjectStoreReadControls, type OperatorSnapshotValue, type OrderByTerm, type OutputManifest, type OutputManifestEntry, PACKAGE, type ParquetEnginePlan, type ParquetEngineTable, type ParquetLakeConfig, type ParquetMetadata, type ParquetRowBatch, type ParquetRowGroupPlan, ParquetScanAdapter, type PartitionedParquetOutputEntryOptions, type PathQueryInit, type PlanIcebergFilesOptions, type PlanParquetRowGroupsOptions, type PlannedIcebergFile, type PlannedParquetRowGroup, type PredicatePlan, type PredicatePlanOptions, type ProjectIcebergRowOptions, type PutOptions, type QueryBudget, QueryBuilder, type QueryPolicy, type QueryPolicyContext, QueryResult, type QueryRunOptions, type QueryStats, type QueueAdapter, type ReadParquetBatchOptions, type ReadParquetOptions, type ResumableBatchOptions, ResumedQuery, type Row, type RuntimeSubstrate, type Scalar, type ScanAdapter, type ScanBatch, type ScanEngineOptions, type ScanOptions, type ScanPlannedIcebergRowsOptions, type ScanTaskPlan, type ScanTaskPlanOptions, type SidecarFileIndex, type SliceOptions, type SliceResult, type Snapshot, type SortOperatorState, type SortOptions, type SortResult, type SortRunState, type SpillAdapter, type SpillRef, type SpillUsage, type SqlBoolean, type TaskCheckpoint, type TaskInput, type TaskManifest, type TaskManifestTask, type TaskState, type TaskTransitionInput, type TopKOperatorState, type TopKOptions, type TopKResult, type ValueInput, type WriteParquetOptions, type WriteParquetRowsOptions, type WritePartitionedParquetFile, type WritePartitionedParquetResult, type WritePartitionedParquetTaskOptions, type WritePartitionedParquetTaskResult, advanceTaskCheckpoint, and, applyIcebergDeletes, assertBookmarkMatches, asyncBufferFromStore, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, classifyPredicate, col, createBookmark, createParquetLake as createLake, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, encodeJsonLine, eq, evaluate, fingerprint, fn, gt, gte, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, ilike, isIn, isLaQLError, isNotNull, isNull, jsonSafeValue, like, lit, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, lookupJoin, lt, lte, matches, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, ne, not, notIn, or, parquetScanner, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planFiles, planRowGroups, planRowGroupsFromMetadata, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, rowGroupMayMatch, scanBatches, scanPlannedIcebergRows, scanRows, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, signPaginationToken, stableStringify, throwIfAborted, transitionTaskCheckpoint, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask };
+export { type AggregateExpr, type AggregateGroupSnapshot, type AggregateOp, type AggregateOperatorState, type AggregateOptions, type AggregateParquetGroupTaskOptions, type AggregateParquetGroupTasksOptions, type AggregateParquetTaskOptions, type AggregateParquetTasksOptions, type AggregateResult, type AggregateSnapshotValue, type AggregateSpec, type AggregateStateSnapshot, AggregationBuilder, type ApplyIcebergDeletesOptions, type ArithmeticExpr, type BBox, type BBoxIndex, type Batch, type BatchExprValues, type BetweenExpr, type Bookmark, type BookmarkInit, type BookmarkPosition, type BookmarkQuery, type BroadcastJoinOptions, type CacheAdapter, CacheApiCache, type CacheApiCacheOptions, type CacheEntry, type CachePolicy, type CallExpr, type CaseExpr, type CaseWhenExpr, type CheckpointAdapter, type CheckpointStore, type Clock, type ColumnExpr, type ColumnInput, type CompareExpr, type CompareOp, type ConditionalObjectStore, type ConditionalPutOptions, type CreateParquetTableAsOptions, type CreateParquetTableAsQuery, type CreateParquetTableAsResult, type CsvStreamOptions, type DecodedIcebergDeletes, type DecodedIcebergParquetDeletes, ERROR_CODES, type EngineFilePlan, type EngineTable, type ErrorDetails, type ExplainJson, type ExplainResult, type Expr, type GeoBackend, type GeoJsonGeometry, type IcebergAppendFile, type IcebergAppendOptions, type IcebergAppendOutputManifestOptions, type IcebergAppendResult, type IcebergCatalog, type IcebergCommitCatalog, type IcebergCommitInput, type IcebergCommitResult, type IcebergDeleteFile, type IcebergDeletionVector, type IcebergEnginePlan, type IcebergEngineTable, type IcebergEqualityDelete, type IcebergField, type IcebergGlueCatalogOptions, type IcebergNessieCatalogOptions, type IcebergParquetDeleteFile, type IcebergParquetDeleteFileContent, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPlan, type IcebergPositionDelete, type IcebergReadMode, IcebergRestCatalog, type IcebergRestCatalogOptions, type IcebergRowBatch, type IcebergSortOrder, IcebergTable, type IcebergTableIdentifier, IcebergUnsupportedCatalog, type IdGenerator, type InExpr, type InMemoryLakeOptions, InMemoryRowsScanner, InMemoryRowsStore, type InMemoryTableOptions, type IndexPruneResult, type IndexValue, type InsertValidationRules, type JoinKey, type JoinType, type JsonExpr, type JsonOrderByTerm, type JsonQueryV1, Lake, type LakeConfig, LakeqlError, type LakeqlErrorCode, type LikeExpr, type ListOptions, type LiteralExpr, type LoadIcebergEngineTableOptions, type LoadIcebergTableFromObjectStoreOptions, type LoadIcebergTableFromRestOptions, type LoadIcebergTableOptions, type LoadParquetEngineTableOptions, type LoadTableOptions, type LockAdapter, type LogHook, type LogicalExpr, type LookupJoinFunction, type LookupJoinOptions, type Manifest, type ManifestDeleteFile, type ManifestFile, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, type MemorySpillAdapterOptions, type MetadataFile, type MetricsHook, type MinMaxColumnIndex, type NotExpr, type NullCheckExpr, type ObjectHead, type ObjectInfo, type ObjectStore, type ObjectStoreCacheOptions, ObjectStoreIcebergCommitCatalog, type ObjectStoreReadControls, type OperatorSnapshotValue, type OrderByTerm, type OutputManifest, type OutputManifestEntry, PACKAGE, type ParquetColumnBatch, type ParquetEnginePlan, type ParquetEngineTable, type ParquetLakeConfig, type ParquetMetadata, type ParquetRowBatch, type ParquetRowGroupPlan, ParquetScanAdapter, type PartitionedParquetOutputEntryOptions, type PathQueryInit, type PlanIcebergFilesOptions, type PlanParquetRowGroupsOptions, type PlanParquetTaskWorkUnitsOptions, type PlannedIcebergFile, type PlannedParquetRowGroup, type PredicatePlan, type PredicatePlanOptions, type ProjectIcebergRowOptions, type PutOptions, type QueryBudget, QueryBuilder, type QueryPolicy, type QueryPolicyContext, QueryResult, type QueryRunOptions, type QueryStats, type QueueAdapter, type ReadParquetBatchOptions, type ReadParquetOptions, type ResumableBatchOptions, ResumedQuery, type Row, type RuntimeSubstrate, type Scalar, type ScanAdapter, type ScanBatch, type ScanColumnBatch, type ScanEngineOptions, type ScanOptions, type ScanParquetTaskOptions, type ScanPlannedIcebergRowsOptions, type ScanTaskPlan, type ScanTaskPlanOptions, type ScanVectorBatch, type Selection, type SharedCacheEntry, type SharedCacheSetOptions, SharedMemoryCache, type SidecarFileIndex, type SliceOptions, type SliceResult, type Snapshot, type SortOperatorState, type SortOptions, type SortResult, type SortRunState, type SpillAdapter, type SpillRef, type SpillUsage, type SqlBoolean, type TaskCheckpoint, type TaskInput, type TaskManifest, type TaskManifestTask, type TaskState, type TaskTransitionInput, type TimestampUnit, TimestampValue, type TopKOperatorState, type TopKOptions, type TopKResult, type ValueInput, type Vector, type VectorAggregateOptions, type VectorAggregateSnapshotValue, type VectorAggregateState, type VectorAggregateStateSnapshot, type VectorAggregateStateSnapshots, type VectorAggregateStates, type VectorAggregateValue, type VectorGroup, type VectorGroupByGroupSnapshot, type VectorGroupByOptions, type VectorGroupBySnapshotValue, type VectorGroupByState, type VectorGroupByStateSnapshot, type VectorHashJoinOptions, type VectorProjectionSpec, type VectorTopKOptions, type WorkUnitFanInOptions, type WriteParquetOptions, type WriteParquetRowsOptions, type WritePartitionedParquetFile, type WritePartitionedParquetResult, type WritePartitionedParquetTaskOptions, type WritePartitionedParquetTaskResult, add, advanceTaskCheckpoint, aggregateParquetGroupTask, aggregateParquetGroupTasks, aggregateParquetGroupTasksBatch, aggregateParquetTask, aggregateParquetTasks, and, applyIcebergDeletes, assertBookmarkMatches, asyncBufferFromStore, batchExprValues, batchFromColumns, batchFromVectors, bboxIntersects, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, cachedObjectStore, classifyPredicate, col, compareTimestampValues, concatBatches, createBookmark, createInMemoryLake, createParquetLake as createLake, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, createVectorAggregateStates, createVectorGroupByState, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, div, encodeJsonLine, enforceVectorGroupByBudget, ensureGeoBackendForExprs, envelopeFromGeometry, envelopeOf, eq, evaluate, fanInWorkUnits, finalizeVectorAggregateStates, finalizeVectorGroupByBatch, finalizeVectorGroupByRows, fingerprint, fn, gatherBatch, getOrCreateVectorGroup, gt, gte, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, ilike, inMemoryRowsScanner, isIn, isLakeqlError, isNotNull, isNull, isTimestampValue, jsonSafeValue, jsonWorkUnitBoundary, like, lit, loadGeoBackend, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, lookupJoin, lt, lte, matches, materializeBatchRows, materializeSelectedBatchRows, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mergeVectorAggregateStateSnapshots, mergeVectorAggregateStates, mergeVectorGroupByStates, mod, mul, ne, not, notIn, or, parquetScanner, parseGeometry, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planFiles, planParquetTaskWorkUnits, planRowGroups, planRowGroupsFromMetadata, predicateSelection, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetColumnBatches, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, requireGeoBackend, restoreVectorAggregateStates, restoreVectorGroupByState, rowGroupMayMatch, rowGroupMustMatch, scalarVectorValue, scanBatches, scanParquetTaskBatches, scanParquetTaskColumnBatches, scanPlannedIcebergRows, scanRows, selectedRowCount, selectedRowIndices, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, setGeoBackend, signPaginationToken, snapshotVectorAggregateStates, snapshotVectorGroupByState, stableStringify, sub, throwIfAborted, timestampEpochForUnit, timestampFromEpoch, timestampToIsoString, timestampValueFromIso, toGeometry, transitionTaskCheckpoint, tryPredicateSelection, updateVectorAggregateStateValue, updateVectorAggregateStates, updateVectorGroupAggregateValue, updateVectorGroupByState, vectorAggregateBatch, vectorFromValues, vectorGroupByBatch, vectorHashJoin, vectorLength, vectorOrderByBatch, vectorProjectBatch, vectorSortIndices, vectorTopKBatch, vectorTopKIndices, vectorValue, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask };