@slexisvn/query-engine 0.1.0

package/dist/execution/operators/merge-join.d.ts

@@ -0,0 +1,38 @@
+import { DataChunk } from '../../storage/chunk.js';
+import { JoinType } from '../../planner/logical-plan.js';
+import { type ColumnValue } from '../../storage/data-type.js';
+import type { CompiledExpr, EvalValue } from '../execution-types.js';
+type JoinKey = EvalValue | EvalValue[];
+interface JoinRow {
+    chunk: DataChunk;
+    idx: number;
+    key: JoinKey;
+}
+interface RowAdapterColumn {
+    get(): ColumnValue;
+}
+interface RowAdapter {
+    row: ColumnValue[] | null;
+    columns: RowAdapterColumn[];
+    setRow(r: ColumnValue[]): void;
+}
+export declare class MergeJoinOperator {
+    buildChunks: DataChunk[];
+    probeChunks: DataChunk[];
+    buildKeyExtractors: CompiledExpr[];
+    probeKeyExtractors: CompiledExpr[];
+    buildColCount: number;
+    probeColCount: number;
+    joinType: JoinType;
+    conditionEvaluator: CompiledExpr | null;
+    constructor(buildChunks: DataChunk[], probeChunks: DataChunk[], buildKeyExtractors: CompiledExpr[], probeKeyExtractors: CompiledExpr[], buildColCount: number, probeColCount: number, joinType?: JoinType, conditionEvaluator?: CompiledExpr | null);
+    execute(): Promise<DataChunk[]>;
+    _flattenAndExtractKeys(chunks: DataChunk[], extractors: CompiledExpr[]): JoinRow[];
+    _compareKeys(k1: JoinKey, k2: JoinKey): number;
+    _combineRow(bRow: JoinRow, pRow: JoinRow): ColumnValue[];
+    _extractProbeRow(rowObj: JoinRow): ColumnValue[];
+    _combineRowWithNulls(rowObj: JoinRow, isBuild: boolean): ColumnValue[];
+    _buildOutputChunk(outputRows: ColumnValue[][]): DataChunk | null;
+    createAdapter(): RowAdapter;
+}
+export {};

package/dist/execution/operators/nested-loop-join.d.ts

@@ -0,0 +1,28 @@
+import { DataChunk } from '../../storage/chunk.js';
+import type { ColumnValue } from '../../storage/data-type.js';
+import { JoinType } from '../../planner/logical-plan.js';
+import type { CompiledExpr } from '../execution-types.js';
+type JoinRow = ColumnValue[];
+interface RowAdapter {
+    chunk: DataChunk;
+    setRow(r: JoinRow): void;
+}
+export declare class NestedLoopJoinOperator {
+    outerChunks: DataChunk[];
+    innerChunks: DataChunk[];
+    outerColCount: number;
+    innerColCount: number;
+    joinType: JoinType;
+    conditionEvaluator: CompiledExpr | null;
+    constructor(outerChunks: DataChunk[], innerChunks: DataChunk[], outerColCount: number, innerColCount: number, joinType?: JoinType, conditionEvaluator?: CompiledExpr | null);
+    execute(): Promise<DataChunk[]>;
+    _flattenChunks(chunks: DataChunk[]): JoinRow[];
+    _combineRow(outerRow: JoinRow, innerRow: JoinRow): JoinRow;
+    _extractOuter(outerRow: JoinRow): JoinRow;
+    _outerWithNulls(outerRow: JoinRow): JoinRow;
+    _innerWithNulls(innerRow: JoinRow): JoinRow;
+    _outerWithMark(outerRow: JoinRow, markValue: boolean): JoinRow;
+    _buildOutputChunk(rows: JoinRow[]): DataChunk;
+    _createAdapter(): RowAdapter;
+}
+export {};

package/dist/execution/operators/projection.d.ts

@@ -0,0 +1,20 @@
+import { DataType } from '../../storage/data-type.js';
+import { DataChunk } from '../../storage/chunk.js';
+import type { BoundExpr, BoundColumnRefNode } from '../../binder/expression-binder.js';
+import type { CompiledExpr, ColumnMapping } from '../execution-types.js';
+interface ParallelDispatchLike {
+    canParallelize(op: string, dt: DataType, n: number): boolean;
+}
+export declare class ProjectionOperator {
+    expressions: BoundExpr[];
+    evaluators: CompiledExpr[];
+    resultTypes: DataType[] | null;
+    columnMapping: ColumnMapping | null;
+    parallelDispatch: ParallelDispatchLike | null;
+    colRefIndices: number[];
+    constructor(expressions: BoundExpr[], evaluators: CompiledExpr[], resultTypes: DataType[] | null | undefined, columnMapping: (ColumnMapping | null) | undefined, parallelDispatch: ParallelDispatchLike | null);
+    init(): Promise<void>;
+    process(chunk: DataChunk): Promise<DataChunk>;
+    _resolveColIdx(expr: BoundColumnRefNode): number;
+}
+export {};

package/dist/execution/operators/scan.d.ts

@@ -0,0 +1,17 @@
+import type { DataChunk } from '../../storage/chunk.js';
+import type { ExecSchema } from '../execution-types.js';
+interface TableStorageLike {
+    getSchema(): ExecSchema;
+    scan(): AsyncGenerator<DataChunk>;
+    scanAll(): Promise<DataChunk[]>;
+    rowCount(): number;
+}
+export declare class ScanOperator {
+    table: TableStorageLike;
+    projectedColumns: number[] | null;
+    constructor(table: TableStorageLike, projectedColumns: number[] | null);
+    init(): Promise<void>;
+    scan(): AsyncGenerator<DataChunk>;
+    estimatedRows(): number;
+}
+export {};

package/dist/execution/operators/sort.d.ts

@@ -0,0 +1,48 @@
+import { DataChunk } from '../../storage/chunk.js';
+import type { ColumnValue, DataType } from '../../storage/data-type.js';
+import type { CompiledExpr } from '../execution-types.js';
+interface SortKey {
+    eval: CompiledExpr;
+    direction: string;
+}
+interface SortRow {
+    row: ColumnValue[];
+    sortKeys: ColumnValue[];
+}
+interface SpillManagerLike {
+    appendChunk(id: string, chunk: DataChunk | null): Promise<void>;
+    readChunks(id: string): AsyncGenerator<DataChunk>;
+    clearAll(): Promise<void>;
+}
+export declare class SortOperator {
+    keyExtractors: SortKey[];
+    limit: number | null;
+    offset: number;
+    topN: number | null;
+    rows: SortRow[];
+    schema: DataType[] | null;
+    spillManager: SpillManagerLike;
+    runCount: number;
+    constructor(keyExtractors: SortKey[], limit: number | null, offset: number, spillManager: SpillManagerLike);
+    init(): Promise<void>;
+    consume(chunk: DataChunk): Promise<void>;
+    spillCurrentRun(): Promise<void>;
+    finalize(): Promise<DataChunk[]>;
+    chunkToItems(chunk: DataChunk): SortRow[];
+    rowsToChunk(items: SortRow[]): DataChunk;
+    compareRows(a: SortRow, b: SortRow): number;
+}
+export declare class LimitOperator {
+    limit: number;
+    offset: number;
+    seen: number;
+    emitted: number;
+    chunks: DataChunk[];
+    schema: DataType[] | null;
+    done: boolean;
+    constructor(limit: number, offset?: number);
+    init(): Promise<void>;
+    consume(chunk: DataChunk): Promise<void>;
+    finalize(): Promise<DataChunk[]>;
+}
+export {};

package/dist/execution/operators/stream-aggregate.d.ts

@@ -0,0 +1,30 @@
+import { DataChunk } from '../../storage/chunk.js';
+import { DataType, type ColumnValue } from '../../storage/data-type.js';
+import type { BoundExpr } from '../../binder/expression-binder.js';
+import type { CompiledExpr, ColumnMapping, EvalValue } from '../execution-types.js';
+interface Accumulator {
+    add(val: EvalValue): void;
+    result(): ColumnValue;
+}
+interface AggregateDef {
+    name: string;
+    valueKey: string | null;
+    resultType: DataType;
+    createAccumulator: () => Accumulator;
+    extractValue: CompiledExpr;
+    _wasmColIndex?: number;
+    _sourceExpr: BoundExpr | null;
+    _columnMapping: ColumnMapping;
+}
+export declare class StreamAggregateOperator {
+    groupByExtractors: CompiledExpr[];
+    groupByTypes: DataType[];
+    aggregateDefs: AggregateDef[];
+    hasCachedValues: boolean;
+    constructor(groupByExtractors: CompiledExpr[], groupByTypes: DataType[], aggregateDefs: AggregateDef[]);
+    init(): Promise<void>;
+    _tryWasmUngrouped(chunks: DataChunk[]): Promise<DataChunk[] | null>;
+    execute(chunks: DataChunk[]): Promise<DataChunk[]>;
+    extractGroupKey(chunk: DataChunk, rowIdx: number): string;
+}
+export {};

package/dist/execution/operators/union.d.ts

@@ -0,0 +1,13 @@
+import { DataChunk } from '../../storage/chunk.js';
+import type { DataType } from '../../storage/data-type.js';
+export declare class UnionOperator {
+    isAll: boolean;
+    seen: Set<string> | null;
+    schema: DataType[] | null;
+    _legacyChunks: DataChunk[];
+    constructor(isAll: boolean);
+    init(): Promise<void>;
+    process(chunk: DataChunk): Promise<DataChunk>;
+    consume(chunk: DataChunk): Promise<void>;
+    finalize(): Promise<DataChunk[]>;
+}

package/dist/execution/operators/vector-aggregate.d.ts

@@ -0,0 +1,73 @@
+import { DictionaryColumn } from '../../storage/dictionary-column.js';
+import { Column } from '../../storage/column.js';
+import { DataType } from '../../storage/data-type.js';
+import { DataChunk } from '../../storage/chunk.js';
+import type { BoundExpr } from '../../binder/expression-binder.js';
+import { StageKind } from '../fragment-spec.js';
+declare enum KeyKind {
+    INT = "int",
+    DICT = "dict"
+}
+interface AggDescriptor {
+    op: number;
+    colIdx: number;
+}
+interface AggState {
+    sums?: Float64Array;
+    counts?: Float64Array;
+    vals?: Float64Array;
+    has?: Uint8Array;
+}
+interface AvgState {
+    sum: number;
+    count: number;
+}
+type SlotState = number | AvgState | null;
+interface VectorSpecSchemaColumn {
+    name: string;
+    dataType: DataType;
+    tableAlias?: string;
+}
+interface VectorSpecAggregate {
+    name: string;
+    distinct: boolean;
+    args: BoundExpr[];
+}
+interface VectorSpecStage {
+    kind: StageKind;
+}
+interface VectorSpec {
+    groupBy: BoundExpr[];
+    stages: VectorSpecStage[];
+    baseSchema: VectorSpecSchemaColumn[];
+    aggregates: VectorSpecAggregate[];
+}
+interface PartialGroup {
+    key: number | string | null;
+    groupValues: (number | string | null)[];
+    states: SlotState[];
+}
+export declare function createVectorAggregator(spec: VectorSpec): VectorGroupAggregator | null;
+export declare class VectorGroupAggregator {
+    keyKind: KeyKind;
+    groupIdx: number;
+    aggs: AggDescriptor[];
+    capacity: number;
+    used: number;
+    slotKeys: (number | string | null)[];
+    nullSlot: number;
+    stringSlots: Map<string, number> | null;
+    denseBase: number;
+    dense: Int32Array;
+    state: AggState[];
+    constructor(keyKind: KeyKind, groupIdx: number, aggs: AggDescriptor[]);
+    clear(): void;
+    get groupCount(): number;
+    _allocSlot(key: number | string | null): number;
+    _nullSlot(): number;
+    _dictRemap(col: DictionaryColumn): Int32Array;
+    _ensureDenseRange(chunk: DataChunk, col: Column): boolean;
+    consume(chunk: DataChunk): boolean;
+    exportPartials(partitionCount: number): PartialGroup[][];
+}
+export {};

package/dist/execution/operators/window.d.ts

@@ -0,0 +1,22 @@
+import { DataChunk } from '../../storage/chunk.js';
+import type { ColumnValue } from '../../storage/data-type.js';
+import type { BoundExpr, BoundWindowNode } from '../../binder/expression-binder.js';
+import type { CompiledExpr, ColumnMapping, ExecSchema, EvalValue } from '../execution-types.js';
+type CompileExpressionFn = (expr: BoundExpr, mapping: ColumnMapping) => CompiledExpr;
+interface OrderKey {
+    eval: CompiledExpr;
+    direction: string;
+}
+export declare class WindowOperator {
+    windowExprs: BoundWindowNode[];
+    childSchema: ExecSchema;
+    childColumnMapping: ColumnMapping;
+    compileExpression: CompileExpressionFn;
+    constructor(windowExprs: BoundWindowNode[], childSchema: ExecSchema, childColumnMapping: ColumnMapping, compileExpressionFn: CompileExpressionFn);
+    execute(chunks: DataChunk[]): Promise<DataChunk[]>;
+    computeWindow(wExpr: BoundWindowNode, allRows: ColumnValue[][], chunks: DataChunk[]): EvalValue[];
+    partitionRows(allRows: ColumnValue[][], partitionEvals: CompiledExpr[], chunks: DataChunk[]): number[][];
+    sameOrderKey(idxA: number, idxB: number, orderKeys: OrderKey[], chunks: DataChunk[]): boolean;
+    compareValues(a: EvalValue, b: EvalValue): number;
+}
+export {};

package/dist/execution/physical-plan.d.ts

@@ -0,0 +1,14 @@
+export declare enum PhysicalNodeType {
+    TABLE_SCAN = "TableScan",
+    FILTER = "Filter",
+    PROJECT = "Project",
+    HASH_JOIN = "HashJoin",
+    HASH_AGGREGATE = "HashAggregate",
+    SORT = "Sort",
+    LIMIT = "Limit",
+    TOP_N = "TopN",
+    DISTINCT = "Distinct",
+    UNION = "Union",
+    MATERIALIZE = "Materialize",
+    RESULT_SINK = "ResultSink"
+}

package/dist/execution/pipeline.d.ts

@@ -0,0 +1,28 @@
+import type { Sink, SourceGenerator } from './execution-types.js';
+export interface Pipeline {
+    id: number;
+    sink: Sink;
+    source: SourceGenerator | null;
+    dependencies: Set<number>;
+    dependents: Set<number>;
+    state: 'PENDING' | 'RUNNING' | 'DONE';
+    cancelled: boolean;
+}
+export declare class PipelineGraph {
+    pipelines: Map<number, Pipeline>;
+    nextId: number;
+    constructor();
+    createPipeline(sink: Sink): number;
+    addDependency(pipelineId: number, dependsOnId: number): void;
+    setSource(pipelineId: number, sourceGenerator: SourceGenerator): void;
+    getReadyPipelines(): Pipeline[];
+    markPipelineDone(pipelineId: number): void;
+    cancelPipeline(pipelineId: number): void;
+    isCancelled(pipelineId: number): boolean;
+}
+export declare class CancelToken {
+    cancelled: boolean;
+    constructor();
+    cancel(): void;
+    get isCancelled(): boolean;
+}

package/dist/execution/query-executor.d.ts

@@ -0,0 +1,70 @@
+import { type LogicalPlanNode, type LogicalJoinNode } from '../planner/logical-plan.js';
+import { ResultSink } from './result-sink.js';
+import type { ParallelJoinPrep, MakeBuildSide, MakeProbeOp, FragmentPoolLike as JoinFragmentPoolLike } from './builders/join-builder.js';
+import type { FragmentPoolLike as AggFragmentPoolLike } from './builders/aggregate-builder.js';
+import type { DataChunk } from '../storage/chunk.js';
+import type { DataType } from '../storage/data-type.js';
+import type { BoundAggregateNode, BoundExpr } from '../binder/expression-binder.js';
+import type { CompiledPipeline, ExecSchema, ExecColumn, ColumnMapping } from './execution-types.js';
+interface TableStorageLike {
+    rowCount(): number;
+    scan(): AsyncGenerator<DataChunk>;
+    getSchema(): ExecSchema;
+}
+type BTreeLike = object;
+interface CatalogLike {
+    getTableStorage(name: string): TableStorageLike | null;
+    getIndexForColumn(table: string, column: string): BTreeLike | null;
+}
+interface TempManagerLike {
+    allocate(category: string, label: string): string;
+}
+interface SpillManagerLike {
+    appendChunk(id: string, chunk: DataChunk | null): Promise<void>;
+    readChunks(id: string): AsyncGenerator<DataChunk>;
+    clearAll(): Promise<void>;
+}
+interface StorageBackendLike {
+    createTempSpace(): object;
+    createPageStore(): object;
+    createSpillManager(handle: string): SpillManagerLike;
+}
+type WorkerPoolLike = object;
+type ParallelDispatchLike = object;
+type FragmentPoolLike = JoinFragmentPoolLike & AggFragmentPoolLike;
+type DistributedContextLike = object;
+interface ExecuteResult {
+    sink: ResultSink;
+    columnNames: string[];
+}
+export declare class QueryExecutor {
+    catalog: CatalogLike;
+    tempManager: TempManagerLike;
+    storageBackend: StorageBackendLike;
+    cteResults: Map<string, DataChunk[]>;
+    cteDefinitions: Map<string, LogicalPlanNode>;
+    workerPool: WorkerPoolLike | null;
+    parallelDispatch: ParallelDispatchLike | null;
+    fragmentPool: FragmentPoolLike | null;
+    _distributedContext: DistributedContextLike | null;
+    constructor(catalog: CatalogLike, tempManager: TempManagerLike, storageBackend?: StorageBackendLike | null);
+    setParallelContext(workerPool: WorkerPoolLike, parallelDispatch: ParallelDispatchLike, fragmentPool?: FragmentPoolLike | null): void;
+    setDistributedContext(ctx: DistributedContextLike): void;
+    _shouldParallelize(storage: TableStorageLike): boolean;
+    execute(logicalPlan: LogicalPlanNode, outputColumns: ExecColumn[], streaming?: boolean): Promise<ExecuteResult>;
+    executeStreaming(logicalPlan: LogicalPlanNode, outputColumns: ExecColumn[]): Promise<ExecuteResult>;
+    executePlan(logicalPlan: LogicalPlanNode, streaming?: boolean): Promise<ResultSink>;
+    buildPipeline(node: LogicalPlanNode): Promise<CompiledPipeline>;
+    resolveProjectedColumnIndexes(storageSchema: ExecSchema, planColumns: ExecColumn[] | null): number[] | null;
+    buildSchemaMapping(schema: ExecSchema, alias: string): ColumnMapping;
+    findCTEPlan(cteName: string): LogicalPlanNode | null;
+    _estimatePlanRows(planNode: LogicalPlanNode): number;
+    _executeSubPipeline(compiled: CompiledPipeline): Promise<DataChunk[]>;
+    _prepareParallelJoin(node: LogicalJoinNode, buildInput: CompiledPipeline, probeInput: CompiledPipeline, buildNode: LogicalPlanNode, probeNode: LogicalPlanNode, buildKeys: BoundExpr[], probeKeys: BoundExpr[], residualCondition: BoundExpr | null, combinedMapping: ColumnMapping): ParallelJoinPrep | null;
+    _runBufferedSerialJoin(makeBuildSide: MakeBuildSide, makeProbeOp: MakeProbeOp, buildChunks: DataChunk[], probeChunks: DataChunk[], node: LogicalJoinNode, probeColCount: number): Promise<DataChunk[]>;
+    normalizeExecType(dt: DataType | string): DataType;
+    normalizeAggResultType(agg: BoundAggregateNode | {
+        name?: string;
+    }): DataType;
+}
+export {};

package/dist/execution/query-result.d.ts

@@ -0,0 +1,15 @@
+import type { DataChunk } from '../storage/chunk.js';
+import type { ColumnValue } from '../storage/data-type.js';
+export type ResultRow = Record<string, ColumnValue>;
+export interface AsyncChunkSource {
+    [Symbol.asyncIterator](): AsyncIterator<DataChunk>;
+}
+export declare class QueryResult {
+    _columnNames: string[];
+    _sink: AsyncChunkSource;
+    constructor(columnNames: string[], sink: AsyncChunkSource);
+    get columns(): string[];
+    toArray(): Promise<ResultRow[]>;
+    [Symbol.asyncIterator](): AsyncGenerator<ResultRow>;
+    chunks(): AsyncGenerator<DataChunk>;
+}

package/dist/execution/result-sink.d.ts

@@ -0,0 +1,26 @@
+import type { DataChunk } from '../storage/chunk.js';
+import type { Sink } from './execution-types.js';
+export declare class ResultSink implements Sink {
+    _streaming: boolean;
+    _capacity: number;
+    _queue: (DataChunk | undefined)[];
+    _head: number;
+    _tail: number;
+    _count: number;
+    _totalRows: number;
+    _done: boolean;
+    _error: Error | null;
+    _producerResolve: (() => void) | null;
+    _consumerResolve: (() => void) | null;
+    _collected: DataChunk[];
+    constructor(streaming?: boolean);
+    init(): Promise<void>;
+    consume(chunk: DataChunk): Promise<void>;
+    finalize(): Promise<void>;
+    error(err: Error): void;
+    get totalRows(): number;
+    get chunks(): DataChunk[];
+    _dequeue(): DataChunk | undefined;
+    [Symbol.asyncIterator](): AsyncIterator<DataChunk>;
+    collect(): Promise<DataChunk[]>;
+}

package/dist/execution/scheduler.d.ts

@@ -0,0 +1,8 @@
+import type { PipelineGraph, Pipeline } from './pipeline.js';
+export declare class TaskScheduler {
+    concurrency: number;
+    constructor(concurrency?: number);
+    schedule(pipelineGraph: PipelineGraph): Promise<void>;
+    executePipelines(pipelines: Pipeline[], graph: PipelineGraph): Promise<void>;
+    runPipelineSource(pipeline: Pipeline, graph: PipelineGraph): Promise<void>;
+}

package/dist/execution/vector-ops.d.ts

@@ -0,0 +1,32 @@
+import { DataChunk } from '../storage/chunk.js';
+import type { AnyColumn } from '../storage/chunk.js';
+import type { BoundExpr } from '../binder/expression-binder.js';
+import { DataType } from '../storage/data-type.js';
+import type { EvalValue, CompiledExpr, ColumnMapping } from './execution-types.js';
+interface VectorResult {
+    ref?: boolean;
+    colIdx?: number;
+    column?: AnyColumn;
+    constant?: boolean;
+    value?: EvalValue;
+    size?: number;
+    data?: EvalValue[];
+}
+type VectorFn = (chunk: DataChunk) => VectorResult | null;
+interface BuildRef {
+    chunkIdx: number;
+    rowIdx: number;
+}
+export declare function compileVectorExpression(expr: BoundExpr | null, columnMapping: ColumnMapping | null): VectorFn | null;
+export declare function vectorGet(result: VectorResult | null, chunk: DataChunk, i: number): EvalValue;
+export declare function vectorizedFilter(chunk: DataChunk, scalarEvalFn: CompiledExpr): {
+    sv: Uint32Array;
+    count: number;
+};
+export declare function vectorizedHashProbe(probeChunk: DataChunk, probeKeyFn: CompiledExpr, hashTable: Map<string, BuildRef[]>, maxMatches?: number): {
+    buildRefs: BuildRef[];
+    probeIndices: number[];
+};
+export declare function buildJoinOutputDirect(buildChunks: DataChunk[], buildRefs: BuildRef[], probeChunk: DataChunk, probeIndices: number[], buildColCount: number, probeColCount: number): DataChunk;
+export declare function vectorizedProject(chunk: DataChunk, evaluators: CompiledExpr[], expressions: (BoundExpr | null)[], resultTypes: DataType[] | null): DataChunk;
+export {};

package/dist/execution/wasm-expr-eval.d.ts

@@ -0,0 +1,7 @@
+import type { BoundExpr } from '../binder/expression-binder.js';
+import type { DataChunk } from '../storage/chunk.js';
+import type { ColumnMapping } from './execution-types.js';
+type VecData = Float64Array | Int32Array;
+export declare function isVectorizableExpr(expr: BoundExpr | null): boolean;
+export declare function evalVectorized(expr: BoundExpr, chunk: DataChunk, columnMapping: ColumnMapping | null, size: number): Promise<VecData | number | null>;
+export {};