@slexisvn/query-engine 0.1.0

package/dist/parallel/worker-messages.d.ts

@@ -0,0 +1,243 @@
+import type { DataType, ColumnValue } from '../storage/data-type.js';
+import type { FragmentSpec, JoinSpec } from '../execution/fragment-spec.js';
+export type FilterOperation = 'filterEq' | 'filterLt' | 'filterGt' | 'filterLe' | 'filterGe' | 'filterBetween';
+export type CompoundCombineOp = 'and' | 'or';
+export type ScalarAggType = 'sum' | 'min' | 'max' | 'count';
+export type WasmDataType = 'INT32' | 'FLOAT64' | 'DATE';
+export type ProjectOp = 'scalarAddF64' | 'scalarSubF64' | 'scalarMulF64' | 'scalarDivF64' | 'scalarSubRevF64' | 'scalarDivRevF64' | 'vecAddF64' | 'vecSubF64' | 'vecMulF64' | 'vecDivF64' | 'negF64' | 'widenI32ToF64';
+export declare enum WorkerTaskType {
+    FILTER = "filter",
+    AGGREGATE = "aggregate",
+    FILTER_COMPOUND = "filter_compound",
+    PIPELINE = "pipeline",
+    PROJECT = "project"
+}
+export interface FilterDescriptor {
+    operation: FilterOperation;
+    value?: number;
+    low?: number;
+    high?: number;
+}
+export interface PipelineFilterStage {
+    kind: 'filter';
+    operation: FilterOperation;
+    value?: number;
+    low?: number;
+    high?: number;
+}
+export interface PipelineAggregateStage {
+    kind: 'aggregate';
+    aggType: ScalarAggType;
+}
+export interface PipelineCountStage {
+    kind: 'count';
+}
+export type PipelineStage = PipelineFilterStage | PipelineAggregateStage | PipelineCountStage;
+export interface FilterTask {
+    type: WorkerTaskType.FILTER;
+    dataOffset: number;
+    count: number;
+    operation: FilterOperation;
+    dataType: WasmDataType;
+    baseIndex: number;
+    value?: number;
+    low?: number;
+    high?: number;
+}
+export interface AggregateTask {
+    type: WorkerTaskType.AGGREGATE;
+    dataOffset: number;
+    count: number;
+    aggType: ScalarAggType;
+    dataType: WasmDataType;
+}
+export interface CompoundFilterTask {
+    type: WorkerTaskType.FILTER_COMPOUND;
+    dataOffset: number;
+    count: number;
+    filters: FilterDescriptor[];
+    combineOp: CompoundCombineOp;
+    dataType: WasmDataType;
+    baseIndex: number;
+}
+export interface PipelineTask {
+    type: WorkerTaskType.PIPELINE;
+    dataOffset: number;
+    count: number;
+    dataType: WasmDataType;
+    baseIndex: number;
+    stages: PipelineStage[];
+}
+export interface ProjectTask {
+    type: WorkerTaskType.PROJECT;
+    dataOffset: number;
+    dataBOffset?: number;
+    count: number;
+    op: ProjectOp;
+    scalar?: number;
+}
+export type WorkerTask = FilterTask | AggregateTask | CompoundFilterTask | PipelineTask | ProjectTask;
+export type WorkerTaskMessage = WorkerTask & {
+    taskId: number;
+};
+export interface ScalarAggregatePartial {
+    aggType: ScalarAggType;
+    result: number;
+    count: number;
+}
+export interface SelectionVectorResult {
+    matchCount: number;
+    selectionVector?: Uint32Array;
+}
+export interface ScalarAggregateResult {
+    result: number;
+    count: number;
+}
+export interface PipelineAggregatesResult {
+    aggregates: ScalarAggregatePartial[];
+}
+export interface ProjectResult {
+    count: number;
+    resultData?: Float64Array;
+}
+export type WorkerResultData = SelectionVectorResult | ScalarAggregateResult | PipelineAggregatesResult | ProjectResult;
+export interface WorkerReadyMessage {
+    type: 'ready';
+    workerId: number;
+}
+export interface WorkerResultMessage {
+    type: 'result';
+    taskId: number;
+    data: WorkerResultData;
+}
+export interface WorkerErrorMessage {
+    type: 'error';
+    taskId: number;
+    error: string;
+}
+export type WorkerOutboundMessage = WorkerReadyMessage | WorkerResultMessage | WorkerErrorMessage;
+export interface WorkerInitData {
+    workerId: number;
+    wasmModule: WebAssembly.Module;
+    wasmMemory: WebAssembly.Memory;
+    regionId: number;
+    regionStart: number;
+    regionCapacity: number;
+}
+export interface MorselSchedulerDescriptor {
+    counterBuffer: SharedArrayBuffer;
+    totalUnits: number;
+    unitsPerMorsel: number;
+}
+export interface MorselRange {
+    start: number;
+    end: number;
+}
+export interface EncodedChunkSet {
+    buffers: ArrayBufferLike[];
+    meta: Uint32Array;
+    sizes: Uint32Array;
+    colCount: number;
+}
+export type AccumulatorState = ColumnValue | {
+    sum: number;
+    count: number;
+} | ColumnValue[];
+export interface PartialGroupRecord {
+    key: string | number | bigint | boolean | null;
+    groupValues: ColumnValue[];
+    states: AccumulatorState[];
+}
+export interface SpillDescriptor {
+    dir: string;
+    tag: string;
+    groupLimit: number;
+}
+export interface SpillRef {
+    file: string;
+    partition: number;
+}
+export interface JoinOutputTypes {
+    build: DataType[];
+    probe: DataType[];
+}
+export declare enum FragmentTaskType {
+    AGGREGATE = "aggregate",
+    COMBINE = "combine",
+    JOIN_PARTITION = "joinPartition",
+    JOIN_PROBE = "joinProbe"
+}
+export interface FragmentAggregateRequest {
+    type: FragmentTaskType.AGGREGATE;
+    spec: FragmentSpec;
+    chunks: EncodedChunkSet;
+    scheduler: MorselSchedulerDescriptor;
+    partitionCount: number;
+    spill: SpillDescriptor | null;
+}
+export interface FragmentCombineRequest {
+    type: FragmentTaskType.COMBINE;
+    spec: FragmentSpec;
+    partials: PartialGroupRecord[][];
+    spillRefs: SpillRef[];
+}
+export interface FragmentJoinPartitionRequest {
+    type: FragmentTaskType.JOIN_PARTITION;
+    spec: JoinSpec;
+    buildChunks: EncodedChunkSet;
+    probeChunks: EncodedChunkSet;
+    buildScheduler: MorselSchedulerDescriptor;
+    probeScheduler: MorselSchedulerDescriptor;
+    partitionCount: number;
+}
+export interface FragmentJoinProbeRequest {
+    type: FragmentTaskType.JOIN_PROBE;
+    spec: JoinSpec;
+    buildChunks: EncodedChunkSet;
+    probeChunks: EncodedChunkSet;
+    buildRefs: Uint32Array;
+    probeRefs: Uint32Array;
+    hasNullKey: boolean;
+    outputTypes: JoinOutputTypes;
+}
+export type FragmentRequest = FragmentAggregateRequest | FragmentCombineRequest | FragmentJoinPartitionRequest | FragmentJoinProbeRequest;
+export type FragmentRequestMessage = FragmentRequest & {
+    reqId: number;
+};
+export interface FragmentAggregateResult {
+    partitions: PartialGroupRecord[][];
+    spillFiles: string[];
+}
+export interface FragmentCombineResult {
+    partials: PartialGroupRecord[];
+}
+export interface FragmentJoinPartitionResult {
+    buildPartitions: Uint32Array[];
+    buildNullCount: number;
+    probePartitions: Uint32Array[];
+    probeNullRows: ColumnValue[][];
+    buildNullRows: ColumnValue[][];
+}
+export interface FragmentJoinProbeResult {
+    chunks: EncodedChunkSet | null;
+}
+export type FragmentResult = FragmentAggregateResult | FragmentCombineResult | FragmentJoinPartitionResult | FragmentJoinProbeResult;
+export interface FragmentSuccessMessage {
+    reqId: number;
+    ok: true;
+    partitions?: PartialGroupRecord[][];
+    spillFiles?: string[];
+    partials?: PartialGroupRecord[] | PartialGroupRecord[][];
+    buildPartitions?: Uint32Array[];
+    buildNullCount?: number;
+    probePartitions?: Uint32Array[];
+    probeNullRows?: ColumnValue[][];
+    buildNullRows?: ColumnValue[][];
+    chunks?: EncodedChunkSet | null;
+}
+export interface FragmentFailureMessage {
+    reqId: number;
+    ok: false;
+    error: string;
+}
+export type FragmentReplyMessage = FragmentSuccessMessage | FragmentFailureMessage;

package/dist/parallel/worker-pool.d.ts

@@ -0,0 +1,62 @@
+import { Worker } from 'worker_threads';
+import type { WorkerTask, WorkerResultData } from './worker-messages.js';
+declare const WORKER_STATE: {
+    readonly IDLE: 0;
+    readonly BUSY: 1;
+    readonly TERMINATED: 2;
+};
+type WorkerStateValue = typeof WORKER_STATE[keyof typeof WORKER_STATE];
+interface RegionBounds {
+    start: number;
+    end: number;
+}
+interface RegionAllocatorLike {
+    addRegion(): number;
+    getRegionBounds(regionId: number): RegionBounds;
+}
+interface WorkerPoolOptions {
+    maxWorkers: number;
+    wasmModule: WebAssembly.Module;
+    wasmMemory: WebAssembly.Memory;
+    regionAllocator: RegionAllocatorLike;
+}
+interface WorkerEntry {
+    worker: Worker;
+    id: number;
+    regionId: number;
+    state: WorkerStateValue;
+}
+interface PendingTask {
+    taskId: number;
+    task: WorkerTask;
+    resolve: (value: WorkerResultData) => void;
+    reject: (reason: Error) => void;
+    targetWorkerId: number | undefined;
+    workerId: number | null;
+}
+export declare class WorkerPool {
+    maxWorkers: number;
+    wasmModule: WebAssembly.Module;
+    wasmMemory: WebAssembly.Memory;
+    regionAllocator: RegionAllocatorLike;
+    workers: Map<number, WorkerEntry>;
+    taskQueue: PendingTask[];
+    taskIdCounter: number;
+    pendingTasks: Map<number, PendingTask>;
+    initialized: boolean;
+    constructor({ maxWorkers, wasmModule, wasmMemory, regionAllocator }: WorkerPoolOptions);
+    init(): Promise<void>;
+    _spawnWorker(id: number): Promise<void>;
+    execute(tasks: WorkerTask[]): Promise<WorkerResultData[]>;
+    executeOnWorker(workerId: number, task: WorkerTask): Promise<WorkerResultData>;
+    _submitTask(task: WorkerTask, targetWorkerId?: number): Promise<WorkerResultData>;
+    _dispatch(entry: WorkerEntry, taskId: number, task: WorkerTask, pending: PendingTask): void;
+    _drainQueue(): void;
+    _findIdleWorker(): WorkerEntry | null;
+    activeWorkerCount(): number;
+    idleWorkerCount(): number;
+    resize(newCount: number): Promise<void>;
+    _terminateWorker(id: number): Promise<void>;
+    shutdown(): Promise<void>;
+}
+export {};

package/dist/parallel/worker-thread.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/parser/ast.d.ts

@@ -0,0 +1,298 @@
+export declare enum NodeKind {
+    EXPLAIN_STMT = "ExplainStmt",
+    SELECT_STMT = "SelectStmt",
+    SET_OP = "SetOp",
+    WITH_CLAUSE = "WithClause",
+    CTE = "CTE",
+    SELECT_ITEM = "SelectItem",
+    ALL_COLUMNS = "AllColumns",
+    TABLE_REF = "TableRef",
+    JOIN_REF = "JoinRef",
+    SUBQUERY_REF = "SubqueryRef",
+    COLUMN_REF = "ColumnRef",
+    LITERAL = "Literal",
+    BINARY_EXPR = "BinaryExpr",
+    UNARY_EXPR = "UnaryExpr",
+    BETWEEN_EXPR = "BetweenExpr",
+    IN_EXPR = "InExpr",
+    LIKE_EXPR = "LikeExpr",
+    IS_NULL_EXPR = "IsNullExpr",
+    EXISTS_EXPR = "ExistsExpr",
+    SUBQUERY_EXPR = "SubqueryExpr",
+    FUNCTION_CALL = "FunctionCall",
+    AGGREGATE_CALL = "AggregateCall",
+    CASE_EXPR = "CaseExpr",
+    WHEN_CLAUSE = "WhenClause",
+    CAST_EXPR = "CastExpr",
+    EXTRACT_EXPR = "ExtractExpr",
+    SUBSTRING_EXPR = "SubstringExpr",
+    INTERVAL_EXPR = "IntervalExpr",
+    ORDER_KEY = "OrderKey",
+    TYPE_NAME = "TypeName",
+    WINDOW_SPEC = "WindowSpec",
+    WINDOW_CALL = "WindowCall",
+    CREATE_TABLE_STMT = "CreateTableStmt",
+    DROP_TABLE_STMT = "DropTableStmt",
+    COLUMN_DEF = "ColumnDef",
+    EXPLAIN_ANALYZE_STMT = "ExplainAnalyzeStmt"
+}
+export type Expr = ColumnRefNode | LiteralNode | BinaryExprNode | UnaryExprNode | BetweenExprNode | InExprNode | LikeExprNode | IsNullExprNode | ExistsExprNode | SubqueryExprNode | FunctionCallNode | AggregateCallNode | CaseExprNode | CastExprNode | ExtractExprNode | SubstringExprNode | IntervalExprNode | WindowCallNode | AllColumnsNode;
+export type FromItem = TableRefNode | JoinRefNode | SubqueryRefNode;
+export type QueryStmt = SelectStmtNode | SetOpNode;
+export type Statement = QueryStmt | ExplainStmtNode | ExplainAnalyzeStmtNode | CreateTableStmtNode | DropTableStmtNode;
+export type ASTNode = Statement | WithClauseNode | CTENode | SelectItemNode | FromItem | Expr | WhenClauseNode | OrderKeyNode | TypeNameNode | WindowSpecNode | ColumnDefNode | QuantifiedSubqueryNode;
+export interface ExplainStmtNode {
+    kind: NodeKind.EXPLAIN_STMT;
+    query: QueryStmt;
+}
+export interface SelectStmtNode {
+    kind: NodeKind.SELECT_STMT;
+    withClause: WithClauseNode | null;
+    distinct: boolean;
+    selectItems: SelectItemNode[];
+    from: FromItem | null;
+    where: Expr | null;
+    groupBy: Expr[] | null;
+    having: Expr | null;
+    orderBy: OrderKeyNode[] | null;
+    limit: Expr | null;
+    offset: Expr | null;
+}
+export interface SetOpNode {
+    kind: NodeKind.SET_OP;
+    op: string;
+    left: QueryStmt;
+    right: QueryStmt;
+    all: boolean;
+}
+export interface WithClauseNode {
+    kind: NodeKind.WITH_CLAUSE;
+    ctes: CTENode[];
+}
+export interface CTENode {
+    kind: NodeKind.CTE;
+    name: string;
+    query: QueryStmt;
+    columnAliases: string[] | null;
+}
+export interface SelectItemNode {
+    kind: NodeKind.SELECT_ITEM;
+    expr: Expr;
+    alias: string | null;
+}
+export interface AllColumnsNode {
+    kind: NodeKind.ALL_COLUMNS;
+    table: string | null;
+}
+export interface TableRefNode {
+    kind: NodeKind.TABLE_REF;
+    name: string;
+    alias: string;
+}
+export interface JoinRefNode {
+    kind: NodeKind.JOIN_REF;
+    left: FromItem;
+    right: FromItem;
+    joinType: string;
+    condition: Expr | null;
+    natural?: boolean;
+    usingColumns?: string[];
+}
+export interface SubqueryRefNode {
+    kind: NodeKind.SUBQUERY_REF;
+    query: QueryStmt;
+    alias: string | null;
+}
+export interface ColumnRefNode {
+    kind: NodeKind.COLUMN_REF;
+    name: string;
+    table: string | null;
+}
+export interface LiteralNode {
+    kind: NodeKind.LITERAL;
+    value: string | number | boolean | null;
+    dataType: string | null;
+}
+export interface BinaryExprNode {
+    kind: NodeKind.BINARY_EXPR;
+    op: string;
+    left: Expr;
+    right: Expr | QuantifiedSubqueryNode;
+}
+export interface UnaryExprNode {
+    kind: NodeKind.UNARY_EXPR;
+    op: string;
+    operand: Expr;
+}
+export interface BetweenExprNode {
+    kind: NodeKind.BETWEEN_EXPR;
+    expr: Expr;
+    low: Expr;
+    high: Expr;
+    negated: boolean;
+}
+export interface InExprNode {
+    kind: NodeKind.IN_EXPR;
+    expr: Expr;
+    list: Expr[] | SubqueryExprNode;
+    negated: boolean;
+}
+export interface LikeExprNode {
+    kind: NodeKind.LIKE_EXPR;
+    expr: Expr;
+    pattern: Expr;
+    negated: boolean;
+}
+export interface IsNullExprNode {
+    kind: NodeKind.IS_NULL_EXPR;
+    expr: Expr;
+    negated: boolean;
+}
+export interface ExistsExprNode {
+    kind: NodeKind.EXISTS_EXPR;
+    query: QueryStmt;
+    negated: boolean;
+}
+export interface SubqueryExprNode {
+    kind: NodeKind.SUBQUERY_EXPR;
+    query: QueryStmt;
+}
+export interface FunctionCallNode {
+    kind: NodeKind.FUNCTION_CALL;
+    name: string;
+    args: Expr[];
+}
+export interface AggregateCallNode {
+    kind: NodeKind.AGGREGATE_CALL;
+    name: string;
+    args: Expr[];
+    distinct: boolean;
+}
+export interface CaseExprNode {
+    kind: NodeKind.CASE_EXPR;
+    operand: Expr | null;
+    whenClauses: WhenClauseNode[];
+    elseExpr: Expr | null;
+}
+export interface WhenClauseNode {
+    kind: NodeKind.WHEN_CLAUSE;
+    condition: Expr;
+    result: Expr;
+}
+export interface CastExprNode {
+    kind: NodeKind.CAST_EXPR;
+    expr: Expr;
+    targetType: TypeNameNode;
+}
+export interface ExtractExprNode {
+    kind: NodeKind.EXTRACT_EXPR;
+    field: string;
+    source: Expr;
+}
+export interface SubstringExprNode {
+    kind: NodeKind.SUBSTRING_EXPR;
+    expr: Expr;
+    from: Expr;
+    length: Expr | null;
+}
+export interface IntervalExprNode {
+    kind: NodeKind.INTERVAL_EXPR;
+    value: string;
+    unit: string;
+}
+export interface OrderKeyNode {
+    kind: NodeKind.ORDER_KEY;
+    expr: Expr;
+    direction: string;
+    nullOrder: string | null;
+}
+export interface TypeNameNode {
+    kind: NodeKind.TYPE_NAME;
+    name: string;
+    params: number[];
+}
+export interface WindowSpecNode {
+    kind: NodeKind.WINDOW_SPEC;
+    partitionBy: Expr[];
+    orderBy: OrderKeyNode[];
+}
+export interface WindowCallNode {
+    kind: NodeKind.WINDOW_CALL;
+    name: string;
+    args: Expr[];
+    windowSpec: WindowSpecNode;
+}
+export interface CreateTableStmtNode {
+    kind: NodeKind.CREATE_TABLE_STMT;
+    name: string;
+    columns: ColumnDefNode[] | null;
+    ifNotExists: boolean;
+    as: QueryStmt | null;
+}
+export interface DropTableStmtNode {
+    kind: NodeKind.DROP_TABLE_STMT;
+    name: string;
+    ifExists: boolean;
+}
+export interface ColumnDefNode {
+    kind: NodeKind.COLUMN_DEF;
+    name: string;
+    typeName: TypeNameNode;
+}
+export interface ExplainAnalyzeStmtNode {
+    kind: NodeKind.EXPLAIN_ANALYZE_STMT;
+    query: QueryStmt;
+}
+export interface QuantifiedSubqueryNode {
+    kind: 'QuantifiedSubquery';
+    quantifier: string;
+    query: QueryStmt;
+}
+export interface SelectStmtOptions {
+    withClause?: WithClauseNode | null;
+    distinct?: boolean;
+    selectItems: SelectItemNode[];
+    from?: FromItem | null;
+    where?: Expr | null;
+    groupBy?: Expr[] | null;
+    having?: Expr | null;
+    orderBy?: OrderKeyNode[] | null;
+    limit?: Expr | null;
+    offset?: Expr | null;
+}
+export declare function ExplainStmt(query: QueryStmt): ExplainStmtNode;
+export declare function SelectStmt({ withClause, distinct, selectItems, from, where, groupBy, having, orderBy, limit, offset, }: SelectStmtOptions): SelectStmtNode;
+export declare function SetOp(op: string, left: QueryStmt, right: QueryStmt, all?: boolean): SetOpNode;
+export declare function WithClause(ctes: CTENode[]): WithClauseNode;
+export declare function CTE(name: string, query: QueryStmt, columnAliases?: string[] | null): CTENode;
+export declare function SelectItem(expr: Expr, alias?: string | null): SelectItemNode;
+export declare function AllColumns(table?: string | null): AllColumnsNode;
+export declare function TableRef(name: string, alias?: string | null): TableRefNode;
+export declare function JoinRef(left: FromItem, right: FromItem, joinType: string, condition?: Expr | null): JoinRefNode;
+export declare function SubqueryRef(query: QueryStmt, alias: string | null): SubqueryRefNode;
+export declare function ColumnRef(name: string, table?: string | null): ColumnRefNode;
+export declare function Literal(value: string | number | boolean | null, dataType?: string | null): LiteralNode;
+export declare function BinaryExpr(op: string, left: Expr, right: Expr | QuantifiedSubqueryNode): BinaryExprNode;
+export declare function UnaryExpr(op: string, operand: Expr): UnaryExprNode;
+export declare function BetweenExpr(expr: Expr, low: Expr, high: Expr, negated?: boolean): BetweenExprNode;
+export declare function InExpr(expr: Expr, list: Expr[] | SubqueryExprNode, negated?: boolean): InExprNode;
+export declare function LikeExpr(expr: Expr, pattern: Expr, negated?: boolean): LikeExprNode;
+export declare function IsNullExpr(expr: Expr, negated?: boolean): IsNullExprNode;
+export declare function ExistsExpr(query: QueryStmt, negated?: boolean): ExistsExprNode;
+export declare function SubqueryExpr(query: QueryStmt): SubqueryExprNode;
+export declare function FunctionCall(name: string, args: Expr[]): FunctionCallNode;
+export declare function AggregateCall(name: string, args: Expr[], distinct?: boolean): AggregateCallNode;
+export declare function CaseExpr(operand: Expr | null, whenClauses: WhenClauseNode[], elseExpr?: Expr | null): CaseExprNode;
+export declare function WhenClause(condition: Expr, result: Expr): WhenClauseNode;
+export declare function CastExpr(expr: Expr, targetType: TypeNameNode): CastExprNode;
+export declare function ExtractExpr(field: string, source: Expr): ExtractExprNode;
+export declare function SubstringExpr(expr: Expr, from: Expr, length?: Expr | null): SubstringExprNode;
+export declare function IntervalExpr(value: string, unit: string): IntervalExprNode;
+export declare function OrderKey(expr: Expr, direction?: string, nullOrder?: string | null): OrderKeyNode;
+export declare function TypeName(name: string, params?: number[]): TypeNameNode;
+export declare function WindowSpec(partitionBy?: Expr[], orderBy?: OrderKeyNode[]): WindowSpecNode;
+export declare function WindowCall(name: string, args: Expr[], windowSpec: WindowSpecNode): WindowCallNode;
+export declare function CreateTableStmt(name: string, columns: ColumnDefNode[] | null, ifNotExists?: boolean, as?: QueryStmt | null): CreateTableStmtNode;
+export declare function DropTableStmt(name: string, ifExists?: boolean): DropTableStmtNode;
+export declare function ColumnDef(name: string, typeName: TypeNameNode): ColumnDefNode;
+export declare function ExplainAnalyzeStmt(query: QueryStmt): ExplainAnalyzeStmtNode;