@slexisvn/query-engine 0.1.0

package/dist/config.d.ts

@@ -0,0 +1,42 @@
+export declare const DEFAULT_CHUNK_SIZE = 2048;
+export declare const Config: {
+    memoryLimit: number;
+    hashJoinPartitions: number;
+    flushBatchSize: number;
+    bufferPoolPages: number;
+    wasmMinChunkSize: number;
+    sinkQueueCapacity: number;
+    btreeOrder: number;
+    indexScanSelectivityThreshold: number;
+    dependentJoinConcurrency: number;
+    parallelWorkers: number;
+    parallelThreshold: number;
+    parallelAggThreshold: number;
+    aggMorselRows: number;
+    sabColumns: boolean;
+    sabArenaSegmentBytes: number;
+    parallelAggMemoryBytes: number;
+    parallelCombineMinGroups: number;
+    aggSpillGroups: number;
+    vectorGroupRange: number;
+    parallelJoinThreshold: number;
+    transportMaxBuffers: number;
+    aggRadixMultiplier: number;
+    regionSize: number;
+    morselSize: number;
+    clusterPort: number;
+    coordinatorSchemaSampleRows: number;
+    heartbeatIntervalMs: number;
+    heartbeatTimeoutMs: number;
+    defaultPartitionCount: number;
+    broadcastThreshold: number;
+    exchangeBatchSize: number;
+    exchangeBufferCapacity: number;
+    fragmentRetryLimit: number;
+    coordinatorTimeoutMs: number;
+    codecCompression: number;
+    distributedWorkers: number;
+    phiAccrualWindowSize: number;
+    phiAccrualThreshold: number;
+    networkCostPerByte: number;
+};

package/dist/dataframe/column-expr.d.ts

@@ -0,0 +1,67 @@
+import type { BoundExpr } from '../binder/expression-binder.js';
+import { DataType } from '../storage/data-type.js';
+import type { ColumnValue } from '../storage/data-type.js';
+import type { CatalogLike } from '../binder/binder.js';
+type LiteralInput = ColumnValue | Date | undefined;
+type ColInput = Col | LiteralInput;
+interface SchemaFieldLike {
+    tableAlias: string;
+    name: string;
+    index: number;
+    dataType: string | null;
+}
+interface SchemaLike {
+    fields: SchemaFieldLike[];
+    resolve(name: string, tableAlias: string | null): SchemaFieldLike;
+}
+interface BindContextLike {
+    catalog: CatalogLike;
+    functionRegistry: object;
+}
+type BuildFn = (schema: SchemaLike, ctx: BindContextLike) => BoundExpr;
+type BinaryTypeFn = (left: DataType, right: DataType) => DataType;
+interface ColBindResult {
+    expr: BoundExpr;
+    outputName: string | null;
+    dataType: string | null;
+}
+export declare class Col {
+    _build: BuildFn;
+    _name: string | null;
+    _alias: string | null;
+    constructor(buildFn: BuildFn, name?: string | null);
+    alias(name: string): Col;
+    as(name: string): Col;
+    bind(schema: SchemaLike, ctx: BindContextLike): ColBindResult;
+    _binary(op: string, other: ColInput, typeFn: BinaryTypeFn): Col;
+    add(other: ColInput): Col;
+    sub(other: ColInput): Col;
+    mul(other: ColInput): Col;
+    div(other: ColInput): Col;
+    eq(other: ColInput): Col;
+    ne(other: ColInput): Col;
+    lt(other: ColInput): Col;
+    le(other: ColInput): Col;
+    gt(other: ColInput): Col;
+    ge(other: ColInput): Col;
+    and(other: ColInput): Col;
+    or(other: ColInput): Col;
+    not(): Col;
+    isNull(): Col;
+    isNotNull(): Col;
+    like(pattern: string): Col;
+    between(low: ColInput, high: ColInput): Col;
+    isin(...values: ColInput[]): Col;
+    cast(targetType: DataType): Col;
+}
+export declare function toCol(value: ColInput): Col;
+export declare function col(spec: string): Col;
+export declare function lit(value: LiteralInput): Col;
+export declare function expr(sqlString: string): Col;
+export declare const sum: (column: Col | string) => Col;
+export declare const avg: (column: Col | string) => Col;
+export declare const min: (column: Col | string) => Col;
+export declare const max: (column: Col | string) => Col;
+export declare const count: (column: Col | string) => Col;
+export declare function countStar(): Col;
+export {};

package/dist/dataframe/dataframe.d.ts

@@ -0,0 +1,96 @@
+import type { LogicalPlanNode } from '../planner/logical-plan.js';
+import type { BoundExpr } from '../binder/expression-binder.js';
+import { DataType } from '../storage/data-type.js';
+import type { ColumnValue } from '../storage/data-type.js';
+import type { DataChunk } from '../storage/chunk.js';
+import { DFSchema } from './schema.js';
+import { Col } from './column-expr.js';
+import type { CatalogLike } from '../binder/binder.js';
+import type { Catalog } from '../catalog/catalog.js';
+import type { defaultFunctionRegistry } from '../catalog/function-registry.js';
+type FunctionRegistry = typeof defaultFunctionRegistry;
+type CteMap = Map<string, LogicalPlanNode>;
+interface BindContext {
+    catalog: CatalogLike;
+    functionRegistry: FunctionRegistry;
+}
+interface BoundColumn {
+    expr: BoundExpr;
+    outputName: string | null;
+    dataType: string | null;
+}
+interface OutputColumnLike {
+    name: string;
+    expr: BoundExpr | null;
+    dataType: DataType | string | null;
+}
+interface ResultLike {
+    toArray(): Promise<Record<string, ColumnValue>[]>;
+    chunks(): AsyncGenerator<DataChunk>;
+}
+interface SqlFrameLike {
+    name: string;
+    columns: {
+        name: string;
+        dataType: DataType;
+    }[];
+    plan: LogicalPlanNode;
+    cteMap: CteMap | null;
+}
+interface EngineLike {
+    catalog: Catalog;
+    functionRegistry: FunctionRegistry;
+    sql(sqlString: string, options: {
+        frames: SqlFrameLike[];
+    }): DataFrame;
+    _nextDfId(): number;
+    _runPlan(plan: LogicalPlanNode, outputColumns: OutputColumnLike[], streaming: boolean, cteMap: CteMap | null): Promise<ResultLike>;
+}
+interface OrderByDescriptor {
+    col: Col | string;
+    desc?: boolean;
+}
+type OrderBySpec = OrderByDescriptor | Col | string;
+export declare class DataFrame {
+    _engine: EngineLike;
+    _plan: LogicalPlanNode;
+    _schema: DFSchema;
+    _cteMap: CteMap | null;
+    constructor(engine: EngineLike, plan: LogicalPlanNode, schema: DFSchema, cteMap?: CteMap | null);
+    _ctx(): BindContext;
+    _derive(plan: LogicalPlanNode, schema: DFSchema, extraCteMap?: CteMap | null): DataFrame;
+    columns(): string[];
+    schema(): DFSchema;
+    explain(): string;
+    sql(sqlString: string): DataFrame;
+    select(...items: (Col | string)[]): DataFrame;
+    filter(condition: Col | string): DataFrame;
+    where(condition: Col | string): DataFrame;
+    withColumn(name: string, column: Col | string): DataFrame;
+    drop(...names: string[]): DataFrame;
+    groupBy(...items: (Col | string)[]): GroupedData;
+    orderBy(...specs: OrderBySpec[]): DataFrame;
+    sort(...specs: OrderBySpec[]): DataFrame;
+    limit(count: number, offset?: number): DataFrame;
+    distinct(): DataFrame;
+    union(other: DataFrame): DataFrame;
+    unionAll(other: DataFrame): DataFrame;
+    _union(other: DataFrame, all: boolean): DataFrame;
+    join(other: DataFrame, on: string | string[], joinType?: string): DataFrame;
+    _outputColumns(): OutputColumnLike[];
+    collect(): Promise<Record<string, ColumnValue>[]>;
+    toArray(): Promise<Record<string, ColumnValue>[]>;
+    count(): Promise<number>;
+    chunks(): AsyncGenerator<DataChunk>;
+    show(n?: number): Promise<string>;
+}
+export declare class GroupedData {
+    _engine: EngineLike;
+    _childPlan: LogicalPlanNode;
+    _childSchema: DFSchema;
+    _groupBounds: BoundColumn[];
+    _cteMap: CteMap | null;
+    constructor(engine: EngineLike, childPlan: LogicalPlanNode, childSchema: DFSchema, groupBounds: BoundColumn[], cteMap: CteMap | null);
+    agg(...aggColumns: Col[]): DataFrame;
+}
+export {};

package/dist/dataframe/expr-types.d.ts

@@ -0,0 +1,6 @@
+import { DataType } from '../storage/data-type.js';
+export declare function inferArithmeticType(left: DataType, right: DataType): DataType;
+export declare function inferComparisonType(): DataType;
+export declare function inferLogicalType(): DataType;
+export declare function inferConcatType(): DataType;
+export declare function inferAggregateResultType(name: string, argType?: DataType): DataType;

package/dist/dataframe/in-memory-relation.d.ts

@@ -0,0 +1,31 @@
+import { DataChunk } from '../storage/chunk.js';
+import type { ColumnSchema, ColumnValue } from '../storage/data-type.js';
+type RowInput = Record<string, ColumnValue> | ColumnValue[];
+type ColumnsInput = Record<string, ColumnValue[]>;
+export declare class InMemoryRelation {
+    schema: ColumnSchema[];
+    chunks: DataChunk[];
+    _rowCount: number;
+    constructor(schema: ColumnSchema[], chunks: DataChunk[]);
+    getSchema(): ColumnSchema[];
+    rowCount(): number;
+    getColumnIndex(name: string): number;
+    scan(): AsyncGenerator<DataChunk, void, void>;
+    static fromRows(rows: RowInput[], declaredSchema?: ColumnSchema[] | null): InMemoryRelation;
+    static fromColumns(columns: ColumnsInput, declaredSchema?: ColumnSchema[] | null): InMemoryRelation;
+    static builder(declaredSchema?: ColumnSchema[] | null): RelationBuilder;
+}
+export declare class RelationBuilder {
+    _declaredSchema: ColumnSchema[] | null;
+    schema: ColumnSchema[] | null;
+    _names: string[] | null;
+    _extractByIndex: boolean;
+    _chunks: DataChunk[];
+    _chunk: DataChunk | null;
+    _finished: boolean;
+    constructor(declaredSchema?: ColumnSchema[] | null);
+    _initFrom(rows: RowInput[]): void;
+    appendRows(rows: RowInput[]): this;
+    finish(): InMemoryRelation;
+}
+export {};

package/dist/dataframe/index.d.ts

@@ -0,0 +1,4 @@
+export { DataFrame, GroupedData } from './dataframe.js';
+export { Col, col, lit, expr, sum, avg, min, max, count, countStar } from './column-expr.js';
+export { DFSchema, DFField, UnknownColumnError, AmbiguousColumnError } from './schema.js';
+export { InMemoryRelation } from './in-memory-relation.js';

package/dist/dataframe/schema.d.ts

@@ -0,0 +1,35 @@
+import type { DataType, ColumnSchema } from '../storage/data-type.js';
+interface FieldLike {
+    name: string;
+    dataType: DataType;
+    tableAlias?: string;
+}
+export declare class UnknownColumnError extends Error {
+}
+export declare class AmbiguousColumnError extends Error {
+}
+export declare class DFField {
+    name: string;
+    dataType: DataType;
+    index: number;
+    tableAlias: string;
+    constructor(name: string, dataType: DataType, index: number, tableAlias?: string);
+}
+export declare class DFSchema {
+    _fields: DFField[];
+    constructor(fields: DFField[]);
+    static fromStorageSchema(storageSchema: ColumnSchema[], alias?: string): DFSchema;
+    static fromFields(fields: FieldLike[]): DFSchema;
+    get fields(): DFField[];
+    get length(): number;
+    field(i: number): DFField;
+    names(): string[];
+    resolve(name: string, tableAlias?: string | null): DFField;
+    has(name: string, tableAlias?: string | null): boolean;
+    project(fields: FieldLike[]): DFSchema;
+    drop(names: string[]): DFSchema;
+    append(other: DFSchema): DFSchema;
+    requalify(alias: string): DFSchema;
+    static aggregateOutput(groupFields: DFField[], aggFields: DFField[]): DFSchema;
+}
+export {};

package/dist/dataframe/sql-expr-binder.d.ts

@@ -0,0 +1,17 @@
+import type { CatalogLike } from '../binder/binder.js';
+import type { BoundExpr } from '../binder/expression-binder.js';
+interface SchemaFieldLike {
+    name: string;
+    dataType: string | null;
+    tableAlias?: string | null;
+}
+interface SchemaLike {
+    fields: SchemaFieldLike[];
+}
+interface ScalarBindResult {
+    expr: BoundExpr;
+    dataType: string | null;
+    outputName: string | null;
+}
+export declare function bindScalarSql(sqlString: string, schema: SchemaLike, catalog: CatalogLike, functionRegistry: object): ScalarBindResult;
+export {};

package/dist/dataframe/type-inference.d.ts

@@ -0,0 +1,8 @@
+import { DataType } from '../storage/data-type.js';
+import type { ColumnValue } from '../storage/data-type.js';
+type InferableValue = ColumnValue | Date | undefined;
+export declare function inferValueType(value: InferableValue): DataType | null;
+export declare function reconcileTypes(a: DataType | null, b: DataType | null): DataType | null;
+export declare function inferColumnType(values: Iterable<InferableValue>): DataType;
+export declare function coerceForColumn(value: InferableValue, dataType: DataType): ColumnValue;
+export {};

package/dist/distributed/cluster/cluster-manager.d.ts

@@ -0,0 +1,54 @@
+import { NodeStatus } from './node-descriptor.js';
+import type { NodeRoleValue } from './node-descriptor.js';
+import { HeartbeatMonitor } from './heartbeat-monitor.js';
+import type { NodeId, PartitionId, NodeCapacity, NodeDescriptorJSON, ClusterSnapshot } from '../distributed-types.js';
+type NodeStatusValue = (typeof NodeStatus)[keyof typeof NodeStatus];
+interface NodeDescriptorLike {
+    nodeId: NodeId;
+    host: string;
+    port: number;
+    role: NodeRoleValue;
+    capacity: NodeCapacity;
+    status: NodeStatusValue;
+    canExecuteFragments(): boolean;
+    hasPartition(partitionId: string | PartitionId): boolean;
+    assignPartition(partitionId: string | PartitionId): void;
+    toJSON(): NodeDescriptorJSON;
+}
+interface PartitionAssignmentLive {
+    node: NodeDescriptorLike;
+    partitionIds: PartitionId[];
+}
+interface ClusterManagerOptions {
+    phiWindowSize?: number;
+    phiThreshold?: number;
+    heartbeatIntervalMs?: number;
+}
+type NodeFailureCallbackLive = (node: NodeDescriptorLike) => void;
+type NodeJoinCallbackLive = (descriptor: NodeDescriptorLike) => void;
+export declare class ClusterManager {
+    _localNode: NodeDescriptorLike;
+    _nodeMap: Map<NodeId, NodeDescriptorLike>;
+    _failureCallbacks: NodeFailureCallbackLive[];
+    _joinCallbacks: NodeJoinCallbackLive[];
+    _heartbeat: HeartbeatMonitor;
+    constructor(localNode: NodeDescriptorLike, options?: ClusterManagerOptions);
+    get localNode(): NodeDescriptorLike;
+    get nodeCount(): number;
+    addNode(descriptor: NodeDescriptorLike): NodeDescriptorLike;
+    removeNode(nodeId: NodeId): boolean;
+    getNode(nodeId: NodeId): NodeDescriptorLike | null;
+    getAliveNodes(): NodeDescriptorLike[];
+    getWorkerNodes(): NodeDescriptorLike[];
+    getNodesForPartitions(partitionIds: PartitionId[]): Map<PartitionId, NodeDescriptorLike>;
+    getNodesByPartition(tableName: string, partitionIds: PartitionId[]): Map<NodeId, PartitionAssignmentLive>;
+    recordHeartbeat(nodeId: NodeId, timestamp: number): void;
+    onNodeFailure(callback: NodeFailureCallbackLive): void;
+    onNodeJoin(callback: NodeJoinCallbackLive): void;
+    startMonitoring(): void;
+    stopMonitoring(): void;
+    _handleStatusChange(nodeId: NodeId, newStatus: NodeStatusValue): void;
+    assignPartition(nodeId: NodeId, partitionKey: string | PartitionId): void;
+    snapshot(): ClusterSnapshot;
+}
+export {};

package/dist/distributed/cluster/heartbeat-monitor.d.ts

@@ -0,0 +1,27 @@
+import type { NodeId, NodeStatus as NodeStatusType, NodeStatusChange } from '../distributed-types.js';
+export interface HeartbeatMonitorOptions {
+    windowSize?: number;
+    threshold?: number;
+    intervalMs?: number;
+}
+export type HeartbeatSendFn = () => void;
+export declare class HeartbeatMonitor {
+    private _windowSize;
+    private _threshold;
+    private _interval;
+    private _windows;
+    private _timer;
+    private _sendFn;
+    private _statusCallback;
+    constructor(options?: HeartbeatMonitorOptions);
+    onSend(fn: HeartbeatSendFn): void;
+    onStatusChange(callback: NodeStatusChange): void;
+    start(): void;
+    stop(): void;
+    recordHeartbeat(nodeId: NodeId, timestamp: number): void;
+    phi(nodeId: NodeId, now: number): number;
+    getStatus(nodeId: NodeId, now: number): NodeStatusType;
+    removeNode(nodeId: NodeId): void;
+    _tick(): void;
+    _evaluateAll(): void;
+}

package/dist/distributed/cluster/node-descriptor.d.ts

@@ -0,0 +1,41 @@
+import type { NodeId, NodeCapacity, NodeDescriptorJSON } from '../distributed-types.js';
+export declare const NodeRole: {
+    readonly COORDINATOR: "coordinator";
+    readonly WORKER: "worker";
+    readonly HYBRID: "hybrid";
+};
+export type NodeRoleValue = (typeof NodeRole)[keyof typeof NodeRole];
+export declare const NodeStatus: {
+    readonly ALIVE: "alive";
+    readonly SUSPECT: "suspect";
+    readonly DEAD: "dead";
+};
+export type NodeStatusValue = (typeof NodeStatus)[keyof typeof NodeStatus];
+export interface NodeDescriptorParams {
+    nodeId: NodeId;
+    host: string;
+    port: number;
+    role?: NodeRoleValue;
+    capacity?: NodeCapacity;
+    status?: NodeStatusValue;
+    partitions?: string[];
+}
+export declare class NodeDescriptor {
+    nodeId: NodeId;
+    host: string;
+    port: number;
+    role: NodeRoleValue;
+    capacity: NodeCapacity;
+    status: NodeStatusValue;
+    partitions: Set<string>;
+    constructor({ nodeId, host, port, role, capacity, status }: NodeDescriptorParams);
+    get address(): string;
+    isAlive(): boolean;
+    canExecuteFragments(): boolean;
+    canCoordinate(): boolean;
+    assignPartition(partitionId: string): void;
+    removePartition(partitionId: string): void;
+    hasPartition(partitionId: string): boolean;
+    toJSON(): NodeDescriptorJSON;
+    static fromJSON(json: NodeDescriptorJSON): NodeDescriptor;
+}