@housekit/orm 0.1.1

package/dist/engines.d.ts

@@ -0,0 +1,558 @@
+/**
+ * HouseKit Engine DSL - First-class ClickHouse Engine Support
+ *
+ * This module provides type-safe engine configurations for ClickHouse tables.
+ * Unlike generic ORMs which often treat engines as raw strings, HouseKit validates engine
+ * parameters at compile-time and provides intelligent defaults.
+ */
+import type { TableDefinition, TableColumns } from './table';
+/**
+ * Base configuration shared by all MergeTree-family engines
+ */
+export interface MergeTreeBaseConfig {
+    /**
+     * Experimental: Enable lightweight DELETE/UPDATE operations
+     * Requires ClickHouse 23.3+
+     */
+    enableLightweightDeletes?: boolean;
+}
+/**
+ * Configuration for basic MergeTree engine
+ */
+export interface MergeTreeConfig extends MergeTreeBaseConfig {
+    type: 'MergeTree';
+}
+/**
+ * Configuration for ReplacingMergeTree engine
+ * Used for deduplication scenarios where the last version of a row should be kept
+ */
+export interface ReplacingMergeTreeConfig extends MergeTreeBaseConfig {
+    type: 'ReplacingMergeTree';
+    /** Column used to determine which row version to keep (newer wins) */
+    versionColumn?: string;
+    /**
+     * Column indicating if row is deleted (ClickHouse 23.2+)
+     * When 1, the row is considered deleted during FINAL merges
+     */
+    isDeletedColumn?: string;
+}
+/**
+ * Configuration for SummingMergeTree engine
+ * Automatically sums numeric columns during merges
+ */
+export interface SummingMergeTreeConfig extends MergeTreeBaseConfig {
+    type: 'SummingMergeTree';
+    /** Columns to sum. If empty, sums all numeric columns not in ORDER BY */
+    columns?: string[];
+}
+/**
+ * Configuration for AggregatingMergeTree engine
+ * Used with AggregateFunction columns for pre-aggregated materialized views
+ */
+export interface AggregatingMergeTreeConfig extends MergeTreeBaseConfig {
+    type: 'AggregatingMergeTree';
+}
+/**
+ * Configuration for CollapsingMergeTree engine
+ * Uses a sign column to collapse pairs of rows with opposite signs
+ */
+export interface CollapsingMergeTreeConfig extends MergeTreeBaseConfig {
+    type: 'CollapsingMergeTree';
+    /** Column containing 1 or -1 to indicate row state */
+    signColumn: string;
+}
+/**
+ * Configuration for VersionedCollapsingMergeTree engine
+ * Like CollapsingMergeTree but with version column for more robust deduplication
+ */
+export interface VersionedCollapsingMergeTreeConfig extends MergeTreeBaseConfig {
+    type: 'VersionedCollapsingMergeTree';
+    /** Column containing 1 or -1 to indicate row state */
+    signColumn: string;
+    /** Version column for ordering rows */
+    versionColumn: string;
+}
+/**
+ * Configuration for GraphiteMergeTree engine
+ * Optimized for storing Graphite metrics data
+ */
+export interface GraphiteMergeTreeConfig extends MergeTreeBaseConfig {
+    type: 'GraphiteMergeTree';
+    /** Name of the Graphite rollup configuration */
+    configSection: string;
+}
+/**
+ * Configuration for ReplicatedMergeTree engine
+ * Provides data replication across ClickHouse cluster nodes
+ */
+export interface ReplicatedMergeTreeConfig extends MergeTreeBaseConfig {
+    type: 'ReplicatedMergeTree';
+    /**
+     * Path in ZooKeeper for this table's replication
+     * Supports macros: {shard}, {replica}, {database}, {table}
+     * @default '/clickhouse/tables/{shard}/{database}/{table}'
+     */
+    zkPath?: string;
+    /**
+     * Unique replica identifier
+     * Supports macros: {replica}, {hostname}
+     * @default '{replica}'
+     */
+    replicaName?: string;
+    /** Base engine type to replicate (defaults to MergeTree) */
+    baseEngine?: 'MergeTree' | 'ReplacingMergeTree' | 'SummingMergeTree' | 'AggregatingMergeTree' | 'CollapsingMergeTree' | 'VersionedCollapsingMergeTree';
+    /** Configuration for ReplacingMergeTree base */
+    versionColumn?: string;
+    isDeletedColumn?: string;
+    /** Configuration for SummingMergeTree base */
+    sumColumns?: string[];
+    /** Configuration for CollapsingMergeTree base */
+    signColumn?: string;
+}
+/**
+ * Configuration for Buffer engine
+ * Buffers writes in memory before flushing to a target table
+ * Excellent for high-throughput insert scenarios
+ */
+export interface BufferConfig {
+    type: 'Buffer';
+    /** Database of the target table (use 'currentDatabase()' for same database) */
+    database: string;
+    /** Name of the target table */
+    table: string;
+    /** Number of buffer layers (typically 16) */
+    layers: number;
+    /** Minimum time (seconds) before flush */
+    minTime: number;
+    /** Maximum time (seconds) before flush */
+    maxTime: number;
+    /** Minimum rows before flush */
+    minRows: number;
+    /** Maximum rows before flush */
+    maxRows: number;
+    /** Minimum bytes before flush */
+    minBytes: number;
+    /** Maximum bytes before flush */
+    maxBytes: number;
+}
+/**
+ * Configuration for Distributed engine
+ * Routes queries across a cluster of ClickHouse nodes
+ */
+export interface DistributedConfig {
+    type: 'Distributed';
+    /** Cluster name as defined in ClickHouse configuration */
+    cluster: string;
+    /** Database name on remote servers */
+    database: string;
+    /** Table name on remote servers */
+    table: string;
+    /**
+     * Expression to determine which shard receives each row
+     * @default 'rand()'
+     */
+    shardingKey?: string;
+    /** Policy name for selecting replicas */
+    policyName?: string;
+}
+/**
+ * Configuration for Null engine
+ * Data is discarded (useful for testing or as data sink)
+ */
+export interface NullConfig {
+    type: 'Null';
+}
+/**
+ * Configuration for Log engine
+ * Simple append-only storage, no indices
+ */
+export interface LogConfig {
+    type: 'Log';
+}
+/**
+ * Configuration for TinyLog engine
+ * Like Log but stores each column in a separate file
+ */
+export interface TinyLogConfig {
+    type: 'TinyLog';
+}
+/**
+ * Configuration for Memory engine
+ * Stores data in RAM, lost on restart
+ */
+export interface MemoryConfig {
+    type: 'Memory';
+    /** Maximum number of rows to store */
+    maxRows?: number;
+    /** Maximum bytes to store */
+    maxBytes?: number;
+    /** Compress data in memory */
+    compress?: boolean;
+}
+/**
+ * Configuration for Join engine
+ * Stores data for JOIN operations
+ */
+export interface JoinConfig {
+    type: 'Join';
+    /** Join strictness: any match or all matches */
+    strictness: 'Any' | 'All' | 'Semi' | 'Anti';
+    /** Join type */
+    joinType: 'Inner' | 'Left' | 'Right' | 'Full' | 'Cross';
+    /** Key columns for the join */
+    keys: string[];
+}
+/**
+ * Configuration for Dictionary engine
+ * Wraps a ClickHouse dictionary as a table
+ */
+export interface DictionaryConfig {
+    type: 'Dictionary';
+    /** Name of the dictionary */
+    dictionaryName: string;
+}
+/**
+ * Configuration for File engine
+ * Reads/writes data from/to a file in a specified format
+ */
+export interface FileConfig {
+    type: 'File';
+    /** Data format (e.g., 'TabSeparated', 'CSV', 'JSONEachRow') */
+    format: string;
+    /** Optional: compression type */
+    compression?: 'none' | 'gzip' | 'lz4' | 'zstd';
+}
+/**
+ * Configuration for URL engine
+ * Reads data from a remote URL
+ */
+export interface URLConfig {
+    type: 'URL';
+    /** URL to read from */
+    url: string;
+    /** Data format */
+    format: string;
+    /** Optional: compression type */
+    compression?: 'none' | 'gzip' | 'lz4' | 'zstd';
+}
+/**
+ * Configuration for S3 engine
+ * Reads/writes data to Amazon S3
+ */
+export interface S3Config {
+    type: 'S3';
+    /** S3 URL pattern */
+    path: string;
+    /** Data format */
+    format: string;
+    /** Optional: AWS access key ID */
+    accessKeyId?: string;
+    /** Optional: AWS secret access key */
+    secretAccessKey?: string;
+    /** Optional: compression type */
+    compression?: 'none' | 'gzip' | 'lz4' | 'zstd';
+}
+/**
+ * Configuration for Kafka engine
+ * Consumes messages from Apache Kafka
+ */
+export interface KafkaConfig {
+    type: 'Kafka';
+    /** Kafka broker list */
+    brokerList: string;
+    /** Topic name(s) */
+    topicList: string | string[];
+    /** Consumer group ID */
+    groupName: string;
+    /** Data format for messages */
+    format: string;
+    /** Number of polling threads */
+    numConsumers?: number;
+    /** Max rows per poll */
+    maxBlockSize?: number;
+    /** Skip broken messages */
+    skipBroken?: number;
+}
+/**
+ * Configuration for PostgreSQL engine
+ * Reads data from PostgreSQL database
+ */
+export interface PostgreSQLConfig {
+    type: 'PostgreSQL';
+    /** PostgreSQL host */
+    host: string;
+    /** PostgreSQL port */
+    port: number;
+    /** Database name */
+    database: string;
+    /** Table name */
+    table: string;
+    /** Username */
+    user: string;
+    /** Password */
+    password: string;
+    /** Schema name */
+    schema?: string;
+}
+/**
+ * Configuration for MySQL engine
+ * Reads data from MySQL database
+ */
+export interface MySQLConfig {
+    type: 'MySQL';
+    /** MySQL host */
+    host: string;
+    /** MySQL port */
+    port: number;
+    /** Database name */
+    database: string;
+    /** Table name */
+    table: string;
+    /** Username */
+    user: string;
+    /** Password */
+    password: string;
+}
+/**
+ * Union type of all supported engine configurations
+ */
+export type EngineConfiguration = MergeTreeConfig | ReplacingMergeTreeConfig | SummingMergeTreeConfig | AggregatingMergeTreeConfig | CollapsingMergeTreeConfig | VersionedCollapsingMergeTreeConfig | GraphiteMergeTreeConfig | ReplicatedMergeTreeConfig | BufferConfig | DistributedConfig | NullConfig | LogConfig | TinyLogConfig | MemoryConfig | JoinConfig | DictionaryConfig | FileConfig | URLConfig | S3Config | KafkaConfig | PostgreSQLConfig | MySQLConfig;
+/**
+ * Factory functions for creating type-safe engine configurations.
+ *
+ * @example
+ * ```typescript
+ * import { table, t, Engine } from '@housekit/orm';
+ *
+ * const events = table('events', (t) => ({
+ *   id: text('id'),
+ *   timestamp: timestamp('timestamp'),
+ * }, {
+ *   engine: Engine.ReplicatedMergeTree(),
+ *   orderBy: 'timestamp',
+ * });
+ * ```
+ */
+export declare const Engine: {
+    /**
+     * The most versatile and powerful ClickHouse engine.
+     * Designed for high-volume data insertion.
+     * @see https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree
+     */
+    MergeTree: (options?: Omit<MergeTreeConfig, "type">) => MergeTreeConfig;
+    /**
+     * Removes duplicates with the same sorting key during merges.
+     * @param versionColumn - Column (UInt* or DateTime) to determine which row is the latest.
+     */
+    ReplacingMergeTree: (versionColumn?: string, isDeletedColumn?: string, options?: Omit<ReplacingMergeTreeConfig, "type" | "versionColumn" | "isDeletedColumn">) => ReplacingMergeTreeConfig;
+    /**
+     * SummingMergeTree - automatic summation of numeric columns during merges
+     *
+     * @example
+     * ```typescript
+     * engine: Engine.SummingMergeTree(['amount', 'count'])
+     * ```
+     */
+    SummingMergeTree: (columns?: string[], options?: Omit<SummingMergeTreeConfig, "type" | "columns">) => SummingMergeTreeConfig;
+    /**
+     * AggregatingMergeTree - for use with AggregateFunction columns
+     */
+    AggregatingMergeTree: (options?: Omit<AggregatingMergeTreeConfig, "type">) => AggregatingMergeTreeConfig;
+    /**
+     * CollapsingMergeTree - uses sign column to collapse row pairs
+     *
+     * @example
+     * ```typescript
+     * engine: Engine.CollapsingMergeTree('sign')
+     * ```
+     */
+    CollapsingMergeTree: (signColumn: string, options?: Omit<CollapsingMergeTreeConfig, "type" | "signColumn">) => CollapsingMergeTreeConfig;
+    /**
+     * VersionedCollapsingMergeTree - CollapsingMergeTree with version support
+     *
+     * @example
+     * ```typescript
+     * engine: Engine.VersionedCollapsingMergeTree('sign', 'version')
+     * ```
+     */
+    VersionedCollapsingMergeTree: (signColumn: string, versionColumn: string, options?: Omit<VersionedCollapsingMergeTreeConfig, "type" | "signColumn" | "versionColumn">) => VersionedCollapsingMergeTreeConfig;
+    /**
+     * GraphiteMergeTree - optimized for Graphite metrics
+     *
+     * @example
+     * ```typescript
+     * engine: Engine.GraphiteMergeTree('graphite_rollup')
+     * ```
+     */
+    GraphiteMergeTree: (configSection: string, options?: Omit<GraphiteMergeTreeConfig, "type" | "configSection">) => GraphiteMergeTreeConfig;
+    /**
+     * ReplicatedMergeTree - data replication across cluster nodes
+     *
+     * HouseKit provides sensible defaults using ClickHouse macros that work
+     * out-of-the-box in most cluster configurations.
+     *
+     * @example
+     * ```typescript
+     * // Basic usage with defaults
+     * engine: Engine.ReplicatedMergeTree()
+     *
+     * // Custom ZK path
+     * engine: Engine.ReplicatedMergeTree({
+     *   zkPath: '/clickhouse/prod/tables/{shard}/events',
+     *   replicaName: '{replica}'
+     * })
+     *
+     * // Replicated ReplacingMergeTree
+     * engine: Engine.ReplicatedMergeTree({
+     *   baseEngine: 'ReplacingMergeTree',
+     *   versionColumn: 'updated_at'
+     * })
+     * ```
+     */
+    ReplicatedMergeTree: (config?: Omit<ReplicatedMergeTreeConfig, "type">) => ReplicatedMergeTreeConfig;
+    /**
+     * Buffer engine - buffers inserts before flushing to a target table
+     *
+     * Excellent for high-throughput scenarios where you want to reduce
+     * the number of parts created by batching inserts.
+     *
+     * @example
+     * ```typescript
+     * // Create buffer with target table reference
+     * const eventsBuffer = table('events_buffer', events.$columns, {
+     *   engine: Engine.Buffer(events, { minRows: 1000, maxRows: 10000 })
+     * });
+     * ```
+     */
+    Buffer: <T extends TableColumns>(targetTable: TableDefinition<T>, opts: {
+        minRows: number;
+        maxRows: number;
+        layers?: number;
+        minTime?: number;
+        maxTime?: number;
+        minBytes?: number;
+        maxBytes?: number;
+    }) => BufferConfig;
+    /**
+     * Buffer engine with explicit database and table names
+     */
+    BufferExplicit: (config: Omit<BufferConfig, "type">) => BufferConfig;
+    /**
+     * Distributed engine - distributes queries across cluster shards
+     *
+     * @example
+     * ```typescript
+     * const eventsDistributed = table('events_distributed', events.$columns, {
+     *   engine: Engine.Distributed({
+     *     cluster: 'my_cluster',
+     *     database: 'default',
+     *     table: 'events_local',
+     *     shardingKey: 'user_id'
+     *   })
+     * });
+     * ```
+     */
+    Distributed: (config: Omit<DistributedConfig, "type">) => DistributedConfig;
+    /**
+     * Null engine - discards all data (useful for testing)
+     */
+    Null: () => NullConfig;
+    /**
+     * Log engine - simple append-only storage
+     */
+    Log: () => LogConfig;
+    /**
+     * TinyLog engine - lightweight logging
+     */
+    TinyLog: () => TinyLogConfig;
+    /**
+     * Memory engine - stores all data in RAM
+     *
+     * @example
+     * ```typescript
+     * engine: Engine.Memory({ maxRows: 100000 })
+     * ```
+     */
+    Memory: (config?: Omit<MemoryConfig, "type">) => MemoryConfig;
+    /**
+     * Join engine - optimized for JOIN operations
+     *
+     * @example
+     * ```typescript
+     * engine: Engine.Join('Any', 'Left', ['user_id'])
+     * ```
+     */
+    Join: (strictness: JoinConfig["strictness"], joinType: JoinConfig["joinType"], keys: string[]) => JoinConfig;
+    /**
+     * Dictionary engine - wraps a dictionary as a table
+     */
+    Dictionary: (dictionaryName: string) => DictionaryConfig;
+    /**
+     * File engine - read/write from files
+     *
+     * @example
+     * ```typescript
+     * engine: Engine.File('CSV')
+     * engine: Engine.File('JSONEachRow', 'gzip')
+     * ```
+     */
+    File: (format: string, compression?: FileConfig["compression"]) => FileConfig;
+    /**
+     * URL engine - read from remote URLs
+     */
+    URL: (url: string, format: string, compression?: URLConfig["compression"]) => URLConfig;
+    /**
+     * S3 engine - read/write to Amazon S3
+     *
+     * @example
+     * ```typescript
+     * engine: Engine.S3({
+     *   path: 's3://bucket/path/data.parquet',
+     *   format: 'Parquet'
+     * })
+     * ```
+     */
+    S3: (config: Omit<S3Config, "type">) => S3Config;
+    /**
+     * Kafka engine - consume from Kafka topics
+     *
+     * @example
+     * ```typescript
+     * engine: Engine.Kafka({
+     *   brokerList: 'kafka:9092',
+     *   topicList: 'events',
+     *   groupName: 'clickhouse_consumer',
+     *   format: 'JSONEachRow'
+     * })
+     * ```
+     */
+    Kafka: (config: Omit<KafkaConfig, "type">) => KafkaConfig;
+    /**
+     * PostgreSQL engine - read from PostgreSQL
+     */
+    PostgreSQL: (config: Omit<PostgreSQLConfig, "type">) => PostgreSQLConfig;
+    /**
+     * MySQL engine - read from MySQL
+     */
+    MySQL: (config: Omit<MySQLConfig, "type">) => MySQLConfig;
+};
+/**
+ * Renders an EngineConfiguration to its SQL representation.
+ * This is used internally by defineTable() but can be useful for debugging.
+ *
+ * @param engine - The engine configuration or raw string
+ * @returns SQL string for the ENGINE clause
+ */
+export declare function renderEngineSQL(engine: EngineConfiguration | undefined): string;
+/**
+ * Checks if an engine configuration is from the MergeTree family
+ */
+export declare function isMergeTreeFamily(engine: string | EngineConfiguration | undefined): boolean;
+/**
+ * Checks if an engine configuration is replicated
+ */
+export declare function isReplicatedEngine(engine: string | EngineConfiguration | undefined): boolean;
+/**
+ * Extracts the version column from an engine configuration if applicable
+ */
+export declare function getVersionColumn(engine: string | EngineConfiguration | undefined): string | undefined;

package/dist/expressions.d.ts

@@ -0,0 +1,72 @@
+import { ClickHouseColumn } from './core';
+export type SQLWrapper = SQLExpression | ClickHouseColumn | SQLValue;
+export type SQLValue = string | number | boolean | Date | null | string[] | number[];
+export interface ToSQLOptions {
+    ignoreTablePrefix?: boolean;
+    table?: {
+        $columns: Record<string, any>;
+    };
+}
+export type AliasedExpression<TResult = any, TAlias extends string = string> = SQLExpression<TResult> & {
+    _alias: TAlias;
+};
+export interface SQLExpression<TResult = any> {
+    _type: TResult;
+    toSQL(options?: ToSQLOptions): {
+        sql: string;
+        params: Record<string, unknown>;
+    };
+    as<TAlias extends string>(alias: TAlias): AliasedExpression<TResult, TAlias>;
+    walk(visitor: (value: any, type: string) => void): string;
+}
+export declare class SQL<TResult = any> implements SQLExpression<TResult> {
+    readonly queryChunks: string[];
+    readonly params: any[];
+    readonly _type: TResult;
+    constructor(queryChunks: string[], params: any[]);
+    as<TAlias extends string>(alias: TAlias): AliasedExpression<TResult, TAlias>;
+    walk(visitor: (value: any, type: string) => void): string;
+    private formatColumn;
+    toSQL(options?: ToSQLOptions): {
+        sql: string;
+        params: Record<string, unknown>;
+    };
+}
+export declare function sql<T = any>(strings: TemplateStringsArray, ...args: any[]): SQLExpression<T>;
+export declare namespace sql {
+    var raw: (rawSql: string) => SQLExpression;
+    var join: (expressions: (SQLExpression | ClickHouseColumn | SQLValue)[], separator?: SQLExpression | string) => SQLExpression;
+}
+/**
+ * Typed SQL helper that preserves types for columns from a table definition.
+ */
+export declare function typedSQL<T extends Record<string, ClickHouseColumn>>(strings: TemplateStringsArray, ...args: Array<SQLValue | SQLExpression | T[keyof T]>): SQL<any>;
+/**
+ * Generic function call helper for any ClickHouse function
+ * @param name Function name
+ * @param args Function arguments
+ * @example fn('hex', md5(users.email))
+ * @example fn('length', users.interests)
+ */
+export declare function fn(name: string, ...args: (ClickHouseColumn | SQLExpression | SQLValue)[]): SQLExpression;
+export declare function eq<T>(col: ClickHouseColumn<T, any, any> | SQLExpression, val: T | SQLValue | ClickHouseColumn | SQLExpression): SQLExpression<any>;
+export declare function ne<T>(col: ClickHouseColumn<T, any, any> | SQLExpression, val: T | SQLValue | ClickHouseColumn | SQLExpression): SQLExpression<any>;
+export declare function gt<T>(col: ClickHouseColumn<T, any, any> | SQLExpression, val: T | SQLValue | ClickHouseColumn | SQLExpression): SQLExpression<any>;
+export declare function gte<T>(col: ClickHouseColumn<T, any, any> | SQLExpression, val: T | SQLValue | ClickHouseColumn | SQLExpression): SQLExpression<any>;
+export declare function lt<T>(col: ClickHouseColumn<T, any, any> | SQLExpression, val: T | SQLValue | ClickHouseColumn | SQLExpression): SQLExpression<any>;
+export declare function lte<T>(col: ClickHouseColumn<T, any, any> | SQLExpression, val: T | SQLValue | ClickHouseColumn | SQLExpression): SQLExpression<any>;
+export declare function inArray(col: ClickHouseColumn | SQLExpression, values: SQLValue[] | SQLExpression): SQLExpression<any>;
+export declare function notInArray(col: ClickHouseColumn | SQLExpression, values: SQLValue[] | SQLExpression): SQLExpression<any>;
+export declare function between(col: ClickHouseColumn | SQLExpression, min: SQLValue, max: SQLValue): SQLExpression<any>;
+export declare function notBetween(col: ClickHouseColumn | SQLExpression, min: SQLValue, max: SQLValue): SQLExpression<any>;
+export declare function has(col: ClickHouseColumn | SQLExpression, value: SQLValue): SQLExpression<any>;
+export declare function hasAll(col: ClickHouseColumn | SQLExpression, values: SQLValue[]): SQLExpression<any>;
+export declare function hasAny(col: ClickHouseColumn | SQLExpression, values: SQLValue[]): SQLExpression<any>;
+export declare function asc(col: ClickHouseColumn | SQLExpression): {
+    col: ClickHouseColumn<any, true, false> | SQLExpression<any>;
+    dir: "ASC";
+};
+export declare function desc(col: ClickHouseColumn | SQLExpression): {
+    col: ClickHouseColumn<any, true, false> | SQLExpression<any>;
+    dir: "DESC";
+};