@housekit/orm 0.1.47 → 0.1.48

package/dist/logger.js

@@ -0,0 +1,36 @@
+export function wrapClientWithLogger(client, logger) {
+    if (!logger)
+        return client;
+    const extractStats = (summary) => {
+        if (!summary)
+            return undefined;
+        const readRows = summary.read_rows ?? summary.rows_read ?? summary.rows ?? summary.result_rows ?? 0;
+        const readBytes = summary.read_bytes ?? summary.bytes_read ?? summary.bytes ?? 0;
+        return { readRows, readBytes };
+    };
+    const withTiming = async (sql, params, fn) => {
+        const start = Date.now();
+        try {
+            const res = await fn();
+            const duration = Date.now() - start;
+            logger.logQuery(sql, params, duration, extractStats(res?.summary));
+            return res;
+        }
+        catch (err) {
+            logger.logError(err, sql);
+            throw err;
+        }
+    };
+    const wrapped = {
+        ...client,
+        query: async (params) => {
+            const sql = typeof params === 'string' ? params : params?.query || '';
+            return withTiming(sql, params?.query_params, () => client.query(params));
+        },
+        insert: async (params) => {
+            const sql = `INSERT INTO ${params?.table ?? ''}`;
+            return withTiming(sql, params?.values, () => client.insert(params));
+        }
+    };
+    return wrapped;
+}

package/dist/materialized-views.d.ts

@@ -1,92 +1,23 @@
-/**
- * HouseKit Materialized Views DSL - Type-Safe Materialized Views
- *
- * Unlike generic ORMs which often treat materialized views as static SQL strings,
- * HouseKit allows defining MV queries using the Query Builder for
- * compile-time type safety. If you rename a column in the source table,
- * TypeScript will catch the error before deployment.
- */
 import { ClickHouseColumn } from './column';
 import { type TableDefinition, type TableColumns } from './table';
 import { EngineConfiguration } from './engines';
 import { type SQLExpression } from './expressions';
-/**
- * Helper type to extract columns from a TableDefinition
- */
 export type InferTableColumns<T> = T extends TableDefinition<infer TCols> ? TCols : TableColumns;
-/**
- * Helper type to ensure we have access to $columns
- */
 type TableWithColumns<TCols extends TableColumns> = {
     $table: string;
     $columns: TCols;
 };
-/**
- * Configuration for type-safe materialized views
- */
 export interface MaterializedViewConfig<TSource extends TableDefinition<any>, TTargetCols extends TableColumns = TableColumns> {
-    /**
-     * Source table(s) for the materialized view.
-     * This is used to provide type information for the query builder.
-     */
     source: TSource;
-    /**
-     * The query definition for the materialized view.
-     * Use the query builder to ensure type safety.
-     *
-     * @example
-     * ```typescript
-     * query: (qb) => qb
-     *   .from(events)
-     *   .select({
-     *     eventType: events.event_type,
-     *     count: sql`count()`
-     *   })
-     *   .groupBy(events.event_type)
-     * ```
-     */
     query: (qb: MaterializedViewQueryBuilder<TSource>) => MaterializedViewQueryBuilder<any>;
-    /**
-     * Target table to write materialized data to.
-     * If not specified, ClickHouse creates an internal table.
-     *
-     * Can be:
-     * - A string table name
-     * - A TableDefinition reference (type-safe)
-     */
     toTable?: string | TableDefinition<TTargetCols>;
-    /**
-     * Cluster name for distributed materialized view
-     */
     onCluster?: string;
-    /**
-     * Whether to populate the MV with existing data on creation.
-     * Warning: Can be slow and resource-intensive for large tables.
-     */
     populate?: boolean;
-    /**
-     * Engine configuration for the internal table (when toTable is not specified).
-     * Uses the type-safe Engine DSL.
-     */
     engine?: EngineConfiguration;
-    /**
-     * Whether to use OR REPLACE semantics
-     */
     orReplace?: boolean;
-    /**
-     * Optional ORDER BY for the internal storage table
-     */
     orderBy?: string | string[];
-    /**
-     * Optional partition by for the internal storage table
-     */
     partitionBy?: string | string[];
 }
-/**
- * Simplified query builder for materialized view definitions.
- * This is a subset of the full ClickHouseQueryBuilder that generates
- * SQL without executing queries.
- */
 export interface MaterializedViewQueryBuilder<TTable extends TableDefinition<any>> {
     select<TSelection extends Record<string, ClickHouseColumn | SQLExpression>>(fields: TSelection): MaterializedViewQueryBuilder<TTable>;
     from<TNewTable extends TableDefinition<any>>(table: TNewTable): MaterializedViewQueryBuilder<TNewTable>;
@@ -102,10 +33,6 @@ export interface MaterializedViewQueryBuilder<TTable extends TableDefinition<any
         params: Record<string, unknown>;
     };
 }
-/**
- * Extended type for materialized views - contains table-like properties
- * plus MV-specific metadata for drift detection and query tracking.
- */
 export type MaterializedViewDefinition<TCols extends TableColumns, TSource extends TableDefinition<any>, TOptions = MaterializedViewConfig<TSource>> = {
     $table: string;
     $columns: TCols;
@@ -114,114 +41,27 @@ export type MaterializedViewDefinition<TCols extends TableColumns, TSource exten
     $source: TSource;
     $querySQL: string;
     $config: TOptions;
-    /**
-     * Get the SQL to create the materialized view
-     */
     toSQL(): string;
-    /**
-     * Get all SQL statements (for views that need multiple statements)
-     */
     toSQLs(): string[];
-    /**
-     * Create an aliased version for use in queries
-     */
     as(alias: string): MaterializedViewDefinition<TCols, TSource, TOptions>;
 } & TCols;
-/**
- * Result type for projection definitions using query builder
- */
 export interface TypedProjectionDefinition<TSource extends TableDefinition<any>> {
     name: string;
     query: string;
     sourceTable: TSource;
     orderBy?: string[];
 }
-/**
- * Create a type-safe materialized view definition.
- * The query is defined using a query builder that validates column references
- * at compile time.
- *
- * @example
- * ```typescript
- * import { defineMaterializedView, text, uint64, sql, Engine } from '@housekit/orm';
- *
- * // Source table
- * const events = defineTable('events', {
- *   event_type: t.text('event_type'),
- *   user_id: t.text('user_id'),
- *   revenue: t.uint64('revenue'),
- * }, { engine: Engine.MergeTree(), orderBy: 'event_type' });
- *
- * // Type-safe materialized view
- * export const revenueByEvent = defineMaterializedView('revenue_by_event_mv', {
- *   event_type: t.text('event_type'),
- *   total_revenue: t.uint64('total_revenue'),
- *   event_count: t.uint64('event_count'),
- * }, {
- *   source: events,
- *   query: (qb) => qb
- *     .from(events)  // Type-safe: only accepts events columns
- *     .select({
- *       event_type: events.event_type,
- *       total_revenue: sql`sum(${events.revenue})`,
- *       event_count: sql`count()`,
- *     })
- *     .groupBy(events.event_type),
- *   engine: Engine.SummingMergeTree(['total_revenue', 'event_count']),
- *   orderBy: 'event_type',
- * });
- * ```
- */
 export declare function chMaterializedView<TCols extends TableColumns, TSource extends TableDefinition<any>, TTarget extends TableColumns = TCols>(name: string, columns: TCols, config: MaterializedViewConfig<TSource, TTarget>): MaterializedViewDefinition<TCols, TSource, MaterializedViewConfig<TSource, TTarget>>;
-/**
- * Create a type-safe projection definition.
- *
- * Projections are precomputed aggregations stored alongside the main table.
- * They're automatically used by ClickHouse when the query matches.
- *
- * @example
- * ```typescript
- * const events = defineTable('events', {
- *   event_type: t.text('event_type'),
- *   user_id: t.text('user_id'),
- *   created_at: t.timestamp('created_at'),
- * }, {
- *   engine: Engine.MergeTree(),
- *   orderBy: ['event_type', 'created_at'],
- *   projections: [
- *     // Type-safe projection
- *     chProjection('events_by_user', events, (cols) => ({
- *       select: {
- *         user_id: cols.user_id,
- *         event_type: cols.event_type,
- *         event_count: sql`count()`,
- *       },
- *       groupBy: [cols.user_id, cols.event_type],
- *       orderBy: ['user_id', 'event_type'],
- *     })),
- *   ],
- * });
- * ```
- */
 export declare function chProjection<TCols extends TableColumns, TSource extends TableDefinition<TCols>>(name: string, sourceTable: TSource & TableWithColumns<TCols>, definition: (cols: TCols) => {
     select: Record<string, ClickHouseColumn | SQLExpression>;
     groupBy?: (ClickHouseColumn | SQLExpression)[];
     orderBy?: string[];
 }): TypedProjectionDefinition<TSource>;
-/**
- * Convert a TypedProjectionDefinition to the format expected by defineTable
- */
 export declare function toProjectionDefinition(projection: TypedProjectionDefinition<any>): {
     name: string;
     query: string;
 };
-/**
- * Normalize SQL for comparison (removes extra whitespace, normalizes case)
- */
 export declare function normalizeSQL(sql: string): string;
-/**
- * Compare two materialized view definitions for drift
- */
 export declare function detectMaterializedViewDrift(local: MaterializedViewDefinition<any, any>, remoteSQL: string): {
     hasDrift: boolean;
     localSQL: string;
@@ -229,38 +69,10 @@ export declare function detectMaterializedViewDrift(local: MaterializedViewDefin
     normalizedLocal: string;
     normalizedRemote: string;
 };
-/**
- * Extract the AS query from a CREATE MATERIALIZED VIEW statement
- */
 export declare function extractMVQuery(createStatement: string): string | null;
-/**
- * Generate a Blue-Green migration plan for a Materialized View update.
- *
- * Instead of just dropping and recreating, we:
- * 1. Create a "next" version of the target table (if using TO table)
- * 2. Create a "next" version of the MV
- * 3. Backfill data from source table using the NEW query
- * 4. Swap them using RENAME
- */
 export declare function generateBlueGreenMigration(oldMV: MaterializedViewDefinition<any, any>, newMV: MaterializedViewDefinition<any, any>, options?: {
     backfill?: boolean;
 }): string[];
-/**
- * Creates a "migration bridge" Materialized View that synchronizes data from an old table to a new one.
- * This is extremely useful for zero-downtime migrations where you need to pipe real-time
- * inserts from the v1 table into the v2 table while you perform a separate historical backfill.
- *
- * @example
- * ```typescript
- * const bridge = createMigrationBridge({
- *   from: tableV1,
- *   to: tableV2,
- *   mapping: {
- *     new_col: sql`upper(${tableV1.old_col})`
- *   }
- * });
- * ```
- */
 export declare function createMigrationBridge<TSourceCols extends TableColumns, TTargetCols extends TableColumns>(options: {
     from: TableDefinition<TSourceCols>;
     to: TableDefinition<TTargetCols>;

package/dist/materialized-views.js

@@ -0,0 +1,380 @@
+import { ClickHouseColumn } from './column';
+import { renderEngineSQL } from './engines';
+class VirtualQueryBuilder {
+    _select = null;
+    _table = null;
+    _where = null;
+    _groupBy = [];
+    _having = null;
+    _orderBy = [];
+    _limit = null;
+    _joins = [];
+    constructor(sourceTable) {
+        if (sourceTable) {
+            this._table = sourceTable;
+        }
+    }
+    select(fields) {
+        this._select = fields;
+        return this;
+    }
+    from(table) {
+        this._table = table;
+        return this;
+    }
+    where(expression) {
+        this._where = expression;
+        return this;
+    }
+    groupBy(...cols) {
+        this._groupBy = cols;
+        return this;
+    }
+    having(expression) {
+        this._having = expression;
+        return this;
+    }
+    orderBy(col, dir = 'ASC') {
+        this._orderBy.push({ col, dir });
+        return this;
+    }
+    limit(val) {
+        this._limit = val;
+        return this;
+    }
+    innerJoin(table, on) {
+        this._joins.push({ type: 'INNER', table: table.$table, on });
+        return this;
+    }
+    leftJoin(table, on) {
+        this._joins.push({ type: 'LEFT', table: table.$table, on });
+        return this;
+    }
+    toSQL() {
+        const parts = [];
+        const params = {};
+        if (this._select) {
+            const selectParts = Object.entries(this._select).map(([alias, col]) => {
+                if (col instanceof ClickHouseColumn) {
+                    const tableName = col.tableName ? `\`${col.tableName}\`.` : '';
+                    return `${tableName}\`${col.name}\` AS \`${alias}\``;
+                }
+                else {
+                    const { sql, params: exprParams } = this.resolveSQLExpression(col);
+                    Object.assign(params, exprParams);
+                    return `${sql} AS \`${alias}\``;
+                }
+            });
+            parts.push(`SELECT ${selectParts.join(', ')}`);
+        }
+        else {
+            parts.push('SELECT *');
+        }
+        if (this._table) {
+            parts.push(`FROM \`${this._table.$table}\``);
+        }
+        for (const join of this._joins) {
+            const { sql: onSQL } = this.resolveSQLExpression(join.on);
+            parts.push(`${join.type} JOIN \`${join.table}\` ON ${onSQL}`);
+        }
+        if (this._where) {
+            const { sql: whereSQL, params: whereParams } = this.resolveSQLExpression(this._where);
+            Object.assign(params, whereParams);
+            parts.push(`WHERE ${whereSQL}`);
+        }
+        if (this._groupBy.length > 0) {
+            const groupParts = this._groupBy.map(col => {
+                if (col instanceof ClickHouseColumn) {
+                    const tableName = col.tableName ? `\`${col.tableName}\`.` : '';
+                    return `${tableName}\`${col.name}\``;
+                }
+                else {
+                    const { sql } = this.resolveSQLExpression(col);
+                    return sql;
+                }
+            });
+            parts.push(`GROUP BY ${groupParts.join(', ')}`);
+        }
+        if (this._having) {
+            const { sql: havingSQL, params: havingParams } = this.resolveSQLExpression(this._having);
+            Object.assign(params, havingParams);
+            parts.push(`HAVING ${havingSQL}`);
+        }
+        if (this._orderBy.length > 0) {
+            const orderParts = this._orderBy.map(({ col, dir }) => {
+                if (col instanceof ClickHouseColumn) {
+                    const tableName = col.tableName ? `\`${col.tableName}\`.` : '';
+                    return `${tableName}\`${col.name}\` ${dir}`;
+                }
+                else {
+                    const { sql } = this.resolveSQLExpression(col);
+                    return `${sql} ${dir}`;
+                }
+            });
+            parts.push(`ORDER BY ${orderParts.join(', ')}`);
+        }
+        if (this._limit !== null) {
+            parts.push(`LIMIT ${this._limit}`);
+        }
+        return { query: parts.join(' '), params };
+    }
+    resolveSQLExpression(expr) {
+        const params = {};
+        if (typeof expr.toSQL === 'function') {
+            const result = expr.toSQL();
+            return { sql: result.sql || String(result), params: result.params || {} };
+        }
+        if (typeof expr.sql === 'string') {
+            return { sql: expr.sql, params: expr.params || {} };
+        }
+        if (Array.isArray(expr.strings)) {
+            const exprAny = expr;
+            let sql = '';
+            const strings = exprAny.strings;
+            const values = exprAny.values;
+            for (let i = 0; i < strings.length; i++) {
+                sql += strings[i];
+                if (i < values.length) {
+                    const value = values[i];
+                    if (value instanceof ClickHouseColumn) {
+                        const tableName = value.tableName ? `\`${value.tableName}\`.` : '';
+                        sql += `${tableName}\`${value.name}\``;
+                    }
+                    else if (typeof value === 'object' && value !== null && 'sql' in value) {
+                        const nested = this.resolveSQLExpression(value);
+                        sql += nested.sql;
+                        Object.assign(params, nested.params);
+                    }
+                    else {
+                        const paramName = `p${Object.keys(params).length}`;
+                        params[paramName] = value;
+                        sql += `{${paramName}: String}`;
+                    }
+                }
+            }
+            return { sql, params };
+        }
+        return { sql: String(expr), params };
+    }
+}
+export function chMaterializedView(name, columns, config) {
+    const virtualBuilder = new VirtualQueryBuilder(config.source);
+    const finalBuilder = config.query(virtualBuilder);
+    const { query: querySQL, params } = finalBuilder.toSQL();
+    for (const col of Object.values(columns)) {
+        col.tableName = name;
+    }
+    const toSQL = () => {
+        const parts = [];
+        if (config.orReplace) {
+            parts.push('CREATE OR REPLACE MATERIALIZED VIEW');
+        }
+        else {
+            parts.push('CREATE MATERIALIZED VIEW IF NOT EXISTS');
+        }
+        parts.push(`\`${name}\``);
+        if (config.onCluster) {
+            parts.push(`ON CLUSTER ${config.onCluster}`);
+        }
+        if (config.toTable) {
+            const tableName = typeof config.toTable === 'string'
+                ? config.toTable
+                : config.toTable.$table;
+            parts.push(`TO \`${tableName}\``);
+        }
+        else if (config.engine) {
+            const engineSQL = renderEngineSQL(config.engine);
+            parts.push(`ENGINE = ${engineSQL}`);
+            if (config.orderBy) {
+                const orderBy = Array.isArray(config.orderBy)
+                    ? config.orderBy.join(', ')
+                    : config.orderBy;
+                parts.push(`ORDER BY (${orderBy})`);
+            }
+            if (config.partitionBy) {
+                const partitionBy = Array.isArray(config.partitionBy)
+                    ? config.partitionBy.join(', ')
+                    : config.partitionBy;
+                parts.push(`PARTITION BY (${partitionBy})`);
+            }
+        }
+        if (config.populate) {
+            parts.push('POPULATE');
+        }
+        parts.push('AS');
+        parts.push(querySQL);
+        return parts.join(' ');
+    };
+    const definition = {
+        $table: name,
+        $columns: columns,
+        $options: config,
+        $kind: 'materializedView',
+        $source: config.source,
+        $querySQL: querySQL,
+        $config: config,
+        toSQL,
+        toSQLs: () => [toSQL()],
+        as: (alias) => {
+            const aliased = {
+                $table: alias,
+                $columns: { ...columns },
+                $options: { ...config, externallyManaged: true },
+                $kind: 'materializedView',
+                toSQL: () => ''
+            };
+            for (const [key, col] of Object.entries(columns)) {
+                const column = col;
+                const cloned = Object.create(Object.getPrototypeOf(column));
+                Object.assign(cloned, column);
+                cloned.tableName = alias;
+                aliased[key] = cloned;
+                aliased.$columns[key] = cloned;
+            }
+            return aliased;
+        }
+    };
+    for (const [key, col] of Object.entries(columns)) {
+        definition[key] = col;
+    }
+    return definition;
+}
+export function chProjection(name, sourceTable, definition) {
+    const cols = sourceTable.$columns;
+    const def = definition(cols);
+    const selectParts = Object.entries(def.select).map(([alias, col]) => {
+        if (col instanceof ClickHouseColumn) {
+            return `\`${col.name}\` AS \`${alias}\``;
+        }
+        else {
+            const sqlExpr = col;
+            if (typeof sqlExpr.toSQL === 'function') {
+                const { sql: sqlStr } = sqlExpr.toSQL({ ignoreTablePrefix: true });
+                return `${sqlStr} AS \`${alias}\``;
+            }
+            return `${String(col)} AS \`${alias}\``;
+        }
+    });
+    let query = `SELECT ${selectParts.join(', ')}`;
+    if (def.groupBy && def.groupBy.length > 0) {
+        const groupParts = def.groupBy.map(col => {
+            if (col instanceof ClickHouseColumn) {
+                return `\`${col.name}\``;
+            }
+            const sqlExpr = col;
+            if (typeof sqlExpr.toSQL === 'function') {
+                const { sql: sqlStr } = sqlExpr.toSQL({ ignoreTablePrefix: true });
+                return sqlStr;
+            }
+            return String(col);
+        });
+        query += ` GROUP BY ${groupParts.join(', ')}`;
+    }
+    if (def.orderBy && def.orderBy.length > 0) {
+        query += ` ORDER BY ${def.orderBy.join(', ')}`;
+    }
+    return {
+        name,
+        query,
+        sourceTable,
+        orderBy: def.orderBy
+    };
+}
+export function toProjectionDefinition(projection) {
+    return {
+        name: projection.name,
+        query: projection.query
+    };
+}
+export function normalizeSQL(sql) {
+    return sql
+        .replace(/\s+/g, ' ')
+        .replace(/\(\s+/g, '(')
+        .replace(/\s+\)/g, ')')
+        .replace(/,\s+/g, ', ')
+        .trim()
+        .toUpperCase();
+}
+function hashString(str) {
+    let hash = 0;
+    for (let i = 0; i < str.length; i++) {
+        const char = str.charCodeAt(i);
+        hash = ((hash << 5) - hash) + char;
+        hash = hash & hash;
+    }
+    return Math.abs(hash).toString(16);
+}
+export function detectMaterializedViewDrift(local, remoteSQL) {
+    const localSQL = local.toSQL();
+    const normalizedLocal = normalizeSQL(localSQL);
+    const normalizedRemote = normalizeSQL(remoteSQL);
+    return {
+        hasDrift: normalizedLocal !== normalizedRemote,
+        localSQL,
+        remoteSQL,
+        normalizedLocal,
+        normalizedRemote
+    };
+}
+export function extractMVQuery(createStatement) {
+    const match = createStatement.match(/\bAS\s+(.+)$/is);
+    return match ? match[1].trim() : null;
+}
+export function generateBlueGreenMigration(oldMV, newMV, options = {}) {
+    const name = newMV.$table;
+    const nextName = `${name}_next`;
+    const targetTable = newMV.$options.toTable;
+    const sqls = [];
+    if (targetTable) {
+        const targetTableName = typeof targetTable === 'string' ? targetTable : targetTable.$table;
+        const nextTargetName = `${targetTableName}_next`;
+        if (typeof targetTable !== 'string') {
+            const nextTableSQL = targetTable.toSQL().replace(new RegExp(`\`${targetTableName}\``, 'g'), `\`${nextTargetName}\``);
+            sqls.push(nextTableSQL);
+        }
+        else {
+            sqls.push(`CREATE TABLE \`${nextTargetName}\` AS \`${targetTableName}\``);
+        }
+        const nextMVSQL = newMV.toSQL()
+            .replace(new RegExp(`\`${name}\``, 'g'), `\`${nextName}\``)
+            .replace(new RegExp(`TO \`${targetTableName}\``, 'g'), `TO \`${nextTargetName}\``);
+        sqls.push(nextMVSQL);
+        if (options.backfill) {
+            const querySQL = newMV.$querySQL;
+            sqls.push(`INSERT INTO \`${nextTargetName}\` ${querySQL}`);
+        }
+        sqls.push(`EXCHANGE TABLES \`${name}\` AND \`${nextName}\``);
+        sqls.push(`EXCHANGE TABLES \`${targetTableName}\` AND \`${nextTargetName}\``);
+        sqls.push(`DROP VIEW IF EXISTS \`${nextName}\``);
+        sqls.push(`DROP TABLE IF EXISTS \`${nextTargetName}\``);
+    }
+    else {
+        sqls.push(`DROP VIEW IF EXISTS \`${name}\``);
+        sqls.push(newMV.toSQL());
+    }
+    return sqls;
+}
+export function createMigrationBridge(options) {
+    const fromName = options.from.$table;
+    const toName = options.to.$table;
+    const bridgeName = options.name || `${fromName}_to_${toName}_bridge_mv`;
+    const selection = {};
+    if (options.mapping) {
+        Object.assign(selection, options.mapping);
+    }
+    else {
+        const fromCols = options.from.$columns;
+        const toCols = options.to.$columns;
+        for (const key of Object.keys(fromCols)) {
+            if (toCols[key]) {
+                selection[key] = fromCols[key];
+            }
+        }
+    }
+    return chMaterializedView(bridgeName, options.to.$columns, {
+        source: options.from,
+        toTable: options.to,
+        onCluster: options.onCluster,
+        query: (qb) => qb.from(options.from).select(selection)
+    });
+}