@housekit/orm 0.1.47 → 0.1.49

package/dist/utils/binary-worker-pool.d.ts

@@ -1,9 +1,3 @@
-/**
- * HouseKit Binary Worker Pool - High-Performance Parallel Serialization
- *
- * Manages a pool of worker threads for parallel RowBinary serialization.
- * Automatically distributes work across available CPU cores.
- */
 import { EventEmitter } from 'events';
 interface ColumnConfig {
     name: string;
@@ -11,11 +5,8 @@ interface ColumnConfig {
     isNullable: boolean;
 }
 export interface BinaryWorkerPoolOptions {
-    /** Number of worker threads (default: CPU cores - 1) */
     poolSize?: number;
-    /** Maximum rows per batch sent to a worker */
     batchSize?: number;
-    /** Enable high-water mark backpressure */
     highWaterMark?: number;
 }
 export declare class BinaryWorkerPool extends EventEmitter {
@@ -26,46 +17,21 @@ export declare class BinaryWorkerPool extends EventEmitter {
     private isShutdown;
     private readonly options;
     constructor(options?: BinaryWorkerPoolOptions);
-    /**
-     * Initialize the worker pool with column configuration
-     */
     initialize(columns: ColumnConfig[]): Promise<void>;
     private spawnWorker;
     private handleWorkerMessage;
-    /**
-     * Serialize rows to binary format using worker pool
-     */
     serialize(rows: Array<Record<string, any>>): Promise<Buffer>;
-    /**
-     * Serialize rows in batches, returning an async iterator of buffers
-     */
     serializeStream(rows: AsyncIterable<Record<string, any>> | Iterable<Record<string, any>>): AsyncGenerator<Buffer>;
     private processQueue;
-    /**
-     * Get pool statistics
-     */
     getStats(): {
         workers: number;
         busy: number;
         queueSize: number;
     };
-    /**
-     * Shutdown all workers
-     */
     shutdown(): Promise<void>;
 }
-/**
- * Get or create the default worker pool
- */
 export declare function getDefaultBinaryPool(columns?: ColumnConfig[]): Promise<BinaryWorkerPool>;
-/**
- * Shutdown the default pool
- */
 export declare function shutdownDefaultBinaryPool(): Promise<void>;
-/**
- * Synchronous fallback serializer for environments without Worker support.
- * Uses object pooling for BinaryWriter instances.
- */
 export declare class SyncBinarySerializer {
     private encoders;
     private columns;

package/dist/utils/binary-worker-pool.js

@@ -0,0 +1,206 @@
+import { Worker } from 'worker_threads';
+import { EventEmitter } from 'events';
+import * as os from 'os';
+import { binaryWorkerCode } from './binary-worker-code';
+export class BinaryWorkerPool extends EventEmitter {
+    workers = [];
+    columns = [];
+    batchIdCounter = 0;
+    queue = [];
+    isShutdown = false;
+    options;
+    constructor(options = {}) {
+        super();
+        this.options = {
+            poolSize: options.poolSize ?? Math.max(1, os.cpus().length - 1),
+            batchSize: options.batchSize ?? 10000,
+            highWaterMark: options.highWaterMark ?? 100,
+        };
+    }
+    async initialize(columns) {
+        this.columns = columns;
+        await Promise.all(Array(this.options.poolSize).fill(0).map(() => this.spawnWorker()));
+    }
+    async spawnWorker() {
+        return new Promise((resolve, reject) => {
+            const worker = new Worker(binaryWorkerCode, {
+                eval: true,
+                workerData: { columns: this.columns },
+            });
+            const workerInfo = {
+                worker,
+                busy: false,
+                configured: false,
+                pendingBatches: new Map(),
+            };
+            worker.on('message', (message) => {
+                this.handleWorkerMessage(workerInfo, message);
+                if (message.type === 'ready' && !workerInfo.configured) {
+                    workerInfo.configured = true;
+                    resolve();
+                }
+            });
+            worker.on('error', (error) => {
+                this.emit('error', error);
+                for (const [, pending] of workerInfo.pendingBatches) {
+                    pending.reject(error);
+                }
+                workerInfo.pendingBatches.clear();
+                const index = this.workers.indexOf(workerInfo);
+                if (index !== -1) {
+                    this.workers.splice(index, 1);
+                }
+                if (!this.isShutdown) {
+                    this.spawnWorker().catch(e => this.emit('error', e));
+                }
+            });
+            worker.on('exit', (code) => {
+                if (code !== 0 && !this.isShutdown) {
+                    this.emit('error', new Error(`Worker exited with code ${code}`));
+                }
+            });
+            this.workers.push(workerInfo);
+        });
+    }
+    handleWorkerMessage(workerInfo, message) {
+        switch (message.type) {
+            case 'result': {
+                const pending = workerInfo.pendingBatches.get(message.batchId);
+                if (pending) {
+                    workerInfo.pendingBatches.delete(message.batchId);
+                    pending.resolve(message.buffer);
+                }
+                workerInfo.busy = workerInfo.pendingBatches.size > 0;
+                this.processQueue();
+                break;
+            }
+            case 'error': {
+                const pending = workerInfo.pendingBatches.get(message.batchId);
+                if (pending) {
+                    workerInfo.pendingBatches.delete(message.batchId);
+                    pending.reject(new Error(message.error));
+                }
+                workerInfo.busy = workerInfo.pendingBatches.size > 0;
+                this.processQueue();
+                break;
+            }
+            case 'ready': {
+                workerInfo.configured = true;
+                this.processQueue();
+                break;
+            }
+        }
+    }
+    async serialize(rows) {
+        if (this.isShutdown) {
+            throw new Error('Worker pool is shut down');
+        }
+        return new Promise((resolve, reject) => {
+            this.queue.push({ rows, resolve, reject });
+            this.processQueue();
+        });
+    }
+    async *serializeStream(rows) {
+        let batch = [];
+        const batchSize = this.options.batchSize;
+        for await (const row of rows) {
+            batch.push(row);
+            if (batch.length >= batchSize) {
+                yield await this.serialize(batch);
+                batch = [];
+            }
+        }
+        if (batch.length > 0) {
+            yield await this.serialize(batch);
+        }
+    }
+    processQueue() {
+        if (this.queue.length === 0)
+            return;
+        const available = this.workers.find(w => w.configured && !w.busy);
+        if (!available)
+            return;
+        const { rows, resolve, reject } = this.queue.shift();
+        const batchId = ++this.batchIdCounter;
+        available.busy = true;
+        available.pendingBatches.set(batchId, { batchId, resolve, reject });
+        available.worker.postMessage({
+            type: 'serialize',
+            rows,
+            batchId,
+        });
+    }
+    getStats() {
+        return {
+            workers: this.workers.length,
+            busy: this.workers.filter(w => w.busy).length,
+            queueSize: this.queue.length,
+        };
+    }
+    async shutdown() {
+        this.isShutdown = true;
+        for (const item of this.queue) {
+            item.reject(new Error('Worker pool shutting down'));
+        }
+        this.queue = [];
+        await Promise.all(this.workers.map(w => new Promise(resolve => {
+            w.worker.on('exit', () => resolve());
+            w.worker.postMessage({ type: 'shutdown' });
+        })));
+        this.workers = [];
+    }
+}
+let defaultPool = null;
+export async function getDefaultBinaryPool(columns) {
+    if (!defaultPool) {
+        defaultPool = new BinaryWorkerPool();
+        if (columns) {
+            await defaultPool.initialize(columns);
+        }
+    }
+    return defaultPool;
+}
+export async function shutdownDefaultBinaryPool() {
+    if (defaultPool) {
+        await defaultPool.shutdown();
+        defaultPool = null;
+    }
+}
+import { createBinaryEncoder, acquireWriter, releaseWriter } from './binary-serializer';
+export class SyncBinarySerializer {
+    encoders = [];
+    columns = [];
+    constructor(columns) {
+        this.columns = columns;
+        this.encoders = columns.map(col => createBinaryEncoder(col.type, col.isNullable));
+    }
+    serialize(rows) {
+        const writer = acquireWriter();
+        try {
+            for (const row of rows) {
+                for (let i = 0; i < this.columns.length; i++) {
+                    const col = this.columns[i];
+                    const value = row[col.name];
+                    this.encoders[i](writer, value);
+                }
+            }
+            return writer.toBuffer();
+        }
+        finally {
+            releaseWriter(writer);
+        }
+    }
+    async *serializeStream(rows, batchSize = 10000) {
+        let batch = [];
+        for await (const row of rows) {
+            batch.push(row);
+            if (batch.length >= batchSize) {
+                yield this.serialize(batch);
+                batch = [];
+            }
+        }
+        if (batch.length > 0) {
+            yield this.serialize(batch);
+        }
+    }
+}

package/dist/utils/binary-worker.d.ts

@@ -1,12 +1 @@
-/**
- * HouseKit Binary Worker - Parallel RowBinary Serialization
- *
- * This worker thread handles binary serialization off the main thread,
- * allowing the main thread to focus on I/O while serialization happens in parallel.
- *
- * Benefits:
- * - Main thread stays responsive for network I/O
- * - Serialization uses separate CPU core
- * - Can saturate 10Gbps links without blocking Node.js event loop
- */
 export {};

package/dist/utils/binary-worker.js

@@ -0,0 +1,63 @@
+import { parentPort, workerData } from 'worker_threads';
+import { BinaryWriter, createBinaryEncoder } from './binary-serializer';
+let config = null;
+let writer = null;
+function serializeRows(rows, batchId) {
+    if (!config || !writer) {
+        throw new Error('Worker not configured');
+    }
+    writer.reset();
+    for (const row of rows) {
+        for (let i = 0; i < config.columns.length; i++) {
+            const col = config.columns[i];
+            const value = row[col.name];
+            config.encoders[i](writer, value);
+        }
+    }
+    return writer.toBuffer();
+}
+function handleMessage(message) {
+    try {
+        switch (message.type) {
+            case 'configure': {
+                const encoders = message.columns.map(col => createBinaryEncoder(col.type, col.isNullable));
+                config = {
+                    columns: message.columns,
+                    keyMapping: new Map(),
+                    encoders,
+                };
+                writer = new BinaryWriter(1024 * 1024);
+                parentPort?.postMessage({ type: 'ready' });
+                break;
+            }
+            case 'serialize': {
+                const buffer = serializeRows(message.rows, message.batchId);
+                const arrayBuffer = new ArrayBuffer(buffer.length);
+                new Uint8Array(arrayBuffer).set(buffer);
+                parentPort?.postMessage({
+                    type: 'result',
+                    batchId: message.batchId,
+                    buffer: Buffer.from(arrayBuffer),
+                    rowCount: message.rows.length,
+                }, [arrayBuffer]);
+                break;
+            }
+            case 'shutdown': {
+                process.exit(0);
+            }
+        }
+    }
+    catch (error) {
+        parentPort?.postMessage({
+            type: 'error',
+            batchId: message.batchId,
+            error: error instanceof Error ? error.message : String(error),
+        });
+    }
+}
+if (parentPort) {
+    parentPort.on('message', handleMessage);
+    if (workerData?.columns) {
+        handleMessage({ type: 'configure', columns: workerData.columns });
+    }
+}

package/dist/utils/insert-processing.d.ts

@@ -16,11 +16,9 @@ export type InsertPlan = {
     keyToColumn: Map<string, PreparedInsertColumn>;
     columnNames: string[];
     useCompact: boolean;
-    /** Skip enum validation for maximum performance */
     skipValidation?: boolean;
 };
 export interface InsertPlanOptions {
-    /** Skip enum validation in production for better performance */
     skipValidation?: boolean;
 }
 export declare function buildInsertPlan(table: TableRuntime<any, any>, options?: InsertPlanOptions): InsertPlan;

package/dist/utils/insert-processing.js

@@ -0,0 +1,163 @@
+const hasNativeUUID = typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function';
+let uuidFns = null;
+function loadUUIDFns() {
+    if (uuidFns)
+        return uuidFns;
+    const uuid = require('uuid');
+    uuidFns = {
+        1: uuid.v1,
+        4: hasNativeUUID ? () => crypto.randomUUID() : uuid.v4,
+        6: uuid.v6,
+        7: uuid.v7,
+    };
+    return uuidFns;
+}
+export function buildInsertPlan(table, options) {
+    const columns = [];
+    const keyToColumn = new Map();
+    for (const [propKey, colRaw] of Object.entries(table.$columns)) {
+        const column = colRaw;
+        const columnName = column.name;
+        const transform = createTransform(column, options?.skipValidation);
+        const autoUUIDVersion = column.meta?.autoGenerate?.type === 'uuid'
+            ? normalizeUUIDVersion(column.meta.autoGenerate.version)
+            : null;
+        const prepared = {
+            propKey,
+            column,
+            columnName,
+            hasDefault: column.meta?.default !== undefined,
+            defaultValue: column.meta?.default,
+            defaultFn: column.meta?.defaultFn || null,
+            autoUUIDVersion,
+            useServerUUID: Boolean(autoUUIDVersion && column.meta?.defaultExpr && /generateuuid/i.test(column.meta.defaultExpr)),
+            transform,
+        };
+        columns.push(prepared);
+        keyToColumn.set(propKey, prepared);
+        keyToColumn.set(columnName, prepared);
+    }
+    const columnNames = columns.map(c => c.columnName);
+    const useCompact = columns.every(c => !c.useServerUUID);
+    return { columns, keyToColumn, columnNames, useCompact, skipValidation: options?.skipValidation };
+}
+export function processRowWithPlan(row, plan, mode = plan.useCompact ? 'compact' : 'json') {
+    if (mode === 'compact') {
+        return processRowCompact(row, plan);
+    }
+    return processRowJson(row, plan);
+}
+function processRowJson(row, plan) {
+    const newRow = {};
+    for (const col of plan.columns) {
+        let value = row[col.propKey] !== undefined ? row[col.propKey] : row[col.columnName];
+        if (value === undefined) {
+            if (col.defaultFn) {
+                value = col.defaultFn(row);
+            }
+            else if (col.autoUUIDVersion !== null) {
+                if (!col.useServerUUID) {
+                    value = generateUUID(col.autoUUIDVersion);
+                }
+            }
+            else if (col.hasDefault) {
+                value = col.defaultValue;
+            }
+        }
+        if (value !== undefined) {
+            newRow[col.columnName] = col.transform(value);
+        }
+    }
+    return newRow;
+}
+export async function* processRowsStream(rows, plan, mode = plan.useCompact ? 'compact' : 'json') {
+    for await (const row of rows) {
+        yield processRowWithPlan(row, plan, mode);
+    }
+}
+function createTransform(col, skipValidation) {
+    const hasJson = Boolean(col.meta?.isJson);
+    const enumValues = col.meta?.enumValues;
+    const enumSet = (!skipValidation && enumValues) ? new Set(enumValues) : null;
+    const needsJson = hasJson;
+    const needsEnum = Boolean(enumSet);
+    if (!needsJson && !needsEnum) {
+        return (value) => {
+            if (value instanceof Date) {
+                return formatDate(value);
+            }
+            return value;
+        };
+    }
+    return (value) => {
+        let result = value;
+        if (result instanceof Date) {
+            result = formatDate(result);
+        }
+        if (needsJson && typeof result === 'object' && result !== null) {
+            result = JSON.stringify(result);
+        }
+        if (enumSet && !enumSet.has(result)) {
+            throw new Error(`❌ Invalid value '${result}' for enum column '${col.name}'. Allowed: ${enumValues.join(', ')}`);
+        }
+        return result;
+    };
+}
+function formatDate(date) {
+    const iso = date.toISOString();
+    const [datePart, timePart] = iso.split('T');
+    const [time] = timePart.split('.');
+    return `${datePart} ${time}`;
+}
+function normalizeUUIDVersion(version) {
+    if (version === 1 || version === 3 || version === 4 || version === 5 || version === 6 || version === 7) {
+        return version;
+    }
+    return 4;
+}
+function generateUUID(version) {
+    if (version === 4 && hasNativeUUID) {
+        return crypto.randomUUID();
+    }
+    const fns = loadUUIDFns();
+    const fn = fns[version];
+    if (!fn) {
+        if (version === 3 || version === 5) {
+            throw new Error(`UUID v${version} requires a name and namespace. Use v4, v6, or v7 for auto-generation.`);
+        }
+        return fns[4]();
+    }
+    return fn();
+}
+function processRowCompact(row, plan) {
+    const out = new Array(plan.columns.length);
+    for (let i = 0; i < plan.columns.length; i++) {
+        const col = plan.columns[i];
+        const provided = row[col.propKey];
+        const providedByName = row[col.columnName];
+        const hasValue = provided !== undefined || providedByName !== undefined;
+        if (hasValue) {
+            const value = col.transform(provided !== undefined ? provided : providedByName);
+            out[i] = value;
+            continue;
+        }
+        if (col.defaultFn) {
+            const computed = col.defaultFn(row);
+            out[i] = col.transform(computed);
+            continue;
+        }
+        if (col.autoUUIDVersion !== null) {
+            out[i] = col.useServerUUID ? undefined : generateUUID(col.autoUUIDVersion);
+            continue;
+        }
+        if (col.hasDefault) {
+            out[i] = col.defaultValue;
+            continue;
+        }
+        if (col.useServerUUID) {
+            throw new Error(`❌ Cannot use JSONCompactEachRow: column '${col.columnName}' relies on server-side default`);
+        }
+        throw new Error(`❌ Missing value for column '${col.columnName}'`);
+    }
+    return out;
+}

package/dist/utils/lru-cache.js

@@ -0,0 +1,30 @@
+export class LRUCache {
+    map = new Map();
+    max;
+    constructor(options) {
+        this.max = options.max;
+    }
+    get(key) {
+        const item = this.map.get(key);
+        if (item !== undefined) {
+            this.map.delete(key);
+            this.map.set(key, item);
+        }
+        return item;
+    }
+    set(key, value) {
+        if (this.map.has(key)) {
+            this.map.delete(key);
+        }
+        else if (this.map.size >= this.max) {
+            const firstKey = this.map.keys().next().value;
+            if (firstKey !== undefined) {
+                this.map.delete(firstKey);
+            }
+        }
+        this.map.set(key, value);
+    }
+    clear() {
+        this.map.clear();
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@housekit/orm",
-    "version": "0.1.47",
+    "version": "0.1.49",
     "description": "Type-safe ClickHouse ORM with modern DX and ClickHouse-specific optimizations. Features optimized JSONCompact streaming, full engine support, and advanced query capabilities.",
     "type": "module",
     "main": "./dist/index.js",
@@ -10,6 +10,70 @@
         ".": {
             "types": "./dist/index.d.ts",
             "import": "./dist/index.js"
+        },
+        "./client": {
+            "types": "./dist/client.d.ts",
+            "import": "./dist/client.js"
+        },
+        "./table": {
+            "types": "./dist/table.d.ts",
+            "import": "./dist/table.js"
+        },
+        "./column": {
+            "types": "./dist/column.d.ts",
+            "import": "./dist/column.js"
+        },
+        "./expressions": {
+            "types": "./dist/expressions.d.ts",
+            "import": "./dist/expressions.js"
+        },
+        "./schema-builder": {
+            "types": "./dist/schema-builder.d.ts",
+            "import": "./dist/schema-builder.js"
+        },
+        "./relational": {
+            "types": "./dist/relational.d.ts",
+            "import": "./dist/relational.js"
+        },
+        "./engines": {
+            "types": "./dist/engines.d.ts",
+            "import": "./dist/engines.js"
+        },
+        "./data-types": {
+            "types": "./dist/data-types.d.ts",
+            "import": "./dist/data-types.js"
+        },
+        "./materialized-views": {
+            "types": "./dist/materialized-views.d.ts",
+            "import": "./dist/materialized-views.js"
+        },
+        "./compiler": {
+            "types": "./dist/compiler.d.ts",
+            "import": "./dist/compiler.js"
+        },
+        "./external": {
+            "types": "./dist/external.d.ts",
+            "import": "./dist/external.js"
+        },
+        "./dictionary": {
+            "types": "./dist/dictionary.d.ts",
+            "import": "./dist/dictionary.js"
+        },
+        "./codegen": {
+            "types": "./dist/codegen/index.d.ts",
+            "import": "./dist/codegen/index.js"
+        },
+        "./utils": {
+            "types": "./dist/utils/index.d.ts",
+            "import": "./dist/utils/index.js"
+        },
+        "./modules": {
+            "types": "./dist/modules/index.d.ts",
+            "import": "./dist/modules/index.js"
+        },
+        "./builders": {
+            "types": "./dist/builders/index.d.ts",
+            "import": "./dist/builders/index.js"
         }
     },
     "license": "MIT",
@@ -43,7 +107,7 @@
         "README.md"
     ],
     "scripts": {
-        "build": "bun run build-worker.ts && bun build ./src/index.ts --outdir ./dist --target node --external @clickhouse/client --external uuid --external zod && tsc --project tsconfig.build.json",
+        "build": "bun run build-worker.ts && tsc --project tsconfig.build.json",
         "prepublishOnly": "bun run build",
         "clean": "rm -rf dist"
     },
@@ -57,7 +121,8 @@
     "devDependencies": {
         "@clickhouse/client": "^1.15.0",
         "@types/uuid": "^11.0.0",
-        "typescript": "^5.0.0"
+        "typescript": "^5.0.0",
+        "esbuild": "^0.27.2"
     },
     "publishConfig": {
         "access": "public"