@housekit/orm 0.1.47 → 0.1.49

package/dist/utils/background-batcher.js

@@ -0,0 +1,75 @@
+class BackgroundBatcher {
+    client;
+    queues = new Map();
+    timers = new Map();
+    tables = new Map();
+    constructor(client) {
+        this.client = client;
+    }
+    async add(table, row, config) {
+        const tableName = table.$table;
+        if (!this.queues.has(tableName)) {
+            this.queues.set(tableName, []);
+            this.tables.set(tableName, table);
+        }
+        const queue = this.queues.get(tableName);
+        queue.push(row);
+        if (queue.length >= config.maxRows) {
+            await this.flush(tableName);
+            return;
+        }
+        if (queue.length === 1) {
+            const timer = setTimeout(() => {
+                this.flush(tableName);
+            }, config.flushIntervalMs);
+            if (typeof timer.unref === 'function') {
+                timer.unref();
+            }
+            this.timers.set(tableName, timer);
+        }
+    }
+    async flush(tableName) {
+        const queue = this.queues.get(tableName);
+        const table = this.tables.get(tableName);
+        if (!queue || queue.length === 0 || !table)
+            return;
+        const timer = this.timers.get(tableName);
+        if (timer)
+            clearTimeout(timer);
+        this.timers.delete(tableName);
+        const dataToInsert = [...queue];
+        this.queues.set(tableName, []);
+        try {
+            const { Readable } = await import('stream');
+            const stream = Readable.from(dataToInsert, { objectMode: true });
+            await this.client.insert({
+                table: tableName,
+                values: stream,
+                format: 'JSONEachRow',
+                clickhouse_settings: {
+                    async_insert: 1,
+                    wait_for_async_insert: 0,
+                }
+            });
+        }
+        catch (err) {
+            console.error(`[housekit] Background flush failed for ${tableName}:`, err);
+        }
+    }
+    async flushAll() {
+        const promises = [];
+        for (const tableName of this.queues.keys()) {
+            promises.push(this.flush(tableName));
+        }
+        await Promise.all(promises);
+    }
+}
+const batchers = new WeakMap();
+export const globalBatcher = (client) => {
+    let batcher = batchers.get(client);
+    if (!batcher) {
+        batcher = new BackgroundBatcher(client);
+        batchers.set(client, batcher);
+    }
+    return batcher;
+};

package/dist/utils/batch-transform.js

@@ -0,0 +1,51 @@
+import { Transform } from 'stream';
+import { processRowWithPlan } from './insert-processing';
+export class BatchTransformStream extends Transform {
+    plan;
+    mode;
+    batch = [];
+    batchSize;
+    constructor(plan, mode, options = {}) {
+        super({ objectMode: true });
+        this.plan = plan;
+        this.mode = mode;
+        this.batchSize = options.batchSize || 100;
+    }
+    _transform(chunk, _encoding, callback) {
+        this.batch.push(chunk);
+        if (this.batch.length >= this.batchSize) {
+            this.processBatch(callback);
+        }
+        else {
+            this.scheduleProcessing(callback);
+        }
+    }
+    _flush(callback) {
+        if (this.batch.length > 0) {
+            this.processBatch(callback);
+        }
+        else {
+            callback();
+        }
+    }
+    scheduleProcessing(callback) {
+        callback();
+    }
+    processBatch(callback) {
+        const batchToProcess = this.batch;
+        this.batch = [];
+        try {
+            for (const row of batchToProcess) {
+                const processed = processRowWithPlan(row, this.plan, this.mode);
+                this.push(processed);
+            }
+            callback();
+        }
+        catch (error) {
+            callback(error);
+        }
+    }
+}
+export function createBatchTransformStream(plan, mode, options) {
+    return new BatchTransformStream(plan, mode, options);
+}

package/dist/utils/binary-reader.d.ts

@@ -1,9 +1,3 @@
-/**
- * HouseKit Binary Reader - Ultra-Fast RowBinary Decoding
- *
- * optimized for reading ClickHouse RowBinary format directly from buffers.
- * This is 10-20x faster than JSON.parse() for large datasets.
- */
 export declare class BinaryReader {
     private buffer;
     private offset;

package/dist/utils/binary-reader.js

@@ -0,0 +1,334 @@
+export class BinaryReader {
+    buffer;
+    offset = 0;
+    view;
+    constructor(buffer) {
+        this.buffer = buffer;
+        this.view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+    }
+    reset(buffer) {
+        this.buffer = buffer;
+        this.offset = 0;
+        this.view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+    }
+    getOffset() {
+        return this.offset;
+    }
+    isEOF() {
+        return this.offset >= this.buffer.length;
+    }
+    readInt8() {
+        const val = this.view.getInt8(this.offset);
+        this.offset += 1;
+        return val;
+    }
+    readUInt8() {
+        const val = this.view.getUint8(this.offset);
+        this.offset += 1;
+        return val;
+    }
+    readInt16() {
+        const val = this.view.getInt16(this.offset, true);
+        this.offset += 2;
+        return val;
+    }
+    readUInt16() {
+        const val = this.view.getUint16(this.offset, true);
+        this.offset += 2;
+        return val;
+    }
+    readInt32() {
+        const val = this.view.getInt32(this.offset, true);
+        this.offset += 4;
+        return val;
+    }
+    readUInt32() {
+        const val = this.view.getUint32(this.offset, true);
+        this.offset += 4;
+        return val;
+    }
+    readInt64() {
+        const val = this.view.getBigInt64(this.offset, true);
+        this.offset += 8;
+        return val;
+    }
+    readUInt64() {
+        const val = this.view.getBigUint64(this.offset, true);
+        this.offset += 8;
+        return val;
+    }
+    readInt128() {
+        const low = this.view.getBigUint64(this.offset, true);
+        const high = this.view.getBigInt64(this.offset + 8, true);
+        this.offset += 16;
+        return (high << 64n) | low;
+    }
+    readUInt128() {
+        const low = this.view.getBigUint64(this.offset, true);
+        const high = this.view.getBigUint64(this.offset + 8, true);
+        this.offset += 16;
+        return (high << 64n) | low;
+    }
+    readInt256() {
+        let val = 0n;
+        for (let i = 0; i < 4; i++) {
+            const word = this.view.getBigUint64(this.offset + i * 8, true);
+            val |= word << (BigInt(i) * 64n);
+        }
+        this.offset += 32;
+        return val;
+    }
+    readUInt256() {
+        return this.readInt256();
+    }
+    readFloat32() {
+        const val = this.view.getFloat32(this.offset, true);
+        this.offset += 4;
+        return val;
+    }
+    readFloat64() {
+        const val = this.view.getFloat64(this.offset, true);
+        this.offset += 8;
+        return val;
+    }
+    readVarInt() {
+        let result = 0;
+        let shift = 0;
+        while (true) {
+            const byte = this.buffer[this.offset++];
+            result |= (byte & 0x7f) << shift;
+            if ((byte & 0x80) === 0)
+                break;
+            shift += 7;
+        }
+        return result;
+    }
+    readString() {
+        const len = this.readVarInt();
+        if (len === 0)
+            return '';
+        const str = this.buffer.toString('utf-8', this.offset, this.offset + len);
+        this.offset += len;
+        return str;
+    }
+    readFixedString(length) {
+        const str = this.buffer.toString('utf-8', this.offset, this.offset + length);
+        this.offset += length;
+        return str.replace(/\u0000+$/, '');
+    }
+    readUUID() {
+        const bytes = this.buffer.subarray(this.offset, this.offset + 16);
+        this.offset += 16;
+        const hex = Buffer.allocUnsafe(16);
+        for (let i = 0; i < 8; i++) {
+            hex[i] = bytes[7 - i];
+        }
+        for (let i = 0; i < 8; i++) {
+            hex[8 + i] = bytes[15 - (i)];
+        }
+        const s = hex.toString('hex');
+        return `${s.slice(0, 8)}-${s.slice(8, 12)}-${s.slice(12, 16)}-${s.slice(16, 20)}-${s.slice(20)}`;
+    }
+    readDate() {
+        const days = this.readUInt16();
+        const date = new Date(days * 24 * 60 * 60 * 1000);
+        return date.toISOString().split('T')[0];
+    }
+    readDate32() {
+        const days = this.readInt32();
+        const date = new Date(days * 24 * 60 * 60 * 1000);
+        return date.toISOString().split('T')[0];
+    }
+    readDateTime() {
+        const seconds = this.readUInt32();
+        return new Date(seconds * 1000);
+    }
+    readDateTime64(precision = 3) {
+        const ticks = this.readInt64();
+        const divisor = BigInt(Math.pow(10, precision));
+        let ms;
+        if (precision === 3) {
+            ms = ticks;
+        }
+        else if (precision > 3) {
+            ms = ticks / BigInt(Math.pow(10, precision - 3));
+        }
+        else {
+            ms = ticks * BigInt(Math.pow(10, 3 - precision));
+        }
+        return new Date(Number(ms));
+    }
+    readNullable(reader) {
+        const isNull = this.readUInt8();
+        if (isNull === 1)
+            return null;
+        return reader();
+    }
+    readArray(itemReader) {
+        const lenBig = this.readUInt64();
+        const length = Number(lenBig);
+        const res = new Array(length);
+        for (let i = 0; i < length; i++) {
+            res[i] = itemReader();
+        }
+        return res;
+    }
+    readMap(keyReader, valueReader) {
+        const lenBig = this.readUInt64();
+        const length = Number(lenBig);
+        const res = {};
+        for (let i = 0; i < length; i++) {
+            const key = keyReader();
+            const value = valueReader();
+            res[key] = value;
+        }
+        return res;
+    }
+    readDecimal32(scale) {
+        const val = this.readInt32();
+        return val / Math.pow(10, scale);
+    }
+    readDecimal64(scale) {
+        const val = this.readInt64();
+        return Number(val) / Math.pow(10, scale);
+    }
+    readDecimal128(scale) {
+        const val = this.readInt128();
+        return Number(val) / Math.pow(10, scale);
+    }
+    readBool() {
+        return this.readUInt8() === 1;
+    }
+    readIPv4() {
+        const val = this.readUInt32();
+        return [
+            (val) & 0xFF,
+            (val >> 8) & 0xFF,
+            (val >> 16) & 0xFF,
+            (val >> 24) & 0xFF
+        ].join('.');
+    }
+    readIPv6() {
+        const buffer = this.buffer.subarray(this.offset, this.offset + 16);
+        this.offset += 16;
+        const parts = [];
+        for (let i = 0; i < 16; i += 2) {
+            const group = buffer.readUInt16BE(i).toString(16);
+            parts.push(group);
+        }
+        return parts.join(':').replace(/(^|:)0(:0)*:0(:|$)/, '::');
+    }
+}
+export function createBinaryDecoder(type) {
+    const t = type.toLowerCase().trim();
+    if (t.startsWith('nullable(')) {
+        const inner = t.match(/^nullable\((.+)\)$/)[1];
+        const innerDecoder = createBinaryDecoder(inner);
+        return (r) => r.readNullable(() => innerDecoder(r));
+    }
+    if (t.startsWith('array(')) {
+        const inner = t.match(/^array\((.+)\)$/)[1];
+        const innerDecoder = createBinaryDecoder(inner);
+        return (r) => r.readArray(() => innerDecoder(r));
+    }
+    if (t.startsWith('map(')) {
+        const inner = t.match(/^map\((.+)\)$/)[1];
+        const [keyType, valueType] = parseGenericTypes(inner);
+        const keyDecoder = createBinaryDecoder(keyType);
+        const valueDecoder = createBinaryDecoder(valueType);
+        return (r) => r.readMap(() => keyDecoder(r), () => valueDecoder(r));
+    }
+    if (t.startsWith('tuple(')) {
+        const inner = t.match(/^tuple\((.+)\)$/)[1];
+        const types = parseGenericTypes(inner);
+        const decoders = types.map(type => createBinaryDecoder(type));
+        return (r) => decoders.map(d => d(r));
+    }
+    if (t.startsWith('nested(')) {
+        const inner = t.match(/^nested\((.+)\)$/)[1];
+        const fields = parseGenericTypes(inner);
+        const types = fields.map(f => {
+            const parts = f.trim().split(/\s+/);
+            return parts.length < 2 ? 'String' : parts.slice(1).join(' ');
+        });
+        const tupleDecoders = types.map(type => createBinaryDecoder(type));
+        return (r) => r.readArray(() => tupleDecoders.map(d => d(r)));
+    }
+    if (t.startsWith('lowcardinality(')) {
+        const inner = t.match(/^lowcardinality\((.+)\)$/)[1];
+        return createBinaryDecoder(inner);
+    }
+    if (t.startsWith('fixedstring(')) {
+        const len = parseInt(t.match(/\d+/)[0], 10);
+        return (r) => r.readFixedString(len);
+    }
+    if (t.startsWith('decimal')) {
+        const match = t.match(/^decimal(?:32|64|128)?\((\d+),\s*(\d+)\)$/);
+        if (match) {
+            const scale = parseInt(match[2], 10);
+            if (t.includes('decimal128'))
+                return (r) => r.readDecimal128(scale);
+            if (t.includes('decimal64'))
+                return (r) => r.readDecimal64(scale);
+            return (r) => r.readDecimal32(scale);
+        }
+    }
+    if (t.startsWith('datetime64')) {
+        const match = t.match(/^datetime64\((\d+)/);
+        const precision = match ? parseInt(match[1], 10) : 3;
+        return (r) => r.readDateTime64(precision);
+    }
+    if (t.startsWith('enum')) {
+        if (t.startsWith('enum8'))
+            return (r) => r.readInt8();
+        return (r) => r.readInt16();
+    }
+    switch (t) {
+        case 'uint8': return (r) => r.readUInt8();
+        case 'int8': return (r) => r.readInt8();
+        case 'uint16': return (r) => r.readUInt16();
+        case 'int16': return (r) => r.readInt16();
+        case 'uint32': return (r) => r.readUInt32();
+        case 'int32': return (r) => r.readInt32();
+        case 'uint64': return (r) => r.readUInt64();
+        case 'int64': return (r) => r.readInt64();
+        case 'uint128': return (r) => r.readUInt128();
+        case 'int128': return (r) => r.readInt128();
+        case 'uint256': return (r) => r.readUInt256();
+        case 'int256': return (r) => r.readInt256();
+        case 'float32': return (r) => r.readFloat32();
+        case 'float64': return (r) => r.readFloat64();
+        case 'string': return (r) => r.readString();
+        case 'uuid': return (r) => r.readUUID();
+        case 'bool': return (r) => r.readBool();
+        case 'date': return (r) => r.readDate();
+        case 'date32': return (r) => r.readDate32();
+        case 'datetime': return (r) => r.readDateTime();
+        case 'ipv4': return (r) => r.readIPv4();
+        case 'ipv6': return (r) => r.readIPv6();
+        default: return (r) => r.readString();
+    }
+}
+function parseGenericTypes(typeString) {
+    const args = [];
+    let current = '';
+    let parenDepth = 0;
+    for (let i = 0; i < typeString.length; i++) {
+        const char = typeString[i];
+        if (char === ',' && parenDepth === 0) {
+            args.push(current.trim());
+            current = '';
+        }
+        else {
+            if (char === '(')
+                parenDepth++;
+            if (char === ')')
+                parenDepth--;
+            current += char;
+        }
+    }
+    if (current.trim()) {
+        args.push(current.trim());
+    }
+    return args;
+}

package/dist/utils/binary-serializer.d.ts

@@ -1,47 +1,12 @@
-/**
- * Acquire a BinaryWriter from the pool (or create new if pool empty)
- */
 export declare function acquireWriter(): BinaryWriter;
-/**
- * Release a BinaryWriter back to the pool
- */
 export declare function releaseWriter(writer: BinaryWriter): void;
-/**
- * HouseKit Binary Serializer - Ultra-Fast RowBinary Encoding
- *
- * ClickHouse's RowBinary format sends data directly as bytes with no parsing overhead.
- * This is the fastest possible way to insert data into ClickHouse.
- *
- * Benefits over JSONEachRow:
- * - No JSON.stringify() overhead
- * - No string escaping
- * - No parsing on ClickHouse side
- * - Smaller payload (Int64 = 8 bytes vs up to 20 bytes as string)
- * - Lower GC pressure (no intermediate strings)
- */
-/**
- * Efficient binary buffer writer that minimizes allocations.
- * Uses a pre-allocated buffer that grows as needed.
- */
 export declare class BinaryWriter {
     private buffer;
     private offset;
     constructor(initialSize?: number);
-    /**
-     * Ensure buffer has enough space for n more bytes
-     */
     private ensureCapacity;
-    /**
-     * Reset the writer for reuse (avoids allocating new buffer)
-     */
     reset(): void;
-    /**
-     * Get the final buffer with only written bytes
-     */
     getBuffer(): Buffer;
-    /**
-     * Get a copy of the buffer (safe to use after reset)
-     */
     toBuffer(): Buffer;
     writeInt8(value: number): void;
     writeUInt8(value: number): void;
@@ -57,124 +22,48 @@ export declare class BinaryWriter {
     writeUInt256(value: bigint): void;
     writeFloat32(value: number): void;
     writeFloat64(value: number): void;
-    /**
-     * Write a variable-length integer (LEB128).
-     * Used for string lengths in RowBinary format.
-     */
     writeVarInt(value: number): void;
-    /**
-     * Write a string in RowBinary format: [VarInt length][UTF-8 bytes]
-     */
     writeString(value: string): void;
-    /**
-     * Write a FixedString(N) - padded with null bytes if shorter
-     */
     writeFixedString(value: string, length: number): void;
-    /**
-     * Write raw bytes directly
-     */
     writeBytes(data: Buffer): void;
-    /**
-     * Write a UUID (16 bytes).
-     * ClickHouse stores UUIDs as two UInt64 in big-endian order!
-     * This is different from the usual little-endian storage.
-     */
     writeUUID(value: string): void;
-    /**
-     * Write a Date (days since epoch as UInt16)
-     */
     writeDate(value: Date | number): void;
-    /**
-     * Write a Date32 (days since epoch as Int32)
-     */
     writeDate32(value: Date | number): void;
-    /**
-     * Write a DateTime (seconds since epoch as UInt32)
-     */
     writeDateTime(value: Date | number): void;
-    /**
-     * Write a DateTime64 with specified precision
-     */
     writeDateTime64(value: Date | number, precision?: number): void;
-    /**
-     * Write a nullable prefix (0 = not null, 1 = null)
-     */
     writeNullable(isNull: boolean): void;
-    /**
-     * Write array length prefix
-     */
     writeArrayLength(length: number): void;
-    /**
-     * Write Decimal32 (stored as Int32)
-     */
     writeDecimal32(value: number, scale: number): void;
-    /**
-     * Write Decimal64 (stored as Int64)
-     */
     writeDecimal64(value: number, scale: number): void;
-    /**
-     * Write Decimal128 (stored as Int128)
-     */
     writeDecimal128(value: number | bigint, scale: number): void;
     writeBool(value: boolean): void;
-    /**
-     * Write IPv4 address (UInt32 in network byte order)
-     */
     writeIPv4(value: string | number): void;
-    /**
-     * Write IPv6 address (16 bytes)
-     */
     writeIPv6(value: string | Buffer): void;
     private expandIPv6;
     writeEnum8(value: number): void;
     writeEnum16(value: number): void;
 }
 export type BinaryEncoder = (writer: BinaryWriter, value: any) => void;
-/**
- * Create a binary encoder for a ClickHouse column type
- */
 export declare function createBinaryEncoder(clickhouseType: string, isNullable?: boolean): BinaryEncoder;
-/**
- * Configuration for binary serialization
- */
 export interface BinarySerializationConfig {
-    /** Column names in order */
     columns: Array<{
         name: string;
         type: string;
         isNullable: boolean;
     }>;
-    /** Property key mapping (propKey -> column index) */
     keyMapping: Map<string, number>;
-    /** Pre-compiled encoders for each column */
     encoders: BinaryEncoder[];
 }
-/**
- * Build a binary serialization configuration from a table definition
- */
 export declare function buildBinaryConfig(columns: Array<{
     name: string;
     type: string;
     isNull: boolean;
     propKey: string;
 }>): BinarySerializationConfig;
-/**
- * Serialize a single row to RowBinary format
- */
 export declare function serializeRowBinary(row: Record<string, any>, config: BinarySerializationConfig, writer?: BinaryWriter): Buffer;
-/**
- * Serialize multiple rows to a single RowBinary buffer
- */
 export declare function serializeRowsBinary(rows: Array<Record<string, any>>, config: BinarySerializationConfig): Buffer;
 export type RowAccessor = (row: any) => any;
-/**
- * Create an optimized accessor function for a column.
- * Uses direct property access instead of dynamic lookup.
- */
 export declare function createAccessor(propKey: string, columnName: string): RowAccessor;
-/**
- * Optimized serialization config with pre-compiled accessors
- */
 export interface OptimizedBinaryConfig {
     columns: Array<{
         name: string;
@@ -183,12 +72,8 @@ export interface OptimizedBinaryConfig {
     }>;
     encoders: BinaryEncoder[];
     accessors: RowAccessor[];
-    /** Skip validation for maximum performance (Optimization #4) */
     skipValidation?: boolean;
 }
-/**
- * Build an optimized binary config with pre-compiled accessors
- */
 export declare function buildOptimizedBinaryConfig(columns: Array<{
     name: string;
     type: string;
@@ -197,20 +82,10 @@ export declare function buildOptimizedBinaryConfig(columns: Array<{
 }>, options?: {
     skipValidation?: boolean;
 }): OptimizedBinaryConfig;
-/**
- * Ultra-fast serialization using pre-compiled accessors and pooled writer
- */
 export declare function serializeRowsOptimized(rows: Array<Record<string, any>>, config: OptimizedBinaryConfig): Buffer;
-/**
- * Check if a schema is numeric-heavy (>50% numeric columns)
- */
 export declare function isNumericHeavySchema(columns: Array<{
     type: string;
 }>): boolean;
-/**
- * Batch serialize numeric columns using TypedArrays for better performance.
- * Only use for schemas with mostly numeric columns.
- */
 export declare function serializeNumericBatch(rows: Array<Record<string, any>>, config: OptimizedBinaryConfig, numericIndices: number[]): {
     numericBuffer: ArrayBuffer;
     otherData: any[][];