@tmlmobilidade/writers 20260320.1741.37 → 20260320.1746.41

package/dist/clickhouse.d.ts

@@ -5,7 +5,7 @@ type ClickHouseWriterParams<T> = {
     /**
      * The maximum number of items to hold in memory
      * before flushing to the database.
-     * @default 10000
+     * @default 10_000
      */
     batch_size?: number;
     /**
@@ -15,6 +15,12 @@ type ClickHouseWriterParams<T> = {
      * @default disabled
      */
     batch_timeout?: number;
+    /**
+     * If enabled, starts an async one-time table ensure operation in the constructor.
+     * Use `await writer.init()` if you need to block startup until it is completed.
+     * @default false
+     */
+    ensure_table_on_init?: boolean;
     /**
      * How long to wait, in milliseconds, after the last write operation
      * before flushing the data to the database. This can be used to prevent
@@ -52,55 +58,39 @@ export declare class ClickHouseWriter<T> {
     private batchTimeoutTimer;
     private idleTimeoutTimer;
     private sessionTimer;
+    private isInitialized;
     constructor(params: ClickHouseWriterParams<T>);
     close(): Promise<void>;
+    /**
+     * Initializes the writer by ensuring the table exists.
+     * Safe to call multiple times.
+     */
+    init(): Promise<void>;
     /**
      * Ensures the table exists in ClickHouse by creating it if it doesn't exist.
      * Uses the tableSchema provided in the constructor, or an optional schema parameter.
-     *
      * @param schema Optional schema to use instead of the constructor-provided tableSchema
-     * @param engine The ClickHouse table engine to use (default: MergeTree)
+     * @param engine The ClickHouse table engine to use (default: ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}'))
      * @param orderBy The ORDER BY clause for the table (default: tuple())
      */
     ensureTable(schema?: ClickHouseColumn<T>[], engine?: string, orderBy?: string): Promise<void>;
     /**
-     * Builds a WHERE clause from a filter object.
-     * Supports simple equality, range queries ($gte, $lte, $gt, $lt), and $in queries.
-     *
-     * @param filter Filter object with column names as keys
-     * @returns WHERE clause string (without the WHERE keyword)
-     */
-    private buildWhereClause;
-    /**
-     * Formats a value for use in a SQL query.
-     */
-    private formatValue;
-    /**
-     * Counts the number of documents in the table that match the given filter.
-     * Supports simple equality and range queries ($gte, $lte, $gt, $lt).
-     *
-     * @param filter Optional WHERE clause conditions (e.g., { operational_date: '2024-01-01' } or { received_at: { $gte: 1234567890 } })
-     * @returns The count of matching documents
-     */
-    countDocuments(filter?: Record<string, unknown>): Promise<number>;
-    /**
-     * Returns distinct values for a given column, optionally filtered.
-     * Supports simple equality and range queries ($gte, $lte, $gt, $lt).
-     *
-     * @param column The column name to get distinct values for
-     * @param filter Optional WHERE clause conditions
-     * @returns Array of distinct values
+     * Flushes the current batch of data to ClickHouse.
+     * This method is called internally when the batch size or timeouts are reached,
+     * but can also be called manually if needed.
+     * @param callback Optional callback to execute after the flush is complete, receiving the flushed data as a parameter
      */
-    distinct<K = string>(column: string, filter?: Record<string, unknown>): Promise<K[]>;
     flush(callback?: (data?: T[]) => Promise<void>): Promise<void>;
     /**
      * Write data to the ClickHouse table.
-     *
      * @param data The data to write
      * @param options Options for the write operation (reserved for future use)
      * @param writeCallback Callback function to call after the write operation is complete
      * @param flushCallback Callback function to call after the flush operation is complete
      */
-    write(data: T | T[], writeCallback?: () => Promise<void>, flushCallback?: (data?: T[]) => Promise<void>): Promise<void>;
+    write(data: T | T[], { flushCallback, writeCallback }?: {
+        flushCallback?: (data?: T[]) => Promise<void>;
+        writeCallback?: () => Promise<void>;
+    }): Promise<void>;
 }
 export {};

package/dist/clickhouse.js

@@ -16,6 +16,7 @@ export class ClickHouseWriter {
     batchTimeoutTimer = null;
     idleTimeoutTimer = null;
     sessionTimer = new Timer();
+    isInitialized = false;
     /* * */
     constructor(params) {
         if (!params.table)
@@ -36,21 +37,32 @@ export class ClickHouseWriter {
             throw new Error('CLICKHOUSEWRITER: Either client or clientConfig is required.');
         }
     }
-    /* * */
+    /*
+     * Closes the ClickHouse client connection
+     * and clears any active timers.
+     */
     async close() {
         await this.client.close();
         Logger.info(`CLICKHOUSEWRITER [${this.params.table}]: Connection closed.`);
     }
-    /* * */
+    /**
+     * Initializes the writer by ensuring the table exists.
+     * Safe to call multiple times.
+     */
+    async init() {
+        if (this.isInitialized)
+            return;
+        await this.ensureTable();
+        this.isInitialized = true;
+    }
     /**
      * Ensures the table exists in ClickHouse by creating it if it doesn't exist.
      * Uses the tableSchema provided in the constructor, or an optional schema parameter.
-     *
      * @param schema Optional schema to use instead of the constructor-provided tableSchema
-     * @param engine The ClickHouse table engine to use (default: MergeTree)
+     * @param engine The ClickHouse table engine to use (default: ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}'))
      * @param orderBy The ORDER BY clause for the table (default: tuple())
      */
-    async ensureTable(schema, engine = 'MergeTree', orderBy = 'tuple()') {
+    async ensureTable(schema, engine = 'ReplicatedMergeTree(\'/clickhouse/tables/{shard}/{table}\', \'{replica}\')', orderBy = 'tuple()') {
         const tableSchemaToUse = schema ?? this.params.tableSchema;
         const tableSchema = tableSchemaToUse?.map(column => `${column.name} ${column.type}`).join(', ');
         if (!tableSchema) {
@@ -58,7 +70,7 @@ export class ClickHouseWriter {
         }
         try {
             const createTableQuery = `
-				CREATE TABLE IF NOT EXISTS ${this.params.table} (
+				CREATE TABLE IF NOT EXISTS ${this.params.table} ON CLUSTER 'clickhouse-replica' (
 					${tableSchema}
 				) ENGINE = ${engine}
 				ORDER BY ${orderBy}
@@ -71,123 +83,15 @@ export class ClickHouseWriter {
             throw error;
         }
     }
-    /* * */
     /**
-     * Builds a WHERE clause from a filter object.
-     * Supports simple equality, range queries ($gte, $lte, $gt, $lt), and $in queries.
-     *
-     * @param filter Filter object with column names as keys
-     * @returns WHERE clause string (without the WHERE keyword)
+     * Flushes the current batch of data to ClickHouse.
+     * This method is called internally when the batch size or timeouts are reached,
+     * but can also be called manually if needed.
+     * @param callback Optional callback to execute after the flush is complete, receiving the flushed data as a parameter
      */
-    buildWhereClause(filter) {
-        if (!filter || Object.keys(filter).length === 0) {
-            return null;
-        }
-        const conditions = [];
-        for (const [key, value] of Object.entries(filter)) {
-            if (value === null || value === undefined) {
-                continue;
-            }
-            // Handle range queries (MongoDB-style operators)
-            if (typeof value === 'object' && !Array.isArray(value)) {
-                const operators = value;
-                for (const [op, opValue] of Object.entries(operators)) {
-                    switch (op) {
-                        case '$gt':
-                            conditions.push(`${key} > ${this.formatValue(opValue)}`);
-                            break;
-                        case '$gte':
-                            conditions.push(`${key} >= ${this.formatValue(opValue)}`);
-                            break;
-                        case '$in':
-                            if (Array.isArray(opValue)) {
-                                const values = opValue.map(v => this.formatValue(v)).join(', ');
-                                conditions.push(`${key} IN (${values})`);
-                            }
-                            break;
-                        case '$lt':
-                            conditions.push(`${key} < ${this.formatValue(opValue)}`);
-                            break;
-                        case '$lte':
-                            conditions.push(`${key} <= ${this.formatValue(opValue)}`);
-                            break;
-                    }
-                }
-            }
-            else {
-                // Simple equality
-                conditions.push(`${key} = ${this.formatValue(value)}`);
-            }
-        }
-        return conditions.length > 0 ? conditions.join(' AND ') : null;
-    }
-    /**
-     * Formats a value for use in a SQL query.
-     */
-    formatValue(value) {
-        if (typeof value === 'string') {
-            return `'${value}'`;
-        }
-        return String(value);
-    }
-    /* * */
-    /**
-     * Counts the number of documents in the table that match the given filter.
-     * Supports simple equality and range queries ($gte, $lte, $gt, $lt).
-     *
-     * @param filter Optional WHERE clause conditions (e.g., { operational_date: '2024-01-01' } or { received_at: { $gte: 1234567890 } })
-     * @returns The count of matching documents
-     */
-    async countDocuments(filter) {
-        try {
-            let query = `SELECT count() as count FROM ${this.params.table}`;
-            const whereClause = this.buildWhereClause(filter);
-            if (whereClause) {
-                query += ` WHERE ${whereClause}`;
-            }
-            const result = await this.client.query({
-                format: 'JSONEachRow',
-                query,
-            });
-            const data = await result.json();
-            return parseInt(data[0]?.count ?? '0', 10);
-        }
-        catch (error) {
-            Logger.error(`CLICKHOUSEWRITER [${this.params.table}]: Error @ countDocuments(): ${error.message}`);
-            throw error;
-        }
-    }
-    /* * */
-    /**
-     * Returns distinct values for a given column, optionally filtered.
-     * Supports simple equality and range queries ($gte, $lte, $gt, $lt).
-     *
-     * @param column The column name to get distinct values for
-     * @param filter Optional WHERE clause conditions
-     * @returns Array of distinct values
-     */
-    async distinct(column, filter) {
-        try {
-            let query = `SELECT DISTINCT ${column} FROM ${this.params.table}`;
-            const whereClause = this.buildWhereClause(filter);
-            if (whereClause) {
-                query += ` WHERE ${whereClause}`;
-            }
-            const result = await this.client.query({
-                format: 'JSONEachRow',
-                query,
-            });
-            const data = await result.json();
-            return data.map(row => row[column]);
-        }
-        catch (error) {
-            Logger.error(`CLICKHOUSEWRITER [${this.params.table}]: Error @ distinct(): ${error.message}`);
-            throw error;
-        }
-    }
-    /* * */
     async flush(callback) {
         try {
+            await this.init();
             //
             const flushTimer = new Timer();
             const sessionTimerResult = this.sessionTimer.get();
@@ -249,17 +153,16 @@ export class ClickHouseWriter {
             throw error; // Re-throw to allow retry logic at higher level
         }
     }
-    /* * */
     /**
      * Write data to the ClickHouse table.
-     *
      * @param data The data to write
      * @param options Options for the write operation (reserved for future use)
      * @param writeCallback Callback function to call after the write operation is complete
      * @param flushCallback Callback function to call after the flush operation is complete
      */
-    async write(data, writeCallback, flushCallback) {
+    async write(data, { flushCallback, writeCallback } = {}) {
         //
+        await this.init();
         //
         // Invalidate the previously set idle timeout timer
         // since we are performing a write operation again.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@tmlmobilidade/writers",
-	"version": "20260320.1741.37",
+	"version": "20260320.1746.41",
 	"author": {
 		"email": "iso@tmlmobilidade.pt",
 		"name": "TML-ISO"