@tmlmobilidade/writers 20260320.1746.41 → 20260322.1655.45

package/dist/generic.d.ts

@@ -0,0 +1,62 @@
+interface BatchWriterParams<T> {
+    /**
+     * The maximum number of items to hold in memory
+     * before flushing to the database.
+     * @required
+     */
+    batch_size: number;
+    /**
+     * How long, in milliseconds, data should be kept in memory before
+     * flushing to the database. If this feature is enabled, a flush will
+     * be triggered even if the batch is not full. Disabled by default.
+     * @default disabled
+     */
+    batch_timeout?: number;
+    /**
+     * How long to wait, in milliseconds, after the last write operation
+     * before flushing the data to the database. This can be used to prevent
+     * items staying in memory for too long if the batch size is not reached
+     * frequently enough. Disabled by default.
+     * @default disabled
+     */
+    idle_timeout?: number;
+    /**
+     * The insert function to use for inserting data into the batch.
+     * @required
+     */
+    insertFn: (data: T[]) => Promise<void>;
+    /**
+     * The title of this BatchWriter instance,
+     * used to identify the source of the logs.
+     * @required
+     */
+    title: string;
+}
+export declare class BatchWriter<T> {
+    private params;
+    private dataBucketAlwaysAvailable;
+    private dataBucketFlushOps;
+    private batchTimeoutTimer;
+    private idleTimeoutTimer;
+    private sessionTimer;
+    constructor(params: BatchWriterParams<T>);
+    /**
+     * Flushes the current batch of data.
+     * This method is called internally when the batch size or timeouts are reached,
+     * but can also be called manually if needed.
+     * @param callback Optional callback to execute after the flush is complete, receiving the flushed data as a parameter
+     */
+    flush(callback?: (data?: T[]) => Promise<void>): Promise<void>;
+    /**
+     * Write data to the batch.
+     * @param data The data to write
+     * @param options Options for the write operation (reserved for future use)
+     * @param writeCallback Callback function to call after the write operation is complete
+     * @param flushCallback Callback function to call after the flush operation is complete
+     */
+    write(data: T | T[], { flushCallback, writeCallback }?: {
+        flushCallback?: (data?: T[]) => Promise<void>;
+        writeCallback?: () => Promise<void>;
+    }): Promise<void>;
+}
+export {};

package/dist/{clickhouse.js → generic.js}

@@ -1,97 +1,33 @@
 /* eslint-disable perfectionist/sort-classes */
 /* * */
-import { createClient } from '@clickhouse/client';
 import { Logger } from '@tmlmobilidade/logger';
 import { Timer } from '@tmlmobilidade/timer';
 /* * */
-export class ClickHouseWriter {
-    //
+export class BatchWriter {
     //
     params;
-    client;
-    //
     dataBucketAlwaysAvailable = [];
     dataBucketFlushOps = [];
-    //
     batchTimeoutTimer = null;
     idleTimeoutTimer = null;
     sessionTimer = new Timer();
-    isInitialized = false;
-    /* * */
     constructor(params) {
-        if (!params.table)
-            throw new Error('CLICKHOUSEWRITER: Table name is required');
-        if (params.client) {
-            this.params = params;
-            this.client = params.client;
-        }
-        else if (params.clientConfig) {
-            const { database, password, url, username } = params.clientConfig;
-            if (!database || !password || !url || !username) {
-                throw new Error('CLICKHOUSEWRITER: Client configuration is invalid. Ensure database, password, url and username are provided.');
-            }
-            this.params = params;
-            this.client = createClient(params.clientConfig);
-        }
-        else {
-            throw new Error('CLICKHOUSEWRITER: Either client or clientConfig is required.');
-        }
-    }
-    /*
-     * Closes the ClickHouse client connection
-     * and clears any active timers.
-     */
-    async close() {
-        await this.client.close();
-        Logger.info(`CLICKHOUSEWRITER [${this.params.table}]: Connection closed.`);
-    }
-    /**
-     * Initializes the writer by ensuring the table exists.
-     * Safe to call multiple times.
-     */
-    async init() {
-        if (this.isInitialized)
-            return;
-        await this.ensureTable();
-        this.isInitialized = true;
-    }
-    /**
-     * Ensures the table exists in ClickHouse by creating it if it doesn't exist.
-     * Uses the tableSchema provided in the constructor, or an optional schema parameter.
-     * @param schema Optional schema to use instead of the constructor-provided tableSchema
-     * @param engine The ClickHouse table engine to use (default: ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}'))
-     * @param orderBy The ORDER BY clause for the table (default: tuple())
-     */
-    async ensureTable(schema, engine = 'ReplicatedMergeTree(\'/clickhouse/tables/{shard}/{table}\', \'{replica}\')', orderBy = 'tuple()') {
-        const tableSchemaToUse = schema ?? this.params.tableSchema;
-        const tableSchema = tableSchemaToUse?.map(column => `${column.name} ${column.type}`).join(', ');
-        if (!tableSchema) {
-            throw new Error(`CLICKHOUSEWRITER [${this.params.table}]: Cannot ensure table without a schema. Provide tableSchema in constructor or as parameter.`);
-        }
-        try {
-            const createTableQuery = `
-				CREATE TABLE IF NOT EXISTS ${this.params.table} ON CLUSTER 'clickhouse-replica' (
-					${tableSchema}
-				) ENGINE = ${engine}
-				ORDER BY ${orderBy}
-			`;
-            await this.client.command({ query: createTableQuery });
-            Logger.info(`CLICKHOUSEWRITER [${this.params.table}]: Table ensured.`);
-        }
-        catch (error) {
-            Logger.error(`CLICKHOUSEWRITER [${this.params.table}]: Error @ ensureTable(): ${error.message}`);
-            throw error;
-        }
+        if (!params.title)
+            throw new Error('BATCHWRITER: Title is required.');
+        if (!params.insertFn)
+            throw new Error('BATCHWRITER: Insert function is required.');
+        if (!params.batch_size)
+            throw new Error('BATCHWRITER: Batch size is required.');
+        this.params = params;
     }
     /**
-     * Flushes the current batch of data to ClickHouse.
+     * Flushes the current batch of data.
      * This method is called internally when the batch size or timeouts are reached,
      * but can also be called manually if needed.
      * @param callback Optional callback to execute after the flush is complete, receiving the flushed data as a parameter
      */
     async flush(callback) {
         try {
-            await this.init();
             //
             const flushTimer = new Timer();
             const sessionTimerResult = this.sessionTimer.get();
@@ -116,45 +52,32 @@ export class ClickHouseWriter {
             this.dataBucketFlushOps = [...this.dataBucketFlushOps, ...this.dataBucketAlwaysAvailable];
             this.dataBucketAlwaysAvailable = [];
             //
-            // Process the data for ClickHouse insert
+            // Process the data for batch insert
             try {
-                // Transform data if a transformation function is provided
-                const insertData = this.dataBucketFlushOps.map((item) => {
-                    if (this.params.transformFn) {
-                        return this.params.transformFn(item);
-                    }
-                    return item;
-                });
-                // Insert data using ClickHouse client
-                await this.client.insert({
-                    format: 'JSONEachRow',
-                    table: this.params.table,
-                    values: insertData,
-                });
-                Logger.info(`CLICKHOUSEWRITER [${this.params.table}]: Flush | Length: ${this.dataBucketFlushOps.length} (session: ${sessionTimerResult}) (flush: ${flushTimer.get()})`);
-                //
+                // Call the insert function provided in the params to perform the actual database insertion.
+                if (!this.params.insertFn)
+                    throw new Error('BATCHWRITER: No insert function provided in params');
+                await this.params.insertFn(this.dataBucketFlushOps);
+                Logger.info(`BATCHWRITER [${this.params.title}]: Flush | Length: ${this.dataBucketFlushOps.length} (session: ${sessionTimerResult}) (flush: ${flushTimer.get()})`);
                 // Call the flush callback, if provided
-                if (callback) {
+                if (callback)
                     await callback(this.dataBucketFlushOps);
-                }
-                //
                 // Reset the flush bucket
                 this.dataBucketFlushOps = [];
-                //
             }
             catch (error) {
-                Logger.error(`CLICKHOUSEWRITER [${this.params.table}]: Error @ flush().insert(): ${error.message}`);
+                Logger.error(`BATCHWRITER [${this.params.title}]: Error @ flush().insert(): ${error.message}`);
                 throw error; // Re-throw to allow retry logic at higher level
             }
             //
         }
         catch (error) {
-            Logger.error(`CLICKHOUSEWRITER [${this.params.table}]: Error @ flush(): ${error.message}`);
+            Logger.error(`BATCHWRITER [${this.params.title}]: Error @ flush(): ${error.message}`);
             throw error; // Re-throw to allow retry logic at higher level
         }
     }
     /**
-     * Write data to the ClickHouse table.
+     * Write data to the batch.
      * @param data The data to write
      * @param options Options for the write operation (reserved for future use)
      * @param writeCallback Callback function to call after the write operation is complete
@@ -162,7 +85,6 @@ export class ClickHouseWriter {
      */
     async write(data, { flushCallback, writeCallback } = {}) {
         //
-        await this.init();
         //
         // Invalidate the previously set idle timeout timer
         // since we are performing a write operation again.
@@ -174,7 +96,7 @@ export class ClickHouseWriter {
         // Check if the batch is full
         const batchSize = this.params.batch_size ?? 10_000;
         if (this.dataBucketAlwaysAvailable.length >= batchSize) {
-            Logger.info(`CLICKHOUSEWRITER [${this.params.table}]: Batch full. Flushing data...`);
+            Logger.info(`BATCHWRITER [${this.params.title}]: Batch full. Flushing data...`);
             await this.flush(flushCallback);
         }
         //
@@ -201,7 +123,7 @@ export class ClickHouseWriter {
         // since the last write operation. Check if this functionality is enabled.
         if (this.params.idle_timeout && this.params.idle_timeout > 0 && !this.idleTimeoutTimer) {
             this.idleTimeoutTimer = setTimeout(async () => {
-                Logger.info(`CLICKHOUSEWRITER [${this.params.table}]: Idle timeout reached. Flushing data...`);
+                Logger.info(`BATCHWRITER [${this.params.title}]: Idle timeout reached. Flushing data...`);
                 await this.flush(flushCallback);
             }, this.params.idle_timeout);
         }
@@ -210,7 +132,7 @@ export class ClickHouseWriter {
         // even if the batch is not full. Check if this functionality is enabled.
         if (this.params.batch_timeout && this.params.batch_timeout > 0 && !this.batchTimeoutTimer) {
             this.batchTimeoutTimer = setTimeout(async () => {
-                Logger.info(`CLICKHOUSEWRITER [${this.params.table}]: Batch timeout reached. Flushing data...`);
+                Logger.info(`BATCHWRITER [${this.params.title}]: Batch timeout reached. Flushing data...`);
                 await this.flush(flushCallback);
             }, this.params.batch_timeout);
         }

package/dist/index.d.ts

@@ -1,5 +1,5 @@
-export * from './clickhouse.js';
 export * from './csv.js';
+export * from './generic.js';
 export * from './json.js';
 export * from './mongo.js';
 export * from './postgres.js';

package/dist/index.js

@@ -1,5 +1,5 @@
-export * from './clickhouse.js';
 export * from './csv.js';
+export * from './generic.js';
 export * from './json.js';
 export * from './mongo.js';
 export * from './postgres.js';

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@tmlmobilidade/writers",
-	"version": "20260320.1746.41",
+	"version": "20260322.1655.45",
 	"author": {
 		"email": "iso@tmlmobilidade.pt",
 		"name": "TML-ISO"
@@ -36,7 +36,6 @@
 		"watch": "tsc-watch --onSuccess 'resolve-tspaths'"
 	},
 	"dependencies": {
-		"@clickhouse/client": "1.18.2",
 		"@tmlmobilidade/clickhouse": "*",
 		"@tmlmobilidade/logger": "*",
 		"@tmlmobilidade/timer": "*",

package/dist/clickhouse.d.ts

@@ -1,96 +0,0 @@
-import { type ClickHouseClientConfigOptions } from '@clickhouse/client';
-import { NodeClickHouseClient } from '@clickhouse/client/dist/client.js';
-import { ClickHouseColumn } from '@tmlmobilidade/clickhouse';
-type ClickHouseWriterParams<T> = {
-    /**
-     * The maximum number of items to hold in memory
-     * before flushing to the database.
-     * @default 10_000
-     */
-    batch_size?: number;
-    /**
-     * How long, in milliseconds, data should be kept in memory before
-     * flushing to the database. If this feature is enabled, a flush will
-     * be triggered even if the batch is not full. Disabled by default.
-     * @default disabled
-     */
-    batch_timeout?: number;
-    /**
-     * If enabled, starts an async one-time table ensure operation in the constructor.
-     * Use `await writer.init()` if you need to block startup until it is completed.
-     * @default false
-     */
-    ensure_table_on_init?: boolean;
-    /**
-     * How long to wait, in milliseconds, after the last write operation
-     * before flushing the data to the database. This can be used to prevent
-     * items staying in memory for too long if the batch size is not reached
-     * frequently enough. Disabled by default.
-     * @default disabled
-     */
-    idle_timeout?: number;
-    /**
-     * The ClickHouse table name to write to.
-     * @required
-     */
-    table: string;
-    /**
-     * Optional ClickHouse column definitions for auto-creating the table.
-     */
-    tableSchema: ClickHouseColumn<T>[];
-    /**
-     * Optional transformation function to convert documents before writing to ClickHouse.
-     * Use this to map MongoDB document fields to ClickHouse column names.
-     */
-    transformFn?: (data: T) => Record<string, unknown>;
-} & ({
-    client: NodeClickHouseClient;
-    clientConfig?: never;
-} | {
-    client?: never;
-    clientConfig: ClickHouseClientConfigOptions;
-});
-export declare class ClickHouseWriter<T> {
-    private params;
-    private client;
-    private dataBucketAlwaysAvailable;
-    private dataBucketFlushOps;
-    private batchTimeoutTimer;
-    private idleTimeoutTimer;
-    private sessionTimer;
-    private isInitialized;
-    constructor(params: ClickHouseWriterParams<T>);
-    close(): Promise<void>;
-    /**
-     * Initializes the writer by ensuring the table exists.
-     * Safe to call multiple times.
-     */
-    init(): Promise<void>;
-    /**
-     * Ensures the table exists in ClickHouse by creating it if it doesn't exist.
-     * Uses the tableSchema provided in the constructor, or an optional schema parameter.
-     * @param schema Optional schema to use instead of the constructor-provided tableSchema
-     * @param engine The ClickHouse table engine to use (default: ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}'))
-     * @param orderBy The ORDER BY clause for the table (default: tuple())
-     */
-    ensureTable(schema?: ClickHouseColumn<T>[], engine?: string, orderBy?: string): Promise<void>;
-    /**
-     * Flushes the current batch of data to ClickHouse.
-     * This method is called internally when the batch size or timeouts are reached,
-     * but can also be called manually if needed.
-     * @param callback Optional callback to execute after the flush is complete, receiving the flushed data as a parameter
-     */
-    flush(callback?: (data?: T[]) => Promise<void>): Promise<void>;
-    /**
-     * Write data to the ClickHouse table.
-     * @param data The data to write
-     * @param options Options for the write operation (reserved for future use)
-     * @param writeCallback Callback function to call after the write operation is complete
-     * @param flushCallback Callback function to call after the flush operation is complete
-     */
-    write(data: T | T[], { flushCallback, writeCallback }?: {
-        flushCallback?: (data?: T[]) => Promise<void>;
-        writeCallback?: () => Promise<void>;
-    }): Promise<void>;
-}
-export {};