@tmlmobilidade/writers 20251202.1817.5

package/dist/csv.d.ts

@@ -0,0 +1,18 @@
+interface CsvWriterOptions {
+    batch_size?: number;
+    include_bom?: boolean;
+    new_line_character?: string;
+}
+export declare class CsvWriter<T> {
+    private CURRENT_BATCH_DATA;
+    private FILE_PATH;
+    private INCLUDE_BOM;
+    private INSTANCE_NAME;
+    private MAX_BATCH_SIZE;
+    private NEW_LINE_CHARACTER;
+    private SESSION_TIMER;
+    constructor(instanceName: string, filePath: string, options?: CsvWriterOptions);
+    flush(): Promise<void>;
+    write(data: T | T[]): Promise<void>;
+}
+export {};

package/dist/csv.js

@@ -0,0 +1,90 @@
+/* * */
+import { Logger } from '@tmlmobilidade/logger';
+import { Timer } from '@tmlmobilidade/timer';
+import fs from 'node:fs';
+import Papa from 'papaparse';
+/* * */
+export class CsvWriter {
+    //
+    CURRENT_BATCH_DATA = [];
+    FILE_PATH = null;
+    INCLUDE_BOM = false;
+    INSTANCE_NAME = 'Unnamed Instance';
+    MAX_BATCH_SIZE = 5000;
+    NEW_LINE_CHARACTER = '\n';
+    SESSION_TIMER = new Timer();
+    /* * */
+    constructor(instanceName, filePath, options) {
+        this.INSTANCE_NAME = instanceName;
+        this.FILE_PATH = filePath;
+        if (options?.batch_size)
+            this.MAX_BATCH_SIZE = options.batch_size;
+        if (options?.new_line_character)
+            this.NEW_LINE_CHARACTER = options.new_line_character;
+        if (options?.include_bom)
+            this.INCLUDE_BOM = options.include_bom;
+    }
+    /* * */
+    async flush() {
+        try {
+            //
+            if (!this.FILE_PATH)
+                return;
+            const flushTimer = new Timer();
+            const sssionTimerResult = this.SESSION_TIMER.get();
+            if (this.CURRENT_BATCH_DATA.length === 0)
+                return;
+            //
+            // Setup a variable to keep track if the file exists or not
+            const fileAlreadyExists = fs.existsSync(this.FILE_PATH);
+            //
+            // Use papaparse to produce the CSV string
+            let csvData = Papa.unparse(this.CURRENT_BATCH_DATA, { header: !fileAlreadyExists, newline: this.NEW_LINE_CHARACTER, skipEmptyLines: 'greedy' });
+            //
+            // Prepend BOM if this is the first write and BOM is enabled
+            if (!fileAlreadyExists && this.INCLUDE_BOM) {
+                csvData = '\uFEFF' + csvData;
+            }
+            //
+            // Prepend a new line character to csvData string
+            // if it is not the first line on the file.
+            if (fileAlreadyExists) {
+                csvData = this.NEW_LINE_CHARACTER + csvData;
+            }
+            //
+            // Recurseively ensure that the directory
+            // for the file path exists.
+            const dirPath = this.FILE_PATH.substring(0, this.FILE_PATH.lastIndexOf('/'));
+            if (!fs.existsSync(dirPath))
+                fs.mkdirSync(dirPath, { recursive: true });
+            //
+            // Append the csv string to the file
+            fs.appendFileSync(this.FILE_PATH, csvData, { encoding: 'utf-8', flush: true });
+            Logger.info(`CSVWRITER [${this.INSTANCE_NAME}]: Flush | Length: ${this.CURRENT_BATCH_DATA.length} | File Path: ${this.FILE_PATH} (session: ${sssionTimerResult}) (flush: ${flushTimer.get()})`);
+            this.CURRENT_BATCH_DATA = [];
+            //
+        }
+        catch (error) {
+            throw new Error(`Error at flush(): ${error.message}`);
+        }
+    }
+    /* * */
+    async write(data) {
+        //
+        //
+        // Check if the batch is full
+        if (this.CURRENT_BATCH_DATA.length >= this.MAX_BATCH_SIZE)
+            await this.flush();
+        //
+        // Reset the timer
+        if (this.CURRENT_BATCH_DATA.length === 0)
+            this.SESSION_TIMER.reset();
+        //
+        // Add the data to the batch
+        if (Array.isArray(data))
+            this.CURRENT_BATCH_DATA.push(...data);
+        else
+            this.CURRENT_BATCH_DATA.push(data);
+        //
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export * from './csv.js';
+export * from './json.js';
+export * from './mongo.js';
+export * from './postgres.js';

package/dist/index.js

@@ -0,0 +1,4 @@
+export * from './csv.js';
+export * from './json.js';
+export * from './mongo.js';
+export * from './postgres.js';

package/dist/json.d.ts

@@ -0,0 +1,19 @@
+interface JsonWriterOptions {
+    add_after?: string;
+    add_before?: string;
+    batch_size?: number;
+}
+export declare class JsonWriter<T> {
+    private readonly ADD_AFTER;
+    private readonly ADD_BEFORE;
+    private CURRENT_BATCH_DATA;
+    private FILE_PATH;
+    private INSTANCE_NAME;
+    private MAX_BATCH_SIZE;
+    private SESSION_TIMER;
+    constructor(instanceName: string, filePath: string, options?: JsonWriterOptions);
+    close(): void;
+    flush(): void;
+    write(data: T | T[]): void;
+}
+export {};

package/dist/json.js

@@ -0,0 +1,86 @@
+/* * */
+import { Logger } from '@tmlmobilidade/logger';
+import { Timer } from '@tmlmobilidade/timer';
+import fs from 'node:fs';
+/* * */
+export class JsonWriter {
+    //
+    ADD_AFTER = null;
+    ADD_BEFORE = null;
+    CURRENT_BATCH_DATA = [];
+    FILE_PATH = null;
+    INSTANCE_NAME = 'Unnamed Instance';
+    MAX_BATCH_SIZE = 5000;
+    SESSION_TIMER = new Timer();
+    /* * */
+    constructor(instanceName, filePath, options) {
+        this.INSTANCE_NAME = instanceName;
+        this.FILE_PATH = filePath;
+        if (options?.add_after)
+            this.ADD_AFTER = options.add_after;
+        if (options?.add_before)
+            this.ADD_BEFORE = options.add_before;
+        if (options?.batch_size)
+            this.MAX_BATCH_SIZE = options.batch_size;
+    }
+    /* * */
+    close() {
+        this.flush();
+        let writableData = ']';
+        if (this.ADD_AFTER)
+            writableData = writableData + this.ADD_AFTER;
+        fs.appendFileSync(this.FILE_PATH, writableData);
+    }
+    /* * */
+    flush() {
+        //
+        if (!this.FILE_PATH)
+            throw new Error('File path is not set. Please provide a valid file path.');
+        const flushTimer = new Timer();
+        const sssionTimerResult = this.SESSION_TIMER.get();
+        if (this.CURRENT_BATCH_DATA.length === 0)
+            return;
+        // Setup a variable to keep track if the file exists or not
+        const fileAlreadyExists = fs.existsSync(this.FILE_PATH);
+        // Use papaparse to produce the CSV string
+        let writableData = JSON.stringify(this.CURRENT_BATCH_DATA);
+        // Remove the first and last characters if they are brackets
+        if (writableData.startsWith('['))
+            writableData = writableData.slice(1);
+        if (writableData.endsWith(']'))
+            writableData = writableData.slice(0, -1);
+        // If the file already does not yet exist then this is the first flush,
+        // which means we need to add an opening bracket. If the file already exists,
+        // we need to add a comma before the data.
+        if (!fileAlreadyExists)
+            writableData = '[' + writableData;
+        else
+            writableData = ',' + writableData;
+        // If the file does not exist add the value for ADD_BEFORE, if it is set.
+        if (!fileAlreadyExists && this.ADD_BEFORE)
+            writableData = this.ADD_BEFORE + writableData;
+        // Append the csv string to the file
+        fs.appendFileSync(this.FILE_PATH, writableData);
+        Logger.info(`JSONWRITER [${this.INSTANCE_NAME}]: Flush | Length: ${this.CURRENT_BATCH_DATA.length} | File Path: ${this.FILE_PATH} (session: ${sssionTimerResult}) (flush: ${flushTimer.get()})`);
+        this.CURRENT_BATCH_DATA = [];
+    }
+    /* * */
+    write(data) {
+        // Check if the batch is full
+        if (this.CURRENT_BATCH_DATA.length >= this.MAX_BATCH_SIZE) {
+            this.flush();
+        }
+        // Reset the timer
+        if (this.CURRENT_BATCH_DATA.length === 0) {
+            this.SESSION_TIMER.reset();
+        }
+        // Add the data to the batch
+        if (Array.isArray(data)) {
+            this.CURRENT_BATCH_DATA = [...this.CURRENT_BATCH_DATA, ...data];
+        }
+        else {
+            this.CURRENT_BATCH_DATA.push(data);
+        }
+        //
+    }
+}

package/dist/mongo.d.ts

@@ -0,0 +1,62 @@
+interface MongoDbWriterParams {
+    /**
+     * The maximum number of items to hold in memory
+     * before flushing to the database.
+     * @default 3000
+     */
+    batch_size?: number;
+    /**
+     * How long, in milliseconds, data should be kept in memory before
+     * flushing to the database. If this feature is enabled, a flush will
+     * be triggered even if the batch is not full. Disabled by default.
+     * @default disabled
+     */
+    batch_timeout?: number;
+    /**
+     * The MongoDB collection to write to.
+     * @required
+     */
+    collection: any;
+    /**
+     * How long to wait, in milliseconds, after the last write operation
+     * before flushing the data to the database. This can be used to prevent
+     * items staying in memory for too long if the batch size is not reached
+     * frequently enough. Disabled by default.
+     * @default disabled
+     */
+    idle_timeout?: number;
+}
+export interface MongoDbWriterWriteOptions {
+    filter?: object;
+    upsert?: boolean;
+    write_mode?: 'replace' | 'update';
+}
+export interface MongoDBWriterWriteOps<T> {
+    data: T;
+    options: MongoDbWriterWriteOptions;
+}
+export declare class MongoDbWriter<T> {
+    private BATCH_SIZE;
+    private BATCH_TIMEOUT_ENABLED;
+    private BATCH_TIMEOUT_TIMER;
+    private BATCH_TIMEOUT_VALUE;
+    private DATA_BUCKET_ALWAYS_AVAILABLE;
+    private DATA_BUCKET_FLUSH_OPS;
+    private DB_COLLECTION;
+    private IDLE_TIMEOUT_ENABLED;
+    private IDLE_TIMEOUT_TIMER;
+    private IDLE_TIMEOUT_VALUE;
+    private SESSION_TIMER;
+    constructor(params: MongoDbWriterParams);
+    flush(callback?: (data?: MongoDBWriterWriteOps<T>[]) => Promise<void>): Promise<void>;
+    /**
+     * Write data to the MongoDB collection.
+     *
+     * @param data The data to write
+     * @param options Options for the write operation
+     * @param writeCallback Callback function to call after the write operation is complete
+     * @param flushCallback Callback function to call after the flush operation is complete
+     */
+    write(data: T | T[], options?: MongoDbWriterWriteOptions, writeCallback?: () => Promise<void>, flushCallback?: (data?: MongoDBWriterWriteOps<T>[]) => Promise<void>): Promise<void>;
+}
+export {};

package/dist/mongo.js

@@ -0,0 +1,174 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* * */
+import { Logger } from '@tmlmobilidade/logger';
+import { Timer } from '@tmlmobilidade/timer';
+/* * */
+export class MongoDbWriter {
+    //
+    BATCH_SIZE = 3000;
+    BATCH_TIMEOUT_ENABLED = false;
+    BATCH_TIMEOUT_TIMER = null;
+    BATCH_TIMEOUT_VALUE = -1;
+    DATA_BUCKET_ALWAYS_AVAILABLE = [];
+    DATA_BUCKET_FLUSH_OPS = [];
+    DB_COLLECTION = null;
+    IDLE_TIMEOUT_ENABLED = false;
+    IDLE_TIMEOUT_TIMER = null;
+    IDLE_TIMEOUT_VALUE = -1;
+    SESSION_TIMER = new Timer();
+    /* * */
+    constructor(params) {
+        // Ensure that the MongoDB Collection is provided
+        if (!params.collection)
+            throw new Error('MONGODBWRITER: Collection is required');
+        this.DB_COLLECTION = params.collection;
+        // Setup the optional idle timeout functionality
+        if (params.idle_timeout > 0) {
+            this.IDLE_TIMEOUT_ENABLED = true;
+            this.IDLE_TIMEOUT_VALUE = params.idle_timeout;
+        }
+        // Override the default batch size
+        if (params.batch_size > 0) {
+            this.BATCH_SIZE = params.batch_size;
+        }
+        // Setup the optional batch timeout functionality
+        if (params.batch_timeout > 0) {
+            this.BATCH_TIMEOUT_ENABLED = true;
+            this.BATCH_TIMEOUT_VALUE = params.batch_timeout;
+        }
+    }
+    /* * */
+    async flush(callback) {
+        try {
+            //
+            const flushTimer = new Timer();
+            const sssionTimerResult = this.SESSION_TIMER.get();
+            //
+            // Invalidate all timers since a flush operation is being performed
+            if (this.IDLE_TIMEOUT_TIMER) {
+                clearTimeout(this.IDLE_TIMEOUT_TIMER);
+                this.IDLE_TIMEOUT_TIMER = null;
+            }
+            if (this.BATCH_TIMEOUT_TIMER) {
+                clearTimeout(this.BATCH_TIMEOUT_TIMER);
+                this.BATCH_TIMEOUT_TIMER = null;
+            }
+            //
+            // Skip if there is no data to flush
+            if (this.DATA_BUCKET_ALWAYS_AVAILABLE.length === 0)
+                return;
+            //
+            // Copy everything in data DATA_BUCKET_ALWAYS_AVAILABLE to DATA_BUCKET_FLUSH_OPS
+            // to prevent any new incoming data to be added to the batch. This is to ensure
+            // that the batch is not modified while it is being processed.
+            this.DATA_BUCKET_FLUSH_OPS = [...this.DATA_BUCKET_FLUSH_OPS, ...this.DATA_BUCKET_ALWAYS_AVAILABLE];
+            this.DATA_BUCKET_ALWAYS_AVAILABLE = [];
+            //
+            // Process the data into MongoDB insert/update operations
+            try {
+                const writeOperations = this.DATA_BUCKET_FLUSH_OPS.map((item) => {
+                    switch (item.options?.write_mode) {
+                        case 'update':
+                            return {
+                                updateOne: {
+                                    filter: item.options.filter,
+                                    update: item.data,
+                                    upsert: true,
+                                },
+                            };
+                        case 'replace':
+                        default:
+                            return {
+                                replaceOne: {
+                                    filter: item.options.filter,
+                                    replacement: item.data,
+                                    upsert: item.options?.upsert ? true : false,
+                                },
+                            };
+                    }
+                });
+                await this.DB_COLLECTION.bulkWrite(writeOperations);
+                Logger.info(`MONGODBWRITER [${this.DB_COLLECTION.collectionName}]: Flush | Length: ${this.DATA_BUCKET_FLUSH_OPS.length} (session: ${sssionTimerResult}) (flush: ${flushTimer.get()})`);
+                //
+                // Call the flush callback, if provided
+                if (callback) {
+                    await callback(this.DATA_BUCKET_FLUSH_OPS);
+                }
+                //
+                // Reset the flush bucket
+                this.DATA_BUCKET_FLUSH_OPS = [];
+                //
+            }
+            catch (error) {
+                Logger.error(`MONGODBWRITER [${this.DB_COLLECTION.collectionName}]: Error @ flush().writeOperations(): ${error.message}`);
+            }
+            //
+        }
+        catch (error) {
+            Logger.error(`MONGODBWRITER [${this.DB_COLLECTION.collectionName}]: Error @ flush(): ${error.message}`);
+        }
+    }
+    /* * */
+    /**
+     * Write data to the MongoDB collection.
+     *
+     * @param data The data to write
+     * @param options Options for the write operation
+     * @param writeCallback Callback function to call after the write operation is complete
+     * @param flushCallback Callback function to call after the flush operation is complete
+     */
+    async write(data, options = {}, writeCallback, flushCallback) {
+        //
+        //
+        // Invalidate the previously set idle timeout timer
+        // since we are performing a write operation again.
+        if (this.IDLE_TIMEOUT_TIMER) {
+            clearTimeout(this.IDLE_TIMEOUT_TIMER);
+            this.IDLE_TIMEOUT_TIMER = null;
+        }
+        //
+        // Check if the batch is full
+        if (this.DATA_BUCKET_ALWAYS_AVAILABLE.length >= this.BATCH_SIZE) {
+            Logger.info(`MONGODBWRITER [${this.DB_COLLECTION.collectionName}]: Batch full. Flushing data...`);
+            await this.flush(flushCallback);
+        }
+        //
+        // Reset the session timer (for logging purposes)
+        if (this.DATA_BUCKET_ALWAYS_AVAILABLE.length === 0) {
+            this.SESSION_TIMER.reset();
+        }
+        //
+        // Add the current data to the batch
+        if (Array.isArray(data)) {
+            const combinedDataWithOptions = data.map(item => ({ data: item, options: options }));
+            this.DATA_BUCKET_ALWAYS_AVAILABLE = [...this.DATA_BUCKET_ALWAYS_AVAILABLE, ...combinedDataWithOptions];
+        }
+        else {
+            this.DATA_BUCKET_ALWAYS_AVAILABLE.push({ data: data, options: options });
+        }
+        //
+        // Call the write callback, if provided
+        if (writeCallback) {
+            await writeCallback();
+        }
+        //
+        // Setup the idle timeout timer to flush the data if too long has passed
+        // since the last write operation. Check if this functionality is enabled.
+        if (this.IDLE_TIMEOUT_ENABLED && !this.IDLE_TIMEOUT_TIMER) {
+            this.IDLE_TIMEOUT_TIMER = setTimeout(async () => {
+                Logger.info(`MONGODBWRITER [${this.DB_COLLECTION.collectionName}]: Idle timeout reached. Flushing data...`);
+                await this.flush(flushCallback);
+            }, this.IDLE_TIMEOUT_VALUE);
+        }
+        //
+        // Setup the batch timeout timer to flush the data, if the timeout value is reached,
+        // even if the batch is not full. Check if this functionality is enabled.
+        if (this.BATCH_TIMEOUT_ENABLED && !this.BATCH_TIMEOUT_TIMER) {
+            this.BATCH_TIMEOUT_TIMER = setTimeout(async () => {
+                Logger.info(`MONGODBWRITER [${this.DB_COLLECTION.collectionName}]: Batch timeout reached. Flushing data...`);
+                await this.flush(flushCallback);
+            }, this.BATCH_TIMEOUT_VALUE);
+        }
+        //
+    }
+}

package/dist/postgres.d.ts

@@ -0,0 +1,15 @@
+interface PostgresWriterOptions {
+    batch_size?: number;
+}
+export declare class PostgresWriter {
+    private CURRENT_BATCH_DATA;
+    private DB_CLIENT;
+    private DB_TABLE;
+    private INSTANCE_NAME;
+    private MAX_BATCH_SIZE;
+    private SESSION_TIMER;
+    constructor(instanceName: string, client: any, table: any, options?: PostgresWriterOptions);
+    flush(): Promise<void>;
+    write(data: any, options?: {}): Promise<void>;
+}
+export {};

package/dist/postgres.js

@@ -0,0 +1,77 @@
+/* * */
+import { Logger } from '@tmlmobilidade/logger';
+import { Timer } from '@tmlmobilidade/timer';
+/* * */
+export class PostgresWriter {
+    //
+    CURRENT_BATCH_DATA = [];
+    DB_CLIENT = null;
+    DB_TABLE = null;
+    INSTANCE_NAME = 'Unnamed Instance';
+    MAX_BATCH_SIZE = 250;
+    SESSION_TIMER = new Timer();
+    /* * */
+    constructor(instanceName, client, table, options) {
+        if (instanceName)
+            this.INSTANCE_NAME = instanceName;
+        if (client)
+            this.DB_CLIENT = client;
+        if (table)
+            this.DB_TABLE = table;
+        if (options?.batch_size)
+            this.MAX_BATCH_SIZE = options.batch_size;
+    }
+    /* * */
+    async flush() {
+        try {
+            //
+            const flushTimer = new Timer();
+            const sssionTimerResult = this.SESSION_TIMER.get();
+            if (this.CURRENT_BATCH_DATA.length === 0)
+                return;
+            const columns = Object.keys(this.CURRENT_BATCH_DATA[0]);
+            const values = [];
+            const placeholders = this.CURRENT_BATCH_DATA.map((item) => {
+                const rowValues = columns.map((key) => {
+                    values.push(item[key]);
+                    return `$${values.length}`;
+                });
+                return `(${rowValues.join(', ')})`;
+            });
+            const insertQuery = `
+				BEGIN;
+				DELETE FROM ${this.DB_TABLE} WHERE code = $1;
+				INSERT INTO ${this.DB_TABLE} (${columns.join(', ')})
+				VALUES (${placeholders.join(', ')});
+				COMMIT;
+			`;
+            await this.DB_CLIENT.query(insertQuery, values);
+            Logger.info(`POSTGRESWRITER [${this.INSTANCE_NAME}]: Flush | Length: ${this.CURRENT_BATCH_DATA.length} | DB Table: ${this.DB_TABLE} (session: ${sssionTimerResult}) (flush: ${flushTimer.get()})`);
+            this.CURRENT_BATCH_DATA = [];
+            //
+        }
+        catch (error) {
+            Logger.error(`POSTGRESWRITER [${this.INSTANCE_NAME}]: Error @ flush(): ${error.message}`);
+        }
+    }
+    /* * */
+    async write(data, options = {}) {
+        // Check if the batch is full
+        if (this.CURRENT_BATCH_DATA.length >= this.MAX_BATCH_SIZE) {
+            await this.flush();
+        }
+        // Reset the timer
+        if (this.CURRENT_BATCH_DATA.length === 0) {
+            this.SESSION_TIMER.reset();
+        }
+        // Add the data to the batch
+        if (Array.isArray(data)) {
+            const combinedDataWithOptions = data.map(item => ({ data: item, options: options }));
+            this.CURRENT_BATCH_DATA = [...this.CURRENT_BATCH_DATA, ...combinedDataWithOptions];
+        }
+        else {
+            this.CURRENT_BATCH_DATA.push(data);
+        }
+        //
+    }
+}

package/package.json

@@ -0,0 +1,51 @@
+{
+	"name": "@tmlmobilidade/writers",
+	"version": "20251202.1817.5",
+	"author": {
+		"email": "iso@tmlmobilidade.pt",
+		"name": "TML-ISO"
+	},
+	"license": "AGPL-3.0-or-later",
+	"homepage": "https://github.com/tmlmobilidade/go#readme",
+	"bugs": {
+		"url": "https://github.com/tmlmobilidade/go/issues"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/tmlmobilidade/go.git"
+	},
+	"keywords": [
+		"public transit",
+		"tml",
+		"transportes metropolitanos de lisboa",
+		"go"
+	],
+	"publishConfig": {
+		"access": "public"
+	},
+	"type": "module",
+	"files": [
+		"dist"
+	],
+	"main": "./dist/index.js",
+	"types": "./dist/index.d.ts",
+	"scripts": {
+		"build": "tsc && resolve-tspaths",
+		"lint": "eslint ./src/ && tsc --noEmit",
+		"lint:fix": "eslint ./src/ --fix",
+		"watch": "tsc-watch --onSuccess 'resolve-tspaths'"
+	},
+	"dependencies": {
+		"@tmlmobilidade/logger": "*",
+		"@tmlmobilidade/timer": "*",
+		"papaparse": "5.5.3"
+	},
+	"devDependencies": {
+		"@tmlmobilidade/tsconfig": "*",
+		"@tmlmobilidade/types": "*",
+		"@types/node": "24.10.1",
+		"resolve-tspaths": "0.8.23",
+		"tsc-watch": "7.2.0",
+		"typescript": "5.9.3"
+	}
+}