@tmlmobilidade/utils 20260330.1756.23 → 20260331.1620.53

package/dist/batching/batch-writer.d.ts

@@ -0,0 +1,81 @@
+interface BatchWriterParams<T> {
+    /**
+     * The maximum number of items to hold in memory
+     * before flushing to the database.
+     * @required
+     */
+    batch_size: number;
+    /**
+     * How long, in milliseconds, data should be kept in memory before
+     * flushing to the database. If this feature is enabled, a flush will
+     * be triggered even if the batch is not full. Disabled by default.
+     * @default disabled
+     */
+    batch_timeout?: number;
+    /**
+     * How long to wait, in milliseconds, after the last write operation
+     * before flushing the data to the database. This can be used to prevent
+     * items staying in memory for too long if the batch size is not reached
+     * frequently enough. Disabled by default.
+     * @default disabled
+     */
+    idle_timeout?: number;
+    /**
+     * The insert function to use for inserting data into the batch.
+     * @required
+     */
+    insertFn: (data: T[]) => Promise<void>;
+    /**
+     * Maximum number of retries for transient insert errors.
+     * @default 3
+     */
+    max_retries?: number;
+    /**
+     * Base delay in milliseconds for exponential backoff.
+     * @default 1000
+     */
+    retry_base_delay_ms?: number;
+    /**
+     * The title of this BatchWriter instance,
+     * used to identify the source of the logs.
+     * @required
+     */
+    title: string;
+}
+export declare class BatchWriter<T> {
+    private params;
+    private dataBucketAlwaysAvailable;
+    private dataBucketFlushOps;
+    private batchTimeoutTimer;
+    private idleTimeoutTimer;
+    private sessionTimer;
+    constructor(params: BatchWriterParams<T>);
+    /**
+     * Flushes the current batch of data.
+     * This method is called internally when the batch size or timeouts are reached,
+     * but can also be called manually if needed.
+     * @param callback Optional callback to execute after the flush is complete, receiving the flushed data as a parameter
+     */
+    flush(callback?: (data?: T[]) => Promise<void>): Promise<void>;
+    /**
+     * Helper method to perform insert operations with retry logic for transient errors.
+     * This method will attempt to insert the data using the provided insert function,
+     * and if an error occurs, it will retry the operation with exponential backoff
+     * until the maximum number of retries is reached.
+     * @param data The data to insert.
+     * @returns A promise that resolves when the insert operation is successful, or rejects if all retries fail.
+     */
+    private insertWithRetry;
+    /**
+     * Write data to the batch.
+     * @param data The data to write.
+     * @param options Options for the write operation (reserved for future use).
+     * @param writeCallback Callback function to call after the write operation is complete.
+     * @param flushCallback Callback function to call after the flush operation is complete.
+     */
+    write(data: T | T[], { flushCallback, writeCallback }?: {
+        flushCallback?: (data?: T[]) => Promise<void>;
+        writeCallback?: () => Promise<void>;
+    }): Promise<void>;
+}
+export {};

package/dist/batching/batch-writer.js

@@ -0,0 +1,166 @@
+/* eslint-disable perfectionist/sort-classes */
+/* * */
+import { Logger } from '@tmlmobilidade/logger';
+import { Timer } from '@tmlmobilidade/timer';
+/* * */
+export class BatchWriter {
+    //
+    params;
+    dataBucketAlwaysAvailable = [];
+    dataBucketFlushOps = [];
+    batchTimeoutTimer = null;
+    idleTimeoutTimer = null;
+    sessionTimer = new Timer();
+    constructor(params) {
+        if (!params.title)
+            throw new Error('BATCHWRITER: Title is required.');
+        if (!params.insertFn)
+            throw new Error('BATCHWRITER: Insert function is required.');
+        if (!params.batch_size)
+            throw new Error('BATCHWRITER: Batch size is required.');
+        this.params = params;
+    }
+    /**
+     * Flushes the current batch of data.
+     * This method is called internally when the batch size or timeouts are reached,
+     * but can also be called manually if needed.
+     * @param callback Optional callback to execute after the flush is complete, receiving the flushed data as a parameter
+     */
+    async flush(callback) {
+        try {
+            //
+            const flushTimer = new Timer();
+            const sessionTimerResult = this.sessionTimer.get();
+            //
+            // Invalidate all timers since a flush operation is being performed
+            if (this.idleTimeoutTimer) {
+                clearTimeout(this.idleTimeoutTimer);
+                this.idleTimeoutTimer = null;
+            }
+            if (this.batchTimeoutTimer) {
+                clearTimeout(this.batchTimeoutTimer);
+                this.batchTimeoutTimer = null;
+            }
+            //
+            // Skip if there is no data to flush
+            if (this.dataBucketAlwaysAvailable.length === 0)
+                return;
+            //
+            // Copy everything in dataBucketAlwaysAvailable to dataBucketFlushOps
+            // to prevent any new incoming data to be added to the batch. This is to ensure
+            // that the batch is not modified while it is being processed.
+            this.dataBucketFlushOps = [...this.dataBucketFlushOps, ...this.dataBucketAlwaysAvailable];
+            this.dataBucketAlwaysAvailable = [];
+            //
+            // Process the data for batch insert
+            try {
+                // Call the insert function provided in the params to perform the actual database insertion.
+                if (!this.params.insertFn)
+                    throw new Error('BATCHWRITER: No insert function provided in params');
+                await this.insertWithRetry(this.dataBucketFlushOps);
+                Logger.info(`BATCHWRITER [${this.params.title}]: Flush | Length: ${this.dataBucketFlushOps.length} (session: ${sessionTimerResult}) (flush: ${flushTimer.get()})`);
+                // Call the flush callback, if provided
+                if (callback)
+                    await callback(this.dataBucketFlushOps);
+                // Reset the flush bucket
+                this.dataBucketFlushOps = [];
+            }
+            catch (error) {
+                Logger.error(`BATCHWRITER [${this.params.title}]: Error @ flush().insert(): ${error.message}`);
+                throw error; // Re-throw to allow retry logic at higher level
+            }
+            //
+        }
+        catch (error) {
+            Logger.error(`BATCHWRITER [${this.params.title}]: Error @ flush(): ${error.message}`);
+            throw error; // Re-throw to allow retry logic at higher level
+        }
+    }
+    /**
+     * Helper method to perform insert operations with retry logic for transient errors.
+     * This method will attempt to insert the data using the provided insert function,
+     * and if an error occurs, it will retry the operation with exponential backoff
+     * until the maximum number of retries is reached.
+     * @param data The data to insert.
+     * @returns A promise that resolves when the insert operation is successful, or rejects if all retries fail.
+     */
+    async insertWithRetry(data) {
+        const maxRetries = this.params.max_retries ?? 3;
+        const retryBaseDelayMs = this.params.retry_base_delay_ms ?? 1000;
+        for (let attempt = 0; attempt <= maxRetries; attempt++) {
+            try {
+                await this.params.insertFn(data);
+                return;
+            }
+            catch (error) {
+                const parsedError = error;
+                const nextAttempt = attempt + 1;
+                const delayMs = retryBaseDelayMs * (2 ** attempt);
+                Logger.error(`BATCHWRITER [${this.params.title}]: Transient insert error (${parsedError.code ?? 'unknown'}). Retrying ${nextAttempt}/${maxRetries} in ${delayMs}ms. ${parsedError.message}`);
+                await new Promise(resolve => setTimeout(resolve, delayMs));
+            }
+        }
+    }
+    /**
+     * Write data to the batch.
+     * @param data The data to write.
+     * @param options Options for the write operation (reserved for future use).
+     * @param writeCallback Callback function to call after the write operation is complete.
+     * @param flushCallback Callback function to call after the flush operation is complete.
+     */
+    async write(data, { flushCallback, writeCallback } = {}) {
+        //
+        //
+        // Invalidate the previously set idle timeout timer
+        // since we are performing a write operation again.
+        if (this.idleTimeoutTimer) {
+            clearTimeout(this.idleTimeoutTimer);
+            this.idleTimeoutTimer = null;
+        }
+        //
+        // Check if the batch is full
+        const batchSize = this.params.batch_size ?? 10_000;
+        if (this.dataBucketAlwaysAvailable.length >= batchSize) {
+            Logger.info(`BATCHWRITER [${this.params.title}]: Batch full. Flushing data...`);
+            await this.flush(flushCallback);
+        }
+        //
+        // Reset the session timer (for logging purposes)
+        if (this.dataBucketAlwaysAvailable.length === 0) {
+            this.sessionTimer.reset();
+        }
+        //
+        // Add the current data to the batch
+        if (Array.isArray(data)) {
+            const combinedDataWithOptions = data.map(item => item);
+            this.dataBucketAlwaysAvailable = [...this.dataBucketAlwaysAvailable, ...combinedDataWithOptions];
+        }
+        else {
+            this.dataBucketAlwaysAvailable.push(data);
+        }
+        //
+        // Call the write callback, if provided
+        if (writeCallback) {
+            await writeCallback();
+        }
+        //
+        // Setup the idle timeout timer to flush the data if too long has passed
+        // since the last write operation. Check if this functionality is enabled.
+        if (this.params.idle_timeout && this.params.idle_timeout > 0 && !this.idleTimeoutTimer) {
+            this.idleTimeoutTimer = setTimeout(async () => {
+                Logger.info(`BATCHWRITER [${this.params.title}]: Idle timeout reached. Flushing data...`);
+                await this.flush(flushCallback);
+            }, this.params.idle_timeout);
+        }
+        //
+        // Setup the batch timeout timer to flush the data, if the timeout value is reached,
+        // even if the batch is not full. Check if this functionality is enabled.
+        if (this.params.batch_timeout && this.params.batch_timeout > 0 && !this.batchTimeoutTimer) {
+            this.batchTimeoutTimer = setTimeout(async () => {
+                Logger.info(`BATCHWRITER [${this.params.title}]: Batch timeout reached. Flushing data...`);
+                await this.flush(flushCallback);
+            }, this.params.batch_timeout);
+        }
+        //
+    }
+}

package/dist/batching/index.d.ts

@@ -1,3 +1,4 @@
+export * from './batch-writer.js';
 export * from './perform-in-chunks.js';
 export * from './perform-in-time-chunks.js';
 export * from './replicate.js';

package/dist/batching/index.js

@@ -1,3 +1,4 @@
+export * from './batch-writer.js';
 export * from './perform-in-chunks.js';
 export * from './perform-in-time-chunks.js';
 export * from './replicate.js';

package/dist/http.d.ts

@@ -73,6 +73,16 @@ export declare function uploadFile<T>(url: string, file: File): Promise<HttpResp
  * ```
  */
 export declare const swrFetcher: <T>(url: string) => Promise<T>;
+/**
+ * Fetches data from a URL using the SWR fetcher function.
+ * @param url - The URL to fetch from
+ * @returns Promise resolving to the fetched data
+ * @example
+ * ```ts
+ * const data = await swrFetcher('/api/users/123');
+ * ```
+ */
+export declare const unauthenticatedSwrFetcher: <T>(url: string) => Promise<T>;
 /**
  * Fetches data from a URL using the SWR fetcher function without authentication.
  * @param url - The URL to fetch from

package/dist/http.js

@@ -160,6 +160,23 @@ export const swrFetcher = async (url) => {
     }
     return data.data;
 };
+/**
+ * Fetches data from a URL using the SWR fetcher function.
+ * @param url - The URL to fetch from
+ * @returns Promise resolving to the fetched data
+ * @example
+ * ```ts
+ * const data = await swrFetcher('/api/users/123');
+ * ```
+ */
+export const unauthenticatedSwrFetcher = async (url) => {
+    const res = await fetch(url, { credentials: 'omit' });
+    const data = await res.json();
+    if (!res.ok) {
+        throw new HttpException(res.status, data.error ?? 'An error occurred');
+    }
+    return data.data;
+};
 /**
  * Fetches data from a URL using the SWR fetcher function without authentication.
  * @param url - The URL to fetch from

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@tmlmobilidade/utils",
-	"version": "20260330.1756.23",
+	"version": "20260331.1620.53",
 	"author": {
 		"email": "iso@tmlmobilidade.pt",
 		"name": "TML-ISO"