@tmlmobilidade/go-utils-exec 20260828.1700.4

package/dist/batch/batch-writer.d.ts

@@ -0,0 +1,91 @@
+interface BatchWriterParams<T> {
+    /**
+     * The maximum number of items to hold in memory
+     * before flushing to the database.
+     * @required
+     */
+    batch_size: number;
+    /**
+     * How long, in milliseconds, data should be kept in memory before
+     * flushing to the database. If this feature is enabled, a flush will
+     * be triggered even if the batch is not full. Disabled by default.
+     * @default disabled
+     */
+    batch_timeout?: number;
+    /**
+     * How long to wait, in milliseconds, after the last write operation
+     * before flushing the data to the database. This can be used to prevent
+     * items staying in memory for too long if the batch size is not reached
+     * frequently enough. Disabled by default.
+     * @default disabled
+     */
+    idle_timeout?: number;
+    /**
+     * The insert function to use for inserting data into the batch.
+     * @required
+     */
+    insertFn: (data: T[]) => Promise<void>;
+    /**
+     * Maximum number of retries for transient insert errors.
+     * @default 3
+     */
+    max_retries?: number;
+    /**
+     * Base delay in milliseconds for exponential backoff.
+     * @default 1000
+     */
+    retry_base_delay_ms?: number;
+    /**
+     * The title of this BatchWriter instance,
+     * used to identify the source of the logs.
+     * @required
+     */
+    title: string;
+}
+export declare class BatchWriter<T> {
+    private params;
+    private dataBucketAlwaysAvailable;
+    private dataBucketFlushOps;
+    private batchTimeoutTimer;
+    private idleTimeoutTimer;
+    private sessionTimer;
+    private flushInProgress;
+    constructor(params: BatchWriterParams<T>);
+    /**
+     * Flushes the current batch of data.
+     * This method is called internally when the batch size or timeouts are reached,
+     * but can also be called manually if needed.
+     * @param callback Optional callback to execute after the flush is complete, receiving the flushed data as a parameter
+     */
+    flush(callback?: (data?: T[]) => Promise<void>): Promise<void>;
+    /**
+     * Timer-triggered flush that guarantees the buffer is emptied even when a
+     * flush was already in progress. If the guarded flush() coalesced into a
+     * running flush, data written during that flush would strand on an idle tail
+     * (timers are cleared at flush start and only re-armed by the next write()).
+     * Re-flush while data remains and no timer is pending.
+     */
+    private drain;
+    private runFlush;
+    /**
+     * Helper method to perform insert operations with retry logic for transient errors.
+     * This method will attempt to insert the data using the provided insert function,
+     * and if an error occurs, it will retry the operation with exponential backoff
+     * until the maximum number of retries is reached.
+     * @param data The data to insert.
+     * @returns A promise that resolves when the insert operation is successful, or rejects if all retries fail.
+     */
+    private insertWithRetry;
+    /**
+     * Write data to the batch.
+     * @param data The data to write.
+     * @param options Options for the write operation (reserved for future use).
+     * @param writeCallback Callback function to call after the write operation is complete.
+     * @param flushCallback Callback function to call after the flush operation is complete.
+     */
+    write(data: T | T[], { flushCallback, writeCallback }?: {
+        flushCallback?: (data?: T[]) => Promise<void>;
+        writeCallback?: () => Promise<void>;
+    }): Promise<void>;
+}
+export {};

package/dist/batch/batch-writer.js

@@ -0,0 +1,198 @@
+/* eslint-disable perfectionist/sort-classes */
+/* * */
+import { Timer } from '@tmlmobilidade/timer';
+/* * */
+export class BatchWriter {
+    //
+    params;
+    dataBucketAlwaysAvailable = [];
+    dataBucketFlushOps = [];
+    batchTimeoutTimer = null;
+    idleTimeoutTimer = null;
+    sessionTimer = new Timer();
+    // ponytail: single in-flight guard, not a queue. Serializes overlapping
+    // flushes (timer-triggered vs batch-full) so inserts never run concurrently
+    // and callers applying backpressure await the same promise.
+    flushInProgress = null;
+    constructor(params) {
+        if (!params.title)
+            throw new Error('BATCHWRITER: Title is required.');
+        if (!params.insertFn)
+            throw new Error('BATCHWRITER: Insert function is required.');
+        if (!params.batch_size)
+            throw new Error('BATCHWRITER: Batch size is required.');
+        this.params = params;
+    }
+    /**
+     * Flushes the current batch of data.
+     * This method is called internally when the batch size or timeouts are reached,
+     * but can also be called manually if needed.
+     * @param callback Optional callback to execute after the flush is complete, receiving the flushed data as a parameter
+     */
+    async flush(callback) {
+        // If a flush is already running, await it instead of starting a
+        // concurrent insert. This is the backpressure seam: the change-stream
+        // handler awaits write() -> flush() and cannot outrun the insert.
+        if (this.flushInProgress) {
+            await this.flushInProgress;
+            return;
+        }
+        this.flushInProgress = this.runFlush(callback);
+        try {
+            await this.flushInProgress;
+        }
+        finally {
+            this.flushInProgress = null;
+        }
+    }
+    /**
+     * Timer-triggered flush that guarantees the buffer is emptied even when a
+     * flush was already in progress. If the guarded flush() coalesced into a
+     * running flush, data written during that flush would strand on an idle tail
+     * (timers are cleared at flush start and only re-armed by the next write()).
+     * Re-flush while data remains and no timer is pending.
+     */
+    async drain(callback) {
+        await this.flush(callback);
+        if (this.dataBucketAlwaysAvailable.length > 0 && !this.idleTimeoutTimer && !this.batchTimeoutTimer && !this.flushInProgress) {
+            await this.flush(callback);
+        }
+    }
+    async runFlush(callback) {
+        try {
+            //
+            const flushTimer = new Timer();
+            const sessionTimerResult = this.sessionTimer.get();
+            //
+            // Invalidate all timers since a flush operation is being performed
+            if (this.idleTimeoutTimer) {
+                clearTimeout(this.idleTimeoutTimer);
+                this.idleTimeoutTimer = null;
+            }
+            if (this.batchTimeoutTimer) {
+                clearTimeout(this.batchTimeoutTimer);
+                this.batchTimeoutTimer = null;
+            }
+            //
+            // Skip if there is no data to flush
+            if (this.dataBucketAlwaysAvailable.length === 0)
+                return;
+            //
+            // Copy everything in dataBucketAlwaysAvailable to dataBucketFlushOps
+            // to prevent any new incoming data to be added to the batch. This is to ensure
+            // that the batch is not modified while it is being processed.
+            this.dataBucketFlushOps = [...this.dataBucketFlushOps, ...this.dataBucketAlwaysAvailable];
+            this.dataBucketAlwaysAvailable = [];
+            //
+            // Process the data for batch insert
+            try {
+                // Call the insert function provided in the params to perform the actual database insertion.
+                if (!this.params.insertFn)
+                    throw new Error('BATCHWRITER: No insert function provided in params');
+                await this.insertWithRetry(this.dataBucketFlushOps);
+                console.info(`BATCHWRITER [${this.params.title}]: Flush | Length: ${this.dataBucketFlushOps.length} (session: ${sessionTimerResult}) (flush: ${flushTimer.get()})`);
+                // Call the flush callback, if provided
+                if (callback)
+                    await callback(this.dataBucketFlushOps);
+                // Reset the flush bucket
+                this.dataBucketFlushOps = [];
+            }
+            catch (error) {
+                console.error(`BATCHWRITER [${this.params.title}]: Error @ flush().insert(): ${error.message}`);
+                throw error; // Re-throw to allow retry logic at higher level
+            }
+            //
+        }
+        catch (error) {
+            console.error(`BATCHWRITER [${this.params.title}]: Error @ flush(): ${error.message}`);
+            throw error; // Re-throw to allow retry logic at higher level
+        }
+    }
+    /**
+     * Helper method to perform insert operations with retry logic for transient errors.
+     * This method will attempt to insert the data using the provided insert function,
+     * and if an error occurs, it will retry the operation with exponential backoff
+     * until the maximum number of retries is reached.
+     * @param data The data to insert.
+     * @returns A promise that resolves when the insert operation is successful, or rejects if all retries fail.
+     */
+    async insertWithRetry(data) {
+        const maxRetries = this.params.max_retries ?? 3;
+        const retryBaseDelayMs = this.params.retry_base_delay_ms ?? 1000;
+        for (let attempt = 0; attempt <= maxRetries; attempt++) {
+            try {
+                await this.params.insertFn(data);
+                return;
+            }
+            catch (error) {
+                const parsedError = error;
+                const nextAttempt = attempt + 1;
+                const delayMs = retryBaseDelayMs * (2 ** attempt);
+                console.error(`BATCHWRITER [${this.params.title}]: Transient insert error (${parsedError.code ?? 'unknown'}). Retrying ${nextAttempt}/${maxRetries} in ${delayMs}ms. ${parsedError.message}`);
+                await new Promise(resolve => setTimeout(resolve, delayMs));
+            }
+        }
+    }
+    /**
+     * Write data to the batch.
+     * @param data The data to write.
+     * @param options Options for the write operation (reserved for future use).
+     * @param writeCallback Callback function to call after the write operation is complete.
+     * @param flushCallback Callback function to call after the flush operation is complete.
+     */
+    async write(data, { flushCallback, writeCallback } = {}) {
+        //
+        //
+        // Invalidate the previously set idle timeout timer
+        // since we are performing a write operation again.
+        if (this.idleTimeoutTimer) {
+            clearTimeout(this.idleTimeoutTimer);
+            this.idleTimeoutTimer = null;
+        }
+        //
+        // Check if the batch is full
+        const batchSize = this.params.batch_size ?? 10_000;
+        if (this.dataBucketAlwaysAvailable.length >= batchSize) {
+            console.info(`BATCHWRITER [${this.params.title}]: Batch full. Flushing data...`);
+            await this.flush(flushCallback);
+        }
+        //
+        // Reset the session timer (for logging purposes)
+        if (this.dataBucketAlwaysAvailable.length === 0) {
+            this.sessionTimer.reset();
+        }
+        //
+        // Add the current data to the batch
+        if (Array.isArray(data)) {
+            const combinedDataWithOptions = data.map(item => item);
+            this.dataBucketAlwaysAvailable = [...this.dataBucketAlwaysAvailable, ...combinedDataWithOptions];
+        }
+        else {
+            this.dataBucketAlwaysAvailable.push(data);
+        }
+        //
+        // Call the write callback, if provided
+        if (writeCallback) {
+            await writeCallback();
+        }
+        //
+        // Setup the idle timeout timer to flush the data if too long has passed
+        // since the last write operation. Check if this functionality is enabled.
+        if (this.params.idle_timeout && this.params.idle_timeout > 0 && !this.idleTimeoutTimer) {
+            this.idleTimeoutTimer = setTimeout(async () => {
+                console.info(`BATCHWRITER [${this.params.title}]: Idle timeout reached. Flushing data...`);
+                await this.drain(flushCallback);
+            }, this.params.idle_timeout);
+        }
+        //
+        // Setup the batch timeout timer to flush the data, if the timeout value is reached,
+        // even if the batch is not full. Check if this functionality is enabled.
+        if (this.params.batch_timeout && this.params.batch_timeout > 0 && !this.batchTimeoutTimer) {
+            this.batchTimeoutTimer = setTimeout(async () => {
+                console.info(`BATCHWRITER [${this.params.title}]: Batch timeout reached. Flushing data...`);
+                await this.drain(flushCallback);
+            }, this.params.batch_timeout);
+        }
+        //
+    }
+}

package/dist/batch/index.d.ts

@@ -0,0 +1,4 @@
+export * from './batch-writer.js';
+export * from './perform-in-chunks.js';
+export * from './perform-in-time-chunks.js';
+export * from './replicate.js';

package/dist/batch/index.js

@@ -0,0 +1,4 @@
+export * from './batch-writer.js';
+export * from './perform-in-chunks.js';
+export * from './perform-in-time-chunks.js';
+export * from './replicate.js';

package/dist/batch/perform-in-chunks.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Performs an operation on large array of data in chunks.
+ * @param data The data to be processed in chunks.
+ * @param operation The operation to be performed on each chunk of data.
+ * @param chunkSize The size of each chunk. Defaults to 5000.
+ */
+export declare function performInChunks<T>(data: T[], operation: (chunk: T[]) => Promise<void>, chunkSize?: number): Promise<void>;

package/dist/batch/perform-in-chunks.js

@@ -0,0 +1,23 @@
+/**
+ * Performs an operation on large array of data in chunks.
+ * @param data The data to be processed in chunks.
+ * @param operation The operation to be performed on each chunk of data.
+ * @param chunkSize The size of each chunk. Defaults to 5000.
+ */
+export async function performInChunks(data, operation, chunkSize = 5000) {
+    //
+    //
+    // Define an array to hold arrays of data (called chunks).
+    const allChunksOfData = [];
+    //
+    // Split the orignal data into chunks
+    for (let i = 0; i < data.length; i += chunkSize) {
+        allChunksOfData.push(data.slice(i, i + chunkSize));
+    }
+    //
+    // Process each chunk of data
+    for (const chunk of allChunksOfData) {
+        await operation(chunk);
+    }
+    //
+}

package/dist/batch/perform-in-time-chunks.d.ts

@@ -0,0 +1,14 @@
+import { type UnixTimestamp } from '@tmlmobilidade/go-types-shared';
+export interface PerformInTimeChunksItem {
+    end: UnixTimestamp;
+    index: number;
+    start: UnixTimestamp;
+    total: number;
+}
+export interface PerformInTimeChunksOptions {
+    endDate?: UnixTimestamp;
+    intervalHrs: number;
+    onChunk: (chunk: PerformInTimeChunksItem) => Promise<void>;
+    startDate: UnixTimestamp;
+}
+export declare function performInTimeChunks({ endDate, intervalHrs, onChunk, startDate }: PerformInTimeChunksOptions): Promise<void>;

package/dist/batch/perform-in-time-chunks.js

@@ -0,0 +1,33 @@
+/* * */
+import { Dates, splitTimeIntervals } from '@tmlmobilidade/go-utils-dates';
+export async function performInTimeChunks({ endDate, intervalHrs, onChunk, startDate }) {
+    //
+    // In order to sync both collections in a manageable way, due to the high volume of data,
+    // it is necessary to divide the process into smaller blocks. Instead of syncing all documents at once,
+    // divide the process by timestamps chunks and iterate over each one, getting all document IDs from both databases.
+    // Like this we can more easily compare the IDs in memory and sync only the missing documents.
+    // More recent data is more important than older data, so we start syncing the most recent data first.
+    // It makes sense to divide chunks by day, but this should be adjusted according to the volume of data in each chunk.
+    const endDateValue = endDate
+        ? Dates.fromUnixTimestamp(endDate).unix_timestamp
+        : Dates.now('utc').minus({ seconds: 30 }).unix_timestamp;
+    const startDateValue = Dates.fromUnixTimestamp(startDate).unix_timestamp;
+    const allTimestampChunks = splitTimeIntervals(startDateValue, endDateValue, intervalHrs);
+    //
+    // Iterate over each timestamp chunk and sync the documents.
+    // Timestamp chunks are sorted in descending order, so that more recent data is processed first.
+    // Timestamp chunks are in the format { start: day1, end: day2 }, so end is always greater than start.
+    // This might be confusing as the array of chunks itself is sorted in descending order, but the chunks individually are not.
+    for (const [chunkIndex, chunkData] of allTimestampChunks.entries()) {
+        //
+        const chunkStartDate = Dates
+            .fromUnixTimestamp(chunkData.start)
+            .setZone('Europe/Lisbon', 'offset_only');
+        const chunkEndDate = Dates
+            .fromUnixTimestamp(chunkData.end)
+            .setZone('Europe/Lisbon', 'offset_only');
+        await onChunk({ end: chunkEndDate.unix_timestamp, index: chunkIndex, start: chunkStartDate.unix_timestamp, total: allTimestampChunks.length });
+    }
+    //
+}
+;

package/dist/batch/replicate.d.ts

@@ -0,0 +1,80 @@
+interface ReplicateProps<SourceDocType> {
+    /**
+     * A function to count the total number of documents
+     * in the destination database. This must return a number.
+     * @returns A promise that resolves to a number.
+     */
+    countDestinationDbFn: () => Promise<number>;
+    /**
+     * A function to count the total number of documents
+     * in the source database. This must return a number.
+     * @returns A promise that resolves to a number.
+     */
+    countSourceDbFn: () => Promise<number>;
+    /**
+     * A function that deletes documents in the destination database,
+     * from an array of unique document IDs. This is used to remove any extra documents
+     * that are present in the destination database but not in the source database.
+     * This function should return a promise that resolves when the deletion is complete.
+     * @param uniqueIds An array of unique document IDs to be deleted from the destination database.
+     * @returns A promise that resolves when the deletion is complete.
+     */
+    deleteDestinationDbFn: (uniqueIds: string[]) => Promise<void>;
+    /**
+     * A function to get the distinct document IDs from the destination database.
+     * This must return an array of strings.
+     * @returns A promise that resolves to an array of strings.
+     */
+    distinctDestinationDbFn: () => Promise<string[]>;
+    /**
+     * A function to get the distinct document IDs from the source database.
+     * This must return an array of strings.
+     * @returns A promise that resolves to an array of strings.
+     */
+    distinctSourceDbFn: () => Promise<string[]>;
+    /**
+     * This is the function that should query the source database for the missing documents based on their IDs.
+     * It should return an async iterable (e.g., an async generator or a MongoDB `.stream()`) that yields
+     * the missing documents one by one.
+     * @param missingDocumentIds An array of document IDs that are missing in the destination database.
+     * @returns An async iterable that yields source documents one by one.
+     */
+    missingDocumentsSourceDbAsyncIterator: (missingDocumentIds: string[]) => AsyncIterable<SourceDocType>;
+    /**
+     * An optional callback function that will be executed after the replication process is complete.
+     * This can be used to perform any necessary cleanup tasks, such as flushing writers or logging.
+     */
+    onCompleteCallbackFn?: () => Promise<void>;
+    /**
+     * This function receives a document from the source database and should write it to the destination database.
+     * You can use any method you prefer to write the document to the destination database, such as a bulk insert or individual writes,
+     * and perform any necessary transformations on the document before writing it.
+     * @param sourceDocument The source document to be written to the destination database.
+     * @returns A promise that resolves when the document has been successfully written to the destination database.
+     */
+    writeSourceDocumentToDestinationDbFn: (sourceDocument: SourceDocType) => Promise<void>;
+}
+/**
+ * Copy documents from a source database to a destination database in multiple steps.
+ * The goal of this function is to ensure that the destination database has the same documents
+ * as the source database. The replication process is designed to be efficient and to minimize
+ * the amount of data transferred between the two databases by only syncing the missing documents.
+ *
+ * 1. First count the total number of documents in both databases to check if they match. This is a
+ * crucial optimization step, as it allows us to skip the replication process if both databases already
+ * have the same number of documents, which would indicate that they are already in sync.
+ * Though, it's important to note that having the same document count does not guarantee
+ * that the documents are identical, but it is a quick check to potentially avoid unnecessary replication.
+ *
+ * 2. If the counts do not match, get the distinct document IDs from both databases and compare them
+ * to find out which ones are missing in the destination database. This step is essential to identify
+ * the specific documents that need to be replicated, rather than syncing all documents again.
+ * Sync only the missing documents from the source database to the destination database,
+ *
+ * 3. Delete any extra documents in the destination database that are not present in the source database.
+ *
+ * 4. Run the onComplete callback function if provided. This allows for any additional actions to be performed after
+ * the replication process is complete, such as logging, flushing writers or any other necessary cleanup tasks.
+ */
+export declare function replicate<SourceDocType>({ countDestinationDbFn, countSourceDbFn, deleteDestinationDbFn, distinctDestinationDbFn, distinctSourceDbFn, missingDocumentsSourceDbAsyncIterator, onCompleteCallbackFn, writeSourceDocumentToDestinationDbFn }: ReplicateProps<SourceDocType>): Promise<void>;
+export {};

package/dist/batch/replicate.js

@@ -0,0 +1,83 @@
+/* * */
+import { Timer } from '@tmlmobilidade/timer';
+/**
+ * Copy documents from a source database to a destination database in multiple steps.
+ * The goal of this function is to ensure that the destination database has the same documents
+ * as the source database. The replication process is designed to be efficient and to minimize
+ * the amount of data transferred between the two databases by only syncing the missing documents.
+ *
+ * 1. First count the total number of documents in both databases to check if they match. This is a
+ * crucial optimization step, as it allows us to skip the replication process if both databases already
+ * have the same number of documents, which would indicate that they are already in sync.
+ * Though, it's important to note that having the same document count does not guarantee
+ * that the documents are identical, but it is a quick check to potentially avoid unnecessary replication.
+ *
+ * 2. If the counts do not match, get the distinct document IDs from both databases and compare them
+ * to find out which ones are missing in the destination database. This step is essential to identify
+ * the specific documents that need to be replicated, rather than syncing all documents again.
+ * Sync only the missing documents from the source database to the destination database,
+ *
+ * 3. Delete any extra documents in the destination database that are not present in the source database.
+ *
+ * 4. Run the onComplete callback function if provided. This allows for any additional actions to be performed after
+ * the replication process is complete, such as logging, flushing writers or any other necessary cleanup tasks.
+ */
+export async function replicate({ countDestinationDbFn, countSourceDbFn, deleteDestinationDbFn, distinctDestinationDbFn, distinctSourceDbFn, missingDocumentsSourceDbAsyncIterator, onCompleteCallbackFn, writeSourceDocumentToDestinationDbFn }) {
+    //
+    const globalTimer = new Timer();
+    //
+    // Run the count functions for both databases, if enabled, to get the total number
+    // of documents that match a given query. This is done to check if the document count
+    // is the same for both databases, which would indicate that all documents are already synced.
+    const countStepTimer = new Timer();
+    const sourceDbCount = await countSourceDbFn();
+    const destinationDbCount = await countDestinationDbFn();
+    if (sourceDbCount === destinationDbCount) {
+        console.info(`MATCH: Found the same number of documents in both databases: ${sourceDbCount} Source = ${destinationDbCount} Destination (${countStepTimer.get()})`);
+        return;
+    }
+    console.info(`MISMATCH: Document count was different for both databases: ${sourceDbCount} Source != ${destinationDbCount} Destination (${countStepTimer.get()})`);
+    //
+    // If the document count was different, then check which documents are missing.
+    // Instead of syncing all documents again, only the missing IDs are synced.
+    // This is done to get the distinct values from each database and comparing
+    // them to find the missing ones.
+    const distinctStepTimer = new Timer();
+    const sourceDbDocIds = await distinctSourceDbFn();
+    const sourceDbDocIdsUnique = new Set(sourceDbDocIds);
+    const destinationDbDocIds = await distinctDestinationDbFn();
+    const destinationDbDocIdsUnique = new Set(destinationDbDocIds);
+    const missingDocumentIds = sourceDbDocIds.filter((documentId) => !destinationDbDocIdsUnique.has(documentId));
+    const extraDocumentIds = destinationDbDocIds.filter(doc => !sourceDbDocIdsUnique.has(doc));
+    console.info(`Source Total: ${sourceDbCount} | Source Unique: ${sourceDbDocIdsUnique.size} | Source ▲: ${sourceDbCount - sourceDbDocIdsUnique.size} | Destination Total: ${destinationDbCount} | Destination Unique: ${destinationDbDocIdsUnique.size} | Destination ▲: ${destinationDbCount - destinationDbDocIdsUnique.size} | Destination Missing: ${missingDocumentIds.length} | Destination Extra: ${extraDocumentIds.length} (${distinctStepTimer.get()})`);
+    //
+    // If there are missing documents, then they are synced.
+    // We query the Source database for the missing documents
+    // and write them to the Destination database.
+    const missingStepTimer = new Timer();
+    if (missingDocumentIds.length > 0) {
+        console.info(`Syncing ${missingDocumentIds.length} missing documents to the Destination database...`);
+        for await (const sourceDbDocument of missingDocumentsSourceDbAsyncIterator(missingDocumentIds)) {
+            await writeSourceDocumentToDestinationDbFn(sourceDbDocument);
+        }
+        console.info(`Synced ${missingDocumentIds.length} missing documents to the Destination database. (${missingStepTimer.get()})`);
+    }
+    //
+    // Extra documents in the destination database should be removed,
+    // as they are not present in the source database.
+    const deleteStepTimer = new Timer();
+    if (extraDocumentIds.length > 0 && deleteDestinationDbFn) {
+        console.info(`Deleting ${extraDocumentIds.length} extra documents in the Destination database...`);
+        await deleteDestinationDbFn(extraDocumentIds);
+        console.info(`Deleted ${extraDocumentIds.length} extra documents in the Destination database. (${deleteStepTimer.get()})`);
+    }
+    //
+    // After syncing the missing documents,
+    // run the onComplete callback function if provided.
+    if (onCompleteCallbackFn) {
+        console.info(`Running onComplete callback function...`);
+        await onCompleteCallbackFn();
+    }
+    console.info(`Replication complete (${globalTimer.get()})`);
+    //
+}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export * from './batch/index.js';
+export * from './lifecycle/index.js';

package/dist/index.js

@@ -0,0 +1,2 @@
+export * from './batch/index.js';
+export * from './lifecycle/index.js';

package/dist/lifecycle/index.d.ts

@@ -0,0 +1,2 @@
+export * from './run-on-interval.js';
+export * from './run-with-concurrency.js';

package/dist/lifecycle/index.js

@@ -0,0 +1,2 @@
+export * from './run-on-interval.js';
+export * from './run-with-concurrency.js';

package/dist/lifecycle/run-on-interval.d.ts

@@ -0,0 +1,21 @@
+import { type TimeSlot } from '@tmlmobilidade/go-types-shared';
+interface RunOnIntervalOptions {
+    /**
+     * Interval in milliseconds between the end of one invocation
+     * and the start of the next. The first invocation happens immediately.
+     * @required
+     */
+    intervalMs: number | TimeSlot;
+    /**
+     * Whether to throw errors after logging them.
+     * If true, errors are logged and then re-thrown, stopping code execution.
+     * @default false
+     */
+    throwOnError?: boolean;
+}
+/**
+ * Runs an asynchronous function at regular intervals, ensuring that each invocation
+ * completes before the next one starts. Errors are logged and can optionally be re-thrown.
+ */
+export declare function runOnInterval(fn: () => Promise<void>, options: RunOnIntervalOptions): Promise<void>;
+export {};

package/dist/lifecycle/run-on-interval.js

@@ -0,0 +1,36 @@
+/* * */
+import { TimeSlotMap } from '@tmlmobilidade/go-types-shared';
+/**
+ * Runs an asynchronous function at regular intervals, ensuring that each invocation
+ * completes before the next one starts. Errors are logged and can optionally be re-thrown.
+ */
+export async function runOnInterval(fn, options) {
+    //
+    //
+    // Validate and determine the interval in milliseconds,
+    // or throw an error if the provided interval is invalid.
+    let intervalMs;
+    if (typeof options.intervalMs === 'number')
+        intervalMs = options.intervalMs;
+    else if (!TimeSlotMap[options.intervalMs])
+        throw new Error(`Invalid TimeSlot: ${options.intervalMs}`);
+    else
+        intervalMs = TimeSlotMap[options.intervalMs];
+    //
+    // Define the runner function that will execute
+    // the provided function and schedule the next execution.
+    const runner = async () => {
+        try {
+            await fn();
+        }
+        catch (error) {
+            console.error('Error in runOnInterval:', error);
+            if (options.throwOnError)
+                throw error;
+        }
+        finally {
+            setTimeout(runner, intervalMs);
+        }
+    };
+    await runner();
+}

package/dist/lifecycle/run-with-concurrency.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Run an async function over all items in the array, with a specific concurrency limit.
+ * Each result is a { status, value } or { status, reason } object, as from Promise.allSettled().
+ * The pool will never run more than `concurrency` functions at once.
+ * @param items - The array of items to process.
+ * @param concurrency - The maximum number of concurrent executions.
+ * @param fn - Async function to run for each item. Receives the item and its index.
+ * @returns A Promise resolving to an array of PromiseSettledResult objects, preserving input order.
+ */
+export declare function runWithConcurrency<T, R>(items: T[], concurrency: number, fn: (item: T, index: number) => Promise<R>): Promise<PromiseSettledResult<R>[]>;

package/dist/lifecycle/run-with-concurrency.js

@@ -0,0 +1,28 @@
+/**
+ * Run an async function over all items in the array, with a specific concurrency limit.
+ * Each result is a { status, value } or { status, reason } object, as from Promise.allSettled().
+ * The pool will never run more than `concurrency` functions at once.
+ * @param items - The array of items to process.
+ * @param concurrency - The maximum number of concurrent executions.
+ * @param fn - Async function to run for each item. Receives the item and its index.
+ * @returns A Promise resolving to an array of PromiseSettledResult objects, preserving input order.
+ */
+export async function runWithConcurrency(items, concurrency, fn) {
+    if (items.length === 0)
+        return [];
+    const results = new Array(items.length);
+    let next = 0;
+    async function worker() {
+        while (next < items.length) {
+            const index = next++;
+            try {
+                results[index] = { status: 'fulfilled', value: await fn(items[index], index) };
+            }
+            catch (reason) {
+                results[index] = { reason, status: 'rejected' };
+            }
+        }
+    }
+    await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()));
+    return results;
+}

package/package.json

@@ -0,0 +1,51 @@
+{
+	"name": "@tmlmobilidade/go-utils-exec",
+	"version": "20260828.1700.4",
+	"author": {
+		"email": "iso@tmlmobilidade.pt",
+		"name": "TML-ISO"
+	},
+	"license": "AGPL-3.0-or-later",
+	"homepage": "https://go.tmlmobilidade.pt",
+	"bugs": {
+		"url": "https://github.com/tmlmobilidade/go/issues"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/tmlmobilidade/go.git"
+	},
+	"keywords": [
+		"public transit",
+		"tml",
+		"transportes metropolitanos de lisboa",
+		"go"
+	],
+	"publishConfig": {
+		"access": "public"
+	},
+	"type": "module",
+	"files": [
+		"dist"
+	],
+	"main": "./dist/index.js",
+	"types": "./dist/index.d.ts",
+	"scripts": {
+		"build": "tsc && resolve-tspaths",
+		"lint": "eslint ./src/ && tsc --noEmit",
+		"lint:fix": "eslint ./src/ --fix",
+		"watch": "tsc-watch --onSuccess 'resolve-tspaths'"
+	},
+	"dependencies": {
+		"@tmlmobilidade/go-types-geo": "*",
+		"@tmlmobilidade/go-types-shared": "*",
+		"@tmlmobilidade/go-utils-dates": "*",
+		"@tmlmobilidade/timer": "*"
+	},
+	"devDependencies": {
+		"@tmlmobilidade/go-utils-tsconfig": "*",
+		"@types/node": "26.1.2",
+		"resolve-tspaths": "0.8.23",
+		"tsc-watch": "7.2.1",
+		"typescript": "6.0.3"
+	}
+}