@tmlmobilidade/utils 20260320.1741.37 → 20260320.1746.41

package/dist/batching/index.d.ts

@@ -1 +1,3 @@
 export * from './perform-in-chunks.js';
+export * from './perform-in-time-chunks.js';
+export * from './replicate.js';

package/dist/batching/index.js

@@ -1 +1,3 @@
 export * from './perform-in-chunks.js';
+export * from './perform-in-time-chunks.js';
+export * from './replicate.js';

package/dist/batching/perform-in-time-chunks.d.ts

@@ -0,0 +1,14 @@
+import { UnixTimestamp } from '@tmlmobilidade/types';
+import { DurationObjectUnits } from 'luxon';
+export interface PerformInTimeChunksItem {
+    end: UnixTimestamp;
+    index: number;
+    start: UnixTimestamp;
+    total: number;
+}
+export interface PerformInTimeChunksOptions {
+    onChunk: (chunk: PerformInTimeChunksItem) => Promise<void>;
+    splitBy: DurationObjectUnits;
+    startDate: UnixTimestamp;
+}
+export declare function performInTimeChunks({ onChunk, splitBy, startDate }: PerformInTimeChunksOptions): Promise<void>;

package/dist/batching/perform-in-time-chunks.js

@@ -0,0 +1,38 @@
+/* * */
+import { Dates } from '@tmlmobilidade/dates';
+import { Interval } from 'luxon';
+export async function performInTimeChunks({ onChunk, splitBy, startDate }) {
+    //
+    // In order to sync both collections in a manageable way, due to the high volume of data,
+    // it is necessary to divide the process into smaller blocks. Instead of syncing all documents at once,
+    // divide the process by timestamps chunks and iterate over each one, getting all document IDs from both databases.
+    // Like this we can more easily compare the IDs in memory and sync only the missing documents.
+    // More recent data is more important than older data, so we start syncing the most recent data first.
+    // It makes sense to divide chunks by day, but this should be adjusted according to the volume of data in each chunk.
+    const thirtySecondsAgo = Dates
+        .now('Europe/Lisbon')
+        .minus({ seconds: 30 });
+    const earliestDataNeeded = Dates.fromUnixTimestamp(startDate);
+    const allTimestampChunks = Interval
+        .fromISO(`${earliestDataNeeded.iso}/${thirtySecondsAgo.iso}`)
+        .splitBy(splitBy)
+        .map(interval => ({ end: interval.end.toMillis(), start: interval.start.toMillis() }))
+        .sort((a, b) => b.start - a.start);
+    //
+    // Iterate over each timestamp chunk and sync the documents.
+    // Timestamp chunks are sorted in descending order, so that more recent data is processed first.
+    // Timestamp chunks are in the format { start: day1, end: day2 }, so end is always greater than start.
+    // This might be confusing as the array of chunks itself is sorted in descending order, but the chunks individually are not.
+    for (const [chunkIndex, chunkData] of allTimestampChunks.entries()) {
+        //
+        const chunkStartDate = Dates
+            .fromUnixTimestamp(chunkData.start)
+            .setZone('Europe/Lisbon', 'offset_only');
+        const chunkEndDate = Dates
+            .fromUnixTimestamp(chunkData.end)
+            .setZone('Europe/Lisbon', 'offset_only');
+        await onChunk({ end: chunkEndDate.unix_timestamp, index: chunkIndex, start: chunkStartDate.unix_timestamp, total: allTimestampChunks.length });
+    }
+    //
+}
+;

package/dist/batching/replicate.d.ts

@@ -0,0 +1,80 @@
+interface ReplicateProps<SourceDocType> {
+    /**
+     * A function to count the total number of documents
+     * in the destination database. This must return a number.
+     * @returns A promise that resolves to a number.
+     */
+    countDestinationDbFn: () => Promise<number>;
+    /**
+     * A function to count the total number of documents
+     * in the source database. This must return a number.
+     * @returns A promise that resolves to a number.
+     */
+    countSourceDbFn: () => Promise<number>;
+    /**
+     * A function that deletes documents in the destination database,
+     * from an array of unique document IDs. This is used to remove any extra documents
+     * that are present in the destination database but not in the source database.
+     * This function should return a promise that resolves when the deletion is complete.
+     * @param uniqueIds An array of unique document IDs to be deleted from the destination database.
+     * @returns A promise that resolves when the deletion is complete.
+     */
+    deleteDestinationDbFn: (uniqueIds: string[]) => Promise<void>;
+    /**
+     * A function to get the distinct document IDs from the destination database.
+     * This must return an array of strings.
+     * @returns A promise that resolves to an array of strings.
+     */
+    distinctDestinationDbFn: () => Promise<string[]>;
+    /**
+     * A function to get the distinct document IDs from the source database.
+     * This must return an array of strings.
+     * @returns A promise that resolves to an array of strings.
+     */
+    distinctSourceDbFn: () => Promise<string[]>;
+    /**
+     * This is the function that should query the source database for the missing documents based on their IDs.
+     * It should return an async iterable (e.g., an async generator or a MongoDB `.stream()`) that yields
+     * the missing documents one by one.
+     * @param missingDocumentIds An array of document IDs that are missing in the destination database.
+     * @returns An async iterable that yields source documents one by one.
+     */
+    missingDocumentsSourceDbAsyncIterator: (missingDocumentIds: string[]) => AsyncIterable<SourceDocType>;
+    /**
+     * An optional callback function that will be executed after the replication process is complete.
+     * This can be used to perform any necessary cleanup tasks, such as flushing writers or logging.
+     */
+    onCompleteCallbackFn?: () => Promise<void>;
+    /**
+     * This function receives a document from the source database and should write it to the destination database.
+     * You can use any method you prefer to write the document to the destination database, such as a bulk insert or individual writes,
+     * and perform any necessary transformations on the document before writing it.
+     * @param sourceDocument The source document to be written to the destination database.
+     * @returns A promise that resolves when the document has been successfully written to the destination database.
+     */
+    writeSourceDocumentToDestinationDbFn: (sourceDocument: SourceDocType) => Promise<void>;
+}
+/**
+ * Copy documents from a source database to a destination database in multiple steps.
+ * The goal of this function is to ensure that the destination database has the same documents
+ * as the source database. The replication process is designed to be efficient and to minimize
+ * the amount of data transferred between the two databases by only syncing the missing documents.
+ *
+ * 1. First count the total number of documents in both databases to check if they match. This is a
+ * crucial optimization step, as it allows us to skip the replication process if both databases already
+ * have the same number of documents, which would indicate that they are already in sync.
+ * Though, it's important to note that having the same document count does not guarantee
+ * that the documents are identical, but it is a quick check to potentially avoid unnecessary replication.
+ *
+ * 2. If the counts do not match, get the distinct document IDs from both databases and compare them
+ * to find out which ones are missing in the destination database. This step is essential to identify
+ * the specific documents that need to be replicated, rather than syncing all documents again.
+ * Sync only the missing documents from the source database to the destination database,
+ *
+ * 3. Delete any extra documents in the destination database that are not present in the source database.
+ *
+ * 4. Run the onComplete callback function if provided. This allows for any additional actions to be performed after
+ * the replication process is complete, such as logging, flushing writers or any other necessary cleanup tasks.
+ */
+export declare function replicate<SourceDocType>({ countDestinationDbFn, countSourceDbFn, deleteDestinationDbFn, distinctDestinationDbFn, distinctSourceDbFn, missingDocumentsSourceDbAsyncIterator, onCompleteCallbackFn, writeSourceDocumentToDestinationDbFn }: ReplicateProps<SourceDocType>): Promise<void>;
+export {};

package/dist/batching/replicate.js

@@ -0,0 +1,82 @@
+/* * */
+import { Logger } from '@tmlmobilidade/logger';
+import { Timer } from '@tmlmobilidade/timer';
+/**
+ * Copy documents from a source database to a destination database in multiple steps.
+ * The goal of this function is to ensure that the destination database has the same documents
+ * as the source database. The replication process is designed to be efficient and to minimize
+ * the amount of data transferred between the two databases by only syncing the missing documents.
+ *
+ * 1. First count the total number of documents in both databases to check if they match. This is a
+ * crucial optimization step, as it allows us to skip the replication process if both databases already
+ * have the same number of documents, which would indicate that they are already in sync.
+ * Though, it's important to note that having the same document count does not guarantee
+ * that the documents are identical, but it is a quick check to potentially avoid unnecessary replication.
+ *
+ * 2. If the counts do not match, get the distinct document IDs from both databases and compare them
+ * to find out which ones are missing in the destination database. This step is essential to identify
+ * the specific documents that need to be replicated, rather than syncing all documents again.
+ * Sync only the missing documents from the source database to the destination database,
+ *
+ * 3. Delete any extra documents in the destination database that are not present in the source database.
+ *
+ * 4. Run the onComplete callback function if provided. This allows for any additional actions to be performed after
+ * the replication process is complete, such as logging, flushing writers or any other necessary cleanup tasks.
+ */
+export async function replicate({ countDestinationDbFn, countSourceDbFn, deleteDestinationDbFn, distinctDestinationDbFn, distinctSourceDbFn, missingDocumentsSourceDbAsyncIterator, onCompleteCallbackFn, writeSourceDocumentToDestinationDbFn }) {
+    //
+    const globalTimer = new Timer();
+    //
+    // Run the count functions for both databases, if enabled, to get the total number
+    // of documents that match a given query. This is done to check if the document count
+    // is the same for both databases, which would indicate that all documents are already synced.
+    const countStepTimer = new Timer();
+    const sourceDbCount = await countSourceDbFn();
+    const destinationDbCount = await countDestinationDbFn();
+    if (sourceDbCount === destinationDbCount) {
+        Logger.success(`MATCH: Found the same number of documents in both databases: ${sourceDbCount} Source = ${destinationDbCount} Destination (${countStepTimer.get()})`);
+        return;
+    }
+    Logger.info(`MISMATCH: Document count was different for both databases: ${sourceDbCount} Source != ${destinationDbCount} Destination (${countStepTimer.get()})`);
+    //
+    // If the document count was different, then check which documents are missing.
+    // Instead of syncing all documents again, only the missing IDs are synced.
+    // This is done to get the distinct values from each database and comparing
+    // them to find the missing ones.
+    const distinctStepTimer = new Timer();
+    const sourceDbDocIds = await distinctSourceDbFn();
+    const sourceDbDocIdsUnique = new Set(sourceDbDocIds);
+    const destinationDbDocIds = await distinctDestinationDbFn();
+    const destinationDbDocIdsUnique = new Set(destinationDbDocIds);
+    const missingDocumentIds = sourceDbDocIds.filter((documentId) => !destinationDbDocIdsUnique.has(documentId));
+    const extraDocumentIds = destinationDbDocIds.filter(doc => !sourceDbDocIdsUnique.has(doc));
+    Logger.info(`Source Total: ${sourceDbCount} | Source Unique: ${sourceDbDocIdsUnique.size} | Source ▲: ${sourceDbCount - sourceDbDocIdsUnique.size} | Destination Total: ${destinationDbCount} | Destination Unique: ${destinationDbDocIdsUnique.size} | Destination ▲: ${destinationDbCount - destinationDbDocIdsUnique.size} | Destination Missing: ${missingDocumentIds.length} | Destination Extra: ${extraDocumentIds.length} (${distinctStepTimer.get()})`);
+    if (missingDocumentIds.length === 0) {
+        Logger.success(`Chunk complete. All document IDs matched. (${distinctStepTimer.get()})`);
+        return;
+    }
+    //
+    // Extra documents in the destination database should be removed,
+    // as they are not present in the source database.
+    const deleteStepTimer = new Timer();
+    if (extraDocumentIds.length > 0 && deleteDestinationDbFn) {
+        await deleteDestinationDbFn(extraDocumentIds);
+        Logger.info(`Deleted ${extraDocumentIds.length} extra documents in the Destination database. (${deleteStepTimer.get()})`);
+    }
+    //
+    // If there are missing documents, then they are synced.
+    // We query the Source database for the missing documents
+    // and write them to the Destination database.
+    const missingStepTimer = new Timer();
+    Logger.info(`Found ${missingDocumentIds.length} missing documents in the Destination database. (${missingStepTimer.get()})`);
+    for await (const sourceDbDocument of missingDocumentsSourceDbAsyncIterator(missingDocumentIds)) {
+        await writeSourceDocumentToDestinationDbFn(sourceDbDocument);
+    }
+    //
+    // After syncing the missing documents,
+    // run the onComplete callback function if provided.
+    if (onCompleteCallbackFn)
+        await onCompleteCallbackFn();
+    Logger.success(`Replication complete (${globalTimer.get()})`);
+    //
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@tmlmobilidade/utils",
-	"version": "20260320.1741.37",
+	"version": "20260320.1746.41",
 	"author": {
 		"email": "iso@tmlmobilidade.pt",
 		"name": "TML-ISO"
@@ -37,7 +37,11 @@
 	},
 	"dependencies": {
 		"@tmlmobilidade/consts": "*",
+		"@tmlmobilidade/dates": "*",
+		"@tmlmobilidade/logger": "*",
+		"@tmlmobilidade/timer": "*",
 		"@tmlmobilidade/types": "*",
+		"luxon": "3.7.2",
 		"mergekit": "3.0.6"
 	},
 	"devDependencies": {