@tmlmobilidade/utils 20260617.1318.33 → 20260617.1731.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
/* eslint-disable perfectionist/sort-classes */
|
|
2
2
|
/* * */
|
|
3
|
-
import { Logger } from '@tmlmobilidade/logger';
|
|
4
3
|
import { Timer } from '@tmlmobilidade/timer';
|
|
5
4
|
/* * */
|
|
6
5
|
export class BatchWriter {
|
|
@@ -58,7 +57,7 @@ export class BatchWriter {
|
|
|
58
57
|
if (!this.params.insertFn)
|
|
59
58
|
throw new Error('BATCHWRITER: No insert function provided in params');
|
|
60
59
|
await this.insertWithRetry(this.dataBucketFlushOps);
|
|
61
|
-
|
|
60
|
+
console.info(`BATCHWRITER [${this.params.title}]: Flush | Length: ${this.dataBucketFlushOps.length} (session: ${sessionTimerResult}) (flush: ${flushTimer.get()})`);
|
|
62
61
|
// Call the flush callback, if provided
|
|
63
62
|
if (callback)
|
|
64
63
|
await callback(this.dataBucketFlushOps);
|
|
@@ -66,13 +65,13 @@ export class BatchWriter {
|
|
|
66
65
|
this.dataBucketFlushOps = [];
|
|
67
66
|
}
|
|
68
67
|
catch (error) {
|
|
69
|
-
|
|
68
|
+
console.error(`BATCHWRITER [${this.params.title}]: Error @ flush().insert(): ${error.message}`);
|
|
70
69
|
throw error; // Re-throw to allow retry logic at higher level
|
|
71
70
|
}
|
|
72
71
|
//
|
|
73
72
|
}
|
|
74
73
|
catch (error) {
|
|
75
|
-
|
|
74
|
+
console.error(`BATCHWRITER [${this.params.title}]: Error @ flush(): ${error.message}`);
|
|
76
75
|
throw error; // Re-throw to allow retry logic at higher level
|
|
77
76
|
}
|
|
78
77
|
}
|
|
@@ -96,7 +95,7 @@ export class BatchWriter {
|
|
|
96
95
|
const parsedError = error;
|
|
97
96
|
const nextAttempt = attempt + 1;
|
|
98
97
|
const delayMs = retryBaseDelayMs * (2 ** attempt);
|
|
99
|
-
|
|
98
|
+
console.error(`BATCHWRITER [${this.params.title}]: Transient insert error (${parsedError.code ?? 'unknown'}). Retrying ${nextAttempt}/${maxRetries} in ${delayMs}ms. ${parsedError.message}`);
|
|
100
99
|
await new Promise(resolve => setTimeout(resolve, delayMs));
|
|
101
100
|
}
|
|
102
101
|
}
|
|
@@ -121,7 +120,7 @@ export class BatchWriter {
|
|
|
121
120
|
// Check if the batch is full
|
|
122
121
|
const batchSize = this.params.batch_size ?? 10_000;
|
|
123
122
|
if (this.dataBucketAlwaysAvailable.length >= batchSize) {
|
|
124
|
-
|
|
123
|
+
console.info(`BATCHWRITER [${this.params.title}]: Batch full. Flushing data...`);
|
|
125
124
|
await this.flush(flushCallback);
|
|
126
125
|
}
|
|
127
126
|
//
|
|
@@ -148,7 +147,7 @@ export class BatchWriter {
|
|
|
148
147
|
// since the last write operation. Check if this functionality is enabled.
|
|
149
148
|
if (this.params.idle_timeout && this.params.idle_timeout > 0 && !this.idleTimeoutTimer) {
|
|
150
149
|
this.idleTimeoutTimer = setTimeout(async () => {
|
|
151
|
-
|
|
150
|
+
console.info(`BATCHWRITER [${this.params.title}]: Idle timeout reached. Flushing data...`);
|
|
152
151
|
await this.flush(flushCallback);
|
|
153
152
|
}, this.params.idle_timeout);
|
|
154
153
|
}
|
|
@@ -157,7 +156,7 @@ export class BatchWriter {
|
|
|
157
156
|
// even if the batch is not full. Check if this functionality is enabled.
|
|
158
157
|
if (this.params.batch_timeout && this.params.batch_timeout > 0 && !this.batchTimeoutTimer) {
|
|
159
158
|
this.batchTimeoutTimer = setTimeout(async () => {
|
|
160
|
-
|
|
159
|
+
console.info(`BATCHWRITER [${this.params.title}]: Batch timeout reached. Flushing data...`);
|
|
161
160
|
await this.flush(flushCallback);
|
|
162
161
|
}, this.params.batch_timeout);
|
|
163
162
|
}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { Logger } from '@tmlmobilidade/logger';
|
|
3
2
|
import { Timer } from '@tmlmobilidade/timer';
|
|
4
3
|
/**
|
|
5
4
|
* Copy documents from a source database to a destination database in multiple steps.
|
|
@@ -34,10 +33,10 @@ export async function replicate({ countDestinationDbFn, countSourceDbFn, deleteD
|
|
|
34
33
|
const sourceDbCount = await countSourceDbFn();
|
|
35
34
|
const destinationDbCount = await countDestinationDbFn();
|
|
36
35
|
if (sourceDbCount === destinationDbCount) {
|
|
37
|
-
|
|
36
|
+
console.info(`MATCH: Found the same number of documents in both databases: ${sourceDbCount} Source = ${destinationDbCount} Destination (${countStepTimer.get()})`);
|
|
38
37
|
return;
|
|
39
38
|
}
|
|
40
|
-
|
|
39
|
+
console.info(`MISMATCH: Document count was different for both databases: ${sourceDbCount} Source != ${destinationDbCount} Destination (${countStepTimer.get()})`);
|
|
41
40
|
//
|
|
42
41
|
// If the document count was different, then check which documents are missing.
|
|
43
42
|
// Instead of syncing all documents again, only the missing IDs are synced.
|
|
@@ -50,7 +49,7 @@ export async function replicate({ countDestinationDbFn, countSourceDbFn, deleteD
|
|
|
50
49
|
const destinationDbDocIdsUnique = new Set(destinationDbDocIds);
|
|
51
50
|
const missingDocumentIds = sourceDbDocIds.filter((documentId) => !destinationDbDocIdsUnique.has(documentId));
|
|
52
51
|
const extraDocumentIds = destinationDbDocIds.filter(doc => !sourceDbDocIdsUnique.has(doc));
|
|
53
|
-
|
|
52
|
+
console.info(`Source Total: ${sourceDbCount} | Source Unique: ${sourceDbDocIdsUnique.size} | Source ▲: ${sourceDbCount - sourceDbDocIdsUnique.size} | Destination Total: ${destinationDbCount} | Destination Unique: ${destinationDbDocIdsUnique.size} | Destination ▲: ${destinationDbCount - destinationDbDocIdsUnique.size} | Destination Missing: ${missingDocumentIds.length} | Destination Extra: ${extraDocumentIds.length} (${distinctStepTimer.get()})`);
|
|
54
53
|
//
|
|
55
54
|
// If there are missing documents, then they are synced.
|
|
56
55
|
// We query the Source database for the missing documents
|
|
@@ -60,7 +59,7 @@ export async function replicate({ countDestinationDbFn, countSourceDbFn, deleteD
|
|
|
60
59
|
for await (const sourceDbDocument of missingDocumentsSourceDbAsyncIterator(missingDocumentIds)) {
|
|
61
60
|
await writeSourceDocumentToDestinationDbFn(sourceDbDocument);
|
|
62
61
|
}
|
|
63
|
-
|
|
62
|
+
console.info(`Synced ${missingDocumentIds.length} missing documents to the Destination database. (${missingStepTimer.get()})`);
|
|
64
63
|
}
|
|
65
64
|
//
|
|
66
65
|
// Extra documents in the destination database should be removed,
|
|
@@ -68,13 +67,13 @@ export async function replicate({ countDestinationDbFn, countSourceDbFn, deleteD
|
|
|
68
67
|
const deleteStepTimer = new Timer();
|
|
69
68
|
if (extraDocumentIds.length > 0 && deleteDestinationDbFn) {
|
|
70
69
|
await deleteDestinationDbFn(extraDocumentIds);
|
|
71
|
-
|
|
70
|
+
console.info(`Deleted ${extraDocumentIds.length} extra documents in the Destination database. (${deleteStepTimer.get()})`);
|
|
72
71
|
}
|
|
73
72
|
//
|
|
74
73
|
// After syncing the missing documents,
|
|
75
74
|
// run the onComplete callback function if provided.
|
|
76
75
|
if (onCompleteCallbackFn)
|
|
77
76
|
await onCompleteCallbackFn();
|
|
78
|
-
|
|
77
|
+
console.info(`Replication complete (${globalTimer.get()})`);
|
|
79
78
|
//
|
|
80
79
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tmlmobilidade/utils",
|
|
3
|
-
"version": "20260617.
|
|
3
|
+
"version": "20260617.1731.32",
|
|
4
4
|
"author": {
|
|
5
5
|
"email": "iso@tmlmobilidade.pt",
|
|
6
6
|
"name": "TML-ISO"
|
|
@@ -38,7 +38,6 @@
|
|
|
38
38
|
"dependencies": {
|
|
39
39
|
"@tmlmobilidade/consts": "*",
|
|
40
40
|
"@tmlmobilidade/dates": "*",
|
|
41
|
-
"@tmlmobilidade/logger": "*",
|
|
42
41
|
"@tmlmobilidade/timer": "*",
|
|
43
42
|
"@tmlmobilidade/types": "*",
|
|
44
43
|
"luxon": "3.7.2",
|