@tmlmobilidade/go-utils-exec 20260920.2153.33 → 20260922.1543.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* eslint-disable perfectionist/sort-classes */
|
|
2
2
|
/* * */
|
|
3
|
-
import { Timer } from '@tmlmobilidade/
|
|
3
|
+
import { Logger, Timer } from '@tmlmobilidade/go-utils-telemetry';
|
|
4
4
|
/* * */
|
|
5
5
|
export class BatchWriter {
|
|
6
6
|
//
|
|
@@ -76,13 +76,13 @@ export class BatchWriter {
|
|
|
76
76
|
try {
|
|
77
77
|
const flushTimer = new Timer();
|
|
78
78
|
await this.insertWithRetry(batch);
|
|
79
|
-
|
|
79
|
+
Logger.info({ message: `BATCHWRITER [${this.params.title}]: Flush | Length: ${batch.length} (session: ${this.sessionTimer.get()}) (flush: ${flushTimer.get()})` });
|
|
80
80
|
// Call the flush callback, if provided
|
|
81
81
|
if (callback)
|
|
82
82
|
await callback(batch);
|
|
83
83
|
}
|
|
84
84
|
catch (error) {
|
|
85
|
-
|
|
85
|
+
Logger.error({ message: `BATCHWRITER [${this.params.title}]: Error @ flush(): ${error.message}` });
|
|
86
86
|
throw error;
|
|
87
87
|
}
|
|
88
88
|
}
|
|
@@ -108,7 +108,7 @@ export class BatchWriter {
|
|
|
108
108
|
const parsedError = error;
|
|
109
109
|
const nextAttempt = attempt + 1;
|
|
110
110
|
const delayMs = retryBaseDelayMs * (2 ** attempt);
|
|
111
|
-
|
|
111
|
+
Logger.error({ message: `BATCHWRITER [${this.params.title}]: Transient insert error (${parsedError.code ?? 'unknown'}). Retrying ${nextAttempt}/${maxRetries} in ${delayMs}ms. ${parsedError.message}` });
|
|
112
112
|
await new Promise(resolve => setTimeout(resolve, delayMs));
|
|
113
113
|
}
|
|
114
114
|
}
|
|
@@ -133,7 +133,7 @@ export class BatchWriter {
|
|
|
133
133
|
// Check if the batch is full
|
|
134
134
|
const batchSize = this.params.batch_size ?? 10_000;
|
|
135
135
|
if (this.dataBucket.length >= batchSize) {
|
|
136
|
-
|
|
136
|
+
Logger.info({ message: `BATCHWRITER [${this.params.title}]: Batch full. Flushing data...` });
|
|
137
137
|
await this.flush(flushCallback);
|
|
138
138
|
}
|
|
139
139
|
//
|
|
@@ -161,7 +161,7 @@ export class BatchWriter {
|
|
|
161
161
|
// since the last write operation. Check if this functionality is enabled.
|
|
162
162
|
if (this.params.idle_timeout && this.params.idle_timeout > 0 && !this.idleTimeoutTimer) {
|
|
163
163
|
this.idleTimeoutTimer = setTimeout(async () => {
|
|
164
|
-
|
|
164
|
+
Logger.info({ message: `BATCHWRITER [${this.params.title}]: Idle timeout reached. Flushing data...` });
|
|
165
165
|
await this.drain(flushCallback);
|
|
166
166
|
}, this.params.idle_timeout);
|
|
167
167
|
}
|
|
@@ -170,7 +170,7 @@ export class BatchWriter {
|
|
|
170
170
|
// even if the batch is not full. Check if this functionality is enabled.
|
|
171
171
|
if (this.params.batch_timeout && this.params.batch_timeout > 0 && !this.batchTimeoutTimer) {
|
|
172
172
|
this.batchTimeoutTimer = setTimeout(async () => {
|
|
173
|
-
|
|
173
|
+
Logger.info({ message: `BATCHWRITER [${this.params.title}]: Batch timeout reached. Flushing data...` });
|
|
174
174
|
await this.drain(flushCallback);
|
|
175
175
|
}, this.params.batch_timeout);
|
|
176
176
|
}
|
package/dist/batch/replicate.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { Timer } from '@tmlmobilidade/
|
|
2
|
+
import { Logger, Timer } from '@tmlmobilidade/go-utils-telemetry';
|
|
3
3
|
/**
|
|
4
4
|
* Copy documents from a source database to a destination database in multiple steps.
|
|
5
5
|
* The goal of this function is to ensure that the destination database has the same documents
|
|
@@ -33,10 +33,10 @@ export async function replicate({ countDestinationDbFn, countSourceDbFn, deleteD
|
|
|
33
33
|
const sourceDbCount = await countSourceDbFn();
|
|
34
34
|
const destinationDbCount = await countDestinationDbFn();
|
|
35
35
|
if (sourceDbCount === destinationDbCount) {
|
|
36
|
-
|
|
36
|
+
Logger.info({ message: `MATCH: Found the same number of documents in both databases: ${sourceDbCount} Source = ${destinationDbCount} Destination (${countStepTimer.get()})` });
|
|
37
37
|
return;
|
|
38
38
|
}
|
|
39
|
-
|
|
39
|
+
Logger.info({ message: `MISMATCH: Document count was different for both databases: ${sourceDbCount} Source != ${destinationDbCount} Destination (${countStepTimer.get()})` });
|
|
40
40
|
//
|
|
41
41
|
// If the document count was different, then check which documents are missing.
|
|
42
42
|
// Instead of syncing all documents again, only the missing IDs are synced.
|
|
@@ -49,35 +49,35 @@ export async function replicate({ countDestinationDbFn, countSourceDbFn, deleteD
|
|
|
49
49
|
const destinationDbDocIdsUnique = new Set(destinationDbDocIds);
|
|
50
50
|
const missingDocumentIds = sourceDbDocIds.filter((documentId) => !destinationDbDocIdsUnique.has(documentId));
|
|
51
51
|
const extraDocumentIds = destinationDbDocIds.filter(doc => !sourceDbDocIdsUnique.has(doc));
|
|
52
|
-
|
|
52
|
+
Logger.info({ message: `Source Total: ${sourceDbCount} | Source Unique: ${sourceDbDocIdsUnique.size} | Source ▲: ${sourceDbCount - sourceDbDocIdsUnique.size} | Destination Total: ${destinationDbCount} | Destination Unique: ${destinationDbDocIdsUnique.size} | Destination ▲: ${destinationDbCount - destinationDbDocIdsUnique.size} | Destination Missing: ${missingDocumentIds.length} | Destination Extra: ${extraDocumentIds.length} (${distinctStepTimer.get()})` });
|
|
53
53
|
//
|
|
54
54
|
// If there are missing documents, then they are synced.
|
|
55
55
|
// We query the Source database for the missing documents
|
|
56
56
|
// and write them to the Destination database.
|
|
57
57
|
const missingStepTimer = new Timer();
|
|
58
58
|
if (missingDocumentIds.length > 0) {
|
|
59
|
-
|
|
59
|
+
Logger.info({ message: `Syncing ${missingDocumentIds.length} missing documents to the Destination database...` });
|
|
60
60
|
for await (const sourceDbDocument of missingDocumentsSourceDbAsyncIterator(missingDocumentIds)) {
|
|
61
61
|
await writeSourceDocumentToDestinationDbFn(sourceDbDocument);
|
|
62
62
|
}
|
|
63
|
-
|
|
63
|
+
Logger.info({ message: `Synced ${missingDocumentIds.length} missing documents to the Destination database. (${missingStepTimer.get()})` });
|
|
64
64
|
}
|
|
65
65
|
//
|
|
66
66
|
// Extra documents in the destination database should be removed,
|
|
67
67
|
// as they are not present in the source database.
|
|
68
68
|
const deleteStepTimer = new Timer();
|
|
69
69
|
if (extraDocumentIds.length > 0 && deleteDestinationDbFn) {
|
|
70
|
-
|
|
70
|
+
Logger.info({ message: `Deleting ${extraDocumentIds.length} extra documents in the Destination database...` });
|
|
71
71
|
await deleteDestinationDbFn(extraDocumentIds);
|
|
72
|
-
|
|
72
|
+
Logger.info({ message: `Deleted ${extraDocumentIds.length} extra documents in the Destination database. (${deleteStepTimer.get()})` });
|
|
73
73
|
}
|
|
74
74
|
//
|
|
75
75
|
// After syncing the missing documents,
|
|
76
76
|
// run the onComplete callback function if provided.
|
|
77
77
|
if (onCompleteCallbackFn) {
|
|
78
|
-
|
|
78
|
+
Logger.info({ message: 'Running onComplete callback function...' });
|
|
79
79
|
await onCompleteCallbackFn();
|
|
80
80
|
}
|
|
81
|
-
|
|
81
|
+
Logger.info({ message: `Replication complete (${globalTimer.get()})` });
|
|
82
82
|
//
|
|
83
83
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
2
|
import { TimeSlotMap } from '@tmlmobilidade/go-types-shared';
|
|
3
|
+
import { Logger } from '@tmlmobilidade/go-utils-telemetry';
|
|
3
4
|
/**
|
|
4
5
|
* Runs an asynchronous function at regular intervals, ensuring that each invocation
|
|
5
6
|
* completes before the next one starts. Errors are logged and can optionally be re-thrown.
|
|
@@ -24,7 +25,7 @@ export async function runOnInterval(fn, options) {
|
|
|
24
25
|
await fn();
|
|
25
26
|
}
|
|
26
27
|
catch (error) {
|
|
27
|
-
|
|
28
|
+
Logger.error({ error, message: 'Error in runOnInterval' });
|
|
28
29
|
if (options.throwOnError)
|
|
29
30
|
throw error;
|
|
30
31
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tmlmobilidade/go-utils-exec",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "20260922.1543.51",
|
|
4
4
|
"author": {
|
|
5
5
|
"email": "iso@tmlmobilidade.pt",
|
|
6
6
|
"name": "TML-ISO"
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
"dependencies": {
|
|
39
39
|
"@tmlmobilidade/go-types-shared": "*",
|
|
40
40
|
"@tmlmobilidade/go-utils-dates": "*",
|
|
41
|
-
"@tmlmobilidade/
|
|
41
|
+
"@tmlmobilidade/go-utils-telemetry": "*",
|
|
42
42
|
"csv-parse": "7.0.2",
|
|
43
43
|
"unzipper": "0.12.5"
|
|
44
44
|
},
|