@tmlmobilidade/export-data 20260527.939.40 → 20260527.1447.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -34,14 +34,8 @@ export async function exportValidationsAggregated({ context, groupFields, messag
|
|
|
34
34
|
is_passenger: true,
|
|
35
35
|
};
|
|
36
36
|
filterQuery.created_at = {
|
|
37
|
-
$gte: Dates
|
|
38
|
-
|
|
39
|
-
.set({ hour: 4, millisecond: 0, minute: 0, second: 0 })
|
|
40
|
-
.unix_timestamp,
|
|
41
|
-
$lt: Dates
|
|
42
|
-
.fromOperationalDate(context.dates.end, 'Europe/Lisbon')
|
|
43
|
-
.set({ hour: 4, millisecond: 0, minute: 0, second: 0 })
|
|
44
|
-
.unix_timestamp,
|
|
37
|
+
$gte: Dates.fromOperationalDate(context.dates.start, 'Europe/Lisbon').unix_timestamp,
|
|
38
|
+
$lt: Dates.fromOperationalDate(context.dates.end, 'Europe/Lisbon').unix_timestamp,
|
|
45
39
|
};
|
|
46
40
|
if (context.filters.agency_ids.length) {
|
|
47
41
|
filterQuery.agency_id = { $in: context.filters.agency_ids };
|
|
@@ -82,6 +76,7 @@ export async function exportValidationsAggregated({ context, groupFields, messag
|
|
|
82
76
|
// Prepare the result key
|
|
83
77
|
const operationalDate = Dates
|
|
84
78
|
.fromUnixTimestamp(document.created_at)
|
|
79
|
+
.setZone('Europe/Lisbon', 'offset_only')
|
|
85
80
|
.operational_date;
|
|
86
81
|
// Build the key from date + all selected group fields
|
|
87
82
|
const keyParts = [operationalDate, ...groupFields.map(field => String(document[field]))];
|
|
@@ -109,11 +104,11 @@ export async function exportValidationsAggregated({ context, groupFields, messag
|
|
|
109
104
|
}
|
|
110
105
|
message(`A escrever os resultados no ficheiro CSV...`);
|
|
111
106
|
// Write results in batches to avoid stack overflow with large datasets
|
|
112
|
-
const
|
|
107
|
+
const writeBatchSize = 10000;
|
|
113
108
|
const resultKeys = Object.keys(result);
|
|
114
109
|
const totalResults = resultKeys.length;
|
|
115
|
-
for (let i = 0; i < totalResults; i +=
|
|
116
|
-
const batch = resultKeys.slice(i, i +
|
|
110
|
+
for (let i = 0; i < totalResults; i += writeBatchSize) {
|
|
111
|
+
const batch = resultKeys.slice(i, i + writeBatchSize).map(key => result[key]);
|
|
117
112
|
await csvWriter.write(batch);
|
|
118
113
|
if (i % 50000 === 0 && i > 0) {
|
|
119
114
|
message(`Escritos ${i} de ${totalResults} resultados...`);
|
|
@@ -13,14 +13,8 @@ export async function exportValidationsRaw({ context, message }) {
|
|
|
13
13
|
// Prepare the filter params
|
|
14
14
|
const filterQuery = {};
|
|
15
15
|
filterQuery.created_at = {
|
|
16
|
-
$gte: Dates
|
|
17
|
-
|
|
18
|
-
.set({ hour: 4, millisecond: 0, minute: 0, second: 0 })
|
|
19
|
-
.unix_timestamp,
|
|
20
|
-
$lt: Dates
|
|
21
|
-
.fromOperationalDate(context.dates.end, 'Europe/Lisbon')
|
|
22
|
-
.set({ hour: 4, millisecond: 0, minute: 0, second: 0 })
|
|
23
|
-
.unix_timestamp,
|
|
16
|
+
$gte: Dates.fromOperationalDate(context.dates.start, 'Europe/Lisbon').unix_timestamp,
|
|
17
|
+
$lt: Dates.fromOperationalDate(context.dates.end, 'Europe/Lisbon').unix_timestamp,
|
|
24
18
|
};
|
|
25
19
|
if (context.filters.agency_ids.length) {
|
|
26
20
|
filterQuery.agency_id = { $in: context.filters.agency_ids };
|