@tmlmobilidade/export-data 20260121.2332.4 → 20260123.1211.24

package/dist/index.js

@@ -9,11 +9,8 @@ import { promptFilterByStopIds } from './prompts/filter-stop-ids.js';
 import { promptFilterTypes } from './prompts/filter-types.js';
 import { promptFilterByVehicleIds } from './prompts/filter-vehicle-ids.js';
 import { promptHashedShapeIds } from './prompts/hashedshape-ids.js';
-import { exportValidationsByLine } from './tasks/apex-validations/validations-by-line.js';
-import { exportValidationsByPattern } from './tasks/apex-validations/validations-by-pattern.js';
-import { exportValidationsByStopByPattern } from './tasks/apex-validations/validations-by-stop-by-pattern.js';
-import { exportValidationsByStopByTrip } from './tasks/apex-validations/validations-by-stop-by-trip.js';
-import { exportValidationsByStop } from './tasks/apex-validations/validations-by-stop.js';
+import { promptValidationGroupFields } from './prompts/validation-group-fields.js';
+import { exportValidationsAggregated } from './tasks/apex-validations/validations-aggregated.js';
 import { exportValidationsRaw } from './tasks/apex-validations/validations-raw.js';
 import { exportHashedShapesGeoJSON } from './tasks/hashed-shapes/hashed-shapes-geojson.js';
 import { exportRidesRaw } from './tasks/rides/rides-raw.js';
@@ -53,6 +50,12 @@ import { ASCII_CM_SHORT } from '@tmlmobilidade/consts';
         hashedShapeIds = await promptHashedShapeIds();
     }
     //
+    // For validations-aggregated export, prompt for group fields
+    let validationGroupFields = [];
+    if (exportTypes.includes('validations-aggregated')) {
+        validationGroupFields = await promptValidationGroupFields();
+    }
+    //
     // Skip filters and dates if all selected export types don't require them
     if (!shouldSkipFilters) {
         const filterTypes = await promptFilterTypes();
@@ -79,29 +82,9 @@ import { ASCII_CM_SHORT } from '@tmlmobilidade/consts';
             title: exportTypeLabels['validations-raw'],
         },
         {
-            enabled: exportTypes.includes('validations-by-stop-by-trip'),
-            task: async (message) => await exportValidationsByStopByTrip({ context, message }),
-            title: exportTypeLabels['validations-by-stop-by-trip'],
-        },
-        {
-            enabled: exportTypes.includes('validations-by-stop-by-pattern'),
-            task: async (message) => await exportValidationsByStopByPattern({ context, message }),
-            title: exportTypeLabels['validations-by-stop-by-pattern'],
-        },
-        {
-            enabled: exportTypes.includes('validations-by-stop'),
-            task: async (message) => await exportValidationsByStop({ context, message }),
-            title: exportTypeLabels['validations-by-stop'],
-        },
-        {
-            enabled: exportTypes.includes('validations-by-pattern'),
-            task: async (message) => await exportValidationsByPattern({ context, message }),
-            title: exportTypeLabels['validations-by-pattern'],
-        },
-        {
-            enabled: exportTypes.includes('validations-by-line'),
-            task: async (message) => await exportValidationsByLine({ context, message }),
-            title: exportTypeLabels['validations-by-line'],
+            enabled: exportTypes.includes('validations-aggregated'),
+            task: async (message) => await exportValidationsAggregated({ context, groupFields: validationGroupFields, message }),
+            title: exportTypeLabels['validations-aggregated'],
         },
         {
             enabled: exportTypes.includes('rides-raw'),

package/dist/prompts/export-types.js

@@ -9,11 +9,7 @@ export async function promptExportTypes() {
         options: {
             '1. Validações APEX': [
                 { label: exportTypeLabels['validations-raw'], value: 'validations-raw' },
-                { label: exportTypeLabels['validations-by-stop-by-trip'], value: 'validations-by-stop-by-trip' },
-                { label: exportTypeLabels['validations-by-stop-by-pattern'], value: 'validations-by-stop-by-pattern' },
-                { label: exportTypeLabels['validations-by-stop'], value: 'validations-by-stop' },
-                { label: exportTypeLabels['validations-by-pattern'], value: 'validations-by-pattern' },
-                { label: exportTypeLabels['validations-by-line'], value: 'validations-by-line' },
+                { label: exportTypeLabels['validations-aggregated'], value: 'validations-aggregated' },
             ],
             '2. Rides': [
                 { label: exportTypeLabels['rides-raw'], value: 'rides-raw' },

package/dist/prompts/validation-group-fields.d.ts

@@ -0,0 +1,2 @@
+import { type ValidationGroupField } from '../tasks/apex-validations/validations-aggregated.js';
+export declare function promptValidationGroupFields(): Promise<ValidationGroupField[]>;

package/dist/prompts/validation-group-fields.js

@@ -0,0 +1,22 @@
+/* * */
+import { validationGroupFieldLabels, validationGroupFields } from '../tasks/apex-validations/validations-aggregated.js';
+import { cancel, isCancel, multiselect } from '@clack/prompts';
+/* * */
+export async function promptValidationGroupFields() {
+    //
+    const values = await multiselect({
+        message: 'Escolhe os campos para agrupar as validações (a data é sempre incluída):',
+        options: validationGroupFields.map(field => ({
+            label: validationGroupFieldLabels[field],
+            value: field,
+        })),
+        required: true,
+    });
+    if (isCancel(values)) {
+        cancel('Operação cancelada pelo utilizador.');
+        process.exit(0);
+    }
+    if (!values)
+        return [];
+    return values;
+}

package/dist/tasks/apex-validations/validations-aggregated.d.ts

@@ -0,0 +1,18 @@
+import { type TaskProps } from '../../types.js';
+/**
+ * Available fields for grouping validations.
+ * 'date' is always included by default.
+ */
+export declare const validationGroupFields: readonly ["line_id", "pattern_id", "product_id", "trip_id", "stop_id", "agency_id", "vehicle_id"];
+export type ValidationGroupField = typeof validationGroupFields[number];
+export declare const validationGroupFieldLabels: Record<ValidationGroupField, string>;
+interface ValidationAggregatedTaskProps extends TaskProps {
+    groupFields: ValidationGroupField[];
+}
+/**
+ * Export Validations aggregated by user-selected fields.
+ * Always groups by date, plus any additional fields selected by the user.
+ * The 'validations' count is always included.
+ */
+export declare function exportValidationsAggregated({ context, groupFields, message }: ValidationAggregatedTaskProps): Promise<void>;
+export {};

package/dist/tasks/apex-validations/{validations-by-stop-by-pattern.js → validations-aggregated.js}

@@ -4,13 +4,30 @@ import { simplifiedApexValidations } from '@tmlmobilidade/interfaces';
 import { CsvWriter } from '@tmlmobilidade/writers';
 import fs from 'node:fs';
 /* * */
-const TASK_ID = 'validations-by-stop-by-pattern';
 /**
- * Export Validations By Stop By Pattern data applying the given filters.
+ * Available fields for grouping validations.
+ * 'date' is always included by default.
  */
-export async function exportValidationsByStopByPattern({ context, message }) {
+export const validationGroupFields = ['line_id', 'pattern_id', 'product_id', 'trip_id', 'stop_id', 'agency_id', 'vehicle_id'];
+export const validationGroupFieldLabels = {
+    agency_id: 'Operador (agency_id)',
+    line_id: 'Linha (line_id)',
+    pattern_id: 'Pattern (pattern_id)',
+    product_id: 'Tipo de passe (product_id)',
+    stop_id: 'Paragem (stop_id)',
+    trip_id: 'Viagem (trip_id)',
+    vehicle_id: 'Veículo (vehicle_id)',
+};
+/* * */
+/**
+ * Export Validations aggregated by user-selected fields.
+ * Always groups by date, plus any additional fields selected by the user.
+ * The 'validations' count is always included.
+ */
+export async function exportValidationsAggregated({ context, groupFields, message }) {
     //
-    message('A iniciar a exportação de Validações APEX por Paragem e por Pattern...');
+    const fieldsList = ['date', ...groupFields].join(', ');
+    message(`A iniciar a exportação de Validações APEX agrupadas por: ${fieldsList}...`);
     //
     // Prepare the filter params
     const filterQuery = {
@@ -51,7 +68,9 @@ export async function exportValidationsByStopByPattern({ context, message }) {
     message(`A preparar a pasta para guardar os resultados...`);
     if (!fs.existsSync(context.output))
         fs.mkdirSync(context.output, { recursive: true });
-    const csvWriter = new CsvWriter('output', `${context.output}/${TASK_ID}-${context.dates.start}-${context.dates.end}.csv`, { batch_size: 100000, logs: false });
+    // Build filename based on selected fields
+    const fieldsSlug = groupFields.length > 0 ? `by-${groupFields.join('-')}` : 'by-date';
+    const csvWriter = new CsvWriter('output', `${context.output}/validations-${fieldsSlug}-${context.dates.start}-${context.dates.end}.csv`, { batch_size: 100000, logs: false });
     //
     // Export the data
     let counter = 0;
@@ -64,25 +83,42 @@ export async function exportValidationsByStopByPattern({ context, message }) {
         const operationalDate = Dates
             .fromUnixTimestamp(document.created_at)
             .operational_date;
-        const resultKey = `${operationalDate}:${document.stop_id}:${document.pattern_id}`;
+        // Build the key from date + all selected group fields
+        const keyParts = [operationalDate, ...groupFields.map(field => String(document[field]))];
+        const resultKey = keyParts.join(':');
         //
         // Update the result with the current document
         if (!result[resultKey]) {
-            result[resultKey] = {
+            // Initialize the result object with date first, then selected fields in order
+            const resultObj = {
                 date: operationalDate,
-                pattern_id: document.pattern_id,
-                stop_id: document.stop_id,
-                validations: 0,
             };
+            // Add each selected group field
+            for (const field of groupFields) {
+                resultObj[field] = document[field];
+            }
+            // Always add validations count at the end
+            resultObj.validations = 0;
+            result[resultKey] = resultObj;
         }
-        result[resultKey].validations += 1;
+        result[resultKey].validations = result[resultKey].validations + 1;
         if (counter % 1000 === 0)
             message(`Processados ${counter} documentos até agora...`);
         counter++;
         //
     }
     message(`A escrever os resultados no ficheiro CSV...`);
-    await csvWriter.write(Object.values(result));
+    // Write results in batches to avoid stack overflow with large datasets
+    const WRITE_BATCH_SIZE = 10000;
+    const resultKeys = Object.keys(result);
+    const totalResults = resultKeys.length;
+    for (let i = 0; i < totalResults; i += WRITE_BATCH_SIZE) {
+        const batch = resultKeys.slice(i, i + WRITE_BATCH_SIZE).map(key => result[key]);
+        await csvWriter.write(batch);
+        if (i % 50000 === 0 && i > 0) {
+            message(`Escritos ${i} de ${totalResults} resultados...`);
+        }
+    }
     await csvWriter.flush();
     //
 }

package/dist/types.d.ts

@@ -3,11 +3,7 @@ export declare const exportTypeLabels: {
     readonly 'hashed-shapes-geojson': "4.0. HashedShapes para GeoJSON";
     readonly 'rides-raw': "2.0. Rides em bruto (SLAs)";
     readonly 'sams-raw': "5.0. SAMs em bruto (Sequencialidade)";
-    readonly 'validations-by-line': "1.5. Validações por Line ID";
-    readonly 'validations-by-pattern': "1.4. Validações por Pattern ID";
-    readonly 'validations-by-stop': "1.3. Validações por Stop ID";
-    readonly 'validations-by-stop-by-pattern': "1.2. Validações por Stop ID, por Pattern ID";
-    readonly 'validations-by-stop-by-trip': "1.1. Validações por Stop ID, por Trip ID";
+    readonly 'validations-aggregated': "1.1. Validações agregadas (escolher campos)";
     readonly 'validations-raw': "1.0. Validações em bruto";
     readonly 'vehicle-events-raw': "3.0. Vehicle Events em bruto";
 };

package/dist/types.js

@@ -4,11 +4,7 @@ export const exportTypeLabels = {
     'hashed-shapes-geojson': '4.0. HashedShapes para GeoJSON',
     'rides-raw': '2.0. Rides em bruto (SLAs)',
     'sams-raw': '5.0. SAMs em bruto (Sequencialidade)',
-    'validations-by-line': '1.5. Validações por Line ID',
-    'validations-by-pattern': '1.4. Validações por Pattern ID',
-    'validations-by-stop': '1.3. Validações por Stop ID',
-    'validations-by-stop-by-pattern': '1.2. Validações por Stop ID, por Pattern ID',
-    'validations-by-stop-by-trip': '1.1. Validações por Stop ID, por Trip ID',
+    'validations-aggregated': '1.1. Validações agregadas (escolher campos)',
     'validations-raw': '1.0. Validações em bruto',
     'vehicle-events-raw': '3.0. Vehicle Events em bruto',
 };

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@tmlmobilidade/export-data",
 	"description": "CLI tool to export data from GO.",
-	"version": "20260121.2332.4",
+	"version": "20260123.1211.24",
 	"author": {
 		"email": "iso@tmlmobilidade.pt",
 		"name": "TML-ISO"

package/dist/tasks/apex-validations/validations-by-line.d.ts

@@ -1,2 +0,0 @@
-import { type TaskProps } from '../../types.js';
-export declare function exportValidationsByLine({ context, message }: TaskProps): Promise<void>;

package/dist/tasks/apex-validations/validations-by-line.js

@@ -1,85 +0,0 @@
-/* * */
-import { Dates } from '@tmlmobilidade/dates';
-import { simplifiedApexValidations } from '@tmlmobilidade/interfaces';
-import { CsvWriter } from '@tmlmobilidade/writers';
-import fs from 'node:fs';
-/* * */
-const TASK_ID = 'validations-by-line';
-/* * */
-export async function exportValidationsByLine({ context, message }) {
-    //
-    message('A iniciar a exportação de Validações APEX por Linha...');
-    //
-    // Prepare the filter params
-    const filterQuery = {
-        is_passenger: true,
-    };
-    filterQuery.created_at = {
-        $gte: Dates
-            .fromOperationalDate(context.dates.start, 'Europe/Lisbon')
-            .set({ hour: 4, millisecond: 0, minute: 0, second: 0 })
-            .unix_timestamp,
-        $lt: Dates
-            .fromOperationalDate(context.dates.end, 'Europe/Lisbon')
-            .set({ hour: 4, millisecond: 0, minute: 0, second: 0 })
-            .unix_timestamp,
-    };
-    if (context.filters.agency_ids.length) {
-        filterQuery.agency_id = { $in: context.filters.agency_ids };
-    }
-    if (context.filters.line_ids.length) {
-        filterQuery.line_id = { $in: context.filters.line_ids };
-    }
-    if (context.filters.pattern_ids.length) {
-        filterQuery.pattern_id = { $in: context.filters.pattern_ids };
-    }
-    if (context.filters.stop_ids.length) {
-        filterQuery.stop_id = { $in: context.filters.stop_ids };
-    }
-    if (context.filters.vehicle_ids.length) {
-        filterQuery.vehicle_id = { $in: context.filters.vehicle_ids };
-    }
-    //
-    // Setup a database stream to export data
-    message(`A iniciar ligação à base de dados...`);
-    const simplifiedApexValidationsCollection = await simplifiedApexValidations.getCollection();
-    const stream = simplifiedApexValidationsCollection.find(filterQuery).stream();
-    //
-    // Prepare the output directory and CSV writer
-    message(`A preparar a pasta para guardar os resultados...`);
-    if (!fs.existsSync(context.output))
-        fs.mkdirSync(context.output, { recursive: true });
-    const csvWriter = new CsvWriter('output', `${context.output}/${TASK_ID}-${context.dates.start}-${context.dates.end}.csv`, { batch_size: 100000, logs: false });
-    //
-    // Export the data
-    let counter = 0;
-    message(`A aguardar o resultado da pesquisa...`);
-    const result = {};
-    for await (const doc of stream) {
-        const document = doc;
-        //
-        // Prepare the result key
-        const operationalDate = Dates
-            .fromUnixTimestamp(document.created_at)
-            .operational_date;
-        const resultKey = `${operationalDate}:${document.line_id}`;
-        //
-        // Update the result with the current document
-        if (!result[resultKey]) {
-            result[resultKey] = {
-                date: operationalDate,
-                line_id: document.line_id,
-                validations: 0,
-            };
-        }
-        result[resultKey].validations += 1;
-        if (counter % 1000 === 0)
-            message(`Processados ${counter} documentos até agora...`);
-        counter++;
-        //
-    }
-    message(`A escrever os resultados no ficheiro CSV...`);
-    await csvWriter.write(Object.values(result));
-    await csvWriter.flush();
-    //
-}

package/dist/tasks/apex-validations/validations-by-pattern.d.ts

@@ -1,5 +0,0 @@
-import { type TaskProps } from '../../types.js';
-/**
- * Export Validations By Pattern data applying the given filters.
- */
-export declare function exportValidationsByPattern({ context, message }: TaskProps): Promise<void>;

package/dist/tasks/apex-validations/validations-by-pattern.js

@@ -1,87 +0,0 @@
-/* * */
-import { Dates } from '@tmlmobilidade/dates';
-import { simplifiedApexValidations } from '@tmlmobilidade/interfaces';
-import { CsvWriter } from '@tmlmobilidade/writers';
-import fs from 'node:fs';
-/* * */
-const TASK_ID = 'validations-by-pattern';
-/**
- * Export Validations By Pattern data applying the given filters.
- */
-export async function exportValidationsByPattern({ context, message }) {
-    //
-    message('A iniciar a exportação de Validações APEX por Pattern...');
-    //
-    // Prepare the filter params
-    const filterQuery = {
-        is_passenger: true,
-    };
-    filterQuery.created_at = {
-        $gte: Dates
-            .fromOperationalDate(context.dates.start, 'Europe/Lisbon')
-            .set({ hour: 4, millisecond: 0, minute: 0, second: 0 })
-            .unix_timestamp,
-        $lt: Dates
-            .fromOperationalDate(context.dates.end, 'Europe/Lisbon')
-            .set({ hour: 4, millisecond: 0, minute: 0, second: 0 })
-            .unix_timestamp,
-    };
-    if (context.filters.agency_ids.length) {
-        filterQuery.agency_id = { $in: context.filters.agency_ids };
-    }
-    if (context.filters.line_ids.length) {
-        filterQuery.line_id = { $in: context.filters.line_ids };
-    }
-    if (context.filters.pattern_ids.length) {
-        filterQuery.pattern_id = { $in: context.filters.pattern_ids };
-    }
-    if (context.filters.stop_ids.length) {
-        filterQuery.stop_id = { $in: context.filters.stop_ids };
-    }
-    if (context.filters.vehicle_ids.length) {
-        filterQuery.vehicle_id = { $in: context.filters.vehicle_ids };
-    }
-    //
-    // Setup a database stream to export data
-    message(`A iniciar ligação à base de dados...`);
-    const simplifiedApexValidationsCollection = await simplifiedApexValidations.getCollection();
-    const stream = simplifiedApexValidationsCollection.find(filterQuery).stream();
-    //
-    // Prepare the output directory and CSV writer
-    message(`A preparar a pasta para guardar os resultados...`);
-    if (!fs.existsSync(context.output))
-        fs.mkdirSync(context.output, { recursive: true });
-    const csvWriter = new CsvWriter('output', `${context.output}/${TASK_ID}-${context.dates.start}-${context.dates.end}.csv`, { batch_size: 100000, logs: false });
-    //
-    // Export the data
-    let counter = 0;
-    message(`A aguardar o resultado da pesquisa...`);
-    const result = {};
-    for await (const doc of stream) {
-        const document = doc;
-        //
-        // Prepare the result key
-        const operationalDate = Dates
-            .fromUnixTimestamp(document.created_at)
-            .operational_date;
-        const resultKey = `${operationalDate}:${document.pattern_id}`;
-        //
-        // Update the result with the current document
-        if (!result[resultKey]) {
-            result[resultKey] = {
-                date: operationalDate,
-                pattern_id: document.pattern_id,
-                validations: 0,
-            };
-        }
-        result[resultKey].validations += 1;
-        if (counter % 1000 === 0)
-            message(`Processados ${counter} documentos até agora...`);
-        counter++;
-        //
-    }
-    message(`A escrever os resultados no ficheiro CSV...`);
-    await csvWriter.write(Object.values(result));
-    await csvWriter.flush();
-    //
-}

package/dist/tasks/apex-validations/validations-by-stop-by-pattern.d.ts

@@ -1,5 +0,0 @@
-import { type TaskProps } from '../../types.js';
-/**
- * Export Validations By Stop By Pattern data applying the given filters.
- */
-export declare function exportValidationsByStopByPattern({ context, message }: TaskProps): Promise<void>;

package/dist/tasks/apex-validations/validations-by-stop-by-trip.d.ts

@@ -1,5 +0,0 @@
-import { type TaskProps } from '../../types.js';
-/**
- * Export Validations By Stop By Trip data applying the given filters.
- */
-export declare function exportValidationsByStopByTrip({ context, message }: TaskProps): Promise<void>;

package/dist/tasks/apex-validations/validations-by-stop-by-trip.js

@@ -1,88 +0,0 @@
-/* * */
-import { Dates } from '@tmlmobilidade/dates';
-import { simplifiedApexValidations } from '@tmlmobilidade/interfaces';
-import { CsvWriter } from '@tmlmobilidade/writers';
-import fs from 'node:fs';
-/* * */
-const TASK_ID = 'validations-by-stop-by-trip';
-/**
- * Export Validations By Stop By Trip data applying the given filters.
- */
-export async function exportValidationsByStopByTrip({ context, message }) {
-    //
-    message('A iniciar a exportação de Validações APEX por Paragem e por Viagem...');
-    //
-    // Prepare the filter params
-    const filterQuery = {
-        is_passenger: true,
-    };
-    filterQuery.created_at = {
-        $gte: Dates
-            .fromOperationalDate(context.dates.start, 'Europe/Lisbon')
-            .set({ hour: 4, millisecond: 0, minute: 0, second: 0 })
-            .unix_timestamp,
-        $lt: Dates
-            .fromOperationalDate(context.dates.end, 'Europe/Lisbon')
-            .set({ hour: 4, millisecond: 0, minute: 0, second: 0 })
-            .unix_timestamp,
-    };
-    if (context.filters.agency_ids.length) {
-        filterQuery.agency_id = { $in: context.filters.agency_ids };
-    }
-    if (context.filters.line_ids.length) {
-        filterQuery.line_id = { $in: context.filters.line_ids };
-    }
-    if (context.filters.pattern_ids.length) {
-        filterQuery.pattern_id = { $in: context.filters.pattern_ids };
-    }
-    if (context.filters.stop_ids.length) {
-        filterQuery.stop_id = { $in: context.filters.stop_ids };
-    }
-    if (context.filters.vehicle_ids.length) {
-        filterQuery.vehicle_id = { $in: context.filters.vehicle_ids };
-    }
-    //
-    // Setup a database stream to export data
-    message(`A iniciar ligação à base de dados...`);
-    const simplifiedApexValidationsCollection = await simplifiedApexValidations.getCollection();
-    const stream = simplifiedApexValidationsCollection.find(filterQuery).stream();
-    //
-    // Prepare the output directory and CSV writer
-    message(`A preparar a pasta para guardar os resultados...`);
-    if (!fs.existsSync(context.output))
-        fs.mkdirSync(context.output, { recursive: true });
-    const csvWriter = new CsvWriter('output', `${context.output}/${TASK_ID}-${context.dates.start}-${context.dates.end}.csv`, { batch_size: 100000, logs: false });
-    //
-    // Export the data
-    let counter = 0;
-    message(`A aguardar o resultado da pesquisa...`);
-    const result = {};
-    for await (const doc of stream) {
-        const document = doc;
-        //
-        // Prepare the result key
-        const operationalDate = Dates
-            .fromUnixTimestamp(document.created_at)
-            .operational_date;
-        const resultKey = `${operationalDate}:${document.stop_id}:${document.trip_id}`;
-        //
-        // Update the result with the current document
-        if (!result[resultKey]) {
-            result[resultKey] = {
-                date: operationalDate,
-                stop_id: document.stop_id,
-                trip_id: document.trip_id,
-                validations: 0,
-            };
-        }
-        result[resultKey].validations += 1;
-        if (counter % 1000 === 0)
-            message(`Processados ${counter} documentos até agora...`);
-        counter++;
-        //
-    }
-    message(`A escrever os resultados no ficheiro CSV...`);
-    await csvWriter.write(Object.values(result));
-    await csvWriter.flush();
-    //
-}

package/dist/tasks/apex-validations/validations-by-stop.d.ts

@@ -1,5 +0,0 @@
-import { type TaskProps } from '../../types.js';
-/**
- * Export Validations By Stop data applying the given filters.
- */
-export declare function exportValidationsByStop({ context, message }: TaskProps): Promise<void>;

package/dist/tasks/apex-validations/validations-by-stop.js

@@ -1,87 +0,0 @@
-/* * */
-import { Dates } from '@tmlmobilidade/dates';
-import { simplifiedApexValidations } from '@tmlmobilidade/interfaces';
-import { CsvWriter } from '@tmlmobilidade/writers';
-import fs from 'node:fs';
-/* * */
-const TASK_ID = 'validations-by-stop';
-/**
- * Export Validations By Stop data applying the given filters.
- */
-export async function exportValidationsByStop({ context, message }) {
-    //
-    message('A iniciar a exportação de Validações APEX por Paragem...');
-    //
-    // Prepare the filter params
-    const filterQuery = {
-        is_passenger: true,
-    };
-    filterQuery.created_at = {
-        $gte: Dates
-            .fromOperationalDate(context.dates.start, 'Europe/Lisbon')
-            .set({ hour: 4, millisecond: 0, minute: 0, second: 0 })
-            .unix_timestamp,
-        $lt: Dates
-            .fromOperationalDate(context.dates.end, 'Europe/Lisbon')
-            .set({ hour: 4, millisecond: 0, minute: 0, second: 0 })
-            .unix_timestamp,
-    };
-    if (context.filters.agency_ids.length) {
-        filterQuery.agency_id = { $in: context.filters.agency_ids };
-    }
-    if (context.filters.line_ids.length) {
-        filterQuery.line_id = { $in: context.filters.line_ids };
-    }
-    if (context.filters.pattern_ids.length) {
-        filterQuery.pattern_id = { $in: context.filters.pattern_ids };
-    }
-    if (context.filters.stop_ids.length) {
-        filterQuery.stop_id = { $in: context.filters.stop_ids };
-    }
-    if (context.filters.vehicle_ids.length) {
-        filterQuery.vehicle_id = { $in: context.filters.vehicle_ids };
-    }
-    //
-    // Setup a database stream to export data
-    message(`A iniciar ligação à base de dados...`);
-    const simplifiedApexValidationsCollection = await simplifiedApexValidations.getCollection();
-    const stream = simplifiedApexValidationsCollection.find(filterQuery).stream();
-    //
-    // Prepare the output directory and CSV writer
-    message(`A preparar a pasta para guardar os resultados...`);
-    if (!fs.existsSync(context.output))
-        fs.mkdirSync(context.output, { recursive: true });
-    const csvWriter = new CsvWriter('output', `${context.output}/${TASK_ID}-${context.dates.start}-${context.dates.end}.csv`, { batch_size: 100000, logs: false });
-    //
-    // Export the data
-    let counter = 0;
-    message(`A aguardar o resultado da pesquisa...`);
-    const result = {};
-    for await (const doc of stream) {
-        const document = doc;
-        //
-        // Prepare the result key
-        const operationalDate = Dates
-            .fromUnixTimestamp(document.created_at)
-            .operational_date;
-        const resultKey = `${operationalDate}:${document.stop_id}`;
-        //
-        // Update the result with the current document
-        if (!result[resultKey]) {
-            result[resultKey] = {
-                date: operationalDate,
-                stop_id: document.stop_id,
-                validations: 0,
-            };
-        }
-        result[resultKey].validations += 1;
-        if (counter % 1000 === 0)
-            message(`Processados ${counter} documentos até agora...`);
-        counter++;
-        //
-    }
-    message(`A escrever os resultados no ficheiro CSV...`);
-    await csvWriter.write(Object.values(result));
-    await csvWriter.flush();
-    //
-}