@tmlmobilidade/export-data 20260706.1756.43 → 20260707.1126.48

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/tasks/rides/rides-raw.js +13 -2
  2. package/dist/utils/parse-ride.js +92 -0
  3. package/package.json +12 -12
package/dist/tasks/rides/rides-raw.js CHANGED
@@ -1,5 +1,6 @@
1
1
  /* * */
2
- import { rides } from '@tmlmobilidade/interfaces';
2
+ import { parseRide } from '../../utils/parse-ride.js';
3
+ import { rides, ridesBatchAggregationPipeline } from '@tmlmobilidade/interfaces';
3
4
  import { CsvWriter } from '@tmlmobilidade/writers';
4
5
  import fs from 'node:fs';
5
6
  /* * */
@@ -33,6 +34,16 @@ export async function exportRidesRaw({ context, message }) {
33
34
  const ridesCollection = await rides.getCollection();
34
35
  const stream = ridesCollection.find(filterQuery).stream();
35
36
  //
37
+ // Get the rides batch using native MongoDB cursor with batchSize to prevent memory issues
38
+ const pipeline = ridesBatchAggregationPipeline({
39
+ agency_ids: context.filters.agency_ids,
40
+ line_ids: context.filters.line_ids,
41
+ operational_date_end: context.dates.end,
42
+ operational_date_start: context.dates.start,
43
+ pattern_ids: context.filters.pattern_ids,
44
+ vehicle_ids: context.filters.vehicle_ids,
45
+ });
46
+ //
36
47
  // Prepare the output directory and CSV writer
37
48
  message(`A preparar a pasta para guardar os resultados...`);
38
49
  if (!fs.existsSync(context.output))
@@ -44,7 +55,7 @@ export async function exportRidesRaw({ context, message }) {
44
55
  message(`A aguardar o resultado da pesquisa...`);
45
56
  for await (const doc of stream) {
46
57
  const document = doc;
47
- await csvWriter.write(document);
58
+ await csvWriter.write(parseRide(document));
48
59
  if (counter % 1000 === 0)
49
60
  message(`Processados ${counter} documentos até agora...`);
50
61
  counter++;
package/dist/utils/parse-ride.js ADDED
@@ -0,0 +1,92 @@
1
+ import { Dates } from '@tmlmobilidade/dates';
2
+ function parseTime(time) {
3
+ if (!time) {
4
+ return null;
5
+ }
6
+ return Dates.fromUnixTimestamp(time).setZone('Europe/Lisbon', 'offset_only').toLocaleString(Dates.FORMATS.TIME_SIMPLE, 'pt-Pt');
7
+ }
8
+ export function parseRide(ride) {
9
+ return {
10
+ /* META */
11
+ /* * */
12
+ _id: ride._id,
13
+ agency_id: ride.agency_id,
14
+ driver_ids: ride.driver_ids?.join('|') ?? null,
15
+ headsign: ride.headsign,
16
+ line_id: ride.line_id,
17
+ pattern_id: ride.pattern_id,
18
+ plan_id: ride.plan_id,
19
+ route_id: ride.route_id,
20
+ trip_id: ride.trip_id,
21
+ vehicle_ids: ride.vehicle_ids?.join('|') ?? null,
22
+ /* TIME & STATUS */
23
+ /* * */
24
+ operational_date: ride.operational_date,
25
+ operational_status: ride.operational_status,
26
+ //
27
+ start_delay_status: ride.start_delay_status,
28
+ start_time_observed: parseTime(ride.start_time_observed),
29
+ start_time_scheduled: parseTime(ride.start_time_scheduled),
30
+ //
31
+ end_delay_status: ride.end_delay_status,
32
+ end_time_observed: parseTime(ride.end_time_observed),
33
+ end_time_scheduled: parseTime(ride.end_time_scheduled),
34
+ extension_observed: ride.extension_observed,
35
+ extension_scheduled: ride.extension_scheduled,
36
+ //
37
+ seen_first_at: parseTime(ride.seen_first_at),
38
+ seen_last_at: parseTime(ride.seen_last_at),
39
+ seen_status: ride.seen_status,
40
+ /* PASSENGERS */
41
+ /* * */
42
+ passengers_estimated: ride.passengers_estimated,
43
+ passengers_observed: ride.passengers_observed,
44
+ passengers_observed_on_board_sales_amount: ride.passengers_observed_on_board_sales_amount,
45
+ passengers_observed_on_board_sales_qty: ride.passengers_observed_on_board_sales_qty,
46
+ passengers_observed_prepaid_amount: ride.passengers_observed_prepaid_amount,
47
+ passengers_observed_prepaid_qty: ride.passengers_observed_prepaid_qty,
48
+ passengers_observed_subscription_qty: ride.passengers_observed_subscription_qty,
49
+ /* APEX */
50
+ /* * */
51
+ apex_locations_qty: ride.apex_locations_qty,
52
+ apex_on_board_refunds_amount: ride.apex_on_board_refunds_amount,
53
+ apex_on_board_refunds_qty: ride.apex_on_board_refunds_qty,
54
+ apex_on_board_sales_amount: ride.apex_on_board_sales_amount,
55
+ apex_on_board_sales_qty: ride.apex_on_board_sales_qty,
56
+ apex_validations_qty: ride.apex_validations_qty,
57
+ /* ANALYSIS */
58
+ /* * */
59
+ analysis_AT_LEAST_ONE_VEHICLE_EVENT_ON_FIRST_STOP: ride.analysis?.AT_LEAST_ONE_VEHICLE_EVENT_ON_FIRST_STOP?.grade ?? null,
60
+ analysis_ENDED_AT_LAST_STOP: ride.analysis?.ENDED_AT_LAST_STOP?.grade ?? null,
61
+ analysis_EXPECTED_APEX_VALIDATION_INTERVAL: ride.analysis?.EXPECTED_APEX_VALIDATION_INTERVAL?.grade ?? null,
62
+ analysis_EXPECTED_DRIVER_ID_QTY: ride.analysis?.EXPECTED_DRIVER_ID_QTY?.grade ?? null,
63
+ analysis_EXPECTED_START_TIME: ride.analysis?.EXPECTED_START_TIME?.grade ?? null,
64
+ analysis_EXPECTED_START_TIME_value: ride.analysis?.EXPECTED_START_TIME?.value ?? null,
65
+ analysis_EXPECTED_VEHICLE_EVENT_DELAY: ride.analysis?.EXPECTED_VEHICLE_EVENT_DELAY?.grade ?? null,
66
+ analysis_EXPECTED_VEHICLE_EVENT_INTERVAL: ride.analysis?.EXPECTED_VEHICLE_EVENT_INTERVAL?.grade ?? null,
67
+ analysis_EXPECTED_VEHICLE_EVENT_QTY: ride.analysis?.EXPECTED_VEHICLE_EVENT_QTY?.grade ?? null,
68
+ analysis_EXPECTED_VEHICLE_EVENT_QTY_expected_qty: ride.analysis?.EXPECTED_VEHICLE_EVENT_QTY?.expected_qty ?? null,
69
+ analysis_EXPECTED_VEHICLE_EVENT_QTY_found_qty: ride.analysis?.EXPECTED_VEHICLE_EVENT_QTY?.found_qty ?? null,
70
+ analysis_EXPECTED_VEHICLE_ID_QTY: ride.analysis?.EXPECTED_VEHICLE_ID_QTY?.grade ?? null,
71
+ analysis_MATCHING_APEX_LOCATIONS: ride.analysis?.MATCHING_APEX_LOCATIONS?.grade ?? null,
72
+ analysis_MATCHING_VEHICLE_IDS: ride.analysis?.MATCHING_VEHICLE_IDS?.grade ?? null,
73
+ analysis_SIMPLE_ONE_APEX_VALIDATION: ride.analysis?.SIMPLE_ONE_APEX_VALIDATION?.grade ?? null,
74
+ analysis_SIMPLE_ONE_VEHICLE_EVENT_OR_APEX_VALIDATION: ride.analysis?.SIMPLE_ONE_VEHICLE_EVENT_OR_APEX_VALIDATION?.grade ?? null,
75
+ analysis_SIMPLE_THREE_VEHICLE_EVENTS: ride.analysis?.SIMPLE_THREE_VEHICLE_EVENTS?.grade ?? null,
76
+ analysis_SIMPLE_THREE_VEHICLE_EVENTS_reason: ride.analysis?.SIMPLE_THREE_VEHICLE_EVENTS?.reason ?? null,
77
+ analysis_SIMPLE_THREE_VEHICLE_EVENTS_stop_ids_first: ride.analysis?.SIMPLE_THREE_VEHICLE_EVENTS?.stop_ids_first?.join('|') ?? null,
78
+ analysis_SIMPLE_THREE_VEHICLE_EVENTS_stop_ids_last: ride.analysis?.SIMPLE_THREE_VEHICLE_EVENTS?.stop_ids_last?.join('|') ?? null,
79
+ analysis_SIMPLE_THREE_VEHICLE_EVENTS_stop_ids_middle: ride.analysis?.SIMPLE_THREE_VEHICLE_EVENTS?.stop_ids_middle?.join('|') ?? null,
80
+ analysis_TRANSACTION_SEQUENTIALITY: ride.analysis?.TRANSACTION_SEQUENTIALITY?.grade ?? null,
81
+ analysis_TRANSACTION_SEQUENTIALITY_expected_qty: ride.analysis?.TRANSACTION_SEQUENTIALITY?.expected_qty ?? null,
82
+ analysis_TRANSACTION_SEQUENTIALITY_found_qty: ride.analysis?.TRANSACTION_SEQUENTIALITY?.found_qty ?? null,
83
+ analysis_TRANSACTION_SEQUENTIALITY_missing_qty: ride.analysis?.TRANSACTION_SEQUENTIALITY?.missing_qty ?? null,
84
+ /* ACCEPTANCE / JUSTIFICATION */
85
+ /* * */
86
+ acceptance_status: ride.acceptance?.acceptance_status,
87
+ justification_cause: ride.acceptance?.justification?.justification_cause,
88
+ justification_source: ride.acceptance?.justification?.justification_source,
89
+ manual_trip_id: ride.acceptance?.justification?.manual_trip_id?.replaceAll('\n', ' ')?.replaceAll(',', ' ')?.replaceAll(';', ' ')?.replaceAll(' ', ' '),
90
+ pto_message: ride.acceptance?.justification?.pto_message?.replaceAll('\n', ' ')?.replaceAll(',', ' ')?.replaceAll(';', ' ')?.replaceAll(' ', ' '),
91
+ };
92
+ }
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@tmlmobilidade/export-data",
3
3
  "description": "CLI tool to export data from GO.",
4
- "version": "20260706.1756.43",
4
+ "version": "20260707.1126.48",
5
5
  "author": {
6
6
  "email": "iso@tmlmobilidade.pt",
7
7
  "name": "TML-ISO"
@@ -37,20 +37,20 @@
37
37
  },
38
38
  "dependencies": {
39
39
  "@clack/prompts": "1.7.0",
40
- "@tmlmobilidade/consts": "20260706.1523.32",
41
- "@tmlmobilidade/dates": "20260706.1523.32",
42
- "@tmlmobilidade/go-types-apex": "20260706.1523.32",
43
- "@tmlmobilidade/go-types-shared": "20260706.1523.32",
44
- "@tmlmobilidade/interfaces": "20260706.1523.32",
45
- "@tmlmobilidade/logger": "20260706.1523.32",
46
- "@tmlmobilidade/strings": "20260706.1523.32",
47
- "@tmlmobilidade/timer": "20260706.1523.32",
48
- "@tmlmobilidade/types": "20260706.1523.32",
49
- "@tmlmobilidade/writers": "20260706.1523.32",
40
+ "@tmlmobilidade/consts": "20260707.1036.5",
41
+ "@tmlmobilidade/dates": "20260707.1036.5",
42
+ "@tmlmobilidade/go-types-apex": "20260707.1036.5",
43
+ "@tmlmobilidade/go-types-shared": "20260707.1036.5",
44
+ "@tmlmobilidade/interfaces": "20260707.1036.5",
45
+ "@tmlmobilidade/logger": "20260707.1036.5",
46
+ "@tmlmobilidade/strings": "20260707.1036.5",
47
+ "@tmlmobilidade/timer": "20260707.1036.5",
48
+ "@tmlmobilidade/types": "20260707.1036.5",
49
+ "@tmlmobilidade/writers": "20260707.1036.5",
50
50
  "exceljs": "^4.4.0"
51
51
  },
52
52
  "devDependencies": {
53
- "@tmlmobilidade/tsconfig": "20260706.1523.32",
53
+ "@tmlmobilidade/tsconfig": "20260707.1036.5",
54
54
  "@types/node": "26.1.0",
55
55
  "resolve-tspaths": "0.8.23",
56
56
  "typescript": "6.0.3"