@tmlmobilidade/import-gtfs 20260903.2201.22 → 20260904.1059.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/shared/extract-source.js +1 -1
- package/dist/versions/standard/processors/calendar-dates.js +2 -2
- package/dist/versions/standard/processors/calendar.js +2 -2
- package/dist/versions/standard/processors/routes.js +2 -2
- package/dist/versions/standard/processors/shapes.js +2 -2
- package/dist/versions/standard/processors/stop-times.js +2 -2
- package/dist/versions/standard/processors/stops.js +2 -2
- package/dist/versions/standard/processors/trips.js +2 -2
- package/dist/versions/v29/processors/calendar-dates.js +2 -2
- package/dist/versions/v29/processors/routes.js +2 -2
- package/dist/versions/v29/processors/shapes.js +2 -2
- package/dist/versions/v29/processors/stop-times.js +2 -2
- package/dist/versions/v29/processors/stops.js +2 -2
- package/dist/versions/v29/processors/trips.js +2 -2
- package/dist/versions/v29-ext/processors/calendar-dates.js +2 -2
- package/dist/versions/v29-ext/processors/calendar.js +2 -2
- package/dist/versions/v29-ext/processors/routes.js +2 -2
- package/dist/versions/v29-ext/processors/shapes.js +2 -2
- package/dist/versions/v29-ext/processors/stop-times.js +2 -2
- package/dist/versions/v29-ext/processors/stops.js +2 -2
- package/dist/versions/v29-ext/processors/trips.js +2 -2
- package/dist/versions/v30/processors/calendar-dates.js +2 -2
- package/dist/versions/v30/processors/calendar.js +2 -2
- package/dist/versions/v30/processors/routes.js +2 -2
- package/dist/versions/v30/processors/shapes.js +2 -2
- package/dist/versions/v30/processors/stop-times.js +2 -2
- package/dist/versions/v30/processors/stops.js +2 -2
- package/dist/versions/v30/processors/trips.js +2 -2
- package/package.json +4 -4
- package/dist/shared/parse-csv.d.ts +0 -7
- package/dist/shared/parse-csv.js +0 -25
- package/dist/shared/unzip-file.d.ts +0 -2
- package/dist/shared/unzip-file.js +0 -24
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsCalendarDatesSchema } from '@tmlmobilidade/go-types-gtfs';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
import fs from 'node:fs';
|
|
@@ -69,7 +69,7 @@ export async function processGtfsCalendarDates(context) {
|
|
|
69
69
|
//
|
|
70
70
|
// Setup the CSV parsing operation only if the file exists
|
|
71
71
|
if (fs.existsSync(`${context.workdir.extract_dir_path}/calendar_dates.txt`)) {
|
|
72
|
-
await
|
|
72
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/calendar_dates.txt`, parseEachRow);
|
|
73
73
|
Logger.success(`Finished processing "calendar_dates.txt": ${Object.keys(context.gtfs.calendar_dates).length} rows saved in ${calendarDatesParseTimer.get()}.`, 1);
|
|
74
74
|
}
|
|
75
75
|
else {
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsCalendarSchema } from '@tmlmobilidade/go-types-gtfs';
|
|
4
3
|
import { OperationalDateIntSchema } from '@tmlmobilidade/go-types-shared';
|
|
5
4
|
import { Dates, getOperationalDatesFromRange } from '@tmlmobilidade/go-utils-dates';
|
|
5
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
6
6
|
import { Logger } from '@tmlmobilidade/logger';
|
|
7
7
|
import { Timer } from '@tmlmobilidade/timer';
|
|
8
8
|
import fs from 'node:fs';
|
|
@@ -85,7 +85,7 @@ export async function processGtfsCalendar(context) {
|
|
|
85
85
|
//
|
|
86
86
|
// Setup the CSV parsing operation only if the file exists
|
|
87
87
|
if (fs.existsSync(`${context.workdir.extract_dir_path}/calendar.txt`)) {
|
|
88
|
-
await
|
|
88
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/calendar.txt`, parseEachRow);
|
|
89
89
|
Logger.success(`Finished processing "calendar.txt": ${context.gtfs.calendar_dates.size} rows saved in ${calendarParseTimer.get()}.`, 1);
|
|
90
90
|
}
|
|
91
91
|
else {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsRoutesSchema } from '@tmlmobilidade/go-types-gtfs';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -25,7 +25,7 @@ export async function processGtfsRoutes(context) {
|
|
|
25
25
|
};
|
|
26
26
|
//
|
|
27
27
|
// Setup the CSV parsing operation
|
|
28
|
-
await
|
|
28
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/routes.txt`, parseEachRow);
|
|
29
29
|
context.gtfs.routes.flush();
|
|
30
30
|
Logger.success(`Finished processing "routes.txt": ${context.gtfs.routes.size} rows saved in ${routesParseTimer.get()}.`, 1);
|
|
31
31
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsShapesSchema } from '@tmlmobilidade/go-types-gtfs';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -30,7 +30,7 @@ export async function processGtfsShapes(context) {
|
|
|
30
30
|
};
|
|
31
31
|
//
|
|
32
32
|
// Setup the CSV parsing operation
|
|
33
|
-
await
|
|
33
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/shapes.txt`, parseEachRow);
|
|
34
34
|
context.gtfs.shapes.flush();
|
|
35
35
|
Logger.success(`Finished processing "shapes.txt": ${context.gtfs.shapes.size} rows saved in ${shapesParseTimer.get()}.`, 1);
|
|
36
36
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStopTimesSchema } from '@tmlmobilidade/go-types-gtfs';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -36,7 +36,7 @@ export async function processGtfsStopTimes(context) {
|
|
|
36
36
|
};
|
|
37
37
|
//
|
|
38
38
|
// Setup the CSV parsing operation
|
|
39
|
-
await
|
|
39
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/stop_times.txt`, parseEachRow);
|
|
40
40
|
context.gtfs.stop_times.flush();
|
|
41
41
|
Logger.success(`Finished processing "stop_times.txt": ${context.counters.stop_times} rows saved in ${stopTimesParseTimer.get()}.`, 1);
|
|
42
42
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStopsSchema } from '@tmlmobilidade/go-types-gtfs';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -25,7 +25,7 @@ export async function processGtfsStops(context) {
|
|
|
25
25
|
};
|
|
26
26
|
//
|
|
27
27
|
// Setup the CSV parsing operation
|
|
28
|
-
await
|
|
28
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/stops.txt`, parseEachRow);
|
|
29
29
|
context.gtfs.stops.flush();
|
|
30
30
|
Logger.success(`Finished processing "stops.txt": ${context.gtfs.stops.size} rows saved in ${stopsParseTimer.get()}.`, 1);
|
|
31
31
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsTripsSchema } from '@tmlmobilidade/go-types-gtfs';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -33,7 +33,7 @@ export async function processGtfsTrips(context) {
|
|
|
33
33
|
};
|
|
34
34
|
//
|
|
35
35
|
// Setup the CSV parsing operation
|
|
36
|
-
await
|
|
36
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/trips.txt`, parseEachRow);
|
|
37
37
|
context.gtfs.trips.flush();
|
|
38
38
|
Logger.success(`Finished processing "trips.txt": ${context.gtfs.trips.size} rows saved in ${tripsParseTimer.get()}.`, 1);
|
|
39
39
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV29CalendarDatesSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
import fs from 'node:fs';
|
|
@@ -69,7 +69,7 @@ export async function processGtfsStrictV29CalendarDates(context) {
|
|
|
69
69
|
//
|
|
70
70
|
// Setup the CSV parsing operation only if the file exists
|
|
71
71
|
if (fs.existsSync(`${context.workdir.extract_dir_path}/calendar_dates.txt`)) {
|
|
72
|
-
await
|
|
72
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/calendar_dates.txt`, parseEachRow);
|
|
73
73
|
Logger.success(`Finished processing "calendar_dates.txt": ${Object.keys(context.gtfs.calendar_dates).length} rows saved in ${calendarDatesParseTimer.get()}.`, 1);
|
|
74
74
|
}
|
|
75
75
|
else {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV29RoutesSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -25,7 +25,7 @@ export async function processGtfsStrictV29Routes(context) {
|
|
|
25
25
|
};
|
|
26
26
|
//
|
|
27
27
|
// Setup the CSV parsing operation
|
|
28
|
-
await
|
|
28
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/routes.txt`, parseEachRow);
|
|
29
29
|
context.gtfs.routes.flush();
|
|
30
30
|
Logger.success(`Finished processing "routes.txt": ${context.gtfs.routes.size} rows saved in ${routesParseTimer.get()}.`, 1);
|
|
31
31
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV29ShapesSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -30,7 +30,7 @@ export async function processGtfsStrictV29Shapes(context) {
|
|
|
30
30
|
};
|
|
31
31
|
//
|
|
32
32
|
// Setup the CSV parsing operation
|
|
33
|
-
await
|
|
33
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/shapes.txt`, parseEachRow);
|
|
34
34
|
context.gtfs.shapes.flush();
|
|
35
35
|
Logger.success(`Finished processing "shapes.txt": ${context.gtfs.shapes.size} rows saved in ${shapesParseTimer.get()}.`, 1);
|
|
36
36
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV29StopTimesSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -36,7 +36,7 @@ export async function processGtfsStrictV29StopTimes(context) {
|
|
|
36
36
|
};
|
|
37
37
|
//
|
|
38
38
|
// Setup the CSV parsing operation
|
|
39
|
-
await
|
|
39
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/stop_times.txt`, parseEachRow);
|
|
40
40
|
context.gtfs.stop_times.flush();
|
|
41
41
|
Logger.success(`Finished processing "stop_times.txt": ${context.counters.stop_times} rows saved in ${stopTimesParseTimer.get()}.`, 1);
|
|
42
42
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV29StopsSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -25,7 +25,7 @@ export async function processGtfsStrictV29Stops(context) {
|
|
|
25
25
|
};
|
|
26
26
|
//
|
|
27
27
|
// Setup the CSV parsing operation
|
|
28
|
-
await
|
|
28
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/stops.txt`, parseEachRow);
|
|
29
29
|
context.gtfs.stops.flush();
|
|
30
30
|
Logger.success(`Finished processing "stops.txt": ${context.gtfs.stops.size} rows saved in ${stopsParseTimer.get()}.`, 1);
|
|
31
31
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV29TripsSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -33,7 +33,7 @@ export async function processGtfsStrictV29Trips(context) {
|
|
|
33
33
|
};
|
|
34
34
|
//
|
|
35
35
|
// Setup the CSV parsing operation
|
|
36
|
-
await
|
|
36
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/trips.txt`, parseEachRow);
|
|
37
37
|
context.gtfs.trips.flush();
|
|
38
38
|
Logger.success(`Finished processing "trips.txt": ${context.gtfs.trips.size} rows saved in ${tripsParseTimer.get()}.`, 1);
|
|
39
39
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV29ExtCalendarDatesSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
import fs from 'node:fs';
|
|
@@ -69,7 +69,7 @@ export async function processGtfsStrictV29ExtCalendarDates(context) {
|
|
|
69
69
|
//
|
|
70
70
|
// Setup the CSV parsing operation only if the file exists
|
|
71
71
|
if (fs.existsSync(`${context.workdir.extract_dir_path}/calendar_dates.txt`)) {
|
|
72
|
-
await
|
|
72
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/calendar_dates.txt`, parseEachRow);
|
|
73
73
|
Logger.success(`Finished processing "calendar_dates.txt": ${Object.keys(context.gtfs.calendar_dates).length} rows saved in ${calendarDatesParseTimer.get()}.`, 1);
|
|
74
74
|
}
|
|
75
75
|
else {
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV29ExtCalendarSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
4
3
|
import { OperationalDateIntSchema } from '@tmlmobilidade/go-types-shared';
|
|
5
4
|
import { Dates, getOperationalDatesFromRange } from '@tmlmobilidade/go-utils-dates';
|
|
5
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
6
6
|
import { Logger } from '@tmlmobilidade/logger';
|
|
7
7
|
import { Timer } from '@tmlmobilidade/timer';
|
|
8
8
|
import fs from 'node:fs';
|
|
@@ -85,7 +85,7 @@ export async function processGtfsStrictV29ExtCalendar(context) {
|
|
|
85
85
|
//
|
|
86
86
|
// Setup the CSV parsing operation only if the file exists
|
|
87
87
|
if (fs.existsSync(`${context.workdir.extract_dir_path}/calendar.txt`)) {
|
|
88
|
-
await
|
|
88
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/calendar.txt`, parseEachRow);
|
|
89
89
|
Logger.success(`Finished processing "calendar.txt": ${context.gtfs.calendar_dates.size} rows saved in ${calendarParseTimer.get()}.`, 1);
|
|
90
90
|
}
|
|
91
91
|
else {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV29ExtRoutesSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -25,7 +25,7 @@ export async function processGtfsStrictV29ExtRoutes(context) {
|
|
|
25
25
|
};
|
|
26
26
|
//
|
|
27
27
|
// Setup the CSV parsing operation
|
|
28
|
-
await
|
|
28
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/routes.txt`, parseEachRow);
|
|
29
29
|
context.gtfs.routes.flush();
|
|
30
30
|
Logger.success(`Finished processing "routes.txt": ${context.gtfs.routes.size} rows saved in ${routesParseTimer.get()}.`, 1);
|
|
31
31
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV29ExtShapesSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -30,7 +30,7 @@ export async function processGtfsStrictV29ExtShapes(context) {
|
|
|
30
30
|
};
|
|
31
31
|
//
|
|
32
32
|
// Setup the CSV parsing operation
|
|
33
|
-
await
|
|
33
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/shapes.txt`, parseEachRow);
|
|
34
34
|
context.gtfs.shapes.flush();
|
|
35
35
|
Logger.success(`Finished processing "shapes.txt": ${context.gtfs.shapes.size} rows saved in ${shapesParseTimer.get()}.`, 1);
|
|
36
36
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV29ExtStopTimesSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -36,7 +36,7 @@ export async function processGtfsStrictV29ExtStopTimes(context) {
|
|
|
36
36
|
};
|
|
37
37
|
//
|
|
38
38
|
// Setup the CSV parsing operation
|
|
39
|
-
await
|
|
39
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/stop_times.txt`, parseEachRow);
|
|
40
40
|
context.gtfs.stop_times.flush();
|
|
41
41
|
Logger.success(`Finished processing "stop_times.txt": ${context.counters.stop_times} rows saved in ${stopTimesParseTimer.get()}.`, 1);
|
|
42
42
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV29ExtStopsSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -25,7 +25,7 @@ export async function processGtfsStrictV29ExtStops(context) {
|
|
|
25
25
|
};
|
|
26
26
|
//
|
|
27
27
|
// Setup the CSV parsing operation
|
|
28
|
-
await
|
|
28
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/stops.txt`, parseEachRow);
|
|
29
29
|
context.gtfs.stops.flush();
|
|
30
30
|
Logger.success(`Finished processing "stops.txt": ${context.gtfs.stops.size} rows saved in ${stopsParseTimer.get()}.`, 1);
|
|
31
31
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV29ExtTripsSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -33,7 +33,7 @@ export async function processGtfsStrictV29ExtTrips(context) {
|
|
|
33
33
|
};
|
|
34
34
|
//
|
|
35
35
|
// Setup the CSV parsing operation
|
|
36
|
-
await
|
|
36
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/trips.txt`, parseEachRow);
|
|
37
37
|
context.gtfs.trips.flush();
|
|
38
38
|
Logger.success(`Finished processing "trips.txt": ${context.gtfs.trips.size} rows saved in ${tripsParseTimer.get()}.`, 1);
|
|
39
39
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV30CalendarDatesSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
import fs from 'node:fs';
|
|
@@ -69,7 +69,7 @@ export async function processGtfsStrictV30CalendarDates(context) {
|
|
|
69
69
|
//
|
|
70
70
|
// Setup the CSV parsing operation only if the file exists
|
|
71
71
|
if (fs.existsSync(`${context.workdir.extract_dir_path}/calendar_dates.txt`)) {
|
|
72
|
-
await
|
|
72
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/calendar_dates.txt`, parseEachRow);
|
|
73
73
|
Logger.success(`Finished processing "calendar_dates.txt": ${Object.keys(context.gtfs.calendar_dates).length} rows saved in ${calendarDatesParseTimer.get()}.`, 1);
|
|
74
74
|
}
|
|
75
75
|
else {
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV30CalendarSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
4
3
|
import { OperationalDateIntSchema } from '@tmlmobilidade/go-types-shared';
|
|
5
4
|
import { Dates, getOperationalDatesFromRange } from '@tmlmobilidade/go-utils-dates';
|
|
5
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
6
6
|
import { Logger } from '@tmlmobilidade/logger';
|
|
7
7
|
import { Timer } from '@tmlmobilidade/timer';
|
|
8
8
|
import fs from 'node:fs';
|
|
@@ -85,7 +85,7 @@ export async function processGtfsStrictV30Calendar(context) {
|
|
|
85
85
|
//
|
|
86
86
|
// Setup the CSV parsing operation only if the file exists
|
|
87
87
|
if (fs.existsSync(`${context.workdir.extract_dir_path}/calendar.txt`)) {
|
|
88
|
-
await
|
|
88
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/calendar.txt`, parseEachRow);
|
|
89
89
|
Logger.success(`Finished processing "calendar.txt": ${context.gtfs.calendar_dates.size} rows saved in ${calendarParseTimer.get()}.`, 1);
|
|
90
90
|
}
|
|
91
91
|
else {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV30RoutesSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -25,7 +25,7 @@ export async function processGtfsStrictV30Routes(context) {
|
|
|
25
25
|
};
|
|
26
26
|
//
|
|
27
27
|
// Setup the CSV parsing operation
|
|
28
|
-
await
|
|
28
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/routes.txt`, parseEachRow);
|
|
29
29
|
context.gtfs.routes.flush();
|
|
30
30
|
Logger.success(`Finished processing "routes.txt": ${context.gtfs.routes.size} rows saved in ${routesParseTimer.get()}.`, 1);
|
|
31
31
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV30ShapesSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -30,7 +30,7 @@ export async function processGtfsStrictV30Shapes(context) {
|
|
|
30
30
|
};
|
|
31
31
|
//
|
|
32
32
|
// Setup the CSV parsing operation
|
|
33
|
-
await
|
|
33
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/shapes.txt`, parseEachRow);
|
|
34
34
|
context.gtfs.shapes.flush();
|
|
35
35
|
Logger.success(`Finished processing "shapes.txt": ${context.gtfs.shapes.size} rows saved in ${shapesParseTimer.get()}.`, 1);
|
|
36
36
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV30StopTimesSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -36,7 +36,7 @@ export async function processGtfsStrictV30StopTimes(context) {
|
|
|
36
36
|
};
|
|
37
37
|
//
|
|
38
38
|
// Setup the CSV parsing operation
|
|
39
|
-
await
|
|
39
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/stop_times.txt`, parseEachRow);
|
|
40
40
|
context.gtfs.stop_times.flush();
|
|
41
41
|
Logger.success(`Finished processing "stop_times.txt": ${context.counters.stop_times} rows saved in ${stopTimesParseTimer.get()}.`, 1);
|
|
42
42
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV30StopsSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -25,7 +25,7 @@ export async function processGtfsStrictV30Stops(context) {
|
|
|
25
25
|
};
|
|
26
26
|
//
|
|
27
27
|
// Setup the CSV parsing operation
|
|
28
|
-
await
|
|
28
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/stops.txt`, parseEachRow);
|
|
29
29
|
context.gtfs.stops.flush();
|
|
30
30
|
Logger.success(`Finished processing "stops.txt": ${context.gtfs.stops.size} rows saved in ${stopsParseTimer.get()}.`, 1);
|
|
31
31
|
//
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* * */
|
|
2
|
-
import { parseCsvFile } from '../../../shared/parse-csv.js';
|
|
3
2
|
import { GtfsStrictV30TripsSchema } from '@tmlmobilidade/go-types-gtfs-strict';
|
|
3
|
+
import { streamCsvFile } from '@tmlmobilidade/go-utils-exec';
|
|
4
4
|
import { Logger } from '@tmlmobilidade/logger';
|
|
5
5
|
import { Timer } from '@tmlmobilidade/timer';
|
|
6
6
|
/**
|
|
@@ -33,7 +33,7 @@ export async function processGtfsStrictV30Trips(context) {
|
|
|
33
33
|
};
|
|
34
34
|
//
|
|
35
35
|
// Setup the CSV parsing operation
|
|
36
|
-
await
|
|
36
|
+
await streamCsvFile(`${context.workdir.extract_dir_path}/trips.txt`, parseEachRow);
|
|
37
37
|
context.gtfs.trips.flush();
|
|
38
38
|
Logger.success(`Finished processing "trips.txt": ${context.gtfs.trips.size} rows saved in ${tripsParseTimer.get()}.`, 1);
|
|
39
39
|
//
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tmlmobilidade/import-gtfs",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "20260904.1059.6",
|
|
4
4
|
"author": {
|
|
5
5
|
"email": "iso@tmlmobilidade.pt",
|
|
6
6
|
"name": "TML-ISO"
|
|
@@ -36,19 +36,19 @@
|
|
|
36
36
|
"watch": "tsc-watch --onSuccess 'resolve-tspaths'"
|
|
37
37
|
},
|
|
38
38
|
"dependencies": {
|
|
39
|
+
"@tmlmobilidade/go-clients-sqlite": "*",
|
|
39
40
|
"@tmlmobilidade/go-interfaces-godb": "*",
|
|
40
41
|
"@tmlmobilidade/go-types-gtfs": "*",
|
|
41
42
|
"@tmlmobilidade/go-types-gtfs-strict": "*",
|
|
42
43
|
"@tmlmobilidade/go-types-hub": "*",
|
|
43
44
|
"@tmlmobilidade/go-types-shared": "*",
|
|
44
45
|
"@tmlmobilidade/go-utils-dates": "*",
|
|
46
|
+
"@tmlmobilidade/go-utils-exec": "*",
|
|
45
47
|
"@tmlmobilidade/interfaces": "*",
|
|
46
48
|
"@tmlmobilidade/logger": "*",
|
|
47
|
-
"@tmlmobilidade/go-clients-sqlite": "*",
|
|
48
49
|
"@tmlmobilidade/strings": "*",
|
|
49
50
|
"@tmlmobilidade/timer": "*",
|
|
50
|
-
"csv-parse": "7.0.2"
|
|
51
|
-
"unzipper": "0.12.5"
|
|
51
|
+
"csv-parse": "7.0.2"
|
|
52
52
|
},
|
|
53
53
|
"devDependencies": {
|
|
54
54
|
"@tmlmobilidade/go-utils-tsconfig": "*",
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Parses a CSV file and calls a callback function for each row.
|
|
3
|
-
* @param filePath The path to the CSV file to parse.
|
|
4
|
-
* @param rowParser A callback function that will be called for each row.
|
|
5
|
-
* @returns A promise that resolves when the file is parsed.
|
|
6
|
-
*/
|
|
7
|
-
export declare function parseCsvFile<T>(filePath: string, rowParser: (rowData: T) => Promise<void>): Promise<void>;
|
package/dist/shared/parse-csv.js
DELETED
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
/* * */
|
|
2
|
-
import { parse as csvParser } from 'csv-parse';
|
|
3
|
-
import fs from 'fs';
|
|
4
|
-
/**
|
|
5
|
-
* Parses a CSV file and calls a callback function for each row.
|
|
6
|
-
* @param filePath The path to the CSV file to parse.
|
|
7
|
-
* @param rowParser A callback function that will be called for each row.
|
|
8
|
-
* @returns A promise that resolves when the file is parsed.
|
|
9
|
-
*/
|
|
10
|
-
export async function parseCsvFile(filePath, rowParser) {
|
|
11
|
-
const parser = csvParser({
|
|
12
|
-
bom: true,
|
|
13
|
-
cast: value => value === '' ? undefined : value,
|
|
14
|
-
columns: true,
|
|
15
|
-
record_delimiter: ['\n', '\r', '\r\n'],
|
|
16
|
-
skip_empty_lines: true,
|
|
17
|
-
skipRecordsWithEmptyValues: true,
|
|
18
|
-
trim: true,
|
|
19
|
-
});
|
|
20
|
-
const fileStream = fs.createReadStream(filePath);
|
|
21
|
-
const stream = fileStream.pipe(parser);
|
|
22
|
-
for await (const rowData of stream) {
|
|
23
|
-
await rowParser(rowData);
|
|
24
|
-
}
|
|
25
|
-
}
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
/* * */
|
|
2
|
-
import fs from 'node:fs';
|
|
3
|
-
import unzipper from 'unzipper';
|
|
4
|
-
/* * */
|
|
5
|
-
export async function unzipFile(zipFilePath, outputDir) {
|
|
6
|
-
await fs
|
|
7
|
-
.createReadStream(zipFilePath)
|
|
8
|
-
.pipe(unzipper.Extract({ path: outputDir }))
|
|
9
|
-
.promise();
|
|
10
|
-
setDirectoryPermissions(outputDir);
|
|
11
|
-
}
|
|
12
|
-
/* * */
|
|
13
|
-
export function setDirectoryPermissions(dirPath, mode = 0o666) {
|
|
14
|
-
const files = fs.readdirSync(dirPath, { withFileTypes: true });
|
|
15
|
-
for (const file of files) {
|
|
16
|
-
const filePath = `${dirPath}/${file.name}`;
|
|
17
|
-
if (file.isDirectory()) {
|
|
18
|
-
setDirectoryPermissions(filePath, mode);
|
|
19
|
-
}
|
|
20
|
-
else {
|
|
21
|
-
fs.chmodSync(filePath, mode);
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
}
|