minotor 1.0.0

package/src/gtfs/__tests__/trips.test.ts

@@ -0,0 +1,463 @@
+import assert from 'node:assert';
+import { Readable } from 'node:stream';
+import { describe, it } from 'node:test';
+
+import { Time } from '../../timetable/time.js';
+import {
+  RoutesAdjacency,
+  ServiceRoutesMap,
+} from '../../timetable/timetable.js';
+import { ServiceIds } from '../services.js';
+import { StopIds } from '../stops.js';
+import { TransfersMap } from '../transfers.js';
+import {
+  buildStopsAdjacencyStructure,
+  parseStopTimes,
+  parseTrips,
+  TripIdsMap,
+} from '../trips.js';
+
+describe('buildStopsAdjacencyStructure', () => {
+  it('should correctly build stops adjacency for valid routes and transfers', () => {
+    const validStops: StopIds = new Set(['stop1']);
+    const routesAdjacency: RoutesAdjacency = new Map([
+      [
+        'routeA',
+        {
+          serviceRouteId: 'service1',
+          stops: ['stop1', 'stop2'],
+          stopIndices: new Map([
+            ['stop1', 0],
+            ['stop2', 1],
+          ]),
+          stopTimes: [],
+        },
+      ],
+    ]);
+    const transfersMap: TransfersMap = new Map([
+      ['stop1', [{ destination: 'stop2', type: 'RECOMMENDED' }]],
+    ]);
+
+    const stopsAdjacency = buildStopsAdjacencyStructure(
+      validStops,
+      routesAdjacency,
+      transfersMap,
+    );
+
+    assert.deepEqual(Array.from(stopsAdjacency.entries()), [
+      [
+        'stop1',
+        {
+          routes: ['routeA'],
+          transfers: [],
+        },
+      ],
+    ]);
+  });
+
+  it('should ignore transfers to invalid stops', () => {
+    const validStops: StopIds = new Set(['stop1', 'stop2']);
+    const routesAdjacency: RoutesAdjacency = new Map([
+      [
+        'routeA',
+        {
+          serviceRouteId: 'service1',
+          stops: ['stop1', 'stop2'],
+          stopIndices: new Map([
+            ['stop1', 0],
+            ['stop2', 1],
+          ]),
+          stopTimes: [],
+        },
+      ],
+    ]);
+    const transfersMap: TransfersMap = new Map([
+      ['stop1', [{ destination: 'stop3', type: 'RECOMMENDED' }]],
+    ]);
+
+    const stopsAdjacency = buildStopsAdjacencyStructure(
+      validStops,
+      routesAdjacency,
+      transfersMap,
+    );
+
+    assert.deepEqual(Array.from(stopsAdjacency.entries()), [
+      [
+        'stop1',
+        {
+          routes: ['routeA'],
+          transfers: [],
+        },
+      ],
+      [
+        'stop2',
+        {
+          routes: ['routeA'],
+          transfers: [],
+        },
+      ],
+    ]);
+  });
+});
+describe('GTFS trips parser', () => {
+  it('should correctly parse valid trips', async () => {
+    const mockedStream = new Readable();
+    mockedStream.push('route_id,service_id,trip_id\n');
+    mockedStream.push('"routeA","service1","trip1"\n');
+    mockedStream.push('"routeB","service2","trip2"\n');
+    mockedStream.push(null);
+
+    const validServiceIds: ServiceIds = new Set(['service1', 'service2']);
+    const validRouteIds: ServiceRoutesMap = new Map([
+      ['routeA', { type: 'BUS', name: 'B1' }],
+      ['routeB', { type: 'TRAM', name: 'T1' }],
+    ]);
+
+    const trips = await parseTrips(
+      mockedStream,
+      validServiceIds,
+      validRouteIds,
+    );
+    assert.deepEqual(
+      trips,
+      new Map([
+        ['trip1', 'routeA'],
+        ['trip2', 'routeB'],
+      ]),
+    );
+  });
+
+  it('should ignore trips with invalid service ids', async () => {
+    const mockedStream = new Readable();
+    mockedStream.push('route_id,service_id,trip_id\n');
+    mockedStream.push('"routeA","service1","trip1"\n');
+    mockedStream.push('"routeB","service3","trip2"\n');
+    mockedStream.push(null);
+
+    const validServiceIds: ServiceIds = new Set(['service1', 'service2']);
+    const validRouteIds: ServiceRoutesMap = new Map([
+      ['routeA', { type: 'BUS', name: 'B1' }],
+      ['routeB', { type: 'TRAM', name: 'T1' }],
+    ]);
+
+    const trips = await parseTrips(
+      mockedStream,
+      validServiceIds,
+      validRouteIds,
+    );
+    assert.deepEqual(trips, new Map([['trip1', 'routeA']]));
+  });
+
+  it('should ignore trips with invalid route ids', async () => {
+    const mockedStream = new Readable();
+    mockedStream.push('route_id,service_id,trip_id\n');
+    mockedStream.push('"routeA","service1","trip1"\n');
+    mockedStream.push('"routeC","service2","trip2"\n');
+    mockedStream.push(null);
+
+    const validServiceIds: ServiceIds = new Set(['service1', 'service2']);
+    const validRouteIds: ServiceRoutesMap = new Map([
+      ['routeA', { type: 'BUS', name: 'B1' }],
+      ['routeB', { type: 'TRAM', name: 'T1' }],
+    ]);
+
+    const trips = await parseTrips(
+      mockedStream,
+      validServiceIds,
+      validRouteIds,
+    );
+    assert.deepEqual(trips, new Map([['trip1', 'routeA']]));
+  });
+});
+
+describe('GTFS stop times parser', () => {
+  it('should correctly parse valid stop times', async () => {
+    const mockedStream = new Readable();
+    mockedStream.push(
+      'trip_id,arrival_time,departure_time,stop_id,stop_sequence,pickup_type,drop_off_type\n',
+    );
+    mockedStream.push('"tripA","08:00:00","08:05:00","stop1","1","0","0"\n');
+    mockedStream.push('"tripA","08:10:00","08:15:00","stop2","2","0","0"\n');
+    mockedStream.push(null);
+
+    const validTripIds: TripIdsMap = new Map([['tripA', 'routeA']]);
+    const validStopIds: StopIds = new Set(['stop1', 'stop2']);
+
+    const routes = await parseStopTimes(
+      mockedStream,
+      validTripIds,
+      validStopIds,
+    );
+    assert.deepEqual(
+      routes,
+      new Map([
+        [
+          'routeA_1e0e4u3',
+          {
+            serviceRouteId: 'routeA',
+            stops: ['stop1', 'stop2'],
+            stopIndices: new Map([
+              ['stop1', 0],
+              ['stop2', 1],
+            ]),
+            stopTimes: [
+              {
+                arrival: Time.fromHMS(8, 0, 0),
+                departure: Time.fromHMS(8, 5, 0),
+                pickUpType: 'REGULAR',
+                dropOffType: 'REGULAR',
+              },
+              {
+                arrival: Time.fromHMS(8, 10, 0),
+                departure: Time.fromHMS(8, 15, 0),
+                pickUpType: 'REGULAR',
+                dropOffType: 'REGULAR',
+              },
+            ],
+          },
+        ],
+      ]),
+    );
+  });
+
+  it('should create same route for same GTFS route with same stops', async () => {
+    const mockedStream = new Readable();
+    mockedStream.push(
+      'trip_id,arrival_time,departure_time,stop_id,stop_sequence,pickup_type,drop_off_type\n',
+    );
+    mockedStream.push('"tripA","08:00:00","08:05:00","stop1","1","0","0"\n');
+    mockedStream.push('"tripA","08:10:00","08:15:00","stop2","2","0","0"\n');
+    mockedStream.push('"tripB","09:00:00","09:05:00","stop1","1","0","0"\n');
+    mockedStream.push('"tripB","09:10:00","09:15:00","stop2","2","0","0"\n');
+    mockedStream.push(null);
+
+    const validTripIds: TripIdsMap = new Map([
+      ['tripA', 'routeA'],
+      ['tripB', 'routeA'],
+    ]);
+    const validStopIds: StopIds = new Set(['stop1', 'stop2']);
+
+    const routes = await parseStopTimes(
+      mockedStream,
+      validTripIds,
+      validStopIds,
+    );
+    assert.deepEqual(
+      routes,
+      new Map([
+        [
+          'routeA_1e0e4u3',
+          {
+            serviceRouteId: 'routeA',
+            stops: ['stop1', 'stop2'],
+            stopIndices: new Map([
+              ['stop1', 0],
+              ['stop2', 1],
+            ]),
+            stopTimes: [
+              {
+                arrival: Time.fromHMS(8, 0, 0),
+                departure: Time.fromHMS(8, 5, 0),
+                pickUpType: 'REGULAR',
+                dropOffType: 'REGULAR',
+              },
+              {
+                arrival: Time.fromHMS(8, 10, 0),
+                departure: Time.fromHMS(8, 15, 0),
+                pickUpType: 'REGULAR',
+                dropOffType: 'REGULAR',
+              },
+              {
+                arrival: Time.fromHMS(9, 0, 0),
+                departure: Time.fromHMS(9, 5, 0),
+                pickUpType: 'REGULAR',
+                dropOffType: 'REGULAR',
+              },
+              {
+                arrival: Time.fromHMS(9, 10, 0),
+                departure: Time.fromHMS(9, 15, 0),
+                pickUpType: 'REGULAR',
+                dropOffType: 'REGULAR',
+              },
+            ],
+          },
+        ],
+      ]),
+    );
+  });
+
+  it('should support unsorted trips within a route', async () => {
+    const mockedStream = new Readable();
+    mockedStream.push(
+      'trip_id,arrival_time,departure_time,stop_id,stop_sequence,pickup_type,drop_off_type\n',
+    );
+    mockedStream.push('"tripB","09:00:00","09:05:00","stop1","1","0","0"\n');
+    mockedStream.push('"tripB","09:10:00","09:15:00","stop2","2","0","0"\n');
+    mockedStream.push('"tripA","08:00:00","08:05:00","stop1","1","0","0"\n');
+    mockedStream.push('"tripA","08:10:00","08:15:00","stop2","2","0","0"\n');
+    mockedStream.push(null);
+
+    const validTripIds: TripIdsMap = new Map([
+      ['tripA', 'routeA'],
+      ['tripB', 'routeA'],
+    ]);
+    const validStopIds: StopIds = new Set(['stop1', 'stop2']);
+
+    const routes = await parseStopTimes(
+      mockedStream,
+      validTripIds,
+      validStopIds,
+    );
+    assert.deepEqual(
+      routes,
+      new Map([
+        [
+          'routeA_1e0e4u3',
+          {
+            serviceRouteId: 'routeA',
+            stops: ['stop1', 'stop2'],
+            stopIndices: new Map([
+              ['stop1', 0],
+              ['stop2', 1],
+            ]),
+            stopTimes: [
+              {
+                arrival: Time.fromHMS(8, 0, 0),
+                departure: Time.fromHMS(8, 5, 0),
+                pickUpType: 'REGULAR',
+                dropOffType: 'REGULAR',
+              },
+              {
+                arrival: Time.fromHMS(8, 10, 0),
+                departure: Time.fromHMS(8, 15, 0),
+                pickUpType: 'REGULAR',
+                dropOffType: 'REGULAR',
+              },
+              {
+                arrival: Time.fromHMS(9, 0, 0),
+                departure: Time.fromHMS(9, 5, 0),
+                pickUpType: 'REGULAR',
+                dropOffType: 'REGULAR',
+              },
+              {
+                arrival: Time.fromHMS(9, 10, 0),
+                departure: Time.fromHMS(9, 15, 0),
+                pickUpType: 'REGULAR',
+                dropOffType: 'REGULAR',
+              },
+            ],
+          },
+        ],
+      ]),
+    );
+  });
+
+  it('should create distinct route for same GTFS route with different stops', async () => {
+    const mockedStream = new Readable();
+    mockedStream.push(
+      'trip_id,arrival_time,departure_time,stop_id,stop_sequence,pickup_type,drop_off_type\n',
+    );
+    mockedStream.push('"tripA","08:00:00","08:05:00","stop1","1","0","0"\n');
+    mockedStream.push('"tripA","08:10:00","08:15:00","stop2","2","0","0"\n');
+    mockedStream.push('"tripB","09:00:00","09:15:00","stop1","1","0","0"\n');
+    mockedStream.push(null);
+
+    const validTripIds: TripIdsMap = new Map([
+      ['tripA', 'routeA'],
+      ['tripB', 'routeA'],
+    ]);
+    const validStopIds: StopIds = new Set(['stop1', 'stop2']);
+
+    const routes = await parseStopTimes(
+      mockedStream,
+      validTripIds,
+      validStopIds,
+    );
+    assert.deepEqual(
+      routes,
+      new Map([
+        [
+          'routeA_1e0e4u3',
+          {
+            serviceRouteId: 'routeA',
+            stops: ['stop1', 'stop2'],
+            stopIndices: new Map([
+              ['stop1', 0],
+              ['stop2', 1],
+            ]),
+            stopTimes: [
+              {
+                arrival: Time.fromHMS(8, 0, 0),
+                departure: Time.fromHMS(8, 5, 0),
+                pickUpType: 'REGULAR',
+                dropOffType: 'REGULAR',
+              },
+              {
+                arrival: Time.fromHMS(8, 10, 0),
+                departure: Time.fromHMS(8, 15, 0),
+                pickUpType: 'REGULAR',
+                dropOffType: 'REGULAR',
+              },
+            ],
+          },
+        ],
+        [
+          'routeA_1tcrqn',
+          {
+            serviceRouteId: 'routeA',
+            stops: ['stop1'],
+            stopIndices: new Map([['stop1', 0]]),
+            stopTimes: [
+              {
+                arrival: Time.fromHMS(9, 0, 0),
+                departure: Time.fromHMS(9, 15, 0),
+                pickUpType: 'REGULAR',
+                dropOffType: 'REGULAR',
+              },
+            ],
+          },
+        ],
+      ]),
+    );
+  });
+
+  it('should throw an error for non-increasing stop sequences', async () => {
+    const mockedStream = new Readable();
+    mockedStream.push(
+      'trip_id,arrival_time,departure_time,stop_id,stop_sequence,pickup_type,drop_off_type\n',
+    );
+    mockedStream.push('"tripA","08:00:00","08:05:00","stop1","2","0","0"\n');
+    mockedStream.push('"tripA","08:10:00","08:15:00","stop2","1","0","0"\n');
+    mockedStream.push(null);
+
+    const validTripIds: TripIdsMap = new Map([['tripA', 'routeA']]);
+    const validStopIds: StopIds = new Set(['stop1', 'stop2']);
+
+    const routes = await parseStopTimes(
+      mockedStream,
+      validTripIds,
+      validStopIds,
+    );
+    assert.deepEqual(
+      routes,
+      new Map([
+        [
+          'routeA_1tcrqn',
+          {
+            serviceRouteId: 'routeA',
+            stops: ['stop1'],
+            stopIndices: new Map([['stop1', 0]]),
+            stopTimes: [
+              {
+                arrival: Time.fromHMS(8, 0, 0),
+                departure: Time.fromHMS(8, 5, 0),
+                pickUpType: 'REGULAR',
+                dropOffType: 'REGULAR',
+              },
+            ],
+          },
+        ],
+      ]),
+    );
+  });
+});

package/src/gtfs/__tests__/utils.test.ts

@@ -0,0 +1,13 @@
+import assert from 'node:assert';
+import { describe, it } from 'node:test';
+
+import { hash } from '../utils.js';
+
+describe('Utility hash function', () => {
+  it('should be consistent for a given input', () => {
+    assert.equal(hash('stationA'), 'lswfbh');
+  });
+  it('should not collide with different input', () => {
+    assert.equal(hash('stationA') === hash('stationB'), false);
+  });
+});

package/src/gtfs/parser.ts

@@ -0,0 +1,154 @@
+import log from 'loglevel';
+import { DateTime } from 'luxon';
+import StreamZip from 'node-stream-zip';
+
+import { Platform } from '../stops/stops.js';
+import { StopsIndex } from '../stops/stopsIndex.js';
+import { RouteType, Timetable } from '../timetable/timetable.js';
+import { standardProfile } from './profiles/standard.js';
+import { parseRoutes } from './routes.js';
+import { parseCalendar, parseCalendarDates, ServiceIds } from './services.js';
+import { parseStops, StopEntry, StopIds } from './stops.js';
+import { parseTransfers, TransfersMap } from './transfers.js';
+import {
+  buildStopsAdjacencyStructure,
+  parseStopTimes,
+  parseTrips,
+} from './trips.js';
+import { Maybe } from './utils.js';
+
+const CALENDAR_FILE = 'calendar.txt';
+const CALENDAR_DATES_FILE = 'calendar_dates.txt';
+const ROUTES_FILE = 'routes.txt';
+const TRIPS_FILE = 'trips.txt';
+const STOP_TIMES_FILE = 'stop_times.txt';
+const STOPS_FILE = 'stops.txt';
+const TRANSFERS_FILE = 'transfers.txt';
+
+export type GtfsProfile = {
+  routeTypeParser: (routeType: number) => Maybe<RouteType>;
+  platformParser?: (stopEntry: StopEntry) => Maybe<Platform>;
+};
+
+export class GtfsParser {
+  private path: string;
+  private profile: GtfsProfile;
+
+  constructor(path: string, profile: GtfsProfile = standardProfile) {
+    // TODO: support input from multiple sources
+    this.path = path;
+    this.profile = profile;
+  }
+
+  /**
+   * Parses a GTFS feed to extract all the data relevant to a given day in a transit-planner friendly format.
+   *
+   * @param date The active date.
+   * @param gtfsPath A path to the zipped GTFS feed.
+   * @param gtfsProfile The GTFS profile configuration.
+   * @returns An object containing the timetable and stops map.
+   */
+  async parse(
+    date: Date,
+  ): Promise<{ timetable: Timetable; stopsIndex: StopsIndex }> {
+    const zip = new StreamZip.async({ file: this.path });
+    const entries = await zip.entries();
+    const datetime = DateTime.fromJSDate(date);
+
+    const validServiceIds: ServiceIds = new Set();
+    const validStopIds: StopIds = new Set();
+
+    if (entries[CALENDAR_FILE]) {
+      log.info(`Parsing ${CALENDAR_FILE}`);
+      const calendarStream = await zip.stream(CALENDAR_FILE);
+      await parseCalendar(calendarStream, validServiceIds, datetime);
+      log.info(`${validServiceIds.size} valid services.`);
+    }
+
+    if (entries[CALENDAR_DATES_FILE]) {
+      log.info(`Parsing ${CALENDAR_DATES_FILE}`);
+      const calendarDatesStream = await zip.stream(CALENDAR_DATES_FILE);
+      await parseCalendarDates(calendarDatesStream, validServiceIds, datetime);
+      log.info(`${validServiceIds.size} valid services.`);
+    }
+
+    log.info(`Parsing ${ROUTES_FILE}`);
+    const routesStream = await zip.stream(ROUTES_FILE);
+    const validGtfsRoutes = await parseRoutes(routesStream, this.profile);
+    log.info(`${validGtfsRoutes.size} valid GTFS routes.`);
+
+    log.info(`Parsing ${TRIPS_FILE}`);
+    const tripsStream = await zip.stream(TRIPS_FILE);
+    const trips = await parseTrips(
+      tripsStream,
+      validServiceIds,
+      validGtfsRoutes,
+    );
+    log.info(`${trips.size} valid trips.`);
+
+    let transfers = new Map() as TransfersMap;
+    if (entries[TRANSFERS_FILE]) {
+      log.info(`Parsing ${TRANSFERS_FILE}`);
+      const transfersStream = await zip.stream(TRANSFERS_FILE);
+      transfers = await parseTransfers(transfersStream);
+      log.info(`${transfers.size} valid transfers.`);
+    }
+
+    log.info(`Parsing ${STOP_TIMES_FILE}`);
+    const stopTimesStream = await zip.stream(STOP_TIMES_FILE);
+    const routesAdjacency = await parseStopTimes(
+      stopTimesStream,
+      trips,
+      validStopIds,
+    );
+    const stopsAdjacency = buildStopsAdjacencyStructure(
+      validStopIds,
+      routesAdjacency,
+      transfers,
+    );
+    log.info(`${routesAdjacency.size} valid unique routes.`);
+
+    log.info(`Parsing ${STOPS_FILE}`);
+    const stopsStream = await zip.stream(STOPS_FILE);
+    const stops = await parseStops(
+      stopsStream,
+      this.profile.platformParser,
+      validStopIds,
+    );
+    log.info(`${stops.size} valid stops.`);
+
+    await zip.close();
+
+    const timetable = new Timetable(
+      stopsAdjacency,
+      routesAdjacency,
+      validGtfsRoutes,
+    );
+
+    log.info(`Building stops index.`);
+    const stopsIndex = new StopsIndex(stops);
+
+    log.info('Parsing complete.');
+    return { timetable, stopsIndex };
+  }
+
+  /**
+   * Parses a GTFS feed to extract all stops.
+   *
+   * @param gtfsPath A path the zipped GTFS feed.
+   * @param gtfsProfile The GTFS profile configuration.
+   * @returns An object containing the timetable and stops map.
+   */
+  async parseStops(): Promise<StopsIndex> {
+    const zip = new StreamZip.async({ file: this.path });
+
+    log.info(`Parsing ${STOPS_FILE}`);
+    const stopsStream = await zip.stream(STOPS_FILE);
+    const stops = await parseStops(stopsStream, this.profile.platformParser);
+    log.info(`${stops.size} valid stops.`);
+
+    await zip.close();
+
+    return new StopsIndex(stops);
+  }
+}

package/src/gtfs/profiles/__tests__/ch.test.ts

@@ -0,0 +1,43 @@
+import assert from 'node:assert';
+import { describe, it } from 'node:test';
+
+import { chGtfsProfile } from '../ch.js';
+
+describe('The swiss GTFS feed parser', () => {
+  it('should extract the platform number from a stop entry', () => {
+    assert.ok(chGtfsProfile.platformParser);
+    assert.equal(
+      chGtfsProfile.platformParser({
+        stop_id: '8504100:0:1',
+        stop_name: 'Fribourg/Freiburg',
+        stop_lat: 46.8018210323626,
+        stop_lon: 7.14993389242926,
+        location_type: 1,
+        parent_station: 'Parent8504100',
+      }),
+      '1',
+    );
+  });
+  it('should not extract any platform number when not specified', () => {
+    assert.ok(chGtfsProfile.platformParser);
+    assert.equal(
+      chGtfsProfile.platformParser({
+        stop_id: 'Parent8587255',
+        stop_name: 'Fribourg, Tilleul/Cathédrale',
+        stop_lat: 46.8061375857565,
+        stop_lon: 7.16145029437328,
+        location_type: 1,
+        parent_station: '',
+      }),
+      undefined,
+    );
+  });
+  it('should convert the SBB route type to GTFS route type', () => {
+    assert.ok(chGtfsProfile.routeTypeParser);
+    assert.equal(chGtfsProfile.routeTypeParser(106), 'RAIL');
+  });
+  it('should not convert an unknown SBB route type', () => {
+    assert.ok(chGtfsProfile.routeTypeParser);
+    assert.equal(chGtfsProfile.routeTypeParser(716), undefined);
+  });
+});