@transportme/load-ptv-gtfs 1.0.0

package/test/gtfs-shapes-loader.test.mjs

@@ -0,0 +1,105 @@
+import { expect } from 'chai'
+import path from 'path'
+import url from 'url'
+import { LokiDatabaseConnection } from '@transportme/database'
+import StopsLoader from '../lib/loader/StopsLoader.mjs'
+import RouteLoader from '../lib/loader/RouteLoader.mjs'
+import TripLoader from '../lib/loader/TripLoader.mjs'
+import ShapeLoader from '../lib/loader/ShapeLoader.mjs'
+import { TRANSIT_MODES } from '../lib/constants.mjs'
+import suburbs from './sample-data/suburbs.json' with { type: 'json' }
+
+const __filename = url.fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+const routesFile = path.join(__dirname, 'sample-data', 'routes', 'metro_bus_routes.txt')
+const agencyFile = path.join(__dirname, 'sample-data', 'routes', 'agency.txt')
+
+const calendarFile = path.join(__dirname, 'sample-data', 'calendar', 'calendar.txt')
+const calendarDatesFile = path.join(__dirname, 'sample-data', 'calendar', 'calendar_dates.txt')
+
+const stopsFile = path.join(__dirname, 'sample-data', 'shapes', 'stops.txt')
+const stopTimesFile = path.join(__dirname, 'sample-data', 'shapes', 'stop_times.txt')
+const tripsFile = path.join(__dirname, 'sample-data', 'shapes', 'trips.txt')
+const shapeFile = path.join(__dirname, 'sample-data', 'shapes', 'shapes.txt')
+
+describe('The ShapeLoader class', () => {
+  it('Should process the shapes and append it to the route data', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let stops = await database.createCollection('stops')
+    let routes = await database.createCollection('routes')
+    let trips = await database.createCollection('gtfs timetables')
+
+    let stopLoader = new StopsLoader(stopsFile, suburbs, TRANSIT_MODES.metroBus, database)
+    await stopLoader.loadStops()
+
+    let routeLoader = new RouteLoader(routesFile, agencyFile, TRANSIT_MODES.metroBus, database)
+    await routeLoader.loadRoutes()
+
+    let routeIDMap = routeLoader.getRouteIDMap()
+
+    let tripLoader = new TripLoader({
+      tripsFile, stopTimesFile,
+      calendarFile, calendarDatesFile
+    }, TRANSIT_MODES.metroBus, database)
+   
+    await tripLoader.loadTrips({ routeIDMap })
+    let shapeIDMap = tripLoader.getShapeIDMap()
+
+    let shapeLoader = new ShapeLoader(shapeFile, database)
+
+    await shapeLoader.loadShapes({ shapeIDMap })
+
+    let route = await routes.findDocument({ routeGTFSID: '4-601' })
+
+    expect(route).to.exist
+    expect(route.routePath.length).to.equal(2)
+
+    let monashHuntingdale = route.routePath.find(path => path.fullGTFSIDs.includes('49-601-aus-1.3.H'))
+    expect(monashHuntingdale).to.exist
+    expect(monashHuntingdale.length).to.equal(2848.26)
+    expect(monashHuntingdale.path.coordinates.length).to.equal(38)
+
+    expect(monashHuntingdale.path.coordinates[0]).to.deep.equal([ 145.132081, -37.913926 ])
+  })
+
+  it('Should merge duplicate shapes with the same length and start/end coordinates', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let stops = await database.createCollection('stops')
+    let routes = await database.createCollection('routes')
+    let trips = await database.createCollection('gtfs timetables')
+
+    let stopLoader = new StopsLoader(stopsFile, suburbs, TRANSIT_MODES.metroBus, database)
+    await stopLoader.loadStops()
+
+    let routeLoader = new RouteLoader(routesFile, agencyFile, TRANSIT_MODES.metroBus, database)
+    await routeLoader.loadRoutes()
+
+    let routeIDMap = routeLoader.getRouteIDMap()
+
+    let tripLoader = new TripLoader({
+      tripsFile, stopTimesFile,
+      calendarFile, calendarDatesFile
+    }, TRANSIT_MODES.metroBus, database)
+
+    await tripLoader.loadTrips({ routeIDMap })
+    let shapeIDMap = tripLoader.getShapeIDMap()
+
+    let shapeLoader = new ShapeLoader(shapeFile, database)
+
+    await shapeLoader.loadShapes({ shapeIDMap })
+
+    let testTrip = await trips.findDocument({ tripID: '49-601--1-MF10-999' })
+    expect(testTrip.shapeID).to.equal('49-601-aus-1.4.H')
+
+    let route = await routes.findDocument({ routeGTFSID: testTrip.routeGTFSID })
+    expect(route.routeNumber).to.equal('601')
+
+    let tripShape = route.routePath.find(path => path.fullGTFSIDs.includes(testTrip.shapeID))
+    expect(tripShape).to.exist
+    expect(tripShape.fullGTFSIDs).to.have.members([
+      '49-601-aus-1.3.H',
+      '49-601-aus-1.4.H'
+    ])
+  })
+})

package/test/gtfs-shapes.test.mjs

@@ -0,0 +1,23 @@
+import { expect } from 'chai'
+import path from 'path'
+import url from 'url'
+import GTFSShapeReader from '../lib/gtfs-parser/readers/GTFSShapeReader.mjs'
+
+const __filename = url.fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+const shapeFile = path.join(__dirname, 'sample-data', 'shapes', 'shapes.txt')
+
+describe('The GTFSShapeReader class', () => {
+  it('Should read the trips.txt file and return a GTFSTrip object', async () => {
+    let reader = new GTFSShapeReader(shapeFile)
+    await reader.open()
+
+    let shape = await reader.getNextEntity()
+
+    expect(shape.shapeID).to.equal('49-601-aus-1.2.R')
+    expect(shape.points.length).to.equal(46)
+    expect(shape.shapeLength).to.equal(3069.43)
+    expect(shape.hash).to.equal(`3069.43-145.102663,-37.910905-145.131911,-37.913896`)
+  })
+})

package/test/gtfs-stop-loader-metro.test.mjs

@@ -0,0 +1,49 @@
+import { LokiDatabaseConnection } from '@transportme/database'
+import StopsLoader from '../lib/loader/StopsLoader.mjs'
+import { TRANSIT_MODES } from '../lib/constants.mjs'
+import path from 'path'
+import url from 'url'
+import { expect } from 'chai'
+import suburbs from './sample-data/suburbs.json' with { type: 'json' }
+
+const __filename = url.fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+const stopsFile = path.join(__dirname, 'sample-data', 'gtfs-splitting', 'nov_2024_train_data.txt')
+
+describe('The GTFS Stops Loader (Testing on the projected Nov 2024 data)', () => {
+  it('Should process the stops and add them to the database', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let stops = await database.createCollection('stops')
+
+    let loader = new StopsLoader(stopsFile, suburbs, TRANSIT_MODES.metroTrain, database)
+    await loader.loadStops()
+
+    let stop = await stops.findDocument({
+      'stopName': 'Camberwell Railway Station'
+    })
+
+    expect(stop).to.not.be.null
+  })
+
+  it('Should label the individual platforms with their platform numbers', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let stops = await database.createCollection('stops')
+
+    let loader = new StopsLoader(stopsFile, suburbs, TRANSIT_MODES.metroTrain, database)
+    await loader.loadStops()
+
+    let stop = await stops.findDocument({
+      'stopName': 'Camberwell Railway Station'
+    })
+
+    let platformIDs = ["11207", "11208", "11209"]
+    let platforms = platformIDs.map(id => stop.bays.find(bay => bay.stopGTFSID === id))
+
+    for (let i = 0; i < platformIDs.length; i++) {
+      let platform = platforms[i]
+      expect(platform).to.not.be.undefined
+      expect(platforms[i].platform).to.equal((i + 1).toString())
+    }
+  })
+})

package/test/gtfs-stop-loader.test.mjs

@@ -0,0 +1,223 @@
+import { LokiDatabaseConnection } from '@transportme/database'
+import StopsLoader from '../lib/loader/StopsLoader.mjs'
+import { TRANSIT_MODES } from '../lib/constants.mjs'
+import path from 'path'
+import url from 'url'
+import { expect } from 'chai'
+import suburbs from './sample-data/suburbs.json' with { type: 'json' }
+
+import uniqueStops from './sample-data/stop-data/unique-stops.json' with { type: 'json' }
+import nameOverrides from './sample-data/stop-data/name-overrides.json' with { type: 'json' }
+import GTFSStopsReader from '../lib/gtfs-parser/readers/GTFSStopsReader.mjs'
+
+const __filename = url.fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+const stopsFile = path.join(__dirname, 'sample-data', 'gtfs-splitting', 'sample_stops.txt')
+
+const uniqueStopsBus = path.join(__dirname, 'sample-data', 'gtfs-splitting', 'force_unique_stops_bus.txt')
+const uniqueStopsTram = path.join(__dirname, 'sample-data', 'gtfs-splitting', 'force_unique_stops_tram.txt')
+
+const stopNameOverrides = path.join(__dirname, 'sample-data', 'gtfs-splitting', 'name_overrides.txt')
+
+const specialHeader = path.join(__dirname, 'sample-data', 'gtfs-splitting', 'special_chars_stops_csv.txt')
+
+describe('The GTFS Stops Loader', () => {
+  it('Should process the stops and add them to the database', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let stops = await database.createCollection('stops')
+
+    let loader = new StopsLoader(stopsFile, suburbs, TRANSIT_MODES.bus, database)
+    await loader.loadStops()
+
+    let stop = await stops.findDocument({
+      'bays.stopGTFSID': '10011'
+    })
+    expect(stop).to.not.be.null
+    expect(stop.stopName).to.equal('Moffat Street/Main Road West')
+  })
+
+  it('Should merge street-level stops matching with the exact same name', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let stops = await database.createCollection('stops')
+
+    let loader = new StopsLoader(stopsFile, suburbs, TRANSIT_MODES.bus, database)
+    await loader.loadStops()
+
+    let wavWatson = await stops.findDocument({
+      'bays.stopGTFSID': '16074'
+    })
+
+    expect(wavWatson).to.not.be.null
+    expect(wavWatson.stopName).to.equal('Waverley Road/Watsons Road')
+    expect(wavWatson.bays.length).to.equal(2)
+    expect(wavWatson.suburb).to.deep.equal(['Glen Waverley'])
+    expect(wavWatson.cleanSuburbs).to.deep.equal(['glen-waverley'])
+
+    let wavWarrigal = await stops.findDocument({
+      'bays.stopGTFSID': '9379'
+    })
+    
+    expect(wavWarrigal).to.not.be.null
+    expect(wavWarrigal.stopName).to.equal('Waverley Road/Warrigal Road')
+    expect(wavWarrigal.bays.length).to.equal(2)
+  })
+
+  it('Should merge non street stops, and keep the secondary stop name if they are the same', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let stops = await database.createCollection('stops')
+
+    let loader = new StopsLoader(stopsFile, suburbs, TRANSIT_MODES.bus, database)
+    await loader.loadStops()
+
+    let stop = await stops.findDocument({
+      'bays.stopGTFSID': '19580'
+    })
+
+    expect(stop).to.not.be.null
+    expect(stop.bays.length).to.equal(3)
+    expect(stop.stopName).to.equal('Belgrave Railway Station/Belgrave-Gembrook Road')
+  })
+
+  it('Should merge non street stops, and discard the secondary stop name if they are different', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let stops = await database.createCollection('stops')
+
+    let loader = new StopsLoader(stopsFile, suburbs, TRANSIT_MODES.bus, database)
+    await loader.loadStops()
+
+    let stop = await stops.findDocument({
+      'bays.stopGTFSID': '28191'
+    })
+
+    expect(stop).to.not.be.null
+    expect(stop.bays.length).to.equal(3)
+    expect(stop.stopName).to.equal('Lilydale Railway Station')
+  })
+
+  it('Should update the suburb list as well', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let stops = await database.createCollection('stops')
+
+    let loader = new StopsLoader(stopsFile, suburbs, TRANSIT_MODES.bus, database)
+    await loader.loadStops()
+
+    let stop = await stops.findDocument({
+      'bays.stopGTFSID': '51586'
+    })
+
+    expect(stop).to.not.be.null
+    expect(stop.bays.length).to.equal(2)
+    expect(stop.stopName).to.equal('Huntingdale Railway Station')
+    expect(stop.suburb).to.deep.equal(['Huntingdale', 'Oakleigh'])
+  })
+
+  it('Should not merge stops that are forced to be unique and would otherwise be merged', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let stops = await database.createCollection('stops')
+
+    function getMergeName(stop) {
+      if (uniqueStops.includes(stop.fullStopName)) return stop.fullStopName
+    }
+
+    await (new StopsLoader(uniqueStopsBus, suburbs, TRANSIT_MODES.bus, database)).loadStops({ getMergeName })
+    await (new StopsLoader(uniqueStopsTram, suburbs, 'tram', database)).loadStops({ getMergeName })
+
+    let melbRoyal = await stops.findDocument({
+      'bays.stopGTFSID': '16830'
+    })
+
+    expect(melbRoyal).to.not.be.null
+    expect(melbRoyal.bays.length).to.equal(1)
+    expect(melbRoyal.stopName).to.equal('Melbourne University/Royal Parade')
+
+    let melbGrattan = await stops.findDocument({
+      'bays.stopGTFSID': '28670'
+    })
+
+    expect(melbGrattan).to.not.be.null
+    expect(melbGrattan.bays.length).to.equal(4)
+    expect(melbGrattan.stopName).to.equal('Melbourne University/Grattan Street')
+
+    let melbSwanston = await stops.findDocument({
+      'bays.stopGTFSID': '87'
+    })
+
+    expect(melbSwanston).to.not.be.null
+    expect(melbSwanston.bays.length).to.equal(3)
+    expect(melbSwanston.stopName).to.equal('Melbourne University/Swanston Street')
+  })
+
+  it('Should override stop names if required', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let stops = await database.createCollection('stops')
+
+    await (new StopsLoader(stopNameOverrides, suburbs, TRANSIT_MODES.bus, database)).loadStops({
+      processStop: stop => {
+        let updatedName = nameOverrides[stop.fullStopName]
+        if (updatedName) stop.fullStopName = updatedName
+
+        return stop
+      }
+    })
+
+    let ballarat = await stops.findDocument({
+      'bays.stopGTFSID': '49020'
+    })
+
+    expect(ballarat).to.not.be.null
+    expect(ballarat.bays.length).to.equal(3)
+    expect(ballarat.stopName).to.equal('Ballarat Railway Station')
+    expect(ballarat.bays[0].originalName).to.equal('Ballarat Bus Interchange (Soldiers Hill)')
+  })
+
+  it('Should handle CSV blank spaces and other unicode junk', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let stops = await database.createCollection('stops')
+
+    await (new StopsLoader(specialHeader, suburbs, TRANSIT_MODES.bus, database)).loadStops()
+
+    let gumRd = await stops.findDocument({
+      'bays.stopGTFSID': '10009'
+    })
+
+    expect(gumRd).to.not.be.null
+    expect(gumRd.stopName).to.equal('Gum Road/Main Road West')
+  })
+
+  it('Should allow for individual stops to be loaded in', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let stops = await database.createCollection('stops')
+
+    let loader = new StopsLoader(specialHeader, suburbs, TRANSIT_MODES.bus, database)
+    let reader = new GTFSStopsReader('')
+
+    await loader.loadStop(reader.processEntity({
+      stop_id: '51586',
+      stop_name: 'Huntingdale Station/Haughton Road (Oakleigh)',
+      stop_lat: '-37.9111998619901',
+      stop_lon: '145.103064853799'
+    }))
+
+    let huntingdale = await stops.findDocument({
+      'bays.stopGTFSID': '51586'
+    })
+
+    expect(huntingdale).to.not.be.null
+    expect(huntingdale.stopName).to.equal('Huntingdale Railway Station/Haughton Road')
+
+    await loader.loadStop(reader.processEntity({
+      stop_id: '22440',
+      stop_name: 'Huntingdale Station/Huntingdale Road (Oakleigh)',
+      stop_lat: '-37.911010298723',
+      stop_lon: '145.103604328761'
+    }))
+
+    huntingdale = await stops.findDocument({
+      'bays.stopGTFSID': '22440'
+    })
+
+    expect(huntingdale).to.not.be.null
+    expect(huntingdale.stopName).to.equal('Huntingdale Railway Station')
+  })
+})

package/test/gtfs-stop-times.test.mjs

@@ -0,0 +1,37 @@
+import { expect } from 'chai'
+import path from 'path'
+import url from 'url'
+import GTFSStopTimesReader from '../lib/gtfs-parser/readers/GTFSStopTimesReader.mjs'
+
+const __filename = url.fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+const stopTimesFile = path.join(__dirname, 'sample-data', 'trips', 'stop_times.txt')
+
+describe('The GTFSStopTimesReader class', () => {
+  it('Should read the stop times one trips worth at a time', async () => {
+    let reader = new GTFSStopTimesReader(stopTimesFile)
+
+    await reader.open()
+
+    let stopTimes = await reader.getNextEntity()
+    expect(stopTimes.tripID).to.equal('1.T2.2-ALM-vpt-1.1.R')
+    expect(stopTimes.stops.length).to.equal(7)
+    expect(stopTimes.stops[0].stopID).to.equal('19847')
+    expect(stopTimes.stops[0].departureTime).to.equal('04:57')
+
+    expect(stopTimes.stops[6].stopID).to.equal('19853')
+    expect(stopTimes.stops[6].departureTime).to.equal('05:08')
+
+    await reader.getNextEntity()
+    await reader.getNextEntity()
+
+    let lastTrip = await reader.getNextEntity()
+    expect(lastTrip.tripID).to.equal('101.T2.2-ALM-vpt-1.2.H')
+    expect(lastTrip.stops.length).to.equal(7)
+    expect(lastTrip.stops[0].stopID).to.equal('19853')
+    expect(lastTrip.stops[0].departureTime).to.equal('23:31')
+    
+    expect(reader.available()).to.be.false
+  })
+})