@transportme/load-ptv-gtfs 1.0.0

package/lib/post-processing/set-route-stops.mjs

@@ -0,0 +1,73 @@
+import async, { dir } from 'async'
+import merge from './merge-stops.mjs'
+
+export default async function setRouteStops(database) {
+  let gtfsTimetables = database.getCollection('gtfs timetables')
+  let routes = database.getCollection('routes')
+
+  let allRoutes = await routes.distinct('routeGTFSID')
+
+  let stopsByService = {}
+
+  await async.forEachLimit(allRoutes, 100, async routeGTFSID => {
+    let routeData = await routes.findDocument({ routeGTFSID })
+    let routeVariants = routeData.routePath.map(variant => ({ shapeID: variant.fullGTFSIDs[0], routeGTFSID }))
+    let routeDirections = [[], []]
+
+    for (let variant of routeVariants) {
+      let timetable = await gtfsTimetables.findDocument(variant)
+      if (!timetable) {
+        console.log('No timetable match for shapeID', variant)
+        continue
+      }
+
+      routeDirections[timetable.gtfsDirection].push(timetable.stopTimings.map(stop => ({
+        stopName: stop.stopName,
+        stopNumber: stop.stopNumber,
+        suburb: stop.suburb,
+        stopGTFSID: stop.stopGTFSID
+      })))
+    }
+
+    for (let gtfsDirection = 0; gtfsDirection < routeDirections.length; gtfsDirection++) {
+      let routeDirection = routeDirections[gtfsDirection]
+      if (!routeDirection.length) continue
+
+      let directionStops = merge(routeDirection, (a, b) => a.stopName === b.stopName)
+      let mostCommonDestinations = (await gtfsTimetables.aggregate([
+        { $match: { routeGTFSID, gtfsDirection } },
+        { $sortByCount: '$destination' }
+      ]).toArray()).map(e => e._id)
+
+      let lastStop = directionStops[directionStops.length - 1]
+      let lastStopName = lastStop.stopName
+
+      let directionName = lastStopName
+      if (directionName.includes('School') || directionName.includes('College')) {
+        directionName = mostCommonDestinations[0]
+      }
+
+      if (!stopsByService[routeGTFSID]) stopsByService[routeGTFSID] = []
+      stopsByService[routeGTFSID][gtfsDirection] = {
+        directionName,
+        gtfsDirection,
+        stops: directionStops
+      }
+    }
+  })
+
+  let bulkOperations = []
+  
+  Object.keys(stopsByService).forEach(routeGTFSID => {
+    bulkOperations.push({
+      updateOne: {
+        filter: { routeGTFSID },
+        update: { $set: {
+          directions: stopsByService[routeGTFSID].filter(Boolean) // Apparently some routes dont have GTFS Dir 0
+        } }
+      }
+    })
+  })
+
+  await routes.bulkWrite(bulkOperations)
+}

package/lib/post-processing/set-stop-services.mjs

@@ -0,0 +1,64 @@
+import async from 'async'
+
+const AGGREGATION_GROUP = {
+  $group: {
+    _id: {
+      routeGTFSID: "$routeGTFSID",
+      gtfsDirection: "$gtfsDirection",
+      routeNumber: "$routeNumber"
+    }
+  }
+}
+
+function compareService(a, b) {
+  return a.routeGTFSID.localeCompare(b.routeGTFSID) || a.gtfsDirection - b.gtfsDirection
+}
+
+export default async function setStopServices(database, onMilestone = () => {}) {
+  let gtfsTimetables = database.getCollection('gtfs timetables')
+  let stops = database.getCollection('stops')
+
+  let stopIDs = await stops.distinct('_id')
+
+  let total = stopIDs.length
+
+  let count = 0
+  await async.forEachLimit(stopIDs, 100, async id => {
+    let stop = await stops.findDocument({ _id: id })
+
+    await async.forEach(stop.bays, async bay => {
+      let screenServices = await gtfsTimetables.aggregate([
+        {
+          $match: {
+            mode: bay.mode,
+            stopTimings: {
+              $elemMatch: { stopGTFSID: bay.stopGTFSID, 'stopConditions.pickup': 0 }
+            }
+          }
+        },
+        AGGREGATION_GROUP
+      ]).toArray()
+
+      let services = await gtfsTimetables.aggregate([
+        {
+          $match: {
+            mode: bay.mode,
+            'stopTimings.stopGTFSID': bay.stopGTFSID
+          }
+        },
+        AGGREGATION_GROUP
+      ]).toArray()
+
+      bay.services = services.map(e => e._id).sort(compareService)
+      bay.screenServices = screenServices.map(e => e._id).sort(compareService)
+    })
+
+    await stops.updateDocument({ _id: id }, {
+      $set: {
+        bays: stop.bays
+      }
+    })
+
+    if (++count % 500 === 0) onMilestone(count, total)
+  })
+}

package/lib/utils.mjs

@@ -0,0 +1,16 @@
+import { DateTime } from 'luxon'
+import { TIMEZONES } from './constants.mjs'
+
+// Note: Need to allow for SA time as PTV does not handle cross border times well... or at all
+export function parseDate(date) {
+  return DateTime.fromFormat(date, 'yyyyMMdd', { zone: TIMEZONES.melbourne })
+}
+
+export function toGTFSDate(date) {
+  return date.toFormat('yyyyMMdd')
+}
+
+export function hhmmToMinutesPastMidnight(date) {
+  let [ hours, minutes ] = date.split(':')
+  return parseInt(minutes) + parseInt(hours) * 60
+}

package/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@transportme/load-ptv-gtfs",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "mocha"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@transportme/database": "^1.0.2",
+    "@transportme/transportvic-utils": "^1.0.0",
+    "@turf/turf": "^7.0.0",
+    "adm-zip": "^0.5.14",
+    "async": "^3.2.6",
+    "line-reader": "^0.4.0",
+    "luxon": "^3.5.0",
+    "node-fetch": "^3.3.2"
+  },
+  "devDependencies": {
+    "chai": "^5.1.0",
+    "mocha": "^10.3.0",
+    "nock": "^14.0.0-beta.5",
+    "tmp-promise": "^3.0.3"
+  }
+}

package/test/gtfs-calendar.test.mjs

@@ -0,0 +1,182 @@
+import { expect } from 'chai'
+import GTFSCalendarReader from '../lib/gtfs-parser/readers/GTFSCalendarReader.mjs'
+import GTFSCalendarDateReader from '../lib/gtfs-parser/readers/GTFSCalendarDatesReader.mjs'
+import GTFSCalendar from '../lib/gtfs-parser/GTFSCalendar.mjs'
+import GTFSCalendarDate from '../lib/gtfs-parser/GTFSCalendarDate.mjs'
+import path from 'path'
+import url from 'url'
+import { CALENDAR_DATES } from '../lib/constants.mjs'
+import { toGTFSDate } from '../lib/utils.mjs'
+
+const __filename = url.fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+const calendarFile = path.join(__dirname, 'sample-data', 'calendar', 'calendar.txt')
+const calendarDatesFile = path.join(__dirname, 'sample-data', 'calendar', 'calendar_dates.txt')
+
+describe('The GTFSCalendar class', () => {
+  it('Should process the calendar data to a list of operation days', () => {
+    expect(new GTFSCalendar({
+      id: 'T1',
+      startDay: '20241122',
+      endDay: '20241129',
+      daysOfWeek: ["0","0","0","0","1","1","0"]
+    }).getOperationDays()).to.have.members([
+      "20241122", "20241123", "20241129"
+    ])
+
+    expect(new GTFSCalendar({
+      id: 'T2',
+      startDay: '20241123',
+      endDay: '20241130',
+      daysOfWeek: ["1","1","1","1","1","0","0"]
+    }).getOperationDays()).to.have.members([
+      "20241125", "20241126", "20241127", "20241128", "20241129"
+    ])
+  })
+
+  it('Be inclusive of the end date', () => {
+    expect(new GTFSCalendar({
+      id: 'T1',
+      startDay: '20241122',
+      endDay: '20241122',
+      daysOfWeek: ["1","1","1","1","1","1","1"]
+    }).getOperationDays()).to.have.members([
+      "20241122"
+    ])
+  })
+
+  it('Should allow adding of an extra date into the calendar', () => {
+    let cal = new GTFSCalendar({
+      id: 'T1',
+      startDay: '20241122',
+      endDay: '20241122',
+      daysOfWeek: ["1","1","1","1","1","1","1"]
+    }, [new GTFSCalendarDate({
+      id: 'T1',
+      date: '20241125',
+      type: CALENDAR_DATES.ADDED
+    })])
+    
+    expect(cal.getOperationDays()).to.have.members([
+      "20241122", "20241125"
+    ])
+  })
+
+  it('Should allow removing of a specific date from the calendar', () => {
+    let cal = new GTFSCalendar({
+      id: 'T1',
+      startDay: '20241224',
+      endDay: '20241226',
+      daysOfWeek: ["1","1","1","1","1","1","1"]
+    }, [new GTFSCalendarDate({
+      id: 'T1',
+      date: '20241225',
+      type: CALENDAR_DATES.REMOVED
+    })])
+
+    expect(cal.getOperationDays()).to.deep.equal([
+      "20241224", "20241226"
+    ])
+  })
+
+  it('Should ensure that adding an extra date into the calendar does not introduce duplicates', () => {
+    let cal = new GTFSCalendar({
+      id: 'T1',
+      startDay: '20241122',
+      endDay: '20241122',
+      daysOfWeek: ["1","1","1","1","1","1","1"]
+    }, [new GTFSCalendarDate({
+      id: 'T1',
+      date: '20241122',
+      type: CALENDAR_DATES.ADDED
+    })])
+
+    expect(cal.getOperationDays()).to.deep.equal([
+      "20241122"
+    ])
+  })
+
+  it('Should ensure that duplicate extra dates do not introduce duplicates', () => {
+    let calendarDate = new GTFSCalendarDate({
+      id: 'T1',
+      date: '20241122',
+      type: CALENDAR_DATES.ADDED
+    })
+
+    let cal = new GTFSCalendar({
+      id: 'T1',
+      startDay: '20241122',
+      endDay: '20241122',
+      daysOfWeek: ["1","1","1","1","1","1","1"]
+    }, [calendarDate, calendarDate, calendarDate])
+
+    expect(cal.getOperationDays()).to.deep.equal([
+      "20241122"
+    ])
+  })
+
+  it('Should ensure that the date exists when removing a specific date from the calendar, and not do anything otherwise', () => {
+    let cal = new GTFSCalendar({
+      id: 'T1',
+      startDay: '20241224',
+      endDay: '20241226',
+      daysOfWeek: ["1","1","1","1","1","1","1"]
+    }, [new GTFSCalendarDate({
+      id: 'T1',
+      date: '20241227',
+      type: CALENDAR_DATES.REMOVED
+    })])
+
+    expect(cal.getOperationDays()).to.deep.equal([
+      "20241224", "20241225", "20241226"
+    ])
+  })
+})
+
+describe('The GTFSCalendarDate class', () => {
+  it('Should convert the CSV data into a proper date object', () => {
+    expect(new GTFSCalendarDate({
+      id: 'T6',
+      date: '20241125',
+      type: CALENDAR_DATES.ADDED
+    }).date.toUTC().toISO()).to.equal('2024-11-24T13:00:00.000Z')
+  })
+})
+
+describe('The GTFSCalendarDatesReader class', () => {
+  it('Should read a calendar dates and return a GTFSCalendarDate object', async () => {
+    let reader = new GTFSCalendarDateReader(calendarDatesFile)
+    await reader.open()
+
+    let date = await reader.getNextEntity()
+    expect(date.id).to.equal('T3_2')
+    expect(toGTFSDate(date.date)).to.equal('20250126')
+    expect(date.type).to.equal(CALENDAR_DATES.REMOVED)
+
+    let date2 = await reader.getNextEntity()
+    expect(date2.id).to.equal('T5_2')
+    expect(toGTFSDate(date2.date)).to.equal('20241225')
+    expect(date2.type).to.equal(CALENDAR_DATES.REMOVED)
+  })
+})
+
+describe('The GTFSCalendarReader class', () => {
+  it('Should read both a calendar dates and calendar file and return a GTFSCalendar object with the appropriate exceptions filled', async () => {
+    let reader = new GTFSCalendarReader(calendarFile, calendarDatesFile)
+    await reader.open()
+
+    let allCalendars = {}
+    while (reader.available()) {
+      let calendar = await reader.getNextEntity()
+      allCalendars[calendar.id] = calendar
+    }
+
+    expect(allCalendars['T2'].getOperationDays()).to.deep.equal(["20241109"])
+
+    expect(allCalendars['T3_2'].getOperationDays()).to.contain("20241124")
+    expect(allCalendars['T3_2'].getOperationDays()).to.contain("20241201")
+    expect(allCalendars['T3_2'].getOperationDays()).to.contain("20250209")
+    expect(allCalendars['T3_2'].getOperationDays()).to.not.contain("20250126")
+  })
+})

package/test/gtfs-route-loader.test.mjs

@@ -0,0 +1,190 @@
+import { LokiDatabaseConnection } from '@transportme/database'
+import RouteLoader from '../lib/loader/RouteLoader.mjs'
+import GTFSAgencyReader from '../lib/gtfs-parser/readers/GTFSAgencyReader.mjs'
+
+import { TRANSIT_MODES } from '../lib/constants.mjs'
+
+import path from 'path'
+import url from 'url'
+import { expect } from 'chai'
+
+const __filename = url.fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+const regionalRoutesFile = path.join(__dirname, 'sample-data', 'routes', 'regional_bus_routes.txt')
+const metroRoutesFile = path.join(__dirname, 'sample-data', 'routes', 'metro_bus_routes.txt')
+const metroLinesFile = path.join(__dirname, 'sample-data', 'routes', 'metro_lines.txt')
+const agencyFile = path.join(__dirname, 'sample-data', 'routes', 'agency.txt')
+const wgtFile = path.join(__dirname, 'sample-data', 'routes', 'west_gipp_transit.txt')
+const vlineFile = path.join(__dirname, 'sample-data', 'routes', 'vline.txt')
+
+describe('The GTFS Agency Reader', () => {
+  it('Should read the agencies one line at a time', async () => {
+    let reader = new GTFSAgencyReader(agencyFile)
+    await reader.open()
+    let operator = await reader.getNextEntity()
+
+    expect(operator.id).to.equal('43')
+    expect(operator.name).to.equal('CDC Ballarat')
+    expect(operator.website).to.equal('https://www.ptv.vic.gov.au/footer/customer-service/operator-contact-details/?utm_source=open_data_click_agency&utm_medium=open_data_agency_click&utm_campaign=open_data_click')
+    expect(operator.phone).to.equal('1800 800 007')
+  })
+})
+
+describe('The GTFS Routes Loader', () => {
+  it('Should process the routes and add them to the database', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let routes = await database.createCollection('routes')
+
+    let loader = new RouteLoader(regionalRoutesFile, agencyFile, TRANSIT_MODES.bus, database)
+    await loader.loadRoutes()
+
+    let bal10 = await routes.findDocument({
+      routeGTFSID: '6-10x'
+    })
+
+    expect(bal10).to.not.be.null
+    expect(bal10.routeNumber).to.equal('10')
+    expect(bal10.operators).to.deep.equal(['CDC Ballarat'])
+  })
+
+  it('Should open the agency.txt file and load all operators', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    await database.createCollection('routes')
+
+    let loader = new RouteLoader(regionalRoutesFile, agencyFile, TRANSIT_MODES.bus, database)
+    await loader.loadAgencies()
+    
+    let paynesville = loader.getOperator('99')
+
+    expect(paynesville).to.not.be.undefined
+    expect(paynesville.name).to.equal('Paynesville Bus Lines')
+  })
+
+  it('Should handle smartrak routes', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let routes = await database.createCollection('routes')
+
+    let loader = new RouteLoader(metroRoutesFile, agencyFile, TRANSIT_MODES.bus, database)
+    await loader.loadRoutes()
+
+    let smart900 = await routes.findDocument({
+      routeGTFSID: '4-900'
+    })
+
+    expect(smart900).to.not.be.null
+    expect(smart900.mode).to.equal(TRANSIT_MODES.bus)
+    expect(smart900.routeNumber).to.equal('900')
+    expect(smart900.operators).to.have.members(['CDC', 'Ventura Bus Lines'])
+  })
+
+  it('Should handle metro train routes', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let routes = await database.createCollection('routes')
+
+    let loader = new RouteLoader(metroLinesFile, agencyFile, TRANSIT_MODES.metroTrain, database)
+    await loader.loadRoutes()
+
+    let stony = await routes.findDocument({
+      routeGTFSID: '2-STY'
+    })
+
+    expect(stony).to.not.be.null
+    expect(stony.mode).to.equal('metro train')
+    expect(stony.routeNumber).to.be.null
+    expect(stony.routeName).to.equal('Stony Point')
+    expect(stony.operators).to.have.members(['Metro'])
+  })
+
+  it('Should allow for custom route rewriting', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let routes = await database.createCollection('routes')
+
+    let loader = new RouteLoader(wgtFile, agencyFile, TRANSIT_MODES.bus, database)
+    await loader.loadRoutes({
+      processRoute: route => {
+        if (route.routeGTFSID.match(/6-w\d\d/)) {
+          route.routeGTFSID = '6-WGT'
+          route.routeName = 'West Gippsland Transit'
+        }
+
+        return route
+      }
+    })
+
+    let wgt = await routes.findDocument({
+      routeGTFSID: '6-WGT'
+    })
+
+    expect(wgt, 'Expected West Gippsland Transit route 6-WGT to be created').to.not.be.null
+    expect(wgt.mode).to.equal('bus')
+    expect(wgt.routeNumber).to.be.null
+    expect(wgt.routeName).to.equal('West Gippsland Transit')
+    expect(wgt.operators).to.have.members(['Warragul Bus Lines'])
+
+    expect(await routes.findDocument({
+      routeGTFSID: '6-w40'
+    }), 'Expected 6-w40 Moe - Garfield to not exist').to.be.null
+
+    expect(await routes.findDocument({
+      routeGTFSID: '6-w41'
+    }), 'Expected 6-w41 Warragul - Pakenham to not exist').to.be.null
+
+    let nojee = await routes.findDocument({
+      routeGTFSID: '6-W89'
+    })
+    expect(nojee, 'Expected 6-W89 Warragul - Nojee to exist and not be modified').to.not.be.null
+    expect(nojee.mode).to.equal('bus')
+    expect(nojee.routeNumber).to.equal('89')
+    expect(nojee.routeName).to.equal('Noojee - Warragul Station Via Buln Buln')
+    expect(nojee.operators).to.have.members(['Warragul Bus Lines'])
+  })
+
+
+  it('Should return a route ID map', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let routes = await database.createCollection('routes')
+
+    let loader = new RouteLoader(wgtFile, agencyFile, TRANSIT_MODES.bus, database)
+    await loader.loadRoutes({
+      processRoute: route => {
+        if (route.routeGTFSID.match(/6-w\d\d/)) {
+          route.routeGTFSID = '6-WGT'
+          route.routeName = 'West Gippsland Transit'
+        }
+
+        return route
+      }
+    })
+
+    let map = loader.getRouteIDMap()
+    expect(map['6-w19-mjp-1']).to.equal('6-WGT')
+    expect(map['6-W89-mjp-1']).to.equal('6-W89')
+  })
+
+  it('Should allow for dropping of unwanted routes', async () => {
+    let database = new LokiDatabaseConnection('test-db')
+    let routes = await database.createCollection('routes')
+
+    let loader = new RouteLoader(vlineFile, agencyFile, TRANSIT_MODES.regionalTrain, database)
+    await loader.loadRoutes({
+      processRoute: route => {
+        if (route.routeGTFSID === '1-vPK') return null
+        return route
+      }
+    })
+
+    expect(await routes.findDocument({
+      routeGTFSID: '1-vPK'
+    }), 'Expected 1-vPK V/Line Dandenong Metro Services to not exist').to.be.null
+
+    let traralgon = await routes.findDocument({
+      routeGTFSID: '1-TRN'
+    })
+    expect(traralgon, 'Expected 1-TRN Warragul - Nojee to exist and not be modified').to.not.be.null
+    expect(traralgon.mode).to.equal('regional train')
+    expect(traralgon.routeNumber).to.be.null
+    expect(traralgon.routeName).to.equal('Traralgon - Melbourne Via Pakenham, Moe & Morwell')
+    expect(traralgon.operators).to.have.members(['V/Line'])
+  })
+})

package/test/gtfs-routes.test.mjs

@@ -0,0 +1,77 @@
+import { expect } from 'chai'
+import GTFSRouteReader from '../lib/gtfs-parser/readers/GTFSRouteReader.mjs'
+import { TRANSIT_MODES } from '../lib/constants.mjs'
+import path from 'path'
+import url from 'url'
+
+const __filename = url.fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+const regionalRoutesFile = path.join(__dirname, 'sample-data', 'routes', 'regional_bus_routes.txt')
+const metroRoutesFile = path.join(__dirname, 'sample-data', 'routes', 'metro_bus_routes.txt')
+
+let regionalBusRoute = {
+  route_id: '6-10x-mjp-1',
+  agency_id: '43',
+  route_short_name: '10',
+  route_long_name: 'Alfredton - Ballarat Station Via Wendouree'
+}
+
+describe('The GTFSRouteReader class', () => {
+  describe('The processRoute function', () => {
+    it('Should take in the raw CSV line and process it', () => {
+      let routeData = new GTFSRouteReader('', TRANSIT_MODES.bus).processEntity(regionalBusRoute)
+
+      expect(routeData.routeGTFSID).to.equal('6-10x')
+      expect(routeData.agencyID).to.equal(regionalBusRoute.agency_id)
+      expect(routeData.routeNumber).to.equal(regionalBusRoute.route_short_name)
+      expect(routeData.routeName).to.equal(regionalBusRoute.route_long_name)
+    })
+  })
+
+  describe('The getNextEntity function', () => {
+    it('Should read the CSV file and return the next stop', async () => {
+      let routeReader = new GTFSRouteReader(regionalRoutesFile, TRANSIT_MODES.bus)
+      await routeReader.open()
+
+      await routeReader.getNextEntity()
+      let routeData = await routeReader.getNextEntity()
+
+      expect(routeData.agencyID).to.equal('99')
+      expect(routeData.routeNumber).to.equal('13')
+      expect(routeData.routeName).to.equal('Paynesville - Bairnsdale Via Paynesville')
+    })
+  })
+})
+
+describe('The GTFSRoute class', () => {
+  describe('The route ID parsing', () => {
+    it('Should parse a regular route ID and remove the -mjp- portion', async () => {
+      let routeReader = new GTFSRouteReader(regionalRoutesFile, TRANSIT_MODES.bus)
+      await routeReader.open()
+
+      let routeData = await routeReader.getNextEntity()
+      
+      expect(routeData.routeGTFSID).to.equal('6-10x')
+    })
+
+    it('Should parse a regular route ID and pad with 0s as needed', async () => {
+      let routeReader = new GTFSRouteReader(regionalRoutesFile, TRANSIT_MODES.bus)
+      await routeReader.open()
+
+      await routeReader.getNextEntity()
+      let routeData = await routeReader.getNextEntity()
+      
+      expect(routeData.routeGTFSID).to.equal('6-013')
+    })
+
+    it('Should detect a smartrak route ID and convert to a legacy 4-XXX trip', async () => {
+      let routeReader = new GTFSRouteReader(metroRoutesFile, TRANSIT_MODES.bus)
+      await routeReader.open()
+
+      expect((await routeReader.getNextEntity()).routeGTFSID).to.equal('4-900')
+      expect((await routeReader.getNextEntity()).routeGTFSID).to.equal('4-900')
+      expect((await routeReader.getNextEntity()).routeGTFSID).to.equal('4-601')
+    })
+  })
+})