@transportme/load-ptv-gtfs 1.0.0

package/.github/workflows/unit-test.yml

@@ -0,0 +1,46 @@
+# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-nodejs
+
+name: Node.js CI
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+  
+    - name: Cache node modules
+      id: cache-npm
+      uses: actions/cache@v4
+      env:
+        cache-name: cache-node-modules
+      with:
+        # npm cache files are stored in `~/.npm` on Linux/macOS
+        path: ~/.npm
+        key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-build-${{ env.cache-name }}-
+          ${{ runner.os }}-build-
+          ${{ runner.os }}-
+    
+    - name: Use Node.js 20.x
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20.x'
+        cache: 'npm'
+        
+    - name: Install dependencies
+      run: npm install
+
+    - name: Run test
+      run: npm test

package/lib/constants.mjs

@@ -0,0 +1,18 @@
+import { GTFS_CONSTANTS } from '@transportme/transportvic-utils'
+const { TRANSIT_MODES, GTFS_MODES } = GTFS_CONSTANTS
+
+const TIMEZONES = {
+  melbourne: 'Australia/Melbourne'
+}
+
+const CALENDAR_DATES = {
+  ADDED: '1',
+  REMOVED: '2'
+}
+
+export {
+  TRANSIT_MODES,
+  TIMEZONES,
+  CALENDAR_DATES,
+  GTFS_MODES
+}

package/lib/gtfs-package/PTVGTFS.mjs

@@ -0,0 +1,96 @@
+import fetch from 'node-fetch'
+import AdmZip from 'adm-zip'
+import { pipeline } from 'stream/promises'
+import path from 'path'
+import { createWriteStream } from 'fs'
+import { unlink } from 'fs/promises'
+
+export default class PTVGTFS {
+
+  #url
+  #destinationFolder
+  #gtfsFile
+  #headResponse
+  
+  /**
+   * Constructs a new instance of a PTVGTFS class representing a GTFS release.
+   * 
+   * @param {string} url The URL pointing to the GTFS resource
+   */
+  constructor(url) {
+    this.#url = url
+  }
+
+  /**
+   * Gets when the GTFS file was released.
+   * 
+   * @returns {Promise<Date>} The date/time at which the GTFS file was released.
+   */
+  async getPublishedDate() {
+    this.#headResponse = await fetch(this.#url, {
+      method: 'HEAD'
+    })
+
+    let lastModified = this.#headResponse.headers.get('last-modified')
+    return new Date(lastModified)
+  }
+
+  /**
+   * Downloads the PTV GTFS file to the specified folder. It will be given the name `gtfs.zip`. 
+   * 
+   * @param {string} destinationFolder The folder to save the GTFS archive to.
+   */
+  async download(destinationFolder) {
+    this.#destinationFolder = destinationFolder
+    let response = await fetch(this.#url)
+
+    this.#gtfsFile = path.join(destinationFolder, 'gtfs.zip')
+    let outputStream = createWriteStream(this.#gtfsFile)
+
+    await pipeline(response.body, outputStream)
+  }
+
+  #getModeFolder(mode) {
+    return path.join(this.#destinationFolder, mode)
+  }
+
+  #getModeFile(mode) {
+    let modeFolder = this.#getModeFolder(mode)
+    return path.join(modeFolder, 'google_transit.zip')
+  }
+
+  async #unzipMode(mode) {
+    let modeFolder = this.#getModeFolder(mode)
+    let modeFile = this.#getModeFile(mode)
+    let zip = new AdmZip(modeFile)
+
+    await new Promise(resolve => zip.extractAllToAsync(modeFolder, true, resolve))
+  }
+
+  /**
+   * Unzips the PTV GTFS archive into its individual transit mode folders, then unzips the invididual mode archives as well.
+   */
+  async unzip() {
+    if (!this.#destinationFolder) throw new Error('Need to download GTFS file first!')
+
+    let zip = new AdmZip(this.#gtfsFile)
+    await new Promise(resolve => zip.extractAllToAsync(this.#destinationFolder, true, resolve))
+
+    for (let mode = 1; mode <= 11; mode++) {
+      if (mode === 9) continue
+      await this.#unzipMode(mode.toString())
+    }
+  }
+
+  /**
+   * Cleans up the zip files only.
+   */
+  async cleanup() {
+    await unlink(this.#gtfsFile)
+
+    for (let mode = 1; mode <= 11; mode++) {
+      if (mode === 9) continue
+      await unlink(this.#getModeFile(mode.toString()))
+    }
+  }
+}

package/lib/gtfs-parser/GTFSAgency.mjs

@@ -0,0 +1,15 @@
+export default class GTFSAgency {
+
+  id
+  name
+  website
+  phone
+
+  constructor({ id, name, website, phone }) {
+    this.id = id
+    this.name = name
+    this.website = website
+    this.phone = phone
+  }
+
+}

package/lib/gtfs-parser/GTFSCalendar.mjs

@@ -0,0 +1,63 @@
+import { CALENDAR_DATES } from '../constants.mjs'
+import { parseDate, toGTFSDate } from '../utils.mjs'
+import GTFSCalendarDate from './GTFSCalendarDate.mjs'
+
+export default class GTFSCalendar {
+
+  id
+  daysOfWeek
+  startDay
+  endDay
+
+  #calendarRemovals = []
+  #operationDays = []
+
+  /**
+   * Constructor to create a new GTFS Calendar object.
+   * 
+   * Note that `startDay` and `endDay` are both inclusive.
+   *  
+   * @param {Object} param0 The data as provided by the calendar.txt CSV entry
+   * @param {string} param0.id The calendar ID
+   * @param {string[]} param0.daysOfWeek An array containing the days of the week. This should be 1 to include the day in the interval and 0 to exclude it. Note that it starts on a Monday.
+   * @param {string} param0.startDay The start day of the calendar interval (inclusive).
+   * @param {string} param0.endDay The end day of the calendar interval (inclusive).
+   * @param {GTFSCalendarDate[]} [calendarDates=[]] A list of calendar date exceptions
+   */
+  constructor({ id, daysOfWeek, startDay, endDay }, calendarDates = []) {
+    this.id = id
+    this.daysOfWeek = daysOfWeek.map(day => day === '1')
+    this.startDay = parseDate(startDay)
+    this.endDay = parseDate(endDay)
+
+    for (let date of calendarDates) {
+      if (date.type === CALENDAR_DATES.ADDED) this.#operationDays.push(date.date)
+      else if (date.type === CALENDAR_DATES.REMOVED) this.#calendarRemovals.push(date.date)
+    }
+
+    this.#findOperationDays()
+  }
+
+  #findOperationDays() {
+    let intervalSize = this.endDay.diff(this.startDay, 'day').get('day')
+
+    for (let offset = 0; offset <= intervalSize; offset++) {
+      let intervalDay = this.startDay.plus({ days: offset })
+      let dayOfWeek = intervalDay.weekday - 1
+
+      let hasRemoval = this.#calendarRemovals.some(removalDate => removalDate.equals(intervalDay))
+      if (hasRemoval) continue
+
+      if (this.daysOfWeek[dayOfWeek]) this.#operationDays.push(intervalDay)
+    }
+  }
+
+  /**
+   * Gets the operation days, taking into account any calendar date exceptions
+   * @returns {string[]} A string array of GTFS formatted dates
+   */
+  getOperationDays() {
+    return this.#operationDays.sort((a, b) => a - b).map(day => toGTFSDate(day)).filter((date, i, arr) => i === 0 || arr[i - 1] != date)
+  }
+
+}

package/lib/gtfs-parser/GTFSCalendarDate.mjs

@@ -0,0 +1,15 @@
+import { parseDate } from '../utils.mjs'
+
+export default class GTFSCalendarDate {
+
+  id
+  date
+  type
+
+  constructor({ id, date, type }) {
+    this.id = id
+    this.date = parseDate(date)
+    this.type = type
+  }
+
+}

package/lib/gtfs-parser/GTFSRoute.mjs

@@ -0,0 +1,106 @@
+import { TRANSIT_MODES } from '../constants.mjs'
+
+export default class GTFSRoute {
+
+  originalRouteGTFSID
+  routeGTFSID
+  agencyID
+  operator
+  routeNumber
+  routeName
+
+  /**
+   * 
+   * @param {Object} param0 The route data
+   * @param {string} param0.routeGTFSID The route's GTFS ID
+   * @param {string} param0.agencyID The route's agency ID
+   * @param {string} param0.routeNumber The route's number. Blank if it does not exist
+   * @param {string} param0.routeName The route's name
+   */
+  constructor({ routeGTFSID, agencyID, routeNumber, routeName }) {
+    this.originalRouteGTFSID = routeGTFSID
+    this.routeGTFSID = this.parseRouteID(routeGTFSID)
+    this.agencyID = agencyID
+    this.routeName = routeName
+
+    if (routeNumber !== null && routeNumber.length === 0) routeNumber = null
+    this.routeNumber = routeNumber
+  }
+
+  /**
+   * Parses the route GTFS ID into a standardises format.
+   * @param {string} routeGTFSID The route GTFS ID
+   * @returns A formatted route GTFS ID
+   */
+  parseRouteID(routeGTFSID) {
+    let [ _, mode, id ] = routeGTFSID.match(/^(\d+)\-(\w+)/)
+
+    return `${mode}-${`000${id}`.slice(-3)}`
+  }
+
+  /**
+   * Checks if the GTFSRoute (or subclass) is able to process the given route data
+   * @param {Object} data The route being checked
+   * @param {string} mode The route mode
+   * @returns true if the class is able to process the route
+   */
+  static canProcess(routeGTFSID, mode) {
+    return true 
+  }
+
+  
+  /**
+   * Creates a new GTFSRoute with the route data that was passed in
+   * @param {Object} data The route data being created
+   * @param {string} mode The transit mode
+   * @returns {GTFSRoute} A route instance representing the data being passed in
+   */
+  static create(data, mode) {
+    let { routeGTFSID } = data
+    let routeTypes = [ MetroGTFSRoute, SmartrakGTFSRoute, GTFSRoute ]
+    for (let type of routeTypes) {
+      if (type.canProcess(routeGTFSID, mode)) return new type(data)
+    }
+  }
+}
+
+/**
+ * Used to represent a Smartrak bus route.
+ * 
+ * Route IDs are of the form DEPOT-ROUTE-aus-ROUTE_VARIANT. Note that the route number could contain letters or even dashes
+ * 
+ * Example: 29-737-aus-1
+ * Example: 29-695-F-aus-1
+ * Example: 53-MFR-aus-1
+ */
+class SmartrakGTFSRoute extends GTFSRoute {
+
+  parseRouteID(routeGTFSID) {
+    let [ _, routeNumber ] = routeGTFSID.match(/^\d+\-(\w+)/)
+    return `4-${routeNumber}`
+  }  
+
+  static canProcess(routeGTFSID, mode) {
+    return routeGTFSID.includes('-aus-')
+  }
+
+}
+
+/**
+ * Used to represent a Metro train route
+ *
+ * Used for a technicality where the route name (eg Pakenham) is stored in the route number instead.
+ */
+class MetroGTFSRoute extends GTFSRoute {
+
+  constructor(data) {
+    super(data)
+    this.routeName = this.routeNumber
+    this.routeNumber = null
+  }
+
+  static canProcess(routeGTFSID, mode) {
+    return mode === TRANSIT_MODES.metroTrain
+  }
+
+}

package/lib/gtfs-parser/GTFSShape.mjs

@@ -0,0 +1,39 @@
+export class GTFSShapePoint {
+
+  #latitude
+  #longitude
+  distance
+
+  get coordinates() {
+    return [
+      this.#longitude, this.#latitude
+    ]
+  }
+
+  constructor({ latitude, longitude, distance }) {
+    this.#latitude = latitude
+    this.#longitude = longitude
+
+    this.distance = distance
+  }
+
+}
+
+export class GTFSShape {
+  
+  shapeID
+  points
+
+  shapeLength
+  hash
+
+  constructor(shapeID, points) {
+    this.shapeID = shapeID
+    this.points = points
+
+    let lastPoint = points[points.length - 1]
+    this.shapeLength = lastPoint.distance
+    this.hash = `${this.shapeLength}-${points[0].coordinates.join(',')}-${lastPoint.coordinates.join(',')}`
+  }
+
+}

package/lib/gtfs-parser/GTFSStop.mjs

@@ -0,0 +1,212 @@
+import { booleanPointInPolygon, intersect } from '@turf/turf'
+import { STOP_UTILS } from '@transportme/transportvic-utils'
+
+const { processName, getPrimaryStopName, getSecondaryStopName } = STOP_UTILS
+
+export class GTFSStop {
+ 
+  stopGTFSID
+  originalName
+  rawStopName
+  fullStopName
+  stopNumber
+  location
+  suburb
+
+  #isInterstate = false
+  #suburbInFrontMatch
+
+  constructor({ stopGTFSID, stopName, stopLat, stopLon }, suburbs) {
+    this.stopGTFSID = stopGTFSID
+    this.originalName = stopName
+    this.location = {
+      type: 'Point',
+      coordinates: [parseFloat(stopLon), parseFloat(stopLat)]
+    }
+
+    this.#setSuburb(suburbs)
+    this.#setStopNumber()
+
+    this.fullStopName = this.getFullStopName()
+  }
+
+  /**
+   * Checks if a stop name requires the suburb to be appended
+   * 
+   * @returns {boolean} True if the suburb is required
+   */
+  requiresSuburb() {
+    let { originalName } = this
+    return originalName[originalName.length - 1] !== ')'
+  }
+
+  getSuburbFromLocation(suburbs) {
+    // Kinda inefficient but it's not expected this will be used very often unless PTV breaks the whole dataset (again)
+    let suburb = suburbs.features.find(suburb => booleanPointInPolygon(this.location, suburb))
+    if (suburb) return this.#expandSuburb(suburb.properties.LOC_NAME)
+
+    this.#isInterstate = true
+    return 'Interstate'
+  }
+
+  getSuburbState(stateIndex) {
+    let bracketContent = this.originalName.slice(stateIndex + 1, -2)
+    if (!bracketContent.includes('-')) return bracketContent
+
+    let dashIndex = bracketContent.indexOf('-')
+    return bracketContent.slice(dashIndex + 2)
+  }
+
+  getInterstateSuburb() {
+    let stateIndex = this.originalName.lastIndexOf('(')
+    let state = this.getSuburbState(stateIndex)
+    let suburbIndex = this.originalName.lastIndexOf('(', stateIndex - 1)
+    let suburb = this.originalName.slice(suburbIndex + 1, stateIndex - 1)
+
+    return `${this.#expandSuburb(suburb)}, ${state}`
+  }
+
+  #expandSuburb(suburb) {
+    return suburb.replace(/^St /, 'St. ')
+      .replace(/^Mt /, 'Mount ')
+  }
+
+  suburbIsInFront() {
+    this.#suburbInFrontMatch = this.originalName.match(/^([\w ]+), (.+)/)
+    return !this.originalName.endsWith(')') && !!this.#suburbInFrontMatch
+  }
+
+  getSuburbFromFrontOfName() {
+    return this.#suburbInFrontMatch[1]
+  }
+
+  getSuburbFromName() {
+    if (this.suburbIsInFront()) return this.getSuburbFromFrontOfName()
+
+    this.#isInterstate = this.originalName.endsWith('))')
+    if (this.#isInterstate) return this.getInterstateSuburb()
+
+    let suburbIndex = this.originalName.lastIndexOf('(')
+    return this.#expandSuburb(this.originalName.slice(suburbIndex + 1, -1))
+  }
+
+  #setSuburb(suburbs) {
+    if (this.requiresSuburb()) {
+      this.suburb = this.getSuburbFromLocation(suburbs)
+    } else {
+      this.suburb = this.getSuburbFromName()
+    }
+  }
+
+  getStopNameWithoutSuburb() {
+    if (this.suburbIsInFront()) return this.#suburbInFrontMatch[2]
+    else if (this.requiresSuburb()) return this.originalName
+    else {
+      if (!this.#isInterstate) return this.originalName.slice(0, this.originalName.lastIndexOf('(') - 1)
+
+      let stateIndex = this.originalName.lastIndexOf('(')
+      let suburbIndex = this.originalName.lastIndexOf('(', stateIndex - 1)
+
+      return this.originalName.slice(0, suburbIndex - 1)
+    }
+  }
+
+  matchStopNumber() {
+    let stopNumberParts
+    let stopName = this.getStopNameWithoutSuburb()
+
+    if (stopName.match(/^\d+\-\d+ \w/)) {
+      // Stops in the form 123-456 ABC Road - not a stop number!
+      return { stopNumber: null, stopName: stopName }
+    }
+
+    if (stopNumberParts = stopName.match(/^(D?\d+[A-Za-z]?)-(.+)$/)) {
+      // Stops in the form 123-ABC Road
+      return {
+        stopNumber: stopNumberParts[1].toUpperCase(),
+        stopName: stopNumberParts[2]
+      }
+    }
+
+    if (stopNumberParts = stopName.match(/^(.*) - Stop (D?\d*[A-Za-z]?)$/)) {
+      // Stops in the form ABC Road - Stop 123
+      return {
+        stopNumber: stopNumberParts[2].toUpperCase(),
+        stopName: stopNumberParts[1]
+      }
+    }
+
+    return { stopNumber: null, stopName: stopName }
+  }
+
+  #setStopNumber() {
+    let { stopNumber, stopName } = this.matchStopNumber()
+    this.stopNumber = stopNumber
+    this.rawStopName = stopName
+  }
+
+  getPrimaryStopName() {
+    return getPrimaryStopName(this.rawStopName)
+  }
+
+  getSecondaryStopName() {
+    return getSecondaryStopName(this.rawStopName)
+  }
+
+  getFullStopName() {
+    let primaryName = this.getPrimaryStopName()
+    let secondaryName = this.getSecondaryStopName()
+
+    let fullStopName = processName(primaryName)
+    if (secondaryName.length) fullStopName += `/${processName(secondaryName)}`
+
+    return fullStopName
+  }
+
+  getBayData(mode) {
+    return {
+      originalName: this.originalName,
+      fullStopName: this.fullStopName,
+      stopGTFSID: this.stopGTFSID,
+      location: this.location,
+      stopNumber: this.stopNumber,
+      mode,
+      suburb: this.suburb,
+      services: [],
+      screenServices: []
+    }
+  }
+
+  static canProcess(data) {
+    return true
+  }
+
+  static create(data, suburbs) {
+    let types = [ GTFSPlatformStop, GTFSStop ]
+    for (let type of types) {
+      if (type.canProcess(data)) return new type(data, suburbs)
+    }
+  }
+}
+
+export class GTFSPlatformStop extends GTFSStop {
+
+  platform
+
+  constructor(data, suburbs) {
+    super(data, suburbs)
+    this.platform = data.platform
+  }
+
+  getBayData(mode) {
+    return {
+      ...super.getBayData(mode),
+      platform: this.platform
+    }
+  }
+
+  static canProcess(data) {
+    return data.platform && data.platform.length
+  }
+
+}

package/lib/gtfs-parser/GTFSStopTime.mjs

@@ -0,0 +1,30 @@
+import { hhmmToMinutesPastMidnight } from '../utils.mjs'
+
+export class GTFSStopTime {
+ 
+  arrivalTime
+  arrivalTimeMinutes
+  departureTime
+  departureTimeMinutes
+
+  stopID
+  stopSequence
+  pickup
+  dropoff
+  distanceTravelled
+  
+  constructor({ arrivalTime, departureTime, stopID, stopSequence, pickup, dropoff, distance }) {
+    this.arrivalTime = arrivalTime.slice(0, 5)
+    this.departureTime = departureTime.slice(0, 5)
+
+    this.arrivalTimeMinutes = hhmmToMinutesPastMidnight(this.arrivalTime)
+    this.departureTimeMinutes = hhmmToMinutesPastMidnight(this.departureTime)
+
+    this.stopID = stopID
+    this.stopSequence = stopSequence
+    this.pickup = parseInt(pickup)
+    this.dropoff = parseInt(dropoff)
+    this.distanceTravelled = distance
+  }
+
+}