@transportme/load-ptv-gtfs 1.0.0

package/lib/gtfs-parser/readers/line-reader.mjs

@@ -0,0 +1,112 @@
+import lineReader from 'line-reader'
+
+export default class CSVLineReader {
+
+  /** @type {String[]}  */
+  #headers
+  /** @type {string}  */
+  #file
+  /** @type {boolean}  */
+  #unread
+  /** @type {Object.<string, string>}  */
+  #previousLine
+  /** @type {LineReader}  */
+  #reader
+
+  /**
+   * Constructs a new CSVLineReader for a given file.
+   * 
+   * @param {string} file The CSV file path
+   */
+  constructor(file) {
+    this.#headers = []
+    this.#file = file
+    this.#unread = false
+    this.#previousLine = null
+  }
+
+  /**
+   * Opens the CSV file for reading, and reads the first line for the headers.
+   */
+  async open() {
+    await new Promise((resolve, reject) => {
+      lineReader.open(this.#file, async (err, reader) => {
+        if (err) return reject(err)
+
+        this.#reader = reader
+        this.#headers = (await this.#readNextLine()).split(',')
+        resolve()
+      })
+    })
+  }
+
+  /**
+   * Checks if there is still more data available for reading
+   * 
+   * @returns {boolean} If there is still a line available
+   */
+  available() {
+    return this.#unread || this.#reader.hasNextLine()
+  }
+
+  #readNextLine() {
+    return new Promise(resolve => {
+      this.#reader.nextLine((err, line) => {
+        resolve(line.trim())
+      })
+    })
+  }
+
+  #setHeaders(line) {
+    let parts = line.match(/"([^"]*)"/g).map(f => f.slice(1, -1))
+    return this.#headers.reduce((acc, header, i) => {
+      acc[header] = parts[i]
+      return acc
+    }, {})
+  }
+
+  /**
+   * Returns the next line of CSV data, taking into account any unreads.
+   * 
+   * @async
+   * @returns {Promise<Object.<string,string>>} The line data, parsed according to the headers
+   */
+  async nextLine() {
+    if (this.#unread) {
+      this.#unread = false
+      return this.#previousLine
+    }
+
+    let line = this.#setHeaders(await this.#readNextLine())
+    this.#previousLine = line
+
+    return line
+  }
+
+  /**
+   * Unreads a line, causing it to be returned again during the next call to `nextLine()`
+   */
+  unreadLine() {
+    this.#unread = true
+  }
+
+  /**
+   * Closes the CSV line reader.
+   */
+  async close() {
+    await new Promise(resolve => {
+      this.#reader.close(() => {
+        resolve()
+      })
+    })
+  }
+
+  /**
+   * Returns the CSV headers.
+   * 
+   * @returns {string[]} A list of headers
+   */
+  getHeaders() {
+    return this.#headers
+  }
+}

package/lib/loader/RouteLoader.mjs

@@ -0,0 +1,101 @@
+import GTFSRouteReader from '../gtfs-parser/readers/GTFSRouteReader.mjs'
+import GTFSAgencyReader from '../gtfs-parser/readers/GTFSAgencyReader.mjs'
+import { STOP_UTILS } from '@transportme/transportvic-utils'
+
+const { sanitiseName } = STOP_UTILS
+
+export default class RouteLoader {
+
+  #routes
+  #mode
+  #routesReader
+  #operators = {}
+  #agencyFile
+
+  #routeIDMap = {}
+
+  constructor(routesFile, agencyFile, mode, database) {
+    this.#routes = database.getCollection('routes')
+    this.#mode = mode
+    this.#routesReader = new GTFSRouteReader(routesFile, mode)
+    this.#agencyFile = agencyFile
+  }
+
+  async loadAgencies() {
+    let agencyReader = new GTFSAgencyReader(this.#agencyFile)
+    await agencyReader.open()
+    while (agencyReader.available()) {
+      let agency = await agencyReader.getNextEntity()
+      this.#operators[agency.id] = agency
+    }
+  }
+
+  async findMatchingRoute(routeData) {
+    return await this.#routes.findDocument({
+      routeGTFSID: routeData.routeGTFSID
+    })
+  }
+
+  generateDBRouteData(route) {
+    return {
+      mode: this.#mode,
+      routeName: route.routeName,
+      cleanName: sanitiseName(route.routeName),
+      routeNumber: route.routeNumber,
+      routeGTFSID: route.routeGTFSID,
+      routePath: [],
+      operators: [ this.#operators[route.agencyID].name ],
+      directions: [],
+
+      codedName: sanitiseName(route.routeName)
+    }
+  }
+
+  async mergeRoutes(matchingRoute, routeData) {
+    if (!matchingRoute.operators.includes(routeData.operators[0])) {
+      matchingRoute.operators.push(routeData.operators[0])
+    }
+
+    await this.#routes.replaceDocument({
+      _id: matchingRoute._id
+    }, matchingRoute)
+  }
+
+  /**
+   * Loads the route data into the database.
+   * 
+   * @param {Object} options An optional parameter to allow for custom route processing
+   * @param {function} options.processRoute A function to modify the route data. If a route should be dropped, it should return null.
+   */
+  async loadRoutes({ processRoute } = { processRoute: r => r }) {
+    await this.loadAgencies()
+
+    await this.#routesReader.open()
+    while (this.#routesReader.available()) {
+      let route = await this.#routesReader.getNextEntity()
+
+      let originalRouteID = route.originalRouteGTFSID
+      let routeData = this.generateDBRouteData(route)
+      let processedData = await processRoute(routeData)
+      if (!processedData) continue
+
+      this.#routeIDMap[originalRouteID] = processedData.routeGTFSID
+
+      let matchingRoute = await this.findMatchingRoute(processedData)
+
+      if (matchingRoute) {
+        await this.mergeRoutes(matchingRoute, processedData)
+      } else {
+        await this.#routes.createDocument(processedData)
+      }
+    }
+  }
+
+  getOperator(id) {
+    return this.#operators[id]
+  }
+
+  getRouteIDMap() {
+    return this.#routeIDMap
+  }
+}

package/lib/loader/ShapeLoader.mjs

@@ -0,0 +1,86 @@
+import { GTFSStop } from '../gtfs-parser/GTFSStop.mjs'
+import GTFSShapeReader from '../gtfs-parser/readers/GTFSShapeReader.mjs'
+
+export default class ShapeLoader {
+
+  #routes
+  #shapeReader
+
+  #routeCache = {}
+
+  constructor(shapeFile, database) {
+    this.#routes = database.getCollection('routes')
+    this.#shapeReader = new GTFSShapeReader(shapeFile)
+  }
+
+  async findMatchingRoute(routeGTFSID) {
+    if (this.#routeCache[routeGTFSID]) return this.#routeCache[routeGTFSID]
+
+    let route = await this.#routes.findDocument({
+      routeGTFSID: routeGTFSID
+    })
+
+    this.#routeCache[routeGTFSID] = route
+    return route
+  }
+
+  async updateRoute(routeGTFSID, routePath) {
+    this.#routeCache[routeGTFSID].routePath = routePath
+    await this.#routes.updateDocument({
+      routeGTFSID
+    }, { $set: { routePath } })
+  }
+
+  generateDBShapeData(shape) {
+    return {
+      fullGTFSIDs: [ shape.shapeID ],
+      hash: shape.hash,
+      length: shape.shapeLength,
+      path: {
+        type: 'LineString',
+        coordinates: shape.points.map(point => point.coordinates)
+      }
+    }
+  }
+
+  /**
+   * Loads a shape into the database.
+   *
+   * @param {GTFSStop} shape The shape being added
+   * @param {function} options.shapeIDMap A mapping of shape IDs to their trip IDs.
+   */
+  async loadShape(shape, shapeIDMap) {
+    let routeGTFSID = shapeIDMap[shape.shapeID]
+    if (!routeGTFSID) return // All trips corresponding to that shape must have been dropped
+
+    let routeData = await this.findMatchingRoute(routeGTFSID)
+
+    let matchingShape = routeData.routePath.find(path => path.hash === shape.hash)
+    if (matchingShape) {
+      matchingShape.fullGTFSIDs.push(shape.shapeID)
+    } else {
+      routeData.routePath.push(this.generateDBShapeData(shape))
+    }
+
+    await this.updateRoute(routeGTFSID, routeData.routePath)
+  }
+
+  /**
+   * Loads the shapes into the database.
+   *
+   * @param {object} options Options for loading the shape data in
+   * @param {function} options.shapeIDMap A mapping of shape IDs to their trip IDs.
+   */
+  async loadShapes({ shapeIDMap }) {
+    await this.#shapeReader.open()
+    
+    let shapesLoaded = 0
+
+    while (this.#shapeReader.available()) {
+      let shape = await this.#shapeReader.getNextEntity()
+      await this.loadShape(shape, shapeIDMap)
+
+      if (++shapesLoaded === 1000) this.#routeCache = {}
+    }
+  }
+}

package/lib/loader/StopsLoader.mjs

@@ -0,0 +1,139 @@
+import { STOP_UTILS } from '@transportme/transportvic-utils'
+import { GTFSStop } from '../gtfs-parser/GTFSStop.mjs'
+import GTFSStopsReader from '../gtfs-parser/readers/GTFSStopsReader.mjs'
+
+const { getPrimaryStopName, isStreetStop, sanitiseName } = STOP_UTILS
+
+export default class StopsLoader {
+
+  #stops
+  #mode
+  #stopsReader
+
+  constructor(stopsFile, suburbs, mode, database) {
+    this.#stops = database.getCollection('stops')
+    this.#mode = mode
+    this.#stopsReader = new GTFSStopsReader(stopsFile, suburbs)
+  }
+
+  getMergeName(stop) {
+    if (isStreetStop(stop.fullStopName)) return stop.fullStopName
+
+    return getPrimaryStopName(stop.fullStopName)
+  }
+
+  async findMatchingStop(stop, mergeName) {
+    return await this.#stops.findDocument({
+      mergeName,
+      location: {
+        $nearSphere: {
+          $geometry: stop.location,
+          $maxDistance: 500
+        }
+      }
+    })
+  }
+
+  generateDBStopData(stop, mergeName) {
+    let cleanName = sanitiseName(stop.fullStopName)
+
+    return {
+      stopName: stop.fullStopName,
+      suburb: [ stop.suburb ],
+      cleanSuburbs: [ sanitiseName(stop.suburb) ],
+      bays: [ stop.getBayData(this.#mode) ],
+      location: {
+        type: 'MultiPoint',
+        coordinates: [ stop.location.coordinates ]
+      },
+      mergeName,
+      services: [],
+      screenServices: [],
+      cleanName,
+      cleanNames: [ cleanName ],
+
+      codedSuburbs: [ sanitiseName(stop.suburb) ], // parity purposes
+      codedName: cleanName, // parity purposes
+      codedNames: [ cleanName ] // parity purposes
+    }
+  }
+
+  updateStopName(stop) {
+    stop.cleanNames = stop.bays
+      .map(bay => sanitiseName(bay.fullStopName))
+      .filter((e, i, a) => a.indexOf(e) === i)
+
+    if (stop.cleanNames.length > 1) {
+      stop.stopName = stop.mergeName
+      stop.cleanName = sanitiseName(stop.stopName)
+
+      if (!stop.cleanNames.includes(stop.cleanName)) stop.cleanNames.push(stop.cleanName)
+    }
+
+    stop.codedName = stop.cleanName
+    stop.codedNames = stop.cleanNames
+  }
+
+  updateStopSuburb(stop) {
+    stop.cleanSuburbs = stop.bays
+      .map(bay => sanitiseName(bay.suburb))
+      .filter((e, i, a) => a.indexOf(e) === i)
+
+    stop.suburb = stop.bays
+      .map(bay => bay.suburb)
+      .filter((e, i, a) => a.indexOf(e) === i)
+      .sort()
+
+      stop.codedSuburbs = stop.cleanSuburbs
+  }
+
+  async mergeStops(matchingStop, stopData) {
+    matchingStop.bays.push(stopData.bays[0])
+
+    this.updateStopSuburb(matchingStop)
+    this.updateStopName(matchingStop)
+
+    await this.#stops.replaceDocument({
+      _id: matchingStop._id
+    }, matchingStop)
+  }
+
+  /**
+   * Loads a stop into the database.
+   *
+   * @param {GTFSStop} stop The stop being added
+   * @param {object} options Options for loading the stop data in
+   * @param {function} options.processStop Used to process as stop's data.
+   * @param {function} options.getMergeName Used to get a stop's merge name. Return null to use the default merge name.
+   */
+  async loadStop(stop, {processStop, getMergeName} = {}) {
+    let processedData = processStop ? processStop(stop) : stop
+
+    let defaultMergeName = this.getMergeName(processedData)
+    let mergeName = getMergeName ? (getMergeName(processedData) || defaultMergeName) : defaultMergeName
+
+    let stopData = this.generateDBStopData(processedData, mergeName)
+    let matchingStop = await this.findMatchingStop(processedData, mergeName)
+
+    if (matchingStop) {
+      await this.mergeStops(matchingStop, stopData)
+    } else {
+      await this.#stops.createDocument(stopData)
+    }
+  }
+
+  /**
+   * Loads the stops into the database.
+   *
+   * @param {object} options Options for loading the stop data in
+   * @param {function} options.processStop Used to process as stop's data.
+   * @param {function} options.getMergeName Used to get a stop's merge name. Return null to use the default merge name.
+   */
+  async loadStops(options) {
+    await this.#stopsReader.open()
+    while (this.#stopsReader.available()) {
+      let stop = await this.#stopsReader.getNextEntity()
+      await this.loadStop(stop, options)
+    }
+  }
+}

package/lib/loader/TripLoader.mjs

@@ -0,0 +1,187 @@
+import GTFSCalendarReader from '../gtfs-parser/readers/GTFSCalendarReader.mjs'
+import GTFSStopTimesReader from '../gtfs-parser/readers/GTFSStopTimesReader.mjs'
+import GTFSTripReader from '../gtfs-parser/readers/GTFSTripReader.mjs'
+import async from 'async'
+
+export default class TripLoader {
+
+  #routes
+  #stops
+  #trips
+
+  #tripsReader
+  #stopTimesReader
+  #calendarReader
+
+  #mode
+
+  #tripDetails = {}
+
+  #calendarMap = {}
+  #routeIDMap = {}
+
+  #routeCache = {}
+  #stopCache = {}
+
+  #shapeIDMap = {}
+
+  constructor({ tripsFile, stopTimesFile, calendarFile, calendarDatesFile }, mode, database) {
+    this.#routes = database.getCollection('routes')
+    this.#stops = database.getCollection('stops')
+    this.#trips = database.getCollection('gtfs timetables')
+
+    this.#mode = mode
+
+    this.#stopTimesReader = new GTFSStopTimesReader(stopTimesFile, mode)
+    this.#tripsReader = new GTFSTripReader(tripsFile, null, null, mode)
+    this.#calendarReader = new GTFSCalendarReader(calendarFile, calendarDatesFile)
+  }
+
+  async loadTripsData() {
+    let tripsReader = this.#tripsReader
+    tripsReader.setCalendar(this.#calendarMap)
+    tripsReader.setRouteMappings(this.#routeIDMap)
+
+    await tripsReader.open()
+    while (tripsReader.available()) {
+      let trip = await tripsReader.getNextEntity()
+      this.#tripDetails[trip.getTripID()] = trip
+    }
+  }
+
+  async loadCalendarData() {
+    let calendarReader = this.#calendarReader
+
+    await calendarReader.open()
+    while (calendarReader.available()) {
+      let calendar = await calendarReader.getNextEntity()
+      this.#calendarMap[calendar.id] = calendar
+    }
+  }
+
+  async findMatchingRoute(routeGTFSID) {
+    if (this.#routeCache[routeGTFSID]) return this.#routeCache[routeGTFSID]
+
+    let route = await this.#routes.findDocument({
+      routeGTFSID: routeGTFSID
+    })
+
+    this.#routeCache[routeGTFSID] = route
+    return route
+  }
+
+  async findMatchingStop(stopGTFSID) {
+    if (this.#stopCache[stopGTFSID]) return this.#stopCache[stopGTFSID]
+
+    let stop = await this.#stops.findDocument({
+      'bays.stopGTFSID': stopGTFSID
+    })
+
+    this.#stopCache[stopGTFSID] = stop
+    return stop
+  }
+
+  async generateDBTripData(tripData, stops) {
+    let routeData = await this.findMatchingRoute(tripData.getRouteGTFSID())
+    if (!routeData) return null
+
+    let stopTimings = await async.map(stops, async stop => {
+      let stopData = await this.findMatchingStop(stop.stopID)
+      let bay = stopData.bays.find(bay => bay.stopGTFSID === stop.stopID)
+
+      let data = {
+        stopName: bay.fullStopName,
+        stopNumber: bay.stopNumber,
+        suburb: bay.suburb,
+        stopGTFSID: stop.stopID,
+        arrivalTime: stop.arrivalTime,
+        arrivalTimeMinutes: stop.arrivalTimeMinutes,
+        departureTime: stop.departureTime,
+        departureTimeMinutes: stop.departureTimeMinutes,
+        stopConditions: { pickup: stop.pickup, dropoff: stop.dropoff },
+        stopDistance: stop.distanceTravelled
+      }
+
+      if (bay.platform) data.platform = bay.platform
+
+      return data
+    })
+
+
+    let dbTrip = {
+      mode: this.#mode,
+      ...tripData.getTripData(),
+      routeName: routeData.routeName,
+      routeNumber: routeData.routeNumber,
+      stopTimings
+    }
+
+    dbTrip.origin = stopTimings[0].stopName
+    dbTrip.departureTime = stopTimings[0].departureTime
+    stopTimings[0].arrivalTime = null
+    stopTimings[0].arrivalTimeMinutes = null
+    stopTimings[0].stopConditions.dropoff = 1
+    
+    let lastStop = stopTimings[stopTimings.length - 1]
+    dbTrip.destination = lastStop.stopName
+    dbTrip.destinationArrivalTime = lastStop.arrivalTime
+    lastStop.departureTime = null
+    lastStop.departureTimeMinutes = null
+    lastStop.stopConditions.pickup = 1
+
+    return dbTrip
+  }
+
+  /**
+   * Loads the trip data into the database.
+   * 
+   * @param {object} options A list of options for the trip loader
+   * @param {object} options.routeIDMap A mapping of full route IDs to their simplified IDs.
+   * @param {function} options.processTrip A function to modify trip data on a per-trip basis. It should return null to discard the trip.
+   */
+  async loadTrips({ routeIDMap, processTrip }) {
+    this.#routeIDMap = routeIDMap
+
+    await this.loadCalendarData()
+    await this.loadTripsData()
+
+    let tripsLoaded = 0
+    let trips = []
+
+    await this.#stopTimesReader.open()
+    while (this.#stopTimesReader.available()) {
+      let { tripID, stops } = await this.#stopTimesReader.getNextEntity()
+
+      let rawTripData = this.#tripDetails[tripID]
+      let tripData = await this.generateDBTripData(rawTripData, stops)
+      if (!tripData) continue
+
+      let processedData = processTrip ? await processTrip(tripData) : tripData
+      if (!processedData) continue
+
+      this.#shapeIDMap[tripData.shapeID] = tripData.routeGTFSID
+
+      trips.push(processedData)
+
+      tripsLoaded += 1
+
+      if (tripsLoaded === 5000) { // Flush the cache every 5000 trips
+        this.#routeCache = {}
+        this.#stopCache = {}
+
+        await this.#trips.createDocuments(trips)
+        trips = []
+
+        tripsLoaded = 0
+      }
+    }
+
+    if (trips.length) {
+      await this.#trips.createDocuments(trips)
+    }
+  }
+
+  getShapeIDMap() {
+    return this.#shapeIDMap
+  }
+}

package/lib/post-processing/merge-stops.mjs

@@ -0,0 +1,85 @@
+import { hash } from 'node:crypto'
+
+function tripHash(trip) {
+  return parseInt(hash('md5', trip.map(stop => stop.stopGTFSID).join('-')), 16)
+}
+
+function isLoop(trip) {
+  return trip[0].stopName === trip[trip.length - 1].stopName
+}
+
+/**
+ * Merges different stopping patterns together to determine the overall stops for a route.
+ *
+ * @param {Object[][]} inputVariants A list of different stopping patterns being merged together
+ */
+export default function merge(inputVariants) {
+  let loopVariants = [], nonLoopVariants = []
+  
+  inputVariants.toSorted((a, b) => {
+    return b.length - a.length || tripHash(a) - tripHash(b)
+  }).forEach(trip => {
+    if (trip.length === 1) return
+    if (isLoop(trip)) loopVariants.push(trip)
+    else nonLoopVariants.push(trip)
+  })
+
+  let variants = loopVariants.concat(nonLoopVariants)
+  let stopsList = variants[0]
+
+  for (let variant of variants.slice(1)) {
+    let branch = []
+    let lastMainMatch = 0
+    let isLoopVariant = isLoop(variant)
+
+    let stopIndex = -1
+
+    for (let variantStop of variant) {
+      stopIndex++
+
+      let matchIndex = lastMainMatch - 1
+
+      let hasMatched = false
+      for (let stop of stopsList.slice(lastMainMatch)) {
+        matchIndex++
+
+        hasMatched = stop.stopName === variantStop.stopName
+        if (hasMatched) {
+          if (branch.length > 0) { // lines are out of sync, but match detected = rejoining
+            let firstHalf = stopsList.slice(0, matchIndex)
+            let backHalf = stopsList.slice(matchIndex)
+
+            stopsList = firstHalf.concat(branch).concat(backHalf)
+            branch = []
+          } else { // otherwise we're on sync, all good
+            if (matchIndex < lastMainMatch) {
+              let jumpPart = variant.slice(matchIndex, lastMainMatch + 1)
+              let subBranch = []
+              for (let jumpStop of jumpPart) {
+                if (matched(jumpStop, variantStop)) break
+                subBranch.push(jumpStop)
+              }
+              branch = branch.concat(subBranch)
+            }
+            lastMainMatch = matchIndex
+          }
+          break
+        }
+      }
+
+      // no match, we're on a branch
+      if (!hasMatched) branch.push(variantStop)
+    }
+
+    if (branch.length) { // we're still on a branch after completing the stops, means they have different destiantions
+      // look at where they deviated, and join it in between
+
+      let firstHalf = stopsList.slice(0, lastMainMatch + 1)
+      let backHalf = stopsList.slice(lastMainMatch + 1)
+
+      stopsList = firstHalf.concat(branch).concat(backHalf)
+    }
+  }
+
+  return stopsList
+}