@transportme/load-ptv-gtfs 1.0.0 → 1.0.2

package/index.mjs

@@ -0,0 +1,26 @@
+import RouteLoader from './lib/loader/RouteLoader.mjs'
+import ShapeLoader from './lib/loader/ShapeLoader.mjs'
+import StopsLoader from './lib/loader/StopsLoader.mjs'
+import TripLoader from './lib/loader/TripLoader.mjs'
+
+import GTFSAgency from './lib/gtfs-parser/GTFSAgency.mjs'
+import GTFSCalendar from './lib/gtfs-parser/GTFSCalendar.mjs'
+import GTFSCalendarDate from './lib/gtfs-parser/GTFSCalendarDate.mjs'
+import GTFSRoute from './lib/gtfs-parser/GTFSRoute.mjs'
+import { GTFSShape, GTFSShapePoint } from './lib/gtfs-parser/GTFSShape.mjs'
+import { GTFSStop, GTFSPlatformStop } from './lib/gtfs-parser/GTFSStop.mjs'
+import { GTFSStopTime } from './lib/gtfs-parser/GTFSStopTime.mjs'
+import GTFSTrip from './lib/gtfs-parser/GTFSTrip.mjs'
+
+const GTFSTypes = {
+  GTFSAgency, GTFSCalendar, GTFSCalendarDate,
+  GTFSRoute, GTFSShape, GTFSShapePoint,
+  GTFSStop, GTFSPlatformStop,
+  GTFSStopTime, GTFSTrip
+}
+
+export {
+  RouteLoader, ShapeLoader,
+  StopsLoader, TripLoader,
+  GTFSTypes
+}

package/lib/loader/RouteLoader.mjs

@@ -1,6 +1,7 @@
 import GTFSRouteReader from '../gtfs-parser/readers/GTFSRouteReader.mjs'
 import GTFSAgencyReader from '../gtfs-parser/readers/GTFSAgencyReader.mjs'
 import { STOP_UTILS } from '@transportme/transportvic-utils'
+import GTFSRoute from '../gtfs-parser/GTFSRoute.mjs'
 
 const { sanitiseName } = STOP_UTILS
 
@@ -14,6 +15,13 @@ export default class RouteLoader {
 
   #routeIDMap = {}
 
+  /**
+   * Creates a new Shape Loader.
+   * 
+   * @param {string} routesFile The path for the routes.txt file
+   * @param {string} agencyFile The path for the agency.txt file
+   * @param {DatabaseConnection} database A database connection
+   */
   constructor(routesFile, agencyFile, mode, database) {
     this.#routes = database.getCollection('routes')
     this.#mode = mode
@@ -61,33 +69,42 @@ export default class RouteLoader {
     }, matchingRoute)
   }
 
+  /**
+   * Loads a route into the database
+   * 
+   * @param {GTFSRoute} route The route being added
+   * @param {function} [processRoute] Used to modify the route data. Should return null to drop the route
+   */
+  async loadRoute(route, processRoute) {
+    let originalRouteID = route.originalRouteGTFSID
+    let routeData = this.generateDBRouteData(route)
+    let processedData = processRoute ? await processRoute(routeData) : routeData
+    if (!processedData) return
+
+    this.#routeIDMap[originalRouteID] = processedData.routeGTFSID
+
+    let matchingRoute = await this.findMatchingRoute(processedData)
+
+    if (matchingRoute) {
+      await this.mergeRoutes(matchingRoute, processedData)
+    } else {
+      await this.#routes.createDocument(processedData)
+    }
+  }
+
   /**
    * Loads the route data into the database.
    * 
    * @param {Object} options An optional parameter to allow for custom route processing
-   * @param {function} options.processRoute A function to modify the route data. If a route should be dropped, it should return null.
+   * @param {function} [options.processRoute] A function to modify the route data. If a route should be dropped, it should return null.
    */
-  async loadRoutes({ processRoute } = { processRoute: r => r }) {
+  async loadRoutes({ processRoute } = {}) {
     await this.loadAgencies()
 
     await this.#routesReader.open()
     while (this.#routesReader.available()) {
       let route = await this.#routesReader.getNextEntity()
-
-      let originalRouteID = route.originalRouteGTFSID
-      let routeData = this.generateDBRouteData(route)
-      let processedData = await processRoute(routeData)
-      if (!processedData) continue
-
-      this.#routeIDMap[originalRouteID] = processedData.routeGTFSID
-
-      let matchingRoute = await this.findMatchingRoute(processedData)
-
-      if (matchingRoute) {
-        await this.mergeRoutes(matchingRoute, processedData)
-      } else {
-        await this.#routes.createDocument(processedData)
-      }
+      await this.loadRoute(route, processRoute)
     }
   }
 

package/lib/loader/ShapeLoader.mjs

@@ -8,6 +8,12 @@ export default class ShapeLoader {
 
   #routeCache = {}
 
+  /**
+   * Creates a new Shape Loader.
+   * 
+   * @param {string} shapeFile The path for the shapes.txt file
+   * @param {DatabaseConnection} database A database connection
+   */
   constructor(shapeFile, database) {
     this.#routes = database.getCollection('routes')
     this.#shapeReader = new GTFSShapeReader(shapeFile)

package/lib/loader/StopsLoader.mjs

@@ -10,6 +10,14 @@ export default class StopsLoader {
   #mode
   #stopsReader
 
+  /**
+   * Creates a new Stop Loader.
+   * 
+   * @param {string} stopsFile The path for the stops.txt file
+   * @param {FeatureCollection} suburbs A GeoJSON FeatureCollection containing a list of all suburbs located within Victoria
+   * @param {string} mode The transit mode being loaded
+   * @param {DatabaseConnection} database A database connection
+   */
   constructor(stopsFile, suburbs, mode, database) {
     this.#stops = database.getCollection('stops')
     this.#mode = mode
@@ -106,7 +114,7 @@ export default class StopsLoader {
    * @param {function} options.processStop Used to process as stop's data.
    * @param {function} options.getMergeName Used to get a stop's merge name. Return null to use the default merge name.
    */
-  async loadStop(stop, {processStop, getMergeName} = {}) {
+  async loadStop(stop, { processStop, getMergeName } = {}) {
     let processedData = processStop ? processStop(stop) : stop
 
     let defaultMergeName = this.getMergeName(processedData)

package/lib/loader/TripLoader.mjs

@@ -1,7 +1,9 @@
+import DatabaseConnection from '@transportme/database/lib/DatabaseConnection.mjs'
 import GTFSCalendarReader from '../gtfs-parser/readers/GTFSCalendarReader.mjs'
 import GTFSStopTimesReader from '../gtfs-parser/readers/GTFSStopTimesReader.mjs'
 import GTFSTripReader from '../gtfs-parser/readers/GTFSTripReader.mjs'
 import async from 'async'
+import { GTFSStopTime } from '../gtfs-parser/GTFSStopTime.mjs'
 
 export default class TripLoader {
 
@@ -25,6 +27,19 @@ export default class TripLoader {
 
   #shapeIDMap = {}
 
+  #tripsToLoad = []
+
+  /**
+   * Creates a new Trip Loader.
+   * 
+   * @param {Object} paths Required file paths for the TripLoader
+   * @param {string} paths.tripsFile The path for the trips.txt file
+   * @param {string} paths.stopTimesFile The path for the stop_times.txt file
+   * @param {string} paths.calendarFile The path for the calendar.txt file
+   * @param {string} paths.calendarDatesFile The path for the calendar_dates.txt file
+   * @param {string} mode The transit mode being loaded
+   * @param {DatabaseConnection} database A database connection
+   */
   constructor({ tripsFile, stopTimesFile, calendarFile, calendarDatesFile }, mode, database) {
     this.#routes = database.getCollection('routes')
     this.#stops = database.getCollection('stops')
@@ -132,12 +147,33 @@ export default class TripLoader {
     return dbTrip
   }
 
+  /**
+   * Loads a trip into the database.
+   * 
+   * @param {Object} tripData The trip data
+   * @param {string} tripData.tripID The trip ID
+   * @param {GTFSStopTime[]} tripData.stops The trip's stops
+   * @param {function} [processTrip] Used to process a trip.
+   */
+  async loadTrip({ tripID, stops }, processTrip) {
+    let rawTripData = this.#tripDetails[tripID]
+    let tripData = await this.generateDBTripData(rawTripData, stops)
+    if (!tripData) return
+
+    let processedData = processTrip ? await processTrip(tripData) : tripData
+    if (!processedData) return
+
+    this.#shapeIDMap[tripData.shapeID] = tripData.routeGTFSID
+
+    this.#tripsToLoad.push(processedData)
+  }
+
   /**
    * Loads the trip data into the database.
    * 
    * @param {object} options A list of options for the trip loader
    * @param {object} options.routeIDMap A mapping of full route IDs to their simplified IDs.
-   * @param {function} options.processTrip A function to modify trip data on a per-trip basis. It should return null to discard the trip.
+   * @param {function} [options.processTrip] A function to modify trip data on a per-trip basis. It should return null to discard the trip.
    */
   async loadTrips({ routeIDMap, processTrip }) {
     this.#routeIDMap = routeIDMap
@@ -146,38 +182,29 @@ export default class TripLoader {
     await this.loadTripsData()
 
     let tripsLoaded = 0
-    let trips = []
 
     await this.#stopTimesReader.open()
     while (this.#stopTimesReader.available()) {
-      let { tripID, stops } = await this.#stopTimesReader.getNextEntity()
-
-      let rawTripData = this.#tripDetails[tripID]
-      let tripData = await this.generateDBTripData(rawTripData, stops)
-      if (!tripData) continue
-
-      let processedData = processTrip ? await processTrip(tripData) : tripData
-      if (!processedData) continue
-
-      this.#shapeIDMap[tripData.shapeID] = tripData.routeGTFSID
-
-      trips.push(processedData)
-
-      tripsLoaded += 1
+      let gtfsTrip = await this.#stopTimesReader.getNextEntity()
+      await this.loadTrip(gtfsTrip, processTrip)
 
-      if (tripsLoaded === 5000) { // Flush the cache every 5000 trips
-        this.#routeCache = {}
-        this.#stopCache = {}
-
-        await this.#trips.createDocuments(trips)
-        trips = []
-
-        tripsLoaded = 0
+      if (this.#tripsToLoad.length === 5000) { // Flush the cache every 5000 trips
+        tripsLoaded += this.#tripsToLoad.length
+        await this.writeTrips()
       }
     }
 
-    if (trips.length) {
-      await this.#trips.createDocuments(trips)
+    tripsLoaded += this.#tripsToLoad.length
+    await this.writeTrips()
+  }
+
+  async writeTrips() {
+    if (this.#tripsToLoad.length) {
+      this.#routeCache = {}
+      this.#stopCache = {}
+
+      await this.#trips.createDocuments(this.#tripsToLoad)
+      this.#tripsToLoad = []
     }
   }
 

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@transportme/load-ptv-gtfs",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "",
-  "main": "index.js",
+  "main": "index.mjs",
   "scripts": {
     "test": "mocha"
   },