gtfs 4.14.4 → 4.15.0

package/dist/bin/gtfsrealtime-update.js

@@ -125,15 +125,13 @@ function formatError(error) {
   return colors.red(errorMessage);
 }
 
-// src/lib/import.ts
+// src/lib/import-gtfs.ts
 import { parse } from "csv-parse";
-import pluralize from "pluralize";
+import pluralize2 from "pluralize";
 import stripBomStream from "strip-bom-stream";
 import { temporaryDirectory } from "tempy";
 import untildify3 from "untildify";
-import mapSeries from "promise-map-series";
-import GtfsRealtimeBindings from "gtfs-realtime-bindings";
-import sqlString2 from "sqlstring-sqlite";
+import mapSeries2 from "promise-map-series";
 
 // src/models/gtfs-realtime/trip-updates.ts
 var tripUpdates = {
@@ -588,6 +586,12 @@ import {
 } from "lodash-es";
 import { feature, featureCollection } from "@turf/helpers";
 
+// src/lib/import-gtfs-realtime.ts
+import pluralize from "pluralize";
+import GtfsRealtimeBindings from "gtfs-realtime-bindings";
+import sqlString2 from "sqlstring-sqlite";
+import mapSeries from "promise-map-series";
+
 // src/lib/utils.ts
 import sqlString from "sqlstring-sqlite";
 import Long from "long";
@@ -621,40 +625,10 @@ function convertLongTimeToDate(longDate) {
   return new Date(new Long(low, high, unsigned).toInt() * 1e3).toISOString();
 }
 
-// src/lib/import.ts
-var downloadGtfsRealtimeData = async (urlAndHeaders, task) => {
-  task.log(`Downloading GTFS-Realtime from ${urlAndHeaders.url}`);
-  const response = await fetch(urlAndHeaders.url, {
-    method: "GET",
-    headers: {
-      ...urlAndHeaders.headers ?? {},
-      "Accept-Encoding": "gzip"
-    },
-    signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
-  });
-  if (response.status !== 200) {
-    task.logWarning(
-      `Unable to download GTFS-Realtime from ${urlAndHeaders.url}. Got status ${response.status}.`
-    );
-    return null;
-  }
-  const buffer = await response.arrayBuffer();
-  const message = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(
-    new Uint8Array(buffer)
-  );
-  return GtfsRealtimeBindings.transit_realtime.FeedMessage.toObject(message, {
-    enums: String,
-    longs: String,
-    bytes: String,
-    defaults: true,
-    arrays: true,
-    objects: true,
-    oneofs: true
-  });
-};
-function getDescendantProp(obj, defaultValue, source) {
-  if (source === void 0) return defaultValue;
-  const arr = source.split(".");
+// src/lib/import-gtfs-realtime.ts
+function getNestedProperty(obj, defaultValue, path2) {
+  if (path2 === void 0) return defaultValue;
+  const arr = path2.split(".");
   while (arr.length) {
     const nextKey = arr.shift();
     if (nextKey === void 0) {
@@ -683,7 +657,37 @@ function getDescendantProp(obj, defaultValue, source) {
   if (obj.__isLong__) return convertLongTimeToDate(obj);
   return obj;
 }
-var deleteExpiredRealtimeData = (config) => {
+async function fetchGtfsRealtimeData(urlConfig, task) {
+  task.log(`Downloading GTFS-Realtime from ${urlConfig.url}`);
+  const response = await fetch(urlConfig.url, {
+    method: "GET",
+    headers: {
+      ...urlConfig.headers ?? {},
+      "Accept-Encoding": "gzip"
+    },
+    signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
+  });
+  if (response.status !== 200) {
+    task.logWarning(
+      `Unable to download GTFS-Realtime from ${urlConfig.url}. Got status ${response.status}.`
+    );
+    return null;
+  }
+  const buffer = await response.arrayBuffer();
+  const message = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(
+    new Uint8Array(buffer)
+  );
+  return GtfsRealtimeBindings.transit_realtime.FeedMessage.toObject(message, {
+    enums: String,
+    longs: String,
+    bytes: String,
+    defaults: true,
+    arrays: true,
+    objects: true,
+    oneofs: true
+  });
+}
+function removeExpiredRealtimeData(config) {
   const log2 = log(config);
   const db = openDb(config);
   log2(`Removing expired GTFS-Realtime data`);
@@ -703,8 +707,8 @@ var deleteExpiredRealtimeData = (config) => {
     `DELETE FROM service_alert_targets WHERE expiration_timestamp <= strftime('%s','now')`
   ).run();
   log2(`Removed expired GTFS-Realtime data\r`, true);
-};
-var prepareRealtimeValue = (entity, column, task) => {
+}
+function prepareRealtimeFieldValue(entity, column, task) {
   if (column.name === "created_timestamp") {
     return task.currentTimestamp;
   }
@@ -712,54 +716,102 @@ var prepareRealtimeValue = (entity, column, task) => {
     return task.currentTimestamp + task.gtfsRealtimeExpirationSeconds;
   }
   return sqlString2.escape(
-    getDescendantProp(entity, column.default, column.source)
+    getNestedProperty(entity, column.default, column.source)
   );
-};
-var updateRealtimeData = async (task) => {
+}
+async function processRealtimeAlerts(db, gtfsRealtimeData, task) {
+  task.log(`Download successful`);
+  let totalLineCount = 0;
+  for (const entity of gtfsRealtimeData.entity) {
+    const fieldValues = serviceAlerts.schema.map(
+      (column) => prepareRealtimeFieldValue(entity, column, task)
+    );
+    try {
+      db.prepare(
+        `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
+      ).run();
+    } catch (error) {
+      task.logWarning("Import error: " + error.message);
+    }
+    const alertTargetArray = [];
+    for (const informedEntity of entity.alert.informedEntity) {
+      informedEntity.parent = entity;
+      const subValues = serviceAlertTargets.schema.map(
+        (column) => prepareRealtimeFieldValue(informedEntity, column, task)
+      );
+      alertTargetArray.push(`(${subValues.join(", ")})`);
+      totalLineCount++;
+    }
+    try {
+      db.prepare(
+        `REPLACE INTO ${serviceAlertTargets.filenameBase} (${serviceAlertTargets.schema.map((column) => column.name).join(", ")}) VALUES ${alertTargetArray.join(", ")}`
+      ).run();
+    } catch (error) {
+      task.logWarning("Import error: " + error.message);
+    }
+    task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
+  }
+}
+async function processRealtimeTripUpdates(db, gtfsRealtimeData, task) {
+  task.log(`Download successful`);
+  let totalLineCount = 0;
+  for (const entity of gtfsRealtimeData.entity) {
+    const fieldValues = tripUpdates.schema.map(
+      (column) => prepareRealtimeFieldValue(entity, column, task)
+    );
+    try {
+      db.prepare(
+        `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
+      ).run();
+    } catch (error) {
+      task.logWarning("Import error: " + error.message);
+    }
+    const stopTimeUpdateArray = [];
+    for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
+      stopTimeUpdate.parent = entity;
+      const subValues = stopTimeUpdates.schema.map(
+        (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
+      );
+      stopTimeUpdateArray.push(`(${subValues.join(", ")})`);
+      totalLineCount++;
+    }
+    try {
+      db.prepare(
+        `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(", ")}) VALUES ${stopTimeUpdateArray.join(", ")}`
+      ).run();
+    } catch (error) {
+      task.logWarning("Import error: " + error.message);
+    }
+    task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
+  }
+}
+async function processRealtimeVehiclePositions(db, gtfsRealtimeData, task) {
+  task.log(`Download successful`);
+  let totalLineCount = 0;
+  for (const entity of gtfsRealtimeData.entity) {
+    const fieldValues = vehiclePositions.schema.map(
+      (column) => prepareRealtimeFieldValue(entity, column, task)
+    );
+    try {
+      db.prepare(
+        `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
+      ).run();
+    } catch (error) {
+      task.logWarning("Import error: " + error.message);
+    }
+    task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
+  }
+}
+async function updateGtfsRealtimeData(task) {
   if (task.realtimeAlerts === void 0 && task.realtimeTripUpdates === void 0 && task.realtimeVehiclePositions === void 0) {
     return;
   }
-  const db = openDb({
-    sqlitePath: task.sqlitePath
-  });
+  const db = openDb({ sqlitePath: task.sqlitePath });
   if (task.realtimeAlerts?.url) {
     try {
-      const gtfsRealtimeData = await downloadGtfsRealtimeData(
-        task.realtimeAlerts,
-        task
-      );
-      if (gtfsRealtimeData?.entity) {
-        task.log(`Download successful`);
-        let totalLineCount = 0;
-        for (const entity of gtfsRealtimeData.entity) {
-          const fieldValues = serviceAlerts.schema.map(
-            (column) => prepareRealtimeValue(entity, column, task)
-          );
-          try {
-            db.prepare(
-              `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-            ).run();
-          } catch (error) {
-            task.logWarning("Import error: " + error.message);
-          }
-          const alertTargetArray = [];
-          for (const informedEntity of entity.alert.informedEntity) {
-            informedEntity.parent = entity;
-            const subValues = serviceAlertTargets.schema.map(
-              (column) => prepareRealtimeValue(informedEntity, column, task)
-            );
-            alertTargetArray.push(`(${subValues.join(", ")})`);
-            totalLineCount++;
-          }
-          try {
-            db.prepare(
-              `REPLACE INTO ${serviceAlertTargets.filenameBase} (${serviceAlertTargets.schema.map((column) => column.name).join(", ")}) VALUES ${alertTargetArray.join(", ")}`
-            ).run();
-          } catch (error) {
-            task.logWarning("Import error: " + error.message);
-          }
-          task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
-        }
+      const alertsData = await fetchGtfsRealtimeData(task.realtimeAlerts, task);
+      if (alertsData?.entity) {
+        await processRealtimeAlerts(db, alertsData, task);
       }
     } catch (error) {
       if (task.ignoreErrors) {
@@ -771,42 +823,12 @@ var updateRealtimeData = async (task) => {
   }
   if (task.realtimeTripUpdates?.url) {
     try {
-      const gtfsRealtimeData = await downloadGtfsRealtimeData(
+      const tripUpdatesData = await fetchGtfsRealtimeData(
         task.realtimeTripUpdates,
         task
       );
-      if (gtfsRealtimeData?.entity) {
-        task.log(`Download successful`);
-        let totalLineCount = 0;
-        for (const entity of gtfsRealtimeData.entity) {
-          const fieldValues = tripUpdates.schema.map(
-            (column) => prepareRealtimeValue(entity, column, task)
-          );
-          try {
-            db.prepare(
-              `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-            ).run();
-          } catch (error) {
-            task.logWarning("Import error: " + error.message);
-          }
-          const stopTimeUpdateArray = [];
-          for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
-            stopTimeUpdate.parent = entity;
-            const subValues = stopTimeUpdates.schema.map(
-              (column) => prepareRealtimeValue(stopTimeUpdate, column, task)
-            );
-            stopTimeUpdateArray.push(`(${subValues.join(", ")})`);
-            totalLineCount++;
-          }
-          try {
-            db.prepare(
-              `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(", ")}) VALUES ${stopTimeUpdateArray.join(", ")}`
-            ).run();
-          } catch (error) {
-            task.logWarning("Import error: " + error.message);
-          }
-          task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
-        }
+      if (tripUpdatesData?.entity) {
+        await processRealtimeTripUpdates(db, tripUpdatesData, task);
       }
     } catch (error) {
       if (task.ignoreErrors) {
@@ -818,26 +840,12 @@ var updateRealtimeData = async (task) => {
   }
   if (task.realtimeVehiclePositions?.url) {
     try {
-      const gtfsRealtimeData = await downloadGtfsRealtimeData(
+      const vehiclePositionsData = await fetchGtfsRealtimeData(
         task.realtimeVehiclePositions,
         task
       );
-      if (gtfsRealtimeData?.entity) {
-        task.log(`Download successful`);
-        let totalLineCount = 0;
-        for (const entity of gtfsRealtimeData.entity) {
-          const fieldValues = vehiclePositions.schema.map(
-            (column) => prepareRealtimeValue(entity, column, task)
-          );
-          try {
-            db.prepare(
-              `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-            ).run();
-          } catch (error) {
-            task.logWarning("Import error: " + error.message);
-          }
-          task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
-        }
+      if (vehiclePositionsData?.entity) {
+        await processRealtimeVehiclePositions(db, vehiclePositionsData, task);
       }
     } catch (error) {
       if (task.ignoreErrors) {
@@ -848,7 +856,7 @@ var updateRealtimeData = async (task) => {
     }
   }
   task.log(`GTFS-Realtime data import complete`);
-};
+}
 async function updateGtfsRealtime(initialConfig) {
   const config = setDefaultConfig(initialConfig);
   validateConfigForImport(config);
@@ -865,7 +873,7 @@ async function updateGtfsRealtime(initialConfig) {
         true
       )} using SQLite database at ${config.sqlitePath}`
     );
-    deleteExpiredRealtimeData(config);
+    removeExpiredRealtimeData(config);
     await mapSeries(config.agencies, async (agency2) => {
       try {
         const task = {
@@ -881,7 +889,7 @@ async function updateGtfsRealtime(initialConfig) {
           logWarning: logWarning2,
           logError: logError2
         };
-        await updateRealtimeData(task);
+        await updateGtfsRealtimeData(task);
       } catch (error) {
         if (config.ignoreErrors) {
           logError2(error.message);
@@ -899,21 +907,39 @@ async function updateGtfsRealtime(initialConfig) {
 `
     );
   } catch (error) {
-    if (error?.code === "SQLITE_CANTOPEN") {
-      logError2(
-        `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
-      );
-    }
-    throw error;
+    handleDatabaseError(error, config, logError2);
   }
 }
+function handleDatabaseError(error, config, logError2) {
+  if (error?.code === "SQLITE_CANTOPEN") {
+    logError2(
+      `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
+    );
+  }
+  throw error;
+}
+
+// src/lib/import-gtfs.ts
+var TIME_COLUMN_NAMES = [
+  "start_time",
+  "end_time",
+  "arrival_time",
+  "departure_time",
+  "prior_notice_last_time",
+  "prior_notice_start_time",
+  "start_pickup_drop_off_window"
+];
+var TIME_COLUMN_PAIRS = TIME_COLUMN_NAMES.map((name) => [
+  name,
+  name.endsWith("time") ? `${name}stamp` : `${name}_timestamp`
+]);
 
 // src/lib/export.ts
 import { without, compact as compact2 } from "lodash-es";
-import pluralize2 from "pluralize";
+import pluralize3 from "pluralize";
 import { stringify } from "csv-stringify";
 import sqlString3 from "sqlstring-sqlite";
-import mapSeries2 from "promise-map-series";
+import mapSeries3 from "promise-map-series";
 import untildify4 from "untildify";
 
 // src/lib/advancedQuery.ts