gtfs 4.17.6 → 4.18.0

package/dist/bin/gtfsrealtime-update.js

@@ -8,10 +8,10 @@ import PrettyError from "pretty-error";
 // src/lib/file-utils.ts
 import path from "path";
 import { existsSync } from "fs";
+import { homedir } from "os";
 import { mkdir, readFile, rm } from "fs/promises";
 import { omit, snakeCase } from "lodash-es";
 import sanitize from "sanitize-filename";
-import untildify from "untildify";
 import StreamZip from "node-stream-zip";
 
 // src/lib/log-utils.ts
@@ -65,6 +65,7 @@ function formatError(error) {
 }
 
 // src/lib/file-utils.ts
+var homeDirectory = homedir();
 async function getConfig(argv2) {
   let config;
   let data;
@@ -101,17 +102,17 @@ async function getConfig(argv2) {
     throw error;
   }
 }
+function untildify(pathWithTilde) {
+  return homeDirectory ? pathWithTilde.replace(/^~(?=$|\/|\\)/, homeDirectory) : pathWithTilde;
+}
 
 // src/lib/import-gtfs.ts
 import path2 from "path";
 import { createReadStream, existsSync as existsSync2, lstatSync } from "fs";
 import { cp, readdir, rename, readFile as readFile2, rm as rm2, writeFile } from "fs/promises";
 import { parse } from "csv-parse";
-import pluralize2 from "pluralize";
 import stripBomStream from "strip-bom-stream";
 import { temporaryDirectory } from "tempy";
-import Timer from "timer-machine";
-import untildify3 from "untildify";
 import mapSeries2 from "promise-map-series";
 
 // src/models/gtfs-realtime/trip-updates.ts
@@ -582,10 +583,9 @@ var serviceAlertInformedEntities = {
 
 // src/lib/db.ts
 import Database from "better-sqlite3";
-import untildify2 from "untildify";
 var dbs = {};
 function setupDb(sqlitePath) {
-  const db = new Database(untildify2(sqlitePath));
+  const db = new Database(untildify(sqlitePath));
   db.pragma("journal_mode = OFF");
   db.pragma("synchronous = OFF");
   db.pragma("temp_store = MEMORY");
@@ -632,7 +632,6 @@ import {
 import { feature, featureCollection } from "@turf/helpers";
 
 // src/lib/import-gtfs-realtime.ts
-import pluralize from "pluralize";
 import GtfsRealtimeBindings from "gtfs-realtime-bindings";
 import mapSeries from "promise-map-series";
 import { get } from "lodash-es";
@@ -659,7 +658,8 @@ function setDefaultConfig(initialConfig) {
     ignoreDuplicates: false,
     ignoreErrors: false,
     gtfsRealtimeExpirationSeconds: 0,
-    verbose: true
+    verbose: true,
+    downloadTimeout: 3e4
   };
   return {
     ...defaults,
@@ -678,58 +678,14 @@ function applyPrefixToValue(value, columnShouldBePrefixed, prefix) {
   }
   return `${prefix}${value}`;
 }
+function pluralize(singularWord, pluralWord, count) {
+  return count === 1 ? singularWord : pluralWord;
+}
 
 // src/lib/import-gtfs-realtime.ts
-async function fetchGtfsRealtimeData(urlConfig, task) {
-  task.log(`Downloading GTFS-Realtime from ${urlConfig.url}`);
-  const response = await fetch(urlConfig.url, {
-    method: "GET",
-    headers: {
-      ...urlConfig.headers ?? {},
-      "Accept-Encoding": "gzip"
-    },
-    signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
-  });
-  if (response.status !== 200) {
-    task.logWarning(
-      `Unable to download GTFS-Realtime from ${urlConfig.url}. Got status ${response.status}.`
-    );
-    return null;
-  }
-  const buffer = await response.arrayBuffer();
-  const message = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(
-    new Uint8Array(buffer)
-  );
-  return GtfsRealtimeBindings.transit_realtime.FeedMessage.toObject(message, {
-    enums: String,
-    longs: String,
-    bytes: String,
-    defaults: false,
-    arrays: true,
-    objects: true,
-    oneofs: true
-  });
-}
-function removeExpiredRealtimeData(config) {
-  const db = openDb(config);
-  log(config)(`Removing expired GTFS-Realtime data`);
-  db.prepare(
-    `DELETE FROM vehicle_positions WHERE expiration_timestamp <= strftime('%s','now')`
-  ).run();
-  db.prepare(
-    `DELETE FROM trip_updates WHERE expiration_timestamp <= strftime('%s','now')`
-  ).run();
-  db.prepare(
-    `DELETE FROM stop_time_updates WHERE expiration_timestamp <= strftime('%s','now')`
-  ).run();
-  db.prepare(
-    `DELETE FROM service_alerts WHERE expiration_timestamp <= strftime('%s','now')`
-  ).run();
-  db.prepare(
-    `DELETE FROM service_alert_informed_entities WHERE expiration_timestamp <= strftime('%s','now')`
-  ).run();
-  log(config)(`Removed expired GTFS-Realtime data\r`, true);
-}
+var BATCH_SIZE = 1e3;
+var MAX_RETRIES = 3;
+var RETRY_DELAY = 1e3;
 function prepareRealtimeFieldValue(entity, column, task) {
   if (column.name === "created_timestamp") {
     return task.currentTimestamp;
@@ -746,163 +702,265 @@ function prepareRealtimeFieldValue(entity, column, task) {
   );
   return column.type === "json" ? JSON.stringify(prefixedValue) : prefixedValue;
 }
-async function processRealtimeAlerts(db, gtfsRealtimeData, task) {
-  const alertStmt = db.prepare(
-    `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${serviceAlerts.schema.map(() => "?").join(", ")})`
-  );
-  const informedEntityStmt = db.prepare(
-    `REPLACE INTO ${serviceAlertInformedEntities.filenameBase} (${serviceAlertInformedEntities.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${serviceAlertInformedEntities.schema.map(() => "?").join(", ")})`
+function createPreparedStatement(db, model) {
+  const columns = model.schema.map((column) => column.name);
+  const placeholders = model.schema.map(() => "?").join(", ");
+  return db.prepare(
+    `REPLACE INTO ${model.filenameBase} (${columns.join(", ")}) VALUES (${placeholders})`
   );
-  let totalLineCount = 0;
-  db.transaction(() => {
-    for (const entity of gtfsRealtimeData.entity) {
-      const fieldValues = serviceAlerts.schema.map(
-        (column) => prepareRealtimeFieldValue(entity, column, task)
+}
+async function processBatch(items, batchSize, processor) {
+  let totalRecordCount = 0;
+  let totalErrorCount = 0;
+  for (let i = 0; i < items.length; i += batchSize) {
+    const batch = items.slice(i, i + batchSize);
+    try {
+      const result = await processor(batch);
+      totalRecordCount += result.recordCount;
+      totalErrorCount += result.errorCount;
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      totalErrorCount += batch.length;
+      console.error(`Batch processing error: ${errorMessage}`);
+    }
+  }
+  return { recordCount: totalRecordCount, errorCount: totalErrorCount };
+}
+async function fetchGtfsRealtimeData(type, task) {
+  const urlConfig = getUrlConfig(type, task);
+  if (!urlConfig) {
+    return null;
+  }
+  task.log(`Importing - GTFS-Realtime from ${urlConfig.url}`);
+  for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      const response = await fetch(urlConfig.url, {
+        method: "GET",
+        headers: {
+          ...urlConfig.headers ?? {},
+          "Accept-Encoding": "gzip"
+        },
+        signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
+      });
+      if (response.status !== 200) {
+        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+      }
+      const buffer = await response.arrayBuffer();
+      const message = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(
+        new Uint8Array(buffer)
       );
-      try {
-        alertStmt.run(fieldValues);
-        if (entity.alert.informedEntity?.length) {
-          const informedEntities = entity.alert.informedEntity.map(
-            (informedEntity) => {
+      const feedMessage = GtfsRealtimeBindings.transit_realtime.FeedMessage.toObject(message, {
+        enums: String,
+        longs: String,
+        bytes: String,
+        defaults: false,
+        arrays: true,
+        objects: true,
+        oneofs: true
+      });
+      return feedMessage;
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      if (attempt === MAX_RETRIES) {
+        if (task.ignoreErrors) {
+          task.logError(
+            `Failed to fetch ${type} after ${MAX_RETRIES} attempts: ${errorMessage}`
+          );
+          return null;
+        }
+        throw error;
+      }
+      task.logWarning(`Attempt ${attempt} failed for ${type}: ${errorMessage}`);
+      await new Promise(
+        (resolve) => setTimeout(resolve, RETRY_DELAY * attempt)
+      );
+    }
+  }
+  return null;
+}
+function getUrlConfig(type, task) {
+  switch (type) {
+    case "alerts":
+      return task.realtimeAlerts;
+    case "tripupdates":
+      return task.realtimeTripUpdates;
+    case "vehiclepositions":
+      return task.realtimeVehiclePositions;
+    default:
+      return void 0;
+  }
+}
+function createServiceAlertsProcessor(db, task) {
+  const alertStmt = createPreparedStatement(db, serviceAlerts);
+  const informedEntityStmt = createPreparedStatement(
+    db,
+    serviceAlertInformedEntities
+  );
+  return async (batch) => {
+    let recordCount = 0;
+    let errorCount = 0;
+    db.transaction(() => {
+      for (const entity of batch) {
+        try {
+          const alertValues = serviceAlerts.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
+          alertStmt.run(alertValues);
+          recordCount++;
+          if (entity.alert?.informedEntity?.length) {
+            for (const informedEntity of entity.alert.informedEntity) {
               informedEntity.parent = entity;
-              return serviceAlertInformedEntities.schema.map(
+              const entityValues = serviceAlertInformedEntities.schema.map(
                 (column) => prepareRealtimeFieldValue(informedEntity, column, task)
               );
+              informedEntityStmt.run(entityValues);
+              recordCount++;
             }
-          );
-          for (const values of informedEntities) {
-            informedEntityStmt.run(values);
           }
+        } catch (error) {
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          errorCount++;
+          task.logWarning(`Alert processing error: ${errorMessage}`);
         }
-        totalLineCount++;
-      } catch (error) {
-        task.logWarning(`Import error: ${error.message}`);
       }
-    }
-    task.log(
-      `Importing - GTFS-Realtime service alerts - ${totalLineCount} entries imported\r`,
-      true
-    );
-  })();
+    })();
+    return { recordCount, errorCount };
+  };
 }
-async function processRealtimeTripUpdates(db, gtfsRealtimeData, task) {
-  let totalLineCount = 0;
-  const tripUpdateStmt = db.prepare(
-    `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${tripUpdates.schema.map(() => "?").join(", ")})`
+function createTripUpdatesProcessor(db, task) {
+  const tripUpdateStmt = createPreparedStatement(
+    db,
+    tripUpdates
   );
-  const stopTimeStmt = db.prepare(
-    `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${stopTimeUpdates.schema.map(() => "?").join(", ")})`
+  const stopTimeStmt = createPreparedStatement(
+    db,
+    stopTimeUpdates
   );
-  db.transaction(() => {
-    for (const entity of gtfsRealtimeData.entity) {
-      try {
-        const fieldValues = tripUpdates.schema.map(
-          (column) => prepareRealtimeFieldValue(entity, column, task)
-        );
-        tripUpdateStmt.run(fieldValues);
-        for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
-          stopTimeUpdate.parent = entity;
-          const values = stopTimeUpdates.schema.map(
-            (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
-          );
-          stopTimeStmt.run(values);
+  return async (batch) => {
+    let recordCount = 0;
+    let errorCount = 0;
+    db.transaction(() => {
+      for (const entity of batch) {
+        try {
+          const tripUpdateValues = tripUpdates.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
+          tripUpdateStmt.run(tripUpdateValues);
+          recordCount++;
+          if (entity.tripUpdate?.stopTimeUpdate?.length) {
+            for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
+              stopTimeUpdate.parent = entity;
+              const stopTimeValues = stopTimeUpdates.schema.map(
+                (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
+              );
+              stopTimeStmt.run(stopTimeValues);
+              recordCount++;
+            }
+          }
+        } catch (error) {
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          errorCount++;
+          task.logWarning(`Trip update processing error: ${errorMessage}`);
         }
-        totalLineCount++;
-      } catch (error) {
-        task.logWarning(`Import error: ${error.message}`);
       }
-    }
-    task.log(
-      `Importing - GTFS-Realtime trip updates - ${totalLineCount} entries imported\r`,
-      true
-    );
-  })();
+    })();
+    return { recordCount, errorCount };
+  };
 }
-async function processRealtimeVehiclePositions(db, gtfsRealtimeData, task) {
-  let totalLineCount = 0;
-  const vehiclePositionStmt = db.prepare(
-    `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${vehiclePositions.schema.map(() => "?").join(", ")})`
+function createVehiclePositionsProcessor(db, task) {
+  const vehiclePositionStmt = createPreparedStatement(
+    db,
+    vehiclePositions
   );
-  db.transaction(() => {
-    for (const entity of gtfsRealtimeData.entity) {
-      try {
-        const fieldValues = vehiclePositions.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
-        vehiclePositionStmt.run(fieldValues);
-        totalLineCount++;
-      } catch (error) {
-        task.logWarning(`Import error: ${error.message}`);
+  return async (batch) => {
+    let recordCount = 0;
+    let errorCount = 0;
+    db.transaction(() => {
+      for (const entity of batch) {
+        try {
+          const fieldValues = vehiclePositions.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
+          vehiclePositionStmt.run(fieldValues);
+          recordCount++;
+        } catch (error) {
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          errorCount++;
+          task.logWarning(`Vehicle position processing error: ${errorMessage}`);
+        }
       }
+    })();
+    return { recordCount, errorCount };
+  };
+}
+function removeExpiredRealtimeData(config) {
+  const db = openDb(config);
+  log(config)(`Removing expired GTFS-Realtime data`);
+  db.transaction(() => {
+    const tables = [
+      "vehicle_positions",
+      "trip_updates",
+      "stop_time_updates",
+      "service_alerts",
+      "service_alert_informed_entities"
+    ];
+    for (const table of tables) {
+      db.prepare(
+        `DELETE FROM ${table} WHERE expiration_timestamp <= strftime('%s','now')`
+      ).run();
     }
-    task.log(
-      `Importing - GTFS-Realtime vehicle positions - ${totalLineCount} entries imported\r`,
-      true
-    );
   })();
+  log(config)(`Removed expired GTFS-Realtime data\r`, true);
 }
 async function updateGtfsRealtimeData(task) {
-  if (task.realtimeAlerts === void 0 && task.realtimeTripUpdates === void 0 && task.realtimeVehiclePositions === void 0) {
+  if (!task.realtimeAlerts && !task.realtimeTripUpdates && !task.realtimeVehiclePositions) {
     return;
   }
+  const [alertsData, tripUpdatesData, vehiclePositionsData] = await Promise.all(
+    [
+      task.realtimeAlerts?.url ? fetchGtfsRealtimeData("alerts", task) : null,
+      task.realtimeTripUpdates?.url ? fetchGtfsRealtimeData("tripupdates", task) : null,
+      task.realtimeVehiclePositions?.url ? fetchGtfsRealtimeData("vehiclepositions", task) : null
+    ]
+  );
   const db = openDb({ sqlitePath: task.sqlitePath });
-  if (task.realtimeAlerts?.url) {
-    try {
-      const alertsData = await fetchGtfsRealtimeData(task.realtimeAlerts, task);
-      if (alertsData?.entity) {
-        await processRealtimeAlerts(db, alertsData, task);
-      }
-    } catch (error) {
-      if (task.ignoreErrors) {
-        task.logError(error.message);
-      } else {
-        throw error;
-      }
-    }
+  const recordCounts = {
+    alerts: 0,
+    tripupdates: 0,
+    vehiclepositions: 0
+  };
+  const processingPromises = [];
+  if (alertsData?.entity?.length) {
+    processingPromises.push(
+      processBatch(
+        alertsData.entity,
+        BATCH_SIZE,
+        createServiceAlertsProcessor(db, task)
+      ).then((result) => {
+        recordCounts.alerts = result.recordCount;
+      })
+    );
   }
-  if (task.realtimeTripUpdates?.url) {
-    try {
-      const tripUpdatesData = await fetchGtfsRealtimeData(
-        task.realtimeTripUpdates,
-        task
-      );
-      if (tripUpdatesData?.entity) {
-        await processRealtimeTripUpdates(db, tripUpdatesData, task);
-      }
-    } catch (error) {
-      if (task.ignoreErrors) {
-        task.logError(error.message);
-      } else {
-        throw error;
-      }
-    }
+  if (tripUpdatesData?.entity?.length) {
+    processingPromises.push(
+      processBatch(
+        tripUpdatesData.entity,
+        BATCH_SIZE,
+        createTripUpdatesProcessor(db, task)
+      ).then((result) => {
+        recordCounts.tripupdates = result.recordCount;
+      })
+    );
   }
-  if (task.realtimeVehiclePositions?.url) {
-    try {
-      const vehiclePositionsData = await fetchGtfsRealtimeData(
-        task.realtimeVehiclePositions,
-        task
-      );
-      if (vehiclePositionsData?.entity) {
-        await processRealtimeVehiclePositions(db, vehiclePositionsData, task);
-      }
-    } catch (error) {
-      if (task.ignoreErrors) {
-        task.logError(error.message);
-      } else {
-        throw error;
-      }
-    }
+  if (vehiclePositionsData?.entity?.length) {
+    processingPromises.push(
+      processBatch(
+        vehiclePositionsData.entity,
+        BATCH_SIZE,
+        createVehiclePositionsProcessor(db, task)
+      ).then((result) => {
+        recordCounts.vehiclepositions = result.recordCount;
+      })
+    );
   }
-  task.log(`GTFS-Realtime data import complete`);
+  await Promise.all(processingPromises);
+  task.log(
+    `GTFS-Realtime import complete: ${recordCounts.alerts} alerts, ${recordCounts.tripupdates} trip updates, ${recordCounts.vehiclepositions} vehicle positions`
+  );
 }
 async function updateGtfsRealtime(initialConfig) {
   const config = setDefaultConfig(initialConfig);
@@ -912,9 +970,9 @@ async function updateGtfsRealtime(initialConfig) {
     const agencyCount = config.agencies.length;
     log(config)(
       `Starting GTFS-Realtime refresh for ${pluralize(
+        "agency",
         "agencies",
-        agencyCount,
-        true
+        agencyCount
       )} using SQLite database at ${config.sqlitePath}`
     );
     removeExpiredRealtimeData(config);
@@ -936,8 +994,9 @@ async function updateGtfsRealtime(initialConfig) {
         };
         await updateGtfsRealtimeData(task);
       } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
         if (config.ignoreErrors) {
-          logError(config)(error.message);
+          logError(config)(errorMessage);
         } else {
           throw error;
         }
@@ -945,14 +1004,14 @@ async function updateGtfsRealtime(initialConfig) {
     });
     log(config)(
       `Completed GTFS-Realtime refresh for ${pluralize(
+        "agency",
         "agencies",
-        agencyCount,
-        true
+        agencyCount
       )}
 `
     );
   } catch (error) {
-    if (error?.code === "SQLITE_CANTOPEN") {
+    if (error.code === "SQLITE_CANTOPEN") {
       logError(config)(
         `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
       );
@@ -965,11 +1024,9 @@ async function updateGtfsRealtime(initialConfig) {
 import path3 from "path";
 import { writeFile as writeFile2 } from "fs/promises";
 import { without, compact as compact2 } from "lodash-es";
-import pluralize3 from "pluralize";
 import { stringify } from "csv-stringify";
 import sqlString2 from "sqlstring-sqlite";
 import mapSeries3 from "promise-map-series";
-import untildify4 from "untildify";
 
 // src/lib/advancedQuery.ts
 import sqlString3 from "sqlstring-sqlite";