gtfs 4.14.2 → 4.14.4

package/dist/bin/gtfsrealtime-update.js

@@ -608,6 +608,7 @@ function setDefaultConfig(initialConfig) {
   const defaults = {
     sqlitePath: ":memory:",
     ignoreDuplicates: false,
+    ignoreErrors: false,
     gtfsRealtimeExpirationSeconds: 0
   };
   return {
@@ -633,7 +634,7 @@ var downloadGtfsRealtimeData = async (urlAndHeaders, task) => {
   });
   if (response.status !== 200) {
     task.logWarning(
-      `Unable to download GTFS-Realtime from ${urlAndHeaders.url}`
+      `Unable to download GTFS-Realtime from ${urlAndHeaders.url}. Got status ${response.status}.`
     );
     return null;
   }
@@ -722,103 +723,127 @@ var updateRealtimeData = async (task) => {
     sqlitePath: task.sqlitePath
   });
   if (task.realtimeAlerts?.url) {
-    const gtfsRealtimeData = await downloadGtfsRealtimeData(
-      task.realtimeAlerts,
-      task
-    );
-    if (gtfsRealtimeData?.entity) {
-      task.log(`Download successful`);
-      let totalLineCount = 0;
-      for (const entity of gtfsRealtimeData.entity) {
-        const fieldValues = serviceAlerts.schema.map(
-          (column) => prepareRealtimeValue(entity, column, task)
-        );
-        try {
-          db.prepare(
-            `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-          ).run();
-        } catch (error) {
-          task.logWarning("Import error: " + error.message);
-        }
-        const alertTargetArray = [];
-        for (const informedEntity of entity.alert.informedEntity) {
-          informedEntity.parent = entity;
-          const subValues = serviceAlertTargets.schema.map(
-            (column) => prepareRealtimeValue(informedEntity, column, task)
+    try {
+      const gtfsRealtimeData = await downloadGtfsRealtimeData(
+        task.realtimeAlerts,
+        task
+      );
+      if (gtfsRealtimeData?.entity) {
+        task.log(`Download successful`);
+        let totalLineCount = 0;
+        for (const entity of gtfsRealtimeData.entity) {
+          const fieldValues = serviceAlerts.schema.map(
+            (column) => prepareRealtimeValue(entity, column, task)
           );
-          alertTargetArray.push(`(${subValues.join(", ")})`);
-          totalLineCount++;
-        }
-        try {
-          db.prepare(
-            `REPLACE INTO ${serviceAlertTargets.filenameBase} (${serviceAlertTargets.schema.map((column) => column.name).join(", ")}) VALUES ${alertTargetArray.join(", ")}`
-          ).run();
-        } catch (error) {
-          task.logWarning("Import error: " + error.message);
+          try {
+            db.prepare(
+              `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
+            ).run();
+          } catch (error) {
+            task.logWarning("Import error: " + error.message);
+          }
+          const alertTargetArray = [];
+          for (const informedEntity of entity.alert.informedEntity) {
+            informedEntity.parent = entity;
+            const subValues = serviceAlertTargets.schema.map(
+              (column) => prepareRealtimeValue(informedEntity, column, task)
+            );
+            alertTargetArray.push(`(${subValues.join(", ")})`);
+            totalLineCount++;
+          }
+          try {
+            db.prepare(
+              `REPLACE INTO ${serviceAlertTargets.filenameBase} (${serviceAlertTargets.schema.map((column) => column.name).join(", ")}) VALUES ${alertTargetArray.join(", ")}`
+            ).run();
+          } catch (error) {
+            task.logWarning("Import error: " + error.message);
+          }
+          task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
         }
-        task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
+      }
+    } catch (error) {
+      if (task.ignoreErrors) {
+        task.logError(error.message);
+      } else {
+        throw error;
       }
     }
   }
   if (task.realtimeTripUpdates?.url) {
-    const gtfsRealtimeData = await downloadGtfsRealtimeData(
-      task.realtimeTripUpdates,
-      task
-    );
-    if (gtfsRealtimeData?.entity) {
-      task.log(`Download successful`);
-      let totalLineCount = 0;
-      for (const entity of gtfsRealtimeData.entity) {
-        const fieldValues = tripUpdates.schema.map(
-          (column) => prepareRealtimeValue(entity, column, task)
-        );
-        try {
-          db.prepare(
-            `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-          ).run();
-        } catch (error) {
-          task.logWarning("Import error: " + error.message);
-        }
-        const stopTimeUpdateArray = [];
-        for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
-          stopTimeUpdate.parent = entity;
-          const subValues = stopTimeUpdates.schema.map(
-            (column) => prepareRealtimeValue(stopTimeUpdate, column, task)
+    try {
+      const gtfsRealtimeData = await downloadGtfsRealtimeData(
+        task.realtimeTripUpdates,
+        task
+      );
+      if (gtfsRealtimeData?.entity) {
+        task.log(`Download successful`);
+        let totalLineCount = 0;
+        for (const entity of gtfsRealtimeData.entity) {
+          const fieldValues = tripUpdates.schema.map(
+            (column) => prepareRealtimeValue(entity, column, task)
           );
-          stopTimeUpdateArray.push(`(${subValues.join(", ")})`);
-          totalLineCount++;
-        }
-        try {
-          db.prepare(
-            `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(", ")}) VALUES ${stopTimeUpdateArray.join(", ")}`
-          ).run();
-        } catch (error) {
-          task.logWarning("Import error: " + error.message);
+          try {
+            db.prepare(
+              `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
+            ).run();
+          } catch (error) {
+            task.logWarning("Import error: " + error.message);
+          }
+          const stopTimeUpdateArray = [];
+          for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
+            stopTimeUpdate.parent = entity;
+            const subValues = stopTimeUpdates.schema.map(
+              (column) => prepareRealtimeValue(stopTimeUpdate, column, task)
+            );
+            stopTimeUpdateArray.push(`(${subValues.join(", ")})`);
+            totalLineCount++;
+          }
+          try {
+            db.prepare(
+              `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(", ")}) VALUES ${stopTimeUpdateArray.join(", ")}`
+            ).run();
+          } catch (error) {
+            task.logWarning("Import error: " + error.message);
+          }
+          task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
         }
-        task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
+      }
+    } catch (error) {
+      if (task.ignoreErrors) {
+        task.logError(error.message);
+      } else {
+        throw error;
       }
     }
   }
   if (task.realtimeVehiclePositions?.url) {
-    const gtfsRealtimeData = await downloadGtfsRealtimeData(
-      task.realtimeVehiclePositions,
-      task
-    );
-    if (gtfsRealtimeData?.entity) {
-      task.log(`Download successful`);
-      let totalLineCount = 0;
-      for (const entity of gtfsRealtimeData.entity) {
-        const fieldValues = vehiclePositions.schema.map(
-          (column) => prepareRealtimeValue(entity, column, task)
-        );
-        try {
-          db.prepare(
-            `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-          ).run();
-        } catch (error) {
-          task.logWarning("Import error: " + error.message);
+    try {
+      const gtfsRealtimeData = await downloadGtfsRealtimeData(
+        task.realtimeVehiclePositions,
+        task
+      );
+      if (gtfsRealtimeData?.entity) {
+        task.log(`Download successful`);
+        let totalLineCount = 0;
+        for (const entity of gtfsRealtimeData.entity) {
+          const fieldValues = vehiclePositions.schema.map(
+            (column) => prepareRealtimeValue(entity, column, task)
+          );
+          try {
+            db.prepare(
+              `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
+            ).run();
+          } catch (error) {
+            task.logWarning("Import error: " + error.message);
+          }
+          task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
         }
-        task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
+      }
+    } catch (error) {
+      if (task.ignoreErrors) {
+        task.logError(error.message);
+      } else {
+        throw error;
       }
     }
   }
@@ -841,14 +866,15 @@ async function updateGtfsRealtime(initialConfig) {
       )} using SQLite database at ${config.sqlitePath}`
     );
     deleteExpiredRealtimeData(config);
-    await Promise.all(
-      config.agencies.map(async (agency2) => {
+    await mapSeries(config.agencies, async (agency2) => {
+      try {
         const task = {
           realtimeAlerts: agency2.realtimeAlerts,
           realtimeTripUpdates: agency2.realtimeTripUpdates,
           realtimeVehiclePositions: agency2.realtimeVehiclePositions,
           downloadTimeout: config.downloadTimeout,
           gtfsRealtimeExpirationSeconds: config.gtfsRealtimeExpirationSeconds,
+          ignoreErrors: config.ignoreErrors,
           sqlitePath: config.sqlitePath,
           currentTimestamp: Math.floor(Date.now() / 1e3),
           log: log2,
@@ -856,8 +882,14 @@ async function updateGtfsRealtime(initialConfig) {
           logError: logError2
         };
         await updateRealtimeData(task);
-      })
-    );
+      } catch (error) {
+        if (config.ignoreErrors) {
+          logError2(error.message);
+        } else {
+          throw error;
+        }
+      }
+    });
     log2(
       `Completed GTFS-Realtime refresh for ${pluralize(
         "agencies",