gtfs 4.14.2 → 4.14.4

package/dist/index.js

@@ -3383,6 +3383,7 @@ function setDefaultConfig(initialConfig) {
   const defaults = {
     sqlitePath: ":memory:",
     ignoreDuplicates: false,
+    ignoreErrors: false,
     gtfsRealtimeExpirationSeconds: 0
   };
   return {
@@ -3481,6 +3482,9 @@ function formatOrderByClause(orderBy2) {
 
 // src/lib/import.ts
 var downloadFiles = async (task) => {
+  if (!task.url) {
+    throw new Error("No `url` specified in config");
+  }
   task.log(`Downloading GTFS from ${task.url}`);
   task.path = `${task.downloadDir}/gtfs.zip`;
   const response = await fetch(task.url, {
@@ -3489,7 +3493,9 @@ var downloadFiles = async (task) => {
     signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
   });
   if (response.status !== 200) {
-    throw new Error(`Unable to download GTFS from ${task.url}`);
+    throw new Error(
+      `Unable to download GTFS from ${task.url}. Got status ${response.status}.`
+    );
   }
   const buffer = await response.arrayBuffer();
   await writeFile(task.path, Buffer.from(buffer));
@@ -3507,7 +3513,7 @@ var downloadGtfsRealtimeData = async (urlAndHeaders, task) => {
   });
   if (response.status !== 200) {
     task.logWarning(
-      `Unable to download GTFS-Realtime from ${urlAndHeaders.url}`
+      `Unable to download GTFS-Realtime from ${urlAndHeaders.url}. Got status ${response.status}.`
     );
     return null;
   }
@@ -3596,103 +3602,127 @@ var updateRealtimeData = async (task) => {
     sqlitePath: task.sqlitePath
   });
   if (task.realtimeAlerts?.url) {
-    const gtfsRealtimeData = await downloadGtfsRealtimeData(
-      task.realtimeAlerts,
-      task
-    );
-    if (gtfsRealtimeData?.entity) {
-      task.log(`Download successful`);
-      let totalLineCount = 0;
-      for (const entity of gtfsRealtimeData.entity) {
-        const fieldValues = serviceAlerts.schema.map(
-          (column) => prepareRealtimeValue(entity, column, task)
-        );
-        try {
-          db.prepare(
-            `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-          ).run();
-        } catch (error) {
-          task.logWarning("Import error: " + error.message);
-        }
-        const alertTargetArray = [];
-        for (const informedEntity of entity.alert.informedEntity) {
-          informedEntity.parent = entity;
-          const subValues = serviceAlertTargets.schema.map(
-            (column) => prepareRealtimeValue(informedEntity, column, task)
+    try {
+      const gtfsRealtimeData = await downloadGtfsRealtimeData(
+        task.realtimeAlerts,
+        task
+      );
+      if (gtfsRealtimeData?.entity) {
+        task.log(`Download successful`);
+        let totalLineCount = 0;
+        for (const entity of gtfsRealtimeData.entity) {
+          const fieldValues = serviceAlerts.schema.map(
+            (column) => prepareRealtimeValue(entity, column, task)
           );
-          alertTargetArray.push(`(${subValues.join(", ")})`);
-          totalLineCount++;
-        }
-        try {
-          db.prepare(
-            `REPLACE INTO ${serviceAlertTargets.filenameBase} (${serviceAlertTargets.schema.map((column) => column.name).join(", ")}) VALUES ${alertTargetArray.join(", ")}`
-          ).run();
-        } catch (error) {
-          task.logWarning("Import error: " + error.message);
+          try {
+            db.prepare(
+              `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
+            ).run();
+          } catch (error) {
+            task.logWarning("Import error: " + error.message);
+          }
+          const alertTargetArray = [];
+          for (const informedEntity of entity.alert.informedEntity) {
+            informedEntity.parent = entity;
+            const subValues = serviceAlertTargets.schema.map(
+              (column) => prepareRealtimeValue(informedEntity, column, task)
+            );
+            alertTargetArray.push(`(${subValues.join(", ")})`);
+            totalLineCount++;
+          }
+          try {
+            db.prepare(
+              `REPLACE INTO ${serviceAlertTargets.filenameBase} (${serviceAlertTargets.schema.map((column) => column.name).join(", ")}) VALUES ${alertTargetArray.join(", ")}`
+            ).run();
+          } catch (error) {
+            task.logWarning("Import error: " + error.message);
+          }
+          task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
         }
-        task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
+      }
+    } catch (error) {
+      if (task.ignoreErrors) {
+        task.logError(error.message);
+      } else {
+        throw error;
       }
     }
   }
   if (task.realtimeTripUpdates?.url) {
-    const gtfsRealtimeData = await downloadGtfsRealtimeData(
-      task.realtimeTripUpdates,
-      task
-    );
-    if (gtfsRealtimeData?.entity) {
-      task.log(`Download successful`);
-      let totalLineCount = 0;
-      for (const entity of gtfsRealtimeData.entity) {
-        const fieldValues = tripUpdates.schema.map(
-          (column) => prepareRealtimeValue(entity, column, task)
-        );
-        try {
-          db.prepare(
-            `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-          ).run();
-        } catch (error) {
-          task.logWarning("Import error: " + error.message);
-        }
-        const stopTimeUpdateArray = [];
-        for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
-          stopTimeUpdate.parent = entity;
-          const subValues = stopTimeUpdates.schema.map(
-            (column) => prepareRealtimeValue(stopTimeUpdate, column, task)
+    try {
+      const gtfsRealtimeData = await downloadGtfsRealtimeData(
+        task.realtimeTripUpdates,
+        task
+      );
+      if (gtfsRealtimeData?.entity) {
+        task.log(`Download successful`);
+        let totalLineCount = 0;
+        for (const entity of gtfsRealtimeData.entity) {
+          const fieldValues = tripUpdates.schema.map(
+            (column) => prepareRealtimeValue(entity, column, task)
           );
-          stopTimeUpdateArray.push(`(${subValues.join(", ")})`);
-          totalLineCount++;
-        }
-        try {
-          db.prepare(
-            `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(", ")}) VALUES ${stopTimeUpdateArray.join(", ")}`
-          ).run();
-        } catch (error) {
-          task.logWarning("Import error: " + error.message);
+          try {
+            db.prepare(
+              `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
+            ).run();
+          } catch (error) {
+            task.logWarning("Import error: " + error.message);
+          }
+          const stopTimeUpdateArray = [];
+          for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
+            stopTimeUpdate.parent = entity;
+            const subValues = stopTimeUpdates.schema.map(
+              (column) => prepareRealtimeValue(stopTimeUpdate, column, task)
+            );
+            stopTimeUpdateArray.push(`(${subValues.join(", ")})`);
+            totalLineCount++;
+          }
+          try {
+            db.prepare(
+              `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(", ")}) VALUES ${stopTimeUpdateArray.join(", ")}`
+            ).run();
+          } catch (error) {
+            task.logWarning("Import error: " + error.message);
+          }
+          task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
         }
-        task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
+      }
+    } catch (error) {
+      if (task.ignoreErrors) {
+        task.logError(error.message);
+      } else {
+        throw error;
       }
     }
   }
   if (task.realtimeVehiclePositions?.url) {
-    const gtfsRealtimeData = await downloadGtfsRealtimeData(
-      task.realtimeVehiclePositions,
-      task
-    );
-    if (gtfsRealtimeData?.entity) {
-      task.log(`Download successful`);
-      let totalLineCount = 0;
-      for (const entity of gtfsRealtimeData.entity) {
-        const fieldValues = vehiclePositions.schema.map(
-          (column) => prepareRealtimeValue(entity, column, task)
-        );
-        try {
-          db.prepare(
-            `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-          ).run();
-        } catch (error) {
-          task.logWarning("Import error: " + error.message);
+    try {
+      const gtfsRealtimeData = await downloadGtfsRealtimeData(
+        task.realtimeVehiclePositions,
+        task
+      );
+      if (gtfsRealtimeData?.entity) {
+        task.log(`Download successful`);
+        let totalLineCount = 0;
+        for (const entity of gtfsRealtimeData.entity) {
+          const fieldValues = vehiclePositions.schema.map(
+            (column) => prepareRealtimeValue(entity, column, task)
+          );
+          try {
+            db.prepare(
+              `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
+            ).run();
+          } catch (error) {
+            task.logWarning("Import error: " + error.message);
+          }
+          task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
         }
-        task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
+      }
+    } catch (error) {
+      if (task.ignoreErrors) {
+        task.logError(error.message);
+      } else {
+        throw error;
       }
     }
   }
@@ -3703,6 +3733,9 @@ var getTextFiles = async (folderPath) => {
   return files.filter((filename) => filename.slice(-3) === "txt");
 };
 var readFiles = async (task) => {
+  if (!task.path) {
+    throw new Error("No `path` specified in config");
+  }
   const gtfsPath = untildify3(task.path);
   task.log(`Importing GTFS from ${task.path}\r`);
   if (path.extname(gtfsPath) === ".zip") {
@@ -3999,9 +4032,8 @@ async function importGtfs(initialConfig) {
       )} using SQLite database at ${config.sqlitePath}`
     );
     createTables(db);
-    await mapSeries(
-      config.agencies,
-      async (agency2) => {
+    await mapSeries(config.agencies, async (agency2) => {
+      try {
         const tempPath = temporaryDirectory();
         const task = {
           exclude: agency2.exclude,
@@ -4016,6 +4048,7 @@ async function importGtfs(initialConfig) {
           path: agency2.path,
           csvOptions: config.csvOptions || {},
           ignoreDuplicates: config.ignoreDuplicates,
+          ignoreErrors: config.ignoreErrors,
           sqlitePath: config.sqlitePath,
           prefix: agency2.prefix,
           currentTimestamp: Math.floor(Date.now() / 1e3),
@@ -4023,23 +4056,21 @@ async function importGtfs(initialConfig) {
           logWarning: logWarning2,
           logError: logError2
         };
-        try {
-          if (task.url) {
-            await downloadFiles(task);
-          }
-          await readFiles(task);
-          await importFiles(task);
-          await updateRealtimeData(task);
-          await rm2(tempPath, { recursive: true });
-        } catch (error) {
-          if (config.ignoreErrors) {
-            logError2(error.message);
-          } else {
-            throw error;
-          }
+        if (task.url) {
+          await downloadFiles(task);
+        }
+        await readFiles(task);
+        await importFiles(task);
+        await updateRealtimeData(task);
+        await rm2(tempPath, { recursive: true });
+      } catch (error) {
+        if (config.ignoreErrors) {
+          logError2(error.message);
+        } else {
+          throw error;
         }
       }
-    );
+    });
     log2(
       `Completed GTFS import for ${pluralize("agency", agencyCount, true)}
 `
@@ -4070,14 +4101,15 @@ async function updateGtfsRealtime(initialConfig) {
       )} using SQLite database at ${config.sqlitePath}`
     );
     deleteExpiredRealtimeData(config);
-    await Promise.all(
-      config.agencies.map(async (agency2) => {
+    await mapSeries(config.agencies, async (agency2) => {
+      try {
         const task = {
           realtimeAlerts: agency2.realtimeAlerts,
           realtimeTripUpdates: agency2.realtimeTripUpdates,
           realtimeVehiclePositions: agency2.realtimeVehiclePositions,
           downloadTimeout: config.downloadTimeout,
           gtfsRealtimeExpirationSeconds: config.gtfsRealtimeExpirationSeconds,
+          ignoreErrors: config.ignoreErrors,
           sqlitePath: config.sqlitePath,
           currentTimestamp: Math.floor(Date.now() / 1e3),
           log: log2,
@@ -4085,8 +4117,14 @@ async function updateGtfsRealtime(initialConfig) {
           logError: logError2
         };
         await updateRealtimeData(task);
-      })
-    );
+      } catch (error) {
+        if (config.ignoreErrors) {
+          logError2(error.message);
+        } else {
+          throw error;
+        }
+      }
+    });
     log2(
       `Completed GTFS-Realtime refresh for ${pluralize(
         "agencies",