gtfs 4.14.3 → 4.14.4

package/dist/bin/gtfs-import.js

@@ -3318,6 +3318,7 @@ function setDefaultConfig(initialConfig) {
   const defaults = {
     sqlitePath: ":memory:",
     ignoreDuplicates: false,
+    ignoreErrors: false,
     gtfsRealtimeExpirationSeconds: 0
   };
   return {
@@ -3346,6 +3347,9 @@ function padLeadingZeros(time) {
 
 // src/lib/import.ts
 var downloadFiles = async (task) => {
+  if (!task.url) {
+    throw new Error("No `url` specified in config");
+  }
   task.log(`Downloading GTFS from ${task.url}`);
   task.path = `${task.downloadDir}/gtfs.zip`;
   const response = await fetch(task.url, {
@@ -3354,7 +3358,9 @@ var downloadFiles = async (task) => {
     signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
   });
   if (response.status !== 200) {
-    throw new Error(`Unable to download GTFS from ${task.url}`);
+    throw new Error(
+      `Unable to download GTFS from ${task.url}. Got status ${response.status}.`
+    );
   }
   const buffer = await response.arrayBuffer();
   await writeFile(task.path, Buffer.from(buffer));
@@ -3372,7 +3378,7 @@ var downloadGtfsRealtimeData = async (urlAndHeaders, task) => {
   });
   if (response.status !== 200) {
     task.logWarning(
-      `Unable to download GTFS-Realtime from ${urlAndHeaders.url}`
+      `Unable to download GTFS-Realtime from ${urlAndHeaders.url}. Got status ${response.status}.`
     );
     return null;
   }
@@ -3440,103 +3446,127 @@ var updateRealtimeData = async (task) => {
     sqlitePath: task.sqlitePath
   });
   if (task.realtimeAlerts?.url) {
-    const gtfsRealtimeData = await downloadGtfsRealtimeData(
-      task.realtimeAlerts,
-      task
-    );
-    if (gtfsRealtimeData?.entity) {
-      task.log(`Download successful`);
-      let totalLineCount = 0;
-      for (const entity of gtfsRealtimeData.entity) {
-        const fieldValues = serviceAlerts.schema.map(
-          (column) => prepareRealtimeValue(entity, column, task)
-        );
-        try {
-          db.prepare(
-            `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-          ).run();
-        } catch (error) {
-          task.logWarning("Import error: " + error.message);
-        }
-        const alertTargetArray = [];
-        for (const informedEntity of entity.alert.informedEntity) {
-          informedEntity.parent = entity;
-          const subValues = serviceAlertTargets.schema.map(
-            (column) => prepareRealtimeValue(informedEntity, column, task)
+    try {
+      const gtfsRealtimeData = await downloadGtfsRealtimeData(
+        task.realtimeAlerts,
+        task
+      );
+      if (gtfsRealtimeData?.entity) {
+        task.log(`Download successful`);
+        let totalLineCount = 0;
+        for (const entity of gtfsRealtimeData.entity) {
+          const fieldValues = serviceAlerts.schema.map(
+            (column) => prepareRealtimeValue(entity, column, task)
           );
-          alertTargetArray.push(`(${subValues.join(", ")})`);
-          totalLineCount++;
-        }
-        try {
-          db.prepare(
-            `REPLACE INTO ${serviceAlertTargets.filenameBase} (${serviceAlertTargets.schema.map((column) => column.name).join(", ")}) VALUES ${alertTargetArray.join(", ")}`
-          ).run();
-        } catch (error) {
-          task.logWarning("Import error: " + error.message);
+          try {
+            db.prepare(
+              `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
+            ).run();
+          } catch (error) {
+            task.logWarning("Import error: " + error.message);
+          }
+          const alertTargetArray = [];
+          for (const informedEntity of entity.alert.informedEntity) {
+            informedEntity.parent = entity;
+            const subValues = serviceAlertTargets.schema.map(
+              (column) => prepareRealtimeValue(informedEntity, column, task)
+            );
+            alertTargetArray.push(`(${subValues.join(", ")})`);
+            totalLineCount++;
+          }
+          try {
+            db.prepare(
+              `REPLACE INTO ${serviceAlertTargets.filenameBase} (${serviceAlertTargets.schema.map((column) => column.name).join(", ")}) VALUES ${alertTargetArray.join(", ")}`
+            ).run();
+          } catch (error) {
+            task.logWarning("Import error: " + error.message);
+          }
+          task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
         }
-        task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
+      }
+    } catch (error) {
+      if (task.ignoreErrors) {
+        task.logError(error.message);
+      } else {
+        throw error;
       }
     }
   }
   if (task.realtimeTripUpdates?.url) {
-    const gtfsRealtimeData = await downloadGtfsRealtimeData(
-      task.realtimeTripUpdates,
-      task
-    );
-    if (gtfsRealtimeData?.entity) {
-      task.log(`Download successful`);
-      let totalLineCount = 0;
-      for (const entity of gtfsRealtimeData.entity) {
-        const fieldValues = tripUpdates.schema.map(
-          (column) => prepareRealtimeValue(entity, column, task)
-        );
-        try {
-          db.prepare(
-            `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-          ).run();
-        } catch (error) {
-          task.logWarning("Import error: " + error.message);
-        }
-        const stopTimeUpdateArray = [];
-        for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
-          stopTimeUpdate.parent = entity;
-          const subValues = stopTimeUpdates.schema.map(
-            (column) => prepareRealtimeValue(stopTimeUpdate, column, task)
+    try {
+      const gtfsRealtimeData = await downloadGtfsRealtimeData(
+        task.realtimeTripUpdates,
+        task
+      );
+      if (gtfsRealtimeData?.entity) {
+        task.log(`Download successful`);
+        let totalLineCount = 0;
+        for (const entity of gtfsRealtimeData.entity) {
+          const fieldValues = tripUpdates.schema.map(
+            (column) => prepareRealtimeValue(entity, column, task)
           );
-          stopTimeUpdateArray.push(`(${subValues.join(", ")})`);
-          totalLineCount++;
-        }
-        try {
-          db.prepare(
-            `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(", ")}) VALUES ${stopTimeUpdateArray.join(", ")}`
-          ).run();
-        } catch (error) {
-          task.logWarning("Import error: " + error.message);
+          try {
+            db.prepare(
+              `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
+            ).run();
+          } catch (error) {
+            task.logWarning("Import error: " + error.message);
+          }
+          const stopTimeUpdateArray = [];
+          for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
+            stopTimeUpdate.parent = entity;
+            const subValues = stopTimeUpdates.schema.map(
+              (column) => prepareRealtimeValue(stopTimeUpdate, column, task)
+            );
+            stopTimeUpdateArray.push(`(${subValues.join(", ")})`);
+            totalLineCount++;
+          }
+          try {
+            db.prepare(
+              `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(", ")}) VALUES ${stopTimeUpdateArray.join(", ")}`
+            ).run();
+          } catch (error) {
+            task.logWarning("Import error: " + error.message);
+          }
+          task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
         }
-        task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
+      }
+    } catch (error) {
+      if (task.ignoreErrors) {
+        task.logError(error.message);
+      } else {
+        throw error;
       }
     }
   }
   if (task.realtimeVehiclePositions?.url) {
-    const gtfsRealtimeData = await downloadGtfsRealtimeData(
-      task.realtimeVehiclePositions,
-      task
-    );
-    if (gtfsRealtimeData?.entity) {
-      task.log(`Download successful`);
-      let totalLineCount = 0;
-      for (const entity of gtfsRealtimeData.entity) {
-        const fieldValues = vehiclePositions.schema.map(
-          (column) => prepareRealtimeValue(entity, column, task)
-        );
-        try {
-          db.prepare(
-            `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-          ).run();
-        } catch (error) {
-          task.logWarning("Import error: " + error.message);
+    try {
+      const gtfsRealtimeData = await downloadGtfsRealtimeData(
+        task.realtimeVehiclePositions,
+        task
+      );
+      if (gtfsRealtimeData?.entity) {
+        task.log(`Download successful`);
+        let totalLineCount = 0;
+        for (const entity of gtfsRealtimeData.entity) {
+          const fieldValues = vehiclePositions.schema.map(
+            (column) => prepareRealtimeValue(entity, column, task)
+          );
+          try {
+            db.prepare(
+              `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
+            ).run();
+          } catch (error) {
+            task.logWarning("Import error: " + error.message);
+          }
+          task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
         }
-        task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
+      }
+    } catch (error) {
+      if (task.ignoreErrors) {
+        task.logError(error.message);
+      } else {
+        throw error;
       }
     }
   }
@@ -3547,6 +3577,9 @@ var getTextFiles = async (folderPath) => {
   return files.filter((filename) => filename.slice(-3) === "txt");
 };
 var readFiles = async (task) => {
+  if (!task.path) {
+    throw new Error("No `path` specified in config");
+  }
   const gtfsPath = untildify3(task.path);
   task.log(`Importing GTFS from ${task.path}\r`);
   if (path2.extname(gtfsPath) === ".zip") {
@@ -3843,47 +3876,45 @@ async function importGtfs(initialConfig) {
       )} using SQLite database at ${config.sqlitePath}`
     );
     createTables(db);
-    await mapSeries(
-      config.agencies,
-      async (agency2) => {
-        try {
-          const tempPath = temporaryDirectory();
-          const task = {
-            exclude: agency2.exclude,
-            url: agency2.url,
-            headers: agency2.headers,
-            realtimeAlerts: agency2.realtimeAlerts,
-            realtimeTripUpdates: agency2.realtimeTripUpdates,
-            realtimeVehiclePositions: agency2.realtimeVehiclePositions,
-            downloadDir: tempPath,
-            downloadTimeout: config.downloadTimeout,
-            gtfsRealtimeExpirationSeconds: config.gtfsRealtimeExpirationSeconds,
-            path: agency2.path,
-            csvOptions: config.csvOptions || {},
-            ignoreDuplicates: config.ignoreDuplicates,
-            sqlitePath: config.sqlitePath,
-            prefix: agency2.prefix,
-            currentTimestamp: Math.floor(Date.now() / 1e3),
-            log: log2,
-            logWarning: logWarning2,
-            logError: logError2
-          };
-          if (task.url) {
-            await downloadFiles(task);
-          }
-          await readFiles(task);
-          await importFiles(task);
-          await updateRealtimeData(task);
-          await rm2(tempPath, { recursive: true });
-        } catch (error) {
-          if (config.ignoreErrors) {
-            logError2(error.message);
-          } else {
-            throw error;
-          }
+    await mapSeries(config.agencies, async (agency2) => {
+      try {
+        const tempPath = temporaryDirectory();
+        const task = {
+          exclude: agency2.exclude,
+          url: agency2.url,
+          headers: agency2.headers,
+          realtimeAlerts: agency2.realtimeAlerts,
+          realtimeTripUpdates: agency2.realtimeTripUpdates,
+          realtimeVehiclePositions: agency2.realtimeVehiclePositions,
+          downloadDir: tempPath,
+          downloadTimeout: config.downloadTimeout,
+          gtfsRealtimeExpirationSeconds: config.gtfsRealtimeExpirationSeconds,
+          path: agency2.path,
+          csvOptions: config.csvOptions || {},
+          ignoreDuplicates: config.ignoreDuplicates,
+          ignoreErrors: config.ignoreErrors,
+          sqlitePath: config.sqlitePath,
+          prefix: agency2.prefix,
+          currentTimestamp: Math.floor(Date.now() / 1e3),
+          log: log2,
+          logWarning: logWarning2,
+          logError: logError2
+        };
+        if (task.url) {
+          await downloadFiles(task);
+        }
+        await readFiles(task);
+        await importFiles(task);
+        await updateRealtimeData(task);
+        await rm2(tempPath, { recursive: true });
+      } catch (error) {
+        if (config.ignoreErrors) {
+          logError2(error.message);
+        } else {
+          throw error;
         }
       }
-    );
+    });
     log2(
       `Completed GTFS import for ${pluralize("agency", agencyCount, true)}
 `