gtfs 4.15.0 → 4.15.2

package/dist/bin/gtfs-import.js

@@ -143,6 +143,7 @@ import { parse } from "csv-parse";
 import pluralize2 from "pluralize";
 import stripBomStream from "strip-bom-stream";
 import { temporaryDirectory } from "tempy";
+import Timer from "timer-machine";
 import untildify3 from "untildify";
 import mapSeries2 from "promise-map-series";
 
@@ -3709,41 +3710,50 @@ var createGtfsIndexes = (db) => {
 var formatGtfsLine = (line, model, totalLineCount) => {
   const lineNumber = totalLineCount + 1;
   const formattedLine = {};
+  const filenameBase = model.filenameBase;
+  const filenameExtension = model.filenameExtension;
   for (const columnSchema of model.schema) {
-    const lineValue = line[columnSchema.name];
-    if (columnSchema.type === "date") {
-      if (lineValue !== "" && lineValue !== void 0) {
-        const dateValue = lineValue.replace(/-/g, "");
-        if (dateValue.length !== 8) {
+    const { name, type, required, min, max } = columnSchema;
+    let value = line[name];
+    if (value === "" || value === void 0 || value === null) {
+      formattedLine[name] = null;
+      if (required) {
+        throw new Error(
+          `Missing required value in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}.`
+        );
+      }
+      continue;
+    }
+    switch (type) {
+      case "date":
+        value = value.replace(/-/g, "");
+        if (value.length !== 8) {
           throw new Error(
-            `Invalid date in ${model.filenameBase}.${model.filenameExtension} for ${columnSchema.name} on line ${lineNumber}.`
+            `Invalid date in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}.`
           );
         }
-        formattedLine[columnSchema.name] = Number.parseInt(dateValue, 10);
-      }
-    } else if (columnSchema.type === "integer") {
-      formattedLine[columnSchema.name] = Number.parseInt(lineValue, 10);
-    } else if (columnSchema.type === "real") {
-      formattedLine[columnSchema.name] = Number.parseFloat(lineValue);
-    } else {
-      formattedLine[columnSchema.name] = lineValue;
-    }
-    if (formattedLine[columnSchema.name] === "" || formattedLine[columnSchema.name] === void 0 || formattedLine[columnSchema.name] === null || Number.isNaN(formattedLine[columnSchema.name])) {
-      formattedLine[columnSchema.name] = null;
-    }
-    if (columnSchema.required === true && formattedLine[columnSchema.name] === null) {
+        value = parseInt(value, 10);
+        break;
+      case "integer":
+        value = parseInt(value, 10);
+        break;
+      case "real":
+        value = parseFloat(value);
+        break;
+    }
+    if (Number.isNaN(value)) {
+      formattedLine[name] = null;
+      continue;
+    }
+    formattedLine[name] = value;
+    if (min !== void 0 && value < min) {
       throw new Error(
-        `Missing required value in ${model.filenameBase}.${model.filenameExtension} for ${columnSchema.name} on line ${lineNumber}.`
+        `Invalid value in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}: below minimum value of ${min}.`
       );
     }
-    if (columnSchema.min !== void 0 && formattedLine[columnSchema.name] < columnSchema.min) {
+    if (max !== void 0 && value > max) {
       throw new Error(
-        `Invalid value in ${model.filenameBase}.${model.filenameExtension} for ${columnSchema.name} on line ${lineNumber}: below minimum value of ${columnSchema.min}.`
-      );
-    }
-    if (columnSchema.max !== void 0 && formattedLine[columnSchema.name] > columnSchema.max) {
-      throw new Error(
-        `Invalid value in ${model.filenameBase}.${model.filenameExtension} for ${columnSchema.name} on line ${lineNumber}: above maximum value of ${columnSchema.max}.`
+        `Invalid value in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}: above maximum value of ${max}.`
       );
     }
   }
@@ -3757,6 +3767,7 @@ var formatGtfsLine = (line, model, totalLineCount) => {
   }
   return formattedLine;
 };
+var BATCH_SIZE = 1e5;
 var importGtfsFiles = (db, task) => mapSeries2(
   Object.values(models_exports),
   (model) => new Promise((resolve, reject) => {
@@ -3830,19 +3841,37 @@ var importGtfsFiles = (db, task) => mapSeries2(
         while (record = parser.read()) {
           totalLineCount += 1;
           lines.push(formatGtfsLine(record, model, totalLineCount));
+          if (lines.length >= BATCH_SIZE) {
+            try {
+              insertLines(lines);
+              lines = [];
+            } catch (error) {
+              reject(error);
+            }
+            task.log(
+              `Importing - ${filename} - ${totalLineCount} lines imported\r`,
+              true
+            );
+          }
         }
       });
       parser.on("end", () => {
         try {
-          insertLines(lines);
+          if (lines.length > 0) {
+            try {
+              insertLines(lines);
+            } catch (error) {
+              reject(error);
+            }
+          }
           task.log(
             `Importing - ${filename} - ${totalLineCount} lines imported\r`,
             true
           );
+          resolve();
         } catch (error) {
           reject(error);
         }
-        resolve();
       });
       parser.on("error", reject);
       createReadStream(filepath).pipe(stripBomStream()).pipe(parser);
@@ -3872,6 +3901,8 @@ var importGtfsFiles = (db, task) => mapSeries2(
   })
 );
 async function importGtfs(initialConfig) {
+  const timer = new Timer();
+  timer.start();
   const config = setDefaultConfig(initialConfig);
   validateConfigForImport(config);
   const log2 = log(config);
@@ -3921,8 +3952,10 @@ async function importGtfs(initialConfig) {
     });
     log2(`Creating DB indexes`);
     createGtfsIndexes(db);
+    const seconds = Math.round(timer.time() / 1e3);
+    timer.stop();
     log2(
-      `Completed GTFS import for ${pluralize2("agency", agencyCount, true)}
+      `Completed GTFS import for ${pluralize2("agency", agencyCount, true)} in ${seconds} seconds
 `
     );
   } catch (error) {