gtfs 4.17.6 → 4.17.7

package/dist/index.js

@@ -9,11 +9,8 @@ import path2 from "path";
 import { createReadStream, existsSync as existsSync2, lstatSync } from "fs";
 import { cp, readdir, rename, readFile as readFile2, rm as rm2, writeFile } from "fs/promises";
 import { parse } from "csv-parse";
-import pluralize2 from "pluralize";
 import stripBomStream from "strip-bom-stream";
 import { temporaryDirectory } from "tempy";
-import Timer from "timer-machine";
-import untildify3 from "untildify";
 import mapSeries2 from "promise-map-series";
 
 // src/models/models.ts
@@ -3976,84 +3973,14 @@ var vehicles = {
 // src/lib/db.ts
 import fs from "fs";
 import Database from "better-sqlite3";
-import untildify from "untildify";
-var dbs = {};
-function setupDb(sqlitePath) {
-  const db = new Database(untildify(sqlitePath));
-  db.pragma("journal_mode = OFF");
-  db.pragma("synchronous = OFF");
-  db.pragma("temp_store = MEMORY");
-  dbs[sqlitePath] = db;
-  return db;
-}
-function openDb(config = null) {
-  if (config) {
-    const { sqlitePath = ":memory:", db } = config;
-    if (db) {
-      return db;
-    }
-    if (dbs[sqlitePath]) {
-      return dbs[sqlitePath];
-    }
-    return setupDb(sqlitePath);
-  }
-  if (Object.keys(dbs).length === 0) {
-    return setupDb(":memory:");
-  }
-  if (Object.keys(dbs).length === 1) {
-    const filename = Object.keys(dbs)[0];
-    return dbs[filename];
-  }
-  if (Object.keys(dbs).length > 1) {
-    throw new Error(
-      "Multiple databases open, please specify which one to use."
-    );
-  }
-  throw new Error("Unable to find database connection.");
-}
-function closeDb(db = null) {
-  if (Object.keys(dbs).length === 0) {
-    throw new Error(
-      "No database connection. Call `openDb(config)` before using any methods."
-    );
-  }
-  if (!db) {
-    if (Object.keys(dbs).length > 1) {
-      throw new Error(
-        "Multiple database connections. Pass the db you want to close as a parameter to `closeDb`."
-      );
-    }
-    db = dbs[Object.keys(dbs)[0]];
-  }
-  db.close();
-  delete dbs[db.name];
-}
-function deleteDb(db = null) {
-  if (Object.keys(dbs).length === 0) {
-    throw new Error(
-      "No database connection. Call `openDb(config)` before using any methods."
-    );
-  }
-  if (!db) {
-    if (Object.keys(dbs).length > 1) {
-      throw new Error(
-        "Multiple database connections. Pass the db you want to delete as a parameter to `deleteDb`."
-      );
-    }
-    db = dbs[Object.keys(dbs)[0]];
-  }
-  db.close();
-  fs.unlinkSync(db.name);
-  delete dbs[db.name];
-}
 
 // src/lib/file-utils.ts
 import path from "path";
 import { existsSync } from "fs";
+import { homedir } from "os";
 import { mkdir, readFile, rm } from "fs/promises";
 import { omit, snakeCase } from "lodash-es";
 import sanitize from "sanitize-filename";
-import untildify2 from "untildify";
 import StreamZip from "node-stream-zip";
 
 // src/lib/log-utils.ts
@@ -4107,6 +4034,7 @@ function formatError(error) {
 }
 
 // src/lib/file-utils.ts
+var homeDirectory = homedir();
 async function prepDirectory(exportPath) {
   await rm(exportPath, { recursive: true, force: true });
   await mkdir(exportPath, { recursive: true });
@@ -4128,6 +4056,80 @@ function generateFolderName(folderName) {
   }
   return snakeCase(sanitize(folderName));
 }
+function untildify(pathWithTilde) {
+  return homeDirectory ? pathWithTilde.replace(/^~(?=$|\/|\\)/, homeDirectory) : pathWithTilde;
+}
+
+// src/lib/db.ts
+var dbs = {};
+function setupDb(sqlitePath) {
+  const db = new Database(untildify(sqlitePath));
+  db.pragma("journal_mode = OFF");
+  db.pragma("synchronous = OFF");
+  db.pragma("temp_store = MEMORY");
+  dbs[sqlitePath] = db;
+  return db;
+}
+function openDb(config = null) {
+  if (config) {
+    const { sqlitePath = ":memory:", db } = config;
+    if (db) {
+      return db;
+    }
+    if (dbs[sqlitePath]) {
+      return dbs[sqlitePath];
+    }
+    return setupDb(sqlitePath);
+  }
+  if (Object.keys(dbs).length === 0) {
+    return setupDb(":memory:");
+  }
+  if (Object.keys(dbs).length === 1) {
+    const filename = Object.keys(dbs)[0];
+    return dbs[filename];
+  }
+  if (Object.keys(dbs).length > 1) {
+    throw new Error(
+      "Multiple databases open, please specify which one to use."
+    );
+  }
+  throw new Error("Unable to find database connection.");
+}
+function closeDb(db = null) {
+  if (Object.keys(dbs).length === 0) {
+    throw new Error(
+      "No database connection. Call `openDb(config)` before using any methods."
+    );
+  }
+  if (!db) {
+    if (Object.keys(dbs).length > 1) {
+      throw new Error(
+        "Multiple database connections. Pass the db you want to close as a parameter to `closeDb`."
+      );
+    }
+    db = dbs[Object.keys(dbs)[0]];
+  }
+  db.close();
+  delete dbs[db.name];
+}
+function deleteDb(db = null) {
+  if (Object.keys(dbs).length === 0) {
+    throw new Error(
+      "No database connection. Call `openDb(config)` before using any methods."
+    );
+  }
+  if (!db) {
+    if (Object.keys(dbs).length > 1) {
+      throw new Error(
+        "Multiple database connections. Pass the db you want to delete as a parameter to `deleteDb`."
+      );
+    }
+    db = dbs[Object.keys(dbs)[0]];
+  }
+  db.close();
+  fs.unlinkSync(db.name);
+  delete dbs[db.name];
+}
 
 // src/lib/geojson-utils.ts
 import {
@@ -4249,7 +4251,6 @@ function stopsToGeoJSONFeatureCollection(stops2) {
 }
 
 // src/lib/import-gtfs-realtime.ts
-import pluralize from "pluralize";
 import GtfsRealtimeBindings from "gtfs-realtime-bindings";
 import mapSeries from "promise-map-series";
 import { get } from "lodash-es";
@@ -4419,6 +4420,9 @@ function applyPrefixToValue(value, columnShouldBePrefixed, prefix) {
   }
   return `${prefix}${value}`;
 }
+function pluralize(singularWord, pluralWord, count) {
+  return count === 1 ? singularWord : pluralWord;
+}
 
 // src/lib/import-gtfs-realtime.ts
 async function fetchGtfsRealtimeData(urlConfig, task) {
@@ -4653,9 +4657,9 @@ async function updateGtfsRealtime(initialConfig) {
     const agencyCount = config.agencies.length;
     log(config)(
       `Starting GTFS-Realtime refresh for ${pluralize(
+        "agency",
         "agencies",
-        agencyCount,
-        true
+        agencyCount
       )} using SQLite database at ${config.sqlitePath}`
     );
     removeExpiredRealtimeData(config);
@@ -4686,9 +4690,9 @@ async function updateGtfsRealtime(initialConfig) {
     });
     log(config)(
       `Completed GTFS-Realtime refresh for ${pluralize(
+        "agency",
         "agencies",
-        agencyCount,
-        true
+        agencyCount
       )}
 `
     );
@@ -4731,7 +4735,7 @@ var extractGtfsFiles = async (task) => {
   if (!task.path) {
     throw new Error("No `path` specified in config");
   }
-  const gtfsPath = untildify3(task.path);
+  const gtfsPath = untildify(task.path);
   task.log(`Importing GTFS from ${task.path}\r`);
   if (path2.extname(gtfsPath) === ".zip") {
     try {
@@ -4983,7 +4987,12 @@ var importGtfsFiles = (db, task) => mapSeries2(
             }
           }
         } catch (error) {
-          reject(error);
+          if (task.ignoreErrors) {
+            task.logError(`Error processing ${filename}: ${error.message}`);
+            resolve();
+          } else {
+            reject(error);
+          }
         }
       });
       parser.on("end", () => {
@@ -4992,7 +5001,16 @@ var importGtfsFiles = (db, task) => mapSeries2(
             try {
               insertLines(lines);
             } catch (error) {
-              reject(error);
+              if (task.ignoreErrors) {
+                task.logError(
+                  `Error inserting data for ${filename}: ${error.message}`
+                );
+                resolve();
+                return;
+              } else {
+                reject(error);
+                return;
+              }
             }
           }
           task.log(
@@ -5001,15 +5019,34 @@ var importGtfsFiles = (db, task) => mapSeries2(
           );
           resolve();
         } catch (error) {
+          if (task.ignoreErrors) {
+            task.logError(`Error finalizing ${filename}: ${error.message}`);
+            resolve();
+          } else {
+            reject(error);
+          }
+        }
+      });
+      parser.on("error", (error) => {
+        if (task.ignoreErrors) {
+          task.logError(`Parser error for ${filename}: ${error.message}`);
+          resolve();
+        } else {
           reject(error);
         }
       });
-      parser.on("error", reject);
       createReadStream(filepath).pipe(stripBomStream()).pipe(parser);
     } else if (model.filenameExtension === "geojson") {
       readFile2(filepath, "utf8").then((data) => {
         if (isValidJSON(data) === false) {
-          reject(new Error(`Invalid JSON in ${filename}`));
+          if (task.ignoreErrors) {
+            task.logError(`Invalid JSON in ${filename}`);
+            resolve();
+            return;
+          } else {
+            reject(new Error(`Invalid JSON in ${filename}`));
+            return;
+          }
         }
         totalLineCount += 1;
         const line = formatGtfsLine(
@@ -5017,30 +5054,54 @@ var importGtfsFiles = (db, task) => mapSeries2(
           model,
           totalLineCount
         );
-        insertLines([line]);
-        task.log(
-          `Importing - ${filename} - ${totalLineCount} lines imported\r`,
-          true
+        try {
+          insertLines([line]);
+          task.log(
+            `Importing - ${filename} - ${totalLineCount} lines imported\r`,
+            true
+          );
+          resolve();
+        } catch (error) {
+          if (task.ignoreErrors) {
+            task.logError(
+              `Error inserting data for ${filename}: ${error.message}`
+            );
+            resolve();
+          } else {
+            reject(error);
+          }
+        }
+      }).catch((error) => {
+        if (task.ignoreErrors) {
+          task.logError(`Error reading ${filename}: ${error.message}`);
+          resolve();
+        } else {
+          reject(error);
+        }
+      });
+    } else {
+      if (task.ignoreErrors) {
+        task.logError(
+          `Unsupported file type: ${model.filenameExtension} for ${filename}`
         );
         resolve();
-      }).catch(reject);
-    } else {
-      reject(
-        new Error(`Unsupported file type: ${model.filenameExtension}`)
-      );
+      } else {
+        reject(
+          new Error(`Unsupported file type: ${model.filenameExtension}`)
+        );
+      }
     }
   })
 );
 async function importGtfs(initialConfig) {
-  const timer = new Timer();
-  timer.start();
+  const startTime = process.hrtime.bigint();
   const config = setDefaultConfig(initialConfig);
   validateConfigForImport(config);
   try {
     const db = openDb(config);
     const agencyCount = config.agencies.length;
     log(config)(
-      `Starting GTFS import for ${pluralize2("file", agencyCount, true)} using SQLite database at ${config.sqlitePath}`
+      `Starting GTFS import for ${pluralize("file", "files", agencyCount)} using SQLite database at ${config.sqlitePath}`
     );
     createGtfsTables(db);
     await mapSeries2(config.agencies, async (agency2) => {
@@ -5084,10 +5145,10 @@ async function importGtfs(initialConfig) {
     });
     log(config)(`Creating DB indexes`);
     createGtfsIndexes(db);
-    const seconds = Math.round(timer.time() / 1e3);
-    timer.stop();
+    const endTime = process.hrtime.bigint();
+    const elapsedSeconds = Number(endTime - startTime) / 1e9;
     log(config)(
-      `Completed GTFS import for ${pluralize2("agency", agencyCount, true)} in ${seconds} seconds
+      `Completed GTFS import in ${elapsedSeconds.toFixed(1)} seconds
 `
     );
   } catch (error) {
@@ -5104,11 +5165,9 @@ async function importGtfs(initialConfig) {
 import path3 from "path";
 import { writeFile as writeFile2 } from "fs/promises";
 import { without, compact as compact2 } from "lodash-es";
-import pluralize3 from "pluralize";
 import { stringify } from "csv-stringify";
 import sqlString2 from "sqlstring-sqlite";
 import mapSeries3 from "promise-map-series";
-import untildify4 from "untildify";
 var getAgencies = (db, config) => {
   try {
     return db.prepare("SELECT agency_name FROM agency;").all();
@@ -5138,15 +5197,15 @@ var exportGtfs = async (initialConfig) => {
     );
   }
   log(config)(
-    `Starting GTFS export for ${pluralize3(
+    `Starting GTFS export for ${pluralize(
       "agency",
-      agencyCount,
-      true
+      "agencies",
+      agencyCount
     )} using SQLite database at ${config.sqlitePath}`
   );
   const folderName = generateFolderName(agencies[0].agency_name);
   const defaultExportPath = path3.join(process.cwd(), "gtfs-export", folderName);
-  const exportPath = untildify4(config.exportPath || defaultExportPath);
+  const exportPath = untildify(config.exportPath || defaultExportPath);
   await prepDirectory(exportPath);
   const modelsToExport = Object.values(models_exports).filter(
     (model) => model.extension !== "gtfs-realtime"
@@ -5214,7 +5273,7 @@ var exportGtfs = async (initialConfig) => {
   }
   log(config)(`Completed GTFS export to ${exportPath}`);
   log(config)(
-    `Completed GTFS export for ${pluralize3("agency", agencyCount, true)}
+    `Completed GTFS export for ${pluralize("agency", "agencies", agencyCount)}
 `
   );
 };