gtfs 4.17.6 → 4.18.0

package/README.md

@@ -69,7 +69,7 @@ import { readFile } from 'fs/promises';
 import path from 'node:path';
 
 const config = JSON.parse(
-  await readFile(path.join(import.meta.dirname, 'config.json'))
+  await readFile(path.join(import.meta.dirname, 'config.json'), 'utf8')
 );
 
 try {
@@ -157,7 +157,7 @@ Copy `config-sample.json` to `config.json` and then add your projects configurat
 | [`exportPath`](#exportpath)                                       | string            | A path to a directory to put exported GTFS files. Optional, defaults to `gtfs-export/<agency_name>`.                                                                |
 | [`gtfsRealtimeExpirationSeconds`](#gtfsrealtimeexpirationseconds) | integer           | Amount of time in seconds to allow GTFS-Realtime data to be stored in database before allowing to be deleted. Optional, defaults to 0.                              |
 | [`ignoreDuplicates`](#ignoreduplicates)                           | boolean           | Whether or not to ignore unique constraints on ids when importing GTFS, such as `trip_id`, `calendar_id`. Optional, defaults to false.                              |
-| [`ignoreErrors`](#ignoreerrors)                                   | boolean           | Whether or not to ignore errors during the import process. If true, when importing multiple agencies, failed agencies will be skipped. Optional, defaults to false. |
+| [`ignoreErrors`](#ignoreerrors)                                   | boolean           | Whether or not to ignore errors during the import process. If true, failed files will be skipped while the rest are processed. Optional, defaults to false. |
 | [`sqlitePath`](#sqlitepath)                                       | string            | A path to an SQLite database. Optional, defaults to using an in-memory database.                                                                                    |
 | [`verbose`](#verbose)                                             | boolean           | Whether or not to print output to the console. Optional, defaults to true.                                                                                          |
 
@@ -362,7 +362,7 @@ importGtfs({
 
 ### downloadTimeout
 
-{Integer} A number of milliseconds to wait when downloading GTFS before throwing an error. Optional.
+{Integer} A number of milliseconds to wait when downloading GTFS before throwing an error. Optional, defaults to `30000` (30 seconds).
 
 ```json
 {
@@ -371,7 +371,7 @@ importGtfs({
       "path": "/path/to/the/unzipped/gtfs/"
     }
   ],
-  "downloadTimeout": 5000
+  "downloadTimeout": 30000
 }
 ```
 
@@ -431,7 +431,28 @@ importGtfs({
 
 ### ignoreErrors
 
-{Boolean} When importing GTFS from multiple agencies, if you don't want node-GTFS to throw an error and instead skip failed GTFS importants proceed to the next agency. Defaults to `false`.
+{Boolean} Controls error handling behavior during GTFS import. When `true`, the import process will continue even when encountering errors, logging them instead of stopping execution. Defaults to `false`.
+
+**When enabled, `ignoreErrors` will:**
+- Continue processing other GTFS files when one file fails
+- Log error messages instead of throwing exceptions
+- Skip problematic records within files while importing valid ones
+- Handle various error types including:
+  - Invalid CSV data or malformed records
+  - JSON parsing errors (for GeoJSON files)
+  - Database constraint violations
+  - File read/write errors
+  - GTFS-Realtime API failures
+
+**Use cases:**
+- Importing from multiple GTFS sources where some may have data quality issues
+- Processing large datasets where minor errors shouldn't halt the entire import
+- Development/testing scenarios where you want to see all errors at once
+
+**⚠️ Important considerations:**
+- Errors are logged but not thrown, so you may miss critical data issues
+- Partial imports may result in incomplete or inconsistent data
+- Consider using the `exclude` config option to skip problematic files entirely instead of ignoring errors
 
 ```json
 {
@@ -510,7 +531,7 @@ import { readFile } from 'fs/promises';
 import path from 'node:path';
 
 const config = JSON.parse(
-  await readFile(path.join(import.meta.dirname, 'config.json'))
+  await readFile(path.join(import.meta.dirname, 'config.json'), 'utf8')
 );
 
 await importGtfs(config);
@@ -556,7 +577,7 @@ import { readFile } from 'fs/promises';
 import path from 'node:path';
 
 const config = JSON.parse(
-  await readFile(path.join(import.meta.dirname, 'config.json'))
+  await readFile(path.join(import.meta.dirname, 'config.json'), 'utf8')
 );
 
 await updateGtfsRealtime(config);
@@ -639,7 +660,7 @@ import { readFile } from 'fs/promises';
 import path from 'node:path';
 
 const config = JSON.parse(
-  await readFile(path.join(import.meta.dirname, 'config.json'))
+  await readFile(path.join(import.meta.dirname, 'config.json'), 'utf8')
 );
 
 const db = openDb(config);
@@ -681,7 +702,7 @@ import { readFile } from 'fs/promises';
 import path from 'node:path';
 
 const config = JSON.parse(
-  await readFile(path.join(import.meta.dirname, 'config.json'))
+  await readFile(path.join(import.meta.dirname, 'config.json'), 'utf8')
 );
 
 const db = openDb(config);
@@ -703,7 +724,7 @@ import { readFile } from 'fs/promises';
 import path from 'node:path';
 
 const config = JSON.parse(
-  await readFile(path.join(import.meta.dirname, 'config.json'))
+  await readFile(path.join(import.meta.dirname, 'config.json'), 'utf8')
 );
 
 const db = openDb(config);
@@ -725,7 +746,7 @@ import { readFile } from 'fs/promises';
 import path from 'node:path';
 
 const config = JSON.parse(
-  await readFile(path.join(import.meta.dirname, 'config.json'))
+  await readFile(path.join(import.meta.dirname, 'config.json'), 'utf8')
 );
 const db = openDb(config);
 const stops = getStops(

package/dist/bin/gtfs-export.js

@@ -13,10 +13,10 @@ import PrettyError from "pretty-error";
 // src/lib/file-utils.ts
 import path from "path";
 import { existsSync } from "fs";
+import { homedir } from "os";
 import { mkdir, readFile, rm } from "fs/promises";
 import { omit, snakeCase } from "lodash-es";
 import sanitize from "sanitize-filename";
-import untildify from "untildify";
 import StreamZip from "node-stream-zip";
 
 // src/lib/log-utils.ts
@@ -60,6 +60,7 @@ function formatError(error) {
 }
 
 // src/lib/file-utils.ts
+var homeDirectory = homedir();
 async function getConfig(argv2) {
   let config;
   let data;
@@ -106,17 +107,17 @@ function generateFolderName(folderName) {
   }
   return snakeCase(sanitize(folderName));
 }
+function untildify(pathWithTilde) {
+  return homeDirectory ? pathWithTilde.replace(/^~(?=$|\/|\\)/, homeDirectory) : pathWithTilde;
+}
 
 // src/lib/import-gtfs.ts
 import path2 from "path";
 import { createReadStream, existsSync as existsSync2, lstatSync } from "fs";
 import { cp, readdir, rename, readFile as readFile2, rm as rm2, writeFile } from "fs/promises";
 import { parse } from "csv-parse";
-import pluralize2 from "pluralize";
 import stripBomStream from "strip-bom-stream";
 import { temporaryDirectory } from "tempy";
-import Timer from "timer-machine";
-import untildify3 from "untildify";
 import mapSeries2 from "promise-map-series";
 
 // src/models/models.ts
@@ -4078,10 +4079,9 @@ var vehicles = {
 
 // src/lib/db.ts
 import Database from "better-sqlite3";
-import untildify2 from "untildify";
 var dbs = {};
 function setupDb(sqlitePath) {
-  const db = new Database(untildify2(sqlitePath));
+  const db = new Database(untildify(sqlitePath));
   db.pragma("journal_mode = OFF");
   db.pragma("synchronous = OFF");
   db.pragma("temp_store = MEMORY");
@@ -4128,7 +4128,6 @@ import {
 import { feature, featureCollection } from "@turf/helpers";
 
 // src/lib/import-gtfs-realtime.ts
-import pluralize from "pluralize";
 import GtfsRealtimeBindings from "gtfs-realtime-bindings";
 import mapSeries from "promise-map-series";
 import { get } from "lodash-es";
@@ -4142,7 +4141,8 @@ function setDefaultConfig(initialConfig) {
     ignoreDuplicates: false,
     ignoreErrors: false,
     gtfsRealtimeExpirationSeconds: 0,
-    verbose: true
+    verbose: true,
+    downloadTimeout: 3e4
   };
   return {
     ...defaults,
@@ -4158,20 +4158,21 @@ function formatCurrency(value, currency) {
   const fractionPart = parts.find((part) => part.type === "fraction")?.value ?? "";
   return `${integerPart}${fractionPart !== "" ? `.${fractionPart}` : ""}`;
 }
+function pluralize(singularWord, pluralWord, count) {
+  return count === 1 ? singularWord : pluralWord;
+}
 
 // src/lib/export.ts
 import path3 from "path";
 import { writeFile as writeFile2 } from "fs/promises";
 import { without, compact as compact2 } from "lodash-es";
-import pluralize3 from "pluralize";
 import { stringify } from "csv-stringify";
 import sqlString2 from "sqlstring-sqlite";
 import mapSeries3 from "promise-map-series";
-import untildify4 from "untildify";
 var getAgencies = (db, config) => {
   try {
     return db.prepare("SELECT agency_name FROM agency;").all();
-  } catch (error) {
+  } catch {
     if (config.sqlitePath === ":memory:") {
       throw new Error(
         'No agencies found in SQLite. You are using an in-memory database - if running this from command line be sure to specify a value for `sqlitePath` in config.json other than ":memory:".'
@@ -4197,15 +4198,15 @@ var exportGtfs = async (initialConfig) => {
     );
   }
   log(config)(
-    `Starting GTFS export for ${pluralize3(
+    `Starting GTFS export for ${pluralize(
       "agency",
-      agencyCount,
-      true
+      "agencies",
+      agencyCount
     )} using SQLite database at ${config.sqlitePath}`
   );
   const folderName = generateFolderName(agencies[0].agency_name);
   const defaultExportPath = path3.join(process.cwd(), "gtfs-export", folderName);
-  const exportPath = untildify4(config.exportPath || defaultExportPath);
+  const exportPath = untildify(config.exportPath || defaultExportPath);
   await prepDirectory(exportPath);
   const modelsToExport = Object.values(models_exports).filter(
     (model) => model.extension !== "gtfs-realtime"
@@ -4238,11 +4239,17 @@ var exportGtfs = async (initialConfig) => {
           }
         } else if (model.filenameBase === "fare_attributes") {
           for (const line of lines) {
-            line.price = formatCurrency(line.price, line.currency_type);
+            line.price = formatCurrency(
+              line.price,
+              line.currency_type
+            );
           }
         } else if (model.filenameBase === "fare_products") {
           for (const line of lines) {
-            line.amount = formatCurrency(line.amount, line.currency);
+            line.amount = formatCurrency(
+              line.amount,
+              line.currency
+            );
           }
         }
         const columns = without(
@@ -4273,7 +4280,7 @@ var exportGtfs = async (initialConfig) => {
   }
   log(config)(`Completed GTFS export to ${exportPath}`);
   log(config)(
-    `Completed GTFS export for ${pluralize3("agency", agencyCount, true)}
+    `Completed GTFS export for ${pluralize("agency", "agencies", agencyCount)}
 `
   );
 };