gtfs 4.17.6 → 4.18.0

package/dist/index.js

@@ -9,11 +9,8 @@ import path2 from "path";
 import { createReadStream, existsSync as existsSync2, lstatSync } from "fs";
 import { cp, readdir, rename, readFile as readFile2, rm as rm2, writeFile } from "fs/promises";
 import { parse } from "csv-parse";
-import pluralize2 from "pluralize";
 import stripBomStream from "strip-bom-stream";
 import { temporaryDirectory } from "tempy";
-import Timer from "timer-machine";
-import untildify3 from "untildify";
 import mapSeries2 from "promise-map-series";
 
 // src/models/models.ts
@@ -3976,84 +3973,14 @@ var vehicles = {
 // src/lib/db.ts
 import fs from "fs";
 import Database from "better-sqlite3";
-import untildify from "untildify";
-var dbs = {};
-function setupDb(sqlitePath) {
-  const db = new Database(untildify(sqlitePath));
-  db.pragma("journal_mode = OFF");
-  db.pragma("synchronous = OFF");
-  db.pragma("temp_store = MEMORY");
-  dbs[sqlitePath] = db;
-  return db;
-}
-function openDb(config = null) {
-  if (config) {
-    const { sqlitePath = ":memory:", db } = config;
-    if (db) {
-      return db;
-    }
-    if (dbs[sqlitePath]) {
-      return dbs[sqlitePath];
-    }
-    return setupDb(sqlitePath);
-  }
-  if (Object.keys(dbs).length === 0) {
-    return setupDb(":memory:");
-  }
-  if (Object.keys(dbs).length === 1) {
-    const filename = Object.keys(dbs)[0];
-    return dbs[filename];
-  }
-  if (Object.keys(dbs).length > 1) {
-    throw new Error(
-      "Multiple databases open, please specify which one to use."
-    );
-  }
-  throw new Error("Unable to find database connection.");
-}
-function closeDb(db = null) {
-  if (Object.keys(dbs).length === 0) {
-    throw new Error(
-      "No database connection. Call `openDb(config)` before using any methods."
-    );
-  }
-  if (!db) {
-    if (Object.keys(dbs).length > 1) {
-      throw new Error(
-        "Multiple database connections. Pass the db you want to close as a parameter to `closeDb`."
-      );
-    }
-    db = dbs[Object.keys(dbs)[0]];
-  }
-  db.close();
-  delete dbs[db.name];
-}
-function deleteDb(db = null) {
-  if (Object.keys(dbs).length === 0) {
-    throw new Error(
-      "No database connection. Call `openDb(config)` before using any methods."
-    );
-  }
-  if (!db) {
-    if (Object.keys(dbs).length > 1) {
-      throw new Error(
-        "Multiple database connections. Pass the db you want to delete as a parameter to `deleteDb`."
-      );
-    }
-    db = dbs[Object.keys(dbs)[0]];
-  }
-  db.close();
-  fs.unlinkSync(db.name);
-  delete dbs[db.name];
-}
 
 // src/lib/file-utils.ts
 import path from "path";
 import { existsSync } from "fs";
+import { homedir } from "os";
 import { mkdir, readFile, rm } from "fs/promises";
 import { omit, snakeCase } from "lodash-es";
 import sanitize from "sanitize-filename";
-import untildify2 from "untildify";
 import StreamZip from "node-stream-zip";
 
 // src/lib/log-utils.ts
@@ -4107,6 +4034,7 @@ function formatError(error) {
 }
 
 // src/lib/file-utils.ts
+var homeDirectory = homedir();
 async function prepDirectory(exportPath) {
   await rm(exportPath, { recursive: true, force: true });
   await mkdir(exportPath, { recursive: true });
@@ -4128,6 +4056,80 @@ function generateFolderName(folderName) {
   }
   return snakeCase(sanitize(folderName));
 }
+function untildify(pathWithTilde) {
+  return homeDirectory ? pathWithTilde.replace(/^~(?=$|\/|\\)/, homeDirectory) : pathWithTilde;
+}
+
+// src/lib/db.ts
+var dbs = {};
+function setupDb(sqlitePath) {
+  const db = new Database(untildify(sqlitePath));
+  db.pragma("journal_mode = OFF");
+  db.pragma("synchronous = OFF");
+  db.pragma("temp_store = MEMORY");
+  dbs[sqlitePath] = db;
+  return db;
+}
+function openDb(config = null) {
+  if (config) {
+    const { sqlitePath = ":memory:", db } = config;
+    if (db) {
+      return db;
+    }
+    if (dbs[sqlitePath]) {
+      return dbs[sqlitePath];
+    }
+    return setupDb(sqlitePath);
+  }
+  if (Object.keys(dbs).length === 0) {
+    return setupDb(":memory:");
+  }
+  if (Object.keys(dbs).length === 1) {
+    const filename = Object.keys(dbs)[0];
+    return dbs[filename];
+  }
+  if (Object.keys(dbs).length > 1) {
+    throw new Error(
+      "Multiple databases open, please specify which one to use."
+    );
+  }
+  throw new Error("Unable to find database connection.");
+}
+function closeDb(db = null) {
+  if (Object.keys(dbs).length === 0) {
+    throw new Error(
+      "No database connection. Call `openDb(config)` before using any methods."
+    );
+  }
+  if (!db) {
+    if (Object.keys(dbs).length > 1) {
+      throw new Error(
+        "Multiple database connections. Pass the db you want to close as a parameter to `closeDb`."
+      );
+    }
+    db = dbs[Object.keys(dbs)[0]];
+  }
+  db.close();
+  delete dbs[db.name];
+}
+function deleteDb(db = null) {
+  if (Object.keys(dbs).length === 0) {
+    throw new Error(
+      "No database connection. Call `openDb(config)` before using any methods."
+    );
+  }
+  if (!db) {
+    if (Object.keys(dbs).length > 1) {
+      throw new Error(
+        "Multiple database connections. Pass the db you want to delete as a parameter to `deleteDb`."
+      );
+    }
+    db = dbs[Object.keys(dbs)[0]];
+  }
+  db.close();
+  fs.unlinkSync(db.name);
+  delete dbs[db.name];
+}
 
 // src/lib/geojson-utils.ts
 import {
@@ -4145,7 +4147,7 @@ function isValidJSON(string) {
   try {
     JSON.parse(string);
     return true;
-  } catch (error) {
+  } catch {
     return false;
   }
 }
@@ -4210,7 +4212,7 @@ function formatProperties(properties) {
   if (formattedRouteTextColor) {
     formattedProperties.route_text_color = formattedRouteTextColor;
   }
-  if (properties.routes) {
+  if (properties.routes && Array.isArray(properties.routes)) {
     formattedProperties.routes = properties.routes.map(
       (route) => formatProperties(route)
     );
@@ -4249,7 +4251,6 @@ function stopsToGeoJSONFeatureCollection(stops2) {
 }
 
 // src/lib/import-gtfs-realtime.ts
-import pluralize from "pluralize";
 import GtfsRealtimeBindings from "gtfs-realtime-bindings";
 import mapSeries from "promise-map-series";
 import { get } from "lodash-es";
@@ -4276,7 +4277,8 @@ function setDefaultConfig(initialConfig) {
     ignoreDuplicates: false,
     ignoreErrors: false,
     gtfsRealtimeExpirationSeconds: 0,
-    verbose: true
+    verbose: true,
+    downloadTimeout: 3e4
   };
   return {
     ...defaults,
@@ -4419,58 +4421,14 @@ function applyPrefixToValue(value, columnShouldBePrefixed, prefix) {
   }
   return `${prefix}${value}`;
 }
+function pluralize(singularWord, pluralWord, count) {
+  return count === 1 ? singularWord : pluralWord;
+}
 
 // src/lib/import-gtfs-realtime.ts
-async function fetchGtfsRealtimeData(urlConfig, task) {
-  task.log(`Downloading GTFS-Realtime from ${urlConfig.url}`);
-  const response = await fetch(urlConfig.url, {
-    method: "GET",
-    headers: {
-      ...urlConfig.headers ?? {},
-      "Accept-Encoding": "gzip"
-    },
-    signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
-  });
-  if (response.status !== 200) {
-    task.logWarning(
-      `Unable to download GTFS-Realtime from ${urlConfig.url}. Got status ${response.status}.`
-    );
-    return null;
-  }
-  const buffer = await response.arrayBuffer();
-  const message = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(
-    new Uint8Array(buffer)
-  );
-  return GtfsRealtimeBindings.transit_realtime.FeedMessage.toObject(message, {
-    enums: String,
-    longs: String,
-    bytes: String,
-    defaults: false,
-    arrays: true,
-    objects: true,
-    oneofs: true
-  });
-}
-function removeExpiredRealtimeData(config) {
-  const db = openDb(config);
-  log(config)(`Removing expired GTFS-Realtime data`);
-  db.prepare(
-    `DELETE FROM vehicle_positions WHERE expiration_timestamp <= strftime('%s','now')`
-  ).run();
-  db.prepare(
-    `DELETE FROM trip_updates WHERE expiration_timestamp <= strftime('%s','now')`
-  ).run();
-  db.prepare(
-    `DELETE FROM stop_time_updates WHERE expiration_timestamp <= strftime('%s','now')`
-  ).run();
-  db.prepare(
-    `DELETE FROM service_alerts WHERE expiration_timestamp <= strftime('%s','now')`
-  ).run();
-  db.prepare(
-    `DELETE FROM service_alert_informed_entities WHERE expiration_timestamp <= strftime('%s','now')`
-  ).run();
-  log(config)(`Removed expired GTFS-Realtime data\r`, true);
-}
+var BATCH_SIZE = 1e3;
+var MAX_RETRIES = 3;
+var RETRY_DELAY = 1e3;
 function prepareRealtimeFieldValue(entity, column, task) {
   if (column.name === "created_timestamp") {
     return task.currentTimestamp;
@@ -4487,163 +4445,265 @@ function prepareRealtimeFieldValue(entity, column, task) {
   );
   return column.type === "json" ? JSON.stringify(prefixedValue) : prefixedValue;
 }
-async function processRealtimeAlerts(db, gtfsRealtimeData, task) {
-  const alertStmt = db.prepare(
-    `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${serviceAlerts.schema.map(() => "?").join(", ")})`
-  );
-  const informedEntityStmt = db.prepare(
-    `REPLACE INTO ${serviceAlertInformedEntities.filenameBase} (${serviceAlertInformedEntities.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${serviceAlertInformedEntities.schema.map(() => "?").join(", ")})`
+function createPreparedStatement(db, model) {
+  const columns = model.schema.map((column) => column.name);
+  const placeholders = model.schema.map(() => "?").join(", ");
+  return db.prepare(
+    `REPLACE INTO ${model.filenameBase} (${columns.join(", ")}) VALUES (${placeholders})`
   );
-  let totalLineCount = 0;
-  db.transaction(() => {
-    for (const entity of gtfsRealtimeData.entity) {
-      const fieldValues = serviceAlerts.schema.map(
-        (column) => prepareRealtimeFieldValue(entity, column, task)
+}
+async function processBatch(items, batchSize, processor) {
+  let totalRecordCount = 0;
+  let totalErrorCount = 0;
+  for (let i = 0; i < items.length; i += batchSize) {
+    const batch = items.slice(i, i + batchSize);
+    try {
+      const result = await processor(batch);
+      totalRecordCount += result.recordCount;
+      totalErrorCount += result.errorCount;
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      totalErrorCount += batch.length;
+      console.error(`Batch processing error: ${errorMessage}`);
+    }
+  }
+  return { recordCount: totalRecordCount, errorCount: totalErrorCount };
+}
+async function fetchGtfsRealtimeData(type, task) {
+  const urlConfig = getUrlConfig(type, task);
+  if (!urlConfig) {
+    return null;
+  }
+  task.log(`Importing - GTFS-Realtime from ${urlConfig.url}`);
+  for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      const response = await fetch(urlConfig.url, {
+        method: "GET",
+        headers: {
+          ...urlConfig.headers ?? {},
+          "Accept-Encoding": "gzip"
+        },
+        signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
+      });
+      if (response.status !== 200) {
+        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+      }
+      const buffer = await response.arrayBuffer();
+      const message = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(
+        new Uint8Array(buffer)
       );
-      try {
-        alertStmt.run(fieldValues);
-        if (entity.alert.informedEntity?.length) {
-          const informedEntities = entity.alert.informedEntity.map(
-            (informedEntity) => {
+      const feedMessage = GtfsRealtimeBindings.transit_realtime.FeedMessage.toObject(message, {
+        enums: String,
+        longs: String,
+        bytes: String,
+        defaults: false,
+        arrays: true,
+        objects: true,
+        oneofs: true
+      });
+      return feedMessage;
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      if (attempt === MAX_RETRIES) {
+        if (task.ignoreErrors) {
+          task.logError(
+            `Failed to fetch ${type} after ${MAX_RETRIES} attempts: ${errorMessage}`
+          );
+          return null;
+        }
+        throw error;
+      }
+      task.logWarning(`Attempt ${attempt} failed for ${type}: ${errorMessage}`);
+      await new Promise(
+        (resolve) => setTimeout(resolve, RETRY_DELAY * attempt)
+      );
+    }
+  }
+  return null;
+}
+function getUrlConfig(type, task) {
+  switch (type) {
+    case "alerts":
+      return task.realtimeAlerts;
+    case "tripupdates":
+      return task.realtimeTripUpdates;
+    case "vehiclepositions":
+      return task.realtimeVehiclePositions;
+    default:
+      return void 0;
+  }
+}
+function createServiceAlertsProcessor(db, task) {
+  const alertStmt = createPreparedStatement(db, serviceAlerts);
+  const informedEntityStmt = createPreparedStatement(
+    db,
+    serviceAlertInformedEntities
+  );
+  return async (batch) => {
+    let recordCount = 0;
+    let errorCount = 0;
+    db.transaction(() => {
+      for (const entity of batch) {
+        try {
+          const alertValues = serviceAlerts.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
+          alertStmt.run(alertValues);
+          recordCount++;
+          if (entity.alert?.informedEntity?.length) {
+            for (const informedEntity of entity.alert.informedEntity) {
               informedEntity.parent = entity;
-              return serviceAlertInformedEntities.schema.map(
+              const entityValues = serviceAlertInformedEntities.schema.map(
                 (column) => prepareRealtimeFieldValue(informedEntity, column, task)
               );
+              informedEntityStmt.run(entityValues);
+              recordCount++;
             }
-          );
-          for (const values of informedEntities) {
-            informedEntityStmt.run(values);
           }
+        } catch (error) {
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          errorCount++;
+          task.logWarning(`Alert processing error: ${errorMessage}`);
         }
-        totalLineCount++;
-      } catch (error) {
-        task.logWarning(`Import error: ${error.message}`);
       }
-    }
-    task.log(
-      `Importing - GTFS-Realtime service alerts - ${totalLineCount} entries imported\r`,
-      true
-    );
-  })();
+    })();
+    return { recordCount, errorCount };
+  };
 }
-async function processRealtimeTripUpdates(db, gtfsRealtimeData, task) {
-  let totalLineCount = 0;
-  const tripUpdateStmt = db.prepare(
-    `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${tripUpdates.schema.map(() => "?").join(", ")})`
+function createTripUpdatesProcessor(db, task) {
+  const tripUpdateStmt = createPreparedStatement(
+    db,
+    tripUpdates
   );
-  const stopTimeStmt = db.prepare(
-    `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${stopTimeUpdates.schema.map(() => "?").join(", ")})`
+  const stopTimeStmt = createPreparedStatement(
+    db,
+    stopTimeUpdates
   );
-  db.transaction(() => {
-    for (const entity of gtfsRealtimeData.entity) {
-      try {
-        const fieldValues = tripUpdates.schema.map(
-          (column) => prepareRealtimeFieldValue(entity, column, task)
-        );
-        tripUpdateStmt.run(fieldValues);
-        for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
-          stopTimeUpdate.parent = entity;
-          const values = stopTimeUpdates.schema.map(
-            (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
-          );
-          stopTimeStmt.run(values);
+  return async (batch) => {
+    let recordCount = 0;
+    let errorCount = 0;
+    db.transaction(() => {
+      for (const entity of batch) {
+        try {
+          const tripUpdateValues = tripUpdates.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
+          tripUpdateStmt.run(tripUpdateValues);
+          recordCount++;
+          if (entity.tripUpdate?.stopTimeUpdate?.length) {
+            for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
+              stopTimeUpdate.parent = entity;
+              const stopTimeValues = stopTimeUpdates.schema.map(
+                (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
+              );
+              stopTimeStmt.run(stopTimeValues);
+              recordCount++;
+            }
+          }
+        } catch (error) {
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          errorCount++;
+          task.logWarning(`Trip update processing error: ${errorMessage}`);
         }
-        totalLineCount++;
-      } catch (error) {
-        task.logWarning(`Import error: ${error.message}`);
       }
-    }
-    task.log(
-      `Importing - GTFS-Realtime trip updates - ${totalLineCount} entries imported\r`,
-      true
-    );
-  })();
+    })();
+    return { recordCount, errorCount };
+  };
 }
-async function processRealtimeVehiclePositions(db, gtfsRealtimeData, task) {
-  let totalLineCount = 0;
-  const vehiclePositionStmt = db.prepare(
-    `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${vehiclePositions.schema.map(() => "?").join(", ")})`
+function createVehiclePositionsProcessor(db, task) {
+  const vehiclePositionStmt = createPreparedStatement(
+    db,
+    vehiclePositions
   );
-  db.transaction(() => {
-    for (const entity of gtfsRealtimeData.entity) {
-      try {
-        const fieldValues = vehiclePositions.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
-        vehiclePositionStmt.run(fieldValues);
-        totalLineCount++;
-      } catch (error) {
-        task.logWarning(`Import error: ${error.message}`);
+  return async (batch) => {
+    let recordCount = 0;
+    let errorCount = 0;
+    db.transaction(() => {
+      for (const entity of batch) {
+        try {
+          const fieldValues = vehiclePositions.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
+          vehiclePositionStmt.run(fieldValues);
+          recordCount++;
+        } catch (error) {
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          errorCount++;
+          task.logWarning(`Vehicle position processing error: ${errorMessage}`);
+        }
       }
+    })();
+    return { recordCount, errorCount };
+  };
+}
+function removeExpiredRealtimeData(config) {
+  const db = openDb(config);
+  log(config)(`Removing expired GTFS-Realtime data`);
+  db.transaction(() => {
+    const tables = [
+      "vehicle_positions",
+      "trip_updates",
+      "stop_time_updates",
+      "service_alerts",
+      "service_alert_informed_entities"
+    ];
+    for (const table of tables) {
+      db.prepare(
+        `DELETE FROM ${table} WHERE expiration_timestamp <= strftime('%s','now')`
+      ).run();
     }
-    task.log(
-      `Importing - GTFS-Realtime vehicle positions - ${totalLineCount} entries imported\r`,
-      true
-    );
   })();
+  log(config)(`Removed expired GTFS-Realtime data\r`, true);
 }
 async function updateGtfsRealtimeData(task) {
-  if (task.realtimeAlerts === void 0 && task.realtimeTripUpdates === void 0 && task.realtimeVehiclePositions === void 0) {
+  if (!task.realtimeAlerts && !task.realtimeTripUpdates && !task.realtimeVehiclePositions) {
     return;
   }
+  const [alertsData, tripUpdatesData, vehiclePositionsData] = await Promise.all(
+    [
+      task.realtimeAlerts?.url ? fetchGtfsRealtimeData("alerts", task) : null,
+      task.realtimeTripUpdates?.url ? fetchGtfsRealtimeData("tripupdates", task) : null,
+      task.realtimeVehiclePositions?.url ? fetchGtfsRealtimeData("vehiclepositions", task) : null
+    ]
+  );
   const db = openDb({ sqlitePath: task.sqlitePath });
-  if (task.realtimeAlerts?.url) {
-    try {
-      const alertsData = await fetchGtfsRealtimeData(task.realtimeAlerts, task);
-      if (alertsData?.entity) {
-        await processRealtimeAlerts(db, alertsData, task);
-      }
-    } catch (error) {
-      if (task.ignoreErrors) {
-        task.logError(error.message);
-      } else {
-        throw error;
-      }
-    }
+  const recordCounts = {
+    alerts: 0,
+    tripupdates: 0,
+    vehiclepositions: 0
+  };
+  const processingPromises = [];
+  if (alertsData?.entity?.length) {
+    processingPromises.push(
+      processBatch(
+        alertsData.entity,
+        BATCH_SIZE,
+        createServiceAlertsProcessor(db, task)
+      ).then((result) => {
+        recordCounts.alerts = result.recordCount;
+      })
+    );
   }
-  if (task.realtimeTripUpdates?.url) {
-    try {
-      const tripUpdatesData = await fetchGtfsRealtimeData(
-        task.realtimeTripUpdates,
-        task
-      );
-      if (tripUpdatesData?.entity) {
-        await processRealtimeTripUpdates(db, tripUpdatesData, task);
-      }
-    } catch (error) {
-      if (task.ignoreErrors) {
-        task.logError(error.message);
-      } else {
-        throw error;
-      }
-    }
+  if (tripUpdatesData?.entity?.length) {
+    processingPromises.push(
+      processBatch(
+        tripUpdatesData.entity,
+        BATCH_SIZE,
+        createTripUpdatesProcessor(db, task)
+      ).then((result) => {
+        recordCounts.tripupdates = result.recordCount;
+      })
+    );
   }
-  if (task.realtimeVehiclePositions?.url) {
-    try {
-      const vehiclePositionsData = await fetchGtfsRealtimeData(
-        task.realtimeVehiclePositions,
-        task
-      );
-      if (vehiclePositionsData?.entity) {
-        await processRealtimeVehiclePositions(db, vehiclePositionsData, task);
-      }
-    } catch (error) {
-      if (task.ignoreErrors) {
-        task.logError(error.message);
-      } else {
-        throw error;
-      }
-    }
+  if (vehiclePositionsData?.entity?.length) {
+    processingPromises.push(
+      processBatch(
+        vehiclePositionsData.entity,
+        BATCH_SIZE,
+        createVehiclePositionsProcessor(db, task)
+      ).then((result) => {
+        recordCounts.vehiclepositions = result.recordCount;
+      })
+    );
   }
-  task.log(`GTFS-Realtime data import complete`);
+  await Promise.all(processingPromises);
+  task.log(
+    `GTFS-Realtime import complete: ${recordCounts.alerts} alerts, ${recordCounts.tripupdates} trip updates, ${recordCounts.vehiclepositions} vehicle positions`
+  );
 }
 async function updateGtfsRealtime(initialConfig) {
   const config = setDefaultConfig(initialConfig);
@@ -4653,9 +4713,9 @@ async function updateGtfsRealtime(initialConfig) {
     const agencyCount = config.agencies.length;
     log(config)(
       `Starting GTFS-Realtime refresh for ${pluralize(
+        "agency",
         "agencies",
-        agencyCount,
-        true
+        agencyCount
       )} using SQLite database at ${config.sqlitePath}`
     );
     removeExpiredRealtimeData(config);
@@ -4677,8 +4737,9 @@ async function updateGtfsRealtime(initialConfig) {
         };
         await updateGtfsRealtimeData(task);
       } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
         if (config.ignoreErrors) {
-          logError(config)(error.message);
+          logError(config)(errorMessage);
         } else {
           throw error;
         }
@@ -4686,14 +4747,14 @@ async function updateGtfsRealtime(initialConfig) {
     });
     log(config)(
       `Completed GTFS-Realtime refresh for ${pluralize(
+        "agency",
         "agencies",
-        agencyCount,
-        true
+        agencyCount
       )}
 `
     );
   } catch (error) {
-    if (error?.code === "SQLITE_CANTOPEN") {
+    if (error.code === "SQLITE_CANTOPEN") {
       logError(config)(
         `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
       );
@@ -4731,8 +4792,8 @@ var extractGtfsFiles = async (task) => {
   if (!task.path) {
     throw new Error("No `path` specified in config");
   }
-  const gtfsPath = untildify3(task.path);
-  task.log(`Importing GTFS from ${task.path}\r`);
+  const gtfsPath = untildify(task.path);
+  task.log(`Importing static GTFS from ${task.path}\r`);
   if (path2.extname(gtfsPath) === ".zip") {
     try {
       await unzip(gtfsPath, task.downloadDir);
@@ -4815,11 +4876,11 @@ var createGtfsTables = (db) => {
       if (column.type === "time") {
         sqlColumnCreateStatements.push(
           `${getTimestampColumnName(column.name)} INTEGER GENERATED ALWAYS AS (
-            CASE 
-              WHEN ${column.name} IS NULL OR ${column.name} = '' THEN NULL 
+            CASE
+              WHEN ${column.name} IS NULL OR ${column.name} = '' THEN NULL
               ELSE CAST(
-                substr(${column.name}, 1, instr(${column.name}, ':') - 1) * 3600 + 
-                substr(${column.name}, instr(${column.name}, ':') + 1, 2) * 60 + 
+                substr(${column.name}, 1, instr(${column.name}, ':') - 1) * 3600 +
+                substr(${column.name}, instr(${column.name}, ':') + 1, 2) * 60 +
                 substr(${column.name}, -2) AS INTEGER
               )
             END
@@ -4876,7 +4937,7 @@ var formatGtfsLine = (line, model, totalLineCount) => {
       continue;
     }
     if (type === "date") {
-      value = value.replace(/-/g, "");
+      value = value?.toString().replace(/-/g, "");
       if (value.length !== 8) {
         throw new Error(
           `Invalid date in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}.`
@@ -4892,155 +4953,221 @@ var formatGtfsLine = (line, model, totalLineCount) => {
   }
   return formattedLine;
 };
-var BATCH_SIZE = 1e5;
-var importGtfsFiles = (db, task) => mapSeries2(
-  Object.values(models_exports),
-  (model) => new Promise((resolve, reject) => {
-    let totalLineCount = 0;
-    const filename = `${model.filenameBase}.${model.filenameExtension}`;
-    if (task.exclude && task.exclude.includes(model.filenameBase)) {
-      task.log(`Skipping - ${filename}\r`);
-      resolve();
-      return;
-    }
-    if (model.extension === "gtfs-realtime") {
-      resolve();
-      return;
-    }
-    const filepath = path2.join(task.downloadDir, `${filename}`);
-    if (!existsSync2(filepath)) {
-      if (!model.nonstandard) {
-        task.log(`Importing - ${filename} - No file found\r`);
+var BATCH_SIZE2 = 1e5;
+var importGtfsFiles = async (db, task) => {
+  await mapSeries2(
+    Object.values(models_exports),
+    (model) => new Promise((resolve, reject) => {
+      let totalLineCount = 0;
+      const filename = `${model.filenameBase}.${model.filenameExtension}`;
+      if (task.exclude && task.exclude.includes(model.filenameBase)) {
+        task.log(`Skipping - ${filename}\r`);
+        resolve();
+        return;
       }
-      resolve();
-      return;
-    }
-    task.log(`Importing - ${filename}\r`);
-    const columns = model.schema;
-    const prefixedColumns = new Set(
-      columns.filter((column) => column.prefix).map((column) => column.name)
-    );
-    const prepareStatement = `INSERT ${task.ignoreDuplicates ? "OR IGNORE" : ""} INTO ${model.filenameBase} (${columns.map(({ name }) => name).join(", ")}) VALUES (${columns.map(({ name }) => `@${name}`).join(", ")})`;
-    const insert = db.prepare(prepareStatement);
-    const insertLines = db.transaction((lines) => {
-      for (const [rowNumber, line] of Object.entries(lines)) {
-        try {
-          if (task.prefix === void 0) {
-            insert.run(line);
-          } else {
-            const prefixedLine = Object.fromEntries(
-              Object.entries(
-                line
-              ).map(([columnName, value]) => [
-                columnName,
-                applyPrefixToValue(
-                  value,
-                  prefixedColumns.has(columnName),
-                  task.prefix
-                )
-              ])
-            );
-            insert.run(prefixedLine);
-          }
-        } catch (error) {
-          if (error.code === "SQLITE_CONSTRAINT_PRIMARYKEY") {
-            const primaryColumns = columns.filter(
-              (column) => column.primary
-            );
-            task.logWarning(
-              `Duplicate values for primary key (${primaryColumns.map((column) => column.name).join(", ")}) found in ${filename}. Set the \`ignoreDuplicates\` option to true in config.json to ignore this error`
-            );
-          }
-          task.logWarning(
-            `Check ${filename} for invalid data on line ${rowNumber + 1}.`
-          );
-          throw error;
+      if (model.extension === "gtfs-realtime") {
+        resolve();
+        return;
+      }
+      const filepath = path2.join(task.downloadDir, `${filename}`);
+      if (!existsSync2(filepath)) {
+        if (!model.nonstandard) {
+          task.log(`Importing - ${filename} - No file found\r`);
         }
+        resolve();
+        return;
       }
-    });
-    if (model.filenameExtension === "txt") {
-      const parser = parse({
-        columns: true,
-        relax_quotes: true,
-        trim: true,
-        skip_empty_lines: true,
-        ...task.csvOptions
-      });
-      let lines = [];
-      parser.on("readable", () => {
-        try {
-          let record;
-          while (record = parser.read()) {
-            totalLineCount += 1;
-            lines.push(formatGtfsLine(record, model, totalLineCount));
-            if (lines.length >= BATCH_SIZE) {
-              insertLines(lines);
-              lines = [];
-              task.log(
-                `Importing - ${filename} - ${totalLineCount} lines imported\r`,
-                true
+      task.log(`Importing - ${filename}\r`);
+      const columns = model.schema;
+      const prefixedColumns = new Set(
+        columns.filter((column) => column.prefix).map((column) => column.name)
+      );
+      const prepareStatement = `INSERT ${task.ignoreDuplicates ? "OR IGNORE" : ""} INTO ${model.filenameBase} (${columns.map(({ name }) => name).join(", ")}) VALUES (${columns.map(({ name }) => `@${name}`).join(", ")})`;
+      const insert = db.prepare(prepareStatement);
+      const insertLines = db.transaction((lines) => {
+        for (const [rowNumber, line] of Object.entries(lines)) {
+          try {
+            if (task.prefix === void 0) {
+              insert.run(line);
+            } else {
+              const prefixedLine = Object.fromEntries(
+                Object.entries(
+                  line
+                ).map(([columnName, value]) => [
+                  columnName,
+                  applyPrefixToValue(
+                    value,
+                    prefixedColumns.has(columnName),
+                    task.prefix
+                  )
+                ])
               );
+              insert.run(prefixedLine);
             }
+          } catch (error) {
+            if (error.code === "SQLITE_CONSTRAINT_PRIMARYKEY") {
+              const primaryColumns = columns.filter(
+                (column) => column.primary
+              );
+              task.logWarning(
+                `Duplicate values for primary key (${primaryColumns.map((column) => column.name).join(", ")}) found in ${filename}. Set the \`ignoreDuplicates\` option to true in config.json to ignore this error`
+              );
+            }
+            task.logWarning(
+              `Check ${filename} for invalid data on line ${rowNumber + 1}.`
+            );
+            throw error;
           }
-        } catch (error) {
-          reject(error);
         }
       });
-      parser.on("end", () => {
-        try {
-          if (lines.length > 0) {
-            try {
-              insertLines(lines);
-            } catch (error) {
+      if (model.filenameExtension === "txt") {
+        const parser = parse({
+          columns: true,
+          relax_quotes: true,
+          trim: true,
+          skip_empty_lines: true,
+          ...task.csvOptions
+        });
+        let lines = [];
+        parser.on("readable", () => {
+          try {
+            let record;
+            while (record = parser.read()) {
+              totalLineCount += 1;
+              lines.push(formatGtfsLine(record, model, totalLineCount));
+              if (lines.length >= BATCH_SIZE2) {
+                insertLines(lines);
+                lines = [];
+                task.log(
+                  `Importing - ${filename} - ${totalLineCount} lines imported\r`,
+                  true
+                );
+              }
+            }
+          } catch (error) {
+            if (task.ignoreErrors) {
+              const errorMessage = error instanceof Error ? error.message : String(error);
+              task.logError(`Error processing ${filename}: ${errorMessage}`);
+              resolve();
+            } else {
+              reject(error);
+            }
+          }
+        });
+        parser.on("end", () => {
+          try {
+            if (lines.length > 0) {
+              try {
+                insertLines(lines);
+              } catch (error) {
+                if (task.ignoreErrors) {
+                  const errorMessage = error instanceof Error ? error.message : String(error);
+                  task.logError(
+                    `Error inserting data for ${filename}: ${errorMessage}`
+                  );
+                  resolve();
+                  return;
+                } else {
+                  reject(error);
+                  return;
+                }
+              }
+            }
+            task.log(
+              `Importing - ${filename} - ${totalLineCount} lines imported\r`,
+              true
+            );
+            resolve();
+          } catch (error) {
+            if (task.ignoreErrors) {
+              const errorMessage = error instanceof Error ? error.message : String(error);
+              task.logError(`Error finalizing ${filename}: ${errorMessage}`);
+              resolve();
+            } else {
               reject(error);
             }
           }
-          task.log(
-            `Importing - ${filename} - ${totalLineCount} lines imported\r`,
-            true
+        });
+        parser.on("error", (error) => {
+          if (task.ignoreErrors) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            task.logError(`Parser error for ${filename}: ${errorMessage}`);
+            resolve();
+          } else {
+            reject(error);
+          }
+        });
+        createReadStream(filepath).pipe(stripBomStream()).pipe(parser);
+      } else if (model.filenameExtension === "geojson") {
+        readFile2(filepath, "utf8").then((data) => {
+          if (isValidJSON(data) === false) {
+            if (task.ignoreErrors) {
+              task.logError(`Invalid JSON in ${filename}`);
+              resolve();
+              return;
+            } else {
+              reject(new Error(`Invalid JSON in ${filename}`));
+              return;
+            }
+          }
+          totalLineCount += 1;
+          const line = formatGtfsLine(
+            { geojson: data },
+            model,
+            totalLineCount
+          );
+          try {
+            insertLines([line]);
+            task.log(
+              `Importing - ${filename} - ${totalLineCount} lines imported\r`,
+              true
+            );
+            resolve();
+          } catch (error) {
+            if (task.ignoreErrors) {
+              const errorMessage = error instanceof Error ? error.message : String(error);
+              task.logError(
+                `Error inserting data for ${filename}: ${errorMessage}`
+              );
+              resolve();
+            } else {
+              reject(error);
+            }
+          }
+        }).catch((error) => {
+          if (task.ignoreErrors) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            task.logError(`Error reading ${filename}: ${errorMessage}`);
+            resolve();
+          } else {
+            reject(error);
+          }
+        });
+      } else {
+        if (task.ignoreErrors) {
+          task.logError(
+            `Unsupported file type: ${model.filenameExtension} for ${filename}`
           );
           resolve();
-        } catch (error) {
-          reject(error);
-        }
-      });
-      parser.on("error", reject);
-      createReadStream(filepath).pipe(stripBomStream()).pipe(parser);
-    } else if (model.filenameExtension === "geojson") {
-      readFile2(filepath, "utf8").then((data) => {
-        if (isValidJSON(data) === false) {
-          reject(new Error(`Invalid JSON in ${filename}`));
+        } else {
+          reject(
+            new Error(`Unsupported file type: ${model.filenameExtension}`)
+          );
         }
-        totalLineCount += 1;
-        const line = formatGtfsLine(
-          { geojson: data },
-          model,
-          totalLineCount
-        );
-        insertLines([line]);
-        task.log(
-          `Importing - ${filename} - ${totalLineCount} lines imported\r`,
-          true
-        );
-        resolve();
-      }).catch(reject);
-    } else {
-      reject(
-        new Error(`Unsupported file type: ${model.filenameExtension}`)
-      );
-    }
-  })
-);
+      }
+    })
+  );
+  task.log(`Static GTFS import complete`);
+};
 async function importGtfs(initialConfig) {
-  const timer = new Timer();
-  timer.start();
+  const startTime = process.hrtime.bigint();
   const config = setDefaultConfig(initialConfig);
   validateConfigForImport(config);
   try {
     const db = openDb(config);
     const agencyCount = config.agencies.length;
     log(config)(
-      `Starting GTFS import for ${pluralize2("file", agencyCount, true)} using SQLite database at ${config.sqlitePath}`
+      `Starting GTFS import for ${pluralize("file", "files", agencyCount)} using SQLite database at ${config.sqlitePath}`
     );
     createGtfsTables(db);
     await mapSeries2(config.agencies, async (agency2) => {
@@ -5048,7 +5175,6 @@ async function importGtfs(initialConfig) {
         const tempPath = temporaryDirectory();
         const task = {
           exclude: agency2.exclude,
-          url: agency2.url,
           headers: agency2.headers,
           realtimeAlerts: agency2.realtimeAlerts,
           realtimeTripUpdates: agency2.realtimeTripUpdates,
@@ -5056,7 +5182,6 @@ async function importGtfs(initialConfig) {
           downloadDir: tempPath,
           downloadTimeout: config.downloadTimeout,
           gtfsRealtimeExpirationSeconds: config.gtfsRealtimeExpirationSeconds,
-          path: agency2.path,
           csvOptions: config.csvOptions || {},
           ignoreDuplicates: config.ignoreDuplicates,
           ignoreErrors: config.ignoreErrors,
@@ -5067,8 +5192,13 @@ async function importGtfs(initialConfig) {
           logWarning: logWarning(config),
           logError: logError(config)
         };
-        if (task.url) {
+        if ("url" in agency2) {
+          Object.assign(task, { url: agency2.url });
           await downloadGtfsFiles(task);
+        } else {
+          Object.assign(task, {
+            path: agency2.path
+          });
         }
         await extractGtfsFiles(task);
         await importGtfsFiles(db, task);
@@ -5076,7 +5206,8 @@ async function importGtfs(initialConfig) {
         await rm2(tempPath, { recursive: true });
       } catch (error) {
         if (config.ignoreErrors) {
-          logError(config)(error.message);
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          logError(config)(errorMessage);
         } else {
           throw error;
         }
@@ -5084,14 +5215,14 @@ async function importGtfs(initialConfig) {
     });
     log(config)(`Creating DB indexes`);
     createGtfsIndexes(db);
-    const seconds = Math.round(timer.time() / 1e3);
-    timer.stop();
+    const endTime = process.hrtime.bigint();
+    const elapsedSeconds = Number(endTime - startTime) / 1e9;
     log(config)(
-      `Completed GTFS import for ${pluralize2("agency", agencyCount, true)} in ${seconds} seconds
+      `Completed GTFS import in ${elapsedSeconds.toFixed(1)} seconds
 `
     );
   } catch (error) {
-    if (error?.code === "SQLITE_CANTOPEN") {
+    if (error.code === "SQLITE_CANTOPEN") {
       logError(config)(
         `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
       );
@@ -5104,15 +5235,13 @@ async function importGtfs(initialConfig) {
 import path3 from "path";
 import { writeFile as writeFile2 } from "fs/promises";
 import { without, compact as compact2 } from "lodash-es";
-import pluralize3 from "pluralize";
 import { stringify } from "csv-stringify";
 import sqlString2 from "sqlstring-sqlite";
 import mapSeries3 from "promise-map-series";
-import untildify4 from "untildify";
 var getAgencies = (db, config) => {
   try {
     return db.prepare("SELECT agency_name FROM agency;").all();
-  } catch (error) {
+  } catch {
     if (config.sqlitePath === ":memory:") {
       throw new Error(
         'No agencies found in SQLite. You are using an in-memory database - if running this from command line be sure to specify a value for `sqlitePath` in config.json other than ":memory:".'
@@ -5138,15 +5267,15 @@ var exportGtfs = async (initialConfig) => {
     );
   }
   log(config)(
-    `Starting GTFS export for ${pluralize3(
+    `Starting GTFS export for ${pluralize(
       "agency",
-      agencyCount,
-      true
+      "agencies",
+      agencyCount
     )} using SQLite database at ${config.sqlitePath}`
   );
   const folderName = generateFolderName(agencies[0].agency_name);
   const defaultExportPath = path3.join(process.cwd(), "gtfs-export", folderName);
-  const exportPath = untildify4(config.exportPath || defaultExportPath);
+  const exportPath = untildify(config.exportPath || defaultExportPath);
   await prepDirectory(exportPath);
   const modelsToExport = Object.values(models_exports).filter(
     (model) => model.extension !== "gtfs-realtime"
@@ -5179,11 +5308,17 @@ var exportGtfs = async (initialConfig) => {
           }
         } else if (model.filenameBase === "fare_attributes") {
           for (const line of lines) {
-            line.price = formatCurrency(line.price, line.currency_type);
+            line.price = formatCurrency(
+              line.price,
+              line.currency_type
+            );
           }
         } else if (model.filenameBase === "fare_products") {
           for (const line of lines) {
-            line.amount = formatCurrency(line.amount, line.currency);
+            line.amount = formatCurrency(
+              line.amount,
+              line.currency
+            );
           }
         }
         const columns = without(
@@ -5214,7 +5349,7 @@ var exportGtfs = async (initialConfig) => {
   }
   log(config)(`Completed GTFS export to ${exportPath}`);
   log(config)(
-    `Completed GTFS export for ${pluralize3("agency", agencyCount, true)}
+    `Completed GTFS export for ${pluralize("agency", "agencies", agencyCount)}
 `
   );
 };
@@ -5558,10 +5693,7 @@ function getRoutes(query = {}, fields = [], orderBy2 = [], options = {}) {
   let whereClause = "";
   const orderByClause = formatOrderByClause(orderBy2);
   const routeQuery = omit3(query, ["stop_id", "service_id"]);
-  const tripQuery = pick(query, [
-    "stop_id",
-    "service_id"
-  ]);
+  const tripQuery = pick(query, ["stop_id", "service_id"]);
   const whereClauses = Object.entries(routeQuery).map(
     ([key, value]) => formatWhereClause(key, value)
   );
@@ -5609,7 +5741,12 @@ function getShapes(query = {}, fields = [], orderBy2 = [], options = {}) {
     "service_id",
     "direction_id"
   ]);
-  const tripQuery = pick2(query, ["route_id", "trip_id", "service_id", "direction_id"]);
+  const tripQuery = pick2(query, [
+    "route_id",
+    "trip_id",
+    "service_id",
+    "direction_id"
+  ]);
   const whereClauses = Object.entries(shapeQuery).map(
     ([key, value]) => formatWhereClause(key, value)
   );
@@ -5716,7 +5853,7 @@ function getStops(query = {}, fields = [], orderBy2 = [], options = {}) {
   if (options.bounding_box_side_m !== void 0) {
     stopQueryOmitKeys.push("stop_lat", "stop_lon");
   }
-  let stopQuery = omit5(query, stopQueryOmitKeys);
+  const stopQuery = omit5(query, stopQueryOmitKeys);
   const tripQuery = pick3(query, [
     "route_id",
     "trip_id",
@@ -5781,7 +5918,7 @@ function getStoptimes(query = {}, fields = [], orderBy2 = [], options = {}) {
   let whereClause = "";
   const orderByClause = formatOrderByClause(orderBy2);
   const stoptimeQueryOmitKeys = ["date", "start_time", "end_time"];
-  let stoptimeQuery = omit6(query, stoptimeQueryOmitKeys);
+  const stoptimeQuery = omit6(query, stoptimeQueryOmitKeys);
   const whereClauses = Object.entries(stoptimeQuery).map(
     ([key, value]) => formatWhereClause(key, value)
   );
@@ -5789,7 +5926,7 @@ function getStoptimes(query = {}, fields = [], orderBy2 = [], options = {}) {
     if (typeof query.date !== "number") {
       throw new Error("`date` must be a number in yyyymmdd format");
     }
-    const serviceIds = getServiceIdsByDate(query.date);
+    const serviceIds = getServiceIdsByDate(query.date, options);
     const tripSubquery = `SELECT DISTINCT trip_id FROM trips WHERE service_id IN (${serviceIds.map((id) => sqlString4.escape(id)).join(",")})`;
     whereClauses.push(`trip_id IN (${tripSubquery})`);
   }
@@ -5863,7 +6000,7 @@ function getTrips(query = {}, fields = [], orderBy2 = [], options = {}) {
   let whereClause = "";
   const orderByClause = formatOrderByClause(orderBy2);
   const tripQueryOmitKeys = ["date"];
-  let tripQuery = omit7(query, tripQueryOmitKeys);
+  const tripQuery = omit7(query, tripQueryOmitKeys);
   const whereClauses = Object.entries(tripQuery).map(
     ([key, value]) => formatWhereClause(key, value)
   );
@@ -5871,7 +6008,7 @@ function getTrips(query = {}, fields = [], orderBy2 = [], options = {}) {
     if (typeof query.date !== "number") {
       throw new Error("`date` must be a number in yyyymmdd format");
     }
-    const serviceIds = getServiceIdsByDate(query.date);
+    const serviceIds = getServiceIdsByDate(query.date, options);
     whereClauses.push(
       `service_id IN (${serviceIds.map((id) => sqlString5.escape(id)).join(",")})`
     );
@@ -6153,6 +6290,7 @@ export {
   closeDb,
   deleteDb,
   exportGtfs,
+  generateFolderName,
   getAgencies2 as getAgencies,
   getAreas,
   getAttributions,
@@ -6213,6 +6351,9 @@ export {
   getVehiclePositions,
   importGtfs,
   openDb,
+  prepDirectory,
+  untildify,
+  unzip,
   updateGtfsRealtime
 };
 //# sourceMappingURL=index.js.map