gtfs 4.17.7 → 4.18.0

package/dist/index.js

@@ -4147,7 +4147,7 @@ function isValidJSON(string) {
   try {
     JSON.parse(string);
     return true;
-  } catch (error) {
+  } catch {
     return false;
   }
 }
@@ -4212,7 +4212,7 @@ function formatProperties(properties) {
   if (formattedRouteTextColor) {
     formattedProperties.route_text_color = formattedRouteTextColor;
   }
-  if (properties.routes) {
+  if (properties.routes && Array.isArray(properties.routes)) {
     formattedProperties.routes = properties.routes.map(
       (route) => formatProperties(route)
     );
@@ -4277,7 +4277,8 @@ function setDefaultConfig(initialConfig) {
     ignoreDuplicates: false,
     ignoreErrors: false,
     gtfsRealtimeExpirationSeconds: 0,
-    verbose: true
+    verbose: true,
+    downloadTimeout: 3e4
   };
   return {
     ...defaults,
@@ -4425,56 +4426,9 @@ function pluralize(singularWord, pluralWord, count) {
 }
 
 // src/lib/import-gtfs-realtime.ts
-async function fetchGtfsRealtimeData(urlConfig, task) {
-  task.log(`Downloading GTFS-Realtime from ${urlConfig.url}`);
-  const response = await fetch(urlConfig.url, {
-    method: "GET",
-    headers: {
-      ...urlConfig.headers ?? {},
-      "Accept-Encoding": "gzip"
-    },
-    signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
-  });
-  if (response.status !== 200) {
-    task.logWarning(
-      `Unable to download GTFS-Realtime from ${urlConfig.url}. Got status ${response.status}.`
-    );
-    return null;
-  }
-  const buffer = await response.arrayBuffer();
-  const message = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(
-    new Uint8Array(buffer)
-  );
-  return GtfsRealtimeBindings.transit_realtime.FeedMessage.toObject(message, {
-    enums: String,
-    longs: String,
-    bytes: String,
-    defaults: false,
-    arrays: true,
-    objects: true,
-    oneofs: true
-  });
-}
-function removeExpiredRealtimeData(config) {
-  const db = openDb(config);
-  log(config)(`Removing expired GTFS-Realtime data`);
-  db.prepare(
-    `DELETE FROM vehicle_positions WHERE expiration_timestamp <= strftime('%s','now')`
-  ).run();
-  db.prepare(
-    `DELETE FROM trip_updates WHERE expiration_timestamp <= strftime('%s','now')`
-  ).run();
-  db.prepare(
-    `DELETE FROM stop_time_updates WHERE expiration_timestamp <= strftime('%s','now')`
-  ).run();
-  db.prepare(
-    `DELETE FROM service_alerts WHERE expiration_timestamp <= strftime('%s','now')`
-  ).run();
-  db.prepare(
-    `DELETE FROM service_alert_informed_entities WHERE expiration_timestamp <= strftime('%s','now')`
-  ).run();
-  log(config)(`Removed expired GTFS-Realtime data\r`, true);
-}
+var BATCH_SIZE = 1e3;
+var MAX_RETRIES = 3;
+var RETRY_DELAY = 1e3;
 function prepareRealtimeFieldValue(entity, column, task) {
   if (column.name === "created_timestamp") {
     return task.currentTimestamp;
@@ -4491,163 +4445,265 @@ function prepareRealtimeFieldValue(entity, column, task) {
   );
   return column.type === "json" ? JSON.stringify(prefixedValue) : prefixedValue;
 }
-async function processRealtimeAlerts(db, gtfsRealtimeData, task) {
-  const alertStmt = db.prepare(
-    `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${serviceAlerts.schema.map(() => "?").join(", ")})`
-  );
-  const informedEntityStmt = db.prepare(
-    `REPLACE INTO ${serviceAlertInformedEntities.filenameBase} (${serviceAlertInformedEntities.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${serviceAlertInformedEntities.schema.map(() => "?").join(", ")})`
+function createPreparedStatement(db, model) {
+  const columns = model.schema.map((column) => column.name);
+  const placeholders = model.schema.map(() => "?").join(", ");
+  return db.prepare(
+    `REPLACE INTO ${model.filenameBase} (${columns.join(", ")}) VALUES (${placeholders})`
   );
-  let totalLineCount = 0;
-  db.transaction(() => {
-    for (const entity of gtfsRealtimeData.entity) {
-      const fieldValues = serviceAlerts.schema.map(
-        (column) => prepareRealtimeFieldValue(entity, column, task)
+}
+async function processBatch(items, batchSize, processor) {
+  let totalRecordCount = 0;
+  let totalErrorCount = 0;
+  for (let i = 0; i < items.length; i += batchSize) {
+    const batch = items.slice(i, i + batchSize);
+    try {
+      const result = await processor(batch);
+      totalRecordCount += result.recordCount;
+      totalErrorCount += result.errorCount;
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      totalErrorCount += batch.length;
+      console.error(`Batch processing error: ${errorMessage}`);
+    }
+  }
+  return { recordCount: totalRecordCount, errorCount: totalErrorCount };
+}
+async function fetchGtfsRealtimeData(type, task) {
+  const urlConfig = getUrlConfig(type, task);
+  if (!urlConfig) {
+    return null;
+  }
+  task.log(`Importing - GTFS-Realtime from ${urlConfig.url}`);
+  for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      const response = await fetch(urlConfig.url, {
+        method: "GET",
+        headers: {
+          ...urlConfig.headers ?? {},
+          "Accept-Encoding": "gzip"
+        },
+        signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
+      });
+      if (response.status !== 200) {
+        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+      }
+      const buffer = await response.arrayBuffer();
+      const message = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(
+        new Uint8Array(buffer)
       );
-      try {
-        alertStmt.run(fieldValues);
-        if (entity.alert.informedEntity?.length) {
-          const informedEntities = entity.alert.informedEntity.map(
-            (informedEntity) => {
+      const feedMessage = GtfsRealtimeBindings.transit_realtime.FeedMessage.toObject(message, {
+        enums: String,
+        longs: String,
+        bytes: String,
+        defaults: false,
+        arrays: true,
+        objects: true,
+        oneofs: true
+      });
+      return feedMessage;
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      if (attempt === MAX_RETRIES) {
+        if (task.ignoreErrors) {
+          task.logError(
+            `Failed to fetch ${type} after ${MAX_RETRIES} attempts: ${errorMessage}`
+          );
+          return null;
+        }
+        throw error;
+      }
+      task.logWarning(`Attempt ${attempt} failed for ${type}: ${errorMessage}`);
+      await new Promise(
+        (resolve) => setTimeout(resolve, RETRY_DELAY * attempt)
+      );
+    }
+  }
+  return null;
+}
+function getUrlConfig(type, task) {
+  switch (type) {
+    case "alerts":
+      return task.realtimeAlerts;
+    case "tripupdates":
+      return task.realtimeTripUpdates;
+    case "vehiclepositions":
+      return task.realtimeVehiclePositions;
+    default:
+      return void 0;
+  }
+}
+function createServiceAlertsProcessor(db, task) {
+  const alertStmt = createPreparedStatement(db, serviceAlerts);
+  const informedEntityStmt = createPreparedStatement(
+    db,
+    serviceAlertInformedEntities
+  );
+  return async (batch) => {
+    let recordCount = 0;
+    let errorCount = 0;
+    db.transaction(() => {
+      for (const entity of batch) {
+        try {
+          const alertValues = serviceAlerts.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
+          alertStmt.run(alertValues);
+          recordCount++;
+          if (entity.alert?.informedEntity?.length) {
+            for (const informedEntity of entity.alert.informedEntity) {
               informedEntity.parent = entity;
-              return serviceAlertInformedEntities.schema.map(
+              const entityValues = serviceAlertInformedEntities.schema.map(
                 (column) => prepareRealtimeFieldValue(informedEntity, column, task)
               );
+              informedEntityStmt.run(entityValues);
+              recordCount++;
             }
-          );
-          for (const values of informedEntities) {
-            informedEntityStmt.run(values);
           }
+        } catch (error) {
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          errorCount++;
+          task.logWarning(`Alert processing error: ${errorMessage}`);
         }
-        totalLineCount++;
-      } catch (error) {
-        task.logWarning(`Import error: ${error.message}`);
       }
-    }
-    task.log(
-      `Importing - GTFS-Realtime service alerts - ${totalLineCount} entries imported\r`,
-      true
-    );
-  })();
+    })();
+    return { recordCount, errorCount };
+  };
 }
-async function processRealtimeTripUpdates(db, gtfsRealtimeData, task) {
-  let totalLineCount = 0;
-  const tripUpdateStmt = db.prepare(
-    `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${tripUpdates.schema.map(() => "?").join(", ")})`
+function createTripUpdatesProcessor(db, task) {
+  const tripUpdateStmt = createPreparedStatement(
+    db,
+    tripUpdates
   );
-  const stopTimeStmt = db.prepare(
-    `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${stopTimeUpdates.schema.map(() => "?").join(", ")})`
+  const stopTimeStmt = createPreparedStatement(
+    db,
+    stopTimeUpdates
   );
-  db.transaction(() => {
-    for (const entity of gtfsRealtimeData.entity) {
-      try {
-        const fieldValues = tripUpdates.schema.map(
-          (column) => prepareRealtimeFieldValue(entity, column, task)
-        );
-        tripUpdateStmt.run(fieldValues);
-        for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
-          stopTimeUpdate.parent = entity;
-          const values = stopTimeUpdates.schema.map(
-            (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
-          );
-          stopTimeStmt.run(values);
+  return async (batch) => {
+    let recordCount = 0;
+    let errorCount = 0;
+    db.transaction(() => {
+      for (const entity of batch) {
+        try {
+          const tripUpdateValues = tripUpdates.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
+          tripUpdateStmt.run(tripUpdateValues);
+          recordCount++;
+          if (entity.tripUpdate?.stopTimeUpdate?.length) {
+            for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
+              stopTimeUpdate.parent = entity;
+              const stopTimeValues = stopTimeUpdates.schema.map(
+                (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
+              );
+              stopTimeStmt.run(stopTimeValues);
+              recordCount++;
+            }
+          }
+        } catch (error) {
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          errorCount++;
+          task.logWarning(`Trip update processing error: ${errorMessage}`);
         }
-        totalLineCount++;
-      } catch (error) {
-        task.logWarning(`Import error: ${error.message}`);
       }
-    }
-    task.log(
-      `Importing - GTFS-Realtime trip updates - ${totalLineCount} entries imported\r`,
-      true
-    );
-  })();
+    })();
+    return { recordCount, errorCount };
+  };
 }
-async function processRealtimeVehiclePositions(db, gtfsRealtimeData, task) {
-  let totalLineCount = 0;
-  const vehiclePositionStmt = db.prepare(
-    `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${vehiclePositions.schema.map(() => "?").join(", ")})`
+function createVehiclePositionsProcessor(db, task) {
+  const vehiclePositionStmt = createPreparedStatement(
+    db,
+    vehiclePositions
   );
-  db.transaction(() => {
-    for (const entity of gtfsRealtimeData.entity) {
-      try {
-        const fieldValues = vehiclePositions.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
-        vehiclePositionStmt.run(fieldValues);
-        totalLineCount++;
-      } catch (error) {
-        task.logWarning(`Import error: ${error.message}`);
+  return async (batch) => {
+    let recordCount = 0;
+    let errorCount = 0;
+    db.transaction(() => {
+      for (const entity of batch) {
+        try {
+          const fieldValues = vehiclePositions.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
+          vehiclePositionStmt.run(fieldValues);
+          recordCount++;
+        } catch (error) {
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          errorCount++;
+          task.logWarning(`Vehicle position processing error: ${errorMessage}`);
+        }
       }
+    })();
+    return { recordCount, errorCount };
+  };
+}
+function removeExpiredRealtimeData(config) {
+  const db = openDb(config);
+  log(config)(`Removing expired GTFS-Realtime data`);
+  db.transaction(() => {
+    const tables = [
+      "vehicle_positions",
+      "trip_updates",
+      "stop_time_updates",
+      "service_alerts",
+      "service_alert_informed_entities"
+    ];
+    for (const table of tables) {
+      db.prepare(
+        `DELETE FROM ${table} WHERE expiration_timestamp <= strftime('%s','now')`
+      ).run();
     }
-    task.log(
-      `Importing - GTFS-Realtime vehicle positions - ${totalLineCount} entries imported\r`,
-      true
-    );
   })();
+  log(config)(`Removed expired GTFS-Realtime data\r`, true);
 }
 async function updateGtfsRealtimeData(task) {
-  if (task.realtimeAlerts === void 0 && task.realtimeTripUpdates === void 0 && task.realtimeVehiclePositions === void 0) {
+  if (!task.realtimeAlerts && !task.realtimeTripUpdates && !task.realtimeVehiclePositions) {
     return;
   }
+  const [alertsData, tripUpdatesData, vehiclePositionsData] = await Promise.all(
+    [
+      task.realtimeAlerts?.url ? fetchGtfsRealtimeData("alerts", task) : null,
+      task.realtimeTripUpdates?.url ? fetchGtfsRealtimeData("tripupdates", task) : null,
+      task.realtimeVehiclePositions?.url ? fetchGtfsRealtimeData("vehiclepositions", task) : null
+    ]
+  );
   const db = openDb({ sqlitePath: task.sqlitePath });
-  if (task.realtimeAlerts?.url) {
-    try {
-      const alertsData = await fetchGtfsRealtimeData(task.realtimeAlerts, task);
-      if (alertsData?.entity) {
-        await processRealtimeAlerts(db, alertsData, task);
-      }
-    } catch (error) {
-      if (task.ignoreErrors) {
-        task.logError(error.message);
-      } else {
-        throw error;
-      }
-    }
+  const recordCounts = {
+    alerts: 0,
+    tripupdates: 0,
+    vehiclepositions: 0
+  };
+  const processingPromises = [];
+  if (alertsData?.entity?.length) {
+    processingPromises.push(
+      processBatch(
+        alertsData.entity,
+        BATCH_SIZE,
+        createServiceAlertsProcessor(db, task)
+      ).then((result) => {
+        recordCounts.alerts = result.recordCount;
+      })
+    );
   }
-  if (task.realtimeTripUpdates?.url) {
-    try {
-      const tripUpdatesData = await fetchGtfsRealtimeData(
-        task.realtimeTripUpdates,
-        task
-      );
-      if (tripUpdatesData?.entity) {
-        await processRealtimeTripUpdates(db, tripUpdatesData, task);
-      }
-    } catch (error) {
-      if (task.ignoreErrors) {
-        task.logError(error.message);
-      } else {
-        throw error;
-      }
-    }
+  if (tripUpdatesData?.entity?.length) {
+    processingPromises.push(
+      processBatch(
+        tripUpdatesData.entity,
+        BATCH_SIZE,
+        createTripUpdatesProcessor(db, task)
+      ).then((result) => {
+        recordCounts.tripupdates = result.recordCount;
+      })
+    );
   }
-  if (task.realtimeVehiclePositions?.url) {
-    try {
-      const vehiclePositionsData = await fetchGtfsRealtimeData(
-        task.realtimeVehiclePositions,
-        task
-      );
-      if (vehiclePositionsData?.entity) {
-        await processRealtimeVehiclePositions(db, vehiclePositionsData, task);
-      }
-    } catch (error) {
-      if (task.ignoreErrors) {
-        task.logError(error.message);
-      } else {
-        throw error;
-      }
-    }
+  if (vehiclePositionsData?.entity?.length) {
+    processingPromises.push(
+      processBatch(
+        vehiclePositionsData.entity,
+        BATCH_SIZE,
+        createVehiclePositionsProcessor(db, task)
+      ).then((result) => {
+        recordCounts.vehiclepositions = result.recordCount;
+      })
+    );
   }
-  task.log(`GTFS-Realtime data import complete`);
+  await Promise.all(processingPromises);
+  task.log(
+    `GTFS-Realtime import complete: ${recordCounts.alerts} alerts, ${recordCounts.tripupdates} trip updates, ${recordCounts.vehiclepositions} vehicle positions`
+  );
 }
 async function updateGtfsRealtime(initialConfig) {
   const config = setDefaultConfig(initialConfig);
@@ -4681,8 +4737,9 @@ async function updateGtfsRealtime(initialConfig) {
         };
         await updateGtfsRealtimeData(task);
       } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
         if (config.ignoreErrors) {
-          logError(config)(error.message);
+          logError(config)(errorMessage);
         } else {
           throw error;
         }
@@ -4697,7 +4754,7 @@ async function updateGtfsRealtime(initialConfig) {
 `
     );
   } catch (error) {
-    if (error?.code === "SQLITE_CANTOPEN") {
+    if (error.code === "SQLITE_CANTOPEN") {
       logError(config)(
         `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
       );
@@ -4736,7 +4793,7 @@ var extractGtfsFiles = async (task) => {
     throw new Error("No `path` specified in config");
   }
   const gtfsPath = untildify(task.path);
-  task.log(`Importing GTFS from ${task.path}\r`);
+  task.log(`Importing static GTFS from ${task.path}\r`);
   if (path2.extname(gtfsPath) === ".zip") {
     try {
       await unzip(gtfsPath, task.downloadDir);
@@ -4819,11 +4876,11 @@ var createGtfsTables = (db) => {
       if (column.type === "time") {
         sqlColumnCreateStatements.push(
           `${getTimestampColumnName(column.name)} INTEGER GENERATED ALWAYS AS (
-            CASE 
-              WHEN ${column.name} IS NULL OR ${column.name} = '' THEN NULL 
+            CASE
+              WHEN ${column.name} IS NULL OR ${column.name} = '' THEN NULL
               ELSE CAST(
-                substr(${column.name}, 1, instr(${column.name}, ':') - 1) * 3600 + 
-                substr(${column.name}, instr(${column.name}, ':') + 1, 2) * 60 + 
+                substr(${column.name}, 1, instr(${column.name}, ':') - 1) * 3600 +
+                substr(${column.name}, instr(${column.name}, ':') + 1, 2) * 60 +
                 substr(${column.name}, -2) AS INTEGER
               )
             END
@@ -4880,7 +4937,7 @@ var formatGtfsLine = (line, model, totalLineCount) => {
       continue;
     }
     if (type === "date") {
-      value = value.replace(/-/g, "");
+      value = value?.toString().replace(/-/g, "");
       if (value.length !== 8) {
         throw new Error(
           `Invalid date in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}.`
@@ -4896,203 +4953,212 @@ var formatGtfsLine = (line, model, totalLineCount) => {
   }
   return formattedLine;
 };
-var BATCH_SIZE = 1e5;
-var importGtfsFiles = (db, task) => mapSeries2(
-  Object.values(models_exports),
-  (model) => new Promise((resolve, reject) => {
-    let totalLineCount = 0;
-    const filename = `${model.filenameBase}.${model.filenameExtension}`;
-    if (task.exclude && task.exclude.includes(model.filenameBase)) {
-      task.log(`Skipping - ${filename}\r`);
-      resolve();
-      return;
-    }
-    if (model.extension === "gtfs-realtime") {
-      resolve();
-      return;
-    }
-    const filepath = path2.join(task.downloadDir, `${filename}`);
-    if (!existsSync2(filepath)) {
-      if (!model.nonstandard) {
-        task.log(`Importing - ${filename} - No file found\r`);
+var BATCH_SIZE2 = 1e5;
+var importGtfsFiles = async (db, task) => {
+  await mapSeries2(
+    Object.values(models_exports),
+    (model) => new Promise((resolve, reject) => {
+      let totalLineCount = 0;
+      const filename = `${model.filenameBase}.${model.filenameExtension}`;
+      if (task.exclude && task.exclude.includes(model.filenameBase)) {
+        task.log(`Skipping - ${filename}\r`);
+        resolve();
+        return;
       }
-      resolve();
-      return;
-    }
-    task.log(`Importing - ${filename}\r`);
-    const columns = model.schema;
-    const prefixedColumns = new Set(
-      columns.filter((column) => column.prefix).map((column) => column.name)
-    );
-    const prepareStatement = `INSERT ${task.ignoreDuplicates ? "OR IGNORE" : ""} INTO ${model.filenameBase} (${columns.map(({ name }) => name).join(", ")}) VALUES (${columns.map(({ name }) => `@${name}`).join(", ")})`;
-    const insert = db.prepare(prepareStatement);
-    const insertLines = db.transaction((lines) => {
-      for (const [rowNumber, line] of Object.entries(lines)) {
-        try {
-          if (task.prefix === void 0) {
-            insert.run(line);
-          } else {
-            const prefixedLine = Object.fromEntries(
-              Object.entries(
-                line
-              ).map(([columnName, value]) => [
-                columnName,
-                applyPrefixToValue(
-                  value,
-                  prefixedColumns.has(columnName),
-                  task.prefix
-                )
-              ])
-            );
-            insert.run(prefixedLine);
-          }
-        } catch (error) {
-          if (error.code === "SQLITE_CONSTRAINT_PRIMARYKEY") {
-            const primaryColumns = columns.filter(
-              (column) => column.primary
-            );
-            task.logWarning(
-              `Duplicate values for primary key (${primaryColumns.map((column) => column.name).join(", ")}) found in ${filename}. Set the \`ignoreDuplicates\` option to true in config.json to ignore this error`
-            );
-          }
-          task.logWarning(
-            `Check ${filename} for invalid data on line ${rowNumber + 1}.`
-          );
-          throw error;
+      if (model.extension === "gtfs-realtime") {
+        resolve();
+        return;
+      }
+      const filepath = path2.join(task.downloadDir, `${filename}`);
+      if (!existsSync2(filepath)) {
+        if (!model.nonstandard) {
+          task.log(`Importing - ${filename} - No file found\r`);
         }
+        resolve();
+        return;
       }
-    });
-    if (model.filenameExtension === "txt") {
-      const parser = parse({
-        columns: true,
-        relax_quotes: true,
-        trim: true,
-        skip_empty_lines: true,
-        ...task.csvOptions
-      });
-      let lines = [];
-      parser.on("readable", () => {
-        try {
-          let record;
-          while (record = parser.read()) {
-            totalLineCount += 1;
-            lines.push(formatGtfsLine(record, model, totalLineCount));
-            if (lines.length >= BATCH_SIZE) {
-              insertLines(lines);
-              lines = [];
-              task.log(
-                `Importing - ${filename} - ${totalLineCount} lines imported\r`,
-                true
+      task.log(`Importing - ${filename}\r`);
+      const columns = model.schema;
+      const prefixedColumns = new Set(
+        columns.filter((column) => column.prefix).map((column) => column.name)
+      );
+      const prepareStatement = `INSERT ${task.ignoreDuplicates ? "OR IGNORE" : ""} INTO ${model.filenameBase} (${columns.map(({ name }) => name).join(", ")}) VALUES (${columns.map(({ name }) => `@${name}`).join(", ")})`;
+      const insert = db.prepare(prepareStatement);
+      const insertLines = db.transaction((lines) => {
+        for (const [rowNumber, line] of Object.entries(lines)) {
+          try {
+            if (task.prefix === void 0) {
+              insert.run(line);
+            } else {
+              const prefixedLine = Object.fromEntries(
+                Object.entries(
+                  line
+                ).map(([columnName, value]) => [
+                  columnName,
+                  applyPrefixToValue(
+                    value,
+                    prefixedColumns.has(columnName),
+                    task.prefix
+                  )
+                ])
               );
+              insert.run(prefixedLine);
             }
-          }
-        } catch (error) {
-          if (task.ignoreErrors) {
-            task.logError(`Error processing ${filename}: ${error.message}`);
-            resolve();
-          } else {
-            reject(error);
+          } catch (error) {
+            if (error.code === "SQLITE_CONSTRAINT_PRIMARYKEY") {
+              const primaryColumns = columns.filter(
+                (column) => column.primary
+              );
+              task.logWarning(
+                `Duplicate values for primary key (${primaryColumns.map((column) => column.name).join(", ")}) found in ${filename}. Set the \`ignoreDuplicates\` option to true in config.json to ignore this error`
+              );
+            }
+            task.logWarning(
+              `Check ${filename} for invalid data on line ${rowNumber + 1}.`
+            );
+            throw error;
           }
         }
       });
-      parser.on("end", () => {
-        try {
-          if (lines.length > 0) {
-            try {
-              insertLines(lines);
-            } catch (error) {
-              if (task.ignoreErrors) {
-                task.logError(
-                  `Error inserting data for ${filename}: ${error.message}`
+      if (model.filenameExtension === "txt") {
+        const parser = parse({
+          columns: true,
+          relax_quotes: true,
+          trim: true,
+          skip_empty_lines: true,
+          ...task.csvOptions
+        });
+        let lines = [];
+        parser.on("readable", () => {
+          try {
+            let record;
+            while (record = parser.read()) {
+              totalLineCount += 1;
+              lines.push(formatGtfsLine(record, model, totalLineCount));
+              if (lines.length >= BATCH_SIZE2) {
+                insertLines(lines);
+                lines = [];
+                task.log(
+                  `Importing - ${filename} - ${totalLineCount} lines imported\r`,
+                  true
                 );
-                resolve();
-                return;
-              } else {
-                reject(error);
-                return;
               }
             }
+          } catch (error) {
+            if (task.ignoreErrors) {
+              const errorMessage = error instanceof Error ? error.message : String(error);
+              task.logError(`Error processing ${filename}: ${errorMessage}`);
+              resolve();
+            } else {
+              reject(error);
+            }
           }
-          task.log(
-            `Importing - ${filename} - ${totalLineCount} lines imported\r`,
-            true
-          );
-          resolve();
-        } catch (error) {
-          if (task.ignoreErrors) {
-            task.logError(`Error finalizing ${filename}: ${error.message}`);
+        });
+        parser.on("end", () => {
+          try {
+            if (lines.length > 0) {
+              try {
+                insertLines(lines);
+              } catch (error) {
+                if (task.ignoreErrors) {
+                  const errorMessage = error instanceof Error ? error.message : String(error);
+                  task.logError(
+                    `Error inserting data for ${filename}: ${errorMessage}`
+                  );
+                  resolve();
+                  return;
+                } else {
+                  reject(error);
+                  return;
+                }
+              }
+            }
+            task.log(
+              `Importing - ${filename} - ${totalLineCount} lines imported\r`,
+              true
+            );
             resolve();
-          } else {
-            reject(error);
+          } catch (error) {
+            if (task.ignoreErrors) {
+              const errorMessage = error instanceof Error ? error.message : String(error);
+              task.logError(`Error finalizing ${filename}: ${errorMessage}`);
+              resolve();
+            } else {
+              reject(error);
+            }
           }
-        }
-      });
-      parser.on("error", (error) => {
-        if (task.ignoreErrors) {
-          task.logError(`Parser error for ${filename}: ${error.message}`);
-          resolve();
-        } else {
-          reject(error);
-        }
-      });
-      createReadStream(filepath).pipe(stripBomStream()).pipe(parser);
-    } else if (model.filenameExtension === "geojson") {
-      readFile2(filepath, "utf8").then((data) => {
-        if (isValidJSON(data) === false) {
+        });
+        parser.on("error", (error) => {
           if (task.ignoreErrors) {
-            task.logError(`Invalid JSON in ${filename}`);
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            task.logError(`Parser error for ${filename}: ${errorMessage}`);
             resolve();
-            return;
           } else {
-            reject(new Error(`Invalid JSON in ${filename}`));
-            return;
+            reject(error);
           }
-        }
-        totalLineCount += 1;
-        const line = formatGtfsLine(
-          { geojson: data },
-          model,
-          totalLineCount
-        );
-        try {
-          insertLines([line]);
-          task.log(
-            `Importing - ${filename} - ${totalLineCount} lines imported\r`,
-            true
+        });
+        createReadStream(filepath).pipe(stripBomStream()).pipe(parser);
+      } else if (model.filenameExtension === "geojson") {
+        readFile2(filepath, "utf8").then((data) => {
+          if (isValidJSON(data) === false) {
+            if (task.ignoreErrors) {
+              task.logError(`Invalid JSON in ${filename}`);
+              resolve();
+              return;
+            } else {
+              reject(new Error(`Invalid JSON in ${filename}`));
+              return;
+            }
+          }
+          totalLineCount += 1;
+          const line = formatGtfsLine(
+            { geojson: data },
+            model,
+            totalLineCount
           );
-          resolve();
-        } catch (error) {
-          if (task.ignoreErrors) {
-            task.logError(
-              `Error inserting data for ${filename}: ${error.message}`
+          try {
+            insertLines([line]);
+            task.log(
+              `Importing - ${filename} - ${totalLineCount} lines imported\r`,
+              true
             );
             resolve();
+          } catch (error) {
+            if (task.ignoreErrors) {
+              const errorMessage = error instanceof Error ? error.message : String(error);
+              task.logError(
+                `Error inserting data for ${filename}: ${errorMessage}`
+              );
+              resolve();
+            } else {
+              reject(error);
+            }
+          }
+        }).catch((error) => {
+          if (task.ignoreErrors) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            task.logError(`Error reading ${filename}: ${errorMessage}`);
+            resolve();
           } else {
             reject(error);
           }
-        }
-      }).catch((error) => {
+        });
+      } else {
         if (task.ignoreErrors) {
-          task.logError(`Error reading ${filename}: ${error.message}`);
+          task.logError(
+            `Unsupported file type: ${model.filenameExtension} for ${filename}`
+          );
           resolve();
         } else {
-          reject(error);
+          reject(
+            new Error(`Unsupported file type: ${model.filenameExtension}`)
+          );
         }
-      });
-    } else {
-      if (task.ignoreErrors) {
-        task.logError(
-          `Unsupported file type: ${model.filenameExtension} for ${filename}`
-        );
-        resolve();
-      } else {
-        reject(
-          new Error(`Unsupported file type: ${model.filenameExtension}`)
-        );
       }
-    }
-  })
-);
+    })
+  );
+  task.log(`Static GTFS import complete`);
+};
 async function importGtfs(initialConfig) {
   const startTime = process.hrtime.bigint();
   const config = setDefaultConfig(initialConfig);
@@ -5109,7 +5175,6 @@ async function importGtfs(initialConfig) {
         const tempPath = temporaryDirectory();
         const task = {
           exclude: agency2.exclude,
-          url: agency2.url,
           headers: agency2.headers,
           realtimeAlerts: agency2.realtimeAlerts,
           realtimeTripUpdates: agency2.realtimeTripUpdates,
@@ -5117,7 +5182,6 @@ async function importGtfs(initialConfig) {
           downloadDir: tempPath,
           downloadTimeout: config.downloadTimeout,
           gtfsRealtimeExpirationSeconds: config.gtfsRealtimeExpirationSeconds,
-          path: agency2.path,
           csvOptions: config.csvOptions || {},
           ignoreDuplicates: config.ignoreDuplicates,
           ignoreErrors: config.ignoreErrors,
@@ -5128,8 +5192,13 @@ async function importGtfs(initialConfig) {
           logWarning: logWarning(config),
           logError: logError(config)
         };
-        if (task.url) {
+        if ("url" in agency2) {
+          Object.assign(task, { url: agency2.url });
           await downloadGtfsFiles(task);
+        } else {
+          Object.assign(task, {
+            path: agency2.path
+          });
         }
         await extractGtfsFiles(task);
         await importGtfsFiles(db, task);
@@ -5137,7 +5206,8 @@ async function importGtfs(initialConfig) {
         await rm2(tempPath, { recursive: true });
       } catch (error) {
         if (config.ignoreErrors) {
-          logError(config)(error.message);
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          logError(config)(errorMessage);
         } else {
           throw error;
         }
@@ -5152,7 +5222,7 @@ async function importGtfs(initialConfig) {
 `
     );
   } catch (error) {
-    if (error?.code === "SQLITE_CANTOPEN") {
+    if (error.code === "SQLITE_CANTOPEN") {
       logError(config)(
         `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
       );
@@ -5171,7 +5241,7 @@ import mapSeries3 from "promise-map-series";
 var getAgencies = (db, config) => {
   try {
     return db.prepare("SELECT agency_name FROM agency;").all();
-  } catch (error) {
+  } catch {
     if (config.sqlitePath === ":memory:") {
       throw new Error(
         'No agencies found in SQLite. You are using an in-memory database - if running this from command line be sure to specify a value for `sqlitePath` in config.json other than ":memory:".'
@@ -5238,11 +5308,17 @@ var exportGtfs = async (initialConfig) => {
           }
         } else if (model.filenameBase === "fare_attributes") {
           for (const line of lines) {
-            line.price = formatCurrency(line.price, line.currency_type);
+            line.price = formatCurrency(
+              line.price,
+              line.currency_type
+            );
           }
         } else if (model.filenameBase === "fare_products") {
           for (const line of lines) {
-            line.amount = formatCurrency(line.amount, line.currency);
+            line.amount = formatCurrency(
+              line.amount,
+              line.currency
+            );
           }
         }
         const columns = without(
@@ -5617,10 +5693,7 @@ function getRoutes(query = {}, fields = [], orderBy2 = [], options = {}) {
   let whereClause = "";
   const orderByClause = formatOrderByClause(orderBy2);
   const routeQuery = omit3(query, ["stop_id", "service_id"]);
-  const tripQuery = pick(query, [
-    "stop_id",
-    "service_id"
-  ]);
+  const tripQuery = pick(query, ["stop_id", "service_id"]);
   const whereClauses = Object.entries(routeQuery).map(
     ([key, value]) => formatWhereClause(key, value)
   );
@@ -5668,7 +5741,12 @@ function getShapes(query = {}, fields = [], orderBy2 = [], options = {}) {
     "service_id",
     "direction_id"
   ]);
-  const tripQuery = pick2(query, ["route_id", "trip_id", "service_id", "direction_id"]);
+  const tripQuery = pick2(query, [
+    "route_id",
+    "trip_id",
+    "service_id",
+    "direction_id"
+  ]);
   const whereClauses = Object.entries(shapeQuery).map(
     ([key, value]) => formatWhereClause(key, value)
   );
@@ -5775,7 +5853,7 @@ function getStops(query = {}, fields = [], orderBy2 = [], options = {}) {
   if (options.bounding_box_side_m !== void 0) {
     stopQueryOmitKeys.push("stop_lat", "stop_lon");
   }
-  let stopQuery = omit5(query, stopQueryOmitKeys);
+  const stopQuery = omit5(query, stopQueryOmitKeys);
   const tripQuery = pick3(query, [
     "route_id",
     "trip_id",
@@ -5840,7 +5918,7 @@ function getStoptimes(query = {}, fields = [], orderBy2 = [], options = {}) {
   let whereClause = "";
   const orderByClause = formatOrderByClause(orderBy2);
   const stoptimeQueryOmitKeys = ["date", "start_time", "end_time"];
-  let stoptimeQuery = omit6(query, stoptimeQueryOmitKeys);
+  const stoptimeQuery = omit6(query, stoptimeQueryOmitKeys);
   const whereClauses = Object.entries(stoptimeQuery).map(
     ([key, value]) => formatWhereClause(key, value)
   );
@@ -5848,7 +5926,7 @@ function getStoptimes(query = {}, fields = [], orderBy2 = [], options = {}) {
     if (typeof query.date !== "number") {
       throw new Error("`date` must be a number in yyyymmdd format");
     }
-    const serviceIds = getServiceIdsByDate(query.date);
+    const serviceIds = getServiceIdsByDate(query.date, options);
     const tripSubquery = `SELECT DISTINCT trip_id FROM trips WHERE service_id IN (${serviceIds.map((id) => sqlString4.escape(id)).join(",")})`;
     whereClauses.push(`trip_id IN (${tripSubquery})`);
   }
@@ -5922,7 +6000,7 @@ function getTrips(query = {}, fields = [], orderBy2 = [], options = {}) {
   let whereClause = "";
   const orderByClause = formatOrderByClause(orderBy2);
   const tripQueryOmitKeys = ["date"];
-  let tripQuery = omit7(query, tripQueryOmitKeys);
+  const tripQuery = omit7(query, tripQueryOmitKeys);
   const whereClauses = Object.entries(tripQuery).map(
     ([key, value]) => formatWhereClause(key, value)
   );
@@ -5930,7 +6008,7 @@ function getTrips(query = {}, fields = [], orderBy2 = [], options = {}) {
     if (typeof query.date !== "number") {
       throw new Error("`date` must be a number in yyyymmdd format");
     }
-    const serviceIds = getServiceIdsByDate(query.date);
+    const serviceIds = getServiceIdsByDate(query.date, options);
     whereClauses.push(
       `service_id IN (${serviceIds.map((id) => sqlString5.escape(id)).join(",")})`
     );
@@ -6212,6 +6290,7 @@ export {
   closeDb,
   deleteDb,
   exportGtfs,
+  generateFolderName,
   getAgencies2 as getAgencies,
   getAreas,
   getAttributions,
@@ -6272,6 +6351,9 @@ export {
   getVehiclePositions,
   importGtfs,
   openDb,
+  prepDirectory,
+  untildify,
+  unzip,
   updateGtfsRealtime
 };
 //# sourceMappingURL=index.js.map