gtfs 4.17.6 → 4.18.0

package/dist/bin/gtfs-import.js

@@ -13,10 +13,10 @@ import PrettyError from "pretty-error";
 // src/lib/file-utils.ts
 import path from "path";
 import { existsSync } from "fs";
+import { homedir } from "os";
 import { mkdir, readFile, rm } from "fs/promises";
 import { omit, snakeCase } from "lodash-es";
 import sanitize from "sanitize-filename";
-import untildify from "untildify";
 import StreamZip from "node-stream-zip";
 
 // src/lib/log-utils.ts
@@ -70,6 +70,7 @@ function formatError(error) {
 }
 
 // src/lib/file-utils.ts
+var homeDirectory = homedir();
 async function getConfig(argv2) {
   let config;
   let data;
@@ -117,17 +118,17 @@ async function unzip(zipfilePath, exportPath) {
     );
   }
 }
+function untildify(pathWithTilde) {
+  return homeDirectory ? pathWithTilde.replace(/^~(?=$|\/|\\)/, homeDirectory) : pathWithTilde;
+}
 
 // src/lib/import-gtfs.ts
 import path2 from "path";
 import { createReadStream, existsSync as existsSync2, lstatSync } from "fs";
 import { cp, readdir, rename, readFile as readFile2, rm as rm2, writeFile } from "fs/promises";
 import { parse } from "csv-parse";
-import pluralize2 from "pluralize";
 import stripBomStream from "strip-bom-stream";
 import { temporaryDirectory } from "tempy";
-import Timer from "timer-machine";
-import untildify3 from "untildify";
 import mapSeries2 from "promise-map-series";
 
 // src/models/models.ts
@@ -4089,10 +4090,9 @@ var vehicles = {
 
 // src/lib/db.ts
 import Database from "better-sqlite3";
-import untildify2 from "untildify";
 var dbs = {};
 function setupDb(sqlitePath) {
-  const db = new Database(untildify2(sqlitePath));
+  const db = new Database(untildify(sqlitePath));
   db.pragma("journal_mode = OFF");
   db.pragma("synchronous = OFF");
   db.pragma("temp_store = MEMORY");
@@ -4158,13 +4158,12 @@ function isValidJSON(string) {
   try {
     JSON.parse(string);
     return true;
-  } catch (error) {
+  } catch {
     return false;
   }
 }
 
 // src/lib/import-gtfs-realtime.ts
-import pluralize from "pluralize";
 import GtfsRealtimeBindings from "gtfs-realtime-bindings";
 import mapSeries from "promise-map-series";
 import { get } from "lodash-es";
@@ -4191,7 +4190,8 @@ function setDefaultConfig(initialConfig) {
     ignoreDuplicates: false,
     ignoreErrors: false,
     gtfsRealtimeExpirationSeconds: 0,
-    verbose: true
+    verbose: true,
+    downloadTimeout: 3e4
   };
   return {
     ...defaults,
@@ -4220,38 +4220,14 @@ function applyPrefixToValue(value, columnShouldBePrefixed, prefix) {
   }
   return `${prefix}${value}`;
 }
+function pluralize(singularWord, pluralWord, count) {
+  return count === 1 ? singularWord : pluralWord;
+}
 
 // src/lib/import-gtfs-realtime.ts
-async function fetchGtfsRealtimeData(urlConfig, task) {
-  task.log(`Downloading GTFS-Realtime from ${urlConfig.url}`);
-  const response = await fetch(urlConfig.url, {
-    method: "GET",
-    headers: {
-      ...urlConfig.headers ?? {},
-      "Accept-Encoding": "gzip"
-    },
-    signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
-  });
-  if (response.status !== 200) {
-    task.logWarning(
-      `Unable to download GTFS-Realtime from ${urlConfig.url}. Got status ${response.status}.`
-    );
-    return null;
-  }
-  const buffer = await response.arrayBuffer();
-  const message = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(
-    new Uint8Array(buffer)
-  );
-  return GtfsRealtimeBindings.transit_realtime.FeedMessage.toObject(message, {
-    enums: String,
-    longs: String,
-    bytes: String,
-    defaults: false,
-    arrays: true,
-    objects: true,
-    oneofs: true
-  });
-}
+var BATCH_SIZE = 1e3;
+var MAX_RETRIES = 3;
+var RETRY_DELAY = 1e3;
 function prepareRealtimeFieldValue(entity, column, task) {
   if (column.name === "created_timestamp") {
     return task.currentTimestamp;
@@ -4268,163 +4244,246 @@ function prepareRealtimeFieldValue(entity, column, task) {
   );
   return column.type === "json" ? JSON.stringify(prefixedValue) : prefixedValue;
 }
-async function processRealtimeAlerts(db, gtfsRealtimeData, task) {
-  const alertStmt = db.prepare(
-    `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${serviceAlerts.schema.map(() => "?").join(", ")})`
-  );
-  const informedEntityStmt = db.prepare(
-    `REPLACE INTO ${serviceAlertInformedEntities.filenameBase} (${serviceAlertInformedEntities.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${serviceAlertInformedEntities.schema.map(() => "?").join(", ")})`
+function createPreparedStatement(db, model) {
+  const columns = model.schema.map((column) => column.name);
+  const placeholders = model.schema.map(() => "?").join(", ");
+  return db.prepare(
+    `REPLACE INTO ${model.filenameBase} (${columns.join(", ")}) VALUES (${placeholders})`
   );
-  let totalLineCount = 0;
-  db.transaction(() => {
-    for (const entity of gtfsRealtimeData.entity) {
-      const fieldValues = serviceAlerts.schema.map(
-        (column) => prepareRealtimeFieldValue(entity, column, task)
+}
+async function processBatch(items, batchSize, processor) {
+  let totalRecordCount = 0;
+  let totalErrorCount = 0;
+  for (let i = 0; i < items.length; i += batchSize) {
+    const batch = items.slice(i, i + batchSize);
+    try {
+      const result = await processor(batch);
+      totalRecordCount += result.recordCount;
+      totalErrorCount += result.errorCount;
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      totalErrorCount += batch.length;
+      console.error(`Batch processing error: ${errorMessage}`);
+    }
+  }
+  return { recordCount: totalRecordCount, errorCount: totalErrorCount };
+}
+async function fetchGtfsRealtimeData(type, task) {
+  const urlConfig = getUrlConfig(type, task);
+  if (!urlConfig) {
+    return null;
+  }
+  task.log(`Importing - GTFS-Realtime from ${urlConfig.url}`);
+  for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      const response = await fetch(urlConfig.url, {
+        method: "GET",
+        headers: {
+          ...urlConfig.headers ?? {},
+          "Accept-Encoding": "gzip"
+        },
+        signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
+      });
+      if (response.status !== 200) {
+        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+      }
+      const buffer = await response.arrayBuffer();
+      const message = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(
+        new Uint8Array(buffer)
       );
-      try {
-        alertStmt.run(fieldValues);
-        if (entity.alert.informedEntity?.length) {
-          const informedEntities = entity.alert.informedEntity.map(
-            (informedEntity) => {
+      const feedMessage = GtfsRealtimeBindings.transit_realtime.FeedMessage.toObject(message, {
+        enums: String,
+        longs: String,
+        bytes: String,
+        defaults: false,
+        arrays: true,
+        objects: true,
+        oneofs: true
+      });
+      return feedMessage;
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      if (attempt === MAX_RETRIES) {
+        if (task.ignoreErrors) {
+          task.logError(
+            `Failed to fetch ${type} after ${MAX_RETRIES} attempts: ${errorMessage}`
+          );
+          return null;
+        }
+        throw error;
+      }
+      task.logWarning(`Attempt ${attempt} failed for ${type}: ${errorMessage}`);
+      await new Promise(
+        (resolve) => setTimeout(resolve, RETRY_DELAY * attempt)
+      );
+    }
+  }
+  return null;
+}
+function getUrlConfig(type, task) {
+  switch (type) {
+    case "alerts":
+      return task.realtimeAlerts;
+    case "tripupdates":
+      return task.realtimeTripUpdates;
+    case "vehiclepositions":
+      return task.realtimeVehiclePositions;
+    default:
+      return void 0;
+  }
+}
+function createServiceAlertsProcessor(db, task) {
+  const alertStmt = createPreparedStatement(db, serviceAlerts);
+  const informedEntityStmt = createPreparedStatement(
+    db,
+    serviceAlertInformedEntities
+  );
+  return async (batch) => {
+    let recordCount = 0;
+    let errorCount = 0;
+    db.transaction(() => {
+      for (const entity of batch) {
+        try {
+          const alertValues = serviceAlerts.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
+          alertStmt.run(alertValues);
+          recordCount++;
+          if (entity.alert?.informedEntity?.length) {
+            for (const informedEntity of entity.alert.informedEntity) {
               informedEntity.parent = entity;
-              return serviceAlertInformedEntities.schema.map(
+              const entityValues = serviceAlertInformedEntities.schema.map(
                 (column) => prepareRealtimeFieldValue(informedEntity, column, task)
               );
+              informedEntityStmt.run(entityValues);
+              recordCount++;
             }
-          );
-          for (const values of informedEntities) {
-            informedEntityStmt.run(values);
           }
+        } catch (error) {
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          errorCount++;
+          task.logWarning(`Alert processing error: ${errorMessage}`);
         }
-        totalLineCount++;
-      } catch (error) {
-        task.logWarning(`Import error: ${error.message}`);
       }
-    }
-    task.log(
-      `Importing - GTFS-Realtime service alerts - ${totalLineCount} entries imported\r`,
-      true
-    );
-  })();
+    })();
+    return { recordCount, errorCount };
+  };
 }
-async function processRealtimeTripUpdates(db, gtfsRealtimeData, task) {
-  let totalLineCount = 0;
-  const tripUpdateStmt = db.prepare(
-    `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${tripUpdates.schema.map(() => "?").join(", ")})`
+function createTripUpdatesProcessor(db, task) {
+  const tripUpdateStmt = createPreparedStatement(
+    db,
+    tripUpdates
   );
-  const stopTimeStmt = db.prepare(
-    `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${stopTimeUpdates.schema.map(() => "?").join(", ")})`
+  const stopTimeStmt = createPreparedStatement(
+    db,
+    stopTimeUpdates
   );
-  db.transaction(() => {
-    for (const entity of gtfsRealtimeData.entity) {
-      try {
-        const fieldValues = tripUpdates.schema.map(
-          (column) => prepareRealtimeFieldValue(entity, column, task)
-        );
-        tripUpdateStmt.run(fieldValues);
-        for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
-          stopTimeUpdate.parent = entity;
-          const values = stopTimeUpdates.schema.map(
-            (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
-          );
-          stopTimeStmt.run(values);
+  return async (batch) => {
+    let recordCount = 0;
+    let errorCount = 0;
+    db.transaction(() => {
+      for (const entity of batch) {
+        try {
+          const tripUpdateValues = tripUpdates.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
+          tripUpdateStmt.run(tripUpdateValues);
+          recordCount++;
+          if (entity.tripUpdate?.stopTimeUpdate?.length) {
+            for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
+              stopTimeUpdate.parent = entity;
+              const stopTimeValues = stopTimeUpdates.schema.map(
+                (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
+              );
+              stopTimeStmt.run(stopTimeValues);
+              recordCount++;
+            }
+          }
+        } catch (error) {
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          errorCount++;
+          task.logWarning(`Trip update processing error: ${errorMessage}`);
         }
-        totalLineCount++;
-      } catch (error) {
-        task.logWarning(`Import error: ${error.message}`);
       }
-    }
-    task.log(
-      `Importing - GTFS-Realtime trip updates - ${totalLineCount} entries imported\r`,
-      true
-    );
-  })();
+    })();
+    return { recordCount, errorCount };
+  };
 }
-async function processRealtimeVehiclePositions(db, gtfsRealtimeData, task) {
-  let totalLineCount = 0;
-  const vehiclePositionStmt = db.prepare(
-    `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(
-      ", "
-    )}) VALUES (${vehiclePositions.schema.map(() => "?").join(", ")})`
+function createVehiclePositionsProcessor(db, task) {
+  const vehiclePositionStmt = createPreparedStatement(
+    db,
+    vehiclePositions
   );
-  db.transaction(() => {
-    for (const entity of gtfsRealtimeData.entity) {
-      try {
-        const fieldValues = vehiclePositions.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
-        vehiclePositionStmt.run(fieldValues);
-        totalLineCount++;
-      } catch (error) {
-        task.logWarning(`Import error: ${error.message}`);
+  return async (batch) => {
+    let recordCount = 0;
+    let errorCount = 0;
+    db.transaction(() => {
+      for (const entity of batch) {
+        try {
+          const fieldValues = vehiclePositions.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
+          vehiclePositionStmt.run(fieldValues);
+          recordCount++;
+        } catch (error) {
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          errorCount++;
+          task.logWarning(`Vehicle position processing error: ${errorMessage}`);
+        }
       }
-    }
-    task.log(
-      `Importing - GTFS-Realtime vehicle positions - ${totalLineCount} entries imported\r`,
-      true
-    );
-  })();
+    })();
+    return { recordCount, errorCount };
+  };
 }
 async function updateGtfsRealtimeData(task) {
-  if (task.realtimeAlerts === void 0 && task.realtimeTripUpdates === void 0 && task.realtimeVehiclePositions === void 0) {
+  if (!task.realtimeAlerts && !task.realtimeTripUpdates && !task.realtimeVehiclePositions) {
     return;
   }
+  const [alertsData, tripUpdatesData, vehiclePositionsData] = await Promise.all(
+    [
+      task.realtimeAlerts?.url ? fetchGtfsRealtimeData("alerts", task) : null,
+      task.realtimeTripUpdates?.url ? fetchGtfsRealtimeData("tripupdates", task) : null,
+      task.realtimeVehiclePositions?.url ? fetchGtfsRealtimeData("vehiclepositions", task) : null
+    ]
+  );
   const db = openDb({ sqlitePath: task.sqlitePath });
-  if (task.realtimeAlerts?.url) {
-    try {
-      const alertsData = await fetchGtfsRealtimeData(task.realtimeAlerts, task);
-      if (alertsData?.entity) {
-        await processRealtimeAlerts(db, alertsData, task);
-      }
-    } catch (error) {
-      if (task.ignoreErrors) {
-        task.logError(error.message);
-      } else {
-        throw error;
-      }
-    }
+  const recordCounts = {
+    alerts: 0,
+    tripupdates: 0,
+    vehiclepositions: 0
+  };
+  const processingPromises = [];
+  if (alertsData?.entity?.length) {
+    processingPromises.push(
+      processBatch(
+        alertsData.entity,
+        BATCH_SIZE,
+        createServiceAlertsProcessor(db, task)
+      ).then((result) => {
+        recordCounts.alerts = result.recordCount;
+      })
+    );
   }
-  if (task.realtimeTripUpdates?.url) {
-    try {
-      const tripUpdatesData = await fetchGtfsRealtimeData(
-        task.realtimeTripUpdates,
-        task
-      );
-      if (tripUpdatesData?.entity) {
-        await processRealtimeTripUpdates(db, tripUpdatesData, task);
-      }
-    } catch (error) {
-      if (task.ignoreErrors) {
-        task.logError(error.message);
-      } else {
-        throw error;
-      }
-    }
+  if (tripUpdatesData?.entity?.length) {
+    processingPromises.push(
+      processBatch(
+        tripUpdatesData.entity,
+        BATCH_SIZE,
+        createTripUpdatesProcessor(db, task)
+      ).then((result) => {
+        recordCounts.tripupdates = result.recordCount;
+      })
+    );
   }
-  if (task.realtimeVehiclePositions?.url) {
-    try {
-      const vehiclePositionsData = await fetchGtfsRealtimeData(
-        task.realtimeVehiclePositions,
-        task
-      );
-      if (vehiclePositionsData?.entity) {
-        await processRealtimeVehiclePositions(db, vehiclePositionsData, task);
-      }
-    } catch (error) {
-      if (task.ignoreErrors) {
-        task.logError(error.message);
-      } else {
-        throw error;
-      }
-    }
+  if (vehiclePositionsData?.entity?.length) {
+    processingPromises.push(
+      processBatch(
+        vehiclePositionsData.entity,
+        BATCH_SIZE,
+        createVehiclePositionsProcessor(db, task)
+      ).then((result) => {
+        recordCounts.vehiclepositions = result.recordCount;
+      })
+    );
   }
-  task.log(`GTFS-Realtime data import complete`);
+  await Promise.all(processingPromises);
+  task.log(
+    `GTFS-Realtime import complete: ${recordCounts.alerts} alerts, ${recordCounts.tripupdates} trip updates, ${recordCounts.vehiclepositions} vehicle positions`
+  );
 }
 
 // src/lib/import-gtfs.ts
@@ -4456,8 +4515,8 @@ var extractGtfsFiles = async (task) => {
   if (!task.path) {
     throw new Error("No `path` specified in config");
   }
-  const gtfsPath = untildify3(task.path);
-  task.log(`Importing GTFS from ${task.path}\r`);
+  const gtfsPath = untildify(task.path);
+  task.log(`Importing static GTFS from ${task.path}\r`);
   if (path2.extname(gtfsPath) === ".zip") {
     try {
       await unzip(gtfsPath, task.downloadDir);
@@ -4540,11 +4599,11 @@ var createGtfsTables = (db) => {
       if (column.type === "time") {
         sqlColumnCreateStatements.push(
           `${getTimestampColumnName(column.name)} INTEGER GENERATED ALWAYS AS (
-            CASE 
-              WHEN ${column.name} IS NULL OR ${column.name} = '' THEN NULL 
+            CASE
+              WHEN ${column.name} IS NULL OR ${column.name} = '' THEN NULL
               ELSE CAST(
-                substr(${column.name}, 1, instr(${column.name}, ':') - 1) * 3600 + 
-                substr(${column.name}, instr(${column.name}, ':') + 1, 2) * 60 + 
+                substr(${column.name}, 1, instr(${column.name}, ':') - 1) * 3600 +
+                substr(${column.name}, instr(${column.name}, ':') + 1, 2) * 60 +
                 substr(${column.name}, -2) AS INTEGER
               )
             END
@@ -4601,7 +4660,7 @@ var formatGtfsLine = (line, model, totalLineCount) => {
       continue;
     }
     if (type === "date") {
-      value = value.replace(/-/g, "");
+      value = value?.toString().replace(/-/g, "");
       if (value.length !== 8) {
         throw new Error(
           `Invalid date in ${filenameBase}.${filenameExtension} for ${name} on line ${lineNumber}.`
@@ -4617,155 +4676,221 @@ var formatGtfsLine = (line, model, totalLineCount) => {
   }
   return formattedLine;
 };
-var BATCH_SIZE = 1e5;
-var importGtfsFiles = (db, task) => mapSeries2(
-  Object.values(models_exports),
-  (model) => new Promise((resolve, reject) => {
-    let totalLineCount = 0;
-    const filename = `${model.filenameBase}.${model.filenameExtension}`;
-    if (task.exclude && task.exclude.includes(model.filenameBase)) {
-      task.log(`Skipping - ${filename}\r`);
-      resolve();
-      return;
-    }
-    if (model.extension === "gtfs-realtime") {
-      resolve();
-      return;
-    }
-    const filepath = path2.join(task.downloadDir, `${filename}`);
-    if (!existsSync2(filepath)) {
-      if (!model.nonstandard) {
-        task.log(`Importing - ${filename} - No file found\r`);
+var BATCH_SIZE2 = 1e5;
+var importGtfsFiles = async (db, task) => {
+  await mapSeries2(
+    Object.values(models_exports),
+    (model) => new Promise((resolve, reject) => {
+      let totalLineCount = 0;
+      const filename = `${model.filenameBase}.${model.filenameExtension}`;
+      if (task.exclude && task.exclude.includes(model.filenameBase)) {
+        task.log(`Skipping - ${filename}\r`);
+        resolve();
+        return;
       }
-      resolve();
-      return;
-    }
-    task.log(`Importing - ${filename}\r`);
-    const columns = model.schema;
-    const prefixedColumns = new Set(
-      columns.filter((column) => column.prefix).map((column) => column.name)
-    );
-    const prepareStatement = `INSERT ${task.ignoreDuplicates ? "OR IGNORE" : ""} INTO ${model.filenameBase} (${columns.map(({ name }) => name).join(", ")}) VALUES (${columns.map(({ name }) => `@${name}`).join(", ")})`;
-    const insert = db.prepare(prepareStatement);
-    const insertLines = db.transaction((lines) => {
-      for (const [rowNumber, line] of Object.entries(lines)) {
-        try {
-          if (task.prefix === void 0) {
-            insert.run(line);
-          } else {
-            const prefixedLine = Object.fromEntries(
-              Object.entries(
-                line
-              ).map(([columnName, value]) => [
-                columnName,
-                applyPrefixToValue(
-                  value,
-                  prefixedColumns.has(columnName),
-                  task.prefix
-                )
-              ])
-            );
-            insert.run(prefixedLine);
-          }
-        } catch (error) {
-          if (error.code === "SQLITE_CONSTRAINT_PRIMARYKEY") {
-            const primaryColumns = columns.filter(
-              (column) => column.primary
-            );
-            task.logWarning(
-              `Duplicate values for primary key (${primaryColumns.map((column) => column.name).join(", ")}) found in ${filename}. Set the \`ignoreDuplicates\` option to true in config.json to ignore this error`
-            );
-          }
-          task.logWarning(
-            `Check ${filename} for invalid data on line ${rowNumber + 1}.`
-          );
-          throw error;
+      if (model.extension === "gtfs-realtime") {
+        resolve();
+        return;
+      }
+      const filepath = path2.join(task.downloadDir, `${filename}`);
+      if (!existsSync2(filepath)) {
+        if (!model.nonstandard) {
+          task.log(`Importing - ${filename} - No file found\r`);
         }
+        resolve();
+        return;
       }
-    });
-    if (model.filenameExtension === "txt") {
-      const parser = parse({
-        columns: true,
-        relax_quotes: true,
-        trim: true,
-        skip_empty_lines: true,
-        ...task.csvOptions
-      });
-      let lines = [];
-      parser.on("readable", () => {
-        try {
-          let record;
-          while (record = parser.read()) {
-            totalLineCount += 1;
-            lines.push(formatGtfsLine(record, model, totalLineCount));
-            if (lines.length >= BATCH_SIZE) {
-              insertLines(lines);
-              lines = [];
-              task.log(
-                `Importing - ${filename} - ${totalLineCount} lines imported\r`,
-                true
+      task.log(`Importing - ${filename}\r`);
+      const columns = model.schema;
+      const prefixedColumns = new Set(
+        columns.filter((column) => column.prefix).map((column) => column.name)
+      );
+      const prepareStatement = `INSERT ${task.ignoreDuplicates ? "OR IGNORE" : ""} INTO ${model.filenameBase} (${columns.map(({ name }) => name).join(", ")}) VALUES (${columns.map(({ name }) => `@${name}`).join(", ")})`;
+      const insert = db.prepare(prepareStatement);
+      const insertLines = db.transaction((lines) => {
+        for (const [rowNumber, line] of Object.entries(lines)) {
+          try {
+            if (task.prefix === void 0) {
+              insert.run(line);
+            } else {
+              const prefixedLine = Object.fromEntries(
+                Object.entries(
+                  line
+                ).map(([columnName, value]) => [
+                  columnName,
+                  applyPrefixToValue(
+                    value,
+                    prefixedColumns.has(columnName),
+                    task.prefix
+                  )
+                ])
               );
+              insert.run(prefixedLine);
             }
+          } catch (error) {
+            if (error.code === "SQLITE_CONSTRAINT_PRIMARYKEY") {
+              const primaryColumns = columns.filter(
+                (column) => column.primary
+              );
+              task.logWarning(
+                `Duplicate values for primary key (${primaryColumns.map((column) => column.name).join(", ")}) found in ${filename}. Set the \`ignoreDuplicates\` option to true in config.json to ignore this error`
+              );
+            }
+            task.logWarning(
+              `Check ${filename} for invalid data on line ${rowNumber + 1}.`
+            );
+            throw error;
           }
-        } catch (error) {
-          reject(error);
         }
       });
-      parser.on("end", () => {
-        try {
-          if (lines.length > 0) {
-            try {
-              insertLines(lines);
-            } catch (error) {
+      if (model.filenameExtension === "txt") {
+        const parser = parse({
+          columns: true,
+          relax_quotes: true,
+          trim: true,
+          skip_empty_lines: true,
+          ...task.csvOptions
+        });
+        let lines = [];
+        parser.on("readable", () => {
+          try {
+            let record;
+            while (record = parser.read()) {
+              totalLineCount += 1;
+              lines.push(formatGtfsLine(record, model, totalLineCount));
+              if (lines.length >= BATCH_SIZE2) {
+                insertLines(lines);
+                lines = [];
+                task.log(
+                  `Importing - ${filename} - ${totalLineCount} lines imported\r`,
+                  true
+                );
+              }
+            }
+          } catch (error) {
+            if (task.ignoreErrors) {
+              const errorMessage = error instanceof Error ? error.message : String(error);
+              task.logError(`Error processing ${filename}: ${errorMessage}`);
+              resolve();
+            } else {
               reject(error);
             }
           }
-          task.log(
-            `Importing - ${filename} - ${totalLineCount} lines imported\r`,
-            true
+        });
+        parser.on("end", () => {
+          try {
+            if (lines.length > 0) {
+              try {
+                insertLines(lines);
+              } catch (error) {
+                if (task.ignoreErrors) {
+                  const errorMessage = error instanceof Error ? error.message : String(error);
+                  task.logError(
+                    `Error inserting data for ${filename}: ${errorMessage}`
+                  );
+                  resolve();
+                  return;
+                } else {
+                  reject(error);
+                  return;
+                }
+              }
+            }
+            task.log(
+              `Importing - ${filename} - ${totalLineCount} lines imported\r`,
+              true
+            );
+            resolve();
+          } catch (error) {
+            if (task.ignoreErrors) {
+              const errorMessage = error instanceof Error ? error.message : String(error);
+              task.logError(`Error finalizing ${filename}: ${errorMessage}`);
+              resolve();
+            } else {
+              reject(error);
+            }
+          }
+        });
+        parser.on("error", (error) => {
+          if (task.ignoreErrors) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            task.logError(`Parser error for ${filename}: ${errorMessage}`);
+            resolve();
+          } else {
+            reject(error);
+          }
+        });
+        createReadStream(filepath).pipe(stripBomStream()).pipe(parser);
+      } else if (model.filenameExtension === "geojson") {
+        readFile2(filepath, "utf8").then((data) => {
+          if (isValidJSON(data) === false) {
+            if (task.ignoreErrors) {
+              task.logError(`Invalid JSON in ${filename}`);
+              resolve();
+              return;
+            } else {
+              reject(new Error(`Invalid JSON in ${filename}`));
+              return;
+            }
+          }
+          totalLineCount += 1;
+          const line = formatGtfsLine(
+            { geojson: data },
+            model,
+            totalLineCount
+          );
+          try {
+            insertLines([line]);
+            task.log(
+              `Importing - ${filename} - ${totalLineCount} lines imported\r`,
+              true
+            );
+            resolve();
+          } catch (error) {
+            if (task.ignoreErrors) {
+              const errorMessage = error instanceof Error ? error.message : String(error);
+              task.logError(
+                `Error inserting data for ${filename}: ${errorMessage}`
+              );
+              resolve();
+            } else {
+              reject(error);
+            }
+          }
+        }).catch((error) => {
+          if (task.ignoreErrors) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            task.logError(`Error reading ${filename}: ${errorMessage}`);
+            resolve();
+          } else {
+            reject(error);
+          }
+        });
+      } else {
+        if (task.ignoreErrors) {
+          task.logError(
+            `Unsupported file type: ${model.filenameExtension} for ${filename}`
           );
           resolve();
-        } catch (error) {
-          reject(error);
-        }
-      });
-      parser.on("error", reject);
-      createReadStream(filepath).pipe(stripBomStream()).pipe(parser);
-    } else if (model.filenameExtension === "geojson") {
-      readFile2(filepath, "utf8").then((data) => {
-        if (isValidJSON(data) === false) {
-          reject(new Error(`Invalid JSON in ${filename}`));
+        } else {
+          reject(
+            new Error(`Unsupported file type: ${model.filenameExtension}`)
+          );
         }
-        totalLineCount += 1;
-        const line = formatGtfsLine(
-          { geojson: data },
-          model,
-          totalLineCount
-        );
-        insertLines([line]);
-        task.log(
-          `Importing - ${filename} - ${totalLineCount} lines imported\r`,
-          true
-        );
-        resolve();
-      }).catch(reject);
-    } else {
-      reject(
-        new Error(`Unsupported file type: ${model.filenameExtension}`)
-      );
-    }
-  })
-);
+      }
+    })
+  );
+  task.log(`Static GTFS import complete`);
+};
 async function importGtfs(initialConfig) {
-  const timer = new Timer();
-  timer.start();
+  const startTime = process.hrtime.bigint();
   const config = setDefaultConfig(initialConfig);
   validateConfigForImport(config);
   try {
     const db = openDb(config);
     const agencyCount = config.agencies.length;
     log(config)(
-      `Starting GTFS import for ${pluralize2("file", agencyCount, true)} using SQLite database at ${config.sqlitePath}`
+      `Starting GTFS import for ${pluralize("file", "files", agencyCount)} using SQLite database at ${config.sqlitePath}`
     );
     createGtfsTables(db);
     await mapSeries2(config.agencies, async (agency2) => {
@@ -4773,7 +4898,6 @@ async function importGtfs(initialConfig) {
         const tempPath = temporaryDirectory();
         const task = {
           exclude: agency2.exclude,
-          url: agency2.url,
           headers: agency2.headers,
           realtimeAlerts: agency2.realtimeAlerts,
           realtimeTripUpdates: agency2.realtimeTripUpdates,
@@ -4781,7 +4905,6 @@ async function importGtfs(initialConfig) {
           downloadDir: tempPath,
           downloadTimeout: config.downloadTimeout,
           gtfsRealtimeExpirationSeconds: config.gtfsRealtimeExpirationSeconds,
-          path: agency2.path,
           csvOptions: config.csvOptions || {},
           ignoreDuplicates: config.ignoreDuplicates,
           ignoreErrors: config.ignoreErrors,
@@ -4792,8 +4915,13 @@ async function importGtfs(initialConfig) {
           logWarning: logWarning(config),
           logError: logError(config)
         };
-        if (task.url) {
+        if ("url" in agency2) {
+          Object.assign(task, { url: agency2.url });
           await downloadGtfsFiles(task);
+        } else {
+          Object.assign(task, {
+            path: agency2.path
+          });
         }
         await extractGtfsFiles(task);
         await importGtfsFiles(db, task);
@@ -4801,7 +4929,8 @@ async function importGtfs(initialConfig) {
         await rm2(tempPath, { recursive: true });
       } catch (error) {
         if (config.ignoreErrors) {
-          logError(config)(error.message);
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          logError(config)(errorMessage);
         } else {
           throw error;
         }
@@ -4809,14 +4938,14 @@ async function importGtfs(initialConfig) {
     });
     log(config)(`Creating DB indexes`);
     createGtfsIndexes(db);
-    const seconds = Math.round(timer.time() / 1e3);
-    timer.stop();
+    const endTime = process.hrtime.bigint();
+    const elapsedSeconds = Number(endTime - startTime) / 1e9;
     log(config)(
-      `Completed GTFS import for ${pluralize2("agency", agencyCount, true)} in ${seconds} seconds
+      `Completed GTFS import in ${elapsedSeconds.toFixed(1)} seconds
 `
     );
   } catch (error) {
-    if (error?.code === "SQLITE_CANTOPEN") {
+    if (error.code === "SQLITE_CANTOPEN") {
       logError(config)(
         `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
       );
@@ -4829,11 +4958,9 @@ async function importGtfs(initialConfig) {
 import path3 from "path";
 import { writeFile as writeFile2 } from "fs/promises";
 import { without, compact as compact2 } from "lodash-es";
-import pluralize3 from "pluralize";
 import { stringify } from "csv-stringify";
 import sqlString2 from "sqlstring-sqlite";
 import mapSeries3 from "promise-map-series";
-import untildify4 from "untildify";
 
 // src/lib/advancedQuery.ts
 import sqlString3 from "sqlstring-sqlite";