gtfs 4.14.5 → 4.15.0

package/dist/index.js

@@ -4,18 +4,16 @@ var __export = (target, all) => {
     __defProp(target, name, { get: all[name], enumerable: true });
 };
 
-// src/lib/import.ts
+// src/lib/import-gtfs.ts
 import path from "node:path";
 import { createReadStream, existsSync, lstatSync } from "node:fs";
 import { cp, readdir, rename, readFile as readFile2, rm as rm2, writeFile } from "node:fs/promises";
 import { parse } from "csv-parse";
-import pluralize from "pluralize";
+import pluralize2 from "pluralize";
 import stripBomStream from "strip-bom-stream";
 import { temporaryDirectory } from "tempy";
 import untildify3 from "untildify";
-import mapSeries from "promise-map-series";
-import GtfsRealtimeBindings from "gtfs-realtime-bindings";
-import sqlString2 from "sqlstring-sqlite";
+import mapSeries2 from "promise-map-series";
 
 // src/models/models.ts
 var models_exports = {};
@@ -3309,6 +3307,12 @@ function stopsToGeoJSONFeatureCollection(stops2) {
   return featureCollection(features);
 }
 
+// src/lib/import-gtfs-realtime.ts
+import pluralize from "pluralize";
+import GtfsRealtimeBindings from "gtfs-realtime-bindings";
+import sqlString2 from "sqlstring-sqlite";
+import mapSeries from "promise-map-series";
+
 // src/lib/log-utils.ts
 import { clearLine, cursorTo } from "node:readline";
 import { noop } from "lodash-es";
@@ -3480,60 +3484,10 @@ function formatOrderByClause(orderBy2) {
   return orderByClause;
 }
 
-// src/lib/import.ts
-var downloadFiles = async (task) => {
-  if (!task.url) {
-    throw new Error("No `url` specified in config");
-  }
-  task.log(`Downloading GTFS from ${task.url}`);
-  task.path = `${task.downloadDir}/gtfs.zip`;
-  const response = await fetch(task.url, {
-    method: "GET",
-    headers: task.headers || {},
-    signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
-  });
-  if (response.status !== 200) {
-    throw new Error(
-      `Unable to download GTFS from ${task.url}. Got status ${response.status}.`
-    );
-  }
-  const buffer = await response.arrayBuffer();
-  await writeFile(task.path, Buffer.from(buffer));
-  task.log("Download successful");
-};
-var downloadGtfsRealtimeData = async (urlAndHeaders, task) => {
-  task.log(`Downloading GTFS-Realtime from ${urlAndHeaders.url}`);
-  const response = await fetch(urlAndHeaders.url, {
-    method: "GET",
-    headers: {
-      ...urlAndHeaders.headers ?? {},
-      "Accept-Encoding": "gzip"
-    },
-    signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
-  });
-  if (response.status !== 200) {
-    task.logWarning(
-      `Unable to download GTFS-Realtime from ${urlAndHeaders.url}. Got status ${response.status}.`
-    );
-    return null;
-  }
-  const buffer = await response.arrayBuffer();
-  const message = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(
-    new Uint8Array(buffer)
-  );
-  return GtfsRealtimeBindings.transit_realtime.FeedMessage.toObject(message, {
-    enums: String,
-    longs: String,
-    bytes: String,
-    defaults: true,
-    arrays: true,
-    objects: true,
-    oneofs: true
-  });
-};
-function getDescendantProp(obj, defaultValue, source) {
-  if (source === void 0) return defaultValue;
-  const arr = source.split(".");
+// src/lib/import-gtfs-realtime.ts
+function getNestedProperty(obj, defaultValue, path3) {
+  if (path3 === void 0) return defaultValue;
+  const arr = path3.split(".");
   while (arr.length) {
     const nextKey = arr.shift();
     if (nextKey === void 0) {
@@ -3562,7 +3516,37 @@ function getDescendantProp(obj, defaultValue, source) {
   if (obj.__isLong__) return convertLongTimeToDate(obj);
   return obj;
 }
-var deleteExpiredRealtimeData = (config) => {
+async function fetchGtfsRealtimeData(urlConfig, task) {
+  task.log(`Downloading GTFS-Realtime from ${urlConfig.url}`);
+  const response = await fetch(urlConfig.url, {
+    method: "GET",
+    headers: {
+      ...urlConfig.headers ?? {},
+      "Accept-Encoding": "gzip"
+    },
+    signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
+  });
+  if (response.status !== 200) {
+    task.logWarning(
+      `Unable to download GTFS-Realtime from ${urlConfig.url}. Got status ${response.status}.`
+    );
+    return null;
+  }
+  const buffer = await response.arrayBuffer();
+  const message = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(
+    new Uint8Array(buffer)
+  );
+  return GtfsRealtimeBindings.transit_realtime.FeedMessage.toObject(message, {
+    enums: String,
+    longs: String,
+    bytes: String,
+    defaults: true,
+    arrays: true,
+    objects: true,
+    oneofs: true
+  });
+}
+function removeExpiredRealtimeData(config) {
   const log2 = log(config);
   const db = openDb(config);
   log2(`Removing expired GTFS-Realtime data`);
@@ -3582,8 +3566,8 @@ var deleteExpiredRealtimeData = (config) => {
     `DELETE FROM service_alert_targets WHERE expiration_timestamp <= strftime('%s','now')`
   ).run();
   log2(`Removed expired GTFS-Realtime data\r`, true);
-};
-var prepareRealtimeValue = (entity, column, task) => {
+}
+function prepareRealtimeFieldValue(entity, column, task) {
   if (column.name === "created_timestamp") {
     return task.currentTimestamp;
   }
@@ -3591,54 +3575,102 @@ var prepareRealtimeValue = (entity, column, task) => {
     return task.currentTimestamp + task.gtfsRealtimeExpirationSeconds;
   }
   return sqlString2.escape(
-    getDescendantProp(entity, column.default, column.source)
+    getNestedProperty(entity, column.default, column.source)
   );
-};
-var updateRealtimeData = async (task) => {
+}
+async function processRealtimeAlerts(db, gtfsRealtimeData, task) {
+  task.log(`Download successful`);
+  let totalLineCount = 0;
+  for (const entity of gtfsRealtimeData.entity) {
+    const fieldValues = serviceAlerts.schema.map(
+      (column) => prepareRealtimeFieldValue(entity, column, task)
+    );
+    try {
+      db.prepare(
+        `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
+      ).run();
+    } catch (error) {
+      task.logWarning("Import error: " + error.message);
+    }
+    const alertTargetArray = [];
+    for (const informedEntity of entity.alert.informedEntity) {
+      informedEntity.parent = entity;
+      const subValues = serviceAlertTargets.schema.map(
+        (column) => prepareRealtimeFieldValue(informedEntity, column, task)
+      );
+      alertTargetArray.push(`(${subValues.join(", ")})`);
+      totalLineCount++;
+    }
+    try {
+      db.prepare(
+        `REPLACE INTO ${serviceAlertTargets.filenameBase} (${serviceAlertTargets.schema.map((column) => column.name).join(", ")}) VALUES ${alertTargetArray.join(", ")}`
+      ).run();
+    } catch (error) {
+      task.logWarning("Import error: " + error.message);
+    }
+    task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
+  }
+}
+async function processRealtimeTripUpdates(db, gtfsRealtimeData, task) {
+  task.log(`Download successful`);
+  let totalLineCount = 0;
+  for (const entity of gtfsRealtimeData.entity) {
+    const fieldValues = tripUpdates.schema.map(
+      (column) => prepareRealtimeFieldValue(entity, column, task)
+    );
+    try {
+      db.prepare(
+        `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
+      ).run();
+    } catch (error) {
+      task.logWarning("Import error: " + error.message);
+    }
+    const stopTimeUpdateArray = [];
+    for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
+      stopTimeUpdate.parent = entity;
+      const subValues = stopTimeUpdates.schema.map(
+        (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
+      );
+      stopTimeUpdateArray.push(`(${subValues.join(", ")})`);
+      totalLineCount++;
+    }
+    try {
+      db.prepare(
+        `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(", ")}) VALUES ${stopTimeUpdateArray.join(", ")}`
+      ).run();
+    } catch (error) {
+      task.logWarning("Import error: " + error.message);
+    }
+    task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
+  }
+}
+async function processRealtimeVehiclePositions(db, gtfsRealtimeData, task) {
+  task.log(`Download successful`);
+  let totalLineCount = 0;
+  for (const entity of gtfsRealtimeData.entity) {
+    const fieldValues = vehiclePositions.schema.map(
+      (column) => prepareRealtimeFieldValue(entity, column, task)
+    );
+    try {
+      db.prepare(
+        `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
+      ).run();
+    } catch (error) {
+      task.logWarning("Import error: " + error.message);
+    }
+    task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
+  }
+}
+async function updateGtfsRealtimeData(task) {
   if (task.realtimeAlerts === void 0 && task.realtimeTripUpdates === void 0 && task.realtimeVehiclePositions === void 0) {
     return;
   }
-  const db = openDb({
-    sqlitePath: task.sqlitePath
-  });
+  const db = openDb({ sqlitePath: task.sqlitePath });
   if (task.realtimeAlerts?.url) {
     try {
-      const gtfsRealtimeData = await downloadGtfsRealtimeData(
-        task.realtimeAlerts,
-        task
-      );
-      if (gtfsRealtimeData?.entity) {
-        task.log(`Download successful`);
-        let totalLineCount = 0;
-        for (const entity of gtfsRealtimeData.entity) {
-          const fieldValues = serviceAlerts.schema.map(
-            (column) => prepareRealtimeValue(entity, column, task)
-          );
-          try {
-            db.prepare(
-              `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-            ).run();
-          } catch (error) {
-            task.logWarning("Import error: " + error.message);
-          }
-          const alertTargetArray = [];
-          for (const informedEntity of entity.alert.informedEntity) {
-            informedEntity.parent = entity;
-            const subValues = serviceAlertTargets.schema.map(
-              (column) => prepareRealtimeValue(informedEntity, column, task)
-            );
-            alertTargetArray.push(`(${subValues.join(", ")})`);
-            totalLineCount++;
-          }
-          try {
-            db.prepare(
-              `REPLACE INTO ${serviceAlertTargets.filenameBase} (${serviceAlertTargets.schema.map((column) => column.name).join(", ")}) VALUES ${alertTargetArray.join(", ")}`
-            ).run();
-          } catch (error) {
-            task.logWarning("Import error: " + error.message);
-          }
-          task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
-        }
+      const alertsData = await fetchGtfsRealtimeData(task.realtimeAlerts, task);
+      if (alertsData?.entity) {
+        await processRealtimeAlerts(db, alertsData, task);
       }
     } catch (error) {
       if (task.ignoreErrors) {
@@ -3650,42 +3682,12 @@ var updateRealtimeData = async (task) => {
   }
   if (task.realtimeTripUpdates?.url) {
     try {
-      const gtfsRealtimeData = await downloadGtfsRealtimeData(
+      const tripUpdatesData = await fetchGtfsRealtimeData(
         task.realtimeTripUpdates,
         task
       );
-      if (gtfsRealtimeData?.entity) {
-        task.log(`Download successful`);
-        let totalLineCount = 0;
-        for (const entity of gtfsRealtimeData.entity) {
-          const fieldValues = tripUpdates.schema.map(
-            (column) => prepareRealtimeValue(entity, column, task)
-          );
-          try {
-            db.prepare(
-              `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-            ).run();
-          } catch (error) {
-            task.logWarning("Import error: " + error.message);
-          }
-          const stopTimeUpdateArray = [];
-          for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
-            stopTimeUpdate.parent = entity;
-            const subValues = stopTimeUpdates.schema.map(
-              (column) => prepareRealtimeValue(stopTimeUpdate, column, task)
-            );
-            stopTimeUpdateArray.push(`(${subValues.join(", ")})`);
-            totalLineCount++;
-          }
-          try {
-            db.prepare(
-              `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(", ")}) VALUES ${stopTimeUpdateArray.join(", ")}`
-            ).run();
-          } catch (error) {
-            task.logWarning("Import error: " + error.message);
-          }
-          task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
-        }
+      if (tripUpdatesData?.entity) {
+        await processRealtimeTripUpdates(db, tripUpdatesData, task);
       }
     } catch (error) {
       if (task.ignoreErrors) {
@@ -3697,26 +3699,12 @@ var updateRealtimeData = async (task) => {
   }
   if (task.realtimeVehiclePositions?.url) {
     try {
-      const gtfsRealtimeData = await downloadGtfsRealtimeData(
+      const vehiclePositionsData = await fetchGtfsRealtimeData(
         task.realtimeVehiclePositions,
         task
       );
-      if (gtfsRealtimeData?.entity) {
-        task.log(`Download successful`);
-        let totalLineCount = 0;
-        for (const entity of gtfsRealtimeData.entity) {
-          const fieldValues = vehiclePositions.schema.map(
-            (column) => prepareRealtimeValue(entity, column, task)
-          );
-          try {
-            db.prepare(
-              `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-            ).run();
-          } catch (error) {
-            task.logWarning("Import error: " + error.message);
-          }
-          task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
-        }
+      if (vehiclePositionsData?.entity) {
+        await processRealtimeVehiclePositions(db, vehiclePositionsData, task);
       }
     } catch (error) {
       if (task.ignoreErrors) {
@@ -3727,12 +3715,120 @@ var updateRealtimeData = async (task) => {
     }
   }
   task.log(`GTFS-Realtime data import complete`);
+}
+async function updateGtfsRealtime(initialConfig) {
+  const config = setDefaultConfig(initialConfig);
+  validateConfigForImport(config);
+  const log2 = log(config);
+  const logError2 = logError(config);
+  const logWarning2 = logWarning(config);
+  try {
+    openDb(config);
+    const agencyCount = config.agencies.length;
+    log2(
+      `Starting GTFS-Realtime refresh for ${pluralize(
+        "agencies",
+        agencyCount,
+        true
+      )} using SQLite database at ${config.sqlitePath}`
+    );
+    removeExpiredRealtimeData(config);
+    await mapSeries(config.agencies, async (agency2) => {
+      try {
+        const task = {
+          realtimeAlerts: agency2.realtimeAlerts,
+          realtimeTripUpdates: agency2.realtimeTripUpdates,
+          realtimeVehiclePositions: agency2.realtimeVehiclePositions,
+          downloadTimeout: config.downloadTimeout,
+          gtfsRealtimeExpirationSeconds: config.gtfsRealtimeExpirationSeconds,
+          ignoreErrors: config.ignoreErrors,
+          sqlitePath: config.sqlitePath,
+          currentTimestamp: Math.floor(Date.now() / 1e3),
+          log: log2,
+          logWarning: logWarning2,
+          logError: logError2
+        };
+        await updateGtfsRealtimeData(task);
+      } catch (error) {
+        if (config.ignoreErrors) {
+          logError2(error.message);
+        } else {
+          throw error;
+        }
+      }
+    });
+    log2(
+      `Completed GTFS-Realtime refresh for ${pluralize(
+        "agencies",
+        agencyCount,
+        true
+      )}
+`
+    );
+  } catch (error) {
+    handleDatabaseError(error, config, logError2);
+  }
+}
+function handleDatabaseError(error, config, logError2) {
+  if (error?.code === "SQLITE_CANTOPEN") {
+    logError2(
+      `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
+    );
+  }
+  throw error;
+}
+
+// src/lib/import-gtfs.ts
+var dateCache = {};
+var calculateAndCacheDate = (value) => {
+  const cached = dateCache[value];
+  if (cached != null) {
+    return cached;
+  }
+  const seconds = calculateSecondsFromMidnight(value);
+  const date = padLeadingZeros(value);
+  const computed = [seconds, date];
+  dateCache[value] = computed;
+  return computed;
 };
 var getTextFiles = async (folderPath) => {
   const files = await readdir(folderPath);
   return files.filter((filename) => filename.slice(-3) === "txt");
 };
-var readFiles = async (task) => {
+var TIME_COLUMN_NAMES = [
+  "start_time",
+  "end_time",
+  "arrival_time",
+  "departure_time",
+  "prior_notice_last_time",
+  "prior_notice_start_time",
+  "start_pickup_drop_off_window"
+];
+var TIME_COLUMN_PAIRS = TIME_COLUMN_NAMES.map((name) => [
+  name,
+  name.endsWith("time") ? `${name}stamp` : `${name}_timestamp`
+]);
+var downloadGtfsFiles = async (task) => {
+  if (!task.url) {
+    throw new Error("No `url` specified in config");
+  }
+  task.log(`Downloading GTFS from ${task.url}`);
+  task.path = `${task.downloadDir}/gtfs.zip`;
+  const response = await fetch(task.url, {
+    method: "GET",
+    headers: task.headers || {},
+    signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
+  });
+  if (response.status !== 200) {
+    throw new Error(
+      `Unable to download GTFS from ${task.url}. Got status ${response.status}.`
+    );
+  }
+  const buffer = await response.arrayBuffer();
+  await writeFile(task.path, Buffer.from(buffer));
+  task.log("Download successful");
+};
+var extractGtfsFiles = async (task) => {
   if (!task.path) {
     throw new Error("No `path` specified in config");
   }
@@ -3784,7 +3880,7 @@ var readFiles = async (task) => {
     }
   }
 };
-var createTables = (db) => {
+var createGtfsTables = (db) => {
   for (const model of Object.values(models_exports)) {
     if (!model.schema) {
       return;
@@ -3813,6 +3909,13 @@ var createTables = (db) => {
     db.prepare(
       `CREATE TABLE ${model.filenameBase} (${columns.join(", ")});`
     ).run();
+  }
+};
+var createGtfsIndexes = (db) => {
+  for (const model of Object.values(models_exports)) {
+    if (!model.schema) {
+      return;
+    }
     for (const column of model.schema.filter((column2) => column2.index)) {
       db.prepare(
         `CREATE INDEX idx_${model.filenameBase}_${column.name} ON ${model.filenameBase} (${column.name});`
@@ -3820,7 +3923,7 @@ var createTables = (db) => {
     }
   }
 };
-var formatLine = (line, model, totalLineCount) => {
+var formatGtfsLine = (line, model, totalLineCount) => {
   const lineNumber = totalLineCount + 1;
   const formattedLine = {};
   for (const columnSchema of model.schema) {
@@ -3861,85 +3964,23 @@ var formatLine = (line, model, totalLineCount) => {
       );
     }
   }
-  const timeColumnNames = [
-    "start_time",
-    "end_time",
-    "arrival_time",
-    "departure_time",
-    "prior_notice_last_time",
-    "prior_notice_start_time",
-    "start_pickup_drop_off_window"
-  ];
-  for (const timeColumnName of timeColumnNames) {
-    if (formattedLine[timeColumnName]) {
-      const timestampColumnName = timeColumnName.endsWith("time") ? `${timeColumnName}stamp` : `${timeColumnName}_timestamp`;
-      formattedLine[timestampColumnName] = calculateSecondsFromMidnight(
-        formattedLine[timeColumnName]
-      );
-      formattedLine[timeColumnName] = padLeadingZeros(
-        formattedLine[timeColumnName]
-      );
+  for (const [timeColumnName, timestampColumnName] of TIME_COLUMN_PAIRS) {
+    const value = formattedLine[timeColumnName];
+    if (value) {
+      const [seconds, date] = calculateAndCacheDate(value);
+      formattedLine[timestampColumnName] = seconds;
+      formattedLine[timeColumnName] = date;
     }
   }
   return formattedLine;
 };
-var importLines = (task, lines, model, totalLineCount) => {
-  const db = openDb({
-    sqlitePath: task.sqlitePath
-  });
-  if (lines.length === 0) {
-    return;
-  }
-  const linesToImportCount = lines.length;
-  const columns = model.schema.filter((column) => column.name !== "id");
-  const placeholders = [];
-  const values = [];
-  while (lines.length > 0) {
-    const line = lines.pop();
-    if (line === void 0) {
-      continue;
-    }
-    placeholders.push(`(${columns.map(() => "?").join(", ")})`);
-    values.push(
-      ...columns.map((column) => {
-        if (task.prefix !== void 0 && column.prefix === true) {
-          return `${task.prefix}${line[column.name]}`;
-        }
-        return line[column.name];
-      })
-    );
-  }
-  try {
-    db.prepare(
-      `INSERT ${task.ignoreDuplicates ? "OR IGNORE" : ""} INTO ${model.filenameBase} (${columns.map((column) => column.name).join(", ")}) VALUES ${placeholders.join(",")}`
-    ).run(...values);
-  } catch (error) {
-    if (error.code === "SQLITE_CONSTRAINT_PRIMARYKEY") {
-      const primaryColumns = model.schema.filter((column) => column.primary);
-      task.logWarning(
-        `Duplicate values for primary key (${primaryColumns.map((column) => column.name).join(", ")}) found in ${model.filenameBase}.${model.filenameExtension}. Set the \`ignoreDuplicates\` option to true in config.json to ignore this error`
-      );
-    }
-    task.logWarning(
-      `Check ${model.filenameBase}.${model.filenameExtension} for invalid data between lines ${totalLineCount - linesToImportCount} and ${totalLineCount}.`
-    );
-    throw error;
-  }
-  task.log(
-    `Importing - ${model.filenameBase}.${model.filenameExtension} - ${totalLineCount} lines imported\r`,
-    true
-  );
-};
-var importFiles = (task) => mapSeries(
+var importGtfsFiles = (db, task) => mapSeries2(
   Object.values(models_exports),
   (model) => new Promise((resolve, reject) => {
-    const lines = [];
     let totalLineCount = 0;
-    const maxInsertVariables = 32e3;
+    const filename = `${model.filenameBase}.${model.filenameExtension}`;
     if (task.exclude && task.exclude.includes(model.filenameBase)) {
-      task.log(
-        `Skipping - ${model.filenameBase}.${model.filenameExtension}\r`
-      );
+      task.log(`Skipping - ${filename}\r`);
       resolve();
       return;
     }
@@ -3947,22 +3988,51 @@ var importFiles = (task) => mapSeries(
       resolve();
       return;
     }
-    const filepath = path.join(
-      task.downloadDir,
-      `${model.filenameBase}.${model.filenameExtension}`
-    );
+    const filepath = path.join(task.downloadDir, `${filename}`);
     if (!existsSync(filepath)) {
       if (!model.nonstandard) {
-        task.log(
-          `Importing - ${model.filenameBase}.${model.filenameExtension} - No file found\r`
-        );
+        task.log(`Importing - ${filename} - No file found\r`);
       }
       resolve();
       return;
     }
-    task.log(
-      `Importing - ${model.filenameBase}.${model.filenameExtension}\r`
-    );
+    task.log(`Importing - ${filename}\r`);
+    const columns = model.schema.filter((column) => column.name !== "id");
+    const placeholder = columns.map(({ name }) => `@${name}`).join(", ");
+    const prepareStatement = `INSERT ${task.ignoreDuplicates ? "OR IGNORE" : ""} INTO ${model.filenameBase} (${columns.map((column) => column.name).join(", ")}) VALUES (${placeholder})`;
+    const insert = db.prepare(prepareStatement);
+    const insertLines = db.transaction((lines) => {
+      for (const [rowNumber, line] of Object.entries(lines)) {
+        try {
+          if (task.prefix === void 0) {
+            insert.run(line);
+          } else {
+            const prefixedLine = Object.fromEntries(
+              Object.entries(
+                line
+              ).map(([columnName, value]) => [
+                columnName,
+                columns.find((col) => col.name === columnName)?.prefix ? `${task.prefix}${value}` : value
+              ])
+            );
+            insert.run(prefixedLine);
+          }
+        } catch (error) {
+          if (error.code === "SQLITE_CONSTRAINT_PRIMARYKEY") {
+            const primaryColumns = model.schema.filter(
+              (column) => column.primary
+            );
+            task.logWarning(
+              `Duplicate values for primary key (${primaryColumns.map((column) => column.name).join(", ")}) found in ${filename}. Set the \`ignoreDuplicates\` option to true in config.json to ignore this error`
+            );
+          }
+          task.logWarning(
+            `Check ${filename} for invalid data on row ${rowNumber}.`
+          );
+          throw error;
+        }
+      }
+    });
     if (model.filenameExtension === "txt") {
       const parser = parse({
         columns: true,
@@ -3971,23 +4041,21 @@ var importFiles = (task) => mapSeries(
         skip_empty_lines: true,
         ...task.csvOptions
       });
+      let lines = [];
       parser.on("readable", () => {
         let record;
         while (record = parser.read()) {
-          try {
-            totalLineCount += 1;
-            lines.push(formatLine(record, model, totalLineCount));
-            if (lines.length >= maxInsertVariables / model.schema.length) {
-              importLines(task, lines, model, totalLineCount);
-            }
-          } catch (error) {
-            reject(error);
-          }
+          totalLineCount += 1;
+          lines.push(formatGtfsLine(record, model, totalLineCount));
         }
       });
       parser.on("end", () => {
         try {
-          importLines(task, lines, model, totalLineCount);
+          insertLines(lines);
+          task.log(
+            `Importing - ${filename} - ${totalLineCount} lines imported\r`,
+            true
+          );
         } catch (error) {
           reject(error);
         }
@@ -3998,14 +4066,19 @@ var importFiles = (task) => mapSeries(
     } else if (model.filenameExtension === "geojson") {
       readFile2(filepath, "utf8").then((data) => {
         if (isValidJSON(data) === false) {
-          reject(
-            new Error(
-              `Invalid JSON in ${model.filenameBase}.${model.filenameExtension}`
-            )
-          );
+          reject(new Error(`Invalid JSON in ${filename}`));
         }
-        const line = formatLine({ geojson: data }, model, totalLineCount);
-        importLines(task, [line], model, totalLineCount);
+        totalLineCount += 1;
+        const line = formatGtfsLine(
+          { geojson: data },
+          model,
+          totalLineCount
+        );
+        insertLines([line]);
+        task.log(
+          `Importing - ${filename} - ${totalLineCount} lines imported\r`,
+          true
+        );
         resolve();
       }).catch(reject);
     } else {
@@ -4025,14 +4098,10 @@ async function importGtfs(initialConfig) {
     const db = openDb(config);
     const agencyCount = config.agencies.length;
     log2(
-      `Starting GTFS import for ${pluralize(
-        "file",
-        agencyCount,
-        true
-      )} using SQLite database at ${config.sqlitePath}`
+      `Starting GTFS import for ${pluralize2("file", agencyCount, true)} using SQLite database at ${config.sqlitePath}`
     );
-    createTables(db);
-    await mapSeries(config.agencies, async (agency2) => {
+    createGtfsTables(db);
+    await mapSeries2(config.agencies, async (agency2) => {
       try {
         const tempPath = temporaryDirectory();
         const task = {
@@ -4057,100 +4126,50 @@ async function importGtfs(initialConfig) {
           logError: logError2
         };
         if (task.url) {
-          await downloadFiles(task);
+          await downloadGtfsFiles(task);
         }
-        await readFiles(task);
-        await importFiles(task);
-        await updateRealtimeData(task);
+        await extractGtfsFiles(task);
+        await importGtfsFiles(db, task);
+        await updateGtfsRealtimeData(task);
         await rm2(tempPath, { recursive: true });
       } catch (error) {
-        if (config.ignoreErrors) {
-          logError2(error.message);
-        } else {
-          throw error;
-        }
+        handleImportError(error, config, logError2);
       }
     });
+    log2(`Creating DB indexes`);
+    createGtfsIndexes(db);
     log2(
-      `Completed GTFS import for ${pluralize("agency", agencyCount, true)}
+      `Completed GTFS import for ${pluralize2("agency", agencyCount, true)}
 `
     );
   } catch (error) {
-    if (error?.code === "SQLITE_CANTOPEN") {
-      logError2(
-        `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
-      );
-    }
+    handleDatabaseError2(error, config, logError2);
+  }
+}
+function handleImportError(error, config, logError2) {
+  if (config.ignoreErrors) {
+    logError2(error.message);
+  } else {
     throw error;
   }
 }
-async function updateGtfsRealtime(initialConfig) {
-  const config = setDefaultConfig(initialConfig);
-  validateConfigForImport(config);
-  const log2 = log(config);
-  const logError2 = logError(config);
-  const logWarning2 = logWarning(config);
-  try {
-    openDb(config);
-    const agencyCount = config.agencies.length;
-    log2(
-      `Starting GTFS-Realtime refresh for ${pluralize(
-        "agencies",
-        agencyCount,
-        true
-      )} using SQLite database at ${config.sqlitePath}`
+function handleDatabaseError2(error, config, logError2) {
+  if (error?.code === "SQLITE_CANTOPEN") {
+    logError2(
+      `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
     );
-    deleteExpiredRealtimeData(config);
-    await mapSeries(config.agencies, async (agency2) => {
-      try {
-        const task = {
-          realtimeAlerts: agency2.realtimeAlerts,
-          realtimeTripUpdates: agency2.realtimeTripUpdates,
-          realtimeVehiclePositions: agency2.realtimeVehiclePositions,
-          downloadTimeout: config.downloadTimeout,
-          gtfsRealtimeExpirationSeconds: config.gtfsRealtimeExpirationSeconds,
-          ignoreErrors: config.ignoreErrors,
-          sqlitePath: config.sqlitePath,
-          currentTimestamp: Math.floor(Date.now() / 1e3),
-          log: log2,
-          logWarning: logWarning2,
-          logError: logError2
-        };
-        await updateRealtimeData(task);
-      } catch (error) {
-        if (config.ignoreErrors) {
-          logError2(error.message);
-        } else {
-          throw error;
-        }
-      }
-    });
-    log2(
-      `Completed GTFS-Realtime refresh for ${pluralize(
-        "agencies",
-        agencyCount,
-        true
-      )}
-`
-    );
-  } catch (error) {
-    if (error?.code === "SQLITE_CANTOPEN") {
-      logError2(
-        `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
-      );
-    }
-    throw error;
   }
+  throw error;
 }
 
 // src/lib/export.ts
 import path2 from "node:path";
 import { writeFile as writeFile2 } from "node:fs/promises";
 import { without, compact as compact2 } from "lodash-es";
-import pluralize2 from "pluralize";
+import pluralize3 from "pluralize";
 import { stringify } from "csv-stringify";
 import sqlString3 from "sqlstring-sqlite";
-import mapSeries2 from "promise-map-series";
+import mapSeries3 from "promise-map-series";
 import untildify4 from "untildify";
 var getAgencies = (db, config) => {
   try {
@@ -4183,7 +4202,7 @@ var exportGtfs = async (initialConfig) => {
     );
   }
   log2(
-    `Starting GTFS export for ${pluralize2(
+    `Starting GTFS export for ${pluralize3(
       "agency",
       agencyCount,
       true
@@ -4196,7 +4215,7 @@ var exportGtfs = async (initialConfig) => {
   const modelsToExport = Object.values(models_exports).filter(
     (model) => model.extension !== "gtfs-realtime"
   );
-  const exportedFiles = await mapSeries2(
+  const exportedFiles = await mapSeries3(
     modelsToExport,
     async (model) => {
       const filePath = path2.join(
@@ -4258,7 +4277,7 @@ var exportGtfs = async (initialConfig) => {
     return;
   }
   log2(`Completed GTFS export to ${exportPath}`);
-  log2(`Completed GTFS export for ${pluralize2("agency", agencyCount, true)}
+  log2(`Completed GTFS export for ${pluralize3("agency", agencyCount, true)}
 `);
 };