gtfs 3.8.0 → 4.0.1

package/lib/import.js

@@ -13,7 +13,7 @@ import gtfsrt from 'gtfs-realtime-bindings';
 import sqlString from 'sqlstring-sqlite';
 
 import models from '../models/models.js';
-import { openDb, setupDb } from './db.js';
+import { openDb } from './db.js';
 import { unzip } from './file-utils.js';
 import {
   log as _log,
@@ -82,72 +82,33 @@ function getDescendantProp(obj, desc, defaultvalue) {
   return obj;
 }
 
-const markRealtimeDataStale = async (db, log) => {
-  const vehiclePositionModel = models.find(
-    (x) => x.filenameBase === 'vehicle_positions'
-  );
-  const tripUpdatesModel = models.find(
-    (x) => x.filenameBase === 'trip_updates'
-  );
-  const stopTimesUpdatesModel = models.find(
-    (x) => x.filenameBase === 'stop_times_updates'
-  );
-  const serviceAlertsModel = models.find(
-    (x) => x.filenameBase === 'service_alerts'
-  );
-  const serviceAlertTargetsModel = models.find(
-    (x) => x.filenameBase === 'service_alert_targets'
-  );
+const markRealtimeDataStale = (config, log) => {
+  const db = openDb(config);
 
-  // Mark all data as stale
   log(`Marking GTFS-Realtime data as stale..`);
-  await db.run(`UPDATE ${vehiclePositionModel.filenameBase} SET isUpdated=0`);
-  await db.run(`UPDATE ${tripUpdatesModel.filenameBase} SET isUpdated=0`);
-  await db.run(`UPDATE ${stopTimesUpdatesModel.filenameBase} SET isUpdated=0`);
-  await db.run(`UPDATE ${serviceAlertsModel.filenameBase} SET isUpdated=0`);
-  await db.run(
-    `UPDATE ${serviceAlertTargetsModel.filenameBase} SET isUpdated=0`
-  );
+  db.prepare(`UPDATE vehicle_positions SET isUpdated=0`).run();
+  db.prepare(`UPDATE trip_updates SET isUpdated=0`).run();
+  db.prepare(`UPDATE stop_times_updates SET isUpdated=0`).run();
+  db.prepare(`UPDATE service_alerts SET isUpdated=0`).run();
+  db.prepare(`UPDATE service_alert_targets SET isUpdated=0`).run();
   log(`Marked GTFS-Realtime data as stale\r`, true);
 };
 
-const cleanStaleRealtimeData = async (db, log) => {
-  const vehiclePositionModel = models.find(
-    (x) => x.filenameBase === 'vehicle_positions'
-  );
-  const tripUpdatesModel = models.find(
-    (x) => x.filenameBase === 'trip_updates'
-  );
-  const stopTimesUpdatesModel = models.find(
-    (x) => x.filenameBase === 'stop_times_updates'
-  );
-  const serviceAlertsModel = models.find(
-    (x) => x.filenameBase === 'service_alerts'
-  );
-  const serviceAlertTargetsModel = models.find(
-    (x) => x.filenameBase === 'service_alert_targets'
-  );
+const cleanStaleRealtimeData = (config, log) => {
+  const db = openDb(config);
 
   log(`Cleaning stale GTFS-RT data..`);
-  await db.run(
-    `DELETE FROM ${vehiclePositionModel.filenameBase} WHERE isUpdated=0`
-  );
-  await db.run(
-    `DELETE FROM ${tripUpdatesModel.filenameBase} WHERE isUpdated=0`
-  );
-  await db.run(
-    `DELETE FROM ${stopTimesUpdatesModel.filenameBase} WHERE isUpdated=0`
-  );
-  await db.run(
-    `DELETE FROM ${serviceAlertsModel.filenameBase} WHERE isUpdated=0`
-  );
-  await db.run(
-    `DELETE FROM ${serviceAlertTargetsModel.filenameBase} WHERE isUpdated=0`
-  );
+  db.prepare(`DELETE FROM vehicle_positions WHERE isUpdated=0`).run();
+  db.prepare(`DELETE FROM trip_updates WHERE isUpdated=0`).run();
+  db.prepare(`DELETE FROM stop_times_updates WHERE isUpdated=0`).run();
+  db.prepare(`DELETE FROM service_alerts WHERE isUpdated=0`).run();
+  db.prepare(`DELETE FROM service_alert_targets WHERE isUpdated=0`).run();
   log(`Cleaned stale GTFS-Realtime data\r`, true);
 };
 
 const updateRealtimeData = async (task) => {
+  const db = openDb(task);
+
   const model = {
     vehicle_positions: models.find(
       (x) => x.filenameBase === 'vehicle_positions'
@@ -220,16 +181,15 @@ const updateRealtimeData = async (task) => {
         )
       );
 
-      // eslint-disable-next-line no-await-in-loop
-      await task.db
-        .run(
+      try {
+        db.prepare(
           `REPLACE INTO ${model[gtfsRealtimeType].filenameBase} (${
             fields[gtfsRealtimeType]
           }) VALUES (${fieldValues.join(', ')})`
-        )
-        .catch((error) => {
-          task.warn('Import error: ' + error.message);
-        });
+        ).run();
+      } catch (error) {
+        task.warn('Import error: ' + error.message);
+      }
 
       // Special processing for tripUpdates
       if (entity.tripUpdate) {
@@ -245,16 +205,15 @@ const updateRealtimeData = async (task) => {
           totalLineCount++;
         }
 
-        // eslint-disable-next-line no-await-in-loop
-        await task.db
-          .run(
+        try {
+          db.prepare(
             `REPLACE INTO ${model.stop_times_updates.filenameBase} (${
               fields.stop_times_updates
             }) VALUES ${stopUpdateArray.join(', ')}`
-          )
-          .catch((error) => {
-            task.warn('Import error: ' + error.message);
-          });
+          ).run();
+        } catch (error) {
+          task.warn('Import error: ' + error.message);
+        }
       }
 
       // Special processing for serviceAlerts
@@ -271,16 +230,15 @@ const updateRealtimeData = async (task) => {
           totalLineCount++;
         }
 
-        // eslint-disable-next-line no-await-in-loop
-        await task.db
-          .run(
+        try {
+          db.prepare(
             `REPLACE INTO ${model.service_alert_targets.filenameBase} (${
               fields.service_alert_targets
             }) VALUES ${alertTargetArray.join(', ')}`
-          )
-          .catch((error) => {
-            task.warn('Import error: ' + error.message);
-          });
+          ).run();
+        } catch (error) {
+          task.warn('Import error: ' + error.message);
+        }
       }
 
       task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
@@ -355,159 +313,147 @@ const readFiles = async (task) => {
   }
 };
 
-const deleteTables = (db) =>
-  Promise.all(
-    models.map((model) => db.run(`DROP TABLE IF EXISTS ${model.filenameBase};`))
-  );
+const createTables = (db) => {
+  for (const model of models) {
+    if (!model.schema) {
+      return;
+    }
 
-const createTables = (db) =>
-  Promise.all(
-    models.map(async (model) => {
-      if (!model.schema) {
-        return;
+    const columns = model.schema.map((column) => {
+      let check = '';
+      if (column.min !== undefined && column.max) {
+        check = `CHECK( ${column.name} >= ${column.min} AND ${column.name} <= ${column.max} )`;
+      } else if (column.min) {
+        check = `CHECK( ${column.name} >= ${column.min} )`;
+      } else if (column.max) {
+        check = `CHECK( ${column.name} <= ${column.max} )`;
       }
 
-      const columns = model.schema.map((column) => {
-        let check = '';
-        if (column.min !== undefined && column.max) {
-          check = `CHECK( ${column.name} >= ${column.min} AND ${column.name} <= ${column.max} )`;
-        } else if (column.min) {
-          check = `CHECK( ${column.name} >= ${column.min} )`;
-        } else if (column.max) {
-          check = `CHECK( ${column.name} <= ${column.max} )`;
-        }
+      const primary = column.primary ? 'PRIMARY KEY' : '';
+      const required = column.required ? 'NOT NULL' : '';
+      const columnDefault = column.default ? 'DEFAULT ' + column.default : '';
+      const columnCollation = column.nocase ? 'COLLATE NOCASE' : '';
+      return `${column.name} ${column.type} ${check} ${primary} ${required} ${columnDefault} ${columnCollation}`;
+    });
 
-        const primary = column.primary ? 'PRIMARY KEY' : '';
-        const required = column.required ? 'NOT NULL' : '';
-        const columnDefault = column.default ? 'DEFAULT ' + column.default : '';
-        const columnCollation = column.nocase ? 'COLLATE NOCASE' : '';
-        return `${column.name} ${column.type} ${check} ${primary} ${required} ${columnDefault} ${columnCollation}`;
-      });
+    db.prepare(`DROP TABLE IF EXISTS ${model.filenameBase};`).run();
 
-      await db.run(
-        `CREATE TABLE ${model.filenameBase} (${columns.join(', ')});`
-      );
+    db.prepare(
+      `CREATE TABLE ${model.filenameBase} (${columns.join(', ')});`
+    ).run();
 
-      await Promise.all(
-        model.schema.map(async (column) => {
-          if (column.index) {
-            const unique = column.index === 'unique' ? 'UNIQUE' : '';
-            await db.run(
-              `CREATE ${unique} INDEX idx_${model.filenameBase}_${column.name} ON ${model.filenameBase} (${column.name});`
-            );
-          }
-        })
-      );
-    })
-  );
+    for (const column of model.schema.filter((column) => column.index)) {
+      const unique = column.index === 'unique' ? 'UNIQUE' : '';
+      db.prepare(
+        `CREATE ${unique} INDEX idx_${model.filenameBase}_${column.name} ON ${model.filenameBase} (${column.name});`
+      ).run();
+    }
+  }
+};
 
 const formatLine = (line, model, totalLineCount) => {
   const lineNumber = totalLineCount + 1;
-  for (const fieldName of Object.keys(line)) {
-    const columnSchema = model.schema.find(
-      (schema) => schema.name === fieldName
-    );
 
-    // Remove columns not part of model
-    if (!columnSchema) {
-      delete line[fieldName];
-      continue;
-    }
+  const formattedLine = {};
 
-    // Remove null values
-    if (line[fieldName] === null || line[fieldName] === '') {
-      delete line[fieldName];
-    }
+  for (const columnSchema of model.schema) {
+    const lineValue = line[columnSchema.name];
 
-    // Convert fields that should be integer
     if (columnSchema.type === 'integer') {
-      const value = Number.parseInt(line[fieldName], 10);
-
-      if (Number.isNaN(value)) {
-        delete line[fieldName];
-      } else {
-        line[fieldName] = value;
-      }
+      // Convert fields that should be integer
+      formattedLine[columnSchema.name] = Number.parseInt(lineValue, 10);
+    } else if (columnSchema.type === 'real') {
+      // Convert fields that should be float
+      formattedLine[columnSchema.name] = Number.parseFloat(lineValue);
+    } else {
+      formattedLine[columnSchema.name] = lineValue;
     }
 
-    // Convert fields that should be float
-    if (columnSchema.type === 'real') {
-      const value = Number.parseFloat(line[fieldName]);
-
-      if (Number.isNaN(value)) {
-        delete line[fieldName];
-      } else {
-        line[fieldName] = value;
-      }
+    if (
+      formattedLine[columnSchema.name] === '' ||
+      formattedLine[columnSchema.name] === undefined ||
+      formattedLine[columnSchema.name] === null ||
+      Number.isNaN(formattedLine[columnSchema.name])
+    ) {
+      // Add null values
+      formattedLine[columnSchema.name] = null;
     }
 
     // Validate required
     if (
       columnSchema.required === true &&
-      (line[fieldName] === undefined || line[fieldName] === '')
+      formattedLine[columnSchema.name] === null
     ) {
       throw new Error(
-        `Missing required value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}.`
+        `Missing required value in ${model.filenameBase}.txt for ${columnSchema.name} on line ${lineNumber}.`
       );
     }
 
     // Validate minimum
-    if (columnSchema.min !== undefined && line[fieldName] < columnSchema.min) {
+    if (
+      columnSchema.min !== undefined &&
+      formattedLine[columnSchema.name] < columnSchema.min
+    ) {
       throw new Error(
-        `Invalid value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}: below minimum value of ${columnSchema.min}.`
+        `Invalid value in ${model.filenameBase}.txt for ${columnSchema.name} on line ${lineNumber}: below minimum value of ${columnSchema.min}.`
       );
     }
 
     // Validate maximum
-    if (columnSchema.max !== undefined && line[fieldName] > columnSchema.max) {
+    if (
+      columnSchema.max !== undefined &&
+      formattedLine[columnSchema.name] > columnSchema.max
+    ) {
       throw new Error(
-        `Invalid value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}: above maximum value of ${columnSchema.max}.`
+        `Invalid value in ${model.filenameBase}.txt for ${columnSchema.name} on line ${lineNumber}: above maximum value of ${columnSchema.max}.`
       );
     }
   }
 
   // Convert to midnight timestamp
-  const timestampFormat = [
+  const timestampColumnNames = [
     'start_time',
     'end_time',
     'arrival_time',
     'departure_time',
   ];
 
-  for (const fieldName of timestampFormat) {
-    if (line[fieldName]) {
-      line[`${fieldName}stamp`] = calculateSecondsFromMidnight(line[fieldName]);
+  for (const timestampColumnName of timestampColumnNames) {
+    if (formattedLine[timestampColumnName]) {
+      formattedLine[`${timestampColumnName}stamp`] =
+        calculateSecondsFromMidnight(formattedLine[timestampColumnName]);
     }
   }
 
-  return line;
+  return formattedLine;
 };
 
-const importLines = async (task, lines, model, totalLineCount) => {
+const importLines = (task, lines, model, totalLineCount) => {
+  const db = openDb(task);
+
   if (lines.length === 0) {
     return;
   }
 
   const linesToImportCount = lines.length;
-  const fieldNames = model.schema.map((column) => column.name);
+  const fieldNames = model.schema
+    .filter((column) => column.name !== 'id')
+    .map((column) => column.name);
   const placeholders = [];
   const values = [];
 
   while (lines.length > 0) {
     const line = lines.pop();
     placeholders.push(`(${fieldNames.map(() => '?').join(', ')})`);
-    for (const fieldName of fieldNames) {
-      values.push(line[fieldName]);
-    }
+    values.push(...fieldNames.map((fieldName) => line[fieldName]));
   }
 
   try {
-    await task.db.run(
+    db.prepare(
       `INSERT ${task.ignoreDuplicates ? 'OR IGNORE' : ''} INTO ${
         model.filenameBase
-      }(${fieldNames.join(', ')}) VALUES${placeholders.join(',')}`,
-      values
-    );
+      } (${fieldNames.join(', ')}) VALUES ${placeholders.join(',')}`
+    ).run(...values);
   } catch (error) {
     task.warn(
       `Check ${model.filenameBase}.txt for invalid data between lines ${
@@ -536,7 +482,7 @@ const importFiles = (task) =>
           return;
         }
 
-        // If the model is a database/gtfs-realtime model then just silently exit as we dont really care here
+        // If the model is a database/gtfs-realtime model then silently exit
         if (model.extension === 'gtfs-realtime') {
           resolve();
           return;
@@ -569,18 +515,16 @@ const importFiles = (task) =>
           ...task.csvOptions,
         });
 
-        parser.on('readable', async () => {
+        parser.on('readable', () => {
           let record;
 
           while ((record = parser.read())) {
             try {
               totalLineCount += 1;
               lines.push(formatLine(record, model, totalLineCount));
-
               // If we have a bunch of lines ready to insert, then do it
               if (lines.length >= maxInsertVariables / model.schema.length) {
-                /* eslint-disable-next-line no-await-in-loop */
-                await importLines(task, lines, model, totalLineCount);
+                importLines(task, lines, model, totalLineCount);
               }
             } catch (error) {
               reject(error);
@@ -588,9 +532,9 @@ const importFiles = (task) =>
           }
         });
 
-        parser.on('end', async () => {
+        parser.on('end', () => {
           // Insert all remaining lines
-          await importLines(task, lines, model, totalLineCount).catch(reject);
+          importLines(task, lines, model, totalLineCount);
           resolve();
         });
 
@@ -606,70 +550,65 @@ export async function importGtfs(initialConfig) {
   const log = _log(config);
   const logError = _logError(config);
   const logWarning = _logWarning(config);
-  const db = await openDb(config).catch((error) => {
-    if (error instanceof Error && error.code === 'SQLITE_CANTOPEN') {
-      logError(
-        `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
-      );
-    }
+  try {
+    const db = openDb(config);
+
+    const agencyCount = config.agencies.length;
+    log(
+      `Starting GTFS import for ${pluralize(
+        'file',
+        agencyCount,
+        true
+      )} using SQLite database at ${config.sqlitePath}`
+    );
 
-    throw error;
-  });
+    createTables(db);
+
+    await mapSeries(config.agencies, async (agency) => {
+      const { path, cleanup } = await dir({ unsafeCleanup: true });
+
+      const task = {
+        exclude: agency.exclude,
+        agency_url: agency.url,
+        headers: agency.headers || false,
+        realtime_headers: agency.realtimeHeaders || false,
+        realtime_urls: agency.realtimeUrls || false,
+        downloadDir: path,
+        path: agency.path,
+        csvOptions: config.csvOptions || {},
+        ignoreDuplicates: config.ignoreDuplicates,
+        sqlitePath: config.sqlitePath,
+        log,
+        warn: logWarning,
+        error: logError,
+      };
+
+      if (task.agency_url) {
+        await downloadFiles(task);
+      }
 
-  const agencyCount = config.agencies.length;
-  log(
-    `Starting GTFS import for ${pluralize(
-      'file',
-      agencyCount,
-      true
-    )} using SQLite database at ${config.sqlitePath}`
-  );
+      await readFiles(task);
+      await importFiles(task);
 
-  await deleteTables(db);
-  await createTables(db);
-
-  await setupDb(db);
-
-  await mapSeries(config.agencies, async (agency) => {
-    const { path, cleanup } = await dir({ unsafeCleanup: true });
-
-    const task = {
-      exclude: agency.exclude,
-      agency_url: agency.url,
-      headers: agency.headers || false,
-      realtime_headers: agency.realtimeHeaders || false,
-      realtime_urls: agency.realtimeUrls || false,
-      downloadDir: path,
-      path: agency.path,
-      csvOptions: config.csvOptions || {},
-      ignoreDuplicates: config.ignoreDuplicates,
-      db,
-      log(message, overwrite) {
-        log(message, overwrite);
-      },
-      warn(message) {
-        logWarning(message);
-      },
-      error(message) {
-        logError(message);
-      },
-    };
-
-    if (task.agency_url) {
-      await downloadFiles(task);
-    }
+      if (task.realtime_urls) {
+        await updateRealtimeData(task);
+      }
 
-    await readFiles(task);
-    await importFiles(task);
+      cleanup();
+    });
 
-    if (task.realtime_urls) {
-      await updateRealtimeData(task);
+    log(
+      `Completed GTFS import for ${pluralize('agency', agencyCount, true)}\n`
+    );
+  } catch (error) {
+    if (error instanceof Error && error.code === 'SQLITE_CANTOPEN') {
+      logError(
+        `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
+      );
     }
 
-    cleanup();
-  });
-
-  log(`Completed GTFS import for ${pluralize('agency', agencyCount, true)}\n`);
+    throw error;
+  }
 }
 
 export async function updateGtfsRealtime(initialConfig) {
@@ -678,7 +617,49 @@ export async function updateGtfsRealtime(initialConfig) {
   const log = _log(config);
   const logError = _logError(config);
   const logWarning = _logWarning(config);
-  const db = await openDb(config).catch((error) => {
+
+  try {
+    openDb(config);
+
+    const agencyCount = config.agencies.length;
+    log(
+      `Starting GTFS-Realtime refresh for ${pluralize(
+        'agencies',
+        agencyCount,
+        true
+      )} using SQLite database at ${config.sqlitePath}`
+    );
+
+    markRealtimeDataStale(config, log);
+
+    await Promise.all(
+      config.agencies.map(async (agency) => {
+        if (!agency.realtimeUrls) {
+          return;
+        }
+
+        const task = {
+          realtime_headers: agency.realtimeHeaders || false,
+          realtime_urls: agency.realtimeUrls || false,
+          sqlitePath: config.sqlitePath,
+          log,
+          warn: logWarning,
+          error: logError,
+        };
+
+        await updateRealtimeData(task);
+      })
+    );
+
+    cleanStaleRealtimeData(config, log);
+    log(
+      `Completed GTFS-Realtime refresh for ${pluralize(
+        'agencies',
+        agencyCount,
+        true
+      )}\n`
+    );
+  } catch (error) {
     if (error instanceof Error && error.code === 'SQLITE_CANTOPEN') {
       logError(
         `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
@@ -686,46 +667,5 @@ export async function updateGtfsRealtime(initialConfig) {
     }
 
     throw error;
-  });
-
-  const agencyCount = config.agencies.length;
-  log(
-    `Starting GTFS-Realtime refresh for ${pluralize(
-      'agencies',
-      agencyCount,
-      true
-    )} using SQLite database at ${config.sqlitePath}`
-  );
-
-  await markRealtimeDataStale(db, log);
-
-  await mapSeries(config.agencies, async (agency) => {
-    const task = {
-      realtime_headers: agency.realtimeHeaders || false,
-      realtime_urls: agency.realtimeUrls || false,
-      db,
-      log(message, overwrite) {
-        log(message, overwrite);
-      },
-      warn(message) {
-        logWarning(message);
-      },
-      error(message) {
-        logError(message);
-      },
-    };
-
-    if (task.realtime_urls) {
-      await updateRealtimeData(task);
-    }
-  });
-
-  await cleanStaleRealtimeData(db, log);
-  log(
-    `Completed GTFS-Realtime refresh for ${pluralize(
-      'agencies',
-      agencyCount,
-      true
-    )}\n`
-  );
+  }
 }