gtfs 3.8.0 → 4.0.0

package/lib/import.js

@@ -13,7 +13,7 @@ import gtfsrt from 'gtfs-realtime-bindings';
 import sqlString from 'sqlstring-sqlite';
 
 import models from '../models/models.js';
-import { openDb, setupDb } from './db.js';
+import { openDb } from './db.js';
 import { unzip } from './file-utils.js';
 import {
   log as _log,
@@ -82,7 +82,7 @@ function getDescendantProp(obj, desc, defaultvalue) {
   return obj;
 }
 
-const markRealtimeDataStale = async (db, log) => {
+const markRealtimeDataStale = (db, log) => {
   const vehiclePositionModel = models.find(
     (x) => x.filenameBase === 'vehicle_positions'
   );
@@ -101,17 +101,21 @@ const markRealtimeDataStale = async (db, log) => {
 
   // Mark all data as stale
   log(`Marking GTFS-Realtime data as stale..`);
-  await db.run(`UPDATE ${vehiclePositionModel.filenameBase} SET isUpdated=0`);
-  await db.run(`UPDATE ${tripUpdatesModel.filenameBase} SET isUpdated=0`);
-  await db.run(`UPDATE ${stopTimesUpdatesModel.filenameBase} SET isUpdated=0`);
-  await db.run(`UPDATE ${serviceAlertsModel.filenameBase} SET isUpdated=0`);
-  await db.run(
+  db.prepare(
+    `UPDATE ${vehiclePositionModel.filenameBase} SET isUpdated=0`
+  ).run();
+  db.prepare(`UPDATE ${tripUpdatesModel.filenameBase} SET isUpdated=0`).run();
+  db.prepare(
+    `UPDATE ${stopTimesUpdatesModel.filenameBase} SET isUpdated=0`
+  ).run();
+  db.prepare(`UPDATE ${serviceAlertsModel.filenameBase} SET isUpdated=0`).run();
+  db.prepare(
     `UPDATE ${serviceAlertTargetsModel.filenameBase} SET isUpdated=0`
-  );
+  ).run();
   log(`Marked GTFS-Realtime data as stale\r`, true);
 };
 
-const cleanStaleRealtimeData = async (db, log) => {
+const cleanStaleRealtimeData = (db, log) => {
   const vehiclePositionModel = models.find(
     (x) => x.filenameBase === 'vehicle_positions'
   );
@@ -129,21 +133,21 @@ const cleanStaleRealtimeData = async (db, log) => {
   );
 
   log(`Cleaning stale GTFS-RT data..`);
-  await db.run(
+  db.prepare(
     `DELETE FROM ${vehiclePositionModel.filenameBase} WHERE isUpdated=0`
-  );
-  await db.run(
+  ).run();
+  db.prepare(
     `DELETE FROM ${tripUpdatesModel.filenameBase} WHERE isUpdated=0`
-  );
-  await db.run(
+  ).run();
+  db.prepare(
     `DELETE FROM ${stopTimesUpdatesModel.filenameBase} WHERE isUpdated=0`
-  );
-  await db.run(
+  ).run();
+  db.prepare(
     `DELETE FROM ${serviceAlertsModel.filenameBase} WHERE isUpdated=0`
-  );
-  await db.run(
+  ).run();
+  db.prepare(
     `DELETE FROM ${serviceAlertTargetsModel.filenameBase} WHERE isUpdated=0`
-  );
+  ).run();
   log(`Cleaned stale GTFS-Realtime data\r`, true);
 };
 
@@ -220,16 +224,17 @@ const updateRealtimeData = async (task) => {
         )
       );
 
-      // eslint-disable-next-line no-await-in-loop
-      await task.db
-        .run(
-          `REPLACE INTO ${model[gtfsRealtimeType].filenameBase} (${
-            fields[gtfsRealtimeType]
-          }) VALUES (${fieldValues.join(', ')})`
-        )
-        .catch((error) => {
-          task.warn('Import error: ' + error.message);
-        });
+      try {
+        task.db
+          .prepare(
+            `REPLACE INTO ${model[gtfsRealtimeType].filenameBase} (${
+              fields[gtfsRealtimeType]
+            }) VALUES (${fieldValues.join(', ')})`
+          )
+          .run();
+      } catch (error) {
+        task.warn('Import error: ' + error.message);
+      }
 
       // Special processing for tripUpdates
       if (entity.tripUpdate) {
@@ -245,16 +250,17 @@ const updateRealtimeData = async (task) => {
           totalLineCount++;
         }
 
-        // eslint-disable-next-line no-await-in-loop
-        await task.db
-          .run(
-            `REPLACE INTO ${model.stop_times_updates.filenameBase} (${
-              fields.stop_times_updates
-            }) VALUES ${stopUpdateArray.join(', ')}`
-          )
-          .catch((error) => {
-            task.warn('Import error: ' + error.message);
-          });
+        try {
+          task.db
+            .prepare(
+              `REPLACE INTO ${model.stop_times_updates.filenameBase} (${
+                fields.stop_times_updates
+              }) VALUES ${stopUpdateArray.join(', ')}`
+            )
+            .run();
+        } catch (error) {
+          task.warn('Import error: ' + error.message);
+        }
       }
 
       // Special processing for serviceAlerts
@@ -271,16 +277,17 @@ const updateRealtimeData = async (task) => {
           totalLineCount++;
         }
 
-        // eslint-disable-next-line no-await-in-loop
-        await task.db
-          .run(
-            `REPLACE INTO ${model.service_alert_targets.filenameBase} (${
-              fields.service_alert_targets
-            }) VALUES ${alertTargetArray.join(', ')}`
-          )
-          .catch((error) => {
-            task.warn('Import error: ' + error.message);
-          });
+        try {
+          task.db
+            .prepare(
+              `REPLACE INTO ${model.service_alert_targets.filenameBase} (${
+                fields.service_alert_targets
+              }) VALUES ${alertTargetArray.join(', ')}`
+            )
+            .run();
+        } catch (error) {
+          task.warn('Import error: ' + error.message);
+        }
       }
 
       task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
@@ -355,159 +362,146 @@ const readFiles = async (task) => {
   }
 };
 
-const deleteTables = (db) =>
-  Promise.all(
-    models.map((model) => db.run(`DROP TABLE IF EXISTS ${model.filenameBase};`))
-  );
+const createTables = (db) => {
+  for (const model of models) {
+    if (!model.schema) {
+      return;
+    }
 
-const createTables = (db) =>
-  Promise.all(
-    models.map(async (model) => {
-      if (!model.schema) {
-        return;
+    const columns = model.schema.map((column) => {
+      let check = '';
+      if (column.min !== undefined && column.max) {
+        check = `CHECK( ${column.name} >= ${column.min} AND ${column.name} <= ${column.max} )`;
+      } else if (column.min) {
+        check = `CHECK( ${column.name} >= ${column.min} )`;
+      } else if (column.max) {
+        check = `CHECK( ${column.name} <= ${column.max} )`;
       }
 
-      const columns = model.schema.map((column) => {
-        let check = '';
-        if (column.min !== undefined && column.max) {
-          check = `CHECK( ${column.name} >= ${column.min} AND ${column.name} <= ${column.max} )`;
-        } else if (column.min) {
-          check = `CHECK( ${column.name} >= ${column.min} )`;
-        } else if (column.max) {
-          check = `CHECK( ${column.name} <= ${column.max} )`;
-        }
+      const primary = column.primary ? 'PRIMARY KEY' : '';
+      const required = column.required ? 'NOT NULL' : '';
+      const columnDefault = column.default ? 'DEFAULT ' + column.default : '';
+      const columnCollation = column.nocase ? 'COLLATE NOCASE' : '';
+      return `${column.name} ${column.type} ${check} ${primary} ${required} ${columnDefault} ${columnCollation}`;
+    });
 
-        const primary = column.primary ? 'PRIMARY KEY' : '';
-        const required = column.required ? 'NOT NULL' : '';
-        const columnDefault = column.default ? 'DEFAULT ' + column.default : '';
-        const columnCollation = column.nocase ? 'COLLATE NOCASE' : '';
-        return `${column.name} ${column.type} ${check} ${primary} ${required} ${columnDefault} ${columnCollation}`;
-      });
+    db.prepare(`DROP TABLE IF EXISTS ${model.filenameBase};`).run();
 
-      await db.run(
-        `CREATE TABLE ${model.filenameBase} (${columns.join(', ')});`
-      );
+    db.prepare(
+      `CREATE TABLE ${model.filenameBase} (${columns.join(', ')});`
+    ).run();
 
-      await Promise.all(
-        model.schema.map(async (column) => {
-          if (column.index) {
-            const unique = column.index === 'unique' ? 'UNIQUE' : '';
-            await db.run(
-              `CREATE ${unique} INDEX idx_${model.filenameBase}_${column.name} ON ${model.filenameBase} (${column.name});`
-            );
-          }
-        })
-      );
-    })
-  );
+    for (const column of model.schema.filter((column) => column.index)) {
+      const unique = column.index === 'unique' ? 'UNIQUE' : '';
+      db.prepare(
+        `CREATE ${unique} INDEX idx_${model.filenameBase}_${column.name} ON ${model.filenameBase} (${column.name});`
+      ).run();
+    }
+  }
+};
 
 const formatLine = (line, model, totalLineCount) => {
   const lineNumber = totalLineCount + 1;
-  for (const fieldName of Object.keys(line)) {
-    const columnSchema = model.schema.find(
-      (schema) => schema.name === fieldName
-    );
 
-    // Remove columns not part of model
-    if (!columnSchema) {
-      delete line[fieldName];
-      continue;
-    }
+  const formattedLine = {};
 
-    // Remove null values
-    if (line[fieldName] === null || line[fieldName] === '') {
-      delete line[fieldName];
-    }
+  for (const columnSchema of model.schema) {
+    const lineValue = line[columnSchema.name];
 
-    // Convert fields that should be integer
     if (columnSchema.type === 'integer') {
-      const value = Number.parseInt(line[fieldName], 10);
-
-      if (Number.isNaN(value)) {
-        delete line[fieldName];
-      } else {
-        line[fieldName] = value;
-      }
+      // Convert fields that should be integer
+      formattedLine[columnSchema.name] = Number.parseInt(lineValue, 10);
+    } else if (columnSchema.type === 'real') {
+      // Convert fields that should be float
+      formattedLine[columnSchema.name] = Number.parseFloat(lineValue);
+    } else {
+      formattedLine[columnSchema.name] = lineValue;
     }
 
-    // Convert fields that should be float
-    if (columnSchema.type === 'real') {
-      const value = Number.parseFloat(line[fieldName]);
-
-      if (Number.isNaN(value)) {
-        delete line[fieldName];
-      } else {
-        line[fieldName] = value;
-      }
+    if (
+      formattedLine[columnSchema.name] === '' ||
+      formattedLine[columnSchema.name] === undefined ||
+      formattedLine[columnSchema.name] === null ||
+      Number.isNaN(formattedLine[columnSchema.name])
+    ) {
+      // Add null values
+      formattedLine[columnSchema.name] = null;
     }
 
     // Validate required
     if (
       columnSchema.required === true &&
-      (line[fieldName] === undefined || line[fieldName] === '')
+      formattedLine[columnSchema.name] === null
     ) {
       throw new Error(
-        `Missing required value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}.`
+        `Missing required value in ${model.filenameBase}.txt for ${columnSchema.name} on line ${lineNumber}.`
       );
     }
 
     // Validate minimum
-    if (columnSchema.min !== undefined && line[fieldName] < columnSchema.min) {
+    if (
+      columnSchema.min !== undefined &&
+      formattedLine[columnSchema.name] < columnSchema.min
+    ) {
       throw new Error(
-        `Invalid value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}: below minimum value of ${columnSchema.min}.`
+        `Invalid value in ${model.filenameBase}.txt for ${columnSchema.name} on line ${lineNumber}: below minimum value of ${columnSchema.min}.`
       );
     }
 
     // Validate maximum
-    if (columnSchema.max !== undefined && line[fieldName] > columnSchema.max) {
+    if (
+      columnSchema.max !== undefined &&
+      formattedLine[columnSchema.name] > columnSchema.max
+    ) {
       throw new Error(
-        `Invalid value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}: above maximum value of ${columnSchema.max}.`
+        `Invalid value in ${model.filenameBase}.txt for ${columnSchema.name} on line ${lineNumber}: above maximum value of ${columnSchema.max}.`
       );
     }
   }
 
   // Convert to midnight timestamp
-  const timestampFormat = [
+  const timestampColumnNames = [
     'start_time',
     'end_time',
     'arrival_time',
     'departure_time',
   ];
 
-  for (const fieldName of timestampFormat) {
-    if (line[fieldName]) {
-      line[`${fieldName}stamp`] = calculateSecondsFromMidnight(line[fieldName]);
+  for (const timestampColumnName of timestampColumnNames) {
+    if (formattedLine[timestampColumnName]) {
+      formattedLine[`${timestampColumnName}stamp`] =
+        calculateSecondsFromMidnight(formattedLine[timestampColumnName]);
     }
   }
 
-  return line;
+  return formattedLine;
 };
 
-const importLines = async (task, lines, model, totalLineCount) => {
+const importLines = (task, lines, model, totalLineCount) => {
   if (lines.length === 0) {
     return;
   }
 
   const linesToImportCount = lines.length;
-  const fieldNames = model.schema.map((column) => column.name);
+  const fieldNames = model.schema
+    .filter((column) => column.name !== 'id')
+    .map((column) => column.name);
   const placeholders = [];
   const values = [];
 
   while (lines.length > 0) {
     const line = lines.pop();
     placeholders.push(`(${fieldNames.map(() => '?').join(', ')})`);
-    for (const fieldName of fieldNames) {
-      values.push(line[fieldName]);
-    }
+    values.push(...fieldNames.map((fieldName) => line[fieldName]));
   }
 
   try {
-    await task.db.run(
+    const insert = task.db.prepare(
       `INSERT ${task.ignoreDuplicates ? 'OR IGNORE' : ''} INTO ${
         model.filenameBase
-      }(${fieldNames.join(', ')}) VALUES${placeholders.join(',')}`,
-      values
+      } (${fieldNames.join(', ')}) VALUES ${placeholders.join(',')}`
     );
+    insert.run(...values);
   } catch (error) {
     task.warn(
       `Check ${model.filenameBase}.txt for invalid data between lines ${
@@ -536,7 +530,7 @@ const importFiles = (task) =>
           return;
         }
 
-        // If the model is a database/gtfs-realtime model then just silently exit as we dont really care here
+        // If the model is a database/gtfs-realtime model then silently exit
         if (model.extension === 'gtfs-realtime') {
           resolve();
           return;
@@ -569,18 +563,16 @@ const importFiles = (task) =>
           ...task.csvOptions,
         });
 
-        parser.on('readable', async () => {
+        parser.on('readable', () => {
           let record;
 
           while ((record = parser.read())) {
             try {
               totalLineCount += 1;
               lines.push(formatLine(record, model, totalLineCount));
-
               // If we have a bunch of lines ready to insert, then do it
               if (lines.length >= maxInsertVariables / model.schema.length) {
-                /* eslint-disable-next-line no-await-in-loop */
-                await importLines(task, lines, model, totalLineCount);
+                importLines(task, lines, model, totalLineCount);
               }
             } catch (error) {
               reject(error);
@@ -588,9 +580,9 @@ const importFiles = (task) =>
           }
         });
 
-        parser.on('end', async () => {
+        parser.on('end', () => {
           // Insert all remaining lines
-          await importLines(task, lines, model, totalLineCount).catch(reject);
+          importLines(task, lines, model, totalLineCount);
           resolve();
         });
 
@@ -606,70 +598,71 @@ export async function importGtfs(initialConfig) {
   const log = _log(config);
   const logError = _logError(config);
   const logWarning = _logWarning(config);
-  const db = await openDb(config).catch((error) => {
-    if (error instanceof Error && error.code === 'SQLITE_CANTOPEN') {
-      logError(
-        `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
-      );
-    }
-
-    throw error;
-  });
-
-  const agencyCount = config.agencies.length;
-  log(
-    `Starting GTFS import for ${pluralize(
-      'file',
-      agencyCount,
-      true
-    )} using SQLite database at ${config.sqlitePath}`
-  );
-
-  await deleteTables(db);
-  await createTables(db);
+  try {
+    const db = openDb(config);
+
+    const agencyCount = config.agencies.length;
+    log(
+      `Starting GTFS import for ${pluralize(
+        'file',
+        agencyCount,
+        true
+      )} using SQLite database at ${config.sqlitePath}`
+    );
 
-  await setupDb(db);
+    createTables(db);
+
+    await mapSeries(config.agencies, async (agency) => {
+      const { path, cleanup } = await dir({ unsafeCleanup: true });
+
+      const task = {
+        exclude: agency.exclude,
+        agency_url: agency.url,
+        headers: agency.headers || false,
+        realtime_headers: agency.realtimeHeaders || false,
+        realtime_urls: agency.realtimeUrls || false,
+        downloadDir: path,
+        path: agency.path,
+        csvOptions: config.csvOptions || {},
+        ignoreDuplicates: config.ignoreDuplicates,
+        db,
+        log(message, overwrite) {
+          log(message, overwrite);
+        },
+        warn(message) {
+          logWarning(message);
+        },
+        error(message) {
+          logError(message);
+        },
+      };
+
+      if (task.agency_url) {
+        await downloadFiles(task);
+      }
 
-  await mapSeries(config.agencies, async (agency) => {
-    const { path, cleanup } = await dir({ unsafeCleanup: true });
+      await readFiles(task);
+      await importFiles(task);
 
-    const task = {
-      exclude: agency.exclude,
-      agency_url: agency.url,
-      headers: agency.headers || false,
-      realtime_headers: agency.realtimeHeaders || false,
-      realtime_urls: agency.realtimeUrls || false,
-      downloadDir: path,
-      path: agency.path,
-      csvOptions: config.csvOptions || {},
-      ignoreDuplicates: config.ignoreDuplicates,
-      db,
-      log(message, overwrite) {
-        log(message, overwrite);
-      },
-      warn(message) {
-        logWarning(message);
-      },
-      error(message) {
-        logError(message);
-      },
-    };
-
-    if (task.agency_url) {
-      await downloadFiles(task);
-    }
+      if (task.realtime_urls) {
+        await updateRealtimeData(task);
+      }
 
-    await readFiles(task);
-    await importFiles(task);
+      cleanup();
+    });
 
-    if (task.realtime_urls) {
-      await updateRealtimeData(task);
+    log(
+      `Completed GTFS import for ${pluralize('agency', agencyCount, true)}\n`
+    );
+  } catch (error) {
+    if (error instanceof Error && error.code === 'SQLITE_CANTOPEN') {
+      logError(
+        `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
+      );
     }
 
-    cleanup();
-  });
-
-  log(`Completed GTFS import for ${pluralize('agency', agencyCount, true)}\n`);
+    throw error;
+  }
 }
 
 export async function updateGtfsRealtime(initialConfig) {
@@ -678,7 +671,7 @@ export async function updateGtfsRealtime(initialConfig) {
   const log = _log(config);
   const logError = _logError(config);
   const logWarning = _logWarning(config);
-  const db = await openDb(config).catch((error) => {
+  const db = openDb(config).catch((error) => {
     if (error instanceof Error && error.code === 'SQLITE_CANTOPEN') {
       logError(
         `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`

package/lib/non-standard/directions.js

@@ -1,6 +1,6 @@
 import sqlString from 'sqlstring-sqlite';
 
-import { getDb } from '../db.js';
+import { openDb } from '../db.js';
 
 import {
   formatOrderByClause,
@@ -12,19 +12,21 @@ import directions from '../../models/non-standard/directions.js';
 /*
  * Returns an array of all directions that match the query parameters.
  */
-export async function getDirections(
+export function getDirections(
   query = {},
   fields = [],
   orderBy = [],
   options = {}
 ) {
-  const db = options.db ?? (await getDb());
+  const db = options.db ?? openDb();
   const tableName = sqlString.escapeId(directions.filenameBase);
   const selectClause = formatSelectClause(fields);
   const whereClause = formatWhereClauses(query);
   const orderByClause = formatOrderByClause(orderBy);
 
-  return db.all(
-    `${selectClause} FROM ${tableName} ${whereClause} ${orderByClause};`
-  );
+  return db
+    .prepare(
+      `${selectClause} FROM ${tableName} ${whereClause} ${orderByClause};`
+    )
+    .all();
 }

package/lib/non-standard/stop-attributes.js

@@ -1,6 +1,6 @@
 import sqlString from 'sqlstring-sqlite';
 
-import { getDb } from '../db.js';
+import { openDb } from '../db.js';
 
 import {
   formatOrderByClause,
@@ -12,19 +12,21 @@ import stopAttributes from '../../models/non-standard/stop-attributes.js';
 /*
  * Returns an array of all stop attributes that match the query parameters.
  */
-export async function getStopAttributes(
+export function getStopAttributes(
   query = {},
   fields = [],
   orderBy = [],
   options = {}
 ) {
-  const db = options.db ?? (await getDb());
+  const db = options.db ?? openDb();
   const tableName = sqlString.escapeId(stopAttributes.filenameBase);
   const selectClause = formatSelectClause(fields);
   const whereClause = formatWhereClauses(query);
   const orderByClause = formatOrderByClause(orderBy);
 
-  return db.all(
-    `${selectClause} FROM ${tableName} ${whereClause} ${orderByClause};`
-  );
+  return db
+    .prepare(
+      `${selectClause} FROM ${tableName} ${whereClause} ${orderByClause};`
+    )
+    .all();
 }