gtfs 3.1.1 → 3.2.0

package/lib/gtfs-ride/board-alights.js

@@ -2,18 +2,29 @@ import sqlString from 'sqlstring-sqlite';
 
 import { getDb } from '../db.js';
 
-import { formatOrderByClause, formatSelectClause, formatWhereClauses } from '../utils.js';
+import {
+  formatOrderByClause,
+  formatSelectClause,
+  formatWhereClauses,
+} from '../utils.js';
 import boardAlights from '../../models/gtfs-ride/board-alight.js';
 
 /*
  * Returns an array of all board-alights that match the query parameters.
  */
-export async function getBoardAlights(query = {}, fields = [], orderBy = []) {
-  const db = await getDb();
+export async function getBoardAlights(
+  query = {},
+  fields = [],
+  orderBy = [],
+  options = {}
+) {
+  const db = options.db ?? (await getDb());
   const tableName = sqlString.escapeId(boardAlights.filenameBase);
   const selectClause = formatSelectClause(fields);
   const whereClause = formatWhereClauses(query);
   const orderByClause = formatOrderByClause(orderBy);
 
-  return db.all(`${selectClause} FROM ${tableName} ${whereClause} ${orderByClause};`);
+  return db.all(
+    `${selectClause} FROM ${tableName} ${whereClause} ${orderByClause};`
+  );
 }

package/lib/gtfs-ride/ride-feed-infos.js

@@ -2,18 +2,29 @@ import sqlString from 'sqlstring-sqlite';
 
 import { getDb } from '../db.js';
 
-import { formatOrderByClause, formatSelectClause, formatWhereClauses } from '../utils.js';
+import {
+  formatOrderByClause,
+  formatSelectClause,
+  formatWhereClauses,
+} from '../utils.js';
 import rideFeedInfo from '../../models/gtfs-ride/ride-feed-info.js';
 
 /*
  * Returns an array of all ride-feed-info that match the query parameters.
  */
-export async function getRideFeedInfos(query = {}, fields = [], orderBy = []) {
-  const db = await getDb();
+export async function getRideFeedInfos(
+  query = {},
+  fields = [],
+  orderBy = [],
+  options = {}
+) {
+  const db = options.db ?? (await getDb());
   const tableName = sqlString.escapeId(rideFeedInfo.filenameBase);
   const selectClause = formatSelectClause(fields);
   const whereClause = formatWhereClauses(query);
   const orderByClause = formatOrderByClause(orderBy);
 
-  return db.all(`${selectClause} FROM ${tableName} ${whereClause} ${orderByClause};`);
+  return db.all(
+    `${selectClause} FROM ${tableName} ${whereClause} ${orderByClause};`
+  );
 }

package/lib/gtfs-ride/rider-trips.js

@@ -2,18 +2,29 @@ import sqlString from 'sqlstring-sqlite';
 
 import { getDb } from '../db.js';
 
-import { formatOrderByClause, formatSelectClause, formatWhereClauses } from '../utils.js';
+import {
+  formatOrderByClause,
+  formatSelectClause,
+  formatWhereClauses,
+} from '../utils.js';
 import riderTrip from '../../models/gtfs-ride/rider-trip.js';
 
 /*
  * Returns an array of all rider trips that match the query parameters.
  */
-export async function getRiderTrips(query = {}, fields = [], orderBy = []) {
-  const db = await getDb();
+export async function getRiderTrips(
+  query = {},
+  fields = [],
+  orderBy = [],
+  options = {}
+) {
+  const db = options.db ?? (await getDb());
   const tableName = sqlString.escapeId(riderTrip.filenameBase);
   const selectClause = formatSelectClause(fields);
   const whereClause = formatWhereClauses(query);
   const orderByClause = formatOrderByClause(orderBy);
 
-  return db.all(`${selectClause} FROM ${tableName} ${whereClause} ${orderByClause};`);
+  return db.all(
+    `${selectClause} FROM ${tableName} ${whereClause} ${orderByClause};`
+  );
 }

package/lib/gtfs-ride/riderships.js

@@ -2,18 +2,29 @@ import sqlString from 'sqlstring-sqlite';
 
 import { getDb } from '../db.js';
 
-import { formatOrderByClause, formatSelectClause, formatWhereClauses } from '../utils.js';
+import {
+  formatOrderByClause,
+  formatSelectClause,
+  formatWhereClauses,
+} from '../utils.js';
 import riderships from '../../models/gtfs-ride/ridership.js';
 
 /*
  * Returns an array of all riderships that match the query parameters.
  */
-export async function getRiderships(query = {}, fields = [], orderBy = []) {
-  const db = await getDb();
+export async function getRiderships(
+  query = {},
+  fields = [],
+  orderBy = [],
+  options = {}
+) {
+  const db = options.db ?? (await getDb());
   const tableName = sqlString.escapeId(riderships.filenameBase);
   const selectClause = formatSelectClause(fields);
   const whereClause = formatWhereClauses(query);
   const orderByClause = formatOrderByClause(orderBy);
 
-  return db.all(`${selectClause} FROM ${tableName} ${whereClause} ${orderByClause};`);
+  return db.all(
+    `${selectClause} FROM ${tableName} ${whereClause} ${orderByClause};`
+  );
 }

package/lib/gtfs-ride/trip-capacities.js

@@ -2,18 +2,29 @@ import sqlString from 'sqlstring-sqlite';
 
 import { getDb } from '../db.js';
 
-import { formatOrderByClause, formatSelectClause, formatWhereClauses } from '../utils.js';
+import {
+  formatOrderByClause,
+  formatSelectClause,
+  formatWhereClauses,
+} from '../utils.js';
 import tripCapacity from '../../models/gtfs-ride/trip-capacity.js';
 
 /*
  * Returns an array of all trip-capacities that match the query parameters.
  */
-export async function getTripCapacities(query = {}, fields = [], orderBy = []) {
-  const db = await getDb();
+export async function getTripCapacities(
+  query = {},
+  fields = [],
+  orderBy = [],
+  options = {}
+) {
+  const db = options.db ?? (await getDb());
   const tableName = sqlString.escapeId(tripCapacity.filenameBase);
   const selectClause = formatSelectClause(fields);
   const whereClause = formatWhereClauses(query);
   const orderByClause = formatOrderByClause(orderBy);
 
-  return db.all(`${selectClause} FROM ${tableName} ${whereClause} ${orderByClause};`);
+  return db.all(
+    `${selectClause} FROM ${tableName} ${whereClause} ${orderByClause};`
+  );
 }

package/lib/import.js

@@ -1,9 +1,9 @@
 import path from 'node:path';
 import { createReadStream, existsSync, lstatSync } from 'node:fs';
 import { readdir, rename, writeFile } from 'node:fs/promises';
-import copydir from 'copy-dir';
+import copy from 'recursive-copy';
 import fetch from 'node-fetch';
-import parse from 'csv-parse';
+import { parse } from 'csv-parse';
 import pluralize from 'pluralize';
 import stripBomStream from 'strip-bom-stream';
 import { dir } from 'tmp-promise';
@@ -13,32 +13,43 @@ import mapSeries from 'promise-map-series';
 import models from '../models/models.js';
 import { openDb, setupDb } from './db.js';
 import { unzip } from './file-utils.js';
-import { log as _log, logError as _logError, logWarning as _logWarning } from './log-utils.js';
-import { calculateSecondsFromMidnight, setDefaultConfig, validateConfigForImport } from './utils.js';
-
-const downloadFiles = async task => {
+import {
+  log as _log,
+  logError as _logError,
+  logWarning as _logWarning,
+} from './log-utils.js';
+import {
+  calculateSecondsFromMidnight,
+  setDefaultConfig,
+  validateConfigForImport,
+} from './utils.js';
+
+const downloadFiles = async (task) => {
   task.log(`Downloading GTFS from ${task.agency_url}`);
 
   task.path = `${task.downloadDir}/gtfs.zip`;
 
-  const response = await fetch(task.agency_url, { method: 'GET', headers: task.agency_headers || {} });
+  const response = await fetch(task.agency_url, {
+    method: 'GET',
+    headers: task.agency_headers || {},
+  });
 
   if (response.status !== 200) {
     throw new Error('Couldn’t download files');
   }
 
-  const buffer = await response.buffer();
+  const buffer = await response.arrayBuffer();
 
-  await writeFile(task.path, buffer);
+  await writeFile(task.path, Buffer.from(buffer));
   task.log('Download successful');
 };
 
-const getTextFiles = async folderPath => {
+const getTextFiles = async (folderPath) => {
   const files = await readdir(folderPath);
-  return files.filter(filename => filename.slice(-3) === 'txt');
+  return files.filter((filename) => filename.slice(-3) === 'txt');
 };
 
-const readFiles = async task => {
+const readFiles = async (task) => {
   const gtfsPath = untildify(task.path);
   task.log(`Importing GTFS from ${task.path}\r`);
   if (path.extname(gtfsPath) === '.zip') {
@@ -49,22 +60,37 @@ const readFiles = async task => {
       // If no .txt files in this directory, check for subdirectories and copy them here
       if (textFiles.length === 0) {
         const files = await readdir(task.downloadDir);
-        const folders = files.map(filename => path.join(task.downloadDir, filename)).filter(source => lstatSync(source).isDirectory());
+        const folders = files
+          .map((filename) => path.join(task.downloadDir, filename))
+          .filter((source) => lstatSync(source).isDirectory());
 
         if (folders.length > 1) {
-          throw new Error(`More than one subfolder found in zip file at \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`);
+          throw new Error(
+            `More than one subfolder found in zip file at \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`
+          );
         } else if (folders.length === 0) {
-          throw new Error(`No .txt files found in \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`);
+          throw new Error(
+            `No .txt files found in \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`
+          );
         }
 
         const subfolderName = folders[0];
         const directoryTextFiles = await getTextFiles(subfolderName);
 
         if (directoryTextFiles.length === 0) {
-          throw new Error(`No .txt files found in \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`);
+          throw new Error(
+            `No .txt files found in \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`
+          );
         }
 
-        await Promise.all(directoryTextFiles.map(async fileName => rename(path.join(subfolderName, fileName), path.join(task.downloadDir, fileName))));
+        await Promise.all(
+          directoryTextFiles.map(async (fileName) =>
+            rename(
+              path.join(subfolderName, fileName),
+              path.join(task.downloadDir, fileName)
+            )
+          )
+        );
       }
     } catch (error) {
       task.error(error);
@@ -74,52 +100,66 @@ const readFiles = async task => {
   } else {
     // Local file is unzipped, just copy it from there.
     try {
-      copydir.sync(gtfsPath, task.downloadDir);
+      await copy(gtfsPath, task.downloadDir);
     } catch {
-      throw new Error(`Unable to load files from path \`${gtfsPath}\` defined in configuration. Verify that path exists and contains GTFS files.`);
+      throw new Error(
+        `Unable to load files from path \`${gtfsPath}\` defined in configuration. Verify that path exists and contains GTFS files.`
+      );
     }
   }
 };
 
-const deleteTables = db => Promise.all(models.map(
-  model => db.run(`DROP TABLE IF EXISTS ${model.filenameBase};`)
-));
-
-const createTables = db => Promise.all(models.map(async model => {
-  if (!model.schema) {
-    return;
-  }
-
-  const columns = model.schema.map(column => {
-    let check = '';
-    if (column.min !== undefined && column.max) {
-      check = `CHECK( ${column.name} >= ${column.min} AND ${column.name} <= ${column.max} )`;
-    } else if (column.min) {
-      check = `CHECK( ${column.name} >= ${column.min} )`;
-    } else if (column.max) {
-      check = `CHECK( ${column.name} <= ${column.max} )`;
-    }
+const deleteTables = (db) =>
+  Promise.all(
+    models.map((model) => db.run(`DROP TABLE IF EXISTS ${model.filenameBase};`))
+  );
 
-    const primary = column.primary ? 'PRIMARY KEY' : '';
-    const required = column.required ? 'NOT NULL' : '';
-    const columnDefault = column.default ? 'DEFAULT ' + column.default : '';
-    return `${column.name} ${column.type} ${check} ${primary} ${required} ${columnDefault}`;
-  });
+const createTables = (db) =>
+  Promise.all(
+    models.map(async (model) => {
+      if (!model.schema) {
+        return;
+      }
 
-  await db.run(`CREATE TABLE ${model.filenameBase} (${columns.join(', ')});`);
+      const columns = model.schema.map((column) => {
+        let check = '';
+        if (column.min !== undefined && column.max) {
+          check = `CHECK( ${column.name} >= ${column.min} AND ${column.name} <= ${column.max} )`;
+        } else if (column.min) {
+          check = `CHECK( ${column.name} >= ${column.min} )`;
+        } else if (column.max) {
+          check = `CHECK( ${column.name} <= ${column.max} )`;
+        }
 
-  await Promise.all(model.schema.map(async column => {
-    if (column.index) {
-      const unique = column.index === 'unique' ? 'UNIQUE' : '';
-      await db.run(`CREATE ${unique} INDEX idx_${model.filenameBase}_${column.name} ON ${model.filenameBase} (${column.name});`);
-    }
-  }));
-}));
+        const primary = column.primary ? 'PRIMARY KEY' : '';
+        const required = column.required ? 'NOT NULL' : '';
+        const columnDefault = column.default ? 'DEFAULT ' + column.default : '';
+        return `${column.name} ${column.type} ${check} ${primary} ${required} ${columnDefault}`;
+      });
+
+      await db.run(
+        `CREATE TABLE ${model.filenameBase} (${columns.join(', ')});`
+      );
+
+      await Promise.all(
+        model.schema.map(async (column) => {
+          if (column.index) {
+            const unique = column.index === 'unique' ? 'UNIQUE' : '';
+            await db.run(
+              `CREATE ${unique} INDEX idx_${model.filenameBase}_${column.name} ON ${model.filenameBase} (${column.name});`
+            );
+          }
+        })
+      );
+    })
+  );
 
 const formatLine = (line, model, totalLineCount) => {
   const lineNumber = totalLineCount + 1;
   for (const fieldName of Object.keys(line)) {
-    const columnSchema = model.schema.find(schema => schema.name === fieldName);
+    const columnSchema = model.schema.find(
+      (schema) => schema.name === fieldName
+    );
 
     // Remove columns not part of model
     if (!columnSchema) {
@@ -155,18 +195,27 @@ const formatLine = (line, model, totalLineCount) => {
     }
 
     // Validate required
-    if (columnSchema.required === true && (line[fieldName] === undefined || line[fieldName] === '')) {
-      throw new Error(`Missing required value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}.`);
+    if (
+      columnSchema.required === true &&
+      (line[fieldName] === undefined || line[fieldName] === '')
+    ) {
+      throw new Error(
+        `Missing required value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}.`
+      );
     }
 
     // Validate minimum
     if (columnSchema.min !== undefined && line[fieldName] < columnSchema.min) {
-      throw new Error(`Invalid value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}: below minimum value of ${columnSchema.min}.`);
+      throw new Error(
+        `Invalid value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}: below minimum value of ${columnSchema.min}.`
+      );
     }
 
     // Validate maximum
     if (columnSchema.max !== undefined && line[fieldName] > columnSchema.max) {
-      throw new Error(`Invalid value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}: above maximum value of ${columnSchema.max}.`);
+      throw new Error(
+        `Invalid value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}: above maximum value of ${columnSchema.max}.`
+      );
     }
   }
 
@@ -175,7 +224,7 @@ const formatLine = (line, model, totalLineCount) => {
     'start_time',
     'end_time',
     'arrival_time',
-    'departure_time'
+    'departure_time',
   ];
 
   for (const fieldName of timestampFormat) {
@@ -193,7 +242,7 @@ const importLines = async (task, lines, model, totalLineCount) => {
   }
 
   const linesToImportCount = lines.length;
-  const fieldNames = model.schema.map(column => column.name);
+  const fieldNames = model.schema.map((column) => column.name);
   const placeholders = [];
   const values = [];
 
@@ -206,103 +255,129 @@ const importLines = async (task, lines, model, totalLineCount) => {
   }
 
   try {
-    await task.db.run(`INSERT INTO ${model.filenameBase}(${fieldNames.join(', ')}) VALUES${placeholders.join(',')}`, values);
+    await task.db.run(
+      `INSERT INTO ${model.filenameBase}(${fieldNames.join(
+        ', '
+      )}) VALUES${placeholders.join(',')}`,
+      values
+    );
   } catch (error) {
-    task.warn(`Check ${model.filenameBase}.txt for invalid data between lines ${totalLineCount - linesToImportCount} and ${totalLineCount}.`);
+    task.warn(
+      `Check ${model.filenameBase}.txt for invalid data between lines ${
+        totalLineCount - linesToImportCount
+      } and ${totalLineCount}.`
+    );
     throw error;
   }
 
-  task.log(`Importing - ${model.filenameBase}.txt - ${totalLineCount} lines imported\r`, true);
+  task.log(
+    `Importing - ${model.filenameBase}.txt - ${totalLineCount} lines imported\r`,
+    true
+  );
 };
 
-const importFiles = task => mapSeries(models, model => new Promise((resolve, reject) => {
-  // Loop through each GTFS file
-  // Filter out excluded files from config
-  if (task.exclude && task.exclude.includes(model.filenameBase)) {
-    task.log(`Skipping - ${model.filenameBase}.txt\r`);
-    resolve();
-    return;
-  }
-
-  const filepath = path.join(task.downloadDir, `${model.filenameBase}.txt`);
-
-  if (!existsSync(filepath)) {
-    if (!model.nonstandard) {
-      task.log(`Importing - ${model.filenameBase}.txt - No file found\r`);
-    }
-
-    resolve();
-    return;
-  }
-
-  task.log(`Importing - ${model.filenameBase}.txt\r`);
-
-  const lines = [];
-  let totalLineCount = 0;
-  const maxInsertVariables = 800;
-  const parser = parse({
-    columns: true,
-    relax: true,
-    trim: true,
-    skip_empty_lines: true,
-    ...task.csvOptions
-  });
-
-  parser.on('readable', async () => {
-    let record;
-    /* eslint-disable-next-line no-cond-assign */
-    while (record = parser.read()) {
-      try {
-        totalLineCount += 1;
-        lines.push(formatLine(record, model, totalLineCount));
-
-        // If we have a bunch of lines ready to insert, then do it
-        if (lines.length >= maxInsertVariables / model.schema.length) {
-          /* eslint-disable-next-line no-await-in-loop */
-          await importLines(task, lines, model, totalLineCount);
+const importFiles = (task) =>
+  mapSeries(
+    models,
+    (model) =>
+      new Promise((resolve, reject) => {
+        // Loop through each GTFS file
+        // Filter out excluded files from config
+        if (task.exclude && task.exclude.includes(model.filenameBase)) {
+          task.log(`Skipping - ${model.filenameBase}.txt\r`);
+          resolve();
+          return;
         }
-      } catch (error) {
-        reject(error);
-      }
-    }
-  });
 
-  parser.on('end', async () => {
-    // Insert all remaining lines
-    await importLines(task, lines, model, totalLineCount).catch(reject);
-    resolve();
-  });
+        const filepath = path.join(
+          task.downloadDir,
+          `${model.filenameBase}.txt`
+        );
 
-  parser.on('error', reject);
+        if (!existsSync(filepath)) {
+          if (!model.nonstandard) {
+            task.log(`Importing - ${model.filenameBase}.txt - No file found\r`);
+          }
 
-  createReadStream(filepath)
-    .pipe(stripBomStream())
-    .pipe(parser);
-}));
+          resolve();
+          return;
+        }
 
-const importGtfs = async initialConfig => {
+        task.log(`Importing - ${model.filenameBase}.txt\r`);
+
+        const lines = [];
+        let totalLineCount = 0;
+        const maxInsertVariables = 800;
+        const parser = parse({
+          columns: true,
+          relax: true,
+          trim: true,
+          skip_empty_lines: true,
+          ...task.csvOptions,
+        });
+
+        parser.on('readable', async () => {
+          let record;
+
+          while ((record = parser.read())) {
+            try {
+              totalLineCount += 1;
+              lines.push(formatLine(record, model, totalLineCount));
+
+              // If we have a bunch of lines ready to insert, then do it
+              if (lines.length >= maxInsertVariables / model.schema.length) {
+                /* eslint-disable-next-line no-await-in-loop */
+                await importLines(task, lines, model, totalLineCount);
+              }
+            } catch (error) {
+              reject(error);
+            }
+          }
+        });
+
+        parser.on('end', async () => {
+          // Insert all remaining lines
+          await importLines(task, lines, model, totalLineCount).catch(reject);
+          resolve();
+        });
+
+        parser.on('error', reject);
+
+        createReadStream(filepath).pipe(stripBomStream()).pipe(parser);
+      })
+  );
+
+const importGtfs = async (initialConfig) => {
   const config = setDefaultConfig(initialConfig);
   validateConfigForImport(config);
   const log = _log(config);
   const logError = _logError(config);
   const logWarning = _logWarning(config);
-  const db = await openDb(config).catch(error => {
+  const db = await openDb(config).catch((error) => {
     if (error instanceof Error && error.code === 'SQLITE_CANTOPEN') {
-      logError(`Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`);
+      logError(
+        `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
+      );
     }
 
     throw error;
   });
 
   const agencyCount = config.agencies.length;
-  log(`Starting GTFS import for ${pluralize('file', agencyCount, true)} using SQLite database at ${config.sqlitePath}`);
+  log(
+    `Starting GTFS import for ${pluralize(
+      'file',
+      agencyCount,
+      true
+    )} using SQLite database at ${config.sqlitePath}`
+  );
 
   await deleteTables(db);
   await createTables(db);
 
   await setupDb(db);
 
-  await mapSeries(config.agencies, async agency => {
+  await mapSeries(config.agencies, async (agency) => {
     const { path, cleanup } = await dir({ unsafeCleanup: true });
 
     const task = {
@@ -316,12 +391,12 @@ const importGtfs = async initialConfig => {
       log: (message, overwrite) => {
         log(message, overwrite);
       },
-      warn: message => {
+      warn: (message) => {
         logWarning(message);
       },
-      error: message => {
+      error: (message) => {
         logError(message);
-      }
+      },
     };
 
     if (task.agency_url) {

package/lib/log-utils.js

@@ -38,7 +38,7 @@ export function logWarning(config) {
     return config.logFunction;
   }
 
-  return text => {
+  return (text) => {
     process.stdout.write(`\n${formatWarning(text)}\n`);
   };
 }
@@ -51,7 +51,7 @@ export function logError(config) {
     return config.logFunction;
   }
 
-  return text => {
+  return (text) => {
     process.stdout.write(`\n${formatError(text)}\n`);
   };
 }
@@ -60,7 +60,9 @@ export function logError(config) {
  * Format console warning text
  */
 export function formatWarning(text) {
-  return `${chalk.yellow.underline('Warning')}${chalk.yellow(':')} ${chalk.yellow(text)}`;
+  return `${chalk.yellow.underline('Warning')}${chalk.yellow(
+    ':'
+  )} ${chalk.yellow(text)}`;
 }
 
 /*
@@ -68,5 +70,7 @@ export function formatWarning(text) {
  */
 export function formatError(error) {
   const message = error instanceof Error ? error.message : error;
-  return `${chalk.red.underline('Error')}${chalk.red(':')} ${chalk.red(message.replace('Error: ', ''))}`;
+  return `${chalk.red.underline('Error')}${chalk.red(':')} ${chalk.red(
+    message.replace('Error: ', '')
+  )}`;
 }