gtfs 3.3.1 → 3.5.1

package/lib/import.js

@@ -9,6 +9,8 @@ import stripBomStream from 'strip-bom-stream';
 import { dir } from 'tmp-promise';
 import untildify from 'untildify';
 import mapSeries from 'promise-map-series';
+import gtfsrt from 'gtfs-realtime-bindings';
+import sqlString from 'sqlstring-sqlite';
 
 import models from '../models/models.js';
 import { openDb, setupDb } from './db.js';
@@ -22,6 +24,7 @@ import {
   calculateSecondsFromMidnight,
   setDefaultConfig,
   validateConfigForImport,
+  convertLongTimeToDate,
 } from './utils.js';
 
 const downloadFiles = async (task) => {
@@ -31,7 +34,7 @@ const downloadFiles = async (task) => {
 
   const response = await fetch(task.agency_url, {
     method: 'GET',
-    headers: task.agency_headers || {},
+    headers: task.headers || {},
   });
 
   if (response.status !== 200) {
@@ -44,6 +47,249 @@ const downloadFiles = async (task) => {
   task.log('Download successful');
 };
 
+const downloadGtfsRealtimeData = async (url, headers) => {
+  const response = await fetch(url, {
+    method: 'GET',
+    headers: { ...{}, ...headers, ...{ 'Accept-Encoding': 'gzip' } },
+  });
+
+  if (response.status !== 200) {
+    throw new Error('Couldn’t download files');
+  }
+
+  const buffer = await response.arrayBuffer();
+  return gtfsrt.transit_realtime.FeedMessage.decode(Buffer.from(buffer));
+};
+
+function getDescendantProp(obj, desc, defaultvalue) {
+  if (desc === undefined) return defaultvalue;
+  const arr = desc.split('.');
+  while (arr.length) {
+    const nextKey = arr.shift();
+    if (nextKey.includes('[')) {
+      const arrayKey = nextKey.match(/(\w*)\[(\d+)\]/);
+      if (!obj[arrayKey[1]]) return defaultvalue;
+      if (!obj[arrayKey[1]][arrayKey[2]]) return defaultvalue;
+      obj = obj[arrayKey[1]][arrayKey[2]];
+    } else {
+      if (!obj[nextKey]) return defaultvalue;
+      obj = obj[nextKey];
+    }
+  }
+
+  if (obj.__isLong__) return convertLongTimeToDate(obj);
+
+  return obj;
+}
+
+const markRealtimeDataStale = async (db, log) => {
+  const vehiclePositionModel = models.find(
+    (x) => x.filenameBase === 'vehicle_positions'
+  );
+  const tripUpdatesModel = models.find(
+    (x) => x.filenameBase === 'trip_updates'
+  );
+  const stopTimesUpdatesModel = models.find(
+    (x) => x.filenameBase === 'stop_times_updates'
+  );
+  const serviceAlertsModel = models.find(
+    (x) => x.filenameBase === 'service_alerts'
+  );
+  const serviceAlertTargetsModel = models.find(
+    (x) => x.filenameBase === 'service_alert_targets'
+  );
+
+  // Mark all data as stale
+  log(`Marking GTFS-Realtime data as stale..`);
+  await db.run(`UPDATE ${vehiclePositionModel.filenameBase} SET isUpdated=0`);
+  await db.run(`UPDATE ${tripUpdatesModel.filenameBase} SET isUpdated=0`);
+  await db.run(`UPDATE ${stopTimesUpdatesModel.filenameBase} SET isUpdated=0`);
+  await db.run(`UPDATE ${serviceAlertsModel.filenameBase} SET isUpdated=0`);
+  await db.run(
+    `UPDATE ${serviceAlertTargetsModel.filenameBase} SET isUpdated=0`
+  );
+  log(`Marked GTFS-Realtime data as stale\r`, true);
+};
+
+const cleanStaleRealtimeData = async (db, log) => {
+  const vehiclePositionModel = models.find(
+    (x) => x.filenameBase === 'vehicle_positions'
+  );
+  const tripUpdatesModel = models.find(
+    (x) => x.filenameBase === 'trip_updates'
+  );
+  const stopTimesUpdatesModel = models.find(
+    (x) => x.filenameBase === 'stop_times_updates'
+  );
+  const serviceAlertsModel = models.find(
+    (x) => x.filenameBase === 'service_alerts'
+  );
+  const serviceAlertTargetsModel = models.find(
+    (x) => x.filenameBase === 'service_alert_targets'
+  );
+
+  log(`Cleaning stale GRFS-RT data..`);
+  await db.run(
+    `DELETE FROM ${vehiclePositionModel.filenameBase} WHERE isUpdated=0`
+  );
+  await db.run(
+    `DELETE FROM ${tripUpdatesModel.filenameBase} WHERE isUpdated=0`
+  );
+  await db.run(
+    `DELETE FROM ${stopTimesUpdatesModel.filenameBase} WHERE isUpdated=0`
+  );
+  await db.run(
+    `DELETE FROM ${serviceAlertsModel.filenameBase} WHERE isUpdated=0`
+  );
+  await db.run(
+    `DELETE FROM ${serviceAlertTargetsModel.filenameBase} WHERE isUpdated=0`
+  );
+  log(`Cleaned stale GTFS-Realtime data\r`, true);
+};
+
+const updateRealtimeData = async (task) => {
+  const model = {
+    vehicle_positions: models.find(
+      (x) => x.filenameBase === 'vehicle_positions'
+    ),
+    trip_updates: models.find((x) => x.filenameBase === 'trip_updates'),
+    stop_times_updates: models.find(
+      (x) => x.filenameBase === 'stop_times_updates'
+    ),
+    service_alerts: models.find((x) => x.filenameBase === 'service_alerts'),
+    service_alert_targets: models.find(
+      (x) => x.filenameBase === 'service_alert_targets'
+    ),
+  };
+
+  const fields = {
+    vehicle_positions: model.vehicle_positions.schema
+      .map((column) => column.name)
+      .join(', '),
+    trip_updates: model.trip_updates.schema
+      .map((column) => column.name)
+      .join(', '),
+    stop_times_updates: model.stop_times_updates.schema
+      .map((column) => column.name)
+      .join(', '),
+    service_alerts: model.service_alerts.schema
+      .map((column) => column.name)
+      .join(', '),
+    service_alert_targets: model.service_alert_targets.schema
+      .map((column) => column.name)
+      .join(', '),
+  };
+
+  task.log(
+    `Starting GTFS-Realtime import from ${task.realtime_urls.length} urls`
+  );
+
+  for (const realtimeUrl of task.realtime_urls) {
+    task.log(`Downloading GTFS-Realtime from ${realtimeUrl}`);
+    // eslint-disable-next-line no-await-in-loop
+    const tripUpdateData = await downloadGtfsRealtimeData(
+      realtimeUrl,
+      task.realtime_headers
+    );
+    task.log(`Download successful`);
+
+    let totalLineCount = 0;
+    for (const entity of tripUpdateData.entity) {
+      // Determine the type of GTFS-Realtime
+      let gtfsRealtimeType = null;
+      if (entity.vehicle) {
+        gtfsRealtimeType = 'vehicle_positions';
+      }
+
+      if (entity.tripUpdate) {
+        gtfsRealtimeType = 'trip_updates';
+      }
+
+      if (entity.alert) {
+        gtfsRealtimeType = 'service_alerts';
+      }
+
+      if (!gtfsRealtimeType) {
+        break;
+      }
+
+      // Do base processing
+      const fieldValues = model[gtfsRealtimeType].schema.map((column) =>
+        sqlString.escape(
+          getDescendantProp(entity, column.source, column.default)
+        )
+      );
+
+      // eslint-disable-next-line no-await-in-loop
+      await task.db
+        .run(
+          `REPLACE INTO ${model[gtfsRealtimeType].filenameBase} (${
+            fields[gtfsRealtimeType]
+          }) VALUES (${fieldValues.join(', ')})`
+        )
+        .catch((error) => {
+          task.warn('Import error: ' + error.message);
+        });
+
+      // Special processing for tripUpdates
+      if (entity.tripUpdate) {
+        const stopUpdateArray = [];
+        for (const stopUpdate of entity.tripUpdate.stopTimeUpdate) {
+          stopUpdate.parent = entity;
+          const subValues = model.stop_times_updates.schema.map((column) =>
+            sqlString.escape(
+              getDescendantProp(stopUpdate, column.source, column.default)
+            )
+          );
+          stopUpdateArray.push(`(${subValues.join(', ')})`);
+          totalLineCount++;
+        }
+
+        // eslint-disable-next-line no-await-in-loop
+        await task.db
+          .run(
+            `REPLACE INTO ${model.stop_times_updates.filenameBase} (${
+              fields.stop_times_updates
+            }) VALUES ${stopUpdateArray.join(', ')}`
+          )
+          .catch((error) => {
+            task.warn('Import error: ' + error.message);
+          });
+      }
+
+      // Special processing for serviceAlerts
+      if (entity.alert) {
+        const alertTargetArray = [];
+        for (const informedEntity of entity.alert.informedEntity) {
+          informedEntity.parent = entity;
+          const subValues = model.service_alert_targets.schema.map((column) =>
+            sqlString.escape(
+              getDescendantProp(informedEntity, column.source, column.default)
+            )
+          );
+          alertTargetArray.push(`(${subValues.join(', ')})`);
+          totalLineCount++;
+        }
+
+        // eslint-disable-next-line no-await-in-loop
+        await task.db
+          .run(
+            `REPLACE INTO ${model.service_alert_targets.filenameBase} (${
+              fields.service_alert_targets
+            }) VALUES ${alertTargetArray.join(', ')}`
+          )
+          .catch((error) => {
+            task.warn('Import error: ' + error.message);
+          });
+      }
+
+      task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
+    }
+  }
+
+  task.log(`GTFS-Realtime data import complete`);
+};
+
 const getTextFiles = async (folderPath) => {
   const files = await readdir(folderPath);
   return files.filter((filename) => filename.slice(-3) === 'txt');
@@ -290,6 +536,12 @@ const importFiles = (task) =>
           return;
         }
 
+        // If the model is a database/gtfs-realtime model then just silently exit as we dont really care here
+        if (model.extension === 'gtfs-realtime') {
+          resolve();
+          return;
+        }
+
         const filepath = path.join(
           task.downloadDir,
           `${model.filenameBase}.txt`
@@ -348,7 +600,7 @@ const importFiles = (task) =>
       })
   );
 
-const importGtfs = async (initialConfig) => {
+export async function importGtfs(initialConfig) {
   const config = setDefaultConfig(initialConfig);
   validateConfigForImport(config);
   const log = _log(config);
@@ -384,7 +636,9 @@ const importGtfs = async (initialConfig) => {
     const task = {
       exclude: agency.exclude,
       agency_url: agency.url,
-      agency_headers: agency.headers || false,
+      headers: agency.headers || false,
+      realtime_headers: agency.realtimeHeaders || false,
+      realtime_urls: agency.realtimeUrls || false,
       downloadDir: path,
       path: agency.path,
       csvOptions: config.csvOptions || {},
@@ -407,11 +661,70 @@ const importGtfs = async (initialConfig) => {
     await readFiles(task);
     await importFiles(task);
 
+    if (task.realtime_urls) {
+      await updateRealtimeData(task);
+    }
+
     cleanup();
-    task.log('Completed GTFS import');
   });
 
-  log(`Completed GTFS import for ${pluralize('file', agencyCount, true)}\n`);
-};
+  log(`Completed GTFS import for ${pluralize('agency', agencyCount, true)}\n`);
+}
+
+export async function updateGtfsRealtime(initialConfig) {
+  const config = setDefaultConfig(initialConfig);
+  validateConfigForImport(config);
+  const log = _log(config);
+  const logError = _logError(config);
+  const logWarning = _logWarning(config);
+  const db = await openDb(config).catch((error) => {
+    if (error instanceof Error && error.code === 'SQLITE_CANTOPEN') {
+      logError(
+        `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
+      );
+    }
+
+    throw error;
+  });
+
+  const agencyCount = config.agencies.length;
+  log(
+    `Starting GTFS-Realtime refresh for ${pluralize(
+      'agencies',
+      agencyCount,
+      true
+    )} using SQLite database at ${config.sqlitePath}`
+  );
+
+  await markRealtimeDataStale(db, log);
 
-export default importGtfs;
+  await mapSeries(config.agencies, async (agency) => {
+    const task = {
+      realtime_headers: agency.realtimeHeaders || false,
+      realtime_urls: agency.realtimeUrls || false,
+      db,
+      log(message, overwrite) {
+        log(message, overwrite);
+      },
+      warn(message) {
+        logWarning(message);
+      },
+      error(message) {
+        logError(message);
+      },
+    };
+
+    if (task.realtime_urls) {
+      await updateRealtimeData(task);
+    }
+  });
+
+  await cleanStaleRealtimeData(db, log);
+  log(
+    `Completed GTFS-Realtime refresh for ${pluralize(
+      'agencies',
+      agencyCount,
+      true
+    )}\n`
+  );
+}

package/lib/log-utils.js

@@ -1,7 +1,7 @@
 import { clearLine, cursorTo } from 'node:readline';
 import PrettyError from 'pretty-error';
 import { noop } from 'lodash-es';
-import chalk from 'chalk';
+import * as colors from 'yoctocolors';
 
 const pe = new PrettyError();
 pe.start();
@@ -60,17 +60,18 @@ export function logError(config) {
  * Format console warning text
  */
 export function formatWarning(text) {
-  return `${chalk.yellow.underline('Warning')}${chalk.yellow(
-    ':'
-  )} ${chalk.yellow(text)}`;
+  const warningMessage = `${colors.underline('Warning')}: ${text}`;
+  return colors.yellow(warningMessage);
 }
 
 /*
  * Format console error text
  */
 export function formatError(error) {
-  const message = error instanceof Error ? error.message : error;
-  return `${chalk.red.underline('Error')}${chalk.red(':')} ${chalk.red(
-    message.replace('Error: ', '')
+  const messageText = error instanceof Error ? error.message : error;
+  const errorMessage = `${colors.underline('Error')}: ${messageText.replace(
+    'Error: ',
+    ''
   )}`;
+  return colors.red(errorMessage);
 }

package/lib/non-standard/trips-dated-vehicle-journey.js

@@ -0,0 +1,30 @@
+import sqlString from 'sqlstring-sqlite';
+
+import { getDb } from '../db.js';
+
+import {
+  formatOrderByClause,
+  formatSelectClause,
+  formatWhereClauses,
+} from '../utils.js';
+import directions from '../../models/non-standard/trips-dated-vehicle-journey.js';
+
+/*
+ * Returns an array of all directions that match the query parameters.
+ */
+export async function getTripsDatedVehicleJourneys(
+  query = {},
+  fields = [],
+  orderBy = [],
+  options = {}
+) {
+  const db = options.db ?? (await getDb());
+  const tableName = sqlString.escapeId(directions.filenameBase);
+  const selectClause = formatSelectClause(fields);
+  const whereClause = formatWhereClauses(query);
+  const orderByClause = formatOrderByClause(orderBy);
+
+  return db.all(
+    `${selectClause} FROM ${tableName} ${whereClause} ${orderByClause};`
+  );
+}

package/lib/utils.js

@@ -1,4 +1,5 @@
 import sqlString from 'sqlstring-sqlite';
+import Long from 'long';
 
 /*
  * Validate configuration.
@@ -33,6 +34,11 @@ export function setDefaultConfig(initialConfig) {
   };
 }
 
+export function convertLongTimeToDate(longDate) {
+  const { high, low, unsigned } = longDate;
+  return new Date(new Long(low, high, unsigned).toInt() * 1000).toISOString();
+}
+
 /*
  * Calculate seconds from midnight for HH:mm:ss
  */
@@ -46,14 +52,33 @@ export function calculateSecondsFromMidnight(time) {
 }
 
 export function formatSelectClause(fields) {
-  const selectItem =
-    fields.length > 0
-      ? fields.map((fieldName) => sqlString.escapeId(fieldName)).join(', ')
-      : '*';
+  if (Array.isArray(fields)) {
+    const selectItem =
+      fields.length > 0
+        ? fields.map((fieldName) => sqlString.escapeId(fieldName)).join(', ')
+        : '*';
+    return `SELECT ${selectItem}`;
+  }
 
+  const selectItem = Object.entries(fields)
+    .map(
+      (key) => `${sqlString.escapeId(key[0])} AS ${sqlString.escapeId(key[1])}`
+    )
+    .join(', ');
   return `SELECT ${selectItem}`;
 }
 
+export function formatJoinClause(joinObject) {
+  return joinObject
+    .map(
+      (data) =>
+        `${data.type ? data.type + ' JOIN' : 'INNER JOIN'} ${sqlString.escapeId(
+          data.table
+        )} ON ${data.on}`
+    )
+    .join(' ');
+}
+
 export function formatWhereClause(key, value) {
   if (Array.isArray(value)) {
     return `${sqlString.escapeId(key)} IN (${value

package/models/gtfs-realtime/service-alert-targets.js

@@ -0,0 +1,38 @@
+const model = {
+  filenameBase: 'service_alert_targets',
+  extension: 'gtfs-realtime',
+  schema: [
+    {
+      name: 'alert_id',
+      type: 'varchar(255)',
+      required: true,
+      primary: true,
+      index: true,
+      source: 'parent.id',
+    },
+    {
+      name: 'stop_id',
+      type: 'varchar(255)',
+      index: true,
+      source: 'stopId',
+      default: null,
+    },
+    {
+      name: 'route_id',
+      type: 'varchar(255)',
+      index: true,
+      source: 'routeId',
+      default: null,
+    },
+    {
+      name: 'isUpdated',
+      type: 'integer',
+      required: true,
+      min: 0,
+      max: 1,
+      default: 1,
+    },
+  ],
+};
+
+export default model;

package/models/gtfs-realtime/service-alerts.js

@@ -0,0 +1,60 @@
+const model = {
+  filenameBase: 'service_alerts',
+  extension: 'gtfs-realtime',
+  schema: [
+    {
+      name: 'id',
+      type: 'varchar(255)',
+      required: true,
+      primary: true,
+      index: true,
+      source: 'id',
+    },
+    {
+      name: 'cause',
+      type: 'integer',
+      required: true,
+      min: 0,
+      source: 'alert.cause',
+      default: 0,
+    },
+    {
+      name: 'start_time',
+      type: 'varchar(255)',
+      required: true,
+      source: 'alert.activePeriod[0].start',
+      default: '',
+    },
+    {
+      name: 'end_time',
+      type: 'varchar(255)',
+      required: true,
+      source: 'alert.activePeriod[0].end',
+      default: '',
+    },
+    {
+      name: 'headline',
+      type: 'varchar(2048)',
+      required: true,
+      source: 'alert.headerText.translation[0].text',
+      default: '',
+    },
+    {
+      name: 'description',
+      type: 'varchar(4096)',
+      required: true,
+      source: 'alert.descriptionText.translation[0].text',
+      default: '',
+    },
+    {
+      name: 'isUpdated',
+      type: 'integer',
+      required: true,
+      min: 0,
+      max: 1,
+      default: 1,
+    },
+  ],
+};
+
+export default model;

package/models/gtfs-realtime/stop-times-updates.js

@@ -0,0 +1,67 @@
+const model = {
+  filenameBase: 'stop_times_updates',
+  extension: 'gtfs-realtime',
+  schema: [
+    {
+      name: 'trip_id',
+      type: 'varchar(255)',
+      index: true,
+      source: 'parent.tripUpdate.trip.tripId',
+      default: null,
+    },
+    {
+      name: 'route_id',
+      type: 'varchar(255)',
+      index: true,
+      source: 'parent.tripUpdate.trip.routeId',
+      default: null,
+    },
+    {
+      name: 'stop_id',
+      type: 'varchar(255)',
+      index: true,
+      source: 'stopId',
+      default: null,
+    },
+    {
+      name: 'stop_sequence',
+      type: 'integer',
+      source: 'stopSequence',
+      default: null,
+    },
+    {
+      name: 'arrival_delay',
+      type: 'integer',
+      source: 'arrival.delay',
+      default: null,
+    },
+    {
+      name: 'departure_delay',
+      type: 'integer',
+      source: 'departure.delay',
+      default: null,
+    },
+    {
+      name: 'departure_timestamp',
+      type: 'varchar(255)',
+      source: 'departure.time',
+      default: null,
+    },
+    {
+      name: 'arrival_timestamp',
+      type: 'varchar(255)',
+      source: 'arrival.time',
+      default: null,
+    },
+    {
+      name: 'isUpdated',
+      type: 'integer',
+      required: true,
+      min: 0,
+      max: 1,
+      default: 1,
+    },
+  ],
+};
+
+export default model;