gtfs 4.11.0 → 4.11.2

package/CHANGELOG.md

@@ -5,6 +5,28 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [4.11.2] - 2024-06-13
+
+### Added
+
+- Support for [locations.geojson](https://gtfs.org/schedule/reference/#locationsgeojson)
+
+### Updated
+
+ - Update sqlite type for all models
+ - Dependency updates
+
+## [4.11.1] - 2024-06-05
+
+### Changed
+
+- Don't throw error on GTFS-Realtime import HTTP error
+
+### Updated
+
+- Remove recursive-copy module
+- Dependency updates
+
 ## [4.11.0] - 2024-06-01
 
 ### Added

package/README.md

@@ -29,27 +29,9 @@ You can use it as a [command-line tool](#command-line-examples) or as a [node.js
 
 This library has four parts: the [GTFS import script](#gtfs-import-script), [GTFS export script](#gtfs-export-script) and [GTFS-Realtime update script](#gtfsrealtime-update-script) and the [query methods](#query-methods)
 
-## Breaking changes in Version 4
-
-Version 4 of node-gtfs switched to using the better-sqlite3 library. This allowed all query methods to become synchronous and speeds up import and export.
-
-- All query methods are now synchronous.
-
-```js
-// Version 3
-const routes = await getRoutes();
-
-// Version 4
-const routes = getRoutes();
-```
-
-- `runRawQuery` has been removed. Use [Raw SQLite Query](#raw-sqlite-query) instead.
-- `execRawQuery` has been removed. Use [Raw SQLite Query](#raw-sqlite-query) instead.
-- `getDb` has been removed. Use `openDb` instead.
-
 ## Installation
 
-To use this library as a command-line utility, install it globally [npm](https://npmjs.org):
+To use this library as a command-line utility, install it globally with [npm](https://npmjs.org):
 
     npm install gtfs -g
 
@@ -1224,6 +1206,17 @@ const locationGroups = getLocationGroupStops({
 });
 ```
 
+#### getLocations(query, fields, sortBy, options)
+
+Returns an array of locations that match query parameters. Each location is text that can be parsed into a geojson object. [Details on locations.geojson](https://gtfs.org/schedule/reference/#locationsgeojson)
+
+```js
+import { getLocations } from 'gtfs';
+
+// Get all locations
+const locations = getLocations();
+```
+
 #### getPathways(query, fields, sortBy, options)
 
 Returns an array of pathways that match query parameters. [Details on pathways.txt](https://gtfs.org/schedule/reference/#pathwaystxt)

package/lib/export.js

@@ -66,52 +66,62 @@ const exportGtfs = async (initialConfig) => {
 
   // Loop through each GTFS file
   const exportedFiles = await mapSeries(models, async (model) => {
-    const filepath = path.join(exportPath, `${model.filenameBase}.txt`);
+    const filePath = path.join(
+      exportPath,
+      `${model.filenameBase}.${model.filenameExtension}`,
+    );
     const tableName = sqlString.escapeId(model.filenameBase);
     const lines = db.prepare(`SELECT * FROM ${tableName};`).all();
 
     if (!lines || lines.length === 0) {
       if (!model.nonstandard) {
-        log(`Skipping (no data) - ${model.filenameBase}.txt\r`);
+        log(
+          `Skipping (no data) - ${model.filenameBase}.${model.filenameExtension}\r`,
+        );
       }
 
       return;
     }
 
-    const excludeColumns = [
-      'id',
-      'arrival_timestamp',
-      'departure_timestamp',
-      'start_timestamp',
-      'end_timestamp',
-      'service_arrival_timestamp',
-      'service_departure_timestamp',
-      'boarding_timestamp',
-      'alighting_timestamp',
-      'ridership_start_timestamp',
-      'ridership_end_timestamp',
-    ];
-
-    // If no routes have values for agency_id, add it to the excludeColumns list
-    if (model.filenameBase === 'routes') {
-      const routesWithAgencyId = db
-        .prepare('SELECT agency_id FROM routes WHERE agency_id IS NOT NULL;')
-        .all();
-      if (!routesWithAgencyId || routesWithAgencyId.length === 0) {
-        excludeColumns.push('agency_id');
+    if (model.filenameExtension === 'txt') {
+      const excludeColumns = [
+        'id',
+        'arrival_timestamp',
+        'departure_timestamp',
+        'start_timestamp',
+        'end_timestamp',
+        'service_arrival_timestamp',
+        'service_departure_timestamp',
+        'boarding_timestamp',
+        'alighting_timestamp',
+        'ridership_start_timestamp',
+        'ridership_end_timestamp',
+      ];
+
+      // If no routes have values for agency_id, add it to the excludeColumns list
+      if (model.filenameBase === 'routes') {
+        const routesWithAgencyId = db
+          .prepare('SELECT agency_id FROM routes WHERE agency_id IS NOT NULL;')
+          .all();
+        if (!routesWithAgencyId || routesWithAgencyId.length === 0) {
+          excludeColumns.push('agency_id');
+        }
       }
-    }
 
-    const columns = without(
-      model.schema.map((column) => column.name),
-      ...excludeColumns,
-    );
-    const fileText = await stringify(lines, { columns, header: true });
-    await writeFile(filepath, fileText);
+      const columns = without(
+        model.schema.map((column) => column.name),
+        ...excludeColumns,
+      );
+      const fileText = await stringify(lines, { columns, header: true });
+      await writeFile(filePath, fileText);
+    } else if (model.filenameExtension === 'geojson') {
+      const fileText = lines?.[0].geojson ?? '';
+      await writeFile(filePath, fileText);
+    }
 
-    log(`Exporting - ${model.filenameBase}.txt\r`);
+    log(`Exporting - ${model.filenameBase}.${model.filenameExtension}\r`);
 
-    return `${model.filenameBase}.txt`;
+    return `${model.filenameBase}.${model.filenameExtension}`;
   });
 
   if (compact(exportedFiles).length === 0) {

package/lib/geojson-utils.js

@@ -9,6 +9,15 @@ import {
 } from 'lodash-es';
 import { feature, featureCollection } from '@turf/helpers';
 
+export function isValidJSON(string) {
+  try {
+    JSON.parse(string);
+  } catch (error) {
+    return false;
+  }
+  return true;
+}
+
 function isValidLineString(lineString) {
   if (!lineString) {
     return false;

package/lib/gtfs/locations.js

@@ -0,0 +1,32 @@
+import sqlString from 'sqlstring-sqlite';
+
+import { openDb } from '../db.js';
+
+import {
+  formatOrderByClause,
+  formatSelectClause,
+  formatWhereClauses,
+} from '../utils.js';
+import locations from '../../models/gtfs/locations.js';
+
+/*
+ * Returns an array of all locations that match the query parameters.
+ */
+export function getLocations(
+  query = {},
+  fields = [],
+  orderBy = [],
+  options = {},
+) {
+  const db = options.db ?? openDb();
+  const tableName = sqlString.escapeId(locations.filenameBase);
+  const selectClause = formatSelectClause(fields);
+  const whereClause = formatWhereClauses(query);
+  const orderByClause = formatOrderByClause(orderBy);
+
+  return db
+    .prepare(
+      `${selectClause} FROM ${tableName} ${whereClause} ${orderByClause};`,
+    )
+    .all();
+}

package/lib/gtfs.js

@@ -22,6 +22,7 @@ import { getFrequencies } from './gtfs/frequencies.js';
 import { getLevels } from './gtfs/levels.js';
 import { getLocationGroups } from './gtfs/location-groups.js';
 import { getLocationGroupStops } from './gtfs/location-group-stops.js';
+import { getLocations } from './gtfs/locations.js';
 import { getNetworks } from './gtfs/networks.js';
 import { getPathways } from './gtfs/pathways.js';
 import { getRouteNetworks } from './gtfs/route-networks.js';
@@ -131,6 +132,9 @@ export { _getLocationGroups as getLocationGroups };
 const _getLocationGroupStops = getLocationGroupStops;
 export { _getLocationGroupStops as getLocationGroupStops };
 
+const _getLocations = getLocations;
+export { _getLocations as getLocations };
+
 const _getNetworks = getNetworks;
 export { _getNetworks as getNetworks };
 

package/lib/import.js

@@ -1,7 +1,6 @@
 import path from 'node:path';
 import { createReadStream, existsSync, lstatSync } from 'node:fs';
-import { readdir, rename, writeFile } from 'node:fs/promises';
-import copy from 'recursive-copy';
+import { cp, readdir, rename, readFile, writeFile } from 'node:fs/promises';
 import fetch from 'node-fetch';
 import { parse } from 'csv-parse';
 import pluralize from 'pluralize';
@@ -15,6 +14,7 @@ import sqlString from 'sqlstring-sqlite';
 import models from '../models/models.js';
 import { openDb } from './db.js';
 import { unzip } from './file-utils.js';
+import { isValidJSON } from './geojson-utils.js';
 import {
   log as _log,
   logError as _logError,
@@ -42,7 +42,7 @@ const downloadFiles = async (task) => {
   });
 
   if (response.status !== 200) {
-    throw new Error('Couldn’t download files');
+    throw new Error(`Unable to download GTFS from ${task.agency_url}`);
   }
 
   const buffer = await response.arrayBuffer();
@@ -51,14 +51,19 @@ const downloadFiles = async (task) => {
   task.log('Download successful');
 };
 
-const downloadGtfsRealtimeData = async (url, headers) => {
+const downloadGtfsRealtimeData = async (url, task) => {
   const response = await fetch(url, {
     method: 'GET',
-    headers: { ...{}, ...headers, ...{ 'Accept-Encoding': 'gzip' } },
+    headers: {
+      ...{},
+      ...task.realtime_headers,
+      ...{ 'Accept-Encoding': 'gzip' },
+    },
   });
 
   if (response.status !== 200) {
-    throw new Error('Couldn’t download files');
+    task.warn(`Unable to download GTFS-Realtime from ${url}`);
+    return null;
   }
 
   const buffer = await response.arrayBuffer();
@@ -190,17 +195,14 @@ const updateRealtimeData = async (task) => {
   for (const realtimeUrl of task.realtime_urls) {
     task.log(`Downloading GTFS-Realtime from ${realtimeUrl}`);
     // eslint-disable-next-line no-await-in-loop
-    const gtfsRealtimeData = await downloadGtfsRealtimeData(
-      realtimeUrl,
-      task.realtime_headers,
-    );
-
-    task.log(`Download successful`);
+    const gtfsRealtimeData = await downloadGtfsRealtimeData(realtimeUrl, task);
 
-    if (!gtfsRealtimeData.entity) {
+    if (!gtfsRealtimeData?.entity) {
       continue;
     }
 
+    task.log(`Download successful`);
+
     let totalLineCount = 0;
 
     for (const entity of gtfsRealtimeData.entity) {
@@ -353,7 +355,7 @@ const readFiles = async (task) => {
   } else {
     // Local file is unzipped, just copy it from there.
     try {
-      await copy(gtfsPath, task.downloadDir);
+      await cp(gtfsPath, task.downloadDir, { recursive: true });
     } catch {
       throw new Error(
         `Unable to load files from path \`${gtfsPath}\` defined in configuration. Verify that path exists and contains GTFS files.`,
@@ -444,7 +446,7 @@ const formatLine = (line, model, totalLineCount) => {
       formattedLine[columnSchema.name] === null
     ) {
       throw new Error(
-        `Missing required value in ${model.filenameBase}.txt for ${columnSchema.name} on line ${lineNumber}.`,
+        `Missing required value in ${model.filenameBase}.${model.filenameExtension} for ${columnSchema.name} on line ${lineNumber}.`,
       );
     }
 
@@ -454,7 +456,7 @@ const formatLine = (line, model, totalLineCount) => {
       formattedLine[columnSchema.name] < columnSchema.min
     ) {
       throw new Error(
-        `Invalid value in ${model.filenameBase}.txt for ${columnSchema.name} on line ${lineNumber}: below minimum value of ${columnSchema.min}.`,
+        `Invalid value in ${model.filenameBase}.${model.filenameExtension} for ${columnSchema.name} on line ${lineNumber}: below minimum value of ${columnSchema.min}.`,
       );
     }
 
@@ -464,7 +466,7 @@ const formatLine = (line, model, totalLineCount) => {
       formattedLine[columnSchema.name] > columnSchema.max
     ) {
       throw new Error(
-        `Invalid value in ${model.filenameBase}.txt for ${columnSchema.name} on line ${lineNumber}: above maximum value of ${columnSchema.max}.`,
+        `Invalid value in ${model.filenameBase}.${model.filenameExtension} for ${columnSchema.name} on line ${lineNumber}: above maximum value of ${columnSchema.max}.`,
       );
     }
   }
@@ -538,12 +540,12 @@ const importLines = (task, lines, model, totalLineCount) => {
     if (error.code === 'SQLITE_CONSTRAINT_PRIMARYKEY') {
       const primaryColumns = model.schema.filter((column) => column.primary);
       task.warn(
-        `Duplicate values for primary key (${primaryColumns.map((column) => column.name).join(', ')}) found in ${model.filenameBase}.txt. Set the \`ignoreDuplicates\` option to true in config.json to ignore this error`,
+        `Duplicate values for primary key (${primaryColumns.map((column) => column.name).join(', ')}) found in ${model.filenameBase}.${model.filenameExtension}. Set the \`ignoreDuplicates\` option to true in config.json to ignore this error`,
       );
     }
 
     task.warn(
-      `Check ${model.filenameBase}.txt for invalid data between lines ${
+      `Check ${model.filenameBase}.${model.filenameExtension} for invalid data between lines ${
         totalLineCount - linesToImportCount
       } and ${totalLineCount}.`,
     );
@@ -551,7 +553,7 @@ const importLines = (task, lines, model, totalLineCount) => {
   }
 
   task.log(
-    `Importing - ${model.filenameBase}.txt - ${totalLineCount} lines imported\r`,
+    `Importing - ${model.filenameBase}.${model.filenameExtension} - ${totalLineCount} lines imported\r`,
     true,
   );
 };
@@ -561,10 +563,16 @@ const importFiles = (task) =>
     models,
     (model) =>
       new Promise((resolve, reject) => {
+        const lines = [];
+        let totalLineCount = 0;
+        const maxInsertVariables = 32_000;
+
         // Loop through each GTFS file
         // Filter out excluded files from config
         if (task.exclude && task.exclude.includes(model.filenameBase)) {
-          task.log(`Skipping - ${model.filenameBase}.txt\r`);
+          task.log(
+            `Skipping - ${model.filenameBase}.${model.filenameExtension}\r`,
+          );
           resolve();
           return;
         }
@@ -577,23 +585,43 @@ const importFiles = (task) =>
 
         const filepath = path.join(
           task.downloadDir,
-          `${model.filenameBase}.txt`,
+          `${model.filenameBase}.${model.filenameExtension}`,
         );
 
         if (!existsSync(filepath)) {
+          // Log only missing standard GTFS files
           if (!model.nonstandard) {
-            task.log(`Importing - ${model.filenameBase}.txt - No file found\r`);
+            task.log(
+              `Importing - ${model.filenameBase}.${model.filenameExtension} - No file found\r`,
+            );
           }
 
           resolve();
           return;
         }
 
-        task.log(`Importing - ${model.filenameBase}.txt\r`);
+        task.log(
+          `Importing - ${model.filenameBase}.${model.filenameExtension}\r`,
+        );
+
+        // Handle geojson files
+        if (model.filenameExtension === 'geojson') {
+          readFile(filepath, 'utf8')
+            .then((data) => {
+              if (isValidJSON(data) === false) {
+                reject(
+                  new Error(
+                    `Invalid JSON in ${model.filenameBase}.${model.filenameExtension}`,
+                  ),
+                );
+              }
+              const line = formatLine({ geojson: data }, model, totalLineCount);
+              importLines(task, [line], model, totalLineCount);
+              resolve();
+            })
+            .catch(reject);
+        }
 
-        const lines = [];
-        let totalLineCount = 0;
-        const maxInsertVariables = 32_000;
         const parser = parse({
           columns: true,
           relax_quotes: true,

package/models/gtfs/agency.js

@@ -1,45 +1,46 @@
 const model = {
   filenameBase: 'agency',
+  filenameExtension: 'txt',
   schema: [
     {
       name: 'agency_id',
-      type: 'varchar(255)',
+      type: 'text',
       primary: true,
       prefix: true,
     },
     {
       name: 'agency_name',
-      type: 'varchar(255)',
+      type: 'text',
       required: true,
       nocase: true,
     },
     {
       name: 'agency_url',
-      type: 'varchar(2047)',
+      type: 'text',
       required: true,
     },
     {
       name: 'agency_timezone',
-      type: 'varchar(255)',
+      type: 'text',
       required: true,
     },
     {
       name: 'agency_lang',
-      type: 'varchar(255)',
+      type: 'text',
       nocase: true,
     },
     {
       name: 'agency_phone',
-      type: 'varchar(64)',
+      type: 'text',
       nocase: true,
     },
     {
       name: 'agency_fare_url',
-      type: 'varchar(2047)',
+      type: 'text',
     },
     {
       name: 'agency_email',
-      type: 'varchar(255)',
+      type: 'text',
       nocase: true,
     },
   ],

package/models/gtfs/areas.js

@@ -1,16 +1,17 @@
 const model = {
   filenameBase: 'areas',
+  filenameExtension: 'txt',
   schema: [
     {
       name: 'area_id',
-      type: 'varchar(255)',
+      type: 'text',
       required: true,
       primary: true,
       prefix: true,
     },
     {
       name: 'area_name',
-      type: 'varchar(255)',
+      type: 'text',
     },
   ],
 };

package/models/gtfs/attributions.js

@@ -1,31 +1,32 @@
 const model = {
   filenameBase: 'attributions',
+  filenameExtension: 'txt',
   schema: [
     {
       name: 'attribution_id',
-      type: 'varchar(255)',
+      type: 'text',
       primary: true,
       required: true,
       prefix: true,
     },
     {
       name: 'agency_id',
-      type: 'varchar(255)',
+      type: 'text',
       prefix: true,
     },
     {
       name: 'route_id',
-      type: 'varchar(255)',
+      type: 'text',
       prefix: true,
     },
     {
       name: 'trip_id',
-      type: 'varchar(255)',
+      type: 'text',
       prefix: true,
     },
     {
       name: 'organization_name',
-      type: 'varchar(255)',
+      type: 'text',
       required: true,
       nocase: true,
     },
@@ -49,16 +50,16 @@ const model = {
     },
     {
       name: 'attribution_url',
-      type: 'varchar(2047)',
+      type: 'text',
     },
     {
       name: 'attribution_email',
-      type: 'varchar(255)',
+      type: 'text',
       nocase: true,
     },
     {
       name: 'attribution_phone',
-      type: 'varchar(255)',
+      type: 'text',
       nocase: true,
     },
   ],

package/models/gtfs/booking-rules.js

@@ -1,9 +1,10 @@
 const model = {
   filenameBase: 'booking_rules',
+  filenameExtension: 'txt',
   schema: [
     {
       name: 'booking_rule_id',
-      type: 'varchar(255)',
+      type: 'text',
       primary: true,
       prefix: true,
     },
@@ -31,7 +32,7 @@ const model = {
     },
     {
       name: 'prior_notice_last_time',
-      type: 'varchar(255)',
+      type: 'text',
     },
     {
       name: 'prior_notice_last_timestamp',
@@ -45,7 +46,7 @@ const model = {
     },
     {
       name: 'prior_notice_start_time',
-      type: 'varchar(255)',
+      type: 'text',
     },
     {
       name: 'prior_notice_start_timestamp',
@@ -54,36 +55,36 @@ const model = {
     },
     {
       name: 'prior_notice_service_id',
-      type: 'varchar(255)',
+      type: 'text',
       prefix: true,
     },
     {
       name: 'message',
-      type: 'varchar(2047)',
+      type: 'text',
       nocase: true,
     },
     {
       name: 'pickup_message',
-      type: 'varchar(2047)',
+      type: 'text',
       nocase: true,
     },
     {
       name: 'drop_off_message',
-      type: 'varchar(2047)',
+      type: 'text',
       nocase: true,
     },
     {
       name: 'phone_number',
-      type: 'varchar(64)',
+      type: 'text',
       nocase: true,
     },
     {
       name: 'info_url',
-      type: 'varchar(2047)',
+      type: 'text',
     },
     {
       name: 'booking_url',
-      type: 'varchar(2047)',
+      type: 'text',
     },
   ],
 };

package/models/gtfs/calendar-dates.js

@@ -1,9 +1,10 @@
 const model = {
   filenameBase: 'calendar_dates',
+  filenameExtension: 'txt',
   schema: [
     {
       name: 'service_id',
-      type: 'varchar(255)',
+      type: 'text',
       required: true,
       primary: true,
       prefix: true,
@@ -24,7 +25,7 @@ const model = {
     },
     {
       name: 'holiday_name',
-      type: 'varchar(255)',
+      type: 'text',
       nocase: true,
     },
   ],

package/models/gtfs/calendar.js

@@ -1,9 +1,10 @@
 const model = {
   filenameBase: 'calendar',
+  filenameExtension: 'txt',
   schema: [
     {
       name: 'service_id',
-      type: 'varchar(255)',
+      type: 'text',
       required: true,
       primary: true,
       prefix: true,