gtfs 4.11.2 → 4.12.0

package/.github/workflows/nodejs.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [ 20, 21 ]
+        node: [ 20, 22 ]
     name: Use Node.js ${{ matrix.node }}
     steps:
       - uses: actions/checkout@v4

package/CHANGELOG.md

@@ -5,6 +5,23 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [4.12.0] - 2024-06-17
+
+### Added
+
+- Support for `service_id` in `getRoutes`
+
+### Changed
+
+- Use tempy for temporary directory creation
+
+## [4.11.3] - 2024-06-13
+
+### Fixed
+
+- Better tests for locations.geojson
+- Handle unknown filename extensions in models
+
 ## [4.11.2] - 2024-06-13
 
 ### Added

package/lib/export.js

@@ -117,6 +117,10 @@ const exportGtfs = async (initialConfig) => {
     } else if (model.filenameExtension === 'geojson') {
       const fileText = lines?.[0].geojson ?? '';
       await writeFile(filePath, fileText);
+    } else {
+      throw new Error(
+        `Unexpected filename extension: ${model.filenameExtension}`,
+      );
     }
 
     log(`Exporting - ${model.filenameBase}.${model.filenameExtension}\r`);

package/lib/gtfs/routes.js

@@ -17,14 +17,30 @@ function buildStoptimeSubquery(query) {
 }
 
 function buildTripSubquery(query) {
-  return `SELECT DISTINCT route_id FROM trips WHERE trip_id IN (${buildStoptimeSubquery(
-    query,
-  )})`;
+  let whereClause = '';
+  const tripQuery = omit(query, ['stop_id']);
+  const stoptimeQuery = pick(query, ['stop_id']);
+
+  const whereClauses = Object.entries(tripQuery).map(([key, value]) =>
+    formatWhereClause(key, value),
+  );
+
+  if (Object.values(stoptimeQuery).length > 0) {
+    whereClauses.push(`trip_id IN (${buildStoptimeSubquery(stoptimeQuery)})`);
+  }
+
+  if (whereClauses.length > 0) {
+    whereClause = `WHERE ${whereClauses.join(' AND ')}`;
+  }
+
+  return `SELECT DISTINCT route_id FROM trips ${whereClause}`;
 }
 
 /*
  * Returns an array of routes that match the query parameters. A `stop_id`
  * query parameter may be passed to find all routes that contain that stop.
+ * A `service_id` query parameter may be passed to limit routes to specific
+ * calendars.
  */
 export function getRoutes(query = {}, fields = [], orderBy = [], options = {}) {
   const db = options.db ?? openDb();
@@ -33,15 +49,15 @@ export function getRoutes(query = {}, fields = [], orderBy = [], options = {}) {
   let whereClause = '';
   const orderByClause = formatOrderByClause(orderBy);
 
-  const routeQuery = omit(query, ['stop_id']);
-  const stoptimeQuery = pick(query, ['stop_id']);
+  const routeQuery = omit(query, ['stop_id', 'service_id']);
+  const tripQuery = pick(query, ['stop_id', 'service_id']);
 
   const whereClauses = Object.entries(routeQuery).map(([key, value]) =>
     formatWhereClause(key, value),
   );
 
-  if (Object.values(stoptimeQuery).length > 0) {
-    whereClauses.push(`route_id IN (${buildTripSubquery(stoptimeQuery)})`);
+  if (Object.values(tripQuery).length > 0) {
+    whereClauses.push(`route_id IN (${buildTripSubquery(tripQuery)})`);
   }
 
   if (whereClauses.length > 0) {

package/lib/import.js

@@ -1,11 +1,11 @@
 import path from 'node:path';
 import { createReadStream, existsSync, lstatSync } from 'node:fs';
-import { cp, readdir, rename, readFile, writeFile } from 'node:fs/promises';
+import { cp, readdir, rename, readFile, rm, writeFile } from 'node:fs/promises';
 import fetch from 'node-fetch';
 import { parse } from 'csv-parse';
 import pluralize from 'pluralize';
 import stripBomStream from 'strip-bom-stream';
-import { dir } from 'tmp-promise';
+import { temporaryDirectory } from 'tempy';
 import untildify from 'untildify';
 import mapSeries from 'promise-map-series';
 import GtfsRealtimeBindings from 'gtfs-realtime-bindings';
@@ -604,8 +604,46 @@ const importFiles = (task) =>
           `Importing - ${model.filenameBase}.${model.filenameExtension}\r`,
         );
 
-        // Handle geojson files
-        if (model.filenameExtension === 'geojson') {
+        if (model.filenameExtension === 'txt') {
+          const parser = parse({
+            columns: true,
+            relax_quotes: true,
+            trim: true,
+            skip_empty_lines: true,
+            ...task.csvOptions,
+          });
+
+          parser.on('readable', () => {
+            let record;
+
+            while ((record = parser.read())) {
+              try {
+                totalLineCount += 1;
+                lines.push(formatLine(record, model, totalLineCount));
+                // If we have a bunch of lines ready to insert, then do it
+                if (lines.length >= maxInsertVariables / model.schema.length) {
+                  importLines(task, lines, model, totalLineCount);
+                }
+              } catch (error) {
+                reject(error);
+              }
+            }
+          });
+
+          parser.on('end', () => {
+            try {
+              // Insert all remaining lines
+              importLines(task, lines, model, totalLineCount);
+            } catch (error) {
+              reject(error);
+            }
+            resolve();
+          });
+
+          parser.on('error', reject);
+
+          createReadStream(filepath).pipe(stripBomStream()).pipe(parser);
+        } else if (model.filenameExtension === 'geojson') {
           readFile(filepath, 'utf8')
             .then((data) => {
               if (isValidJSON(data) === false) {
@@ -620,46 +658,11 @@ const importFiles = (task) =>
               resolve();
             })
             .catch(reject);
+        } else {
+          reject(
+            new Error(`Unsupported file type: ${model.filenameExtension}`),
+          );
         }
-
-        const parser = parse({
-          columns: true,
-          relax_quotes: true,
-          trim: true,
-          skip_empty_lines: true,
-          ...task.csvOptions,
-        });
-
-        parser.on('readable', () => {
-          let record;
-
-          while ((record = parser.read())) {
-            try {
-              totalLineCount += 1;
-              lines.push(formatLine(record, model, totalLineCount));
-              // If we have a bunch of lines ready to insert, then do it
-              if (lines.length >= maxInsertVariables / model.schema.length) {
-                importLines(task, lines, model, totalLineCount);
-              }
-            } catch (error) {
-              reject(error);
-            }
-          }
-        });
-
-        parser.on('end', () => {
-          try {
-            // Insert all remaining lines
-            importLines(task, lines, model, totalLineCount);
-          } catch (error) {
-            reject(error);
-          }
-          resolve();
-        });
-
-        parser.on('error', reject);
-
-        createReadStream(filepath).pipe(stripBomStream()).pipe(parser);
       }),
   );
 
@@ -684,7 +687,7 @@ export async function importGtfs(initialConfig) {
     createTables(db);
 
     await mapSeries(config.agencies, async (agency) => {
-      const { path, cleanup } = await dir({ unsafeCleanup: true });
+      const tempPath = temporaryDirectory();
 
       const task = {
         exclude: agency.exclude,
@@ -692,7 +695,7 @@ export async function importGtfs(initialConfig) {
         headers: agency.headers || false,
         realtime_headers: agency.realtimeHeaders || false,
         realtime_urls: agency.realtimeUrls || false,
-        downloadDir: path,
+        downloadDir: tempPath,
         downloadTimeout: config.downloadTimeout,
         path: agency.path,
         csvOptions: config.csvOptions || {},
@@ -716,7 +719,7 @@ export async function importGtfs(initialConfig) {
           await updateRealtimeData(task);
         }
 
-        cleanup();
+        await rm(tempPath, { recursive: true });
       } catch (error) {
         if (config.ignoreErrors) {
           logError(error.message);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gtfs",
-  "version": "4.11.2",
+  "version": "4.12.0",
   "description": "Import GTFS transit data into SQLite and query routes, stops, times, fares and more",
   "keywords": [
     "transit",
@@ -91,7 +91,7 @@
     "sanitize-filename": "^1.6.3",
     "sqlstring-sqlite": "^0.1.1",
     "strip-bom-stream": "^5.0.0",
-    "tmp-promise": "^3.0.3",
+    "tempy": "^3.1.0",
     "untildify": "^5.0.0",
     "yargs": "^17.7.2",
     "yoctocolors": "^2.0.2"

package/test/mocha/get-locations.js

@@ -17,7 +17,6 @@ const locationsConfig = {
     },
   ],
   verbose: false,
-  sqlitePath: '/tmp/locations',
 };
 
 describe('getLocations():', () => {
@@ -25,7 +24,6 @@ describe('getLocations():', () => {
     openDb(locationsConfig);
 
     // Add locations.geojson to test GTFS dataset
-    const temporaryDir = path.join(import.meta.dirname, '../fixture/tmp2/');
     await prepDirectory(temporaryDir);
     await unzip(config.agencies[0].path, temporaryDir);