gtfs 4.7.1 → 4.8.0

package/@types/index.d.ts

@@ -63,6 +63,11 @@ export interface DbConfig {
    * A path to a SQLite database. Defaults to using an in-memory database.
    */
   sqlitePath?: string;
+
+  /**
+   * A better-sqlite3 database object. If provided, sqlitePath will be ignored.
+   */
+  db?: Database.Database;
 }
 
 export interface ExportConfig extends DbConfig, VerboseConfig {
@@ -114,6 +119,16 @@ export interface ImportConfig extends DbConfig, VerboseConfig {
    * Options passed to csv-parse for parsing GTFS CSV files.
    */
   csvOptions?: CsvParse.Options;
+
+  /**
+   * A timeout in milliseconds for downloading GTFS files. Defaults to no timeout.
+   */
+  downloadTimeout?: number;
+
+  /**
+   * Whether or not to ignore errors during import. Defaults to false.
+   */
+  ignoreErrors?: boolean;
 }
 
 export interface QueryOptions {

package/CHANGELOG.md

@@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [4.8.0] - 2024-03-10
+
+### Added
+
+- `ignoreErrors` config option
+- `downloadTimeout` config option
+- `db` config option
+
+## [4.7.2] - 2024-03-05
+
+### Updated
+
+- Show info about ignoreDuplicates option when primary key constraint issue happens
+- Dependency updates
+
 ## [4.7.1] - 2024-02-18
 
 ### Updated

package/README.md

@@ -161,14 +161,17 @@ Copy `config-sample.json` to `config.json` and then add your projects configurat
 
     cp config-sample.json config.json
 
-| option                                  | type    | description                                                                                                                            |
-| --------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------- |
-| [`agencies`](#agencies)                 | array   | An array of GTFS files to be imported, and which files to exclude.                                                                     |
-| [`csvOptions`](#csvOptions)             | object  | Options passed to `csv-parse` for parsing GTFS CSV files. Optional.                                                                    |
-| [`exportPath`](#exportPath)             | string  | A path to a directory to put exported GTFS files. Optional, defaults to `gtfs-export/<agency_name>`.                                   |
-| [`ignoreDuplicates`](#ignoreduplicates) | boolean | Whether or not to ignore unique constraints on ids when importing GTFS, such as `trip_id`, `calendar_id`. Optional, defaults to false. |
-| [`sqlitePath`](#sqlitePath)             | string  | A path to an SQLite database. Optional, defaults to using an in-memory database.                                                       |
-| [`verbose`](#verbose)                   | boolean | Whether or not to print output to the console. Optional, defaults to true.                                                             |
+| option                                  | type              | description                                                                                                                                                         |
+| --------------------------------------- | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [`agencies`](#agencies)                 | array             | An array of GTFS files to be imported, and which files to exclude.                                                                                                  |
+| [`csvOptions`](#csvOptions)             | object            | Options passed to `csv-parse` for parsing GTFS CSV files. Optional.                                                                                                 |
+| [`db`](#db)                             | database instance | An existing database instance to use instead of relying on node-gtfs to connect. Optional.                                                                         |
+| [`downloadTimeout`](#downloadtimeout)   | integer           | The number of milliseconds to wait before throwing an error when downloading GTFS. Optional.                                                                        |
+| [`exportPath`](#exportPath)             | string            | A path to a directory to put exported GTFS files. Optional, defaults to `gtfs-export/<agency_name>`.                                                                |
+| [`ignoreDuplicates`](#ignoreduplicates) | boolean           | Whether or not to ignore unique constraints on ids when importing GTFS, such as `trip_id`, `calendar_id`. Optional, defaults to false.                              |
+| [`ignoreErrors`](#ignoreerrors)         | boolean           | Whether or not to ignore errors during the import process. If true, when importing multiple agencies, failed agencies will be skipped. Optional, defaults to false. |
+| [`sqlitePath`](#sqlitePath)             | string            | A path to an SQLite database. Optional, defaults to using an in-memory database.                                                                                    |
+| [`verbose`](#verbose)                   | boolean           | Whether or not to print output to the console. Optional, defaults to true.                                                                                          |
 
 ### agencies
 
@@ -320,6 +323,60 @@ For instance, if you wanted to skip importing invalid lines in the GTFS file:
 
 See [full list of options](https://csv.js.org/parse/options/).
 
+### db
+
+{Database Instance} When passing configuration to `importGtfs` in javascript, you can pass a `db` parameter with an existing database instance. This is not possible using a json configuration file Optional.
+
+```js
+// Using better-sqlite3 to open database
+import { importGtfs } from 'gtfs';
+import Database from 'better-sqlite3';
+
+const db = new Database('/path/to/database');
+
+importGtfs({
+  agencies: [
+    {
+      path: '/path/to/the/unzipped/gtfs/',
+    },
+  ],
+  db: db,
+});
+```
+
+```js
+// Using `openDb` from node-gtfs to open database
+import { importGtfs, openDb } from 'gtfs';
+
+const db = openDb({
+  sqlitePath: '/path/to/database',
+});
+
+importGtfs({
+  agencies: [
+    {
+      path: '/path/to/the/unzipped/gtfs/',
+    },
+  ],
+  db: db,
+});
+```
+
+### downloadTimeout
+
+{Integer} A number of milliseconds to wait when downloading GTFS before throwing an error. Optional.
+
+```json
+{
+  "agencies": [
+    {
+      "path": "/path/to/the/unzipped/gtfs/"
+    }
+  ],
+  "downloadTimeout": 5000
+}
+```
+
 ### exportPath
 
 {String} A path to a directory to put exported GTFS files. If the directory does not exist, it will be created. Used when running `gtfs-export` script or `exportGtfs()`. Optional, defaults to `gtfs-export/<agency_name>` where `<agency_name>` is a sanitized, [snake-cased](https://en.wikipedia.org/wiki/Snake_case) version of the first `agency_name` in `agency.txt`.
@@ -350,6 +407,21 @@ See [full list of options](https://csv.js.org/parse/options/).
 }
 ```
 
+### ignoreErrors
+
+{Boolean} When importing GTFS from multiple agencies, if you don't want node-GTFS to throw an error and instead skip failed GTFS importants proceed to the next agency. Defaults to `false`.
+
+```json
+{
+  "agencies": [
+    {
+      "path": "/path/to/the/unzipped/gtfs/"
+    }
+  ],
+  "ignoreErrors": true
+}
+```
+
 ### sqlitePath
 
 {String} A path to an SQLite database. Optional, defaults to using an in-memory database with a value of `:memory:`.

package/config-sample-full.json

@@ -12,7 +12,9 @@
   "csvOptions": {
     "skip_lines_with_error": true
   },
+  "downloadTimeout": 5000,
   "ignoreDuplicates": false,
+  "ignoreErrors": false,
   "sqlitePath": "/tmp/gtfs",
   "exportPath": "~/path/to/export/gtfs"
 }

package/lib/db.js

@@ -17,7 +17,10 @@ function setupDb(sqlitePath) {
 export function openDb(config) {
   // If config is passed, use that to open or return db
   if (config) {
-    const { sqlitePath } = setDefaultConfig(config);
+    const { sqlitePath, db } = setDefaultConfig(config);
+    if (db) {
+      return db;
+    }
 
     if (dbs[sqlitePath]) {
       return dbs[sqlitePath];
@@ -34,7 +37,7 @@ export function openDb(config) {
 
   if (Object.keys(dbs).length > 1) {
     throw new Error(
-      'Multiple databases open, please specify which one to use.'
+      'Multiple databases open, please specify which one to use.',
     );
   }
 
@@ -44,14 +47,14 @@ export function openDb(config) {
 export function closeDb(db) {
   if (Object.keys(dbs).length === 0) {
     throw new Error(
-      'No database connection. Call `openDb(config)` before using any methods.'
+      'No database connection. Call `openDb(config)` before using any methods.',
     );
   }
 
   if (!db) {
     if (Object.keys(dbs).length > 1) {
       throw new Error(
-        'Multiple database connections. Pass the db you want to close as a parameter to `closeDb`.'
+        'Multiple database connections. Pass the db you want to close as a parameter to `closeDb`.',
       );
     }
 

package/lib/import.js

@@ -36,6 +36,9 @@ const downloadFiles = async (task) => {
   const response = await fetch(task.agency_url, {
     method: 'GET',
     headers: task.headers || {},
+    signal: task.downloadTimeout
+      ? AbortSignal.timeout(task.downloadTimeout)
+      : undefined,
   });
 
   if (response.status !== 200) {
@@ -509,6 +512,13 @@ const importLines = (task, lines, model, totalLineCount) => {
         .join(', ')}) VALUES ${placeholders.join(',')}`,
     ).run(...values);
   } catch (error) {
+    if (error.code === 'SQLITE_CONSTRAINT_PRIMARYKEY') {
+      const primaryColumns = model.schema.filter((column) => column.primary);
+      task.warn(
+        `Duplicate values for primary key (${primaryColumns.map((column) => column.name).join(', ')}) found in ${model.filenameBase}.txt. Set the \`ignoreDuplicates\` option to true in config.json to ignore this error`,
+      );
+    }
+
     task.warn(
       `Check ${model.filenameBase}.txt for invalid data between lines ${
         totalLineCount - linesToImportCount
@@ -587,8 +597,12 @@ const importFiles = (task) =>
         });
 
         parser.on('end', () => {
-          // Insert all remaining lines
-          importLines(task, lines, model, totalLineCount);
+          try {
+            // Insert all remaining lines
+            importLines(task, lines, model, totalLineCount);
+          } catch (error) {
+            reject(error);
+          }
           resolve();
         });
 
@@ -628,6 +642,7 @@ export async function importGtfs(initialConfig) {
         realtime_headers: agency.realtimeHeaders || false,
         realtime_urls: agency.realtimeUrls || false,
         downloadDir: path,
+        downloadTimeout: config.downloadTimeout,
         path: agency.path,
         csvOptions: config.csvOptions || {},
         ignoreDuplicates: config.ignoreDuplicates,
@@ -638,18 +653,26 @@ export async function importGtfs(initialConfig) {
         error: logError,
       };
 
-      if (task.agency_url) {
-        await downloadFiles(task);
-      }
+      try {
+        if (task.agency_url) {
+          await downloadFiles(task);
+        }
 
-      await readFiles(task);
-      await importFiles(task);
+        await readFiles(task);
+        await importFiles(task);
 
-      if (task.realtime_urls) {
-        await updateRealtimeData(task);
-      }
+        if (task.realtime_urls) {
+          await updateRealtimeData(task);
+        }
 
-      cleanup();
+        cleanup();
+      } catch (error) {
+        if (config.ignoreErrors) {
+          logError(error.message);
+        } else {
+          throw error;
+        }
+      }
     });
 
     log(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gtfs",
-  "version": "4.7.1",
+  "version": "4.8.0",
   "description": "Import GTFS transit data into SQLite and query routes, stops, times, fares and more",
   "keywords": [
     "transit",
@@ -60,7 +60,8 @@
     "David Abell",
     "Matthias Feist <matze@matf.de>",
     "Oliv4945",
-    "Kyle Ramey"
+    "Kyle Ramey",
+    "Anton Bracke"
   ],
   "type": "module",
   "main": "index.js",
@@ -75,9 +76,9 @@
   },
   "dependencies": {
     "@turf/helpers": "^6.5.0",
-    "better-sqlite3": "^9.4.1",
-    "csv-parse": "^5.5.3",
-    "csv-stringify": "^6.4.5",
+    "better-sqlite3": "^9.4.3",
+    "csv-parse": "^5.5.5",
+    "csv-stringify": "^6.4.6",
     "gtfs-realtime-bindings": "^1.1.1",
     "lodash-es": "^4.17.21",
     "long": "^5.2.3",
@@ -93,7 +94,7 @@
     "tmp-promise": "^3.0.3",
     "untildify": "^5.0.0",
     "yargs": "^17.7.2",
-    "yoctocolors": "^1.0.0"
+    "yoctocolors": "^2.0.0"
   },
   "devDependencies": {
     "husky": "^9.0.11",

package/test/mocha/import-gtfs.js

@@ -30,7 +30,7 @@ const agenciesFixturesLocal = [
   {
     path: path.join(
       path.dirname(fileURLToPath(import.meta.url)),
-      '../fixture/caltrain_20160406.zip'
+      '../fixture/caltrain_20160406.zip',
     ),
   },
 ];
@@ -58,6 +58,18 @@ describe('importGtfs():', function () {
       routes.length.should.equal(4);
     });
 
+    it('should be able to download and import from HTTP with a downloadTimeout', async () => {
+      try {
+        await importGtfs({
+          ...config,
+          agencies: agenciesFixturesRemote,
+          downloadTimeout: 1,
+        });
+      } catch (error) {
+        error.name.should.equal('AbortError');
+      }
+    });
+
     it('should be able to download and import from local filesystem', async () => {
       await importGtfs({
         ...config,
@@ -105,7 +117,7 @@ describe('importGtfs():', function () {
     const countData = {};
     const temporaryDir = path.join(
       path.dirname(fileURLToPath(import.meta.url)),
-      '../fixture/tmp/'
+      '../fixture/tmp/',
     );
 
     before(async () => {
@@ -135,7 +147,7 @@ describe('importGtfs():', function () {
               }
 
               countData[model.filenameBase] = data.length;
-            }
+            },
           );
 
           return createReadStream(filePath)
@@ -144,7 +156,7 @@ describe('importGtfs():', function () {
               countData[model.collection] = 0;
               throw new Error(error);
             });
-        })
+        }),
       );
 
       await importGtfs({

package/test/mocha/open-db.js

@@ -7,14 +7,25 @@ import config from '../test-config.js';
 import { openDb, closeDb, importGtfs, getShapes } from '../../index.js';
 
 const db2Config = {
-  ...config,
   agencies: [
     {
       ...config.agencies[0],
       exclude: ['shapes'],
     },
   ],
-  sqlitePath: './tmpdb',
+  verbose: false,
+  sqlitePath: './tmpdb2',
+};
+
+const db3Config = {
+  agencies: [
+    {
+      ...config.agencies[0],
+      exclude: ['shapes'],
+    },
+  ],
+  verbose: false,
+  sqlitePath: './tmpdb3',
 };
 
 describe('openDb():', () => {
@@ -31,6 +42,11 @@ describe('openDb():', () => {
     const db2 = openDb(db2Config);
     closeDb(db2);
     fs.unlinkSync(db2Config.sqlitePath);
+
+    // Close db3 and then delete it
+    const db3 = openDb(db3Config);
+    closeDb(db3);
+    fs.unlinkSync(db3Config.sqlitePath);
   });
 
   it('should allow raw db queries: calendar_dates', () => {
@@ -40,7 +56,7 @@ describe('openDb():', () => {
       .prepare(
         `SELECT * FROM calendar_dates WHERE exception_type = 1 AND service_id NOT IN (${serviceIds
           .map((serviceId) => `'${serviceId}'`)
-          .join(', ')})`
+          .join(', ')})`,
       )
       .all();
 
@@ -55,7 +71,7 @@ describe('openDb():', () => {
     const db = openDb();
     const results = db
       .prepare(
-        'SELECT * from trips where trips.trip_id IN (SELECT start_stop_times.trip_id FROM stop_times as start_stop_times WHERE stop_id = ? AND start_stop_times.stop_sequence < (SELECT end_stop_times.stop_sequence FROM stop_times as end_stop_times WHERE end_stop_times.stop_sequence > start_stop_times.stop_sequence AND end_stop_times.trip_id = start_stop_times.trip_id AND end_stop_times.stop_id = ? ))'
+        'SELECT * from trips where trips.trip_id IN (SELECT start_stop_times.trip_id FROM stop_times as start_stop_times WHERE stop_id = ? AND start_stop_times.stop_sequence < (SELECT end_stop_times.stop_sequence FROM stop_times as end_stop_times WHERE end_stop_times.stop_sequence > start_stop_times.stop_sequence AND end_stop_times.trip_id = start_stop_times.trip_id AND end_stop_times.stop_id = ? ))',
       )
       .all(startStopId, endStopId);
     should.exists(results);
@@ -69,7 +85,7 @@ describe('openDb():', () => {
     const db1 = openDb(config);
 
     db1.name.should.equal(':memory:');
-    db2.name.should.equal('./tmpdb');
+    db2.name.should.equal('./tmpdb2');
 
     // Query db1 for shapes
     const shapeId = 'cal_sf_tam';
@@ -79,7 +95,7 @@ describe('openDb():', () => {
       },
       [],
       [],
-      { db: db1 }
+      { db: db1 },
     );
 
     const expectedResult = {
@@ -101,10 +117,33 @@ describe('openDb():', () => {
       },
       [],
       [],
-      { db: db2 }
+      { db: db2 },
     );
 
     should.exist(results2);
     results2.length.should.equal(0);
   });
+
+  it('should allow `db` configuration option', async () => {
+    const db3 = openDb(db3Config);
+
+    await importGtfs({
+      ...db3Config,
+      db: db3,
+    });
+
+    // Query db3 for shapes, none should exist
+    const shapeId = 'cal_sf_tam';
+    const results = getShapes(
+      {
+        shape_id: shapeId,
+      },
+      [],
+      [],
+      { db: db3 },
+    );
+
+    should.exist(results);
+    results.length.should.equal(0);
+  });
 });