gtfs 3.6.1 → 3.7.0

package/CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.7.0] - 2022-11-17
+
+### Updated
+
+- Dependency updates
+
+### Added
+
+- Add ignoreDuplicates config option
+
 ## [3.6.1] - 2022-11-08
 
 ### Updated

package/README.md

@@ -139,13 +139,14 @@ Copy `config-sample.json` to `config.json` and then add your projects configurat
 
     cp config-sample.json config.json
 
-| option                      | type    | description                                                                                          |
-| --------------------------- | ------- | ---------------------------------------------------------------------------------------------------- |
-| [`agencies`](#agencies)     | array   | An array of GTFS files to be imported.                                                               |
-| [`csvOptions`](#csvOptions) | object  | Options passed to `csv-parse` for parsing GTFS CSV files. Optional.                                  |
-| [`exportPath`](#exportPath) | string  | A path to a directory to put exported GTFS files. Optional, defaults to `gtfs-export/<agency_name>`. |
-| [`sqlitePath`](#sqlitePath) | string  | A path to an SQLite database. Optional, defaults to using an in-memory database.                     |
-| [`verbose`](#verbose)       | boolean | Whether or not to print output to the console. Optional, defaults to true.                           |
+| option                                  | type    | description                                                                                                                            |
+| --------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------- |
+| [`agencies`](#agencies)                 | array   | An array of GTFS files to be imported.                                                                                                 |
+| [`csvOptions`](#csvOptions)             | object  | Options passed to `csv-parse` for parsing GTFS CSV files. Optional.                                                                    |
+| [`exportPath`](#exportPath)             | string  | A path to a directory to put exported GTFS files. Optional, defaults to `gtfs-export/<agency_name>`.                                   |
+| [`ignoreDuplicates`](#ignoreduplicates) | boolean | Whether or not to ignore unique constraints on ids when importing GTFS, such as `trip_id`, `calendar_id`. Optional, defaults to false. |
+| [`sqlitePath`](#sqlitePath)             | string  | A path to an SQLite database. Optional, defaults to using an in-memory database.                                                       |
+| [`verbose`](#verbose)                   | boolean | Whether or not to print output to the console. Optional, defaults to true.                                                             |
 
 ### agencies
 
@@ -274,6 +275,32 @@ See [full list of options](https://csv.js.org/parse/options/).
 
 {String} A path to a directory to put exported GTFS files. If the directory does not exist, it will be created. Used when running `gtfs-export` script or `exportGtfs()`. Optional, defaults to `gtfs-export/<agency_name>` where `<agency_name>` is a sanitized, [snake-cased](https://en.wikipedia.org/wiki/Snake_case) version of the first `agency_name` in `agency.txt`.
 
+```json
+{
+  "agencies": [
+    {
+      "path": "/path/to/the/unzipped/gtfs/"
+    }
+  ],
+  "exportPath": "~/path/to/export/gtfs"
+}
+```
+
+### ignoreDuplicates
+
+{Boolean} If you don't want node-GTFS to throw an error when it encounters a duplicate id on GTFS import. If `true`, it will skip importing duplicate records where unique constraints are violated, such as`trip_id`, `stop_id`, `calendar_id`. Useful if importing GTFS from multiple sources into one SQlite database that share routes or stops. Defaults to `false`.
+
+```json
+{
+  "agencies": [
+    {
+      "path": "/path/to/the/unzipped/gtfs/"
+    }
+  ],
+  "ignoreDuplicates": false
+}
+```
+
 ### sqlitePath
 
 {String} A path to an SQLite database. Optional, defaults to using an in-memory database with a value of `:memory:`.

package/config-sample-full.json

@@ -12,6 +12,7 @@
   "csvOptions": {
     "skip_lines_with_error": true
   },
+  "ignoreDuplicates": false,
   "sqlitePath": "/tmp/gtfs",
   "exportPath": "~/path/to/export/gtfs"
 }

package/lib/import.js

@@ -128,7 +128,7 @@ const cleanStaleRealtimeData = async (db, log) => {
     (x) => x.filenameBase === 'service_alert_targets'
   );
 
-  log(`Cleaning stale GRFS-RT data..`);
+  log(`Cleaning stale GTFS-RT data..`);
   await db.run(
     `DELETE FROM ${vehiclePositionModel.filenameBase} WHERE isUpdated=0`
   );
@@ -503,9 +503,9 @@ const importLines = async (task, lines, model, totalLineCount) => {
 
   try {
     await task.db.run(
-      `INSERT INTO ${model.filenameBase}(${fieldNames.join(
-        ', '
-      )}) VALUES${placeholders.join(',')}`,
+      `INSERT ${task.ignoreDuplicates ? 'OR IGNORE' : ''} INTO ${
+        model.filenameBase
+      }(${fieldNames.join(', ')}) VALUES${placeholders.join(',')}`,
       values
     );
   } catch (error) {
@@ -642,6 +642,7 @@ export async function importGtfs(initialConfig) {
       downloadDir: path,
       path: agency.path,
       csvOptions: config.csvOptions || {},
+      ignoreDuplicates: config.ignoreDuplicates,
       db,
       log(message, overwrite) {
         log(message, overwrite);

package/lib/utils.js

@@ -26,6 +26,7 @@ export function validateConfigForImport(config) {
 export function setDefaultConfig(initialConfig) {
   const defaults = {
     sqlitePath: ':memory:',
+    ignoreDuplicates: false,
   };
 
   return {
@@ -85,11 +86,11 @@ export function formatWhereClause(key, value) {
       .filter((v) => v !== null)
       .map((v) => sqlString.escape(v))
       .join(', ')})`;
-      
+
     if (value.includes(null)) {
       whereClause = `(${whereClause} OR ${sqlString.escapeId(key)} IS NULL)`;
     }
-    
+
     return whereClause;
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gtfs",
-  "version": "3.6.1",
+  "version": "3.7.0",
   "description": "Import GTFS transit data into SQLite and query routes, stops, times, fares and more",
   "keywords": [
     "transit",
@@ -78,7 +78,7 @@
     "gtfs-realtime-bindings": "^0.0.6",
     "lodash-es": "^4.17.21",
     "long": "^5.2.1",
-    "node-fetch": "^3.2.10",
+    "node-fetch": "^3.3.0",
     "pluralize": "^8.0.0",
     "pretty-error": "^4.0.0",
     "promise-map-series": "^0.3.0",