gtfs 3.6.0 → 3.7.0

package/CHANGELOG.md

@@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.7.0] - 2022-11-17
+
+### Updated
+
+- Dependency updates
+
+### Added
+
+- Add ignoreDuplicates config option
+
+## [3.6.1] - 2022-11-08
+
+### Updated
+
+- Support for querying null values as part of an array
+
 ## [3.6.0] - 2022-08-08
 
 ### Added

package/README.md

@@ -93,6 +93,10 @@ try {
     <td><img src="https://github.com/BlinkTagInc/gtfs-tts/raw/main/docs/images/gtfs-tts-logo.svg" alt="GTFS-TTS" width="200"></td>
     <td><a href="https://github.com/blinktaginc/gtfs-tts">GTFS-Text-to-Speech</a> app tests GTFS stop name pronunciation for text-to-speech. It uses `node-gtfs` for loading stop names from GTFS data.</td>
   </tr>
+  <tr>
+    <td><img src="https://github.com/BlinkTagInc/transit-arrivals-widget/raw/master/docs/images/transit-arrivals-widget-logo.svg" alt="Transit Arrivals Widget" width="200"></td>
+    <td><a href="https://github.com/BlinkTagInc/transit-arrivals-widget">Transit Arrivals Widget</a> creates a realtime transit arrivals tool from GTFS and GTFS-RT data.</td>
+  </tr>
 </table>
 
 ## Command-Line Usage
@@ -135,13 +139,14 @@ Copy `config-sample.json` to `config.json` and then add your projects configurat
 
     cp config-sample.json config.json
 
-| option                      | type    | description                                                                                          |
-| --------------------------- | ------- | ---------------------------------------------------------------------------------------------------- |
-| [`agencies`](#agencies)     | array   | An array of GTFS files to be imported.                                                               |
-| [`csvOptions`](#csvOptions) | object  | Options passed to `csv-parse` for parsing GTFS CSV files. Optional.                                  |
-| [`exportPath`](#exportPath) | string  | A path to a directory to put exported GTFS files. Optional, defaults to `gtfs-export/<agency_name>`. |
-| [`sqlitePath`](#sqlitePath) | string  | A path to an SQLite database. Optional, defaults to using an in-memory database.                     |
-| [`verbose`](#verbose)       | boolean | Whether or not to print output to the console. Optional, defaults to true.                           |
+| option                                  | type    | description                                                                                                                            |
+| --------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------- |
+| [`agencies`](#agencies)                 | array   | An array of GTFS files to be imported.                                                                                                 |
+| [`csvOptions`](#csvOptions)             | object  | Options passed to `csv-parse` for parsing GTFS CSV files. Optional.                                                                    |
+| [`exportPath`](#exportPath)             | string  | A path to a directory to put exported GTFS files. Optional, defaults to `gtfs-export/<agency_name>`.                                   |
+| [`ignoreDuplicates`](#ignoreduplicates) | boolean | Whether or not to ignore unique constraints on ids when importing GTFS, such as `trip_id`, `calendar_id`. Optional, defaults to false. |
+| [`sqlitePath`](#sqlitePath)             | string  | A path to an SQLite database. Optional, defaults to using an in-memory database.                                                       |
+| [`verbose`](#verbose)                   | boolean | Whether or not to print output to the console. Optional, defaults to true.                                                             |
 
 ### agencies
 
@@ -270,6 +275,32 @@ See [full list of options](https://csv.js.org/parse/options/).
 
 {String} A path to a directory to put exported GTFS files. If the directory does not exist, it will be created. Used when running `gtfs-export` script or `exportGtfs()`. Optional, defaults to `gtfs-export/<agency_name>` where `<agency_name>` is a sanitized, [snake-cased](https://en.wikipedia.org/wiki/Snake_case) version of the first `agency_name` in `agency.txt`.
 
+```json
+{
+  "agencies": [
+    {
+      "path": "/path/to/the/unzipped/gtfs/"
+    }
+  ],
+  "exportPath": "~/path/to/export/gtfs"
+}
+```
+
+### ignoreDuplicates
+
+{Boolean} If you don't want node-GTFS to throw an error when it encounters a duplicate id on GTFS import. If `true`, it will skip importing duplicate records where unique constraints are violated, such as`trip_id`, `stop_id`, `calendar_id`. Useful if importing GTFS from multiple sources into one SQlite database that share routes or stops. Defaults to `false`.
+
+```json
+{
+  "agencies": [
+    {
+      "path": "/path/to/the/unzipped/gtfs/"
+    }
+  ],
+  "ignoreDuplicates": false
+}
+```
+
 ### sqlitePath
 
 {String} A path to an SQLite database. Optional, defaults to using an in-memory database with a value of `:memory:`.

package/config-sample-full.json

@@ -12,6 +12,7 @@
   "csvOptions": {
     "skip_lines_with_error": true
   },
+  "ignoreDuplicates": false,
   "sqlitePath": "/tmp/gtfs",
   "exportPath": "~/path/to/export/gtfs"
 }

package/lib/db.js

@@ -26,7 +26,7 @@ export async function setupDb(db) {
   if (!db) {
     if (Object.keys(dbs).length > 1) {
       throw new Error(
-        'Multiple database connections. Pass the db you want to close as a parameter to `setupDb`.'
+        'Multiple database connections. Pass the db you want to setup as a parameter to `setupDb`.'
       );
     }
 

package/lib/import.js

@@ -128,7 +128,7 @@ const cleanStaleRealtimeData = async (db, log) => {
     (x) => x.filenameBase === 'service_alert_targets'
   );
 
-  log(`Cleaning stale GRFS-RT data..`);
+  log(`Cleaning stale GTFS-RT data..`);
   await db.run(
     `DELETE FROM ${vehiclePositionModel.filenameBase} WHERE isUpdated=0`
   );
@@ -503,9 +503,9 @@ const importLines = async (task, lines, model, totalLineCount) => {
 
   try {
     await task.db.run(
-      `INSERT INTO ${model.filenameBase}(${fieldNames.join(
-        ', '
-      )}) VALUES${placeholders.join(',')}`,
+      `INSERT ${task.ignoreDuplicates ? 'OR IGNORE' : ''} INTO ${
+        model.filenameBase
+      }(${fieldNames.join(', ')}) VALUES${placeholders.join(',')}`,
       values
     );
   } catch (error) {
@@ -642,6 +642,7 @@ export async function importGtfs(initialConfig) {
       downloadDir: path,
       path: agency.path,
       csvOptions: config.csvOptions || {},
+      ignoreDuplicates: config.ignoreDuplicates,
       db,
       log(message, overwrite) {
         log(message, overwrite);

package/lib/utils.js

@@ -26,6 +26,7 @@ export function validateConfigForImport(config) {
 export function setDefaultConfig(initialConfig) {
   const defaults = {
     sqlitePath: ':memory:',
+    ignoreDuplicates: false,
   };
 
   return {
@@ -81,9 +82,16 @@ export function formatJoinClause(joinObject) {
 
 export function formatWhereClause(key, value) {
   if (Array.isArray(value)) {
-    return `${sqlString.escapeId(key)} IN (${value
+    let whereClause = `${sqlString.escapeId(key)} IN (${value
+      .filter((v) => v !== null)
       .map((v) => sqlString.escape(v))
       .join(', ')})`;
+
+    if (value.includes(null)) {
+      whereClause = `(${whereClause} OR ${sqlString.escapeId(key)} IS NULL)`;
+    }
+
+    return whereClause;
   }
 
   if (value === null) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gtfs",
-  "version": "3.6.0",
+  "version": "3.7.0",
   "description": "Import GTFS transit data into SQLite and query routes, stops, times, fares and more",
   "keywords": [
     "transit",
@@ -35,6 +35,7 @@
     "Jean-François Vial <jeff@modulaweb.fr>",
     "wdalrymple",
     "Ivan Yulaev",
+    "Ivy Rose",
     "Adam Pitchie",
     "Daniel Demidov",
     "gerlacdt",
@@ -72,34 +73,34 @@
   },
   "dependencies": {
     "@turf/helpers": "^6.5.0",
-    "csv-parse": "^5.3.0",
-    "csv-stringify": "^6.2.0",
+    "csv-parse": "^5.3.2",
+    "csv-stringify": "^6.2.1",
     "gtfs-realtime-bindings": "^0.0.6",
     "lodash-es": "^4.17.21",
-    "long": "^5.2.0",
-    "node-fetch": "^3.2.10",
+    "long": "^5.2.1",
+    "node-fetch": "^3.3.0",
     "pluralize": "^8.0.0",
     "pretty-error": "^4.0.0",
     "promise-map-series": "^0.3.0",
     "recursive-copy": "^2.0.14",
     "sanitize-filename": "^1.6.3",
     "sqlite": "^4.1.2",
-    "sqlite3": "^5.0.11",
+    "sqlite3": "^5.1.2",
     "sqlstring-sqlite": "^0.1.1",
     "strip-bom-stream": "^5.0.0",
     "tmp-promise": "^3.0.3",
     "untildify": "^4.0.0",
     "unzipper": "^0.10.11",
-    "yargs": "^17.5.1",
+    "yargs": "^17.6.2",
     "yoctocolors": "^1.0.0"
   },
   "devDependencies": {
     "dtslint": "^4.2.1",
-    "eslint": "^8.21.0",
+    "eslint": "^8.27.0",
     "eslint-config-prettier": "^8.5.0",
-    "eslint-config-xo": "^0.41.0",
-    "husky": "^8.0.1",
-    "mocha": "^10.0.0",
+    "eslint-config-xo": "^0.43.1",
+    "husky": "^8.0.2",
+    "mocha": "^10.1.0",
     "prettier": "^2.7.1",
     "pretty-quick": "^3.1.3",
     "should": "^13.2.3"