gtfs 3.6.1 → 3.8.0

package/.github/workflows/nodejs.yml

@@ -4,22 +4,22 @@ on: [push, pull_request]
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        node-version: [14.x, 16.x]
+        node-version: [14.x, 16.x, 18.x]
 
     steps:
-    - uses: actions/checkout@v1
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: npm install and test
-      run: |
-        npm ci
-        npm test
-      env:
-        CI: true
+      - uses: actions/checkout@v1
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: npm install and test
+        run: |
+          npm install -g npm@9
+          npm ci
+          npm test
+        env:
+          CI: true

package/CHANGELOG.md

@@ -5,6 +5,27 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.8.0] - 2022-12-22
+
+### Updated
+
+- Dependency updates
+- Add Node 18 tests
+
+### Changed
+
+- Make transfer_type not required in transfers.txt
+
+## [3.7.0] - 2022-11-17
+
+### Updated
+
+- Dependency updates
+
+### Added
+
+- Add ignoreDuplicates config option
+
 ## [3.6.1] - 2022-11-08
 
 ### Updated

package/README.md

@@ -139,13 +139,14 @@ Copy `config-sample.json` to `config.json` and then add your projects configurat
 
     cp config-sample.json config.json
 
-| option                      | type    | description                                                                                          |
-| --------------------------- | ------- | ---------------------------------------------------------------------------------------------------- |
-| [`agencies`](#agencies)     | array   | An array of GTFS files to be imported.                                                               |
-| [`csvOptions`](#csvOptions) | object  | Options passed to `csv-parse` for parsing GTFS CSV files. Optional.                                  |
-| [`exportPath`](#exportPath) | string  | A path to a directory to put exported GTFS files. Optional, defaults to `gtfs-export/<agency_name>`. |
-| [`sqlitePath`](#sqlitePath) | string  | A path to an SQLite database. Optional, defaults to using an in-memory database.                     |
-| [`verbose`](#verbose)       | boolean | Whether or not to print output to the console. Optional, defaults to true.                           |
+| option                                  | type    | description                                                                                                                            |
+| --------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------- |
+| [`agencies`](#agencies)                 | array   | An array of GTFS files to be imported, and which files to exclude.                                                                     |
+| [`csvOptions`](#csvOptions)             | object  | Options passed to `csv-parse` for parsing GTFS CSV files. Optional.                                                                    |
+| [`exportPath`](#exportPath)             | string  | A path to a directory to put exported GTFS files. Optional, defaults to `gtfs-export/<agency_name>`.                                   |
+| [`ignoreDuplicates`](#ignoreduplicates) | boolean | Whether or not to ignore unique constraints on ids when importing GTFS, such as `trip_id`, `calendar_id`. Optional, defaults to false. |
+| [`sqlitePath`](#sqlitePath)             | string  | A path to an SQLite database. Optional, defaults to using an in-memory database.                                                       |
+| [`verbose`](#verbose)                   | boolean | Whether or not to print output to the console. Optional, defaults to true.                                                             |
 
 ### agencies
 
@@ -274,6 +275,32 @@ See [full list of options](https://csv.js.org/parse/options/).
 
 {String} A path to a directory to put exported GTFS files. If the directory does not exist, it will be created. Used when running `gtfs-export` script or `exportGtfs()`. Optional, defaults to `gtfs-export/<agency_name>` where `<agency_name>` is a sanitized, [snake-cased](https://en.wikipedia.org/wiki/Snake_case) version of the first `agency_name` in `agency.txt`.
 
+```json
+{
+  "agencies": [
+    {
+      "path": "/path/to/the/unzipped/gtfs/"
+    }
+  ],
+  "exportPath": "~/path/to/export/gtfs"
+}
+```
+
+### ignoreDuplicates
+
+{Boolean} If you don't want node-GTFS to throw an error when it encounters a duplicate id on GTFS import. If `true`, it will skip importing duplicate records where unique constraints are violated, such as`trip_id`, `stop_id`, `calendar_id`. Useful if importing GTFS from multiple sources into one SQlite database that share routes or stops. Defaults to `false`.
+
+```json
+{
+  "agencies": [
+    {
+      "path": "/path/to/the/unzipped/gtfs/"
+    }
+  ],
+  "ignoreDuplicates": false
+}
+```
+
 ### sqlitePath
 
 {String} A path to an SQLite database. Optional, defaults to using an in-memory database with a value of `:memory:`.

package/config-sample-full.json

@@ -12,6 +12,7 @@
   "csvOptions": {
     "skip_lines_with_error": true
   },
+  "ignoreDuplicates": false,
   "sqlitePath": "/tmp/gtfs",
   "exportPath": "~/path/to/export/gtfs"
 }

package/lib/import.js

@@ -128,7 +128,7 @@ const cleanStaleRealtimeData = async (db, log) => {
     (x) => x.filenameBase === 'service_alert_targets'
   );
 
-  log(`Cleaning stale GRFS-RT data..`);
+  log(`Cleaning stale GTFS-RT data..`);
   await db.run(
     `DELETE FROM ${vehiclePositionModel.filenameBase} WHERE isUpdated=0`
   );
@@ -503,9 +503,9 @@ const importLines = async (task, lines, model, totalLineCount) => {
 
   try {
     await task.db.run(
-      `INSERT INTO ${model.filenameBase}(${fieldNames.join(
-        ', '
-      )}) VALUES${placeholders.join(',')}`,
+      `INSERT ${task.ignoreDuplicates ? 'OR IGNORE' : ''} INTO ${
+        model.filenameBase
+      }(${fieldNames.join(', ')}) VALUES${placeholders.join(',')}`,
       values
     );
   } catch (error) {
@@ -642,6 +642,7 @@ export async function importGtfs(initialConfig) {
       downloadDir: path,
       path: agency.path,
       csvOptions: config.csvOptions || {},
+      ignoreDuplicates: config.ignoreDuplicates,
       db,
       log(message, overwrite) {
         log(message, overwrite);

package/lib/utils.js

@@ -26,6 +26,7 @@ export function validateConfigForImport(config) {
 export function setDefaultConfig(initialConfig) {
   const defaults = {
     sqlitePath: ':memory:',
+    ignoreDuplicates: false,
   };
 
   return {
@@ -85,11 +86,11 @@ export function formatWhereClause(key, value) {
       .filter((v) => v !== null)
       .map((v) => sqlString.escape(v))
       .join(', ')})`;
-      
+
     if (value.includes(null)) {
       whereClause = `(${whereClause} OR ${sqlString.escapeId(key)} IS NULL)`;
     }
-    
+
     return whereClause;
   }
 

package/models/gtfs/transfers.js

@@ -37,7 +37,8 @@ const model = {
       type: 'integer',
       min: 0,
       max: 5,
-      required: true,
+      required: false,
+      default: 0
     },
     {
       name: 'min_transfer_time',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gtfs",
-  "version": "3.6.1",
+  "version": "3.8.0",
   "description": "Import GTFS transit data into SQLite and query routes, stops, times, fares and more",
   "keywords": [
     "transit",
@@ -55,7 +55,8 @@
     "Darius MARTIN",
     "Mike Brocks",
     "Matt Moran",
-    "Daniel Sörlöv"
+    "Daniel Sörlöv",
+    "Ali Zarghami <alizarghami@gmail.com>"
   ],
   "type": "module",
   "main": "index.js",
@@ -73,19 +74,19 @@
   },
   "dependencies": {
     "@turf/helpers": "^6.5.0",
-    "csv-parse": "^5.3.2",
-    "csv-stringify": "^6.2.1",
+    "csv-parse": "^5.3.3",
+    "csv-stringify": "^6.2.3",
     "gtfs-realtime-bindings": "^0.0.6",
     "lodash-es": "^4.17.21",
     "long": "^5.2.1",
-    "node-fetch": "^3.2.10",
+    "node-fetch": "^3.3.0",
     "pluralize": "^8.0.0",
     "pretty-error": "^4.0.0",
     "promise-map-series": "^0.3.0",
     "recursive-copy": "^2.0.14",
     "sanitize-filename": "^1.6.3",
     "sqlite": "^4.1.2",
-    "sqlite3": "^5.1.2",
+    "sqlite3": "^5.1.4",
     "sqlstring-sqlite": "^0.1.1",
     "strip-bom-stream": "^5.0.0",
     "tmp-promise": "^3.0.3",
@@ -96,12 +97,12 @@
   },
   "devDependencies": {
     "dtslint": "^4.2.1",
-    "eslint": "^8.27.0",
+    "eslint": "^8.30.0",
     "eslint-config-prettier": "^8.5.0",
     "eslint-config-xo": "^0.43.1",
     "husky": "^8.0.2",
-    "mocha": "^10.1.0",
-    "prettier": "^2.7.1",
+    "mocha": "^10.2.0",
+    "prettier": "^2.8.1",
     "pretty-quick": "^3.1.3",
     "should": "^13.2.3"
   },