gtfs 4.1.1 → 4.3.0

package/.github/workflows/nodejs.yml

@@ -8,12 +8,12 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [14.x, 16.x, 18.x]
+        node-version: [16.x, 18.x, 20.x]
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v3
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.node-version }}
       - name: npm install and test

package/CHANGELOG.md

@@ -5,6 +5,25 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [4.3.0] - 2023-05-04
+
+### Updated
+
+- Updated Readme to add closeDb documentation
+- Use node-stream-zip instead of unzipper
+
+### Added
+
+- Support for prefixes when importing multiple GTFS files
+
+## [4.2.0] - 2023-04-13
+
+### Updated
+
+- Add route_attributes to getShapesAsGeoJSON() function as geojson properties
+- Add stop_attributes to getStopsAsGeoJSON() function as geojson properties
+- Support for multi-column primary keys
+
 ## [4.1.1] - 2023-04-12
 
 ### Added

package/README.md

@@ -176,11 +176,21 @@ Copy `config-sample.json` to `config.json` and then add your projects configurat
 
 For GTFS files that contain more than one agency, you only need to list each GTFS file once in the `agencies` array, not once per agency that it contains.
 
-To find an agency's GTFS file, visit [transitfeeds.com](http://transitfeeds.com). You can use the
-URL from the agency's website or you can use a URL generated from the transitfeeds.com
-API along with your API token.
+To find an agency's GTFS file, visit [transitfeeds.com](http://transitfeeds.com).
 
-- Specify a download URL:
+#### agencies options
+
+| option            | type   | description                                                                                                                        |
+| ----------------- | ------ | ---------------------------------------------------------------------------------------------------------------------------------- |
+| `url`             | string | The URL to a zipped GTFS file. Required if `path` not present.                                                                     |
+| `path`            | string | A path to a zipped GTFS file or a directory of unzipped .txt files. Required if `url` is not present.                              |
+| `headers`         | object | An object of HTTP headers in key:value format to use when fetching GTFS from the `url` specified. Optional.                        |
+| `prefix`          | string | A prefix to be added to every ID field maintain uniqueness when importing multiple GTFS from multiple agencies. Optional.          |
+| `exclude`         | array  | An array of GTFS file names (without `.txt`) to exclude when importing. Optional.                                                  |
+| `realtimeUrls`    | array  | An array of GTFS-Realtime urls to import. Optional.                                                                                |
+| `realtimeHeaders` | array  | An object of HTTP headers in key:value format to use when fetching GTFS-Realtime data from the `realtimeUrls` specified. Optional. |
+
+- Specify a `url` to download GTFS:
 
 ```json
 {
@@ -192,7 +202,7 @@ API along with your API token.
 }
 ```
 
-- Specify a download URL with custom headers:
+- Specify a download URL with custom headers using the `headers` field:
 
 ```json
 {
@@ -208,7 +218,7 @@ API along with your API token.
 }
 ```
 
-- Specify a path to a zipped GTFS file:
+- Specify a `path` to a zipped GTFS file:
 
 ```json
 {
@@ -220,7 +230,7 @@ API along with your API token.
 }
 ```
 
-- Specify a path to an unzipped GTFS file:
+- Specify a `path` to an unzipped GTFS file:
 
 ```json
 {
@@ -232,7 +242,7 @@ API along with your API token.
 }
 ```
 
-- Exclude files - if you don't want all GTFS files to be imported, you can specify an array of files to exclude.
+- If you don't want all GTFS files to be imported, you can specify an array of files to `exclude`. This can save a lot of time for larger GTFS.
 
 ```json
 {
@@ -279,6 +289,23 @@ API along with your API token.
 }
 ```
 
+- When importing multiple agencies their IDs may overlap. Specify a `prefix` to be added to every ID field to maintain uniqueness.
+
+```json
+{
+  "agencies": [
+    {
+      "path": "/path/to/the/gtfs.zip",
+      "prefix": "A"
+    },
+    {
+      "path": "/path/to/the/othergtfs.zip",
+      "prefix": 10000
+    }
+  ]
+}
+```
+
 ### csvOptions
 
 {Object} Add options to be passed to [`csv-parse`](https://csv.js.org/parse/) with the key `csvOptions`. This is an optional parameter.
@@ -506,13 +533,29 @@ Most query methods accept three optional arguments: `query`, `fields`, `sortBy`
 
 For more advanced queries, you can use `advancedQuery` or raw SQL queries using query method from [better-sqlite3](#raw-sqlite-query).
 
-### Setup
+### Database Setup
 
-To use any of the query methods, first open the database before making any queries:
+To use any of the query methods, first open the database using `openDb` before making any queries:
 
 ```js
 import { openDb } from 'gtfs';
+import { readFile } from 'fs/promises';
+const config = JSON.parse(
+  await readFile(new URL('./config.json', import.meta.url))
+);
+const db = openDb(config);
+```
+
+If you no longer need a database (especially if using an in-memory database) you can use `closeDb`:
+
+```js
+import { closeDb, openDb } from 'gtfs';
 const db = openDb(config);
+
+// Do some stuff here
+
+// Close database connection when done.
+closeDb(db);
 ```
 
 ### Examples
@@ -520,7 +563,7 @@ const db = openDb(config);
 For example, to get a list of all routes with just `route_id`, `route_short_name` and `route_color` sorted by `route_short_name`:
 
 ```js
-import { openDb, getRoutes } from 'gtfs';
+import { closeDb, openDb, getRoutes } from 'gtfs';
 import { readFile } from 'fs/promises';
 const config = JSON.parse(
   await readFile(new URL('./config.json', import.meta.url))
@@ -533,12 +576,14 @@ const routes = getRoutes(
   [['route_short_name', 'ASC']], // Sort by this field and direction
   { db: db } // Options for the query. Can specify which database to use if more than one are open
 );
+
+closeDb(db);
 ```
 
 To get a list of all trip_ids for a specific route:
 
 ```js
-import { openDb, getTrips } from 'gtfs';
+import { closeDb, openDb, getTrips } from 'gtfs';
 import { readFile } from 'fs/promises';
 const config = JSON.parse(
   await readFile(new URL('./config.json', import.meta.url))
@@ -551,12 +596,14 @@ const trips = getTrips(
   },
   ['trip_id']
 );
+
+closeDb(db);
 ```
 
 To get a few stops by specific stop_ids:
 
 ```js
-import { openDb, getStops } from 'gtfs';
+import { closeDb, openDb, getStops } from 'gtfs';
 import { readFile } from 'fs/promises';
 const config = JSON.parse(await readFile(new URL('./config.json', import.meta.url)));
 
@@ -570,6 +617,8 @@ const stops = getStops(
     ]
   }
 );
+
+closeDb(db);
 ```
 
 ### Static GTFS Files
@@ -692,7 +741,7 @@ const stops = getStops({
 
 #### getStopsAsGeoJSON(query, options)
 
-Returns geoJSON object of stops that match query parameters. All valid queries for `getStops()` work for `getStopsAsGeoJSON()`.
+Returns geoJSON object of stops that match query parameters. Stops will include all properties of each stop from stops.txt and stop_attributes.txt if present. All valid queries for `getStops()` work for `getStopsAsGeoJSON()`.
 
 ```js
 import { getStopsAsGeoJSON } from 'gtfs';
@@ -804,7 +853,7 @@ const shapes = getShapes({
 
 #### getShapesAsGeoJSON(query, options)
 
-Returns a geoJSON object of shapes that match query parameters. All valid queries for `getShapes()` work for `getShapesAsGeoJSON()`.
+Returns a geoJSON object of shapes that match query parameters. Shapes will include all properties of each route from routes.txt and route_attributes.txt if present. All valid queries for `getShapes()` work for `getShapesAsGeoJSON()`.
 
 ```js
 import { getShapesAsGeoJSON } from 'gtfs';

package/lib/db.js

@@ -4,21 +4,26 @@ import untildify from 'untildify';
 import { setDefaultConfig } from './utils.js';
 const dbs = {};
 
-const getOrCreateDbConnection = (sqlitePath) => {
-  if (dbs[sqlitePath]) {
-    return dbs[sqlitePath];
-  }
-
+function setupDb(sqlitePath) {
   const db = new Database(untildify(sqlitePath));
-  setupDb(db);
-
+  db.pragma('journal_mode = OFF');
+  db.pragma('synchronous = OFF');
+  db.pragma('temp_store = MEMORY');
   dbs[sqlitePath] = db;
+
   return db;
-};
+}
 
 export function openDb(config) {
+  // If config is passed, use that to open or return db
   if (config) {
-    return getOrCreateDbConnection(setDefaultConfig(config).sqlitePath);
+    const { sqlitePath } = setDefaultConfig(config);
+
+    if (dbs[sqlitePath]) {
+      return dbs[sqlitePath];
+    }
+
+    return setupDb(sqlitePath);
   }
 
   // If only one db connection already exists, use it
@@ -36,12 +41,6 @@ export function openDb(config) {
   throw new Error('Unable to find database connection.');
 }
 
-export function setupDb(db) {
-  db.pragma('journal_mode = OFF');
-  db.pragma('synchronous = OFF');
-  db.pragma('temp_store = MEMORY');
-}
-
 export function closeDb(db) {
   if (Object.keys(dbs).length === 0) {
     throw new Error(

package/lib/file-utils.js

@@ -1,10 +1,10 @@
 import path from 'node:path';
-import { createReadStream, existsSync } from 'node:fs';
+import { existsSync } from 'node:fs';
 import { mkdir, readFile, rm } from 'node:fs/promises';
 import { omit, snakeCase } from 'lodash-es';
 import sanitize from 'sanitize-filename';
 import untildify from 'untildify';
-import { Extract } from 'unzipper';
+import StreamZip from 'node-stream-zip';
 
 /*
  * Attempt to parse any config JSON file and read values from CLI.
@@ -91,13 +91,11 @@ export async function prepDirectory(exportPath) {
 /*
  * Unzip a zipfile into a specified directory
  */
-export function unzip(zipfilePath, exportPath) {
-  /* eslint-disable new-cap */
-  return createReadStream(zipfilePath)
-    .pipe(Extract({ path: exportPath }))
-    .on('entry', (entry) => entry.autodrain())
-    .promise();
-  /* eslint-enable new-cap */
+export async function unzip(zipfilePath, exportPath) {
+  /* eslint-disable-next-line new-cap */
+  const zip = new StreamZip.async({ file: zipfilePath });
+  await zip.extract(null, exportPath);
+  await zip.close();
 }
 
 /*

package/lib/gtfs/shapes.js

@@ -14,6 +14,7 @@ import { shapesToGeoJSONFeatures } from '../geojson-utils.js';
 import shapes from '../../models/gtfs/shapes.js';
 import { getAgencies } from './agencies.js';
 import { getRoutes } from './routes.js';
+import { getRouteAttributes } from '../gtfs-plus/route-attributes.js';
 
 function buildTripSubquery(query) {
   const whereClause = formatWhereClauses(query);
@@ -92,6 +93,12 @@ export function getShapesAsGeoJSON(query = {}, options = {}) {
       [],
       options
     );
+    const routeAttributes = getRouteAttributes(
+      { route_id: route.route_id },
+      [],
+      [],
+      options
+    );
 
     const agency = agencies.find(
       (agency) => agency.agency_id === route.agency_id
@@ -101,6 +108,7 @@ export function getShapesAsGeoJSON(query = {}, options = {}) {
       agency_name: agency ? agency.agency_name : undefined,
       shape_id: query.shape_id,
       ...route,
+      ...(routeAttributes?.[0] || []),
     };
     features.push(...shapesToGeoJSONFeatures(shapes, geojsonProperties));
   }

package/lib/gtfs/stops.js

@@ -12,6 +12,7 @@ import {
 import { stopsToGeoJSON } from '../geojson-utils.js';
 import stops from '../../models/gtfs/stops.js';
 import { getAgencies } from './agencies.js';
+import { getStopAttributes } from '../gtfs-plus/stop-attributes.js';
 
 function buildTripSubquery(query) {
   const whereClause = formatWhereClauses(query);
@@ -89,12 +90,17 @@ export function getStopsAsGeoJSON(query = {}, options = {}) {
       .prepare(`SELECT * FROM routes WHERE route_id IN (${routeSubquery})`)
       .all(stop.stop_id);
 
+    const stopAttributes = getStopAttributes({ stop_id: stop.stop_id });
+
     stop.routes = orderBy(routes, (route) =>
       Number.parseInt(route.route_short_name, 10)
     );
     stop.agency_name = agencies[0].agency_name;
 
-    return stop;
+    return {
+      ...stop,
+      ...(stopAttributes?.[0] || []),
+    };
   });
 
   // Exclude stops not part of any route

package/lib/import.js

@@ -329,13 +329,23 @@ const createTables = (db) => {
         check = `CHECK( ${column.name} <= ${column.max} )`;
       }
 
-      const primary = column.primary ? 'PRIMARY KEY' : '';
       const required = column.required ? 'NOT NULL' : '';
       const columnDefault = column.default ? 'DEFAULT ' + column.default : '';
       const columnCollation = column.nocase ? 'COLLATE NOCASE' : '';
-      return `${column.name} ${column.type} ${check} ${primary} ${required} ${columnDefault} ${columnCollation}`;
+      return `${column.name} ${column.type} ${check} ${required} ${columnDefault} ${columnCollation}`;
     });
 
+    // Find Primary Key fields
+    const primaryColumns = model.schema.filter((column) => column.primary);
+
+    if (primaryColumns.length > 0) {
+      columns.push(
+        `PRIMARY KEY (${primaryColumns
+          .map((column) => column.name)
+          .join(', ')})`
+      );
+    }
+
     db.prepare(`DROP TABLE IF EXISTS ${model.filenameBase};`).run();
 
     db.prepare(
@@ -436,23 +446,32 @@ const importLines = (task, lines, model, totalLineCount) => {
   }
 
   const linesToImportCount = lines.length;
-  const fieldNames = model.schema
-    .filter((column) => column.name !== 'id')
-    .map((column) => column.name);
+  const columns = model.schema.filter((column) => column.name !== 'id');
   const placeholders = [];
   const values = [];
 
   while (lines.length > 0) {
     const line = lines.pop();
-    placeholders.push(`(${fieldNames.map(() => '?').join(', ')})`);
-    values.push(...fieldNames.map((fieldName) => line[fieldName]));
+    placeholders.push(`(${columns.map(() => '?').join(', ')})`);
+    values.push(
+      ...columns.map((column) => {
+        if (task.prefix !== undefined && column.prefix === true) {
+          // Add prefixes to field values if needed
+          return `${task.prefix}${line[column.name]}`;
+        }
+
+        return line[column.name];
+      })
+    );
   }
 
   try {
     db.prepare(
       `INSERT ${task.ignoreDuplicates ? 'OR IGNORE' : ''} INTO ${
         model.filenameBase
-      } (${fieldNames.join(', ')}) VALUES ${placeholders.join(',')}`
+      } (${columns
+        .map((column) => column.name)
+        .join(', ')}) VALUES ${placeholders.join(',')}`
     ).run(...values);
   } catch (error) {
     task.warn(
@@ -578,6 +597,7 @@ export async function importGtfs(initialConfig) {
         csvOptions: config.csvOptions || {},
         ignoreDuplicates: config.ignoreDuplicates,
         sqlitePath: config.sqlitePath,
+        prefix: agency.prefix,
         log,
         warn: logWarning,
         error: logError,

package/models/gtfs/agency.js

@@ -1,14 +1,11 @@
 const model = {
   filenameBase: 'agency',
   schema: [
-    {
-      name: 'id',
-      type: 'integer',
-      primary: true,
-    },
     {
       name: 'agency_id',
       type: 'varchar(255)',
+      primary: true,
+      prefix: true,
     },
     {
       name: 'agency_name',

package/models/gtfs/areas.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'area_name',

package/models/gtfs/attributions.js

@@ -5,18 +5,23 @@ const model = {
       name: 'attribution_id',
       type: 'varchar(255)',
       primary: true,
+      required: true,
+      prefix: true,
     },
     {
       name: 'agency_id',
-      type: 'varchar(255)',
+      type: 'varchar(255)',      
+      prefix: true,
     },
     {
       name: 'route_id',
       type: 'varchar(255)',
+      prefix: true,      
     },
     {
       name: 'trip_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'organization_name',

package/models/gtfs/calendar-dates.js

@@ -1,22 +1,18 @@
 const model = {
   filenameBase: 'calendar_dates',
   schema: [
-    {
-      name: 'id',
-      type: 'integer',
-      primary: true,
-    },
     {
       name: 'service_id',
       type: 'varchar(255)',
       required: true,
-      index: true,
+      primary: true,
+      prefix: true,
     },
     {
       name: 'date',
       type: 'integer',
       required: true,
-      index: true,
+      primary: true,
     },
     {
       name: 'exception_type',

package/models/gtfs/calendar.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'monday',

package/models/gtfs/fare-attributes.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'price',
@@ -33,6 +34,7 @@ const model = {
     {
       name: 'agency_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'transfer_duration',

package/models/gtfs/fare-leg-rules.js

@@ -4,23 +4,32 @@ const model = {
     {
       name: 'leg_group_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'network_id',
       type: 'varchar(255)',
+      primary: true,
+      prefix: true,
     },
     {
       name: 'from_area_id',
       type: 'varchar(255)',
+      primary: true,
+      prefix: true,
     },
     {
       name: 'to_area_id',
       type: 'varchar(255)',
+      primary: true,
+      prefix: true,
     },
     {
       name: 'fare_product_id',
       type: 'varchar(255)',
       required: true,
+      primary: true,
+      prefix: true,
     },
   ],
 };

package/models/gtfs/fare-products.js

@@ -6,11 +6,18 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'fare_product_name',
       type: 'varchar(255)',
     },
+    {
+      name: 'fare_media_id',
+      type: 'varchar(255)',
+      primary: true,
+      prefix: true,
+    },
     {
       name: 'amount',
       type: 'real',

package/models/gtfs/fare-rules.js

@@ -1,31 +1,31 @@
 const model = {
   filenameBase: 'fare_rules',
   schema: [
-    {
-      name: 'id',
-      type: 'integer',
-      primary: true,
-    },
     {
       name: 'fare_id',
       type: 'varchar(255)',
       required: true,
+      prefix: true,
     },
     {
       name: 'route_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'origin_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'destination_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'contains_id',
       type: 'varchar(255)',
+      prefix: true,
     },
   ],
 };

package/models/gtfs/fare-transfer-rules.js

@@ -4,24 +4,31 @@ const model = {
     {
       name: 'from_leg_group_id',
       type: 'varchar(255)',
+      primary: true,
+      prefix: true,
     },
     {
       name: 'to_leg_group_id',
       type: 'varchar(255)',
+      primary: true,
+      prefix: true,
     },
     {
       name: 'transfer_count',
       type: 'integer',
       min: -1,
+      primary: true,
     },
     {
       name: 'transfer_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'duration_limit',
       type: 'integer',
       min: 0,
+      primary: true,
     },
     {
       name: 'duration_limit_type',
@@ -34,10 +41,13 @@ const model = {
       type: 'integer',
       min: 0,
       max: 2,
+      required: true,
     },
     {
       name: 'fare_product_id',
       type: 'varchar(255)',
+      primary: true,
+      prefix: true,
     },
   ],
 };