gtfs 4.2.0 → 4.3.0

package/.github/workflows/nodejs.yml

@@ -8,12 +8,12 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [14.x, 16.x, 18.x]
+        node-version: [16.x, 18.x, 20.x]
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v3
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.node-version }}
       - name: npm install and test

package/CHANGELOG.md

@@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [4.3.0] - 2023-05-04
+
+### Updated
+
+- Updated Readme to add closeDb documentation
+- Use node-stream-zip instead of unzipper
+
+### Added
+
+- Support for prefixes when importing multiple GTFS files
+
 ## [4.2.0] - 2023-04-13
 
 ### Updated

package/README.md

@@ -176,11 +176,21 @@ Copy `config-sample.json` to `config.json` and then add your projects configurat
 
 For GTFS files that contain more than one agency, you only need to list each GTFS file once in the `agencies` array, not once per agency that it contains.
 
-To find an agency's GTFS file, visit [transitfeeds.com](http://transitfeeds.com). You can use the
-URL from the agency's website or you can use a URL generated from the transitfeeds.com
-API along with your API token.
+To find an agency's GTFS file, visit [transitfeeds.com](http://transitfeeds.com).
 
-- Specify a download URL:
+#### agencies options
+
+| option            | type   | description                                                                                                                        |
+| ----------------- | ------ | ---------------------------------------------------------------------------------------------------------------------------------- |
+| `url`             | string | The URL to a zipped GTFS file. Required if `path` not present.                                                                     |
+| `path`            | string | A path to a zipped GTFS file or a directory of unzipped .txt files. Required if `url` is not present.                              |
+| `headers`         | object | An object of HTTP headers in key:value format to use when fetching GTFS from the `url` specified. Optional.                        |
+| `prefix`          | string | A prefix to be added to every ID field maintain uniqueness when importing multiple GTFS from multiple agencies. Optional.          |
+| `exclude`         | array  | An array of GTFS file names (without `.txt`) to exclude when importing. Optional.                                                  |
+| `realtimeUrls`    | array  | An array of GTFS-Realtime urls to import. Optional.                                                                                |
+| `realtimeHeaders` | array  | An object of HTTP headers in key:value format to use when fetching GTFS-Realtime data from the `realtimeUrls` specified. Optional. |
+
+- Specify a `url` to download GTFS:
 
 ```json
 {
@@ -192,7 +202,7 @@ API along with your API token.
 }
 ```
 
-- Specify a download URL with custom headers:
+- Specify a download URL with custom headers using the `headers` field:
 
 ```json
 {
@@ -208,7 +218,7 @@ API along with your API token.
 }
 ```
 
-- Specify a path to a zipped GTFS file:
+- Specify a `path` to a zipped GTFS file:
 
 ```json
 {
@@ -220,7 +230,7 @@ API along with your API token.
 }
 ```
 
-- Specify a path to an unzipped GTFS file:
+- Specify a `path` to an unzipped GTFS file:
 
 ```json
 {
@@ -232,7 +242,7 @@ API along with your API token.
 }
 ```
 
-- Exclude files - if you don't want all GTFS files to be imported, you can specify an array of files to exclude.
+- If you don't want all GTFS files to be imported, you can specify an array of files to `exclude`. This can save a lot of time for larger GTFS.
 
 ```json
 {
@@ -279,6 +289,23 @@ API along with your API token.
 }
 ```
 
+- When importing multiple agencies their IDs may overlap. Specify a `prefix` to be added to every ID field to maintain uniqueness.
+
+```json
+{
+  "agencies": [
+    {
+      "path": "/path/to/the/gtfs.zip",
+      "prefix": "A"
+    },
+    {
+      "path": "/path/to/the/othergtfs.zip",
+      "prefix": 10000
+    }
+  ]
+}
+```
+
 ### csvOptions
 
 {Object} Add options to be passed to [`csv-parse`](https://csv.js.org/parse/) with the key `csvOptions`. This is an optional parameter.
@@ -506,13 +533,29 @@ Most query methods accept three optional arguments: `query`, `fields`, `sortBy`
 
 For more advanced queries, you can use `advancedQuery` or raw SQL queries using query method from [better-sqlite3](#raw-sqlite-query).
 
-### Setup
+### Database Setup
 
-To use any of the query methods, first open the database before making any queries:
+To use any of the query methods, first open the database using `openDb` before making any queries:
 
 ```js
 import { openDb } from 'gtfs';
+import { readFile } from 'fs/promises';
+const config = JSON.parse(
+  await readFile(new URL('./config.json', import.meta.url))
+);
+const db = openDb(config);
+```
+
+If you no longer need a database (especially if using an in-memory database) you can use `closeDb`:
+
+```js
+import { closeDb, openDb } from 'gtfs';
 const db = openDb(config);
+
+// Do some stuff here
+
+// Close database connection when done.
+closeDb(db);
 ```
 
 ### Examples
@@ -520,7 +563,7 @@ const db = openDb(config);
 For example, to get a list of all routes with just `route_id`, `route_short_name` and `route_color` sorted by `route_short_name`:
 
 ```js
-import { openDb, getRoutes } from 'gtfs';
+import { closeDb, openDb, getRoutes } from 'gtfs';
 import { readFile } from 'fs/promises';
 const config = JSON.parse(
   await readFile(new URL('./config.json', import.meta.url))
@@ -533,12 +576,14 @@ const routes = getRoutes(
   [['route_short_name', 'ASC']], // Sort by this field and direction
   { db: db } // Options for the query. Can specify which database to use if more than one are open
 );
+
+closeDb(db);
 ```
 
 To get a list of all trip_ids for a specific route:
 
 ```js
-import { openDb, getTrips } from 'gtfs';
+import { closeDb, openDb, getTrips } from 'gtfs';
 import { readFile } from 'fs/promises';
 const config = JSON.parse(
   await readFile(new URL('./config.json', import.meta.url))
@@ -551,12 +596,14 @@ const trips = getTrips(
   },
   ['trip_id']
 );
+
+closeDb(db);
 ```
 
 To get a few stops by specific stop_ids:
 
 ```js
-import { openDb, getStops } from 'gtfs';
+import { closeDb, openDb, getStops } from 'gtfs';
 import { readFile } from 'fs/promises';
 const config = JSON.parse(await readFile(new URL('./config.json', import.meta.url)));
 
@@ -570,6 +617,8 @@ const stops = getStops(
     ]
   }
 );
+
+closeDb(db);
 ```
 
 ### Static GTFS Files

package/lib/db.js

@@ -4,21 +4,26 @@ import untildify from 'untildify';
 import { setDefaultConfig } from './utils.js';
 const dbs = {};
 
-const getOrCreateDbConnection = (sqlitePath) => {
-  if (dbs[sqlitePath]) {
-    return dbs[sqlitePath];
-  }
-
+function setupDb(sqlitePath) {
   const db = new Database(untildify(sqlitePath));
-  setupDb(db);
-
+  db.pragma('journal_mode = OFF');
+  db.pragma('synchronous = OFF');
+  db.pragma('temp_store = MEMORY');
   dbs[sqlitePath] = db;
+
   return db;
-};
+}
 
 export function openDb(config) {
+  // If config is passed, use that to open or return db
   if (config) {
-    return getOrCreateDbConnection(setDefaultConfig(config).sqlitePath);
+    const { sqlitePath } = setDefaultConfig(config);
+
+    if (dbs[sqlitePath]) {
+      return dbs[sqlitePath];
+    }
+
+    return setupDb(sqlitePath);
   }
 
   // If only one db connection already exists, use it
@@ -36,12 +41,6 @@ export function openDb(config) {
   throw new Error('Unable to find database connection.');
 }
 
-export function setupDb(db) {
-  db.pragma('journal_mode = OFF');
-  db.pragma('synchronous = OFF');
-  db.pragma('temp_store = MEMORY');
-}
-
 export function closeDb(db) {
   if (Object.keys(dbs).length === 0) {
     throw new Error(

package/lib/file-utils.js

@@ -1,10 +1,10 @@
 import path from 'node:path';
-import { createReadStream, existsSync } from 'node:fs';
+import { existsSync } from 'node:fs';
 import { mkdir, readFile, rm } from 'node:fs/promises';
 import { omit, snakeCase } from 'lodash-es';
 import sanitize from 'sanitize-filename';
 import untildify from 'untildify';
-import { Extract } from 'unzipper';
+import StreamZip from 'node-stream-zip';
 
 /*
  * Attempt to parse any config JSON file and read values from CLI.
@@ -91,13 +91,11 @@ export async function prepDirectory(exportPath) {
 /*
  * Unzip a zipfile into a specified directory
  */
-export function unzip(zipfilePath, exportPath) {
-  /* eslint-disable new-cap */
-  return createReadStream(zipfilePath)
-    .pipe(Extract({ path: exportPath }))
-    .on('entry', (entry) => entry.autodrain())
-    .promise();
-  /* eslint-enable new-cap */
+export async function unzip(zipfilePath, exportPath) {
+  /* eslint-disable-next-line new-cap */
+  const zip = new StreamZip.async({ file: zipfilePath });
+  await zip.extract(null, exportPath);
+  await zip.close();
 }
 
 /*

package/lib/import.js

@@ -446,23 +446,32 @@ const importLines = (task, lines, model, totalLineCount) => {
   }
 
   const linesToImportCount = lines.length;
-  const fieldNames = model.schema
-    .filter((column) => column.name !== 'id')
-    .map((column) => column.name);
+  const columns = model.schema.filter((column) => column.name !== 'id');
   const placeholders = [];
   const values = [];
 
   while (lines.length > 0) {
     const line = lines.pop();
-    placeholders.push(`(${fieldNames.map(() => '?').join(', ')})`);
-    values.push(...fieldNames.map((fieldName) => line[fieldName]));
+    placeholders.push(`(${columns.map(() => '?').join(', ')})`);
+    values.push(
+      ...columns.map((column) => {
+        if (task.prefix !== undefined && column.prefix === true) {
+          // Add prefixes to field values if needed
+          return `${task.prefix}${line[column.name]}`;
+        }
+
+        return line[column.name];
+      })
+    );
   }
 
   try {
     db.prepare(
       `INSERT ${task.ignoreDuplicates ? 'OR IGNORE' : ''} INTO ${
         model.filenameBase
-      } (${fieldNames.join(', ')}) VALUES ${placeholders.join(',')}`
+      } (${columns
+        .map((column) => column.name)
+        .join(', ')}) VALUES ${placeholders.join(',')}`
     ).run(...values);
   } catch (error) {
     task.warn(
@@ -588,6 +597,7 @@ export async function importGtfs(initialConfig) {
         csvOptions: config.csvOptions || {},
         ignoreDuplicates: config.ignoreDuplicates,
         sqlitePath: config.sqlitePath,
+        prefix: agency.prefix,
         log,
         warn: logWarning,
         error: logError,

package/models/gtfs/agency.js

@@ -5,6 +5,7 @@ const model = {
       name: 'agency_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'agency_name',

package/models/gtfs/areas.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'area_name',

package/models/gtfs/attributions.js

@@ -6,18 +6,22 @@ const model = {
       type: 'varchar(255)',
       primary: true,
       required: true,
+      prefix: true,
     },
     {
       name: 'agency_id',
-      type: 'varchar(255)',
+      type: 'varchar(255)',      
+      prefix: true,
     },
     {
       name: 'route_id',
       type: 'varchar(255)',
+      prefix: true,      
     },
     {
       name: 'trip_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'organization_name',

package/models/gtfs/calendar-dates.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'date',

package/models/gtfs/calendar.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'monday',

package/models/gtfs/fare-attributes.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'price',
@@ -33,6 +34,7 @@ const model = {
     {
       name: 'agency_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'transfer_duration',

package/models/gtfs/fare-leg-rules.js

@@ -4,27 +4,32 @@ const model = {
     {
       name: 'leg_group_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'network_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'from_area_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'to_area_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'fare_product_id',
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
   ],
 };

package/models/gtfs/fare-products.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'fare_product_name',
@@ -15,6 +16,7 @@ const model = {
       name: 'fare_media_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'amount',

package/models/gtfs/fare-rules.js

@@ -5,22 +5,27 @@ const model = {
       name: 'fare_id',
       type: 'varchar(255)',
       required: true,
+      prefix: true,
     },
     {
       name: 'route_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'origin_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'destination_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'contains_id',
       type: 'varchar(255)',
+      prefix: true,
     },
   ],
 };

package/models/gtfs/fare-transfer-rules.js

@@ -5,11 +5,13 @@ const model = {
       name: 'from_leg_group_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'to_leg_group_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'transfer_count',
@@ -20,6 +22,7 @@ const model = {
     {
       name: 'transfer_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'duration_limit',
@@ -44,6 +47,7 @@ const model = {
       name: 'fare_product_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
   ],
 };

package/models/gtfs/frequencies.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'start_time',

package/models/gtfs/levels.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       primary: true,
       required: true,
+      prefix: true,
     },
     {
       name: 'level_index',

package/models/gtfs/pathways.js

@@ -6,16 +6,19 @@ const model = {
       type: 'varchar(255)',
       primary: true,
       required: true,
+      prefix: true,
     },
     {
       name: 'from_stop_id',
       type: 'varchar(255)',
       required: true,
+      prefix: true,
     },
     {
       name: 'to_stop_id',
       type: 'varchar(255)',
       required: true,
+      prefix: true,
     },
     {
       name: 'pathway_mode',

package/models/gtfs/routes.js

@@ -6,10 +6,12 @@ const model = {
       type: 'varchar(255)',
       primary: true,
       required: true,
+      prefix: true,
     },
     {
       name: 'agency_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'route_short_name',
@@ -68,6 +70,7 @@ const model = {
     {
       name: 'network_id',
       type: 'varchar(255)',
+      prefix: true,
     },
   ],
 };

package/models/gtfs/shapes.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'shape_pt_lat',

package/models/gtfs/stop-areas.js

@@ -5,11 +5,13 @@ const model = {
       name: 'area_id',
       type: 'varchar(255)',
       required: true,
+      prefix: true,
     },
     {
       name: 'stop_id',
       type: 'varchar(255)',
       required: true,
+      prefix: true,
     },
   ],
 };

package/models/gtfs/stop-times.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'arrival_time',
@@ -29,6 +30,7 @@ const model = {
       name: 'stop_id',
       type: 'varchar(255)',
       required: true,
+      prefix: true,
     },
     {
       name: 'stop_sequence',

package/models/gtfs/stops.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       primary: true,
       required: true,
+      prefix: true,
     },
     {
       name: 'stop_code',
@@ -41,6 +42,7 @@ const model = {
     {
       name: 'zone_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'stop_url',
@@ -70,6 +72,7 @@ const model = {
     {
       name: 'level_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'platform_code',

package/models/gtfs/transfers.js

@@ -5,31 +5,37 @@ const model = {
       name: 'from_stop_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'to_stop_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'from_route_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'to_route_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'from_trip_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'to_trip_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'transfer_type',

package/models/gtfs/translations.js

@@ -28,11 +28,13 @@ const model = {
       name: 'record_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'record_sub_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'field_value',

package/models/gtfs/trips.js

@@ -6,18 +6,21 @@ const model = {
       type: 'varchar(255)',
       required: true,
       index: true,
+      prefix: true,
     },
     {
       name: 'service_id',
       type: 'varchar(255)',
       required: true,
       index: true,
+      prefix: true,
     },
     {
       name: 'trip_id',
       type: 'varchar(255)',
       primary: true,
       required: true,
+      prefix: true,
     },
     {
       name: 'trip_headsign',
@@ -40,11 +43,13 @@ const model = {
       name: 'block_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'shape_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'wheelchair_accessible',

package/models/gtfs-plus/calendar-attributes.js

@@ -7,6 +7,7 @@ const model = {
       name: 'service_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'service_description',

package/models/gtfs-plus/directions.js

@@ -8,6 +8,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'direction_id',

package/models/gtfs-plus/route-attributes.js

@@ -7,6 +7,7 @@ const model = {
       name: 'route_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'category',

package/models/gtfs-plus/stop-attributes.js

@@ -8,6 +8,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'accessibility_id',

package/models/gtfs-ride/board-alight.js

@@ -8,12 +8,14 @@ const model = {
       type: 'varchar(255)',
       required: true,
       index: true,
+      prefix: true,
     },
     {
       name: 'stop_id',
       type: 'varchar(255)',
       required: true,
       index: true,
+      prefix: true,
     },
     {
       name: 'stop_sequence',

package/models/gtfs-ride/rider-trip.js

@@ -7,21 +7,25 @@ const model = {
       name: 'rider_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'agency_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'trip_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'boarding_stop_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'boarding_stop_sequence',
@@ -33,6 +37,7 @@ const model = {
       name: 'alighting_stop_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'alighting_stop_sequence',

package/models/gtfs-ride/ridership.js

@@ -47,6 +47,7 @@ const model = {
       name: 'service_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'monday',
@@ -94,11 +95,13 @@ const model = {
       name: 'agency_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'route_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'direction_id',
@@ -110,10 +113,12 @@ const model = {
     {
       name: 'trip_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'stop_id',
       type: 'varchar(255)',
+      prefix: true,
     },
   ],
 };

package/models/gtfs-ride/trip-capacity.js

@@ -7,11 +7,13 @@ const model = {
       name: 'agency_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'trip_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'service_date',

package/models/non-standard/timetable-notes-references.js

@@ -5,26 +5,31 @@ const model = {
     {
       name: 'note_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'timetable_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'route_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'trip_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'stop_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'stop_sequence',

package/models/non-standard/timetable-notes.js

@@ -6,6 +6,7 @@ const model = {
       name: 'note_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'symbol',

package/models/non-standard/timetable-pages.js

@@ -6,6 +6,7 @@ const model = {
       name: 'timetable_page_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'timetable_page_label',

package/models/non-standard/timetable-stop-order.js

@@ -6,15 +6,18 @@ const model = {
       name: 'id',
       type: 'integer',
       primary: true,
+      prefix: true,
     },
     {
       name: 'timetable_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'stop_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'stop_sequence',

package/models/non-standard/timetables.js

@@ -6,14 +6,17 @@ const model = {
       name: 'id',
       type: 'integer',
       primary: true,
+      prefix: true,
     },
     {
       name: 'timetable_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'route_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'direction_id',

package/models/non-standard/trips-dated-vehicle-journey.js

@@ -7,6 +7,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       index: true,
+      prefix: true,
     },
     {
       name: 'operating_day_date',

package/models/ods/deadhead-times.js

@@ -7,12 +7,14 @@ const model = {
       name: 'id',
       type: 'integer',
       primary: true,
+      prefix: true,
     },
     {
       name: 'deadhead_id',
       type: 'varchar(255)',
       required: true,
       index: true,
+      prefix: true,
     },
     {
       name: 'arrival_time',
@@ -37,10 +39,12 @@ const model = {
     {
       name: 'ops_location_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'stop_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'location_sequence',

package/models/ods/deadheads.js

@@ -8,42 +8,50 @@ const model = {
       type: 'varchar(255)',
       primary: true,
       required: true,
+      prefix: true,
     },
     {
       name: 'service_id',
       type: 'varchar(255)',
       required: true,
+      prefix: true,
     },
     {
       name: 'block_id',
       type: 'varchar(255)',
       required: true,
       index: true,
+      prefix: true,
     },
     {
       name: 'shape_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'to_trip_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'from_trip_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'to_deadhead_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'from_deadhead_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
   ],
 };

package/models/ods/ops-locations.js

@@ -8,6 +8,7 @@ const model = {
       type: 'varchar(255)',
       primary: true,
       required: true,
+      prefix: true,
     },
     {
       name: 'ops_location_code',

package/models/ods/run-events.js

@@ -8,11 +8,13 @@ const model = {
       type: 'varchar(255)',
       primary: true,
       required: true,
+      prefix: true,
     },
     {
       name: 'piece_id',
       type: 'varchar(255)',
       required: true,
+      prefix: true,
     },
     {
       name: 'event_type',
@@ -47,6 +49,7 @@ const model = {
     {
       name: 'event_from_location_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'event_to_location_type',
@@ -58,6 +61,7 @@ const model = {
     {
       name: 'event_to_location_id',
       type: 'varchar(255)',
+      prefix: true,
     },
   ],
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gtfs",
-  "version": "4.2.0",
+  "version": "4.3.0",
   "description": "Import GTFS transit data into SQLite and query routes, stops, times, fares and more",
   "keywords": [
     "transit",
@@ -75,12 +75,13 @@
   "dependencies": {
     "@turf/helpers": "^6.5.0",
     "better-sqlite3": "^8.3.0",
-    "csv-parse": "^5.3.6",
-    "csv-stringify": "^6.3.0",
+    "csv-parse": "^5.3.8",
+    "csv-stringify": "^6.3.2",
     "gtfs-realtime-bindings": "^1.1.1",
     "lodash-es": "^4.17.21",
-    "long": "^5.2.1",
+    "long": "^5.2.3",
     "node-fetch": "^3.3.1",
+    "node-stream-zip": "^1.15.0",
     "pluralize": "^8.0.0",
     "pretty-error": "^4.0.0",
     "promise-map-series": "^0.3.0",
@@ -90,19 +91,18 @@
     "strip-bom-stream": "^5.0.0",
     "tmp-promise": "^3.0.3",
     "untildify": "^4.0.0",
-    "unzipper": "^0.10.11",
     "yargs": "^17.7.1",
     "yoctocolors": "^1.0.0"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.4",
     "dtslint": "^4.2.1",
-    "eslint": "^8.38.0",
+    "eslint": "^8.39.0",
     "eslint-config-prettier": "^8.8.0",
     "eslint-config-xo": "^0.43.1",
     "husky": "^8.0.3",
     "mocha": "^10.2.0",
-    "prettier": "^2.8.7",
+    "prettier": "^2.8.8",
     "pretty-quick": "^3.1.3",
     "should": "^13.2.3"
   },

package/test/mocha/export-gtfs.js

@@ -8,7 +8,11 @@ import { rm } from 'node:fs/promises';
 import { parse } from 'csv-parse';
 import should from 'should';
 
-import { unzip, generateFolderName } from '../../lib/file-utils.js';
+import {
+  unzip,
+  generateFolderName,
+  prepDirectory,
+} from '../../lib/file-utils.js';
 import config from '../test-config.js';
 import {
   openDb,
@@ -45,6 +49,7 @@ describe('exportGtfs():', function () {
     );
 
     before(async () => {
+      await prepDirectory(temporaryDir);
       await unzip(config.agencies[0].path, temporaryDir);
 
       await Promise.all(

package/test/mocha/import-gtfs.js

@@ -7,9 +7,15 @@ import { fileURLToPath } from 'node:url';
 import { parse } from 'csv-parse';
 import should from 'should';
 
-import { unzip } from '../../lib/file-utils.js';
+import { prepDirectory, unzip } from '../../lib/file-utils.js';
 import config from '../test-config.js';
-import { openDb, closeDb, importGtfs, getRoutes } from '../../index.js';
+import {
+  openDb,
+  closeDb,
+  importGtfs,
+  getRoutes,
+  getStops,
+} from '../../index.js';
 import models from '../../models/models.js';
 
 let db;
@@ -63,15 +69,35 @@ describe('importGtfs():', function () {
       routes.length.should.equal(4);
     });
 
-    it("should throw an error when importing from local filesystem which doesn't exist", async () => {
-      return importGtfs({
+    it("should throw an error when importing from local filesystem which doesn't exist", async () =>
+      importGtfs({
         ...config,
         agencies: [
           {
             path: '/does/not/exist',
           },
         ],
-      }).should.be.rejected();
+      }).should.be.rejected());
+
+    it('should add a prefix to imported data if present in config', async () => {
+      const prefix = 'test-prefix';
+      await importGtfs({
+        ...config,
+        agencies: [
+          {
+            ...agenciesFixturesLocal[0],
+            prefix,
+          },
+        ],
+      });
+
+      const routes = getRoutes();
+      should.exist(routes);
+      routes[0].route_id.should.startWith(prefix);
+
+      const stops = getStops();
+      should.exist(stops);
+      stops[0].stop_id.should.startWith(prefix);
     });
   });
 
@@ -83,6 +109,7 @@ describe('importGtfs():', function () {
     );
 
     before(async () => {
+      await prepDirectory(temporaryDir);
       await unzip(agenciesFixturesLocal[0].path, temporaryDir);
 
       await Promise.all(