gtfs 4.4.3 → 4.5.1

package/CHANGELOG.md

@@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [4.5.1] - 2023-11-09
+
+### Updated
+
+- Dependency updates
+- Use path to types file instead of directory in package.json
+
+### Fixed
+
+- Exclude agency_id from export if empty
+
+## [4.5.0] - 2023-08-23
+
+### Updated
+
+- Increase `maxInsertVariables` to 32,000
+- Dependency updates
+
 ## [4.4.3] - 2023-07-18
 
 ### Updated

package/README.md

@@ -116,8 +116,8 @@ try {
     <td><a href="https://github.com/blinktaginc/gtfs-tts">GTFS-Text-to-Speech</a> app tests GTFS stop name pronunciation for text-to-speech. It uses `node-gtfs` for loading stop names from GTFS data.</td>
   </tr>
   <tr>
-    <td><img src="https://github.com/BlinkTagInc/transit-arrivals-widget/raw/master/docs/images/transit-arrivals-widget-logo.svg" alt="Transit Arrivals Widget" width="200"></td>
-    <td><a href="https://github.com/BlinkTagInc/transit-arrivals-widget">Transit Arrivals Widget</a> creates a realtime transit arrivals tool from GTFS and GTFS-RT data.</td>
+    <td><img src="https://raw.githubusercontent.com/BlinkTagInc/transit-departures-widget/main/docs/images/transit-departures-widget-logo.svg" alt="Transit Departures Widget" width="200"></td>
+    <td><a href="https://github.com/BlinkTagInc/transit-departures-widget">Transit Departures Widget</a> creates a realtime transit departures widget from GTFS and GTFS-Realtime data.</td>
   </tr>
 </table>
 

package/config-sample.json

@@ -1,7 +1,7 @@
 {
   "agencies": [
     {
-      "url": "~/path/to/my/gtfs"
+      "path": "~/path/to/my/gtfs"
     }
   ],
   "sqlitePath": "/tmp/gtfs"

package/lib/export.js

@@ -20,12 +20,12 @@ const getAgencies = (db, config) => {
   } catch (error) {
     if (config.sqlitePath === ':memory:') {
       throw new Error(
-        'No agencies found in SQLite. You are using an in-memory database - if running this from command line be sure to specify a value for `sqlitePath` in config.json other than ":memory:".'
+        'No agencies found in SQLite. You are using an in-memory database - if running this from command line be sure to specify a value for `sqlitePath` in config.json other than ":memory:".',
       );
     }
 
     throw new Error(
-      'No agencies found in SQLite. Be sure to first import data into SQLite using `gtfs-import` or `importGtfs(config);`'
+      'No agencies found in SQLite. Be sure to first import data into SQLite using `gtfs-import` or `importGtfs(config);`',
     );
   }
 };
@@ -42,11 +42,11 @@ const exportGtfs = async (initialConfig) => {
   const agencyCount = agencies.length;
   if (agencyCount === 0) {
     throw new Error(
-      'No agencies found in SQLite. Be sure to first import data into SQLite using `gtfs-import` or `importGtfs(config);`'
+      'No agencies found in SQLite. Be sure to first import data into SQLite using `gtfs-import` or `importGtfs(config);`',
     );
   } else if (agencyCount > 1) {
     logWarning(
-      'More than one agency is defined in config.json. Export will merge all into one GTFS file.'
+      'More than one agency is defined in config.json. Export will merge all into one GTFS file.',
     );
   }
 
@@ -54,8 +54,8 @@ const exportGtfs = async (initialConfig) => {
     `Starting GTFS export for ${pluralize(
       'agency',
       agencyCount,
-      true
-    )} using SQLite database at ${config.sqlitePath}`
+      true,
+    )} using SQLite database at ${config.sqlitePath}`,
   );
 
   const folderName = generateFolderName(agencies[0].agency_name);
@@ -92,9 +92,19 @@ const exportGtfs = async (initialConfig) => {
       'ridership_end_timestamp',
     ];
 
+    // If no routes have values for agency_id, add it to the excludeColumns list
+    if (model.filenameBase === 'routes') {
+      const routesWithAgencyId = db
+        .prepare('SELECT agency_id FROM routes WHERE agency_id IS NOT NULL;')
+        .all();
+      if (!routesWithAgencyId || routesWithAgencyId.length === 0) {
+        excludeColumns.push('agency_id');
+      }
+    }
+
     const columns = without(
       model.schema.map((column) => column.name),
-      ...excludeColumns
+      ...excludeColumns,
     );
     const fileText = await stringify(lines, { columns, header: true });
     await writeFile(filepath, fileText);

package/lib/import.js

@@ -112,15 +112,15 @@ const updateRealtimeData = async (task) => {
 
   const model = {
     vehicle_positions: models.find(
-      (x) => x.filenameBase === 'vehicle_positions'
+      (x) => x.filenameBase === 'vehicle_positions',
     ),
     trip_updates: models.find((x) => x.filenameBase === 'trip_updates'),
     stop_times_updates: models.find(
-      (x) => x.filenameBase === 'stop_times_updates'
+      (x) => x.filenameBase === 'stop_times_updates',
     ),
     service_alerts: models.find((x) => x.filenameBase === 'service_alerts'),
     service_alert_targets: models.find(
-      (x) => x.filenameBase === 'service_alert_targets'
+      (x) => x.filenameBase === 'service_alert_targets',
     ),
   };
 
@@ -143,7 +143,7 @@ const updateRealtimeData = async (task) => {
   };
 
   task.log(
-    `Starting GTFS-Realtime import from ${task.realtime_urls.length} urls`
+    `Starting GTFS-Realtime import from ${task.realtime_urls.length} urls`,
   );
 
   for (const realtimeUrl of task.realtime_urls) {
@@ -151,7 +151,7 @@ const updateRealtimeData = async (task) => {
     // eslint-disable-next-line no-await-in-loop
     const tripUpdateData = await downloadGtfsRealtimeData(
       realtimeUrl,
-      task.realtime_headers
+      task.realtime_headers,
     );
     task.log(`Download successful`);
 
@@ -178,15 +178,15 @@ const updateRealtimeData = async (task) => {
       // Do base processing
       const fieldValues = model[gtfsRealtimeType].schema.map((column) =>
         sqlString.escape(
-          getDescendantProp(entity, column.source, column.default)
-        )
+          getDescendantProp(entity, column.source, column.default),
+        ),
       );
 
       try {
         db.prepare(
           `REPLACE INTO ${model[gtfsRealtimeType].filenameBase} (${
             fields[gtfsRealtimeType]
-          }) VALUES (${fieldValues.join(', ')})`
+          }) VALUES (${fieldValues.join(', ')})`,
         ).run();
       } catch (error) {
         task.warn('Import error: ' + error.message);
@@ -199,8 +199,8 @@ const updateRealtimeData = async (task) => {
           stopUpdate.parent = entity;
           const subValues = model.stop_times_updates.schema.map((column) =>
             sqlString.escape(
-              getDescendantProp(stopUpdate, column.source, column.default)
-            )
+              getDescendantProp(stopUpdate, column.source, column.default),
+            ),
           );
           stopUpdateArray.push(`(${subValues.join(', ')})`);
           totalLineCount++;
@@ -210,7 +210,7 @@ const updateRealtimeData = async (task) => {
           db.prepare(
             `REPLACE INTO ${model.stop_times_updates.filenameBase} (${
               fields.stop_times_updates
-            }) VALUES ${stopUpdateArray.join(', ')}`
+            }) VALUES ${stopUpdateArray.join(', ')}`,
           ).run();
         } catch (error) {
           task.warn('Import error: ' + error.message);
@@ -224,8 +224,8 @@ const updateRealtimeData = async (task) => {
           informedEntity.parent = entity;
           const subValues = model.service_alert_targets.schema.map((column) =>
             sqlString.escape(
-              getDescendantProp(informedEntity, column.source, column.default)
-            )
+              getDescendantProp(informedEntity, column.source, column.default),
+            ),
           );
           alertTargetArray.push(`(${subValues.join(', ')})`);
           totalLineCount++;
@@ -235,7 +235,7 @@ const updateRealtimeData = async (task) => {
           db.prepare(
             `REPLACE INTO ${model.service_alert_targets.filenameBase} (${
               fields.service_alert_targets
-            }) VALUES ${alertTargetArray.join(', ')}`
+            }) VALUES ${alertTargetArray.join(', ')}`,
           ).run();
         } catch (error) {
           task.warn('Import error: ' + error.message);
@@ -273,11 +273,11 @@ const readFiles = async (task) => {
 
         if (folders.length > 1) {
           throw new Error(
-            `More than one subfolder found in zip file at \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`
+            `More than one subfolder found in zip file at \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`,
           );
         } else if (folders.length === 0) {
           throw new Error(
-            `No .txt files found in \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`
+            `No .txt files found in \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`,
           );
         }
 
@@ -286,7 +286,7 @@ const readFiles = async (task) => {
 
         if (directoryTextFiles.length === 0) {
           throw new Error(
-            `No .txt files found in \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`
+            `No .txt files found in \`${task.path}\`. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`,
           );
         }
 
@@ -294,9 +294,9 @@ const readFiles = async (task) => {
           directoryTextFiles.map(async (fileName) =>
             rename(
               path.join(subfolderName, fileName),
-              path.join(task.downloadDir, fileName)
-            )
-          )
+              path.join(task.downloadDir, fileName),
+            ),
+          ),
         );
       }
     } catch (error) {
@@ -310,7 +310,7 @@ const readFiles = async (task) => {
       await copy(gtfsPath, task.downloadDir);
     } catch {
       throw new Error(
-        `Unable to load files from path \`${gtfsPath}\` defined in configuration. Verify that path exists and contains GTFS files.`
+        `Unable to load files from path \`${gtfsPath}\` defined in configuration. Verify that path exists and contains GTFS files.`,
       );
     }
   }
@@ -345,20 +345,20 @@ const createTables = (db) => {
       columns.push(
         `PRIMARY KEY (${primaryColumns
           .map((column) => column.name)
-          .join(', ')})`
+          .join(', ')})`,
       );
     }
 
     db.prepare(`DROP TABLE IF EXISTS ${model.filenameBase};`).run();
 
     db.prepare(
-      `CREATE TABLE ${model.filenameBase} (${columns.join(', ')});`
+      `CREATE TABLE ${model.filenameBase} (${columns.join(', ')});`,
     ).run();
 
     for (const column of model.schema.filter((column) => column.index)) {
       const unique = column.index === 'unique' ? 'UNIQUE' : '';
       db.prepare(
-        `CREATE ${unique} INDEX idx_${model.filenameBase}_${column.name} ON ${model.filenameBase} (${column.name});`
+        `CREATE ${unique} INDEX idx_${model.filenameBase}_${column.name} ON ${model.filenameBase} (${column.name});`,
       ).run();
     }
   }
@@ -398,7 +398,7 @@ const formatLine = (line, model, totalLineCount) => {
       formattedLine[columnSchema.name] === null
     ) {
       throw new Error(
-        `Missing required value in ${model.filenameBase}.txt for ${columnSchema.name} on line ${lineNumber}.`
+        `Missing required value in ${model.filenameBase}.txt for ${columnSchema.name} on line ${lineNumber}.`,
       );
     }
 
@@ -408,7 +408,7 @@ const formatLine = (line, model, totalLineCount) => {
       formattedLine[columnSchema.name] < columnSchema.min
     ) {
       throw new Error(
-        `Invalid value in ${model.filenameBase}.txt for ${columnSchema.name} on line ${lineNumber}: below minimum value of ${columnSchema.min}.`
+        `Invalid value in ${model.filenameBase}.txt for ${columnSchema.name} on line ${lineNumber}: below minimum value of ${columnSchema.min}.`,
       );
     }
 
@@ -418,7 +418,7 @@ const formatLine = (line, model, totalLineCount) => {
       formattedLine[columnSchema.name] > columnSchema.max
     ) {
       throw new Error(
-        `Invalid value in ${model.filenameBase}.txt for ${columnSchema.name} on line ${lineNumber}: above maximum value of ${columnSchema.max}.`
+        `Invalid value in ${model.filenameBase}.txt for ${columnSchema.name} on line ${lineNumber}: above maximum value of ${columnSchema.max}.`,
       );
     }
   }
@@ -438,7 +438,7 @@ const formatLine = (line, model, totalLineCount) => {
 
       // Ensure leading zeros for time columns
       formattedLine[timestampColumnName] = padLeadingZeros(
-        formattedLine[timestampColumnName]
+        formattedLine[timestampColumnName],
       );
     }
   }
@@ -469,7 +469,7 @@ const importLines = (task, lines, model, totalLineCount) => {
         }
 
         return line[column.name];
-      })
+      }),
     );
   }
 
@@ -479,20 +479,20 @@ const importLines = (task, lines, model, totalLineCount) => {
         model.filenameBase
       } (${columns
         .map((column) => column.name)
-        .join(', ')}) VALUES ${placeholders.join(',')}`
+        .join(', ')}) VALUES ${placeholders.join(',')}`,
     ).run(...values);
   } catch (error) {
     task.warn(
       `Check ${model.filenameBase}.txt for invalid data between lines ${
         totalLineCount - linesToImportCount
-      } and ${totalLineCount}.`
+      } and ${totalLineCount}.`,
     );
     throw error;
   }
 
   task.log(
     `Importing - ${model.filenameBase}.txt - ${totalLineCount} lines imported\r`,
-    true
+    true,
   );
 };
 
@@ -517,7 +517,7 @@ const importFiles = (task) =>
 
         const filepath = path.join(
           task.downloadDir,
-          `${model.filenameBase}.txt`
+          `${model.filenameBase}.txt`,
         );
 
         if (!existsSync(filepath)) {
@@ -533,7 +533,7 @@ const importFiles = (task) =>
 
         const lines = [];
         let totalLineCount = 0;
-        const maxInsertVariables = 800;
+        const maxInsertVariables = 32_000;
         const parser = parse({
           columns: true,
           relax_quotes: true,
@@ -568,7 +568,7 @@ const importFiles = (task) =>
         parser.on('error', reject);
 
         createReadStream(filepath).pipe(stripBomStream()).pipe(parser);
-      })
+      }),
   );
 
 export async function importGtfs(initialConfig) {
@@ -585,8 +585,8 @@ export async function importGtfs(initialConfig) {
       `Starting GTFS import for ${pluralize(
         'file',
         agencyCount,
-        true
-      )} using SQLite database at ${config.sqlitePath}`
+        true,
+      )} using SQLite database at ${config.sqlitePath}`,
     );
 
     createTables(db);
@@ -626,12 +626,12 @@ export async function importGtfs(initialConfig) {
     });
 
     log(
-      `Completed GTFS import for ${pluralize('agency', agencyCount, true)}\n`
+      `Completed GTFS import for ${pluralize('agency', agencyCount, true)}\n`,
     );
   } catch (error) {
     if (error instanceof Error && error.code === 'SQLITE_CANTOPEN') {
       logError(
-        `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
+        `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`,
       );
     }
 
@@ -654,8 +654,8 @@ export async function updateGtfsRealtime(initialConfig) {
       `Starting GTFS-Realtime refresh for ${pluralize(
         'agencies',
         agencyCount,
-        true
-      )} using SQLite database at ${config.sqlitePath}`
+        true,
+      )} using SQLite database at ${config.sqlitePath}`,
     );
 
     markRealtimeDataStale(config, log);
@@ -676,7 +676,7 @@ export async function updateGtfsRealtime(initialConfig) {
         };
 
         await updateRealtimeData(task);
-      })
+      }),
     );
 
     cleanStaleRealtimeData(config, log);
@@ -684,13 +684,13 @@ export async function updateGtfsRealtime(initialConfig) {
       `Completed GTFS-Realtime refresh for ${pluralize(
         'agencies',
         agencyCount,
-        true
-      )}\n`
+        true,
+      )}\n`,
     );
   } catch (error) {
     if (error instanceof Error && error.code === 'SQLITE_CANTOPEN') {
       logError(
-        `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
+        `Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`,
       );
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gtfs",
-  "version": "4.4.3",
+  "version": "4.5.1",
   "description": "Import GTFS transit data into SQLite and query routes, stops, times, fares and more",
   "keywords": [
     "transit",
@@ -67,19 +67,19 @@
     "gtfs-import": "bin/gtfs-import.js",
     "gtfsrealtime-update": "bin/gtfsrealtime-update.js"
   },
-  "types": "@types",
+  "types": "@types/index.d.ts",
   "scripts": {
     "test": "NODE_ENV=test mocha ./test/mocha/**/*.js --timeout 2000"
   },
   "dependencies": {
     "@turf/helpers": "^6.5.0",
-    "better-sqlite3": "^8.4.0",
-    "csv-parse": "^5.4.0",
-    "csv-stringify": "^6.4.0",
+    "better-sqlite3": "^9.1.1",
+    "csv-parse": "^5.5.2",
+    "csv-stringify": "^6.4.4",
     "gtfs-realtime-bindings": "^1.1.1",
     "lodash-es": "^4.17.21",
     "long": "^5.2.3",
-    "node-fetch": "^3.3.1",
+    "node-fetch": "^3.3.2",
     "node-stream-zip": "^1.15.0",
     "pluralize": "^8.0.0",
     "pretty-error": "^4.0.0",
@@ -95,9 +95,9 @@
   },
   "devDependencies": {
     "husky": "^8.0.3",
-    "lint-staged": "^13.2.3",
+    "lint-staged": "^15.0.2",
     "mocha": "^10.2.0",
-    "prettier": "^3.0.0",
+    "prettier": "^3.0.3",
     "should": "^13.2.3"
   },
   "engines": {