gtfs 4.19.1 → 4.19.3

package/README.md

@@ -181,6 +181,8 @@ For GTFS files that contain more than one agency, you only need to list each GTF
 | `headers`                  | object | An object of HTTP headers in key:value format to use when fetching GTFS from the `url` specified. Optional.                                                         |
 | `prefix`                   | string | A prefix to be added to every ID field maintain uniqueness when importing multiple GTFS from multiple agencies. Optional.                                           |
 | `exclude`                  | array  | An array of GTFS file names (without `.txt`) to exclude when importing. Optional.                                                                                   |
+| `fillEmptyAgencyId`        | boolean | When `true`, fills empty `agency_id` on routes, fares, and other files for single-agency feeds. Useful for shared databases. Defaults to `false`. Optional.        |
+| `agencyId`                 | string | Explicit `agency_id` to use when `fillEmptyAgencyId` is `true` and `agency.txt` does not define one. Also backfills the `agency_id` on the agency row itself. If `agency.txt` already defines an `agency_id`, that value takes precedence. Optional. |
 | `realtimeAlerts`           | object | An object containing a `url` field for GTFS-Realtime alerts and a `headers` field in key:value format to use when fetching GTFS-Realtime data. Optional.            |
 | `realtimeTripUpdates`      | object | An object containing a `url` field for GTFS-Realtime trip updates and a `headers` field in key:value format to use when fetching GTFS-Realtime data. Optional.      |
 | `realtimeVehiclePositions` | object | An object containing a `url` field for GTFS-Realtime vehicle positions and a `headers` field in key:value format to use when fetching GTFS-Realtime data. Optional. |

package/dist/bin/gtfs-export.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { Tt as formatError, st as exportGtfs, xt as getConfig } from "../src-DuXcyA_N.js";
+import { Tt as formatError, st as exportGtfs, xt as getConfig } from "../src-gj91FR8c.js";
 import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
 import PrettyError from "pretty-error";

package/dist/bin/gtfs-import.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { Tt as formatError, ct as importGtfs, ft as openDb, ut as closeDb, xt as getConfig } from "../src-DuXcyA_N.js";
+import { Tt as formatError, ct as importGtfs, ft as openDb, ut as closeDb, xt as getConfig } from "../src-gj91FR8c.js";
 import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
 import PrettyError from "pretty-error";

package/dist/bin/gtfsrealtime-update.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { Tt as formatError, lt as updateGtfsRealtime, xt as getConfig } from "../src-DuXcyA_N.js";
+import { Tt as formatError, lt as updateGtfsRealtime, xt as getConfig } from "../src-gj91FR8c.js";
 import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
 import PrettyError from "pretty-error";

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import { Options } from "csv-parse";
-import Database, { Database as Database$1 } from "better-sqlite3";
+import Database$1, { Database } from "better-sqlite3";
 
 //#region src/lib/errors.d.ts
 declare enum GtfsErrorCategory {
@@ -121,6 +121,20 @@ interface BaseConfigAgency {
    * A prefix to be added to every ID field maintain uniqueness when importing multiple GTFS from multiple agencies
    */
   prefix?: string;
+  /**
+   * When set to true and the feed contains exactly one agency, populates any empty `agency_id` fields
+   * on routes, fare_attributes, and other relevant files. Useful when merging single-agency feeds into
+   * a shared database.
+   *
+   * @defaultValue false
+   */
+  fillEmptyAgencyId?: boolean;
+  /**
+   * Explicit `agency_id` to use when `fillEmptyAgencyId` is true and `agency.txt` does not define
+   * one. Also backfills the `agency_id` on the agency row itself. If `agency.txt` already defines
+   * an `agency_id` and it differs from this value, the value from `agency.txt` takes precedence.
+   */
+  agencyId?: string;
 }
 type ConfigAgency = BaseConfigAgency & ({
   /**
@@ -137,7 +151,7 @@ interface Config {
   /**
    * An existing database instance to use instead of relying on node-gtfs to connect.
    */
-  db?: Database$1;
+  db?: Database;
   /**
    * A path to an SQLite database. Defaults to using an in-memory database.
    */
@@ -233,7 +247,7 @@ type SqlWhere = Record<string, null | SqlValue | SqlValue[]>;
 type QueryResult<Base extends object, Select extends keyof Base> = [Select] extends [never] ? Base : Pick<Base, Select>;
 type SqlOrderBy = Array<[string, 'ASC' | 'DESC']>;
 interface QueryOptions {
-  db?: Database$1;
+  db?: Database;
   bounding_box_side_m?: number;
 }
 interface Agency {
@@ -830,15 +844,15 @@ declare const exportGtfs: (initialConfig: Config) => Promise<void>;
 //#endregion
 //#region src/lib/db.d.ts
 declare function openDb(config?: {
-  db?: Database.Database;
+  db?: Database$1.Database;
   sqlitePath?: string;
-} | null): Database.Database;
-declare function closeDb(db?: Database.Database | null): void;
-declare function deleteDb(db?: Database.Database | null): void;
+} | null): Database$1.Database;
+declare function closeDb(db?: Database$1.Database | null): void;
+declare function deleteDb(db?: Database$1.Database | null): void;
 //#endregion
 //#region src/lib/advancedQuery.d.ts
 declare function advancedQuery(table: string, advancedQueryOptions: {
-  db?: Database.Database;
+  db?: Database$1.Database;
   query?: SqlWhere;
   fields?: string[];
   orderBy?: SqlOrderBy;

package/dist/index.js

@@ -1,3 +1,3 @@
-import { $ as getCalendars, A as getStopsAsGeoJSON, B as getNetworks, C as getCalendarAttributes, Ct as untildify, D as getTimeframes, E as getTransfers, F as getRouteAttributes, G as getFrequencies, H as getLocationGroupStops, I as getRoutes, J as getFareRules, K as getFeedInfo, L as getRouteNetworks, M as getStopAreas, N as getShapes, O as getStoptimes, P as getShapesAsGeoJSON, Q as getFareAttributes, R as getRiderCategories, S as getDirections, St as prepDirectory, T as getTranslations, U as getLocationGroups, V as getLocations, W as getLevels, X as getFareMedia, Y as getFareProducts, Z as getFareLegRules, _ as getTimetableNotesReferences, _t as formatGtfsError, a as getDeadheads, at as getAgencies, b as getTimetableStopOrders, bt as generateFolderName, c as getVehiclePositions, ct as importGtfs, d as getTripCapacities, dt as deleteDb, et as getServiceIdsByDate, f as getRidership, ft as openDb, g as getTripsDatedVehicleJourneys, gt as GtfsWarningCode, h as getBoardAlights, ht as GtfsErrorCode, i as getDeadheadTimes, it as getAreas, j as getStopAttributes, k as getStops, l as getTripUpdates, lt as updateGtfsRealtime, m as getRideFeedInfo, mt as GtfsErrorCategory, n as getRunEvents, nt as getBookingRules, o as getServiceAlertInformedEntities, ot as advancedQuery, p as getRiderTrips, pt as GtfsError, q as getFareTransferRules, r as getOpsLocations, rt as getAttributions, s as getServiceAlerts, st as exportGtfs, t as getRunsPieces, tt as getCalendarDates, u as getStopTimeUpdates, ut as closeDb, v as getTimetableNotes, vt as isGtfsError, w as getTrips, wt as unzip, x as getTimetables, y as getTimetablePages, yt as isGtfsValidationError, z as getPathways } from "./src-DuXcyA_N.js";
+import { $ as getCalendars, A as getStopsAsGeoJSON, B as getNetworks, C as getCalendarAttributes, Ct as untildify, D as getTimeframes, E as getTransfers, F as getRouteAttributes, G as getFrequencies, H as getLocationGroupStops, I as getRoutes, J as getFareRules, K as getFeedInfo, L as getRouteNetworks, M as getStopAreas, N as getShapes, O as getStoptimes, P as getShapesAsGeoJSON, Q as getFareAttributes, R as getRiderCategories, S as getDirections, St as prepDirectory, T as getTranslations, U as getLocationGroups, V as getLocations, W as getLevels, X as getFareMedia, Y as getFareProducts, Z as getFareLegRules, _ as getTimetableNotesReferences, _t as formatGtfsError, a as getDeadheads, at as getAgencies, b as getTimetableStopOrders, bt as generateFolderName, c as getVehiclePositions, ct as importGtfs, d as getTripCapacities, dt as deleteDb, et as getServiceIdsByDate, f as getRidership, ft as openDb, g as getTripsDatedVehicleJourneys, gt as GtfsWarningCode, h as getBoardAlights, ht as GtfsErrorCode, i as getDeadheadTimes, it as getAreas, j as getStopAttributes, k as getStops, l as getTripUpdates, lt as updateGtfsRealtime, m as getRideFeedInfo, mt as GtfsErrorCategory, n as getRunEvents, nt as getBookingRules, o as getServiceAlertInformedEntities, ot as advancedQuery, p as getRiderTrips, pt as GtfsError, q as getFareTransferRules, r as getOpsLocations, rt as getAttributions, s as getServiceAlerts, st as exportGtfs, t as getRunsPieces, tt as getCalendarDates, u as getStopTimeUpdates, ut as closeDb, v as getTimetableNotes, vt as isGtfsError, w as getTrips, wt as unzip, x as getTimetables, y as getTimetablePages, yt as isGtfsValidationError, z as getPathways } from "./src-gj91FR8c.js";
 
 export { GtfsError, GtfsErrorCategory, GtfsErrorCode, GtfsWarningCode, advancedQuery, closeDb, deleteDb, exportGtfs, formatGtfsError, generateFolderName, getAgencies, getAreas, getAttributions, getBoardAlights, getBookingRules, getCalendarAttributes, getCalendarDates, getCalendars, getDeadheadTimes, getDeadheads, getDirections, getFareAttributes, getFareLegRules, getFareMedia, getFareProducts, getFareRules, getFareTransferRules, getFeedInfo, getFrequencies, getLevels, getLocationGroupStops, getLocationGroups, getLocations, getNetworks, getOpsLocations, getPathways, getRideFeedInfo, getRiderCategories, getRiderTrips, getRidership, getRouteAttributes, getRouteNetworks, getRoutes, getRunEvents, getRunsPieces, getServiceAlertInformedEntities, getServiceAlerts, getServiceIdsByDate, getShapes, getShapesAsGeoJSON, getStopAreas, getStopAttributes, getStopTimeUpdates, getStops, getStopsAsGeoJSON, getStoptimes, getTimeframes, getTimetableNotes, getTimetableNotesReferences, getTimetablePages, getTimetableStopOrders, getTimetables, getTransfers, getTranslations, getTripCapacities, getTripUpdates, getTrips, getTripsDatedVehicleJourneys, getVehiclePositions, importGtfs, isGtfsError, isGtfsValidationError, openDb, prepDirectory, untildify, unzip, updateGtfsRealtime };

package/dist/{src-DuXcyA_N.js → src-gj91FR8c.js}

@@ -7,9 +7,9 @@ import stripBomStream from "strip-bom-stream";
 import { temporaryDirectory } from "tempy";
 import mapSeries from "promise-map-series";
 import fs from "fs";
-import Database from "better-sqlite3";
+import Database$1 from "better-sqlite3";
 import { homedir } from "node:os";
-import { cloneDeep, compact, filter, get, groupBy, last, noop, omit, omitBy, orderBy, pick, snakeCase, sortBy, without } from "lodash-es";
+import { compact, filter, get, groupBy, last, noop, omit, omitBy, orderBy, pick, snakeCase, sortBy, without } from "lodash-es";
 import sanitize from "sanitize-filename";
 import StreamZip from "node-stream-zip";
 import { clearLine, cursorTo } from "node:readline";
@@ -281,7 +281,7 @@ function formatGtfsError(error, options = { verbosity: "developer" }) {
 //#region src/lib/db.ts
 const dbs = {};
 function setupDb(sqlitePath) {
-	const db = new Database(untildify(sqlitePath));
+	const db = new Database$1(untildify(sqlitePath));
 	db.pragma("journal_mode = OFF");
 	db.pragma("synchronous = OFF");
 	db.pragma("temp_store = MEMORY");
@@ -296,7 +296,10 @@ function openDb(config = null) {
 		return setupDb(sqlitePath);
 	}
 	if (Object.keys(dbs).length === 0) return setupDb(":memory:");
-	if (Object.keys(dbs).length === 1) return dbs[Object.keys(dbs)[0]];
+	if (Object.keys(dbs).length === 1) {
+		const filename = Object.keys(dbs)[0];
+		return dbs[filename];
+	}
 	if (Object.keys(dbs).length > 1) throw new GtfsError("Multiple databases open, please specify which one to use.", {
 		code: "GTFS_DB_OPERATION_FAILED",
 		category: "database",
@@ -390,17 +393,16 @@ function consolidateShapes(shapeGroups) {
 	for (const segments of segmentsArray) {
 		consolidatedLineStrings.push([]);
 		for (const segment of segments) {
-			const key1 = segment.flat().join(",");
-			const key2 = segment.reverse().flat().join(",");
+			const [[x1, y1], [x2, y2]] = segment;
+			const key = x1 < x2 || x1 === x2 && y1 <= y2 ? `${x1},${y1},${x2},${y2}` : `${x2},${y2},${x1},${y1}`;
 			const currentLine = last(consolidatedLineStrings);
-			if (!currentLine || keys.has(key1) || keys.has(key2)) {
+			if (!currentLine || keys.has(key)) {
 				consolidatedLineStrings.push([]);
 				continue;
 			}
 			if (currentLine.length === 0) currentLine.push(segment[0]);
 			currentLine.push(segment[1]);
-			keys.add(key1);
-			keys.add(key2);
+			keys.add(key);
 		}
 	}
 	return filter(consolidatedLineStrings, isValidLineString);
@@ -422,7 +424,7 @@ function formatHexColor(color) {
 * @returns {Record<string, unknown>} Formatted properties object
 */
 function formatProperties(properties) {
-	const formattedProperties = cloneDeep(omitBy(properties, (value) => value == null));
+	const formattedProperties = omitBy(properties, (value) => value == null);
 	const formattedRouteColor = formatHexColor(properties.route_color);
 	const formattedRouteTextColor = formatHexColor(properties.route_text_color);
 	if (formattedRouteColor) formattedProperties.route_color = formattedRouteColor;
@@ -1134,6 +1136,42 @@ const extractGtfsFiles = async (task) => {
 		});
 	}
 };
+/**
+* Reads agency.txt from disk after extraction to find the single agency_id for a feed.
+* Returns the raw agency_id string if there is exactly one agency with a non-empty
+* agency_id, or undefined otherwise.
+*/
+const getSingleAgencyId = (downloadDir, csvOptions, logWarning) => new Promise((resolve) => {
+	const filepath = path.join(downloadDir, "agency.txt");
+	if (!existsSync(filepath)) {
+		resolve(void 0);
+		return;
+	}
+	const rows = [];
+	const parser = parse({
+		columns: true,
+		relax_quotes: true,
+		trim: true,
+		skip_empty_lines: true,
+		...csvOptions
+	});
+	parser.on("readable", () => {
+		let record;
+		while (record = parser.read()) rows.push(record);
+	});
+	parser.on("end", () => {
+		if (rows.length !== 1) {
+			resolve(void 0);
+			return;
+		}
+		resolve(rows[0].agency_id?.trim() || void 0);
+	});
+	parser.on("error", (err) => {
+		logWarning(`Unable to parse agency.txt for \`fillEmptyAgencyId\`: ${err.message}`);
+		resolve(void 0);
+	});
+	createReadStream(filepath).pipe(stripBomStream()).pipe(parser);
+});
 const createGtfsTables = (db) => {
 	for (const model of Object.values(models_exports)) {
 		if (!model.schema) continue;
@@ -1179,7 +1217,20 @@ const createGtfsIndexes = (db) => {
 		}
 	}
 };
-const formatGtfsLine = (line, model, totalLineCount) => {
+const AGENCY_ID_BACKFILL_MODELS = /* @__PURE__ */ new Set([
+	"agency",
+	"routes",
+	"fare_attributes",
+	"trip_capacity",
+	"rider_trip",
+	"ridership"
+]);
+function shouldBackfillAgencyId(model, formattedLine) {
+	if (AGENCY_ID_BACKFILL_MODELS.has(model.filenameBase)) return true;
+	if (model.filenameBase === "attributions") return formattedLine.route_id == null && formattedLine.trip_id == null;
+	return false;
+}
+const formatGtfsLine = (line, model, totalLineCount, fillEmptyAgencyId, agencyId) => {
 	const lineNumber = totalLineCount + 1;
 	const formattedLine = {};
 	const filenameBase = model.filenameBase;
@@ -1215,6 +1266,7 @@ const formatGtfsLine = (line, model, totalLineCount) => {
 		if (type === "json") value = JSON.stringify(value);
 		formattedLine[name] = value;
 	}
+	if (fillEmptyAgencyId && agencyId !== void 0 && formattedLine.agency_id == null && shouldBackfillAgencyId(model, formattedLine)) formattedLine.agency_id = agencyId;
 	return formattedLine;
 };
 const BATCH_SIZE = 1e5;
@@ -1291,7 +1343,7 @@ const importGtfsFiles = async (db, task) => {
 					let record;
 					while (record = parser.read()) {
 						totalLineCount += 1;
-						lines.push(formatGtfsLine(record, model, totalLineCount));
+						lines.push(formatGtfsLine(record, model, totalLineCount, task.fillEmptyAgencyId, task.agencyId));
 						if (lines.length >= BATCH_SIZE) {
 							insertLines(lines);
 							lines = [];
@@ -1388,7 +1440,7 @@ const importGtfsFiles = async (db, task) => {
 				return;
 			}
 			totalLineCount += 1;
-			const line = formatGtfsLine({ geojson: data }, model, totalLineCount);
+			const line = formatGtfsLine({ geojson: data }, model, totalLineCount, task.fillEmptyAgencyId, task.agencyId);
 			try {
 				insertLines([line]);
 				task.log(`Importing - ${filename} - ${totalLineCount} lines imported\r`, true);
@@ -1463,6 +1515,8 @@ async function importGtfs(initialConfig) {
 					ignoreErrors: config.ignoreErrors,
 					sqlitePath: config.sqlitePath,
 					prefix: agency.prefix,
+					fillEmptyAgencyId: agency.fillEmptyAgencyId ?? false,
+					agencyId: agency.agencyId,
 					currentTimestamp: Math.floor(Date.now() / 1e3),
 					log: log(config),
 					logWarning: logWarning(config),
@@ -1474,6 +1528,13 @@ async function importGtfs(initialConfig) {
 					await downloadGtfsFiles(task);
 				} else Object.assign(task, { path: agency.path });
 				await extractGtfsFiles(task);
+				if (task.fillEmptyAgencyId) {
+					const agencyIdFromGtfs = await getSingleAgencyId(task.downloadDir, task.csvOptions, task.logWarning);
+					if (agencyIdFromGtfs !== void 0) {
+						if (task.agencyId !== void 0 && task.agencyId !== agencyIdFromGtfs) task.logWarning(`\`agencyId\` "${task.agencyId}" does not match the \`agency_id\` "${agencyIdFromGtfs}" in agency.txt. Using the value from agency.txt.`);
+						task.agencyId = agencyIdFromGtfs;
+					} else if (task.agencyId === void 0) task.logWarning("`fillEmptyAgencyId` is set but a single `agency_id` could not be determined for this feed and no `agencyId` was provided in config. `agency_id` will not be backfilled.");
+				}
 				await importGtfsFiles(db, task);
 				await updateGtfsRealtimeData(task);
 				await rm(tempPath, { recursive: true });
@@ -2308,7 +2369,7 @@ function getVehiclePositions(query = {}, fields = [], orderBy = [], options = {}
 
 //#endregion
 //#region src/lib/gtfs-realtime/service-alerts.ts
-const ENTITY_COLUMNS = new Set([
+const ENTITY_COLUMNS = /* @__PURE__ */ new Set([
 	"alert_id",
 	"stop_id",
 	"route_id",
@@ -2414,4 +2475,4 @@ function getRunsPieces(query = {}, fields = [], orderBy = [], options = {}) {
 
 //#endregion
 export { getCalendars as $, getStopsAsGeoJSON as A, getNetworks as B, getCalendarAttributes as C, untildify as Ct, getTimeframes as D, getTransfers as E, getRouteAttributes as F, getFrequencies as G, getLocationGroupStops as H, getRoutes as I, getFareRules as J, getFeedInfo as K, getRouteNetworks as L, getStopAreas as M, getShapes as N, getStoptimes as O, getShapesAsGeoJSON as P, getFareAttributes as Q, getRiderCategories as R, getDirections as S, prepDirectory as St, getTranslations as T, formatError as Tt, getLocationGroups as U, getLocations as V, getLevels as W, getFareMedia as X, getFareProducts as Y, getFareLegRules as Z, getTimetableNotesReferences as _, formatGtfsError as _t, getDeadheads as a, getAgencies as at, getTimetableStopOrders as b, generateFolderName as bt, getVehiclePositions as c, importGtfs as ct, getTripCapacities as d, deleteDb as dt, getServiceIdsByDate as et, getRidership as f, openDb as ft, getTripsDatedVehicleJourneys as g, GtfsWarningCode as gt, getBoardAlights as h, GtfsErrorCode as ht, getDeadheadTimes as i, getAreas as it, getStopAttributes as j, getStops as k, getTripUpdates as l, updateGtfsRealtime as lt, getRideFeedInfo as m, GtfsErrorCategory as mt, getRunEvents as n, getBookingRules as nt, getServiceAlertInformedEntities as o, advancedQuery as ot, getRiderTrips as p, GtfsError as pt, getFareTransferRules as q, getOpsLocations as r, getAttributions as rt, getServiceAlerts as s, exportGtfs as st, getRunsPieces as t, getCalendarDates as tt, getStopTimeUpdates as u, closeDb as ut, getTimetableNotes as v, isGtfsError as vt, getTrips as w, unzip as wt, getTimetables as x, getConfig as xt, getTimetablePages as y, isGtfsValidationError as yt, getPathways as z };
-//# sourceMappingURL=src-DuXcyA_N.js.map
+//# sourceMappingURL=src-gj91FR8c.js.map