gtfs 4.17.0 → 4.17.2

package/dist/index.js

@@ -945,7 +945,6 @@ var riderCategories = {
     {
       name: "is_default_fare_category",
       type: "integer",
-      required: true,
       min: 0,
       max: 1
     },
@@ -1285,7 +1284,8 @@ var stops = {
     {
       name: "parent_station",
       type: "text",
-      index: true
+      index: true,
+      prefix: true
     },
     {
       name: "stop_timezone",
@@ -2381,21 +2381,24 @@ var tripUpdates = {
       required: true,
       primary: true,
       index: true,
-      source: "id"
+      source: "id",
+      prefix: true
     },
     {
       name: "vehicle_id",
       type: "text",
       index: true,
       source: "tripUpdate.vehicle.id",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "trip_id",
       type: "text",
       index: true,
       source: "tripUpdate.trip.tripId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "trip_start_time",
@@ -2414,7 +2417,8 @@ var tripUpdates = {
       type: "text",
       index: true,
       source: "tripUpdate.trip.routeId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "start_date",
@@ -2457,7 +2461,8 @@ var stopTimeUpdates = {
       type: "text",
       index: true,
       source: "parent.tripUpdate.trip.tripId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "trip_start_time",
@@ -2476,14 +2481,16 @@ var stopTimeUpdates = {
       type: "text",
       index: true,
       source: "parent.tripUpdate.trip.routeId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "stop_id",
       type: "text",
       index: true,
       source: "stopId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "stop_sequence",
@@ -2545,7 +2552,8 @@ var vehiclePositions = {
       required: true,
       primary: true,
       index: true,
-      source: "id"
+      source: "id",
+      prefix: true
     },
     {
       name: "bearing",
@@ -2587,7 +2595,8 @@ var vehiclePositions = {
       type: "text",
       index: true,
       source: "vehicle.trip.tripId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "trip_start_date",
@@ -2632,7 +2641,8 @@ var vehiclePositions = {
       type: "text",
       index: true,
       source: "vehicle.vehicle.id",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "vehicle_label",
@@ -2682,7 +2692,8 @@ var serviceAlerts = {
       required: true,
       primary: true,
       index: true,
-      source: "id"
+      source: "id",
+      prefix: true
     },
     {
       name: "active_period",
@@ -2771,21 +2782,24 @@ var serviceAlertInformedEntities = {
       type: "text",
       required: true,
       primary: true,
-      source: "parent.id"
+      source: "parent.id",
+      prefix: true
     },
     {
       name: "stop_id",
       type: "text",
       index: true,
       source: "stopId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "route_id",
       type: "text",
       index: true,
       source: "routeId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "route_type",
@@ -2799,7 +2813,8 @@ var serviceAlertInformedEntities = {
       type: "text",
       index: true,
       source: "trip.tripId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "direction_id",
@@ -3383,8 +3398,8 @@ function stopsToGeoJSONFeatureCollection(stops2) {
 // src/lib/import-gtfs-realtime.ts
 import pluralize from "pluralize";
 import GtfsRealtimeBindings from "gtfs-realtime-bindings";
-import sqlString2 from "sqlstring-sqlite";
 import mapSeries from "promise-map-series";
+import { get } from "lodash-es";
 
 // src/lib/utils.ts
 import sqlString from "sqlstring-sqlite";
@@ -3545,39 +3560,14 @@ function formatCurrency(value, currency) {
 function getTimestampColumnName(columnName) {
   return columnName.endsWith("time") ? `${columnName}stamp` : `${columnName}_timestamp`;
 }
-
-// src/lib/import-gtfs-realtime.ts
-function getNestedProperty(obj, defaultValue, path3) {
-  if (path3 === void 0) return defaultValue;
-  const arr = path3.split(".");
-  while (arr.length) {
-    const nextKey = arr.shift();
-    if (nextKey === void 0) {
-      return defaultValue;
-    } else if (obj == null) {
-      return defaultValue;
-    } else if (nextKey?.includes("[")) {
-      const arrayKey = nextKey.match(/(\w*)\[(\d+)\]/);
-      if (arrayKey === null) {
-        return defaultValue;
-      }
-      if (obj[arrayKey[1]] === void 0) {
-        return defaultValue;
-      }
-      if (obj[arrayKey[1]][arrayKey[2]] === void 0) {
-        return defaultValue;
-      }
-      obj = obj[arrayKey[1]][arrayKey[2]];
-    } else {
-      if (obj[nextKey] === void 0) {
-        return defaultValue;
-      }
-      obj = obj[nextKey];
-    }
+function applyPrefixToValue(value, columnShouldBePrefixed, prefix) {
+  if (!columnShouldBePrefixed || prefix === void 0 || value === null) {
+    return value;
   }
-  if (obj?.__isLong__) return convertLongTimeToDate(obj);
-  return obj;
+  return `${prefix}${value}`;
 }
+
+// src/lib/import-gtfs-realtime.ts
 async function fetchGtfsRealtimeData(urlConfig, task) {
   task.log(`Downloading GTFS-Realtime from ${urlConfig.url}`);
   const response = await fetch(urlConfig.url, {
@@ -3635,100 +3625,117 @@ function prepareRealtimeFieldValue(entity, column, task) {
   if (column.name === "expiration_timestamp") {
     return task.currentTimestamp + task.gtfsRealtimeExpirationSeconds;
   }
-  const value = getNestedProperty(entity, column.default, column.source);
-  if (column.type === "json") {
-    return sqlString2.escape(JSON.stringify(value));
-  }
-  return sqlString2.escape(value);
+  const baseValue = column.source === void 0 ? column.default : get(entity, column.source, column.default);
+  const timeAdjustedValue = baseValue?.__isLong__ ? convertLongTimeToDate(baseValue) : baseValue;
+  const prefixedValue = applyPrefixToValue(
+    timeAdjustedValue,
+    column.prefix,
+    task.prefix
+  );
+  return column.type === "json" ? JSON.stringify(prefixedValue) : prefixedValue;
 }
 async function processRealtimeAlerts(db, gtfsRealtimeData, task) {
-  task.log(`Download successful`);
+  const alertStmt = db.prepare(
+    `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(
+      ", "
+    )}) VALUES (${serviceAlerts.schema.map(() => "?").join(", ")})`
+  );
+  const informedEntityStmt = db.prepare(
+    `REPLACE INTO ${serviceAlertInformedEntities.filenameBase} (${serviceAlertInformedEntities.schema.map((column) => column.name).join(
+      ", "
+    )}) VALUES (${serviceAlertInformedEntities.schema.map(() => "?").join(", ")})`
+  );
   let totalLineCount = 0;
-  for (const entity of gtfsRealtimeData.entity) {
-    const fieldValues = serviceAlerts.schema.map(
-      (column) => prepareRealtimeFieldValue(entity, column, task)
-    );
-    try {
-      db.prepare(
-        `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-      ).run();
-    } catch (error) {
-      task.logWarning(`Import error: ${error.message}`);
-    }
-    if (!entity.alert.informedEntity || entity.alert.informedEntity.length === 0) {
-      task.logWarning(
-        `Import error: No informed entities found for alert id=${entity.id}`
+  db.transaction(() => {
+    for (const entity of gtfsRealtimeData.entity) {
+      const fieldValues = serviceAlerts.schema.map(
+        (column) => prepareRealtimeFieldValue(entity, column, task)
       );
-    } else {
-      const informedEntities = [];
-      for (const informedEntity of entity.alert.informedEntity) {
-        informedEntity.parent = entity;
-        const subValues = serviceAlertInformedEntities.schema.map(
-          (column) => prepareRealtimeFieldValue(informedEntity, column, task)
-        );
-        informedEntities.push(`(${subValues.join(", ")})`);
-        totalLineCount++;
-      }
       try {
-        db.prepare(
-          `REPLACE INTO ${serviceAlertInformedEntities.filenameBase} (${serviceAlertInformedEntities.schema.map((column) => column.name).join(", ")}) VALUES ${informedEntities.join(", ")}`
-        ).run();
+        alertStmt.run(fieldValues);
+        if (entity.alert.informedEntity?.length) {
+          const informedEntities = entity.alert.informedEntity.map(
+            (informedEntity) => {
+              informedEntity.parent = entity;
+              return serviceAlertInformedEntities.schema.map(
+                (column) => prepareRealtimeFieldValue(informedEntity, column, task)
+              );
+            }
+          );
+          for (const values of informedEntities) {
+            informedEntityStmt.run(values);
+          }
+        }
+        totalLineCount++;
       } catch (error) {
         task.logWarning(`Import error: ${error.message}`);
       }
     }
-    task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
-  }
+    task.log(
+      `Importing - GTFS-Realtime service alerts - ${totalLineCount} entries imported\r`,
+      true
+    );
+  })();
 }
 async function processRealtimeTripUpdates(db, gtfsRealtimeData, task) {
-  task.log(`Download successful`);
   let totalLineCount = 0;
-  for (const entity of gtfsRealtimeData.entity) {
-    const fieldValues = tripUpdates.schema.map(
-      (column) => prepareRealtimeFieldValue(entity, column, task)
-    );
-    try {
-      db.prepare(
-        `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-      ).run();
-    } catch (error) {
-      task.logWarning(`Import error: ${error.message}`);
-    }
-    const stopTimeUpdateArray = [];
-    for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
-      stopTimeUpdate.parent = entity;
-      const subValues = stopTimeUpdates.schema.map(
-        (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
-      );
-      stopTimeUpdateArray.push(`(${subValues.join(", ")})`);
-      totalLineCount++;
-    }
-    try {
-      db.prepare(
-        `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(", ")}) VALUES ${stopTimeUpdateArray.join(", ")}`
-      ).run();
-    } catch (error) {
-      task.logWarning(`Import error: ${error.message}`);
+  const tripUpdateStmt = db.prepare(
+    `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(
+      ", "
+    )}) VALUES (${tripUpdates.schema.map(() => "?").join(", ")})`
+  );
+  const stopTimeStmt = db.prepare(
+    `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(
+      ", "
+    )}) VALUES (${stopTimeUpdates.schema.map(() => "?").join(", ")})`
+  );
+  db.transaction(() => {
+    for (const entity of gtfsRealtimeData.entity) {
+      try {
+        const fieldValues = tripUpdates.schema.map(
+          (column) => prepareRealtimeFieldValue(entity, column, task)
+        );
+        tripUpdateStmt.run(fieldValues);
+        for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
+          stopTimeUpdate.parent = entity;
+          const values = stopTimeUpdates.schema.map(
+            (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
+          );
+          stopTimeStmt.run(values);
+        }
+        totalLineCount++;
+      } catch (error) {
+        task.logWarning(`Import error: ${error.message}`);
+      }
     }
-    task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
-  }
+    task.log(
+      `Importing - GTFS-Realtime trip updates - ${totalLineCount} entries imported\r`,
+      true
+    );
+  })();
 }
 async function processRealtimeVehiclePositions(db, gtfsRealtimeData, task) {
-  task.log(`Download successful`);
   let totalLineCount = 0;
-  for (const entity of gtfsRealtimeData.entity) {
-    const fieldValues = vehiclePositions.schema.map(
-      (column) => prepareRealtimeFieldValue(entity, column, task)
-    );
-    try {
-      db.prepare(
-        `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-      ).run();
-    } catch (error) {
-      task.logWarning(`Import error: ${error.message}`);
+  const vehiclePositionStmt = db.prepare(
+    `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(
+      ", "
+    )}) VALUES (${vehiclePositions.schema.map(() => "?").join(", ")})`
+  );
+  db.transaction(() => {
+    for (const entity of gtfsRealtimeData.entity) {
+      try {
+        const fieldValues = vehiclePositions.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
+        vehiclePositionStmt.run(fieldValues);
+        totalLineCount++;
+      } catch (error) {
+        task.logWarning(`Import error: ${error.message}`);
+      }
     }
-    task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
-  }
+    task.log(
+      `Importing - GTFS-Realtime vehicle positions - ${totalLineCount} entries imported\r`,
+      true
+    );
+  })();
 }
 async function updateGtfsRealtimeData(task) {
   if (task.realtimeAlerts === void 0 && task.realtimeTripUpdates === void 0 && task.realtimeVehiclePositions === void 0) {
@@ -3809,6 +3816,7 @@ async function updateGtfsRealtime(initialConfig) {
           gtfsRealtimeExpirationSeconds: config.gtfsRealtimeExpirationSeconds,
           ignoreErrors: config.ignoreErrors,
           sqlitePath: config.sqlitePath,
+          prefix: agency2.prefix,
           currentTimestamp: Math.floor(Date.now() / 1e3),
           log: log(config),
           logWarning: logWarning(config),
@@ -4092,7 +4100,11 @@ var importGtfsFiles = (db, task) => mapSeries2(
                 line
               ).map(([columnName, value]) => [
                 columnName,
-                prefixedColumns.has(columnName) && value !== null ? `${task.prefix}${value}` : value
+                applyPrefixToValue(
+                  value,
+                  prefixedColumns.has(columnName),
+                  task.prefix
+                )
               ])
             );
             insert.run(prefixedLine);
@@ -4261,7 +4273,7 @@ import { writeFile as writeFile2 } from "node:fs/promises";
 import { without, compact as compact2 } from "lodash-es";
 import pluralize3 from "pluralize";
 import { stringify } from "csv-stringify";
-import sqlString3 from "sqlstring-sqlite";
+import sqlString2 from "sqlstring-sqlite";
 import mapSeries3 from "promise-map-series";
 import untildify4 from "untildify";
 var getAgencies = (db, config) => {
@@ -4313,7 +4325,7 @@ var exportGtfs = async (initialConfig) => {
         exportPath,
         `${model.filenameBase}.${model.filenameExtension}`
       );
-      const tableName = sqlString3.escapeId(model.filenameBase);
+      const tableName = sqlString2.escapeId(model.filenameBase);
       const lines = db.prepare(`SELECT * FROM ${tableName};`).all();
       if (!lines || lines.length === 0) {
         if (!model.nonstandard) {
@@ -4375,7 +4387,7 @@ var exportGtfs = async (initialConfig) => {
 };
 
 // src/lib/advancedQuery.ts
-import sqlString4 from "sqlstring-sqlite";
+import sqlString3 from "sqlstring-sqlite";
 function advancedQuery(table, advancedQueryOptions) {
   const defaultOptions = {
     query: {},
@@ -4386,7 +4398,7 @@ function advancedQuery(table, advancedQueryOptions) {
   };
   const queryOptions = { ...defaultOptions, ...advancedQueryOptions };
   const db = queryOptions.options.db ?? openDb();
-  const tableName = sqlString4.escapeId(table);
+  const tableName = sqlString3.escapeId(table);
   const selectClause = formatSelectClause(queryOptions.fields);
   const whereClause = formatWhereClauses(queryOptions.query);
   const joinClause = formatJoinClause(queryOptions.join);
@@ -4928,7 +4940,7 @@ function getStopsAsGeoJSON(query = {}, options = {}) {
 
 // src/lib/gtfs/stop-times.ts
 import { omit as omit6 } from "lodash-es";
-import sqlString5 from "sqlstring-sqlite";
+import sqlString4 from "sqlstring-sqlite";
 function getStoptimes(query = {}, fields = [], orderBy2 = [], options = {}) {
   const db = options.db ?? openDb();
   const tableName = "stop_times";
@@ -4945,7 +4957,7 @@ function getStoptimes(query = {}, fields = [], orderBy2 = [], options = {}) {
       throw new Error("`date` must be a number in yyyymmdd format");
     }
     const serviceIds = getServiceIdsByDate(query.date);
-    const tripSubquery = `SELECT DISTINCT trip_id FROM trips WHERE service_id IN (${serviceIds.map((id) => sqlString5.escape(id)).join(",")})`;
+    const tripSubquery = `SELECT DISTINCT trip_id FROM trips WHERE service_id IN (${serviceIds.map((id) => sqlString4.escape(id)).join(",")})`;
     whereClauses.push(`trip_id IN (${tripSubquery})`);
   }
   if (query.start_time) {