gtfs 4.16.0 → 4.17.1

package/dist/bin/gtfsrealtime-update.js

@@ -122,21 +122,24 @@ var tripUpdates = {
       required: true,
       primary: true,
       index: true,
-      source: "id"
+      source: "id",
+      prefix: true
     },
     {
       name: "vehicle_id",
       type: "text",
       index: true,
       source: "tripUpdate.vehicle.id",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "trip_id",
       type: "text",
       index: true,
       source: "tripUpdate.trip.tripId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "trip_start_time",
@@ -155,7 +158,8 @@ var tripUpdates = {
       type: "text",
       index: true,
       source: "tripUpdate.trip.routeId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "start_date",
@@ -198,7 +202,8 @@ var stopTimeUpdates = {
       type: "text",
       index: true,
       source: "parent.tripUpdate.trip.tripId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "trip_start_time",
@@ -217,14 +222,16 @@ var stopTimeUpdates = {
       type: "text",
       index: true,
       source: "parent.tripUpdate.trip.routeId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "stop_id",
       type: "text",
       index: true,
       source: "stopId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "stop_sequence",
@@ -286,7 +293,8 @@ var vehiclePositions = {
       required: true,
       primary: true,
       index: true,
-      source: "id"
+      source: "id",
+      prefix: true
     },
     {
       name: "bearing",
@@ -328,7 +336,8 @@ var vehiclePositions = {
       type: "text",
       index: true,
       source: "vehicle.trip.tripId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "trip_start_date",
@@ -373,7 +382,8 @@ var vehiclePositions = {
       type: "text",
       index: true,
       source: "vehicle.vehicle.id",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "vehicle_label",
@@ -423,7 +433,8 @@ var serviceAlerts = {
       required: true,
       primary: true,
       index: true,
-      source: "id"
+      source: "id",
+      prefix: true
     },
     {
       name: "active_period",
@@ -502,9 +513,9 @@ var serviceAlerts = {
   ]
 };
 
-// src/models/gtfs-realtime/service-alert-targets.ts
-var serviceAlertTargets = {
-  filenameBase: "service_alert_targets",
+// src/models/gtfs-realtime/service-alert-informed_entities.ts
+var serviceAlertInformedEntities = {
+  filenameBase: "service_alert_informed_entities",
   extension: "gtfs-realtime",
   schema: [
     {
@@ -512,20 +523,45 @@ var serviceAlertTargets = {
       type: "text",
       required: true,
       primary: true,
-      source: "parent.id"
+      source: "parent.id",
+      prefix: true
     },
     {
       name: "stop_id",
       type: "text",
       index: true,
       source: "stopId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "route_id",
       type: "text",
       index: true,
       source: "routeId",
+      default: null,
+      prefix: true
+    },
+    {
+      name: "route_type",
+      type: "integer",
+      index: true,
+      source: "routeType",
+      default: null
+    },
+    {
+      name: "trip_id",
+      type: "text",
+      index: true,
+      source: "trip.tripId",
+      default: null,
+      prefix: true
+    },
+    {
+      name: "direction_id",
+      type: "integer",
+      index: true,
+      source: "directionId",
       default: null
     },
     {
@@ -595,8 +631,8 @@ import { feature, featureCollection } from "@turf/helpers";
 // src/lib/import-gtfs-realtime.ts
 import pluralize from "pluralize";
 import GtfsRealtimeBindings from "gtfs-realtime-bindings";
-import sqlString2 from "sqlstring-sqlite";
 import mapSeries from "promise-map-series";
+import { get } from "lodash-es";
 
 // src/lib/utils.ts
 import sqlString from "sqlstring-sqlite";
@@ -633,39 +669,14 @@ function convertLongTimeToDate(longDate) {
     Long.fromBits(low, high, unsigned).toNumber() * 1e3
   ).toISOString();
 }
-
-// src/lib/import-gtfs-realtime.ts
-function getNestedProperty(obj, defaultValue, path2) {
-  if (path2 === void 0) return defaultValue;
-  const arr = path2.split(".");
-  while (arr.length) {
-    const nextKey = arr.shift();
-    if (nextKey === void 0) {
-      return defaultValue;
-    } else if (obj == null) {
-      return defaultValue;
-    } else if (nextKey?.includes("[")) {
-      const arrayKey = nextKey.match(/(\w*)\[(\d+)\]/);
-      if (arrayKey === null) {
-        return defaultValue;
-      }
-      if (obj[arrayKey[1]] === void 0) {
-        return defaultValue;
-      }
-      if (obj[arrayKey[1]][arrayKey[2]] === void 0) {
-        return defaultValue;
-      }
-      obj = obj[arrayKey[1]][arrayKey[2]];
-    } else {
-      if (obj[nextKey] === void 0) {
-        return defaultValue;
-      }
-      obj = obj[nextKey];
-    }
+function applyPrefixToValue(value, columnShouldBePrefixed, prefix) {
+  if (!columnShouldBePrefixed || prefix === void 0 || value === null) {
+    return value;
   }
-  if (obj?.__isLong__) return convertLongTimeToDate(obj);
-  return obj;
+  return `${prefix}${value}`;
 }
+
+// src/lib/import-gtfs-realtime.ts
 async function fetchGtfsRealtimeData(urlConfig, task) {
   task.log(`Downloading GTFS-Realtime from ${urlConfig.url}`);
   const response = await fetch(urlConfig.url, {
@@ -690,7 +701,7 @@ async function fetchGtfsRealtimeData(urlConfig, task) {
     enums: String,
     longs: String,
     bytes: String,
-    defaults: true,
+    defaults: false,
     arrays: true,
     objects: true,
     oneofs: true
@@ -712,7 +723,7 @@ function removeExpiredRealtimeData(config) {
     `DELETE FROM service_alerts WHERE expiration_timestamp <= strftime('%s','now')`
   ).run();
   db.prepare(
-    `DELETE FROM service_alert_targets WHERE expiration_timestamp <= strftime('%s','now')`
+    `DELETE FROM service_alert_informed_entities WHERE expiration_timestamp <= strftime('%s','now')`
   ).run();
   log(config)(`Removed expired GTFS-Realtime data\r`, true);
 }
@@ -723,100 +734,117 @@ function prepareRealtimeFieldValue(entity, column, task) {
   if (column.name === "expiration_timestamp") {
     return task.currentTimestamp + task.gtfsRealtimeExpirationSeconds;
   }
-  const value = getNestedProperty(entity, column.default, column.source);
-  if (column.type === "json") {
-    return sqlString2.escape(JSON.stringify(value));
-  }
-  return sqlString2.escape(value);
+  const baseValue = column.source === void 0 ? column.default : get(entity, column.source, column.default);
+  const timeAdjustedValue = baseValue?.__isLong__ ? convertLongTimeToDate(baseValue) : baseValue;
+  const prefixedValue = applyPrefixToValue(
+    timeAdjustedValue,
+    column.prefix,
+    task.prefix
+  );
+  return column.type === "json" ? JSON.stringify(prefixedValue) : prefixedValue;
 }
 async function processRealtimeAlerts(db, gtfsRealtimeData, task) {
-  task.log(`Download successful`);
+  const alertStmt = db.prepare(
+    `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(
+      ", "
+    )}) VALUES (${serviceAlerts.schema.map(() => "?").join(", ")})`
+  );
+  const informedEntityStmt = db.prepare(
+    `REPLACE INTO ${serviceAlertInformedEntities.filenameBase} (${serviceAlertInformedEntities.schema.map((column) => column.name).join(
+      ", "
+    )}) VALUES (${serviceAlertInformedEntities.schema.map(() => "?").join(", ")})`
+  );
   let totalLineCount = 0;
-  for (const entity of gtfsRealtimeData.entity) {
-    const fieldValues = serviceAlerts.schema.map(
-      (column) => prepareRealtimeFieldValue(entity, column, task)
-    );
-    try {
-      db.prepare(
-        `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-      ).run();
-    } catch (error) {
-      task.logWarning(`Import error: ${error.message}`);
-    }
-    if (!entity.alert.informedEntity || entity.alert.informedEntity.length === 0) {
-      task.logWarning(
-        `Import error: No informed entities found for alert id=${entity.id}`
+  db.transaction(() => {
+    for (const entity of gtfsRealtimeData.entity) {
+      const fieldValues = serviceAlerts.schema.map(
+        (column) => prepareRealtimeFieldValue(entity, column, task)
       );
-    } else {
-      const alertTargetArray = [];
-      for (const informedEntity of entity.alert.informedEntity) {
-        informedEntity.parent = entity;
-        const subValues = serviceAlertTargets.schema.map(
-          (column) => prepareRealtimeFieldValue(informedEntity, column, task)
-        );
-        alertTargetArray.push(`(${subValues.join(", ")})`);
-        totalLineCount++;
-      }
       try {
-        db.prepare(
-          `REPLACE INTO ${serviceAlertTargets.filenameBase} (${serviceAlertTargets.schema.map((column) => column.name).join(", ")}) VALUES ${alertTargetArray.join(", ")}`
-        ).run();
+        alertStmt.run(fieldValues);
+        if (entity.alert.informedEntity?.length) {
+          const informedEntities = entity.alert.informedEntity.map(
+            (informedEntity) => {
+              informedEntity.parent = entity;
+              return serviceAlertInformedEntities.schema.map(
+                (column) => prepareRealtimeFieldValue(informedEntity, column, task)
+              );
+            }
+          );
+          for (const values of informedEntities) {
+            informedEntityStmt.run(values);
+          }
+        }
+        totalLineCount++;
       } catch (error) {
         task.logWarning(`Import error: ${error.message}`);
       }
     }
-    task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
-  }
+    task.log(
+      `Importing - GTFS-Realtime service alerts - ${totalLineCount} entries imported\r`,
+      true
+    );
+  })();
 }
 async function processRealtimeTripUpdates(db, gtfsRealtimeData, task) {
-  task.log(`Download successful`);
   let totalLineCount = 0;
-  for (const entity of gtfsRealtimeData.entity) {
-    const fieldValues = tripUpdates.schema.map(
-      (column) => prepareRealtimeFieldValue(entity, column, task)
-    );
-    try {
-      db.prepare(
-        `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-      ).run();
-    } catch (error) {
-      task.logWarning(`Import error: ${error.message}`);
-    }
-    const stopTimeUpdateArray = [];
-    for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
-      stopTimeUpdate.parent = entity;
-      const subValues = stopTimeUpdates.schema.map(
-        (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
-      );
-      stopTimeUpdateArray.push(`(${subValues.join(", ")})`);
-      totalLineCount++;
-    }
-    try {
-      db.prepare(
-        `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(", ")}) VALUES ${stopTimeUpdateArray.join(", ")}`
-      ).run();
-    } catch (error) {
-      task.logWarning(`Import error: ${error.message}`);
+  const tripUpdateStmt = db.prepare(
+    `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(
+      ", "
+    )}) VALUES (${tripUpdates.schema.map(() => "?").join(", ")})`
+  );
+  const stopTimeStmt = db.prepare(
+    `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(
+      ", "
+    )}) VALUES (${stopTimeUpdates.schema.map(() => "?").join(", ")})`
+  );
+  db.transaction(() => {
+    for (const entity of gtfsRealtimeData.entity) {
+      try {
+        const fieldValues = tripUpdates.schema.map(
+          (column) => prepareRealtimeFieldValue(entity, column, task)
+        );
+        tripUpdateStmt.run(fieldValues);
+        for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
+          stopTimeUpdate.parent = entity;
+          const values = stopTimeUpdates.schema.map(
+            (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
+          );
+          stopTimeStmt.run(values);
+        }
+        totalLineCount++;
+      } catch (error) {
+        task.logWarning(`Import error: ${error.message}`);
+      }
     }
-    task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
-  }
+    task.log(
+      `Importing - GTFS-Realtime trip updates - ${totalLineCount} entries imported\r`,
+      true
+    );
+  })();
 }
 async function processRealtimeVehiclePositions(db, gtfsRealtimeData, task) {
-  task.log(`Download successful`);
   let totalLineCount = 0;
-  for (const entity of gtfsRealtimeData.entity) {
-    const fieldValues = vehiclePositions.schema.map(
-      (column) => prepareRealtimeFieldValue(entity, column, task)
-    );
-    try {
-      db.prepare(
-        `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-      ).run();
-    } catch (error) {
-      task.logWarning(`Import error: ${error.message}`);
+  const vehiclePositionStmt = db.prepare(
+    `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(
+      ", "
+    )}) VALUES (${vehiclePositions.schema.map(() => "?").join(", ")})`
+  );
+  db.transaction(() => {
+    for (const entity of gtfsRealtimeData.entity) {
+      try {
+        const fieldValues = vehiclePositions.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
+        vehiclePositionStmt.run(fieldValues);
+        totalLineCount++;
+      } catch (error) {
+        task.logWarning(`Import error: ${error.message}`);
+      }
     }
-    task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
-  }
+    task.log(
+      `Importing - GTFS-Realtime vehicle positions - ${totalLineCount} entries imported\r`,
+      true
+    );
+  })();
 }
 async function updateGtfsRealtimeData(task) {
   if (task.realtimeAlerts === void 0 && task.realtimeTripUpdates === void 0 && task.realtimeVehiclePositions === void 0) {
@@ -897,6 +925,7 @@ async function updateGtfsRealtime(initialConfig) {
           gtfsRealtimeExpirationSeconds: config.gtfsRealtimeExpirationSeconds,
           ignoreErrors: config.ignoreErrors,
           sqlitePath: config.sqlitePath,
+          prefix: agency2.prefix,
           currentTimestamp: Math.floor(Date.now() / 1e3),
           log: log(config),
           logWarning: logWarning(config),
@@ -933,12 +962,12 @@ async function updateGtfsRealtime(initialConfig) {
 import { without, compact as compact2 } from "lodash-es";
 import pluralize3 from "pluralize";
 import { stringify } from "csv-stringify";
-import sqlString3 from "sqlstring-sqlite";
+import sqlString2 from "sqlstring-sqlite";
 import mapSeries3 from "promise-map-series";
 import untildify4 from "untildify";
 
 // src/lib/advancedQuery.ts
-import sqlString4 from "sqlstring-sqlite";
+import sqlString3 from "sqlstring-sqlite";
 
 // src/lib/gtfs/routes.ts
 import { omit as omit3, pick } from "lodash-es";
@@ -952,7 +981,7 @@ import { omit as omit5, orderBy, pick as pick3 } from "lodash-es";
 
 // src/lib/gtfs/stop-times.ts
 import { omit as omit6 } from "lodash-es";
-import sqlString5 from "sqlstring-sqlite";
+import sqlString4 from "sqlstring-sqlite";
 
 // src/bin/gtfsrealtime-update.ts
 var pe = new PrettyError();