gtfs 4.16.0 → 4.17.1

package/dist/bin/gtfs-import.js

@@ -168,7 +168,7 @@ __export(models_exports, {
   routes: () => routes,
   runEvents: () => runEvents,
   runsPieces: () => runsPieces,
-  serviceAlertTargets: () => serviceAlertTargets,
+  serviceAlertInformedEntities: () => serviceAlertInformedEntities,
   serviceAlerts: () => serviceAlerts,
   shapes: () => shapes,
   stopAreas: () => stopAreas,
@@ -1399,7 +1399,8 @@ var stops = {
     {
       name: "parent_station",
       type: "text",
-      index: true
+      index: true,
+      prefix: true
     },
     {
       name: "stop_timezone",
@@ -2495,21 +2496,24 @@ var tripUpdates = {
       required: true,
       primary: true,
       index: true,
-      source: "id"
+      source: "id",
+      prefix: true
     },
     {
       name: "vehicle_id",
       type: "text",
       index: true,
       source: "tripUpdate.vehicle.id",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "trip_id",
       type: "text",
       index: true,
       source: "tripUpdate.trip.tripId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "trip_start_time",
@@ -2528,7 +2532,8 @@ var tripUpdates = {
       type: "text",
       index: true,
       source: "tripUpdate.trip.routeId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "start_date",
@@ -2571,7 +2576,8 @@ var stopTimeUpdates = {
       type: "text",
       index: true,
       source: "parent.tripUpdate.trip.tripId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "trip_start_time",
@@ -2590,14 +2596,16 @@ var stopTimeUpdates = {
       type: "text",
       index: true,
       source: "parent.tripUpdate.trip.routeId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "stop_id",
       type: "text",
       index: true,
       source: "stopId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "stop_sequence",
@@ -2659,7 +2667,8 @@ var vehiclePositions = {
       required: true,
       primary: true,
       index: true,
-      source: "id"
+      source: "id",
+      prefix: true
     },
     {
       name: "bearing",
@@ -2701,7 +2710,8 @@ var vehiclePositions = {
       type: "text",
       index: true,
       source: "vehicle.trip.tripId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "trip_start_date",
@@ -2746,7 +2756,8 @@ var vehiclePositions = {
       type: "text",
       index: true,
       source: "vehicle.vehicle.id",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "vehicle_label",
@@ -2796,7 +2807,8 @@ var serviceAlerts = {
       required: true,
       primary: true,
       index: true,
-      source: "id"
+      source: "id",
+      prefix: true
     },
     {
       name: "active_period",
@@ -2875,9 +2887,9 @@ var serviceAlerts = {
   ]
 };
 
-// src/models/gtfs-realtime/service-alert-targets.ts
-var serviceAlertTargets = {
-  filenameBase: "service_alert_targets",
+// src/models/gtfs-realtime/service-alert-informed_entities.ts
+var serviceAlertInformedEntities = {
+  filenameBase: "service_alert_informed_entities",
   extension: "gtfs-realtime",
   schema: [
     {
@@ -2885,20 +2897,45 @@ var serviceAlertTargets = {
       type: "text",
       required: true,
       primary: true,
-      source: "parent.id"
+      source: "parent.id",
+      prefix: true
     },
     {
       name: "stop_id",
       type: "text",
       index: true,
       source: "stopId",
-      default: null
+      default: null,
+      prefix: true
     },
     {
       name: "route_id",
       type: "text",
       index: true,
       source: "routeId",
+      default: null,
+      prefix: true
+    },
+    {
+      name: "route_type",
+      type: "integer",
+      index: true,
+      source: "routeType",
+      default: null
+    },
+    {
+      name: "trip_id",
+      type: "text",
+      index: true,
+      source: "trip.tripId",
+      default: null,
+      prefix: true
+    },
+    {
+      name: "direction_id",
+      type: "integer",
+      index: true,
+      source: "directionId",
       default: null
     },
     {
@@ -3279,8 +3316,8 @@ function isValidJSON(string) {
 // src/lib/import-gtfs-realtime.ts
 import pluralize from "pluralize";
 import GtfsRealtimeBindings from "gtfs-realtime-bindings";
-import sqlString2 from "sqlstring-sqlite";
 import mapSeries from "promise-map-series";
+import { get } from "lodash-es";
 
 // src/lib/utils.ts
 import sqlString from "sqlstring-sqlite";
@@ -3337,39 +3374,14 @@ function padLeadingZeros(time) {
 function getTimestampColumnName(columnName) {
   return columnName.endsWith("time") ? `${columnName}stamp` : `${columnName}_timestamp`;
 }
-
-// src/lib/import-gtfs-realtime.ts
-function getNestedProperty(obj, defaultValue, path3) {
-  if (path3 === void 0) return defaultValue;
-  const arr = path3.split(".");
-  while (arr.length) {
-    const nextKey = arr.shift();
-    if (nextKey === void 0) {
-      return defaultValue;
-    } else if (obj == null) {
-      return defaultValue;
-    } else if (nextKey?.includes("[")) {
-      const arrayKey = nextKey.match(/(\w*)\[(\d+)\]/);
-      if (arrayKey === null) {
-        return defaultValue;
-      }
-      if (obj[arrayKey[1]] === void 0) {
-        return defaultValue;
-      }
-      if (obj[arrayKey[1]][arrayKey[2]] === void 0) {
-        return defaultValue;
-      }
-      obj = obj[arrayKey[1]][arrayKey[2]];
-    } else {
-      if (obj[nextKey] === void 0) {
-        return defaultValue;
-      }
-      obj = obj[nextKey];
-    }
+function applyPrefixToValue(value, columnShouldBePrefixed, prefix) {
+  if (!columnShouldBePrefixed || prefix === void 0 || value === null) {
+    return value;
   }
-  if (obj?.__isLong__) return convertLongTimeToDate(obj);
-  return obj;
+  return `${prefix}${value}`;
 }
+
+// src/lib/import-gtfs-realtime.ts
 async function fetchGtfsRealtimeData(urlConfig, task) {
   task.log(`Downloading GTFS-Realtime from ${urlConfig.url}`);
   const response = await fetch(urlConfig.url, {
@@ -3394,7 +3406,7 @@ async function fetchGtfsRealtimeData(urlConfig, task) {
     enums: String,
     longs: String,
     bytes: String,
-    defaults: true,
+    defaults: false,
     arrays: true,
     objects: true,
     oneofs: true
@@ -3407,100 +3419,117 @@ function prepareRealtimeFieldValue(entity, column, task) {
   if (column.name === "expiration_timestamp") {
     return task.currentTimestamp + task.gtfsRealtimeExpirationSeconds;
   }
-  const value = getNestedProperty(entity, column.default, column.source);
-  if (column.type === "json") {
-    return sqlString2.escape(JSON.stringify(value));
-  }
-  return sqlString2.escape(value);
+  const baseValue = column.source === void 0 ? column.default : get(entity, column.source, column.default);
+  const timeAdjustedValue = baseValue?.__isLong__ ? convertLongTimeToDate(baseValue) : baseValue;
+  const prefixedValue = applyPrefixToValue(
+    timeAdjustedValue,
+    column.prefix,
+    task.prefix
+  );
+  return column.type === "json" ? JSON.stringify(prefixedValue) : prefixedValue;
 }
 async function processRealtimeAlerts(db, gtfsRealtimeData, task) {
-  task.log(`Download successful`);
+  const alertStmt = db.prepare(
+    `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(
+      ", "
+    )}) VALUES (${serviceAlerts.schema.map(() => "?").join(", ")})`
+  );
+  const informedEntityStmt = db.prepare(
+    `REPLACE INTO ${serviceAlertInformedEntities.filenameBase} (${serviceAlertInformedEntities.schema.map((column) => column.name).join(
+      ", "
+    )}) VALUES (${serviceAlertInformedEntities.schema.map(() => "?").join(", ")})`
+  );
   let totalLineCount = 0;
-  for (const entity of gtfsRealtimeData.entity) {
-    const fieldValues = serviceAlerts.schema.map(
-      (column) => prepareRealtimeFieldValue(entity, column, task)
-    );
-    try {
-      db.prepare(
-        `REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-      ).run();
-    } catch (error) {
-      task.logWarning(`Import error: ${error.message}`);
-    }
-    if (!entity.alert.informedEntity || entity.alert.informedEntity.length === 0) {
-      task.logWarning(
-        `Import error: No informed entities found for alert id=${entity.id}`
+  db.transaction(() => {
+    for (const entity of gtfsRealtimeData.entity) {
+      const fieldValues = serviceAlerts.schema.map(
+        (column) => prepareRealtimeFieldValue(entity, column, task)
       );
-    } else {
-      const alertTargetArray = [];
-      for (const informedEntity of entity.alert.informedEntity) {
-        informedEntity.parent = entity;
-        const subValues = serviceAlertTargets.schema.map(
-          (column) => prepareRealtimeFieldValue(informedEntity, column, task)
-        );
-        alertTargetArray.push(`(${subValues.join(", ")})`);
-        totalLineCount++;
-      }
       try {
-        db.prepare(
-          `REPLACE INTO ${serviceAlertTargets.filenameBase} (${serviceAlertTargets.schema.map((column) => column.name).join(", ")}) VALUES ${alertTargetArray.join(", ")}`
-        ).run();
+        alertStmt.run(fieldValues);
+        if (entity.alert.informedEntity?.length) {
+          const informedEntities = entity.alert.informedEntity.map(
+            (informedEntity) => {
+              informedEntity.parent = entity;
+              return serviceAlertInformedEntities.schema.map(
+                (column) => prepareRealtimeFieldValue(informedEntity, column, task)
+              );
+            }
+          );
+          for (const values of informedEntities) {
+            informedEntityStmt.run(values);
+          }
+        }
+        totalLineCount++;
       } catch (error) {
         task.logWarning(`Import error: ${error.message}`);
       }
     }
-    task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
-  }
+    task.log(
+      `Importing - GTFS-Realtime service alerts - ${totalLineCount} entries imported\r`,
+      true
+    );
+  })();
 }
 async function processRealtimeTripUpdates(db, gtfsRealtimeData, task) {
-  task.log(`Download successful`);
   let totalLineCount = 0;
-  for (const entity of gtfsRealtimeData.entity) {
-    const fieldValues = tripUpdates.schema.map(
-      (column) => prepareRealtimeFieldValue(entity, column, task)
-    );
-    try {
-      db.prepare(
-        `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-      ).run();
-    } catch (error) {
-      task.logWarning(`Import error: ${error.message}`);
-    }
-    const stopTimeUpdateArray = [];
-    for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
-      stopTimeUpdate.parent = entity;
-      const subValues = stopTimeUpdates.schema.map(
-        (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
-      );
-      stopTimeUpdateArray.push(`(${subValues.join(", ")})`);
-      totalLineCount++;
-    }
-    try {
-      db.prepare(
-        `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(", ")}) VALUES ${stopTimeUpdateArray.join(", ")}`
-      ).run();
-    } catch (error) {
-      task.logWarning(`Import error: ${error.message}`);
+  const tripUpdateStmt = db.prepare(
+    `REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(
+      ", "
+    )}) VALUES (${tripUpdates.schema.map(() => "?").join(", ")})`
+  );
+  const stopTimeStmt = db.prepare(
+    `REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(
+      ", "
+    )}) VALUES (${stopTimeUpdates.schema.map(() => "?").join(", ")})`
+  );
+  db.transaction(() => {
+    for (const entity of gtfsRealtimeData.entity) {
+      try {
+        const fieldValues = tripUpdates.schema.map(
+          (column) => prepareRealtimeFieldValue(entity, column, task)
+        );
+        tripUpdateStmt.run(fieldValues);
+        for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
+          stopTimeUpdate.parent = entity;
+          const values = stopTimeUpdates.schema.map(
+            (column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
+          );
+          stopTimeStmt.run(values);
+        }
+        totalLineCount++;
+      } catch (error) {
+        task.logWarning(`Import error: ${error.message}`);
+      }
     }
-    task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
-  }
+    task.log(
+      `Importing - GTFS-Realtime trip updates - ${totalLineCount} entries imported\r`,
+      true
+    );
+  })();
 }
 async function processRealtimeVehiclePositions(db, gtfsRealtimeData, task) {
-  task.log(`Download successful`);
   let totalLineCount = 0;
-  for (const entity of gtfsRealtimeData.entity) {
-    const fieldValues = vehiclePositions.schema.map(
-      (column) => prepareRealtimeFieldValue(entity, column, task)
-    );
-    try {
-      db.prepare(
-        `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(", ")}) VALUES (${fieldValues.join(", ")})`
-      ).run();
-    } catch (error) {
-      task.logWarning(`Import error: ${error.message}`);
+  const vehiclePositionStmt = db.prepare(
+    `REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(
+      ", "
+    )}) VALUES (${vehiclePositions.schema.map(() => "?").join(", ")})`
+  );
+  db.transaction(() => {
+    for (const entity of gtfsRealtimeData.entity) {
+      try {
+        const fieldValues = vehiclePositions.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
+        vehiclePositionStmt.run(fieldValues);
+        totalLineCount++;
+      } catch (error) {
+        task.logWarning(`Import error: ${error.message}`);
+      }
     }
-    task.log(`Importing - ${totalLineCount++} entries imported\r`, true);
-  }
+    task.log(
+      `Importing - GTFS-Realtime vehicle positions - ${totalLineCount} entries imported\r`,
+      true
+    );
+  })();
 }
 async function updateGtfsRealtimeData(task) {
   if (task.realtimeAlerts === void 0 && task.realtimeTripUpdates === void 0 && task.realtimeVehiclePositions === void 0) {
@@ -3809,7 +3838,11 @@ var importGtfsFiles = (db, task) => mapSeries2(
                 line
               ).map(([columnName, value]) => [
                 columnName,
-                prefixedColumns.has(columnName) && value !== null ? `${task.prefix}${value}` : value
+                applyPrefixToValue(
+                  value,
+                  prefixedColumns.has(columnName),
+                  task.prefix
+                )
               ])
             );
             insert.run(prefixedLine);
@@ -3976,12 +4009,12 @@ async function importGtfs(initialConfig) {
 import { without, compact as compact2 } from "lodash-es";
 import pluralize3 from "pluralize";
 import { stringify } from "csv-stringify";
-import sqlString3 from "sqlstring-sqlite";
+import sqlString2 from "sqlstring-sqlite";
 import mapSeries3 from "promise-map-series";
 import untildify4 from "untildify";
 
 // src/lib/advancedQuery.ts
-import sqlString4 from "sqlstring-sqlite";
+import sqlString3 from "sqlstring-sqlite";
 
 // src/lib/gtfs/routes.ts
 import { omit as omit3, pick } from "lodash-es";
@@ -3995,7 +4028,7 @@ import { omit as omit5, orderBy, pick as pick3 } from "lodash-es";
 
 // src/lib/gtfs/stop-times.ts
 import { omit as omit6 } from "lodash-es";
-import sqlString5 from "sqlstring-sqlite";
+import sqlString4 from "sqlstring-sqlite";
 
 // src/bin/gtfs-import.ts
 var pe = new PrettyError();