gtfs 4.18.6 → 4.19.0

package/dist/bin/gtfsrealtime-update.js

@@ -465,16 +465,14 @@ var serviceAlerts = {
     {
       name: "start_time",
       type: "text",
-      required: true,
       source: "alert.activePeriod[0].start",
-      default: ""
+      default: null
     },
     {
       name: "end_time",
       type: "text",
-      required: true,
       source: "alert.activePeriod[0].end",
-      default: ""
+      default: null
     },
     {
       name: "header_text",
@@ -534,7 +532,7 @@ var serviceAlertInformedEntities = {
     {
       name: "stop_id",
       type: "text",
-      index: true,
+      primary: true,
       source: "stopId",
       default: null,
       prefix: true
@@ -542,7 +540,7 @@ var serviceAlertInformedEntities = {
     {
       name: "route_id",
       type: "text",
-      index: true,
+      primary: true,
       source: "routeId",
       default: null,
       prefix: true
@@ -550,14 +548,14 @@ var serviceAlertInformedEntities = {
     {
       name: "route_type",
       type: "integer",
-      index: true,
+      primary: true,
       source: "routeType",
       default: null
     },
     {
       name: "trip_id",
       type: "text",
-      index: true,
+      primary: true,
       source: "trip.tripId",
       default: null,
       prefix: true
@@ -565,7 +563,7 @@ var serviceAlertInformedEntities = {
     {
       name: "direction_id",
       type: "integer",
-      index: true,
+      primary: true,
       source: "directionId",
       default: null
     },
@@ -746,7 +744,7 @@ function convertLongTimeToDate(longDate) {
   ).toISOString();
 }
 function applyPrefixToValue(value, columnShouldBePrefixed, prefix) {
-  if (!columnShouldBePrefixed || prefix === void 0 || value === null) {
+  if (!columnShouldBePrefixed || prefix === void 0 || value === null || value === void 0) {
     return value;
   }
   return `${prefix}${value}`;
@@ -809,13 +807,15 @@ async function fetchGtfsRealtimeData(type, task) {
     try {
       const response = await fetch(urlConfig.url, {
         method: "GET",
+        redirect: "follow",
         headers: {
+          "User-Agent": "node-gtfs",
           ...urlConfig.headers ?? {},
           "Accept-Encoding": "gzip"
         },
         signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
       });
-      if (response.status !== 200) {
+      if (!response.ok) {
         throw new GtfsError(`HTTP ${response.status}: ${response.statusText}`, {
           code: "GTFS_DOWNLOAD_HTTP" /* GTFS_DOWNLOAD_HTTP */,
           category: "download" /* DOWNLOAD */,
@@ -888,12 +888,17 @@ function createServiceAlertsProcessor(db, task) {
     db,
     serviceAlertInformedEntities
   );
+  const deleteInformedEntitiesStmt = db.prepare(
+    `DELETE FROM ${serviceAlertInformedEntities.filenameBase} WHERE alert_id = ?`
+  );
   return async (batch) => {
     let recordCount = 0;
     let errorCount = 0;
     db.transaction(() => {
       for (const entity of batch) {
         try {
+          const alertId = applyPrefixToValue(entity.id, true, task.prefix);
+          deleteInformedEntitiesStmt.run(alertId);
           const alertValues = serviceAlerts.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
           alertStmt.run(alertValues);
           recordCount++;
@@ -926,6 +931,9 @@ function createTripUpdatesProcessor(db, task) {
     db,
     stopTimeUpdates
   );
+  const deleteStopTimesByTripStmt = db.prepare(
+    `DELETE FROM ${stopTimeUpdates.filenameBase} WHERE trip_id = ? AND trip_start_time IS ?`
+  );
   return async (batch) => {
     let recordCount = 0;
     let errorCount = 0;
@@ -936,6 +944,15 @@ function createTripUpdatesProcessor(db, task) {
           tripUpdateStmt.run(tripUpdateValues);
           recordCount++;
           if (entity.tripUpdate?.stopTimeUpdate?.length) {
+            const tripId = applyPrefixToValue(
+              entity.tripUpdate?.trip?.tripId ?? null,
+              true,
+              task.prefix
+            );
+            const tripStartTime = entity.tripUpdate?.trip?.startTime ?? null;
+            if (tripId !== null) {
+              deleteStopTimesByTripStmt.run(tripId, tripStartTime);
+            }
             for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
               stopTimeUpdate.parent = entity;
               const stopTimeValues = stopTimeUpdates.schema.map(
@@ -1015,41 +1032,30 @@ async function updateGtfsRealtimeData(task) {
     tripupdates: 0,
     vehiclepositions: 0
   };
-  const processingPromises = [];
   if (alertsData?.entity?.length) {
-    processingPromises.push(
-      processBatch(
-        alertsData.entity,
-        BATCH_SIZE,
-        createServiceAlertsProcessor(db, task)
-      ).then((result) => {
-        recordCounts.alerts = result.recordCount;
-      })
+    const result = await processBatch(
+      alertsData.entity,
+      BATCH_SIZE,
+      createServiceAlertsProcessor(db, task)
     );
+    recordCounts.alerts = result.recordCount;
   }
   if (tripUpdatesData?.entity?.length) {
-    processingPromises.push(
-      processBatch(
-        tripUpdatesData.entity,
-        BATCH_SIZE,
-        createTripUpdatesProcessor(db, task)
-      ).then((result) => {
-        recordCounts.tripupdates = result.recordCount;
-      })
+    const result = await processBatch(
+      tripUpdatesData.entity,
+      BATCH_SIZE,
+      createTripUpdatesProcessor(db, task)
     );
+    recordCounts.tripupdates = result.recordCount;
   }
   if (vehiclePositionsData?.entity?.length) {
-    processingPromises.push(
-      processBatch(
-        vehiclePositionsData.entity,
-        BATCH_SIZE,
-        createVehiclePositionsProcessor(db, task)
-      ).then((result) => {
-        recordCounts.vehiclepositions = result.recordCount;
-      })
+    const result = await processBatch(
+      vehiclePositionsData.entity,
+      BATCH_SIZE,
+      createVehiclePositionsProcessor(db, task)
     );
+    recordCounts.vehiclepositions = result.recordCount;
   }
-  await Promise.all(processingPromises);
   task.log(
     `GTFS-Realtime import complete: ${recordCounts.alerts} alerts, ${recordCounts.tripupdates} trip updates, ${recordCounts.vehiclepositions} vehicle positions`
   );