gtfs 4.18.7 → 4.19.0

package/dist/bin/gtfs-import.js

@@ -1126,6 +1126,7 @@ var routes = {
     {
       name: "agency_id",
       type: "text",
+      index: true,
       prefix: true
     },
     {
@@ -1378,7 +1379,8 @@ var stops = {
     },
     {
       name: "stop_code",
-      type: "text"
+      type: "text",
+      index: true
     },
     {
       name: "stop_name",
@@ -2880,16 +2882,14 @@ var serviceAlerts = {
     {
       name: "start_time",
       type: "text",
-      required: true,
       source: "alert.activePeriod[0].start",
-      default: ""
+      default: null
     },
     {
       name: "end_time",
       type: "text",
-      required: true,
       source: "alert.activePeriod[0].end",
-      default: ""
+      default: null
     },
     {
       name: "header_text",
@@ -2949,7 +2949,7 @@ var serviceAlertInformedEntities = {
     {
       name: "stop_id",
       type: "text",
-      index: true,
+      primary: true,
       source: "stopId",
       default: null,
       prefix: true
@@ -2957,7 +2957,7 @@ var serviceAlertInformedEntities = {
     {
       name: "route_id",
       type: "text",
-      index: true,
+      primary: true,
       source: "routeId",
       default: null,
       prefix: true
@@ -2965,14 +2965,14 @@ var serviceAlertInformedEntities = {
     {
       name: "route_type",
       type: "integer",
-      index: true,
+      primary: true,
       source: "routeType",
       default: null
     },
     {
       name: "trip_id",
       type: "text",
-      index: true,
+      primary: true,
       source: "trip.tripId",
       default: null,
       prefix: true
@@ -2980,7 +2980,7 @@ var serviceAlertInformedEntities = {
     {
       name: "direction_id",
       type: "integer",
-      index: true,
+      primary: true,
       source: "directionId",
       default: null
     },
@@ -3497,7 +3497,8 @@ var passengerEvents = {
     {
       name: "trip_stop_sequence",
       type: "integer",
-      min: 1
+      min: 1,
+      required: true
     },
     {
       name: "scheduled_stop_sequence",
@@ -3723,7 +3724,8 @@ var stopVisits = {
     },
     {
       name: "ramp_deployed_time",
-      type: "text"
+      type: "real",
+      min: 0
     },
     {
       name: "ramp_failure",
@@ -3731,12 +3733,12 @@ var stopVisits = {
     },
     {
       name: "kneel_deployed_time",
-      type: "integer",
+      type: "real",
       min: 0
     },
     {
       name: "lift_deployed_time",
-      type: "integer",
+      type: "real",
       min: 0
     },
     {
@@ -4336,7 +4338,7 @@ function getTimestampColumnName(columnName) {
   return columnName.endsWith("time") ? `${columnName}stamp` : `${columnName}_timestamp`;
 }
 function applyPrefixToValue(value, columnShouldBePrefixed, prefix) {
-  if (!columnShouldBePrefixed || prefix === void 0 || value === null) {
+  if (!columnShouldBePrefixed || prefix === void 0 || value === null || value === void 0) {
     return value;
   }
   return `${prefix}${value}`;
@@ -4399,13 +4401,15 @@ async function fetchGtfsRealtimeData(type, task) {
     try {
       const response = await fetch(urlConfig.url, {
         method: "GET",
+        redirect: "follow",
         headers: {
+          "User-Agent": "node-gtfs",
           ...urlConfig.headers ?? {},
           "Accept-Encoding": "gzip"
         },
         signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
       });
-      if (response.status !== 200) {
+      if (!response.ok) {
         throw new GtfsError(`HTTP ${response.status}: ${response.statusText}`, {
           code: "GTFS_DOWNLOAD_HTTP" /* GTFS_DOWNLOAD_HTTP */,
           category: "download" /* DOWNLOAD */,
@@ -4478,12 +4482,17 @@ function createServiceAlertsProcessor(db, task) {
     db,
     serviceAlertInformedEntities
   );
+  const deleteInformedEntitiesStmt = db.prepare(
+    `DELETE FROM ${serviceAlertInformedEntities.filenameBase} WHERE alert_id = ?`
+  );
   return async (batch) => {
     let recordCount = 0;
     let errorCount = 0;
     db.transaction(() => {
       for (const entity of batch) {
         try {
+          const alertId = applyPrefixToValue(entity.id, true, task.prefix);
+          deleteInformedEntitiesStmt.run(alertId);
           const alertValues = serviceAlerts.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
           alertStmt.run(alertValues);
           recordCount++;
@@ -4516,6 +4525,9 @@ function createTripUpdatesProcessor(db, task) {
     db,
     stopTimeUpdates
   );
+  const deleteStopTimesByTripStmt = db.prepare(
+    `DELETE FROM ${stopTimeUpdates.filenameBase} WHERE trip_id = ? AND trip_start_time IS ?`
+  );
   return async (batch) => {
     let recordCount = 0;
     let errorCount = 0;
@@ -4526,6 +4538,15 @@ function createTripUpdatesProcessor(db, task) {
           tripUpdateStmt.run(tripUpdateValues);
           recordCount++;
           if (entity.tripUpdate?.stopTimeUpdate?.length) {
+            const tripId = applyPrefixToValue(
+              entity.tripUpdate?.trip?.tripId ?? null,
+              true,
+              task.prefix
+            );
+            const tripStartTime = entity.tripUpdate?.trip?.startTime ?? null;
+            if (tripId !== null) {
+              deleteStopTimesByTripStmt.run(tripId, tripStartTime);
+            }
             for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
               stopTimeUpdate.parent = entity;
               const stopTimeValues = stopTimeUpdates.schema.map(
@@ -4586,41 +4607,30 @@ async function updateGtfsRealtimeData(task) {
     tripupdates: 0,
     vehiclepositions: 0
   };
-  const processingPromises = [];
   if (alertsData?.entity?.length) {
-    processingPromises.push(
-      processBatch(
-        alertsData.entity,
-        BATCH_SIZE,
-        createServiceAlertsProcessor(db, task)
-      ).then((result) => {
-        recordCounts.alerts = result.recordCount;
-      })
+    const result = await processBatch(
+      alertsData.entity,
+      BATCH_SIZE,
+      createServiceAlertsProcessor(db, task)
     );
+    recordCounts.alerts = result.recordCount;
   }
   if (tripUpdatesData?.entity?.length) {
-    processingPromises.push(
-      processBatch(
-        tripUpdatesData.entity,
-        BATCH_SIZE,
-        createTripUpdatesProcessor(db, task)
-      ).then((result) => {
-        recordCounts.tripupdates = result.recordCount;
-      })
+    const result = await processBatch(
+      tripUpdatesData.entity,
+      BATCH_SIZE,
+      createTripUpdatesProcessor(db, task)
     );
+    recordCounts.tripupdates = result.recordCount;
   }
   if (vehiclePositionsData?.entity?.length) {
-    processingPromises.push(
-      processBatch(
-        vehiclePositionsData.entity,
-        BATCH_SIZE,
-        createVehiclePositionsProcessor(db, task)
-      ).then((result) => {
-        recordCounts.vehiclepositions = result.recordCount;
-      })
+    const result = await processBatch(
+      vehiclePositionsData.entity,
+      BATCH_SIZE,
+      createVehiclePositionsProcessor(db, task)
     );
+    recordCounts.vehiclepositions = result.recordCount;
   }
-  await Promise.all(processingPromises);
   task.log(
     `GTFS-Realtime import complete: ${recordCounts.alerts} alerts, ${recordCounts.tripupdates} trip updates, ${recordCounts.vehiclepositions} vehicle positions`
   );
@@ -4648,10 +4658,14 @@ var downloadGtfsFiles = async (task) => {
   try {
     const response = await fetch(task.url, {
       method: "GET",
-      headers: task.headers || {},
+      redirect: "follow",
+      headers: {
+        "User-Agent": "node-gtfs",
+        ...task.headers
+      },
       signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
     });
-    if (response.status !== 200) {
+    if (!response.ok) {
       throw new GtfsError(
         `Unable to download GTFS from ${task.url}. Got status ${response.status}.`,
         {
@@ -4764,18 +4778,18 @@ var extractGtfsFiles = async (task) => {
 var createGtfsTables = (db) => {
   for (const model of Object.values(models_exports)) {
     if (!model.schema) {
-      return;
+      continue;
     }
     const sqlColumnCreateStatements = [];
     for (const column of model.schema) {
       const checks = [];
-      if (column.min !== void 0 && column.max) {
+      if (column.min !== void 0 && column.max !== void 0) {
         checks.push(
           `${column.name} >= ${column.min} AND ${column.name} <= ${column.max}`
         );
-      } else if (column.min) {
+      } else if (column.min !== void 0) {
         checks.push(`${column.name} >= ${column.min}`);
-      } else if (column.max) {
+      } else if (column.max !== void 0) {
         checks.push(`${column.name} <= ${column.max}`);
       }
       if (column.type === "integer") {
@@ -4824,7 +4838,7 @@ var createGtfsTables = (db) => {
 var createGtfsIndexes = (db) => {
   for (const model of Object.values(models_exports)) {
     if (!model.schema) {
-      return;
+      continue;
     }
     for (const column of model.schema) {
       if (column.index) {