@alteriom/painlessmesh 2.0.2 → 2.1.0

package/src/painlessmesh/mesh.hpp

@@ -81,6 +81,55 @@ struct SendOptions {
 typedef std::function<void(bool success, uint16_t httpStatus, TSTRING error)>
     internetResultCallback_t;
 
+/**
+ * Defined when sendToInternet() accepts an InternetResult callback, the
+ * gateway only retries requests that cannot have reached the server, and
+ * each request carries X-Request-Id. Code that must build against older
+ * releases too can test for it with #ifdef.
+ */
+#define PAINLESSMESH_HAS_INTERNET_RESULT 1
+
+/**
+ * Everything the library learned about one sendToInternet() call
+ *
+ * The library applies HTTP's meaning of a reply and nothing more. Whether a
+ * particular service's answer means what the application wanted -- a message
+ * really queued, a record really stored -- depends on that service, so the
+ * application decides, from `httpStatus` and `response`.
+ */
+struct InternetResult {
+  /** The id sendToInternet() returned. */
+  uint32_t messageId = 0;
+  /** True for HTTP 200, 201, 202 and 204. */
+  bool success = false;
+  /** The destination's HTTP status; 0 when no reply arrived. */
+  uint16_t httpStatus = 0;
+  /** Why it failed, for a log or a user; empty on success. */
+  TSTRING error;
+  /** The start of the response body as one line, when a body was read. */
+  TSTRING response;
+  /**
+   * True when the request cannot have reached the server, or the server said
+   * it did not take it. False when it may have been processed: resending it
+   * is then the application's decision, because it could arrive twice.
+   */
+  bool retryable = false;
+  /** How many times the request was issued, retries included. */
+  uint8_t attempts = 0;
+};
+
+/**
+ * Callback type for Internet request results, with the whole result
+ *
+ * \code
+ * mesh.sendToInternet(url, "", [](const InternetResult& result) {
+ *   Serial.printf("HTTP %u: %s\n", result.httpStatus, result.response.c_str());
+ * });
+ * \endcode
+ */
+typedef std::function<void(const InternetResult& result)>
+    internetResponseCallback_t;
+
 /**
  * Pending Internet request entry
  *
@@ -98,6 +147,15 @@ struct PendingInternetRequest {
   TSTRING destination = "";        ///< Internet destination URL
   TSTRING payload = "";            ///< Request payload
   internetResultCallback_t callback;  ///< User callback for result
+  internetResponseCallback_t onResult;  ///< Or the whole-result callback
+  /// Times the request was handed to a gateway: 0 until the first send
+  /// leaves this node, whatever retries were scheduled meanwhile.
+  uint8_t attempts = 0;
+  /// Drawn once for the call, sent on every attempt (GatewayDataPackage).
+  uint32_t requestNonce = 0;
+  /// Why the last attempt failed, when retries were spent waiting for a
+  /// gateway; the final callback reports it instead of a bare retry count.
+  TSTRING lastError = "";
 
   /**
    * Check if this request has timed out
@@ -243,6 +301,37 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
     painlessmesh::ntp::addPackageCallback(this->callbackList, (*this));
     painlessmesh::router::addPackageCallback(this->callbackList, (*this));
 
+    // Every node answers a gateway request it cannot serve. A bridge that
+    // reboots, crashes or is reflashed as a regular node announces nothing
+    // -- only a bridge stepping down in-process sends `leaving` -- so its
+    // peers keep it in knownBridges until its last status ages out
+    // (bridgeTimeoutMs, 60 s) and may route every Internet request to it.
+    // A regular node had no GATEWAY_DATA handler and dropped them without a
+    // word; the origin waited out its request timeout, 30 s, each time.
+    // Found on the HIL rig while validating 2.0.3.
+    //
+    // Registered first, so a gateway's own handler (initGatewayInternetHandler
+    // in wifi.hpp, or a test's stand-in) is always the second: this one
+    // stays silent whenever any other GATEWAY_DATA handler exists.
+    this->callbackList.onPackage(
+        protocol::GATEWAY_DATA,
+        [this](protocol::Variant& variant, std::shared_ptr<T> ingress,
+               uint32_t) {
+          if (this->callbackList.count(protocol::GATEWAY_DATA) > 1) {
+            return false;  // this node is a gateway; its handler answers
+          }
+          auto pkg = variant.to<gateway::GatewayDataPackage>();
+          char errorBuf[96];
+          snprintf(errorBuf, sizeof(errorBuf), "Node %u %s",
+                   static_cast<unsigned>(this->nodeId),
+                   gateway::GATEWAY_NOT_A_GATEWAY_PHRASE);
+          Log(logger::ERROR,
+              "GATEWAY_DATA from node %u answered: %s\n",
+              static_cast<unsigned>(pkg.originNode), errorBuf);
+          this->sendGatewayAck(pkg, false, 0, TSTRING(errorBuf), ingress);
+          return true;
+        });
+
     // Seed delivery-confirmation ids with a random value so they do not
     // restart at 1 after a reboot — a delayed ACK for a pre-reboot
     // message could otherwise match a fresh message's id and report a
@@ -1349,7 +1438,8 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
     
     size_t removed = sizeBefore - knownBridges.size();
     if (removed > 0) {
-      Log(GENERAL, "cleanupExpiredBridges(): Removed %u expired bridges\n", removed);
+      Log(GENERAL, "cleanupExpiredBridges(): Removed %lu expired bridges\n",
+          static_cast<unsigned long>(removed));
     }
   }
 
@@ -1432,6 +1522,31 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
     return internetHealthChecker.hasLocalInternet();
   }
 
+  /**
+   * Can this node serve a gateway request on its own uplink right now?
+   *
+   * hasLocalInternet() is the last periodic probe's answer. On a bridge the
+   * first probe runs while the station is still associating and fails, and
+   * the next is a full interval (30 s) away, so for that whole window a send
+   * fell through to the mesh path and was refused for want of a peer (issue
+   * #450). When the flag says no and the health check is running, spend the
+   * one on-demand probe the checker allows per period: a send made after the
+   * uplink came up is then served locally, and a node whose uplink really is
+   * down pays one probe per interval, not one per call.
+   *
+   * @return true if the request should go to this node's own gateway handler
+   */
+  bool uplinkServesLocally() {
+    if (hasLocalInternet()) return true;
+    if (!isInternetHealthCheckEnabled()) return false;
+    if (internetHealthChecker.checkOnDemand()) {
+      Log(logger::COMMUNICATION,
+          "uplinkServesLocally(): on-demand probe found the uplink up\n");
+      return true;
+    }
+    return false;
+  }
+
   /**
    * Register callback for local Internet connectivity changes
    * 
@@ -1661,6 +1776,61 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
       TSTRING payload,
       internetResultCallback_t callback,
       uint8_t priority = static_cast<uint8_t>(gateway::GatewayPriority::PRIORITY_NORMAL)) {
+    return startInternetRequest(destination, payload, callback, nullptr, priority);
+  }
+
+  /**
+   * Send data to an Internet destination, receiving the whole result
+   *
+   * The same request as the (success, httpStatus, error) form, but the
+   * callback also gets the start of the response body, whether the request
+   * may be resent safely, and how many times it was issued. Use it when a
+   * service's status alone does not say whether it did what was asked -- the
+   * sendToInternet example judges CallMeBot's replies this way.
+   *
+   * @return Unique message ID for tracking, or 0 if immediate failure
+   */
+  uint32_t sendToInternet(
+      TSTRING destination,
+      TSTRING payload,
+      internetResponseCallback_t onResult,
+      uint8_t priority = static_cast<uint8_t>(gateway::GatewayPriority::PRIORITY_NORMAL)) {
+    return startInternetRequest(destination, payload, nullptr, onResult, priority);
+  }
+
+ private:
+  /** Hand a finished request's result to whichever callback it was given. */
+  static void deliverInternetResult(const PendingInternetRequest& request,
+                                    InternetResult result) {
+    result.messageId = request.messageId;
+    result.attempts = request.attempts;
+    // A request that never left this node -- refused for want of a mesh or a
+    // gateway, or whose every routing attempt failed to send -- cannot have
+    // reached a server, whatever ended it: resending it cannot duplicate.
+    if (!result.success && request.attempts == 0) {
+      result.retryable = true;
+    }
+    if (request.onResult) {
+      request.onResult(result);
+    } else if (request.callback) {
+      request.callback(result.success, result.httpStatus, result.error);
+    }
+  }
+
+  static InternetResult failedInternetResult(const TSTRING& error,
+                                             uint16_t httpStatus = 0) {
+    InternetResult result;
+    result.httpStatus = httpStatus;
+    result.error = error;
+    return result;
+  }
+
+  uint32_t startInternetRequest(
+      TSTRING destination,
+      TSTRING payload,
+      internetResultCallback_t callback,
+      internetResponseCallback_t onResult,
+      uint8_t priority) {
     using namespace logger;
 
     // Generate unique message ID
@@ -1672,7 +1842,7 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
     // A shared gateway can execute its own gateway package locally. Keep the
     // same package handler, pending-request tracking and callback semantics,
     // but do not require a mesh peer merely to reach this node's own uplink.
-    if (hasLocalInternet()) {
+    if (uplinkServesLocally()) {
       PendingInternetRequest request;
       request.messageId = messageId;
       request.timestamp = millis();
@@ -1685,20 +1855,32 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
       request.destination = destination;
       request.payload = payload;
       request.callback = callback;
+      request.onResult = onResult;
+      request.requestNonce = gateway::newRequestNonce();
+      // Handed to this node's own handler below, unconditionally.
+      request.attempts = 1;
+      const uint32_t nonce = request.requestNonce;
       pendingInternetRequests[messageId] = request;
 
       dispatchInternetRequestLocally(messageId, priority, 0, destination,
-                                     payload);
+                                     payload, nonce);
       return messageId;
     }
 
+    // A request refused before it is tracked still reports through the
+    // callback it was given, on the scheduler rather than inside this call.
+    PendingInternetRequest refused;
+    refused.callback = callback;
+    refused.onResult = onResult;
+    refused.attempts = 0;
+
     // Validate mesh connectivity before attempting to send
     if (!hasActiveMeshConnections()) {
       Log(ERROR, "sendToInternet(): No active mesh connections\n");
-      if (callback) {
-        // Schedule callback to avoid blocking
-        this->addTask([callback]() {
-          callback(false, 0, "No mesh connections - cannot route to gateway");
+      if (callback || onResult) {
+        this->addTask([refused]() {
+          deliverInternetResult(refused, failedInternetResult(
+              "No mesh connections - cannot route to gateway"));
         });
       }
       return 0;
@@ -1708,10 +1890,9 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
     BridgeInfo* gateway = getPrimaryBridge();
     if (gateway == nullptr) {
       Log(ERROR, "sendToInternet(): No gateway available\n");
-      if (callback) {
-        // Schedule callback to avoid blocking
-        this->addTask([callback]() {
-          callback(false, 0, "No gateway available");
+      if (callback || onResult) {
+        this->addTask([refused]() {
+          deliverInternetResult(refused, failedInternetResult("No gateway available"));
         });
       }
       return 0;
@@ -1730,6 +1911,8 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
     request.destination = destination;
     request.payload = payload;
     request.callback = callback;
+    request.onResult = onResult;
+    request.requestNonce = gateway::newRequestNonce();
 
     // Store pending request
     pendingInternetRequests[messageId] = request;
@@ -1747,6 +1930,7 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
     pkg.contentType = "application/json";
     pkg.retryCount = 0;
     pkg.requiresAck = true;
+    pkg.requestNonce = request.requestNonce;
 
     // Send the package with priority
     bool sent = false;
@@ -1769,6 +1953,8 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
       scheduleInternetRetry(messageId);
     } else {
       Log(COMMUNICATION, "sendToInternet(): Sent to gateway %u\n", gateway->nodeId);
+      // Only a send that left this node is an attempt.
+      pendingInternetRequests[messageId].attempts = 1;
       // Schedule timeout check
       scheduleInternetTimeout(messageId);
     }
@@ -1776,6 +1962,7 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
     return messageId;
   }
 
+ public:
   /**
    * Configure Internet request timeout
    *
@@ -1847,11 +2034,11 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
   bool cancelInternetRequest(uint32_t messageId) {
     auto it = pendingInternetRequests.find(messageId);
     if (it != pendingInternetRequests.end()) {
-      auto callback = it->second.callback;
+      const PendingInternetRequest cancelled = it->second;
       pendingInternetRequests.erase(it);
-      if (callback) {
-        this->addTask([callback]() {
-          callback(false, 0, "Request cancelled");
+      if (cancelled.callback || cancelled.onResult) {
+        this->addTask([cancelled]() {
+          deliverInternetResult(cancelled, failedInternetResult("Request cancelled"));
         });
       }
       Log(logger::GENERAL, "cancelInternetRequest(): Cancelled msgId=%u\n", messageId);
@@ -1913,9 +2100,7 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
 
     // Cancel all pending requests
     for (auto& pair : pendingInternetRequests) {
-      if (pair.second.callback) {
-        pair.second.callback(false, 0, "API disabled");
-      }
+      deliverInternetResult(pair.second, failedInternetResult("API disabled"));
     }
     pendingInternetRequests.clear();
 
@@ -1935,6 +2120,79 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
   /**
    * Handle incoming gateway acknowledgment package
    */
+ public:
+  // Public: the gateway handler in wifi.hpp, a derived class, answers with it.
+  /**
+   * Answer a gateway request: send its GATEWAY_ACK back to the origin node.
+   *
+   * A request this node originated completes locally, without a mesh hop.
+   * Otherwise the ack is routed to the origin; when the NodeTree has not yet
+   * converged enough for findRoute() to resolve it -- a freshly promoted
+   * gateway can receive data before that -- the connection the request
+   * arrived on is a valid reverse path, since every intermediate node routes
+   * the addressed ack onward toward the origin.
+   *
+   * Used by the gateway's own handler (wifi.hpp) and by the fallback every
+   * node registers for requests it cannot serve.
+   */
+  void sendGatewayAck(const gateway::GatewayDataPackage& request, bool success,
+                      uint16_t httpStatus, const TSTRING& error,
+                      std::shared_ptr<T> ingressConnection = nullptr,
+                      const TSTRING& response = TSTRING(), int8_t retryable = -1,
+                      uint32_t retryAfterMs = 0) {
+    using namespace logger;
+
+    gateway::GatewayAckPackage ack;
+    ack.from = this->nodeId;
+    ack.dest = request.originNode;
+    ack.messageId = request.messageId;
+    ack.originNode = request.originNode;
+    ack.success = success;
+    ack.httpStatus = httpStatus;
+    ack.error = error;
+    ack.timestamp = this->getNodeTime();
+    ack.response = response;
+    ack.retryable = retryable;
+    ack.retryAfterMs = retryAfterMs;
+
+    if (request.originNode == this->nodeId) {
+      // Completed from the scheduler, not from inside the gateway handler
+      // that called this. There the handler's HTTPClient, its WiFiClient
+      // and the response buffers are all still allocated, and the
+      // application's callback runs on what is left: on the HIL rig an
+      // ESP8266 shared gateway with ~11 KB free delivered its own result
+      // into a callback that could no longer allocate the event it built.
+      this->addTask([this, ack]() {
+        gateway::GatewayAckPackage local = ack;
+        protocol::Variant variant(&local);
+        this->callbackList.execute(protocol::GATEWAY_ACK, variant, nullptr, 0);
+      });
+      Log(COMMUNICATION,
+          "Completing local GATEWAY_ACK from the scheduler (success=%d, http=%d)\n",
+          success, httpStatus);
+      return;
+    }
+
+    auto conn = router::findRoute<T>((*this), request.originNode);
+    if (!conn && ingressConnection) {
+      conn = ingressConnection;
+      Log(COMMUNICATION,
+          "Routing GATEWAY_ACK to node %u through request ingress while "
+          "topology converges\n",
+          request.originNode);
+    }
+    if (conn) {
+      protocol::Variant variant(&ack);
+      router::send(std::move(variant), conn);
+      Log(COMMUNICATION, "Sent GATEWAY_ACK to node %u (success=%d, http=%d)\n",
+          request.originNode, success, httpStatus);
+    } else {
+      Log(ERROR, "Failed to send GATEWAY_ACK: no route to node %u\n",
+          request.originNode);
+    }
+  }
+
+ private:
   void handleGatewayAck(const gateway::GatewayAckPackage& ack) {
     using namespace logger;
     Log(COMMUNICATION, "handleGatewayAck(): msgId=%u success=%d http=%u\n",
@@ -1948,94 +2206,127 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
 
     PendingInternetRequest& request = it->second;
 
+    InternetResult result;
+    result.success = ack.success;
+    result.httpStatus = ack.httpStatus;
+    result.error = ack.error;
+    result.response = ack.response;
+
     // Check if this is a success response
     if (ack.success) {
-      // Success - call callback and remove request
-      if (request.callback) {
-        request.callback(ack.success, ack.httpStatus, ack.error);
-      }
+      const PendingInternetRequest done = request;
       pendingInternetRequests.erase(it);
+      deliverInternetResult(done, result);
       return;
     }
 
-    // Failure response - determine if retryable
-    bool isRetryable = false;
-    
-    // HTTP 203 (Non-Authoritative Information) indicates cached/proxied response
-    // This is often temporary and retrying may succeed when cache expires
-    if (ack.httpStatus == 203) {
-      isRetryable = true;
-      Log(COMMUNICATION, "handleGatewayAck(): HTTP 203 detected, marking as retryable\n");
-    }
-    // HTTP 5xx server errors are typically transient
-    else if (ack.httpStatus >= 500 && ack.httpStatus < 600) {
-      isRetryable = true;
-      Log(COMMUNICATION, "handleGatewayAck(): HTTP 5xx server error, marking as retryable\n");
-    }
-    // HTTP 429 (Too Many Requests) should be retried with backoff
-    else if (ack.httpStatus == 429) {
-      isRetryable = true;
-      Log(COMMUNICATION, "handleGatewayAck(): HTTP 429 rate limit, marking as retryable\n");
-    }
-    // Network errors (httpStatus == 0) EXCEPT gateway connectivity errors
-    // Gateway connectivity errors are infrastructure issues (not transient)
-    else if (ack.httpStatus == 0) {
-      // Check if this is a gateway-level connectivity error (non-retryable)
-      bool isGatewayConnectivityError = false;
-      
-      // These errors indicate infrastructure issues that won't be fixed by retrying:
-      // - "Router has no internet access" - WAN connection down
-      // - "Gateway WiFi not connected" - ESP not associated with WiFi
-      // - "Captive portal detected" - Router requires web authentication
-      // Use find() for std::string (test env) or indexOf() for Arduino String
-      bool routerError = false, wifiError = false, captivePortalError = false;
-      #if defined(PAINLESSMESH_BOOST)
-        // Test environment: TSTRING is std::string, use find()
-        routerError = (ack.error.find("Router has no internet") != std::string::npos);
-        wifiError = (ack.error.find("Gateway WiFi not connected") != std::string::npos);
-        captivePortalError = (ack.error.find("Captive portal detected") != std::string::npos);
-      #else
-        // Arduino environment: TSTRING is String, use indexOf()
-        routerError = (ack.error.indexOf("Router has no internet") >= 0);
-        wifiError = (ack.error.indexOf("Gateway WiFi not connected") >= 0);
-        captivePortalError = (ack.error.indexOf("Captive portal detected") >= 0);
-      #endif
-      
-      if (routerError || wifiError || captivePortalError) {
-        isGatewayConnectivityError = true;
-        Log(COMMUNICATION, "handleGatewayAck(): Gateway connectivity error detected (non-retryable): %s\n", 
-            ack.error.c_str());
+    // The node this request went to is not a gateway (issue found on the
+    // HIL rig while validating 2.0.3): it rebooted or was reflashed as a
+    // regular node without announcing it, and its last bridge status is
+    // still inside bridgeTimeoutMs. Forget it and send again through the
+    // next gateway at once. The retry budget is not spent: nothing was
+    // attempted against the Internet, and every such answer removes one
+    // entry from knownBridges, so this cannot loop.
+    if (gateway::responseContains(ack.error,
+                                  gateway::GATEWAY_NOT_A_GATEWAY_PHRASE)) {
+      const uint32_t staleGateway = request.gatewayNodeId;
+      bool forgot = this->forgetBridge(staleGateway);
+      if (ack.from != staleGateway) forgot = this->forgetBridge(ack.from) || forgot;
+      Log(ERROR, "handleGatewayAck(): %s; forgot it as a gateway for msgId=%u\n",
+          ack.error.c_str(), ack.messageId);
+
+      if (!forgot) {
+        // Nothing to forget, so a redispatch would reach the same node.
+        // Fall through to the ordinary failure path below.
+      } else if (this->getPrimaryBridge() == nullptr &&
+                 !this->uplinkServesLocally()) {
+        // No gateway left right now. On the rig the live bridge's first
+        // status arrived half a second after the ex-bridge answered, so an
+        // accepted request rides the ordinary retry backoff rather than
+        // failing on the spot; retryInternetRequest() reschedules while no
+        // gateway is known, and the final callback carries this reason.
+        char errorBuf[160];
+        snprintf(errorBuf, sizeof(errorBuf),
+                 "No Internet gateway available: %s", ack.error.c_str());
+        request.lastError = TSTRING(errorBuf);
+        scheduleInternetRetry(ack.messageId);
+        return;
+      } else {
+        const uint32_t messageId = ack.messageId;
+        this->addTask([this, messageId]() {
+          this->retryInternetRequest(messageId);
+        });
+        return;
       }
-      
-      // Only mark as retryable if it's NOT a gateway connectivity error
-      if (!isGatewayConnectivityError) {
-        isRetryable = true;
-        Log(COMMUNICATION, "handleGatewayAck(): Network error, marking as retryable\n");
+    }
+
+    // Failure: may the identical request be sent again?
+    //
+    // A gateway that says so decides (2.1.0 and later): it saw the transport
+    // error or the status, and it knows whether the request can have reached
+    // the server. Resending one that may have been processed delivers it
+    // twice -- on the rig one timed-out send was issued four times -- so only
+    // a request that cannot have arrived, or one the server told us to bring
+    // back (429, 503), is retried.
+    bool isRetryable = false;
+    uint32_t minDelayMs = 0;
+    if (ack.retryable >= 0) {
+      isRetryable = ack.retryable == 1;
+      if (isRetryable && ack.retryAfterMs > gateway::GATEWAY_RETRY_AFTER_MAX_MS) {
+        // The server asked to wait longer than a retry will: say when, and
+        // leave the resend to the application.
+        isRetryable = false;
+        char errorBuf[224];
+        snprintf(errorBuf, sizeof(errorBuf), "%s (the server asked to retry after %lu s)",
+                 ack.error.c_str(),
+                 static_cast<unsigned long>(ack.retryAfterMs / 1000));
+        result.error = TSTRING(errorBuf);
+      } else if (isRetryable) {
+        minDelayMs = ack.retryAfterMs;
       }
+    } else {
+      // A gateway that predates the field. Keep its behaviour -- it has
+      // retried network errors and 5xx since 1.x -- except for a 2xx: the
+      // server answered, so a resend is a second copy.
+      isRetryable = legacyAckIsRetryable(ack);
     }
-    // HTTP 4xx client errors (except 429) are NOT retryable
-    // HTTP 3xx redirects are NOT retryable (should be followed by HTTPClient)
-    // Other status codes are NOT retryable
+    result.retryable = isRetryable;
 
     // If retryable and have retries left, schedule retry
     if (isRetryable && request.retryCount < request.maxRetries) {
       Log(COMMUNICATION, "handleGatewayAck(): Scheduling retry for msgId=%u (attempt %u/%u)\n",
           ack.messageId, request.retryCount + 1, request.maxRetries);
-      scheduleInternetRetry(ack.messageId);
+      request.lastError = result.error;
+      scheduleInternetRetry(ack.messageId, minDelayMs);
     } else {
-      // Not retryable or max retries reached - call callback and remove
-      if (request.retryCount >= request.maxRetries) {
+      if (isRetryable) {
         Log(ERROR, "handleGatewayAck(): Max retries reached for msgId=%u\n", ack.messageId);
       } else {
-        Log(COMMUNICATION, "handleGatewayAck(): Non-retryable failure for msgId=%u (HTTP %u)\n",
+        Log(COMMUNICATION, "handleGatewayAck(): Final failure for msgId=%u (HTTP %u)\n",
             ack.messageId, ack.httpStatus);
       }
-      
-      if (request.callback) {
-        request.callback(ack.success, ack.httpStatus, ack.error);
-      }
+      const PendingInternetRequest done = request;
       pendingInternetRequests.erase(it);
+      deliverInternetResult(done, result);
+    }
+  }
+
+  /**
+   * How an ack from a gateway that does not say whether to retry was read
+   * before 2.1.0, minus retrying a 2xx.
+   */
+  static bool legacyAckIsRetryable(const gateway::GatewayAckPackage& ack) {
+    if (ack.httpStatus >= 500 && ack.httpStatus < 600) return true;
+    if (ack.httpStatus == 429) return true;
+    if (ack.httpStatus != 0) return false;
+    // Gateway-level connectivity failures are infrastructure, not transient.
+    static const char* const FINAL[] = {"Router has no internet",
+                                        "Gateway WiFi not connected",
+                                        "Captive portal detected"};
+    for (const char* phrase : FINAL) {
+      if (gateway::responseContains(ack.error, phrase)) return false;
     }
+    return true;
   }
 
   /**
@@ -2045,14 +2336,16 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
    * acknowledgment, pending-request tracking and callback semantics stay
    * identical to a request served by a peer gateway.
    *
-   * The handler runs synchronously and its acknowledgment can erase the
-   * pending entry, so callers must not hold a reference into
-   * pendingInternetRequests across this call -- pass copies.
+   * The handler runs synchronously. sendGatewayAck() completes a local
+   * acknowledgment from the scheduler, but a handler is free to do otherwise,
+   * so callers must not hold a reference into pendingInternetRequests across
+   * this call -- pass copies.
    */
   void dispatchInternetRequestLocally(uint32_t messageId, uint8_t priority,
                                       uint8_t retryCount,
                                       const TSTRING& destination,
-                                      const TSTRING& payload) {
+                                      const TSTRING& payload,
+                                      uint32_t requestNonce) {
     gateway::GatewayDataPackage pkg;
     pkg.from = this->nodeId;
     pkg.dest = this->nodeId;
@@ -2065,6 +2358,7 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
     pkg.contentType = "application/json";
     pkg.retryCount = retryCount;
     pkg.requiresAck = true;
+    pkg.requestNonce = requestNonce;
     protocol::Variant variant(&pkg);
     this->callbackList.execute(protocol::GATEWAY_DATA, variant, nullptr, 0);
   }
@@ -2072,7 +2366,7 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
   /**
    * Schedule retry for a failed Internet request
    */
-  void scheduleInternetRetry(uint32_t messageId) {
+  void scheduleInternetRetry(uint32_t messageId, uint32_t minDelayMs = 0) {
     auto it = pendingInternetRequests.find(messageId);
     if (it == pendingInternetRequests.end()) {
       return;
@@ -2081,20 +2375,32 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
     PendingInternetRequest& request = it->second;
 
     if (request.retryCount >= request.maxRetries) {
-      // Max retries reached - fail the request
+      // Max retries reached - fail the request, with the reason the retries
+      // were spent on when there is one
       Log(logger::ERROR, "scheduleInternetRetry(): Max retries reached for msgId=%u\n",
           messageId);
-      if (request.callback) {
-        request.callback(false, 0, "Max retries exceeded");
-      }
+      const PendingInternetRequest done = request;
       pendingInternetRequests.erase(it);
+      deliverInternetResult(done, failedInternetResult(
+          done.lastError.length() > 0 ? done.lastError : TSTRING("Max retries exceeded")));
       return;
     }
 
-    // Calculate exponential backoff delay
+    // Exponential backoff, never shorter than what the server asked for
     uint32_t delay = request.retryDelayMs * (1 << request.retryCount);
+    if (delay < minDelayMs) delay = minDelayMs;
     request.retryCount++;
 
+    // A wait the server asked for moves the request's deadline with it:
+    // otherwise a Retry-After longer than what is left of the request
+    // timeout (30 s by default) times the request out before the retry the
+    // server invited can run. The retry gets a full timeout of its own.
+    if (minDelayMs > 0) {
+      const uint32_t elapsed = static_cast<uint32_t>(millis()) - request.timestamp;
+      const uint32_t needed = elapsed + delay + internetRequestTimeout;
+      if (needed > request.timeoutMs) request.timeoutMs = needed;
+    }
+
     Log(logger::COMMUNICATION, "scheduleInternetRetry(): Retry %u for msgId=%u in %u ms\n",
         request.retryCount, messageId, delay);
 
@@ -2124,16 +2430,18 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
     //
     // Copy what the redispatch needs first: the local handler runs
     // synchronously and can erase this entry, leaving `request` dangling.
-    if (request.gatewayNodeId == this->nodeId && hasLocalInternet()) {
+    if (request.gatewayNodeId == this->nodeId && uplinkServesLocally()) {
       const uint8_t priority = request.priority;
       const uint8_t retryCount = request.retryCount;
       const TSTRING destination = request.destination;
       const TSTRING payload = request.payload;
+      const uint32_t nonce = request.requestNonce;
+      request.attempts++;
       Log(logger::COMMUNICATION,
           "retryInternetRequest(): Retrying msgId=%u on the local uplink\n",
           messageId);
       dispatchInternetRequestLocally(messageId, priority, retryCount,
-                                     destination, payload);
+                                     destination, payload, nonce);
       return;
     }
 
@@ -2171,6 +2479,7 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
     pkg.contentType = "application/json";
     pkg.retryCount = request.retryCount;
     pkg.requiresAck = true;
+    pkg.requestNonce = request.requestNonce;
 
     // Send the package
     auto conn = painlessmesh::router::findRoute<T>((*this), gateway->nodeId);
@@ -2184,6 +2493,7 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
       Log(logger::ERROR, "retryInternetRequest(): Retry send failed msgId=%u\n", messageId);
       scheduleInternetRetry(messageId);
     } else {
+      request.attempts++;
       Log(logger::COMMUNICATION, "retryInternetRequest(): Retry sent msgId=%u to gateway %u\n",
           messageId, gateway->nodeId);
     }
@@ -2219,10 +2529,9 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
     if (request.isTimedOut()) {
       Log(logger::ERROR, "checkInternetRequestTimeout(): Request timed out msgId=%u\n",
           messageId);
-      if (request.callback) {
-        request.callback(false, 0, "Request timed out");
-      }
+      const PendingInternetRequest done = request;
       pendingInternetRequests.erase(it);
+      deliverInternetResult(done, failedInternetResult("Request timed out"));
     }
   }
 
@@ -2235,9 +2544,7 @@ class Mesh : public ntp::MeshTime, public plugin::PackageHandler<T> {
       if (it->second.isTimedOut()) {
         Log(logger::GENERAL, "cleanupTimedOutRequests(): Cleaning up msgId=%u\n",
             it->first);
-        if (it->second.callback) {
-          it->second.callback(false, 0, "Request timed out");
-        }
+        deliverInternetResult(it->second, failedInternetResult("Request timed out"));
         it = pendingInternetRequests.erase(it);
       } else {
         ++it;