edge-currency-plugins 3.14.0 → 3.16.0

package/lib/common/plugin/EngineEmitter.js

@@ -86,6 +86,11 @@ var _events = require('events');
 
 
 
+
+
+
+
+
 
 
 
@@ -105,6 +110,11 @@ var EngineEvent; (function (EngineEvent) {
   const NEW_ADDRESS_TRANSACTION = 'address:transaction:changed'; EngineEvent["NEW_ADDRESS_TRANSACTION"] = NEW_ADDRESS_TRANSACTION;
   const ADDRESSES_CHECKED = 'addresses:checked'; EngineEvent["ADDRESSES_CHECKED"] = ADDRESSES_CHECKED;
   const TXIDS_CHANGED = 'txids:changed'; EngineEvent["TXIDS_CHANGED"] = TXIDS_CHANGED;
+  /**
+   * The engine disconnected a server on purpose because it was found to be
+   * out of sync. Data fetched through it may be behind the chain.
+   */
+  const SERVER_DROPPED = 'server:dropped'; EngineEvent["SERVER_DROPPED"] = SERVER_DROPPED;
   const CONNECTION_OPEN = 'connection:open'; EngineEvent["CONNECTION_OPEN"] = CONNECTION_OPEN;
   const CONNECTION_CLOSE = 'connection:close'; EngineEvent["CONNECTION_CLOSE"] = CONNECTION_CLOSE;
   const CONNECTION_TIMER = 'connection:timer'; EngineEvent["CONNECTION_TIMER"] = CONNECTION_TIMER;

package/lib/common/plugin/PluginState.js

@@ -10,9 +10,48 @@ var _memlet = require('memlet');
 
 
 
+
 var _ServerScores = require('./ServerScores');
 
 
+/**
+ * How long a server dropped for being out of sync stays out of the pick
+ * order. Long enough for a lagging indexer to catch up, short enough that a
+ * recovered server is tried again without an app restart. After expiry the
+ * server is probed again on connect and dropped again if still behind.
+ */
+ const SERVER_QUARANTINE_MS = 5 * 60 * 1000; exports.SERVER_QUARANTINE_MS = SERVER_QUARANTINE_MS
+
+/**
+ * Runs a server's health probe, or hands back the one started for the same
+ * URI less than `minIntervalMs` ago, whether it is still in flight or has
+ * already answered.
+ */
+
+
+
+
+
+
+ function makeShareServerProbe() {
+  // Server URI -> the latest probe and when it started (ms since epoch)
+  const latestProbes = new Map
+
+
+()
+
+  return async (uri, minIntervalMs, probe) => {
+    const now = Date.now()
+    const latest = latestProbes.get(uri)
+    if (latest != null && now - latest.startTime < minIntervalMs) {
+      return await latest.result
+    }
+    const result = probe()
+    latestProbes.set(uri, { startTime: now, result })
+    return await result
+  }
+} exports.makeShareServerProbe = makeShareServerProbe;
+
 // The filename for ServerInfoCache data (see ServerScores.ts)
 // Perhaps this should be in ServerScores.ts file, but that'll take some refactoring
 const SERVER_CACHE_FILE = 'serverCache.json'
@@ -38,6 +77,21 @@ const SERVER_CACHE_FILE = 'serverCache.json'
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 
@@ -97,6 +151,20 @@ const SERVER_CACHE_FILE = 'serverCache.json'
   let engines = []
   const memlet = _memlet.makeMemlet.call(void 0, pluginDisklet)
 
+  // Server URI -> time (ms since epoch) at which the quarantine expires
+  const quarantineExpiry = new Map()
+
+  const isServerQuarantined = (uri) => {
+    const expiry = quarantineExpiry.get(uri)
+    if (expiry == null) return false
+    if (expiry > Date.now()) return true
+    quarantineExpiry.delete(uri)
+    return false
+  }
+
+  const hasQuarantinedServers = () =>
+    Array.from(quarantineExpiry.keys()).some(isServerQuarantined)
+
   let serverCache = {
     customServers: {},
     enableCustomServers: defaultSettings.enableCustomServers,
@@ -182,6 +250,9 @@ const SERVER_CACHE_FILE = 'serverCache.json'
 
       return {
         'pluginState.servers_': sanitizeServerList(selectedServerList),
+        quarantinedServers: Array.from(quarantineExpiry.keys())
+          .filter(isServerQuarantined)
+          .map(sanitizeServerUri),
         infoServers,
         customServers: Object.keys(serverCache.customServers).map(
           sanitizeServerUri
@@ -213,6 +284,15 @@ const SERVER_CACHE_FILE = 'serverCache.json'
       serverScores.serverScoreUp(getSelectedServerList(), uri, score)
     },
 
+    quarantineServer(uri, durationMs = exports.SERVER_QUARANTINE_MS) {
+      log.warn(`${pluginId} - quarantining ${uri} for ${durationMs}ms`)
+      quarantineExpiry.set(uri, Date.now() + durationMs)
+    },
+
+    isServerQuarantined,
+    hasQuarantinedServers,
+    shareServerProbe: makeShareServerProbe(),
+
     async clearCache() {
       serverScores.clearServerScoreTimes()
       serverCacheDirty = true
@@ -223,8 +303,14 @@ const SERVER_CACHE_FILE = 'serverCache.json'
       numServersWanted,
       includePatterns = []
     ) {
+      // Withhold quarantined servers before ranking, so they cannot take
+      // slots that lower-ranked healthy servers would otherwise fill:
+      const eligibleServers = {}
+      for (const [uri, serverInfo] of Object.entries(getSelectedServerList())) {
+        if (!isServerQuarantined(uri)) eligibleServers[uri] = serverInfo
+      }
       return serverScores.getServers(
-        getSelectedServerList(),
+        eligibleServers,
         numServersWanted,
         includePatterns
       )

package/lib/common/utxobased/engine/ServerStates.js

@@ -1,4 +1,5 @@
-"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }
+"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _cleaners = require('cleaners');
+
 var _urijs = require('uri-js');
 
 var _EngineEmitter = require('../../plugin/EngineEmitter');
@@ -23,6 +24,26 @@ var _socketQueue = require('../network/socketQueue');
 var _constants = require('./constants');
 
 
+/**
+ * What the last REST health probe said about a connected server:
+ * - `pending`: the probe has not answered yet.
+ * - `healthy`: Blockbook reported `inSync: true`.
+ * - `unhealthy`: Blockbook reported `inSync: false`.
+ * - `unknown`: the server has no HTTP twin to probe, or the probe failed
+ *   (network error, non-200, unparseable body). Never treated as unhealthy.
+ */
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 
@@ -115,6 +136,33 @@ var _constants = require('./constants');
  */
  const BROADCAST_ATTEMPT_TIMEOUT_MS = 30000; exports.BROADCAST_ATTEMPT_TIMEOUT_MS = BROADCAST_ATTEMPT_TIMEOUT_MS
 
+/**
+ * Upper bound on a REST health probe. The probe runs off the socket keepalive
+ * and must never hold the connection's timer for long.
+ */
+ const HEALTH_PROBE_TIMEOUT_MS = 10000; exports.HEALTH_PROBE_TIMEOUT_MS = HEALTH_PROBE_TIMEOUT_MS
+
+/**
+ * Probes of one server closer together than this share a single request,
+ * across every wallet of the plugin. The connect path and the first
+ * keepalive can land within milliseconds of each other, and N wallets each
+ * run their own keepalive; one answer serves them all.
+ */
+ const HEALTH_PROBE_MIN_INTERVAL_MS = 10000; exports.HEALTH_PROBE_MIN_INTERVAL_MS = HEALTH_PROBE_MIN_INTERVAL_MS
+
+/**
+ * The part of Blockbook's REST `GET /api/` response the health probe reads.
+ * The WebSocket `getInfo` message does not carry `inSync`, so the probe has
+ * to go over HTTP. Blockbook itself masks the first five seconds of every
+ * resync, so a `false` here means the indexer has been behind its backend
+ * for longer than that, or the backend RPC is failing.
+ */
+const asBlockbookHealth = _cleaners.asObject.call(void 0, {
+  blockbook: _cleaners.asObject.call(void 0, {
+    inSync: _cleaners.asBoolean
+  })
+})
+
 const withTimeout = async ( 
   promise,
   ms,
@@ -145,9 +193,146 @@ const withTimeout = async (
   } = config
   const { serverConfigs = [] } = pluginInfo.engineInfo
   const { broadcastTimeoutMs = exports.BROADCAST_ATTEMPT_TIMEOUT_MS } = config
+  const {
+    healthProbeMinIntervalMs = exports.HEALTH_PROBE_MIN_INTERVAL_MS,
+    healthProbeTimeoutMs = exports.HEALTH_PROBE_TIMEOUT_MS,
+    keepAliveMs,
+    wakeUpMs
+  } = config
   log('Making server states')
 
   const serverStatesCache = {}
+
+  // Edge's own Blockbook servers expose both a socket and a REST endpoint on
+  // the same host. Index the REST URIs by host so a socket URI can find its
+  // twin for the health probe. Third-party sockets (Trezor, NOWNodes) have no
+  // twin here and are never probed.
+  const httpUriByHost = {}
+  for (const serverConfig of serverConfigs) {
+    if (serverConfig.type !== 'blockbook') continue
+    for (const httpUri of serverConfig.uris) {
+      const { host } = _urijs.parse.call(void 0, httpUri)
+      if (host != null) httpUriByHost[host] = httpUri
+    }
+  }
+
+  const findHttpTwin = (uri) => {
+    const { host } = _urijs.parse.call(void 0, uri)
+    return host == null ? undefined : httpUriByHost[host]
+  }
+
+  const probeServerHealth = async (uri) => {
+    const httpUri = findHttpTwin(uri)
+    if (httpUri == null) return 'unknown'
+    const probe = async () => {
+      const response = await io.fetchCors(`${httpUri}/api/`, { method: 'GET' })
+      if (!response.ok) {
+        log.warn(`${uri} health probe: HTTP ${response.status}`)
+        return 'unknown'
+      }
+      const { blockbook } = asBlockbookHealth(await response.json())
+      return blockbook.inSync ? 'healthy' : 'unhealthy'
+    }
+    try {
+      // The bound covers the body too; a server can send its headers and
+      // then stall:
+      return await withTimeout(
+        probe(),
+        healthProbeTimeoutMs,
+        `Timeout for health probe to ${httpUri}`
+      )
+    } catch (error) {
+      log.warn(`${uri} health probe failed: ${String(error)}`)
+      return 'unknown'
+    }
+  }
+
+  /**
+   * Disconnects a server on purpose. On platforms that deliver a close event
+   * for a locally closed socket (the app's WebView does) the socket's close
+   * path also runs, scoring the server down and throwing its in-flight
+   * requests. The refill is scheduled here regardless, so replacing the
+   * connection never depends on that event arriving.
+   */
+  const dropServer = (uri, reason) => {
+    const serverState = serverStatesCache[uri]
+    if (serverState == null) return
+    log.warn(`${uri} dropped: ${reason}`)
+    // Keep the refill from picking it straight back; its score is still
+    // near the top since it answered every ping:
+    pluginState.quarantineServer(uri)
+    _utils.removeItem.call(void 0, serverStatesCache, uri)
+    serverState.blockbook.disconnect().catch((error) => {
+      log.error(`${uri} disconnect failed: ${String(error)}`)
+    })
+    reconnect()
+    // Let the engine re-query anything this server answered:
+    engineEmitter.emit(_EngineEmitter.EngineEvent.SERVER_DROPPED, uri)
+  }
+
+  /**
+   * Whether some other connection could carry the wallet if `uri` went
+   * away: open, and not known to be out of sync. A peer whose probe has not
+   * answered yet does not count; it may be about to fail the same way.
+   */
+  const hasUsablePeer = (uri) =>
+    Object.entries(serverStatesCache).some(
+      ([otherUri, state]) =>
+        otherUri !== uri &&
+        state.blockbook.isConnected &&
+        (state.health === 'healthy' || state.health === 'unknown')
+    )
+
+  /**
+   * Drops every server known to be out of sync, as long as the wallet keeps
+   * at least one usable connection. A stale server that is the only leg is
+   * kept: behind the chain beats no data at all. It is re-evaluated on every
+   * probe result, so it goes as soon as a usable peer shows up.
+   *
+   * Only one stale leg is kept. Two stale servers would otherwise keep each
+   * other, filling every connection slot so the refill never reaches a
+   * server that could replace them.
+   */
+  const dropUnhealthyServers = () => {
+    let isStaleLegKept = false
+    for (const [uri, state] of Object.entries(serverStatesCache)) {
+      if (state.health !== 'unhealthy') continue
+      if (!hasUsablePeer(uri) && !isStaleLegKept) {
+        log.warn(
+          `${uri} reports inSync: false but is the only usable connection`
+        )
+        isStaleLegKept = true
+        continue
+      }
+      dropServer(uri, 'Blockbook reports inSync: false')
+    }
+  }
+
+  const checkServerHealth = (uri) => {
+    const serverState = serverStatesCache[uri]
+    if (serverState == null) return
+    pluginState
+      .shareServerProbe(
+        uri,
+        healthProbeMinIntervalMs,
+        async () => await probeServerHealth(uri)
+      )
+      .then(health => {
+        const serverState = serverStatesCache[uri]
+        // The connection may have gone away while the probe was in flight:
+        if (serverState == null) return
+        if (serverState.health !== health) {
+          log(`${uri} health: ${serverState.health} -> ${health}`)
+        }
+        serverState.health = health
+        // This result may make this server droppable, or make it the peer
+        // that lets a previously kept stale server go:
+        dropUnhealthyServers()
+      })
+      .catch((error) => {
+        log.error(`${uri} health check failed: ${String(error)}`)
+      })
+  }
   let isEngineOn = true
   let serverList = []
   let reconnectCounter = 0
@@ -181,6 +366,8 @@ const withTimeout = async (
       const queryTime = Date.now() - queryDate
       log(`${uri} returned healthCheck in ${queryTime}ms`)
       pluginState.serverScoreUp(uri, queryTime)
+      // Periodic probe, once per keepalive:
+      checkServerHealth(uri)
     }
   )
   engineEmitter.on(
@@ -199,7 +386,11 @@ const withTimeout = async (
     _EngineEmitter.EngineEvent.BLOCK_HEIGHT_CHANGED,
     (uri, blockHeight) => {
       log(`${uri} block height changed to ${blockHeight}`)
-      serverStatesCache[uri].blockHeight = blockHeight
+      const serverState = serverStatesCache[uri]
+      // The connection may already be gone, as the sibling handler above
+      // allows for:
+      if (serverState == null) return
+      serverState.blockHeight = blockHeight
     }
   )
 
@@ -207,18 +398,26 @@ const withTimeout = async (
 
 
 
-  const makeServerStatesCacheEntry = (blockbook) => ({
+  const makeServerStatesCacheEntry = (
+    uri,
+    blockbook
+  ) => ({
     blockbook,
     blockSubscriptionStatus: 'unsubscribed',
     txids: new Set(),
     addresses: new Set(),
-    blockHeight: 0
+    blockHeight: 0,
+    health: findHttpTwin(uri) == null ? 'unknown' : 'pending'
   })
 
   const reconnect = () => {
     if (isEngineOn) {
       log(`attempting server reconnect number ${reconnectCounter}`)
       const reconnectionDelay = Math.max(5, reconnectCounter++) * 1000
+      // One pending reconnect at a time. A drop and the close event it
+      // causes both land here, and a timer that was overwritten rather than
+      // cleared would outlive stop() and switch the engine back on.
+      clearTimeout(reconnectTimer)
       reconnectTimer = setTimeout(() => {
         clearTimeout(reconnectTimer)
         instance.refillServers()
@@ -244,6 +443,9 @@ const withTimeout = async (
       // Skip reconnecting to an existing connection
       if (serverStatesCache[uri] != null) continue
 
+      // The list may predate a quarantine:
+      if (pluginState.isServerQuarantined(uri)) continue
+
       // Validate the URI of server to make sure it is valid
       const parsed = _urijs.parse.call(void 0, uri)
       if (
@@ -254,7 +456,7 @@ const withTimeout = async (
         continue
       }
 
-      // Ranomize the URI picking
+      // Randomize the URI picking
       chanceToBePicked -= chanceToBePicked > 0.5 ? 0.25 : 0
       if (Math.random() > chanceToBePicked) {
         serverList.push(uri)
@@ -331,17 +533,25 @@ const withTimeout = async (
           log,
           taskGeneratorFn,
           socketEmitter,
-          walletId: walletInfo.id
+          walletId: walletInfo.id,
+          keepAliveMs,
+          wakeUpMs
         })
       }
 
       // Make new ServerStates instance
-      serverStatesCache[uri] = makeServerStatesCacheEntry(blockbook)
+      serverStatesCache[uri] = makeServerStatesCacheEntry(uri, blockbook)
 
       // Initialize blockbook connection for server
       blockbook
         .connect()
         .then(async () => {
+          // This connection is now open and counts as a usable peer, which
+          // may be what a stale server kept for lack of one was waiting on.
+          // Done here rather than on the socket's open event, which fires
+          // before the connect promise marks the blockbook connected:
+          dropUnhealthyServers()
+
           // Fetch block height from blockbook server
           const startTime = Date.now()
           const { bestHeight: blockHeight } = await blockbook.fetchInfo()
@@ -356,11 +566,29 @@ const withTimeout = async (
           // Increment server score using response time
           const responseTime = Date.now() - startTime
           pluginState.serverScoreUp(uri, responseTime)
+
+          // On-connect probe. The keepalive may or may not have fired by
+          // now, depending on how fast the socket opened, so this is the
+          // one place that guarantees a probe before the server does work:
+          checkServerHealth(uri)
         })
         .catch(e => {
           log.error(`${JSON.stringify(e.message)}`)
         })
     }
+
+    // A quarantined server is withheld from the candidate list, so a wallet
+    // left short of connections would otherwise stay that way until an
+    // unrelated close event. Keep trying while a quarantine is in force; the
+    // server comes back into the list when it expires. The reconnect delay
+    // grows with each empty pass, so this does not spin.
+    if (
+      Object.keys(serverStatesCache).length < _constants.MAX_CONNECTIONS &&
+      serverList.length === 0 &&
+      pluginState.hasQuarantinedServers()
+    ) {
+      reconnect()
+    }
   }
 
   const deferredWithServerScoring = (
@@ -581,6 +809,10 @@ const withTimeout = async (
       return serverStatesCache[uri].blockHeight
     },
 
+    getServerHealth(uri) {
+      return _optionalChain([serverStatesCache, 'access', _2 => _2[uri], 'optionalAccess', _3 => _3.health])
+    },
+
     getServerList() {
       return serverList
     },
@@ -674,7 +906,7 @@ const withTimeout = async (
           serverState.addresses.delete(address)
         }
         // Reject the deferredAddressSub promise if provided
-        _optionalChain([deferredAddressSub, 'optionalAccess', _2 => _2.reject, 'call', _3 => _3(err)])
+        _optionalChain([deferredAddressSub, 'optionalAccess', _4 => _4.reject, 'call', _5 => _5(err)])
       })
 
       blockbook.watchAddresses(