edge-currency-plugins 3.15.0 → 3.16.0

package/lib/common/utxobased/engine/UtxoEngineProcessor.js

@@ -156,9 +156,25 @@ var _utils3 = require('./utils');
    * 2. Address UTXO processing
    **/
   const processesPerAddress = 2
-  let processedCount = 0
+  // Which of the two processes have completed, per address (by scriptPubkey).
+  // Flags rather than a running count, so an address re-queued after a server
+  // drop is not counted twice and the ratio cannot pass 1 with work pending.
+  const progressByScriptPubkey = new Map
+
+
+()
   let processedPercent = 0 // last sync ratio emitted
-  const updateProgressRatio = async () => {
+  const updateProgressRatio = async (
+    scriptPubkey,
+    process
+  ) => {
+    const progress = _nullishCoalesce(progressByScriptPubkey.get(scriptPubkey), () => ( {
+      txs: false,
+      utxos: false
+    }))
+    progress[process] = true
+    progressByScriptPubkey.set(scriptPubkey, progress)
+
     // Avoid re-sending sync ratios / sending ratios larger than 1
     if (processedPercent >= 1) return
 
@@ -167,8 +183,11 @@ var _utils3 = require('./utils');
     const expectedProcessCount =
       Object.keys(taskCache.addressSubscribeCache).length * processesPerAddress
 
-    // Increment the processed count
-    processedCount = processedCount + 1
+    let processedCount = 0
+    for (const { txs, utxos } of progressByScriptPubkey.values()) {
+      if (txs) processedCount++
+      if (utxos) processedCount++
+    }
 
     // If we have no addresses, we should not have not yet began processing.
     if (expectedProcessCount === 0) throw new Error('No addresses to process')
@@ -244,8 +263,14 @@ var _utils3 = require('./utils');
     if (running) return
     running = true
 
-    await initializeAddressSubscriptions()
-    await setLookAhead(common)
+    try {
+      await initializeAddressSubscriptions()
+      await setLookAhead(common)
+    } catch (error) {
+      // Let the next call try again instead of skipping the initialization:
+      running = false
+      throw error
+    }
   }
 
   emitter.on(
@@ -265,6 +290,30 @@ var _utils3 = require('./utils');
     }
   )
 
+  emitter.on(_EngineEmitter.EngineEvent.SERVER_DROPPED, (uri) => {
+    // Everything this server told us about may stop short of the chain tip.
+    // Re-open every address so the surviving connection re-queries it. Each
+    // query starts from the address's stored lastQueriedBlockHeight, which is
+    // never above the height of the server that answered it, so the range
+    // the dropped server missed is fetched again.
+    let requeued = 0
+    for (const [address, cacheItem] of Object.entries(
+      taskCache.addressSubscribeCache
+    )) {
+      if (!cacheItem.processing) continue
+      cacheItem.processing = false
+      taskCache.addressForUtxosCache[address] = {
+        processing: false,
+        path: cacheItem.path
+      }
+      // The re-query has to complete again before the address counts as
+      // processed:
+      progressByScriptPubkey.delete(walletTools.addressToScriptPubkey(address))
+      requeued++
+    }
+    log(`${uri} dropped, re-queued ${requeued} addresses`)
+  })
+
   emitter.on(
     _EngineEmitter.EngineEvent.NEW_ADDRESS_TRANSACTION,
     async (_uri, response) => {
@@ -290,6 +339,30 @@ var _utils3 = require('./utils');
     }
   )
 
+  // The DataLayer can count an address that it can no longer read, such as
+  // when a write to disk was cut short. Every address derives from the wallet
+  // keys and its path, so rebuild the missing row instead of failing the
+  // engine. The rebuilt row has no sync state, so the address syncs again.
+  const repairAddress = async (path) => {
+    const { format, changeIndex, addressIndex } = path
+    const pathName = `${format}/${changeIndex}/${addressIndex}`
+    log.warn(`Repairing missing data-layer address with '${pathName}' path`)
+
+    const { scriptPubkey, redeemScript } = walletTools.getScriptPubkey(path)
+    await dataLayer.saveAddress(
+      _types5.makeAddressData.call(void 0, { scriptPubkey, redeemScript, path })
+    )
+
+    // Read the row back, since the save keeps any state that survived:
+    const addressData = await dataLayer.fetchAddress(path)
+    if (addressData == null) {
+      throw new Error(
+        `Missing data-layer address with '${pathName}' path during initialization`
+      )
+    }
+    return addressData
+  }
+
   // Initialize the addressSubscribeCache with the existing addresses already
   // processed by the DataLayer. This happens only once before any call to
   // setLookAhead.
@@ -318,16 +391,13 @@ var _utils3 = require('./utils');
           addressIndex < branchAddressCount;
           addressIndex++
         ) {
-          const addressData = await dataLayer.fetchAddress({
+          const path = {
             format,
             changeIndex: branch,
             addressIndex
-          })
-          if (addressData == null) {
-            throw new Error(
-              `Missing data-layer address with '${format}/${branch}/${addressIndex}' path during initialization`
-            )
           }
+          const addressData =
+            await _asyncNullishCoalesce((await dataLayer.fetchAddress(path)), async () => ( (await repairAddress(path))))
           const { address } = walletTools.scriptPubkeyToAddress({
             changePath,
             scriptPubkey: addressData.scriptPubkey
@@ -360,7 +430,7 @@ var _utils3 = require('./utils');
   return {
     processedPercent,
     async start() {
-      processedCount = 0
+      progressByScriptPubkey.clear()
       processedPercent = 0
 
       await run()
@@ -694,6 +764,12 @@ const internalGetFreshAddress = async (
 
 
 
+
+
+
+
+
+
 
 
 
@@ -1208,6 +1284,17 @@ async function* processAddressForTransactions(
     throw new Error(`could not find address with script pubkey ${scriptPubkey}`)
   }
 
+  // The address was queued with the height of whichever server subscribed
+  // it, but any server can answer the query. Never record more than the
+  // answering server has indexed, so a lagging server cannot move
+  // lastQueriedBlockHeight past blocks it never saw. A height of 0 means this
+  // server has not reported one yet, so nothing it answers can be trusted to
+  // move the cursor; the next query covers the range again.
+  const queriedHeight =
+    serverState.blockHeight > 0
+      ? Math.min(blockHeight, serverState.blockHeight)
+      : addressData.lastQueriedBlockHeight
+
   try {
     const addressResponse = yield* common.serverStates.addressQueryTask(
       serverUri,
@@ -1287,23 +1374,25 @@ async function* processAddressForTransactions(
       addressForTransactionsCache[address] = {
         ...cacheItem,
         processing: false,
-        page: page + 1
+        page: page + 1,
+        blockHeight: queriedHeight
       }
       return true
     }
 
     // Update the lastQueriedBlockHeight for the address
-    addressData.lastQueriedBlockHeight = blockHeight
+    addressData.lastQueriedBlockHeight = queriedHeight
 
     // Save/update the fully-processed address
     await common.dataLayer.saveAddress(addressData)
 
-    // Update the progress now that the transactions for an address have processed
-    await common.updateProgressRatio()
-
-    // Call setLookAhead to update the lookahead
+    // Extend the lookahead before counting progress, so any addresses it adds
+    // are in the total before this address's flag can bring the ratio to 1:
     await setLookAhead(common)
 
+    // Update the progress now that the transactions for an address have processed
+    await common.updateProgressRatio(scriptPubkey, 'txs')
+
     // Add the address to the server cache
     serverState.addresses.add(address)
 
@@ -1573,13 +1662,14 @@ const processDataLayerUtxos = async (
     })
   }
 
-  // Update the progress now that the UTXOs for an address have been processed
-  await common.updateProgressRatio()
-
+  // Extend the lookahead first, as in processAddressForTransactions:
   await setLookAhead(common).catch(err => {
     common.log.error(err)
     throw err
   })
+
+  // Update the progress now that the UTXOs for an address have been processed
+  await common.updateProgressRatio(scriptPubkey, 'utxos')
 }
 
 /**

package/lib/common/utxobased/network/Blockbook.js

@@ -103,6 +103,9 @@ var _Socket = require('./Socket');
 
 
 
+
+
+
 
 
 
@@ -277,7 +280,9 @@ var _Socket = require('./Socket');
     taskGeneratorFn,
     log,
     emitter: socketEmitter,
-    walletId
+    walletId,
+    keepAliveMs: config.keepAliveMs,
+    wakeUpMs: config.wakeUpMs
   })
 
   async function ping() {

package/lib/common/utxobased/network/Socket.js

@@ -87,6 +87,10 @@ const WAKE_UP_MS = 5000 // interval at which we wakeUp and potentially onQueueSp
 
 
 
+
+
+
+
 
 
 
@@ -134,6 +138,8 @@ const asResponseDefault = (raw) => [
   let connected = false
   let cancelConnect = false
   const timeout = 1000 * (_nullishCoalesce(config.timeout, () => ( 30)))
+  const keepAliveMs = _nullishCoalesce(config.keepAliveMs, () => ( KEEP_ALIVE_MS))
+  const wakeUpMs = _nullishCoalesce(config.wakeUpMs, () => ( WAKE_UP_MS))
   let trackedError
   let timer
 
@@ -264,7 +270,7 @@ const asResponseDefault = (raw) => [
   const onTimer = () => {
     log(`socket timer with server ${uri} expired, check if healthCheck needed`)
     const now = Date.now() - TIMER_SLACK
-    if (lastKeepAlive + KEEP_ALIVE_MS < now) {
+    if (lastKeepAlive + keepAliveMs < now) {
       log(`submitting healthCheck to server ${uri}`)
       lastKeepAlive = now
       config
@@ -291,7 +297,7 @@ const asResponseDefault = (raw) => [
 
   const setupTimer = () => {
     log(`setupTimer with server ${uri}`)
-    let nextWakeUp = lastWakeUp + WAKE_UP_MS
+    let nextWakeUp = lastWakeUp + wakeUpMs
     for (const request of Object.values(pendingRequests)) {
       const to = request.startTime + timeout
       if (to < nextWakeUp) nextWakeUp = to

package/lib/src/common/plugin/EngineEmitter.d.ts

@@ -6,11 +6,11 @@ export declare interface EngineEmitter {
     emit: ((event: EngineEvent.SEEN_TX_CHECKPOINT, checkpoint: string) => boolean) & ((event: EngineEvent.TRANSACTIONS, transactionEvents: EdgeTransactionEvent[]) => boolean) & ((event: EngineEvent.TRANSACTIONS_CHANGED, transactions: EdgeTransaction[]) => boolean) & ((event: EngineEvent.ADDRESS_BALANCE_CHANGED, currencyCode: string, addressBalanceChanges: Array<{
         scriptPubkey: string;
         balance: string;
-    }>) => boolean) & ((event: EngineEvent.WALLET_BALANCE_CHANGED, currencyCode: string, nativeBalance: string) => boolean) & ((event: EngineEvent.BLOCK_HEIGHT_CHANGED, uri: string, blockHeight: number) => boolean) & ((event: EngineEvent.NEW_ADDRESS_TRANSACTION, uri: string, newTx: SubscribeAddressResponse) => boolean) & ((event: EngineEvent.ADDRESSES_CHECKED, progressRatio: number) => boolean) & ((event: EngineEvent.TXIDS_CHANGED, txids: EdgeTxidMap) => boolean);
+    }>) => boolean) & ((event: EngineEvent.WALLET_BALANCE_CHANGED, currencyCode: string, nativeBalance: string) => boolean) & ((event: EngineEvent.BLOCK_HEIGHT_CHANGED, uri: string, blockHeight: number) => boolean) & ((event: EngineEvent.NEW_ADDRESS_TRANSACTION, uri: string, newTx: SubscribeAddressResponse) => boolean) & ((event: EngineEvent.ADDRESSES_CHECKED, progressRatio: number) => boolean) & ((event: EngineEvent.TXIDS_CHANGED, txids: EdgeTxidMap) => boolean) & ((event: EngineEvent.SERVER_DROPPED, uri: string) => boolean);
     on: ((event: EngineEvent.SEEN_TX_CHECKPOINT, listener: (checkpoint: string) => Promise<void> | void) => this) & ((event: EngineEvent.TRANSACTIONS, listener: (transactionEvents: EdgeTransactionEvent[]) => Promise<void> | void) => boolean) & ((event: EngineEvent.TRANSACTIONS_CHANGED, listener: (transactions: EdgeTransaction[]) => Promise<void> | void) => this) & ((event: EngineEvent.ADDRESS_BALANCE_CHANGED, listener: (currencyCode: string, addressBalanceChanges: Array<{
         scriptPubkey: string;
         balance: string;
-    }>) => Promise<void> | void) => this) & ((event: EngineEvent.WALLET_BALANCE_CHANGED, listener: (currencyCode: string, nativeBalance: string) => Promise<void> | void) => this) & ((event: EngineEvent.BLOCK_HEIGHT_CHANGED, listener: (uri: string, blockHeight: number) => Promise<void> | void) => this) & ((event: EngineEvent.NEW_ADDRESS_TRANSACTION, listener: (uri: string, newTx: SubscribeAddressResponse) => Promise<void> | void) => this) & ((event: EngineEvent.ADDRESSES_CHECKED, listener: (progressRatio: number) => Promise<void> | void) => this) & ((event: EngineEvent.TXIDS_CHANGED, listener: (txids: EdgeTxidMap) => Promise<void> | void) => this);
+    }>) => Promise<void> | void) => this) & ((event: EngineEvent.WALLET_BALANCE_CHANGED, listener: (currencyCode: string, nativeBalance: string) => Promise<void> | void) => this) & ((event: EngineEvent.BLOCK_HEIGHT_CHANGED, listener: (uri: string, blockHeight: number) => Promise<void> | void) => this) & ((event: EngineEvent.NEW_ADDRESS_TRANSACTION, listener: (uri: string, newTx: SubscribeAddressResponse) => Promise<void> | void) => this) & ((event: EngineEvent.ADDRESSES_CHECKED, listener: (progressRatio: number) => Promise<void> | void) => this) & ((event: EngineEvent.TXIDS_CHANGED, listener: (txids: EdgeTxidMap) => Promise<void> | void) => this) & ((event: EngineEvent.SERVER_DROPPED, listener: (uri: string) => Promise<void> | void) => this);
 }
 export declare class EngineEmitter extends EventEmitter {
 }
@@ -25,6 +25,11 @@ export declare enum EngineEvent {
     NEW_ADDRESS_TRANSACTION = "address:transaction:changed",
     ADDRESSES_CHECKED = "addresses:checked",
     TXIDS_CHANGED = "txids:changed",
+    /**
+     * The engine disconnected a server on purpose because it was found to be
+     * out of sync. Data fetched through it may be behind the chain.
+     */
+    SERVER_DROPPED = "server:dropped",
     CONNECTION_OPEN = "connection:open",
     CONNECTION_CLOSE = "connection:close",
     CONNECTION_TIMER = "connection:timer"

package/lib/src/common/plugin/PluginState.d.ts

@@ -1,8 +1,23 @@
 import { Disklet } from 'disklet';
 import { EdgeIo, EdgeLog } from 'edge-core-js/types';
+import { ServerHealth } from '../utxobased/engine/ServerStates';
 import { UtxoUserSettings } from '../utxobased/engine/types';
 import { UtxoEngineProcessor } from '../utxobased/engine/UtxoEngineProcessor';
 import { InfoPayload } from './types';
+/**
+ * How long a server dropped for being out of sync stays out of the pick
+ * order. Long enough for a lagging indexer to catch up, short enough that a
+ * recovered server is tried again without an app restart. After expiry the
+ * server is probed again on connect and dropped again if still behind.
+ */
+export declare const SERVER_QUARANTINE_MS: number;
+/**
+ * Runs a server's health probe, or hands back the one started for the same
+ * URI less than `minIntervalMs` ago, whether it is still in flight or has
+ * already answered.
+ */
+export declare type ShareServerProbe = (uri: string, minIntervalMs: number, probe: () => Promise<ServerHealth>) => Promise<ServerHealth>;
+export declare function makeShareServerProbe(): ShareServerProbe;
 /** A JSON object (as opposed to an array or primitive). */
 interface JsonObject {
     [name: string]: unknown;
@@ -28,6 +43,21 @@ export interface PluginState {
     load: () => Promise<PluginState>;
     serverScoreDown: (uri: string) => void;
     serverScoreUp: (uri: string, score: number) => void;
+    /**
+     * Keeps a server out of getLocalServers for a while. Shared by every
+     * engine of this plugin, so one wallet's finding spares the others the
+     * same connect-probe-drop cycle. In memory only: it does not outlive the
+     * plugin, and a restart gives the server a fresh chance.
+     */
+    quarantineServer: (uri: string, durationMs?: number) => void;
+    isServerQuarantined: (uri: string) => boolean;
+    hasQuarantinedServers: () => boolean;
+    /**
+     * Shares health probes between every engine of this plugin, so N wallets
+     * connected to the same server cost one REST request per interval rather
+     * than N. Each engine still applies the answer to its own connection.
+     */
+    shareServerProbe: ShareServerProbe;
     clearCache: () => Promise<void>;
     getLocalServers: (numServersWanted: number, includePatterns?: Array<string | RegExp>) => string[];
     refreshServers: (updatedCustomServers?: string[]) => Promise<void>;

package/lib/src/common/utxobased/engine/ServerStates.d.ts

@@ -8,10 +8,20 @@ import { AddressResponse, AddressUtxosResponse, BlockbookTransaction } from '../
 import Deferred from '../network/Deferred';
 import { WsTask, WsTaskGenerator } from '../network/Socket';
 import { UtxoInitOptions } from './types';
+/**
+ * What the last REST health probe said about a connected server:
+ * - `pending`: the probe has not answered yet.
+ * - `healthy`: Blockbook reported `inSync: true`.
+ * - `unhealthy`: Blockbook reported `inSync: false`.
+ * - `unknown`: the server has no HTTP twin to probe, or the probe failed
+ *   (network error, non-200, unparseable body). Never treated as unhealthy.
+ */
+export declare type ServerHealth = 'pending' | 'healthy' | 'unhealthy' | 'unknown';
 export interface ServerState {
     blockbook: Blockbook;
     blockSubscriptionStatus: 'unsubscribed' | 'subscribing' | 'subscribed';
     blockHeight: number;
+    health: ServerHealth;
     txids: Set<string>;
     addresses: Set<string>;
 }
@@ -28,6 +38,14 @@ interface ServerStateConfig {
      * seconds the socket layer uses for a request. Exposed for tests.
      */
     broadcastTimeoutMs?: number;
+    /**
+     * Timing overrides so tests can drive the keepalive probe cycle in
+     * milliseconds rather than minutes. Production uses the defaults.
+     */
+    healthProbeMinIntervalMs?: number;
+    healthProbeTimeoutMs?: number;
+    keepAliveMs?: number;
+    wakeUpMs?: number;
 }
 export interface ServerStates {
     setPickNextTaskCB: (callback: (uri: string) => AsyncGenerator<WsTask<unknown> | boolean>) => void;
@@ -43,6 +61,7 @@ export interface ServerStates {
     watchAddresses: (uri: string, addresses: string[], deferredAddressSub?: Deferred<unknown>) => void;
     watchBlocks: (uri: string) => void;
     getBlockHeight: (uri: string) => number;
+    getServerHealth: (uri: string) => ServerHealth | undefined;
     addressQueryTask: (serverUri: string, address: string, params: {
         lastQueriedBlockHeight: number;
         page: number;
@@ -67,5 +86,17 @@ export declare const NOWNODES_BROADCAST_DELAY_MS = 2000;
  * open. This mirrors the socket figure.
  */
 export declare const BROADCAST_ATTEMPT_TIMEOUT_MS = 30000;
+/**
+ * Upper bound on a REST health probe. The probe runs off the socket keepalive
+ * and must never hold the connection's timer for long.
+ */
+export declare const HEALTH_PROBE_TIMEOUT_MS = 10000;
+/**
+ * Probes of one server closer together than this share a single request,
+ * across every wallet of the plugin. The connect path and the first
+ * keepalive can land within milliseconds of each other, and N wallets each
+ * run their own keepalive; one answer serves them all.
+ */
+export declare const HEALTH_PROBE_MIN_INTERVAL_MS = 10000;
 export declare function makeServerStates(config: ServerStateConfig): ServerStates;
 export {};

package/lib/src/common/utxobased/engine/UtxoEngineProcessor.d.ts

@@ -42,7 +42,7 @@ interface CommonParams {
     emitter: EngineEmitter;
     taskCache: TaskCache;
     pendingTimeouts: Set<NodeJS.Timeout>;
-    updateProgressRatio: () => void;
+    updateProgressRatio: (scriptPubkey: string, process: AddressProcess) => Promise<void>;
     updateSeenTxCheckpoint: () => void;
     io: EdgeIo;
     log: EdgeLog;
@@ -75,6 +75,8 @@ interface AddressForUtxosCache {
         path: ChangePath;
     };
 }
+/** The two per-address jobs that make up the sync progress ratio. */
+declare type AddressProcess = 'txs' | 'utxos';
 interface AddressSubscribeCache {
     [key: string]: {
         processing: boolean;

package/lib/src/common/utxobased/network/Blockbook.d.ts

@@ -42,6 +42,9 @@ interface BlockbookConfig {
     ping?: () => Promise<void>;
     socketEmitter: SocketEmitter;
     walletId: string;
+    /** Socket timing overrides, for tests. */
+    keepAliveMs?: number;
+    wakeUpMs?: number;
 }
 export declare function makeBlockbook(config: BlockbookConfig): Blockbook;
 export {};

package/lib/src/common/utxobased/network/Socket.d.ts

@@ -39,6 +39,10 @@ interface SocketConfig {
     taskGeneratorFn: TaskGeneratorFn;
     timeout?: number;
     walletId: string;
+    /** Keepalive interval override, for tests. Defaults to KEEP_ALIVE_MS. */
+    keepAliveMs?: number;
+    /** Timer wake-up interval override, for tests. Defaults to WAKE_UP_MS. */
+    wakeUpMs?: number;
 }
 export declare type WsResponse = WsResponseMessage[];
 export interface WsResponseMessage {

package/lib/test/common/plugin/PluginState.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/test/common/utxobased/engine/AddressRepair.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "edge-currency-plugins",
-  "version": "3.15.0",
+  "version": "3.16.0",
   "description": "Edge currency plugins to fetch and process wallet accounts, balances and transactions.",
   "homepage": "https://edge.app/",
   "repository": "https://github.com/EdgeApp/edge-currency-plugins",