@exodus/solana-api 3.25.4 → 3.26.1

package/src/connection.js

@@ -1,68 +1,39 @@
 import { WebSocket } from '@exodus/fetch'
 import debugLogger from 'debug'
 import delay from 'delay'
-import lodash from 'lodash'
-import makeConcurrent from 'make-concurrent'
+import assert from 'minimalistic-assert'
 import ms from 'ms'
 
-// WS subscriptions: https://docs.solana.com/developing/clients/jsonrpc-api#subscription-websocket
-
-const SOLANA_DEFAULT_ENDPOINT = 'wss://solana.a.exodus.io/ws'
 const DEFAULT_RECONNECT_DELAY = ms('15s')
-const PING_INTERVAL = ms('60s')
-const TIMEOUT = ms('50s')
+const PING_INTERVAL = ms('30s')
 
 const debug = debugLogger('exodus:solana-api')
 
 export class Connection {
   constructor({
-    endpoint = SOLANA_DEFAULT_ENDPOINT,
+    endpoint,
     address,
     tokensAddresses = [],
-    callback,
+    onConnectionReady,
+    onConnectionClose,
     onMsg,
-    reconnectCallback = () => {},
-    reconnectDelay = DEFAULT_RECONNECT_DELAY,
   }) {
+    assert(endpoint, 'endpoint is required')
     this.address = address
     this.tokensAddresses = tokensAddresses
     this.endpoint = endpoint
-    this.callback = callback
+    this.onConnectionReady = onConnectionReady
+    this.onConnectionClose = onConnectionClose
     this.onMsg = onMsg
-    this.reconnectCallback = reconnectCallback
-    this.reconnectDelay = reconnectDelay
 
     this.shutdown = false
     this.ws = null
-    this.rpcQueue = {}
     this.messageQueue = []
     this.inProcessMessages = false
     this.pingTimeout = null
     this.reconnectTimeout = null
     this.txCache = {}
     this.seq = 0
-
-    this.sendMessage = makeConcurrent(
-      async (method, params = []) => {
-        return new Promise((resolve, reject) => {
-          if (this.isClosed || this.shutdown) return reject(new Error('connection not started'))
-          const id = ++this.seq
-
-          this.rpcQueue[id] = { resolve, reject }
-          this.rpcQueue[id].timeout = setTimeout(() => {
-            delete this.rpcQueue[id]
-            console.log(`solana ws: reply timeout (${method}) - ${JSON.stringify(params)} - ${id}`)
-            resolve(null)
-          }, TIMEOUT)
-          if (typeof this.rpcQueue[id].timeout.unref === 'function') {
-            this.rpcQueue[id].timeout.unref()
-          }
-
-          this.ws.send(JSON.stringify({ jsonrpc: '2.0', method, params, id }))
-        })
-      },
-      { concurrency: 15 }
-    )
   }
 
   newSocket(reqUrl) {
@@ -106,10 +77,25 @@ export class Connection {
     return 'NONE'
   }
 
-  doPing() {
+  startPing() {
     if (this.ws) {
-      this.ws.ping()
-      this.pingTimeout = setTimeout(this.doPing.bind(this), PING_INTERVAL)
+      this.pingTimeout = setInterval(() => {
+        if (this.isOpen) {
+          if (typeof this.ws.ping === 'function') {
+            this.ws.ping()
+          } else {
+            // Some WebSocket implementations (like in browser or Electron renderer) don't have a ping method
+            this.ws.send(
+              JSON.stringify({
+                jsonrpc: '2.0',
+                id: Date.now(),
+                method: 'getVersion',
+                params: [],
+              })
+            )
+          }
+        }
+      }, PING_INTERVAL)
     }
   }
 
@@ -117,75 +103,35 @@ export class Connection {
     // debug('Restarting WS:')
     this.reconnectTimeout = setTimeout(async () => {
       try {
-        debug('reconnecting ws...')
+        console.log('SOL reconnecting ws...')
         this.start()
-        await this.reconnectCallback()
       } catch (e) {
         console.log(`Error in reconnect callback: ${e.message}`)
       }
-    }, this.reconnectDelay)
+    }, DEFAULT_RECONNECT_DELAY)
   }
 
   onMessage(evt) {
     try {
       const json = JSON.parse(evt.data)
       debug('new ws msg:', json)
-      if (json.error) {
-        if (lodash.get(this.rpcQueue, json.id)) {
-          this.rpcQueue[json.id].reject(new Error(json.error.message))
-          clearTimeout(this.rpcQueue[json.id].timeout)
-          delete this.rpcQueue[json.id]
-        } else debug('Unsupported WS message:', json.error.message)
-      } else {
-        if (lodash.get(this.rpcQueue, json.id)) {
-          // json-rpc reply
-          clearTimeout(this.rpcQueue[json.id].timeout)
-          this.rpcQueue[json.id].resolve(json.result)
-          delete this.rpcQueue[json.id]
-        } else if (json.method) {
-          const msg = { method: json.method, ...lodash.get(json, 'params.result', json.result) }
-          debug('pushing msg to queue', msg)
-          this.messageQueue.push(msg) // sub results
-        }
-
-        this.processMessages(json)
-      }
+      this.onMsg(json)
     } catch (e) {
       debug(e)
-      debug('Cannot parse msg:', evt.data)
+      debug('Cannot process msg:', evt.data)
     }
   }
 
   onOpen(evt) {
     debug('Opened WS')
-    // subscribe to each addresses (SOL and ASA addr)
-    const addresses = [...this.tokensAddresses, this.address]
-    addresses.forEach((address) => {
-      // sub for account state changes
-      this.ws.send(
-        JSON.stringify({
-          jsonrpc: '2.0',
-          method: 'accountSubscribe',
-          params: [
-            address,
-            {
-              encoding: 'jsonParsed',
-            },
-          ],
-          id: ++this.seq,
-        })
-      )
-      // sub for incoming/outcoming txs
-      this.ws.send(
-        JSON.stringify({
-          jsonrpc: '2.0',
-          method: 'logsSubscribe',
-          params: [{ mentions: [address] }, { commitment: 'finalized' }],
-          id: ++this.seq,
-        })
-      )
-    })
-    // this.doPing()
+    this.onConnectionReady(evt)
+    this.startPing()
+  }
+
+  send(args) {
+    if (this.isOpen) {
+      this.ws.send(JSON.stringify(args))
+    }
   }
 
   onError(evt) {
@@ -194,32 +140,17 @@ export class Connection {
 
   onClose(evt) {
     debug('Closing WS')
-    clearTimeout(this.pingTimeout)
+    clearInterval(this.pingTimeout)
     clearTimeout(this.reconnectTimeout)
+    this.onConnectionClose(evt)
     if (!this.shutdown) {
       this.doRestart()
     }
   }
 
-  async processMessages(json) {
-    if (this.onMsg) await this.onMsg(json)
-    if (this.inProcessMessages) return null
-    this.inProcessMessages = true
-    try {
-      while (this.messageQueue.length > 0) {
-        const items = this.messageQueue.splice(0, this.messageQueue.length)
-        await this.callback(items)
-      }
-    } catch (e) {
-      console.log(`Solana: error processing streams: ${e.message}`)
-    } finally {
-      this.inProcessMessages = false
-    }
-  }
-
   async close() {
     clearTimeout(this.reconnectTimeout)
-    clearTimeout(this.pingTimeout)
+    clearInterval(this.pingTimeout)
     if (this.ws && (this.isConnecting || this.isOpen)) {
       // this.ws.send(JSON.stringify({ method: 'close' }))
       // Not sending the method above so just no need to wait below
@@ -245,4 +176,4 @@ export class Connection {
     await this.close()
     while (this.running) await delay(ms('50ms'))
   }
-} // Connection
+}

package/src/index.js

@@ -6,6 +6,7 @@ import { Api } from './api.js'
 
 export { SolanaMonitor } from './tx-log/index.js'
 export { SolanaClarityMonitor } from './tx-log/index.js'
+export { SolanaWebsocketMonitor } from './tx-log/index.js'
 export { createAccountState } from './account-state.js'
 export { getStakingInfo } from './staking-utils.js'
 export {
@@ -32,4 +33,5 @@ const serverApi = new Api({ assets }) // TODO: remove it, clean every use from p
 export default serverApi // TODO: remove it
 
 export { Api } from './api.js'
+export { WsApi } from './ws-api.js'
 export { ClarityApi } from './clarity-api.js'

package/src/rpc-api.js

@@ -112,14 +112,6 @@ export class RpcApi {
     return result?.value?.blockhash
   }
 
-  async getPriorityFee(transaction) {
-    // https://docs.helius.dev/solana-rpc-nodes/alpha-priority-fee-api
-    const result = await this.rpcCall('getPriorityFeeEstimate', [
-      { transaction, options: { recommended: true } },
-    ])
-    return result.priorityFeeEstimate
-  }
-
   async getBlockTime(slot) {
     // might result in error if executed on a validator with partial ledger (https://github.com/solana-labs/solana/issues/12413)
     return this.rpcCall('getBlockTime', [slot])

package/src/tx-log/README.md

@@ -0,0 +1,63 @@
+# WS Clarity Monitor
+
+This monitor will use a Mix of Clarity RPC calls and WS events (using Helius Enhanced Websockets: https://www.helius.dev/docs/enhanced-websockets)
+
+## Solana Monitor startup logic using WS and Clarity API
+
+1. on wallet first start we call API to get the current account state (balances + txsHistory + staking), for both SOL and tokens.
+
+2. we subscribe to WS for both SOL and tokens addresses (accountSubscribe, transactionSubscribe).
+
+3. after that we will rely only on WS to get new txs and update balances, but we will still ran regular tick to get a list of new "token accounts" (tokenAccountsByOwner), this to detect if the wallet has new ATA with new tokens. We could replace this with programSubscribe once Helius implements that method.
+
+4. when a new tx is received from WS we will parse it and add it to the tx log immediately.
+
+5. when a new balance update is received from WS we will update the balance immediately, (plus we also get and update stakingInfo).
+
+6. when a new tx is received, it is parsed and if it's a staking tx we update the stakingInfo in the state.
+
+### Notes
+
+- when there's a balance we always fetch and update stakingInfo.
+
+- Both Laserstream gRPC and Geyser enhanced websockets are serviced by Laserstream under the hood.
+
+For `Richat`:
+If you have 10 ATA you will have
+
+- 11 `accountSubscribe` - 1 for wallet and 10 for ATA
+- 1 `transactionSubscribe` - all 11 addresses in account.include
+- 1 `tokenInitSubscribe` - 1 with wallet address
+
+For `transactionSubscribe`, once you have new ATA unfortunately you need to send new `transactionSubscribe`, once you receive messages that subscription is set you need to unsubscribe the old subscription, at that moment you will be able to receive duplicated transactions for a short period of time.
+(But this way you will not miss any transaction and will be easy to filter out duplicates on client side by txId)
+
+Note that `tokenInitSubscribe` is a Richat-only method (in development). It doesn't exist in Laserstream/Geyser. It's used to check for new token accounts. For that we still need to rely on the `getTokenAccountsByOwner` RPC (or clarity call).
+
+## WS RPC
+
+##### `accountSubscribe` and `accountNotification`
+
+When we are subscribed (`accountSubscribe`) to both SOL and SPL tokens and our account sends out an SPL token.
+
+We'll have 2 `accountNotification`. Notifying us about the SPL balance change but also SOL balance change (because of the fees spent to send the SPL token).
+
+How can we distinguish that? looking at the owner. For our SOL account the program owner is `11111111111111111111111111111111` while for tokens we have `TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA`
+
+### What to test in platform
+
+- [x] check on wallet startup if the wallet tick updating balances and txHistory (for txs received when the wallet was close)
+- [x] send/receive SOL token (got expected balance and txLog)
+- [x] send/receive SPL token (got expected balance and txLog)
+- [x] send/receive SPL2022 token, like PYUSD (got expected balance and txLog)
+- [ ] receive SPL token that was not previously received (no token account yet)
+- [x] staking/unstaking/withdraw SOL
+- [x] force WS disconnect (check if the tokenAccounts are re-subscribed, apparently the onClose event is not called and the socket is still open)
+  - [x] disconnect wallet A -> send SOL from a different wallet B (connected) - re-connect A -> the wallet should catch up and see the balance and txLog
+- [x] JUP swaps (and other DEX interactions?)
+
+### Known issues:
+
+- [ ] when receiving a new token with no token account yet (the txLog is not written)
+
+  - if it's a known token (in this.assets) but no tokenAccount yet, the balance will update, txLog won't.

package/src/tx-log/clarity-monitor.js

@@ -63,6 +63,8 @@ export class SolanaClarityMonitor extends BaseMonitor {
     if (unknownTokensList.length > 0) {
       this.emit('unknown-tokens', unknownTokensList)
     }
+
+    return unknownTokensList
   }
 
   async getStakingAddressesFromTxLog({ assetName, walletAccount }) {
@@ -126,13 +128,22 @@ export class SolanaClarityMonitor extends BaseMonitor {
     const isHistoryUpdateTick =
       this.tickCount[walletAccount] % this.ticksBetweenHistoryFetches === 0
 
-    const shouldUpdateHistory = refresh || isHistoryUpdateTick || balanceChanged
+    const baseAssetTxLog = await this.aci.getTxLog({
+      assetName,
+      walletAccount,
+    })
+    const hasUnconfirmedSentTx = [...baseAssetTxLog].some((tx) => tx.pending && tx.sent)
+
+    const shouldUpdateHistory =
+      refresh || isHistoryUpdateTick || balanceChanged || hasUnconfirmedSentTx
     const shouldUpdateOnlyBalance = balanceChanged && !shouldUpdateHistory
 
+    // start a batch
+    const batch = this.aci.createOperationsBatch()
     // getHistory is more likely to fail/be rate limited, so we want to update users balance only on a lot of ticks
     if (this.shouldUpdateBalanceBeforeHistory || shouldUpdateOnlyBalance) {
       // update all state at once
-      await this.updateState({ account, walletAccount, staking })
+      this.updateState({ account, walletAccount, staking, batch })
       await this.emitUnknownTokensEvent({ tokenAccounts })
     }
 
@@ -149,13 +160,21 @@ export class SolanaClarityMonitor extends BaseMonitor {
 
       // update all state at once
       const clearedLogItems = await this.markStaleTransactions({ walletAccount, logItemsByAsset })
-      await this.updateTxLogByAsset({ walletAccount, logItemsByAsset: clearedLogItems, refresh })
-      await this.updateState({ account, cursorState, walletAccount, staking })
+      this.updateTxLogByAssetBatch({
+        logItemsByAsset: clearedLogItems,
+        walletAccount,
+        refresh,
+        batch,
+      })
+      this.updateState({ account, cursorState, walletAccount, staking, batch })
       await this.emitUnknownTokensEvent({ tokenAccounts })
       if (refresh || cursorChanged) {
         this.cursors[walletAccount] = cursorState.cursor
       }
     }
+
+    // close batch
+    await this.aci.executeOperationsBatch(batch)
   }
 
   async getHistory({ address, accountState, refresh, tokenAccounts } = Object.create(null)) {
@@ -172,12 +191,10 @@ export class SolanaClarityMonitor extends BaseMonitor {
     const mappedTransactions = []
     for (const tx of transactions) {
       // we get the token name using the token.mintAddress
-      let tokenName = this.api.tokens.get(tx.token?.mintAddress)?.name
-      if (tx.token && !tokenName) {
-        tokenName = 'unknown' // unknown token
-      }
+      const assetName = tx.token
+        ? this.api.tokens.get(tx.token.mintAddress)?.name ?? 'unknown'
+        : baseAsset.name
 
-      const assetName = tokenName ?? baseAsset.name
       const asset = this.assets[assetName]
       if (assetName === 'unknown' || !asset) continue // skip unknown tokens
       const feeAsset = asset.feeAsset
@@ -249,12 +266,10 @@ export class SolanaClarityMonitor extends BaseMonitor {
   }
 
   async getAccountsAndBalances({ refresh, address, accountState, walletAccount }) {
-    const tokens = Object.keys(this.assets).filter((name) => name !== this.asset.name)
     const [accountInfo, { balances: splBalances, accounts: tokenAccounts }] = await Promise.all([
       this.api.getAccountInfo(address).catch(() => {}),
       this.api.getTokensBalancesAndAccounts({
         address,
-        filterByTokens: tokens,
       }),
     ])
 
@@ -321,7 +336,8 @@ export class SolanaClarityMonitor extends BaseMonitor {
     }
   }
 
-  async updateState({ account, cursorState, walletAccount, staking }) {
+  updateState({ account, cursorState = {}, walletAccount, staking, batch }) {
+    const assetName = this.asset.name
     const { balance, tokenBalances, rentExemptAmount, accountSize, ownerChanged } = account
     const newData = {
       balance,
@@ -332,7 +348,7 @@ export class SolanaClarityMonitor extends BaseMonitor {
       stakingInfo: staking,
       ...cursorState,
     }
-    return this.updateAccountState({ newData, walletAccount })
+    return this.updateAccountStateBatch({ assetName, walletAccount, newData, batch })
   }
 
   async getStakingInfo({ address, accountState, walletAccount }) {

package/src/tx-log/index.js

@@ -1,2 +1,3 @@
-export * from './solana-monitor.js'
-export * from './clarity-monitor.js'
+export { SolanaMonitor } from './solana-monitor.js'
+export { SolanaClarityMonitor } from './clarity-monitor.js'
+export { SolanaWebsocketMonitor } from './ws-monitor.js'

package/src/tx-log/solana-monitor.js

@@ -7,7 +7,6 @@ import { DEFAULT_POOL_ADDRESS } from '../account-state.js'
 
 const DEFAULT_REMOTE_CONFIG = {
   rpcs: [],
-  ws: [],
   staking: { enabled: true, pool: DEFAULT_POOL_ADDRESS },
 }
 
@@ -81,9 +80,8 @@ export class SolanaMonitor extends BaseMonitor {
   }
 
   setServer(config = {}) {
-    const { rpcs, ws, staking = {} } = { ...DEFAULT_REMOTE_CONFIG, ...config }
+    const { rpcs, staking = {} } = { ...DEFAULT_REMOTE_CONFIG, ...config }
     this.api.setServer(rpcs[0])
-    this.api.setWsEndpoint(ws[0])
     this.staking = staking
   }
 
@@ -100,6 +98,8 @@ export class SolanaMonitor extends BaseMonitor {
     if (unknownTokensList.length > 0) {
       this.emit('unknown-tokens', unknownTokensList)
     }
+
+    return unknownTokensList
   }
 
   async getStakingAddressesFromTxLog({ assetName, walletAccount }) {
@@ -170,10 +170,12 @@ export class SolanaMonitor extends BaseMonitor {
     const shouldUpdateHistory = refresh || isHistoryUpdateTick || balanceChanged
     const shouldUpdateOnlyBalance = balanceChanged && !shouldUpdateHistory
 
+    // start a batch
+    const batch = this.aci.createOperationsBatch()
     // getHistory is more likely to fail/be rate limited, so we want to update users balance only on a lot of ticks
     if (this.shouldUpdateBalanceBeforeHistory || shouldUpdateOnlyBalance) {
       // update all state at once
-      await this.updateState({ account, walletAccount, staking })
+      this.updateState({ account, walletAccount, staking, batch })
       await this.emitUnknownTokensEvent({ tokenAccounts })
     }
 
@@ -190,13 +192,21 @@ export class SolanaMonitor extends BaseMonitor {
 
       // update all state at once
       const clearedLogItems = await this.markStaleTransactions({ walletAccount, logItemsByAsset })
-      await this.updateTxLogByAsset({ walletAccount, logItemsByAsset: clearedLogItems, refresh })
-      await this.updateState({ account, cursorState, walletAccount, staking })
+      this.updateTxLogByAssetBatch({
+        logItemsByAsset: clearedLogItems,
+        walletAccount,
+        refresh,
+        batch,
+      })
+      this.updateState({ account, cursorState, walletAccount, staking, batch })
       await this.emitUnknownTokensEvent({ tokenAccounts })
       if (refresh || cursorChanged) {
         this.cursors[walletAccount] = cursorState.cursor
       }
     }
+
+    // close batch
+    await this.aci.executeOperationsBatch(batch)
   }
 
   async getHistory({ address, accountState, refresh, tokenAccounts } = Object.create(null)) {
@@ -212,7 +222,11 @@ export class SolanaMonitor extends BaseMonitor {
 
     const mappedTransactions = []
     for (const tx of transactions) {
-      const assetName = _.get(tx, 'token.tokenName', baseAsset.name)
+      // we get the token name using the token.mintAddress
+      const assetName = tx.token
+        ? this.api.tokens.get(tx.token.mintAddress)?.name ?? 'unknown'
+        : baseAsset.name
+
       const asset = this.assets[assetName]
       if (assetName === 'unknown' || !asset) continue // skip unknown tokens
       const feeAsset = asset.feeAsset
@@ -262,11 +276,7 @@ export class SolanaMonitor extends BaseMonitor {
         item.data.meta = tx.data.meta
       }
 
-      if (
-        asset.assetType === this.api.tokenAssetType &&
-        item.feeAmount &&
-        item.feeAmount.isPositive
-      ) {
+      if (asset.name !== asset.baseAsset.name && item.feeAmount && item.feeAmount.isPositive) {
         const feeItem = {
           ..._.clone(item),
           coinName: feeAsset.name,
@@ -352,7 +362,8 @@ export class SolanaMonitor extends BaseMonitor {
     }
   }
 
-  async updateState({ account, cursorState, walletAccount, staking }) {
+  updateState({ account, cursorState = {}, walletAccount, staking, batch }) {
+    const assetName = this.asset.name
     const { balance, tokenBalances, rentExemptAmount, accountSize, ownerChanged } = account
     const newData = {
       balance,
@@ -363,7 +374,7 @@ export class SolanaMonitor extends BaseMonitor {
       stakingInfo: staking,
       ...cursorState,
     }
-    return this.updateAccountState({ newData, walletAccount })
+    return this.updateAccountStateBatch({ assetName, walletAccount, newData, batch })
   }
 
   async getStakingInfo({ address, accountState, walletAccount }) {