@xyo-network/chain-api 1.15.2 → 1.15.4

package/src/driver/mongo/MongoMap.ts

@@ -4,12 +4,17 @@ import {
 } from '@xylabs/creatable'
 import { BaseMongoSdk } from '@xylabs/mongo'
 import { isNull } from '@xylabs/typeof'
-import { AsynchronousMap } from '@xyo-network/chain-protocol'
+import { AsynchronousMap } from '@xyo-network/xl1-protocol-sdk'
+import { LRUCache } from 'lru-cache'
 import {
   Document, Filter, OptionalUnlessRequiredId, WithId,
 } from 'mongodb'
 
 export interface MongoMapParams<TData extends Document = Document> extends CreatableParams {
+  getCache?: {
+    enabled: boolean
+    maxEntries?: number
+  }
   sdk: BaseMongoSdk<TData>
 }
 
@@ -19,44 +24,77 @@ function stripMongoId<V extends Document>(doc: WithId<V>): V {
 }
 
 @creatable()
-export class MongoMap<K = string, V extends Document = Document>
+export class MongoMap<K extends {} = string, V extends Document = Document>
   extends AbstractCreatable<MongoMapParams<V>>
   implements AsynchronousMap<K, V> {
+  private _getCache: LRUCache<K, V> | undefined
+
   get sdk(): BaseMongoSdk<V> {
     return assertEx(this.params.sdk, () => 'No sdk specified')
   }
 
   async clear(): Promise<void> {
+    this._getCache?.clear()
     await this.sdk.deleteMany({})
   }
 
   async delete(id: K): Promise<boolean> {
+    this._getCache?.delete(id)
+
     const filter = { _id: id } as Filter<V>
     const result = await this.sdk.deleteOne(filter)
     return result.deletedCount > 0
   }
 
   async get(id: K): Promise<V | undefined> {
+    if (this._getCache) {
+      const getCacheResult = this._getCache.get(id)
+      if (getCacheResult) {
+        return getCacheResult
+      }
+    }
     const filter = { _id: id } as Filter<V>
     const doc = await this.sdk.findOne(filter)
     return isNull(doc) ? undefined : stripMongoId(doc)
   }
 
+  async getMany(ids: K[]): Promise<V[]> {
+    const result: V[] = []
+    for (const id of ids) {
+      const value = await this.get(id)
+      if (value) {
+        result.push(value)
+      }
+    }
+    return result
+  }
+
   async has(id: K): Promise<boolean> {
+    if (this._getCache) {
+      const getCacheResult = this._getCache.has(id)
+      if (getCacheResult) {
+        return getCacheResult
+      }
+    }
     const filter = { _id: id } as Filter<V>
     const exists = await this.sdk.findOne(filter)
     return isNull(exists) ? false : true
   }
 
-  async set(id: K, data: V): Promise<this> {
+  async set(id: K, data: V): Promise<void> {
     const filter = { _id: id } as Filter<V>
     const value = { ...data, _id: id } as OptionalUnlessRequiredId<V>
     await this.sdk.replaceOne(filter, value, { upsert: true })
-    return this
+    this._getCache?.set(id, data)
   }
 
   override async startHandler(): Promise<void> {
     await super.startHandler()
     // TODO: Ensure index
+
+    if (this.params.getCache?.enabled === true) {
+      const maxEntries = this.params.getCache?.maxEntries ?? 5000
+      this._getCache = new LRUCache<K, V>({ max: maxEntries })
+    }
   }
 }

package/src/helpers/index.ts

@@ -0,0 +1,4 @@
+export * from './initChainId.ts'
+export * from './initEvmProvider.ts'
+export * from './initInfuraProvider.ts'
+export * from './initJsonRpcProvider.ts'

package/src/helpers/initChainId.ts

@@ -0,0 +1,20 @@
+import { assertEx } from '@xylabs/assert'
+import { hexFrom, isHex } from '@xylabs/hex'
+import { isDefined } from '@xylabs/typeof'
+import type { Config } from '@xyo-network/xl1-protocol-sdk'
+
+export const canUseChainId = (config: Config): boolean => {
+  return isDefined(config.evm.chainId)
+}
+
+export const getChainId = (config: Config) => {
+  const chainId = assertEx(config.evm.chainId, () => 'Missing config.evm.chainId')
+  if (isHex(chainId, { prefix: true })) {
+    const hex = hexFrom(chainId)
+    const parsed = Number.parseInt(hex, 16)
+    return parsed
+  } else {
+    const parsed = Number.parseInt(chainId, 10)
+    return parsed
+  }
+}

package/src/helpers/initEvmProvider.ts

@@ -0,0 +1,24 @@
+import { assertEx } from '@xylabs/assert'
+import type { Logger } from '@xylabs/logger'
+import type { Config } from '@xyo-network/xl1-protocol-sdk'
+import type { Provider } from 'ethers'
+import type { JsonRpcProvider } from 'ethers/providers'
+
+import { canUseInfuraProvider, initInfuraProvider } from './initInfuraProvider.ts'
+import { canUseJsonRpcProvider, initJsonRpcProvider } from './initJsonRpcProvider.ts'
+
+let provider: Promise<JsonRpcProvider> | undefined
+
+export const initEvmProvider = async ({ config }: { config: Config; logger?: Logger }): Promise<Provider> => {
+  if (provider) return provider
+  if (canUseInfuraProvider(config)) {
+    provider = initInfuraProvider(config)
+  } else if (canUseJsonRpcProvider(config)) {
+    provider = initJsonRpcProvider(config)
+  }
+  return assertEx(await provider, () => 'Error: No provider available')
+}
+
+export const canUseEvmProvider = ({ config }: { config: Config }) => {
+  return canUseInfuraProvider(config) || canUseJsonRpcProvider(config)
+}

package/src/helpers/initInfuraProvider.ts

@@ -0,0 +1,27 @@
+import { assertEx } from '@xylabs/assert'
+import { isDefined } from '@xylabs/typeof'
+import type { Config } from '@xyo-network/xl1-protocol-sdk'
+import { InfuraProvider } from 'ethers/providers'
+
+import { canUseChainId, getChainId } from './initChainId.ts'
+
+let instance: Promise<InfuraProvider> | undefined
+
+export const initInfuraProvider = (config: Config) => {
+  if (instance) return instance
+  const providerConfig = getInfuraProviderConfig(config)
+  instance = Promise.resolve(new InfuraProvider(...providerConfig))
+  return instance
+}
+
+export const canUseInfuraProvider = (config: Config): boolean => {
+  return canUseChainId(config)
+    && isDefined(config.evm?.infura?.projectId)
+    && isDefined(config.evm?.infura?.projectSecret)
+}
+
+export const getInfuraProviderConfig = (config: Config) => {
+  const projectId = assertEx(config.evm?.infura?.projectId, () => 'Missing config.evm.infura.projectId')
+  const projectSecret = assertEx(config.evm?.infura?.projectSecret, () => 'Missing config.evm.infura.projectSecret')
+  return [getChainId(config), projectId, projectSecret] as const
+}

package/src/helpers/initJsonRpcProvider.ts

@@ -0,0 +1,21 @@
+import { assertEx } from '@xylabs/assert'
+import { isDefined } from '@xylabs/typeof'
+import type { Config } from '@xyo-network/xl1-protocol-sdk'
+import { JsonRpcProvider } from 'ethers/providers'
+
+import { canUseChainId, getChainId } from './initChainId.ts'
+
+export const initJsonRpcProvider = (config: Config) => {
+  const providerConfig = getJsonRpcProviderConfig(config)
+  return Promise.resolve(new JsonRpcProvider(...providerConfig))
+}
+
+export const canUseJsonRpcProvider = (config: Config) => {
+  return canUseChainId(config)
+    && isDefined(config.evm.jsonRpc?.url)
+}
+
+export const getJsonRpcProviderConfig = (config: Config) => {
+  const jsonRpcUrl = assertEx(config.evm.jsonRpc?.url, () => 'Missing config.evm.jsonRpc.url')
+  return [jsonRpcUrl, getChainId(config)] as const
+}

package/src/manifest/getLocator.ts

@@ -8,17 +8,20 @@ import { MemoryArchivist } from '@xyo-network/archivist-memory'
 import { MongoDBArchivistV2 } from '@xyo-network/archivist-mongodb'
 import { ViewArchivist } from '@xyo-network/archivist-view'
 import {
-  AddressBalanceDivinerV2, ArchivistSyncDiviner, balanceSummaryRepositoryFromMap, HeadValidationDiviner,
+  AddressBalanceDivinerV2, AddressTransferDiviner, ArchivistSyncDiviner, HeadValidationDiviner,
 } from '@xyo-network/chain-modules'
-import type { MapType } from '@xyo-network/chain-protocol'
 import { initTelemetry } from '@xyo-network/chain-telemetry'
 import { AbstractModule, LoggerModuleStatusReporter } from '@xyo-network/module-abstract'
 import { ModuleFactoryLocator } from '@xyo-network/module-factory-locator'
 import type { MongoDBModuleParamsV2 } from '@xyo-network/module-model-mongodb'
 import type { WithStorageMeta } from '@xyo-network/payload-model'
 import { MemorySentinel } from '@xyo-network/sentinel-memory'
-import type { BalancesStepSummary, Config } from '@xyo-network/xl1-protocol-sdk'
+import { StepSizes } from '@xyo-network/xl1-protocol'
+import type {
+  BalancesStepSummary, Config, MapType, TransfersStepSummary,
+} from '@xyo-network/xl1-protocol-sdk'
 import { hasMongoConfig } from '@xyo-network/xl1-protocol-sdk'
+import { Semaphore } from 'async-mutex'
 
 import { MongoMap } from '../driver/index.ts'
 
@@ -51,8 +54,8 @@ export const getLocator = async (context: GetLocatorContext) => {
   const statusReporter = logger ? new LoggerModuleStatusReporter(logger) : undefined
 
   const locator = new ModuleFactoryLocator()
-  // Initialize with an in-memory map for backing the summary repository
-  let summaryMap: MapType<Hash, WithStorageMeta<BalancesStepSummary>> = new Map<Hash, WithStorageMeta<BalancesStepSummary>>()
+  let balanceSummaryMap: MapType<string, WithStorageMeta<BalancesStepSummary>> | undefined
+  let transferSummaryMap: MapType<string, WithStorageMeta<TransfersStepSummary>> | undefined
   // If there's a MongoDB configuration
   const mongoConfig = config.storage?.mongo
   if (hasMongoConfig(mongoConfig)) {
@@ -70,13 +73,25 @@ export const getLocator = async (context: GetLocatorContext) => {
     locator.register(MongoDBArchivistV2.factory(params), undefined, true)
 
     // Use a persistent MongoMap for the summary repository if MongoDB is configured
-    const sdk = new BaseMongoSdk<WithStorageMeta<BalancesStepSummary>>({ ...payloadSdkConfig, collection: 'balance_summary_map' })
-    summaryMap = await MongoMap.create<MongoMap<Hash, WithStorageMeta<BalancesStepSummary>>>({ sdk })
+    const sdkBalanceSummaryMap = new BaseMongoSdk<WithStorageMeta<BalancesStepSummary>>({ ...payloadSdkConfig, collection: 'balance_summary_map' })
+    balanceSummaryMap = await MongoMap.create<MongoMap<Hash, WithStorageMeta<BalancesStepSummary>>>({
+      sdk: sdkBalanceSummaryMap,
+      getCache: { enabled: true, maxEntries: 5000 },
+    })
+
+    const sdkTransferSummaryMap = new BaseMongoSdk<WithStorageMeta<TransfersStepSummary>>({ ...payloadSdkConfig, collection: 'transfer_summary_map' })
+    transferSummaryMap = await MongoMap.create<MongoMap<Hash, WithStorageMeta<TransfersStepSummary>>>({
+      sdk: sdkTransferSummaryMap,
+      getCache: { enabled: true, maxEntries: 5000 },
+    })
   }
 
-  const summaryRepository = balanceSummaryRepositoryFromMap(summaryMap)
   locator.register(AddressBalanceDivinerV2.factory<AddressBalanceDivinerV2>({
-    traceProvider, meterProvider, statusReporter, summaryRepository,
+    traceProvider, meterProvider, statusReporter, summaryMap: balanceSummaryMap, stepSemaphores: StepSizes.map(() => new Semaphore(20)),
+  }))
+
+  locator.register(AddressTransferDiviner.factory<AddressTransferDiviner>({
+    traceProvider, meterProvider, statusReporter, summaryMap: transferSummaryMap, stepSemaphores: StepSizes.map(() => new Semaphore(20)),
   }))
 
   const chainId = isDefined(config.chain.id)

package/src/manifest/public/Chain.json

@@ -30,12 +30,12 @@
               "eventSubscriptions": [
                 {
                   "sourceEvent": "inserted",
-                  "sourceModule": "Chain:Submissions",
+                  "sourceModule": "Submissions",
                   "targetModuleFunction": "divine"
                 }
               ],
-              "inArchivist": "Chain:Submissions",
-              "outArchivist": "Chain:Validated",
+              "inArchivist": "Submissions",
+              "outArchivist": "Validated",
               "name": "HeadValidationDiviner"
             }
           },
@@ -66,7 +66,7 @@
               "accountPath": "1/1'/4'",
               "automations": [
                 {
-                  "frequency": 60000,
+                  "frequency": 10000,
                   "frequencyUnits": "millis",
                   "schema": "network.xyo.automation.interval",
                   "type": "interval"
@@ -82,6 +82,28 @@
                 }
               ]
             }
+          },
+          {
+            "config": {
+              "accountPath": "1/1'/5'",
+              "automations": [
+                {
+                  "frequency": 10000,
+                  "frequencyUnits": "millis",
+                  "schema": "network.xyo.automation.interval",
+                  "type": "interval"
+                }
+              ],
+              "name": "AddressTransferPollingSentinel",
+              "schema": "network.xyo.sentinel.config",
+              "synchronous": true,
+              "tasks": [
+                {
+                  "mod": "AddressTransferDiviner",
+                  "endPoint": "divine"
+                }
+              ]
+            }
           }
         ],
         "public": [
@@ -111,23 +133,38 @@
                 "enabled": true,
                 "maxEntries": 5000
               },
-              "originArchivist": "Chain:Validated",
+              "originArchivist": "Validated",
               "schema": "network.xyo.archivist.view.config"
             }
           },
           {
             "config": {
-              "accountPath": "1/1/3",
-              "schema": "network.xyo.diviner.chain.address.balance.config",
-              "archivist": "Chain:Validated",
+              "accountPath": "1/1/3'",
               "name": "AddressBalanceDiviner",
-              "eventSubscriptions": [
-                {
-                  "sourceEvent": "inserted",
-                  "sourceModule": "Chain:Validated",
-                  "targetModuleFunction": "divine"
+              "schema": "network.xyo.diviner.chain.address.balance.config",
+              "map": {
+                "lmdb": {
+                  "dbName": "summaries",
+                  "storeName": "address_balance",
+                  "location": ".store"
                 }
-              ]
+              },
+              "archivist": "Validated"
+            }
+          },
+          {
+            "config": {
+              "accountPath": "1/1/4'",
+              "name": "AddressTransferDiviner",
+              "schema": "network.xyo.diviner.chain.address.transfer.config",
+              "map": {
+                "lmdb": {
+                  "dbName": "summaries",
+                  "storeName": "address_transfer",
+                  "location": ".store"
+                }
+              },
+              "archivist": "Validated"
             }
           }
         ]
@@ -135,4 +172,4 @@
     }
   ],
   "schema": "network.xyo.manifest"
-}
+}

package/src/server/app.ts

@@ -9,6 +9,7 @@ import {
   standardResponses,
 } from '@xylabs/express'
 import type { NodeInstance } from '@xyo-network/node-model'
+import type { StakedChainContextRead, StakeEventsRead } from '@xyo-network/xl1-protocol-sdk'
 import compression from 'compression'
 import cors from 'cors'
 import type { Express } from 'express'
@@ -17,7 +18,7 @@ import express from 'express'
 import { addInstrumentation } from './instrumentation.ts'
 import { addRoutes } from './routes/index.ts'
 
-export const getApp = (node: NodeInstance): Express => {
+export const getApp = (node: NodeInstance, eventsReader: StakeEventsRead, stakedChainContext: StakedChainContextRead): Express => {
   addInstrumentation()
   const app = express()
   app.set('etag', false)
@@ -31,7 +32,7 @@ export const getApp = (node: NodeInstance): Express => {
   app.use(customPoweredByHeader)
   disableCaseSensitiveRouting(app)
   app.node = node
-  addRoutes(app)
+  addRoutes(app, eventsReader, stakedChainContext)
   app.use(standardErrors)
   return app
 }

package/src/server/routes/addRoutes.ts

@@ -1,11 +1,12 @@
+import type { StakedChainContextRead, StakeEventsRead } from '@xyo-network/xl1-protocol-sdk'
 import type { Express } from 'express'
 
 import { addNodeRoutes } from './address/index.ts'
 import { addDataLakeRoutes } from './dataLake/index.ts'
 import { addRpcRoutes } from './rpc/index.ts'
 
-export const addRoutes = (app: Express) => {
-  addRpcRoutes(app)
+export const addRoutes = (app: Express, eventsReader: StakeEventsRead, stakedChainContext: StakedChainContextRead) => {
+  addRpcRoutes(app, eventsReader, stakedChainContext)
   addDataLakeRoutes(app)
   addNodeRoutes(app)
 }

package/src/server/routes/healthz/get.ts

@@ -17,4 +17,4 @@ const handler: RequestHandler<NoReqParams> = (_req, res) => {
   res.status(200).send(data)
 }
 
-export const getHealthz = handler
+export const getHealthz: RequestHandler<NoReqParams> = handler

package/src/server/routes/rpc/routes/addRpcRoutes.ts

@@ -1,15 +1,16 @@
 import { setRawResponseFormat } from '@xylabs/express'
 import { NodeXyoViewer } from '@xyo-network/chain-rpc'
+import type { StakedChainContextRead, StakeEventsRead } from '@xyo-network/xl1-protocol-sdk'
 import {
   NodeXyoRunner, rpcEngineFromConnection,
   XyoBaseConnection,
 } from '@xyo-network/xl1-rpc'
 import type { Express } from 'express'
 
-export const addRpcRoutes = (app: Express) => {
+export const addRpcRoutes = (app: Express, eventsReader: StakeEventsRead, stakedChainContext: StakedChainContextRead) => {
   const { node } = app
   const runner = new NodeXyoRunner(node)
-  const viewer = new NodeXyoViewer(node)
+  const viewer = new NodeXyoViewer(node, eventsReader, stakedChainContext)
   const connection = new XyoBaseConnection({ runner, viewer })
   const engine = rpcEngineFromConnection(connection)
 

package/src/server/server.ts

@@ -1,12 +1,19 @@
 import { assertEx } from '@xylabs/assert'
+import { toEthAddress, ZERO_HASH } from '@xylabs/hex'
 import type { Logger } from '@xylabs/logger'
 import { isDefined, isString } from '@xylabs/typeof'
+import { asArchivistInstance } from '@xyo-network/archivist-model'
 import { boot } from '@xyo-network/bios'
 import type { BiosExternalInterface } from '@xyo-network/bios-model'
+import { EthereumChainStake, EthereumChainStakeEvents } from '@xyo-network/chain-ethereum'
 import type { NodeInstance } from '@xyo-network/node-model'
+import type { Payload, WithStorageMeta } from '@xyo-network/payload-model'
+import { StakedXyoChainV2__factory } from '@xyo-network/typechain'
 import { HDWallet } from '@xyo-network/wallet'
-import { type Config } from '@xyo-network/xl1-protocol-sdk'
+import type { ChainId } from '@xyo-network/xl1-protocol'
+import { type Config, readPayloadMapFromStore } from '@xyo-network/xl1-protocol-sdk'
 
+import { initInfuraProvider } from '../helpers/index.ts'
 import { getNode } from '../manifest/index.ts'
 import { getApp } from './app.ts'
 
@@ -49,10 +56,24 @@ export const getServer = async (context: GetServerContext) => {
   const bios = await boot()
   const seedPhrase = isDefined(mnemonic) ? mnemonic : await getSeedPhrase(bios, config, logger)
   const wallet = await HDWallet.fromPhrase(seedPhrase)
+  const provider = await initInfuraProvider(config)
+  const contractAddress = assertEx(config.chain.id, () => 'Missing config.evm.chainId') as ChainId
+  const contract = StakedXyoChainV2__factory.connect(toEthAddress(contractAddress), provider)
   const nodeContext = {
     wallet, logger, config,
   }
-  const app = getApp(node ?? await getNode(nodeContext))
+  const eventsReader = new EthereumChainStakeEvents(contract)
+  const stakeChainReader = await EthereumChainStake.create({ contract, eventsReader })
+  await stakeChainReader.start()
+  const rootNode = await getNode(nodeContext)
+  const archivist = assertEx(asArchivistInstance(
+    await rootNode.resolve('Chain:Validated'),
+    { required: true },
+  ), () => 'FinalizedArchivist not found in node')
+  const chainMap = readPayloadMapFromStore<WithStorageMeta<Payload>>(archivist)
+  const app = getApp(node ?? await getNode(nodeContext), eventsReader, {
+    chainId: contractAddress, stake: stakeChainReader, store: { chainMap }, head: () => ZERO_HASH,
+  })
   const server = app.listen(port, hostname, () => logger?.log(`[API] Server listening at http://${hostname}:${port}`))
   server.setTimeout(20_000)
   return server