@xyo-network/chain-mempool 1.16.3

package/src/manifest/getLocator.ts

@@ -0,0 +1,85 @@
+import { assertEx } from '@xylabs/assert'
+import { asAddress, ZERO_ADDRESS } from '@xylabs/hex'
+import type { Logger } from '@xylabs/logger'
+import { type BaseMongoSdkPrivateConfig } from '@xylabs/mongo'
+import { isDefined } from '@xylabs/typeof'
+import { MemoryArchivist } from '@xyo-network/archivist-memory'
+import { MongoDBArchivistV2 } from '@xyo-network/archivist-mongodb'
+import { ViewArchivist } from '@xyo-network/archivist-view'
+import { HeadValidationDiviner } from '@xyo-network/chain-modules'
+import { initTelemetry } from '@xyo-network/chain-telemetry'
+import { AbstractModule, LoggerModuleStatusReporter } from '@xyo-network/module-abstract'
+import { ModuleFactoryLocator } from '@xyo-network/module-factory-locator'
+import type { MongoDBModuleParamsV2 } from '@xyo-network/module-model-mongodb'
+import { MemorySentinel } from '@xyo-network/sentinel-memory'
+import type { Config } from '@xyo-network/xl1-protocol-sdk'
+import { hasMongoConfig } from '@xyo-network/xl1-protocol-sdk'
+
+export interface GetLocatorContext {
+  config: Config
+  logger?: Logger
+}
+
+/**
+ * Used for retrieving a locator with the necessary modules registered for testing
+ * operation of the node (entirely in memory)
+ * @returns A locator with the necessary modules registered
+ */
+export const getLocator = async (context: GetLocatorContext) => {
+  const { config, logger } = context
+  const { otlpEndpoint } = config.telemetry?.otel ?? {}
+  const { traceProvider, meterProvider } = await initTelemetry({
+    attributes: {
+      serviceName: 'xl1-api',
+      serviceVersion: '1.0.0',
+    },
+    otlpEndpoint,
+    metricsConfig: {
+      endpoint: '/metrics',
+      port: 9465,
+    },
+  })
+
+  if (isDefined(logger)) AbstractModule.defaultLogger = logger
+  const statusReporter = logger ? new LoggerModuleStatusReporter(logger) : undefined
+
+  const locator = new ModuleFactoryLocator()
+
+  // If there's a MongoDB configuration
+  const mongoConfig = config.storage?.mongo
+  if (hasMongoConfig(mongoConfig)) {
+    // Create the MongoDB SDK from the configuration
+    const {
+      connectionString: dbConnectionString, database: dbName, domain: dbDomain, password: dbPassword, username: dbUserName,
+    } = mongoConfig
+    const payloadSdkConfig: BaseMongoSdkPrivateConfig = {
+      dbConnectionString, dbDomain, dbName, dbPassword, dbUserName,
+    }
+    const params: Partial<MongoDBModuleParamsV2> = {
+      meterProvider, payloadSdkConfig, statusReporter, traceProvider,
+    }
+    // Register the MongoDB Archivist as the default
+    locator.register(MongoDBArchivistV2.factory(params), undefined, true)
+  }
+
+  const chainId = isDefined(config.chain.id)
+    ? assertEx(asAddress(config.chain.id), () => 'chain.id must be an Address')
+    : ZERO_ADDRESS
+  locator.register(HeadValidationDiviner.factory<HeadValidationDiviner>({
+    traceProvider,
+    meterProvider,
+    statusReporter,
+    chainId,
+    allowedProducers: config.producer.allowlist,
+  }))
+  locator.register(MemoryArchivist.factory({
+    traceProvider, meterProvider, statusReporter,
+  }))
+  locator.register(MemorySentinel.factory({
+    traceProvider, meterProvider, statusReporter,
+  }))
+  locator.register(ViewArchivist.factory({
+    traceProvider, meterProvider, statusReporter,
+  }))
+  return locator
+}

package/src/manifest/getNode.ts

@@ -0,0 +1,32 @@
+import type { Logger } from '@xylabs/logger'
+import { ManifestWrapper } from '@xyo-network/manifest-wrapper'
+import type { WalletInstance } from '@xyo-network/wallet-model'
+import type { Config } from '@xyo-network/xl1-protocol-sdk'
+
+import { getLocator } from './getLocator.ts'
+import { NodeManifest } from './nodeManifest.ts'
+import { PrivateChildManifests } from './private/index.ts'
+import { PublicChildManifests } from './public/index.ts'
+
+export interface GetNodeContext {
+  config: Config
+  logger?: Logger
+  wallet: WalletInstance
+}
+
+/**
+ * Creates a node with the xyo-chain modules registered
+ * @param context The context to use for the node
+ * @returns A node with the xyo-chain modules registered
+ */
+export const getNode = async (context: GetNodeContext) => {
+  const { wallet } = context
+  const locator = await getLocator(context)
+  const wrapper = new ManifestWrapper(NodeManifest, wallet, locator, PublicChildManifests, PrivateChildManifests)
+  const [node, ...childNodes] = await wrapper.loadNodes()
+  if (childNodes?.length > 0) {
+    await Promise.all(childNodes.map(childNode => node.register(childNode)))
+    await Promise.all(childNodes.map(childNode => node.attach(childNode.address, true)))
+  }
+  return node
+}

package/src/manifest/index.ts

@@ -0,0 +1,5 @@
+export * from './getLocator.ts'
+export * from './getNode.ts'
+export * from './nodeManifest.ts'
+export * from './private/index.ts'
+export * from './public/index.ts'

package/src/manifest/node.json

@@ -0,0 +1,17 @@
+{
+  "$schema": "https://raw.githubusercontent.com/XYOracleNetwork/sdk-xyo-client-js/main/packages/manifest/src/schema.json",
+  "nodes": [
+    {
+      "config": {
+        "accountPath": "44'/60'/1",
+        "name": "XYOChain",
+        "schema": "network.xyo.node.config"
+      },
+      "modules": {
+        "private": [],
+        "public": []
+      }
+    }
+  ],
+  "schema": "network.xyo.manifest"
+}

package/src/manifest/nodeManifest.ts

@@ -0,0 +1,8 @@
+import type { PackageManifestPayload } from '@xyo-network/manifest-model'
+
+import node from './node.json' with { type: 'json' }
+
+/**
+ * Root Node Manifest
+ */
+export const NodeManifest = node as PackageManifestPayload

package/src/manifest/private/index.ts

@@ -0,0 +1,4 @@
+/**
+ * Private Child Manifests
+ */
+export const PrivateChildManifests = []

package/src/manifest/public/Chain.json

@@ -0,0 +1,101 @@
+{
+  "$schema": "https://raw.githubusercontent.com/XYOracleNetwork/sdk-xyo-client-js/main/packages/manifest/src/schema.json",
+  "nodes": [
+    {
+      "config": {
+        "accountPath": "1",
+        "name": "Chain",
+        "schema": "network.xyo.node.config"
+      },
+      "modules": {
+        "private": [
+          {
+            "config": {
+              "accountPath": "1/1'/1'",
+              "name": "Validated",
+              "getCache": {
+                "enabled": true,
+                "maxEntries": 5000
+              },
+              "payloadSdkConfig": {
+                "collection": "chain_validated"
+              },
+              "schema": "network.xyo.archivist.config"
+            }
+          },
+          {
+            "config": {
+              "accountPath": "1/1'/2'",
+              "schema": "network.xyo.diviner.chain.head.validation.config",
+              "eventSubscriptions": [
+                {
+                  "sourceEvent": "inserted",
+                  "sourceModule": "Submissions",
+                  "targetModuleFunction": "divine"
+                }
+              ],
+              "inArchivist": "Submissions",
+              "outArchivist": "Validated",
+              "name": "HeadValidationDiviner"
+            }
+          },
+          {
+            "config": {
+              "accountPath": "1/1'/3'",
+              "automations": [
+                {
+                  "frequency": 1000,
+                  "frequencyUnits": "millis",
+                  "schema": "network.xyo.automation.interval",
+                  "type": "interval"
+                }
+              ],
+              "name": "ChainValidationSentinel",
+              "schema": "network.xyo.sentinel.config",
+              "synchronous": true,
+              "tasks": [
+                {
+                  "mod": "HeadValidationDiviner",
+                  "endPoint": "divine"
+                }
+              ]
+            }
+          }
+        ],
+        "public": [
+          {
+            "config": {
+              "accountPath": "1/1/1",
+              "name": "Submissions",
+              "getCache": {
+                "enabled": true,
+                "maxEntries": 5000
+              },
+              "payloadSdkConfig": {
+                "collection": "chain_submissions"
+              },
+              "schema": "network.xyo.archivist.config"
+            }
+          },
+          {
+            "config": {
+              "accountPath": "1/1/2",
+              "name": "Finalized",
+              "allowedQueries": [
+                "network.xyo.query.archivist.get",
+                "network.xyo.query.archivist.next"
+              ],
+              "getCache": {
+                "enabled": true,
+                "maxEntries": 50000
+              },
+              "originArchivist": "Validated",
+              "schema": "network.xyo.archivist.view.config"
+            }
+          }
+        ]
+      }
+    }
+  ],
+  "schema": "network.xyo.manifest"
+}

package/src/manifest/public/Pending.json

@@ -0,0 +1,35 @@
+{
+  "$schema": "https://raw.githubusercontent.com/XYOracleNetwork/sdk-xyo-client-js/main/packages/manifest/src/schema.json",
+  "nodes": [
+    {
+      "config": {
+        "accountPath": "2",
+        "name": "Pending",
+        "schema": "network.xyo.node.config"
+      },
+      "modules": {
+        "private": [],
+        "public": [
+          {
+            "config": {
+              "accountPath": "2/1/1",
+              "name": "PendingTransactions",
+              "getCache": {
+                "enabled": true,
+                "maxEntries": 5000
+              },
+              "labels": {
+                "network.xyo.storage.class": "mongodb"
+              },
+              "payloadSdkConfig": {
+                "collection": "pending_bundles"
+              },
+              "schema": "network.xyo.archivist.config"
+            }
+          }
+        ]
+      }
+    }
+  ],
+  "schema": "network.xyo.manifest"
+}

package/src/manifest/public/index.ts

@@ -0,0 +1,20 @@
+import type { ModuleManifest, PackageManifestPayload } from '@xyo-network/manifest-model'
+
+import Chain from './Chain.json' with { type: 'json' }
+import Pending from './Pending.json' with { type: 'json' }
+
+/**
+ * Chain Node Manifest
+ */
+export const ChainNodeManifest = Chain as PackageManifestPayload
+/**
+ * Pending Node Manifest
+ */
+export const PendingNodeManifest = Pending as PackageManifestPayload
+/**
+ * Public Child Manifests
+ */
+export const PublicChildManifests: ModuleManifest[] = [
+  ...ChainNodeManifest.nodes,
+  ...PendingNodeManifest.nodes,
+]

package/src/manifest/public/spec/Node.spec.ts

@@ -0,0 +1,32 @@
+import type { MemoryNode } from '@xyo-network/node-memory'
+import { HDWallet } from '@xyo-network/wallet'
+import { getDefaultConfig } from '@xyo-network/xl1-protocol-sdk'
+import {
+  beforeAll, describe, expect,
+  it,
+} from 'vitest'
+
+import { getNode, GetNodeContext } from '../../getNode.ts'
+import { SilentLogger } from '@xylabs/logger'
+
+/**
+ * @group manifest
+ */
+describe('Node', () => {
+  let node: MemoryNode
+  let context: GetNodeContext
+  
+  beforeAll(async () => {
+    const config = getDefaultConfig()
+    const logger = new SilentLogger()
+    const wallet = await HDWallet.random()
+    context = {
+      config, logger, wallet,
+    }
+    // Create a node with the default configuration
+    node = await getNode(context)
+  })
+  it('should return node', () => {
+    expect(node).toBeDefined()
+  })
+})

package/src/server/app.ts

@@ -0,0 +1,36 @@
+import {
+  customPoweredByHeader,
+  disableCaseSensitiveRouting,
+  disableExpressDefaultPoweredByHeader,
+  getJsonBodyParser,
+  getJsonBodyParserOptions,
+  responseProfiler,
+  standardErrors,
+  standardResponses,
+} from '@xylabs/express'
+import type { NodeInstance } from '@xyo-network/node-model'
+import compression from 'compression'
+import cors from 'cors'
+import type { Express } from 'express'
+import express from 'express'
+
+import { addInstrumentation } from './instrumentation.ts'
+import { addRoutes } from './routes/index.ts'
+
+export const getApp = (node: NodeInstance): Express => {
+  addInstrumentation()
+  const app = express()
+  app.set('etag', false)
+  app.use(cors())
+  app.use(compression())
+  app.use(responseProfiler)
+  app.use(getJsonBodyParser(getJsonBodyParserOptions({ limit: '1mb' })))
+  app.use(standardResponses)
+  disableExpressDefaultPoweredByHeader(app)
+  app.use(customPoweredByHeader)
+  disableCaseSensitiveRouting(app)
+  app.node = node
+  addRoutes(app)
+  app.use(standardErrors)
+  return app
+}

package/src/server/index.ts

@@ -0,0 +1,13 @@
+export * from './app.ts'
+export * from './server.ts'
+
+import type { NodeInstance } from '@xyo-network/node-model'
+
+declare global {
+  // eslint-disable-next-line @typescript-eslint/no-namespace
+  namespace Express {
+    interface Application {
+      node: NodeInstance
+    }
+  }
+}

package/src/server/instrumentation.ts

@@ -0,0 +1,15 @@
+import { registerInstrumentations } from '@opentelemetry/instrumentation'
+import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express'
+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'
+
+/**
+ * Registers OpenTelemetry instrumentations for HTTP and Express.
+ * This function is used to set up the necessary instrumentations for monitoring
+ * HTTP requests and Express applications. Since it monkey patches the Express
+ * router & middleware system, it should be called before any Express applications
+ * are defined.
+ */
+export const addInstrumentation = () => {
+  const instrumentations = [new HttpInstrumentation(), new ExpressInstrumentation()]
+  registerInstrumentations({ instrumentations })
+}

package/src/server/routes/addRoutes.ts

@@ -0,0 +1,11 @@
+import type { Express } from 'express'
+
+import { addNodeRoutes } from './address/index.ts'
+import { addDataLakeRoutes } from './dataLake/index.ts'
+import { addHealthRoutes } from './healthz/index.ts'
+
+export const addRoutes = (app: Express) => {
+  addHealthRoutes(app)
+  addDataLakeRoutes(app)
+  addNodeRoutes(app)
+}

package/src/server/routes/address/AddressPathParams.ts

@@ -0,0 +1,3 @@
+export type AddressPathParams = {
+  address: string
+}

package/src/server/routes/address/addNodeRoutes.ts

@@ -0,0 +1,21 @@
+import type { Express } from 'express'
+import { StatusCodes } from 'http-status-codes'
+
+import { getAddress } from './get/index.ts'
+import { postAddress } from './post/index.ts'
+
+export const addNodeRoutes = (app: Express) => {
+  const defaultModule = app.node
+  const address = defaultModule.address
+  const defaultModuleEndpoint = `/${address}`
+  app.get('/', (_req, res) => res.redirect(StatusCodes.MOVED_TEMPORARILY, defaultModuleEndpoint))
+  app.post('/', (_req, res) => res.redirect(StatusCodes.TEMPORARY_REDIRECT, defaultModuleEndpoint))
+  app.get('/:address', getAddress)
+  app.post('/:address', postAddress)
+  app.get('/:hash', (_req, res) => {
+    res.sendStatus(StatusCodes.NOT_FOUND)
+  })
+  app.post('/:hash', (_req, res) => {
+    res.sendStatus(StatusCodes.NOT_FOUND)
+  })
+}

package/src/server/routes/address/get/get.ts

@@ -0,0 +1,31 @@
+import { asyncHandler } from '@xylabs/express'
+import { asAddress } from '@xylabs/hex'
+import { isDefined } from '@xylabs/typeof'
+import { isModuleName } from '@xyo-network/module-model'
+import type { Payload } from '@xyo-network/payload-model'
+import type { RequestHandler } from 'express'
+import { StatusCodes } from 'http-status-codes'
+
+import type { AddressPathParams } from '../AddressPathParams.ts'
+
+const handler: RequestHandler<AddressPathParams, Payload[]> = async (req, res, next) => {
+  const { address: moduleIdentifier } = req.params
+  const { node } = req.app
+  const address = asAddress(moduleIdentifier)
+  if (isDefined(address)) {
+    let mod = node.address === address ? node : (await node.resolve(address, { direction: 'down' }))
+    if (mod) {
+      res.json(await mod.state())
+      return
+    }
+  }
+  if (isModuleName(moduleIdentifier)) {
+    const mod = await node.resolve(moduleIdentifier, { direction: 'down' })
+    if (mod) {
+      res.redirect(StatusCodes.MOVED_TEMPORARILY, `/${mod.address}`)
+      return
+    }
+  }
+  next('route')
+}
+export const getAddress = asyncHandler(handler)

package/src/server/routes/address/get/index.ts

@@ -0,0 +1 @@
+export * from './get.ts'

package/src/server/routes/address/index.ts

@@ -0,0 +1 @@
+export * from './addNodeRoutes.ts'

package/src/server/routes/address/post/getQueryConfig.ts

@@ -0,0 +1,23 @@
+import type { BoundWitness, QueryBoundWitness } from '@xyo-network/boundwitness-model'
+import { BoundWitnessSchema } from '@xyo-network/boundwitness-model'
+import type { ModuleConfig, ModuleInstance } from '@xyo-network/module-model'
+import { ModuleConfigSchema } from '@xyo-network/module-model'
+import type { Payload } from '@xyo-network/payload-model'
+import type { Request } from 'express'
+
+const DEFAULT_DEPTH = 5 as const
+
+export const getQueryConfig = (mod: ModuleInstance, req: Request, bw: QueryBoundWitness, payloads?: Payload[]): ModuleConfig | undefined => {
+  // TODO: Filter based on query addresses?
+  // Recurse through payloads for nested BWs
+  const nestedBwAddresses
+    = payloads
+      ?.flat(DEFAULT_DEPTH)
+      .filter<BoundWitness>((payload): payload is BoundWitness => payload?.schema === BoundWitnessSchema)
+      .map(bw => bw.addresses) ?? []
+  // TODO: Do we want to end up with a list of addresses or a list of address lists?
+  const addresses = [bw.addresses, ...nestedBwAddresses].filter(address => address.length > 0)
+  const allowed = addresses.length > 0 ? Object.fromEntries(mod.queries.map(schema => [schema, addresses])) : {}
+  const security = { allowed }
+  return { schema: ModuleConfigSchema, security }
+}

package/src/server/routes/address/post/index.ts

@@ -0,0 +1 @@
+export * from './post.ts'

package/src/server/routes/address/post/post.ts

@@ -0,0 +1,77 @@
+import { assertEx } from '@xylabs/assert'
+import { asyncHandler } from '@xylabs/express'
+import {
+  asAddress, isAddress,
+  toAddress,
+} from '@xylabs/hex'
+import type { JsonObject } from '@xylabs/object'
+import { isQueryBoundWitness, type QueryBoundWitness } from '@xyo-network/boundwitness-model'
+import { ModuleErrorBuilder } from '@xyo-network/module-abstract'
+import type { ModuleInstance, ModuleQueryResult } from '@xyo-network/module-model'
+import type { ModuleError, Payload } from '@xyo-network/payload-model'
+import type { RequestHandler } from 'express'
+import { StatusCodes } from 'http-status-codes'
+
+import type { AddressPathParams } from '../AddressPathParams.ts'
+import { getQueryConfig } from './getQueryConfig.ts'
+
+type PostAddressRequestBody = [QueryBoundWitness, undefined | Payload[]]
+
+const handler: RequestHandler<AddressPathParams, ModuleQueryResult | ModuleError, PostAddressRequestBody> = async (req, res, next) => {
+  const returnError = (code: number, message = 'An error occurred', details?: JsonObject) => {
+    const error = new ModuleErrorBuilder().message(message).details(details).build()
+    res.locals.rawResponse = false
+    res.status(code).json(error)
+    next()
+  }
+
+  const { address } = req.params
+  const { node } = req.app
+  const [bw, payloads] = Array.isArray(req.body) ? req.body : []
+  if (!isAddress(address)) {
+    return returnError(StatusCodes.BAD_REQUEST, 'Missing address')
+  }
+
+  if (!bw) {
+    return returnError(StatusCodes.BAD_REQUEST, 'Missing boundwitness')
+  }
+
+  if (!isQueryBoundWitness(bw)) {
+    return returnError(StatusCodes.BAD_REQUEST, 'Invalid query boundwitness')
+  }
+
+  let modules: ModuleInstance[] = []
+  const normalizedAddress = toAddress(address)
+  if (node.address === normalizedAddress) modules = [node]
+  else {
+    const typedAddress = asAddress(address)
+    const byAddress = (typedAddress === undefined) ? undefined : await node.resolve(typedAddress, { maxDepth: 10 })
+
+    if (byAddress) modules = [byAddress]
+    else {
+      const byName = await node.resolve(address, { direction: 'down' })
+      if (byName) {
+        const moduleAddress = assertEx(byName?.address, () => 'Error redirecting to module by address')
+        res.redirect(StatusCodes.TEMPORARY_REDIRECT, `/${moduleAddress}`)
+        return
+      } else {
+        return returnError(StatusCodes.NOT_FOUND, 'Module not found', { address })
+      }
+    }
+  }
+
+  if (modules.length > 0) {
+    const mod = modules[0]
+    const queryConfig = getQueryConfig(mod, req, bw, payloads)
+    try {
+      const queryResult = await mod.query(bw, payloads, queryConfig)
+      res.json(queryResult)
+    } catch (ex) {
+      return returnError(StatusCodes.INTERNAL_SERVER_ERROR, 'Query Failed', { message: (ex as Error)?.message ?? 'Unknown Error' })
+    }
+  } else {
+    return returnError(StatusCodes.NOT_FOUND, 'Module not found', { address })
+  }
+}
+
+export const postAddress = asyncHandler(handler)

package/src/server/routes/dataLake/addDataLakeRoutes.ts

@@ -0,0 +1,9 @@
+import type { Express } from 'express'
+
+import { archivistMiddleware } from './archivistMiddleware.ts'
+
+export const addDataLakeRoutes = (app: Express) => {
+  const { node } = app
+  const archivistModuleIdentifier = 'Chain:Finalized'
+  app.use('/chain', archivistMiddleware({ node, archivistModuleIdentifier }))
+}