@xyo-network/chain-orchestration 1.17.2 → 1.17.6

package/src/archivists/lib/localPersistentArchivist.ts

@@ -0,0 +1,63 @@
+import { rm } from 'node:fs/promises'
+import Path from 'node:path'
+
+import { LmdbArchivist, LmdbArchivistConfigSchema } from '@xyo-network/archivist-lmdb'
+import type { ArchivistInstance } from '@xyo-network/archivist-model'
+import { getStoreDirectory, type StoreKind } from '@xyo-network/chain-protocol'
+
+const DEFAULT_STORAGE_ROOT = Path.join(process.cwd(), '.store')
+
+/**
+ * Returns a local persistent archivist
+ * @param name The name of the archivist
+ * @param dbName The name of the database
+ * @param storeName The name of the store
+ * @param storageRoot The root directory for storage (default is '.store' in the current working directory)
+ * @param kind The kind of the archivist
+ * @returns an archivist instance
+ */
+export const getLocalPersistentArchivist = (
+  name: string,
+  dbName: string,
+  storeName: string,
+  storageRoot?: string,
+  kind: StoreKind = 'lmdb',
+): Promise<ArchivistInstance> => {
+  switch (kind) {
+    case 'lmdb': {
+      const root = storageRoot ?? DEFAULT_STORAGE_ROOT
+      return LmdbArchivist.create({
+        account: 'random',
+        config: {
+          name,
+          clearStoreOnStart: false,
+          dbName,
+          location: getStoreDirectory(dbName, root, 'lmdb'),
+          schema: LmdbArchivistConfigSchema,
+          storeName,
+        },
+      })
+    }
+  }
+}
+
+/**
+ * Deletes a local persistent archivist
+ * @param dbName The name of the database
+ * @param storageRoot The root directory for storage (default is '.store' in the current working directory)
+ * @param kind The kind of the archivist
+ * @returns A promise that resolves when the archivist is deleted
+ */
+export const deleteLocalPersistentArchivist = async (dbName: string, storageRoot?: string, kind: StoreKind = 'lmdb'): Promise<void> => {
+  await Promise.resolve()
+  console.warn(`Deleting local persistent archivist for ${dbName} of kind ${kind}.`)
+  let location = ''
+  switch (kind) {
+    case 'lmdb': {
+      const root = storageRoot ?? DEFAULT_STORAGE_ROOT
+      location = getStoreDirectory(dbName, root, 'lmdb')
+      break
+    }
+  }
+  await rm(location, { recursive: true, force: true })
+}

package/src/buildTelemetryConfig.ts

@@ -0,0 +1,15 @@
+import type { Config } from '@xyo-network/xl1-protocol-sdk'
+
+export function buildTelemetryConfig(config: Config, serviceName: string, serviceVersion: string) {
+  const { otlpEndpoint } = config.telemetry?.otel ?? {}
+  const { path: endpoint = '/metrics', port: port = 9464 } = config.telemetry?.metrics?.scrape ?? {}
+  const telemetryConfig = {
+    attributes: {
+      serviceName,
+      serviceVersion,
+    },
+    otlpEndpoint,
+    metricsConfig: { endpoint, port },
+  }
+  return telemetryConfig
+}

package/src/health/index.ts

@@ -0,0 +1 @@
+export * from './initHealthEndpoints.ts'

package/src/health/initHealthEndpoints.ts

@@ -0,0 +1,71 @@
+import type {
+  IncomingMessage, Server, ServerResponse,
+} from 'node:http'
+import http from 'node:http'
+
+import type { CreatableStatus, Logger } from '@xylabs/sdk-js'
+import { isDefined } from '@xylabs/sdk-js'
+import type { Initializable } from '@xyo-network/xl1-protocol'
+import type { Config } from '@xyo-network/xl1-protocol-sdk'
+
+import type { RuntimeStatusMonitor } from '../status/index.ts'
+
+const sendStatus = (res: ServerResponse, status: CreatableStatus, errorCode: number) => {
+  const statusCode = status === 'started' ? 200 : errorCode
+  res.writeHead(statusCode, { 'Content-Type': 'application/json' })
+  res.end(JSON.stringify({ status }))
+}
+
+const notFound = (res: ServerResponse) => {
+  res.writeHead(404, { 'Content-Type': 'application/json' })
+  res.end(JSON.stringify({ status: 'not found' }))
+}
+
+/**
+ * Starts an HTTP server with Kubernetes health endpoints:
+ * - /healthz
+ * - /livez
+ * - /readyz
+ * @param port The port to listen on (default: 8080)
+ */
+const createHealthServer = (port: number, statusMonitor: RuntimeStatusMonitor, logger: Logger) => {
+  const server = http.createServer((req: IncomingMessage, res: ServerResponse) => {
+    const url = req.url ?? ''
+
+    const status = statusMonitor.getGlobalStatus()
+
+    switch (url) {
+      case '/healthz': {
+        sendStatus(res, status, 500)
+        break
+      }
+      case '/livez': {
+        sendStatus(res, status, 503)
+        break
+      }
+      case '/readyz': {
+        sendStatus(res, status, 503)
+        break
+      }
+      default: {
+        notFound(res)
+      }
+    }
+  })
+
+  server.listen(port, () => {
+    logger.log(`Health server running on http://localhost:${port}`)
+    logger.log(' - /healthz')
+    logger.log(' - /livez')
+    logger.log(' - /readyz')
+  })
+  return server
+}
+
+export const initHealthEndpoints: Initializable<{ config: Config; logger: Logger; statusReporter: RuntimeStatusMonitor }, Server | undefined> = (params) => {
+  const {
+    config, logger, statusReporter,
+  } = params
+  const { healthCheckPort } = config.producer
+  return isDefined(healthCheckPort) ? createHealthServer(healthCheckPort, statusReporter, logger) : undefined
+}

package/src/host/implementation/DefaultHost.ts

@@ -0,0 +1,25 @@
+import type { Host, ServiceProvider } from '../model/index.ts'
+
+/**
+ * A generic host implementation that can be used as a starting point for
+ * more complex host implementations.
+ */
+export class GenericHost implements Host {
+  services: ServiceProvider
+
+  constructor(services: ServiceProvider) {
+    this.services = services
+  }
+
+  async start(): Promise<void> {
+    await Promise.resolve()
+    // Initialize or start your services here
+    console.log('Host is starting...')
+  }
+
+  async stop(): Promise<void> {
+    await Promise.resolve()
+    // Stop or clean up services here
+    console.log('Host is stopping...')
+  }
+}

package/src/host/implementation/DefaultServiceCollection.ts

@@ -0,0 +1,60 @@
+import type {
+  ServiceCollection, ServiceDescriptor, ServiceProvider,
+} from '../model/index.ts'
+import { ServiceLifetime } from '../model/index.ts'
+import { DefaultServiceProvider } from './DefaultServiceProvider.ts'
+
+/**
+ * A collection of service descriptors, allowing adding/removing services before building.
+ */
+export class DefaultServiceCollection implements ServiceCollection {
+  private readonly descriptors: ServiceDescriptor[] = []
+
+  /**
+   * Register a service as Singleton: a single instance is created and shared.
+   */
+  addSingleton<T>(
+    identifier: string | symbol,
+    implementationFactory: () => T,
+  ): void {
+    this.descriptors.push({
+      identifier,
+      lifetime: ServiceLifetime.Singleton,
+      implementationFactory,
+    })
+  }
+
+  /**
+   * Register a service as Transient: a new instance is created on each request.
+   */
+  addTransient<T>(
+    _identifier: string | symbol,
+    _implementationFactory: () => T,
+  ): void {
+    throw new Error('Not implemented')
+  }
+
+  /**
+   * Build a concrete, immutable service provider from the current descriptors.
+   */
+  build(): ServiceProvider {
+    const services: Record<string, unknown> = {}
+    for (const descriptor of this.descriptors) {
+      if (descriptor.lifetime === ServiceLifetime.Singleton) {
+        services[descriptor.identifier.toString()] = descriptor.implementationFactory()
+      }
+    }
+    return new DefaultServiceProvider(services)
+  }
+
+  /**
+   * Removes all registrations matching the given identifier.
+   */
+  remove(identifier: string | symbol): void {
+    for (let i = this.descriptors.length - 1; i >= 0; i--) {
+      if (this.descriptors[i].identifier === identifier) {
+        this.descriptors.splice(i, 1)
+      }
+    }
+  }
+}

package/src/host/implementation/DefaultServiceProvider.ts

@@ -0,0 +1,12 @@
+import type { ServiceProvider } from '../model/index.ts'
+
+export class DefaultServiceProvider implements ServiceProvider {
+  protected _services: Record<string, unknown>
+  constructor(services: Record<string, unknown>) {
+    this._services = services
+  }
+
+  getService<T>(serviceIdentifier: string): T | undefined {
+    return this._services[serviceIdentifier] as T
+  }
+}

package/src/host/implementation/index.ts

@@ -0,0 +1,2 @@
+export * from './DefaultHost.ts'
+export * from './DefaultServiceProvider.ts'

package/src/host/index.ts

@@ -0,0 +1,2 @@
+export * from './implementation/index.ts'
+export * from './model/index.ts'

package/src/host/model/Host.ts

@@ -0,0 +1,21 @@
+import type { ServiceProvider } from './ServiceProvider.ts'
+
+/**
+ * Represents the host for an application.
+ */
+export interface Host {
+  /**
+   * Provides access to the application's services (for dependency injection).
+   */
+  services: ServiceProvider
+
+  /**
+   * Starts the host and any underlying services (e.g., background tasks).
+   */
+  start(): Promise<void>
+
+  /**
+   * Stops the host and attempts to gracefully shut down services.
+   */
+  stop(): Promise<void>
+}

package/src/host/model/ServiceCollection.ts

@@ -0,0 +1,22 @@
+import type { ServiceProvider } from './ServiceProvider.ts'
+
+export interface ServiceCollection {
+  build(): ServiceProvider
+}
+
+/**
+ * Represents the lifetime of a service
+ */
+export const ServiceLifetime = {
+  Singleton: 'Singleton',
+  Transient: 'Transient',
+} as const
+
+/**
+ * Describes a single service registration
+ */
+export interface ServiceDescriptor<T = unknown> {
+  identifier: string | symbol
+  implementationFactory: () => T
+  lifetime: keyof typeof ServiceLifetime
+}

package/src/host/model/ServiceProvider.ts

@@ -0,0 +1,3 @@
+export interface ServiceProvider {
+  getService<T = unknown>(serviceIdentifier: string): T | undefined
+}

package/src/host/model/index.ts

@@ -0,0 +1,3 @@
+export * from './Host.ts'
+export * from './ServiceCollection.ts'
+export * from './ServiceProvider.ts'

package/src/index.ts

@@ -1,8 +1,13 @@
 export * from './actor/index.ts'
-export * from './archivist/index.ts'
-export * from './bridge.ts'
+export * from './archivists/index.ts'
+export * from './buildTelemetryConfig.ts'
+export * from './ChainInitializableParams.ts'
 export * from './ConfigContext.ts'
 export * from './createDeclarationIntentBlock.ts'
+export * from './evm/index.ts'
+export * from './health/index.ts'
+export * from './host/index.ts'
 export * from './init/index.ts'
 export * from './manifest/index.ts'
 export * from './orchestrator/index.ts'
+export * from './status/index.ts'

package/src/init/index.ts

@@ -1,11 +1,10 @@
 export * from './initBalanceSummaryMap.ts'
 export * from './initBridgedModule.ts'
 export * from './initChainStakeViewer.ts'
-export * from './initPendingBlockArchivist.ts'
-export * from './initPendingTransactionArchivist.ts'
 export * from './initProducerAccount.ts'
 export * from './initSeedPhrase.ts'
 export * from './initServerNode.ts'
 export * from './initServerWallet.ts'
+export * from './initStatusReporter.ts'
 export * from './initTransferSummaryMap.ts'
 export * from './iterator.ts'

package/src/init/initChainStakeViewer.ts

@@ -32,7 +32,7 @@ export async function initSimpleStakeViewer(config: Config, logger?: Logger): Pr
   return stakeChainViewer
 }
 
-export async function initStakeViewer(config: Config, logger?: Logger): Promise<StakeViewer> {
+export async function initStakeViewer({ config, logger}: { config: Config; logger?: Logger }): Promise<StakeViewer> {
   if (canUseEvmProvider({ config })) {
     return await initEvmChainStakeViewer(config, logger)
   } else {

package/src/init/initServerNode.ts

@@ -10,20 +10,32 @@ import type {
 import type { ConfigContext } from '../ConfigContext.ts'
 import { loadNode } from '../manifest/index.ts'
 
-async function chainArchivistFromNode(node: NodeInstance) {
-  const chainArchivist = assertEx(asArchivistInstance(
+async function writableChainArchivistFromNode(node: NodeInstance) {
+  return asArchivistInstance(
     await node.resolve('Chain:Validated'),
     { required: true },
-  ), () => 'FinalizedArchivist not found in node')
-  return chainArchivist
+  )
 }
 
-async function pendingArchivistFromNode(node: NodeInstance) {
-  const pendingArchivist = assertEx(asArchivistInstance(
-    await node.resolve('XYOChain:Pending:PendingTransactions'),
+async function readonlyChainArchivistFromNode(node: NodeInstance) {
+  return assertEx(asArchivistInstance(
+    await node.resolve('Chain:Validated'),
+    { required: true },
+  ))
+}
+
+async function pendingTransactionsArchivistFromNode(node: NodeInstance) {
+  return asArchivistInstance(
+    await node.resolve('XYOChain:Pending:Transactions'),
+    { required: true },
+  )
+}
+
+async function pendingBlocksArchivistFromNode(node: NodeInstance) {
+  return asArchivistInstance(
+    await node.resolve('XYOChain:Pending:Blocks'),
     { required: true },
-  ), () => 'PendingArchivist not found in node')
-  return pendingArchivist
+  )
 }
 
 export async function initServerNode({
@@ -37,9 +49,11 @@ export async function initServerNode({
   const node: NodeInstance = providedNode ?? await loadNode({
     wallet, logger, config, transfersSummaryMap, balancesSummaryMap,
   })
-  const chainArchivist = await chainArchivistFromNode(node)
-  const pendingArchivist = await pendingArchivistFromNode(node)
+  const writableChainArchivist = await writableChainArchivistFromNode(node)
+  const readonlyChainArchivist = await readonlyChainArchivistFromNode(node)
+  const pendingTransactionsArchivist = await pendingTransactionsArchivistFromNode(node)
+  const pendingBlocksArchivist = await pendingBlocksArchivistFromNode(node)
   return {
-    node, chainArchivist, pendingArchivist,
+    node, writableChainArchivist, readonlyChainArchivist, pendingTransactionsArchivist, pendingBlocksArchivist,
   }
 }

package/src/init/initStatusReporter.ts

@@ -0,0 +1,16 @@
+import type { Logger } from '@xylabs/sdk-js'
+
+import { RuntimeStatusMonitor } from '../status/index.ts'
+
+export function initStatusReporter({ logger }: { logger: Logger }) {
+  const statusReporter = new RuntimeStatusMonitor(logger)
+  statusReporter.onGlobalTransition({ to: 'started' }, () => {
+    logger.log('All services started.')
+  })
+  statusReporter.onGlobalTransition({ to: 'error' }, () => {
+    logger.error('Producer encountered an unhandled error!')
+    // eslint-disable-next-line unicorn/no-process-exit
+    process.exit(1)
+  })
+  return statusReporter
+}

package/src/manifest/getLocator.ts

@@ -1,26 +1,18 @@
 import { type BaseMongoSdkPrivateConfig } from '@xylabs/mongo'
-import {
-  asAddress, assertEx, isDefined,
-  ZERO_ADDRESS,
-} from '@xylabs/sdk-js'
+import { isDefined } from '@xylabs/sdk-js'
 import { MemoryArchivist } from '@xyo-network/archivist-memory'
 import { MongoDBArchivistV2 } from '@xyo-network/archivist-mongodb'
 import { ViewArchivist } from '@xyo-network/archivist-view'
-import {
-  AddressBalanceDivinerV2, AddressTransferDiviner, ArchivistSyncDiviner, HeadValidationDiviner,
-} from '@xyo-network/chain-modules'
 import { initTelemetry } from '@xyo-network/chain-telemetry'
 import { AbstractModule, LoggerModuleStatusReporter } from '@xyo-network/module-abstract'
 import { ModuleFactoryLocator } from '@xyo-network/module-factory-locator'
 import type { MongoDBModuleParamsV2 } from '@xyo-network/module-model-mongodb'
 import type { WithStorageMeta } from '@xyo-network/payload-model'
 import { MemorySentinel } from '@xyo-network/sentinel-memory'
-import { StepSizes } from '@xyo-network/xl1-protocol'
 import type {
   BalancesStepSummary, MapType, TransfersStepSummary,
 } from '@xyo-network/xl1-protocol-sdk'
 import { hasMongoConfig } from '@xyo-network/xl1-protocol-sdk'
-import { Semaphore } from 'async-mutex'
 
 import type { ConfigContext } from '../ConfigContext.ts'
 
@@ -29,9 +21,7 @@ import type { ConfigContext } from '../ConfigContext.ts'
  * operation of the node (entirely in memory)
  * @returns A locator with the necessary modules registered
  */
-export const getLocator = async ({
-  config, logger, balancesSummaryMap, transfersSummaryMap,
-}: ConfigContext<{
+export const getLocator = async ({ config, logger}: ConfigContext<{
   balancesSummaryMap: MapType<string, WithStorageMeta<BalancesStepSummary>>
   transfersSummaryMap: MapType<string, WithStorageMeta<TransfersStepSummary>>
 }>) => {
@@ -67,24 +57,6 @@ export const getLocator = async ({
     locator.register(MongoDBArchivistV2.factory(params), undefined, true)
   }
 
-  locator.register(AddressBalanceDivinerV2.factory<AddressBalanceDivinerV2>({
-    traceProvider, meterProvider, statusReporter, summaryMap: balancesSummaryMap, stepSemaphores: StepSizes.map(() => new Semaphore(20)),
-  }))
-
-  locator.register(AddressTransferDiviner.factory<AddressTransferDiviner>({
-    traceProvider, meterProvider, statusReporter, summaryMap: transfersSummaryMap, stepSemaphores: StepSizes.map(() => new Semaphore(20)),
-  }))
-
-  const chainId = isDefined(config.chain.id)
-    ? assertEx(asAddress(config.chain.id), () => 'chain.id must be an Address')
-    : ZERO_ADDRESS
-  locator.register(HeadValidationDiviner.factory<HeadValidationDiviner>({
-    traceProvider,
-    meterProvider,
-    statusReporter,
-    chainId,
-    allowedProducers: config.producer.allowlist,
-  }))
   locator.register(MemoryArchivist.factory({
     traceProvider, meterProvider, statusReporter,
   }))
@@ -94,8 +66,5 @@ export const getLocator = async ({
   locator.register(ViewArchivist.factory({
     traceProvider, meterProvider, statusReporter,
   }))
-  locator.register(ArchivistSyncDiviner.factory({
-    traceProvider, meterProvider, statusReporter,
-  }))
   return locator
 }

package/src/manifest/loadNode.ts

@@ -9,7 +9,7 @@ import type { ConfigContext } from '../ConfigContext.ts'
 import { getLocator } from './getLocator.ts'
 import { NodeManifest } from './nodeManifest.ts'
 import { PrivateChildManifests } from './private/index.ts'
-import { PublicChildManifestsWithMempool, PublicChildManifestsWithoutMempool } from './public/index.ts'
+import { PublicChildManifests } from './public/index.ts'
 
 /**
  * Creates a node with the xyo-chain modules registered
@@ -22,8 +22,6 @@ export const loadNode = async ({
   transfersSummaryMap: MapType<string, WithStorageMeta<TransfersStepSummary>>
   wallet: WalletInstance
 }>) => {
-  const { enabled: mempoolEnabled } = config.mempool
-  const PublicChildManifests = mempoolEnabled ? PublicChildManifestsWithoutMempool : PublicChildManifestsWithMempool
   const locator = await getLocator({
     config, balancesSummaryMap, transfersSummaryMap,
   })

package/src/manifest/public/{WithoutMempool/Pending.json → Chain.json}

@@ -3,42 +3,42 @@
   "nodes": [
     {
       "config": {
-        "accountPath": "2",
-        "name": "Pending",
+        "accountPath": "1",
+        "name": "Chain",
         "schema": "network.xyo.node.config"
       },
       "modules": {
-        "private": [],
-        "public": [
+        "private": [
           {
             "config": {
-              "accountPath": "2/1/1",
-              "name": "PendingTransactions",
+              "accountPath": "1/1'/1'",
+              "name": "Validated",
               "getCache": {
                 "enabled": true,
                 "maxEntries": 5000
               },
-              "labels": {
-                "network.xyo.storage.class": "mongodb"
-              },
               "payloadSdkConfig": {
-                "collection": "pending_transaction_bundles"
+                "collection": "chain_validated"
               },
               "schema": "network.xyo.archivist.config"
             }
-          },
+          }
+        ],
+        "public": [
           {
             "config": {
-              "accountPath": "2/1/2",
-              "name": "PendingBlocks",
+              "accountPath": "1/1/1",
+              "name": "Finalized",
+              "allowedQueries": [
+                "network.xyo.query.archivist.get",
+                "network.xyo.query.archivist.next"
+              ],
               "getCache": {
                 "enabled": true,
-                "maxEntries": 5000
-              },
-              "payloadSdkConfig": {
-                "collection": "pending_block_bundles"
+                "maxEntries": 50000
               },
-              "schema": "network.xyo.archivist.config"
+              "originArchivist": "Validated",
+              "schema": "network.xyo.archivist.view.config"
             }
           }
         ]

package/src/manifest/public/{WithMempool/Pending.json → Pending.json}

@@ -13,7 +13,7 @@
           {
             "config": {
               "accountPath": "2/1/1",
-              "name": "PendingTransactions",
+              "name": "Transactions",
               "getCache": {
                 "enabled": true,
                 "maxEntries": 5000
@@ -30,7 +30,7 @@
           {
             "config": {
               "accountPath": "2/1/2",
-              "name": "PendingBlocks",
+              "name": "Blocks",
               "getCache": {
                 "enabled": true,
                 "maxEntries": 5000

package/src/manifest/public/index.ts

@@ -1,2 +1,20 @@
-export * from './WithMempool/index.ts'
-export * from './WithoutMempool/index.ts'
+import type { ModuleManifest, PackageManifestPayload } from '@xyo-network/manifest-model'
+
+import Chain from './Chain.json' with { type: 'json' }
+import Pending from './Pending.json' with { type: 'json' }
+
+/**
+ * Chain Node Manifest
+ */
+const ChainNodeManifest = Chain as PackageManifestPayload
+/**
+ * Pending Node Manifest
+ */
+const PendingNodeManifest = Pending as PackageManifestPayload
+/**
+ * Public Child Manifests
+ */
+export const PublicChildManifests: ModuleManifest[] = [
+  ...ChainNodeManifest.nodes,
+  ...PendingNodeManifest.nodes,
+]

package/src/orchestrator/Orchestrator.ts

@@ -5,8 +5,6 @@ import type { ActorInstance } from '../actor/index.ts'
 
 export interface OrchestratorInstance extends CreatableInstance {
   registerActor(actor: ActorInstance): Promise<void>
-  start(): Promise<boolean>
-  stop(): Promise<boolean>
 }
 
 @creatable()