@xyo-network/bridge-pub-sub 5.3.22 → 5.3.24

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xyo-network/bridge-pub-sub",
-  "version": "5.3.22",
+  "version": "5.3.24",
   "description": "Primary SDK for using XYO Protocol 2.0",
   "homepage": "https://xyo.network",
   "bugs": {
@@ -30,68 +30,64 @@
   "types": "dist/neutral/index.d.ts",
   "files": [
     "dist",
-    "src",
     "!**/*.bench.*",
     "!**/*.spec.*",
     "!**/*.test.*",
     "README.md"
   ],
   "dependencies": {
-    "@xyo-network/account": "~5.3.22",
-    "@xyo-network/archivist-model": "~5.3.22",
-    "@xyo-network/boundwitness-model": "~5.3.22",
-    "@xyo-network/bridge-abstract": "~5.3.22",
-    "@xyo-network/bridge-model": "~5.3.22",
-    "@xyo-network/config-payload-plugin": "~5.3.22",
-    "@xyo-network/diviner-boundwitness-model": "~5.3.22",
-    "@xyo-network/diviner-model": "~5.3.22",
-    "@xyo-network/module-abstract": "~5.3.22",
-    "@xyo-network/module-model": "~5.3.22",
-    "@xyo-network/node-model": "~5.3.22",
-    "@xyo-network/payload-builder": "~5.3.22",
-    "@xyo-network/payload-model": "~5.3.22",
     "async-mutex": "~0.5.0",
-    "lru-cache": "~11.2.7"
+    "lru-cache": "~11.2.7",
+    "@xyo-network/account": "~5.3.24",
+    "@xyo-network/archivist-model": "~5.3.24",
+    "@xyo-network/boundwitness-model": "~5.3.24",
+    "@xyo-network/bridge-model": "~5.3.24",
+    "@xyo-network/bridge-abstract": "~5.3.24",
+    "@xyo-network/config-payload-plugin": "~5.3.24",
+    "@xyo-network/module-model": "~5.3.24",
+    "@xyo-network/diviner-boundwitness-model": "~5.3.24",
+    "@xyo-network/node-model": "~5.3.24",
+    "@xyo-network/payload-model": "~5.3.24",
+    "@xyo-network/module-abstract": "~5.3.24",
+    "@xyo-network/payload-builder": "~5.3.24",
+    "@xyo-network/diviner-model": "~5.3.24"
   },
   "devDependencies": {
     "@opentelemetry/api": "^1.9.1",
     "@types/node": "^25.5.0",
-    "@xylabs/sdk-js": "^5.0.91",
-    "@xylabs/ts-scripts-common": "~7.6.8",
-    "@xylabs/ts-scripts-yarn3": "~7.6.8",
-    "@xylabs/tsconfig": "~7.6.8",
-    "@xylabs/vitest-extended": "~5.0.91",
-    "@xyo-network/account": "~5.3.22",
-    "@xyo-network/archivist-memory": "~5.3.22",
-    "@xyo-network/archivist-model": "~5.3.22",
-    "@xyo-network/boundwitness-model": "~5.3.22",
-    "@xyo-network/bridge-abstract": "~5.3.22",
-    "@xyo-network/bridge-model": "~5.3.22",
-    "@xyo-network/config-payload-plugin": "~5.3.22",
-    "@xyo-network/diviner-boundwitness-memory": "~5.3.22",
-    "@xyo-network/diviner-boundwitness-model": "~5.3.22",
-    "@xyo-network/diviner-model": "~5.3.22",
-    "@xyo-network/module-abstract": "~5.3.22",
-    "@xyo-network/module-model": "~5.3.22",
-    "@xyo-network/node-memory": "~5.3.22",
-    "@xyo-network/node-model": "~5.3.22",
-    "@xyo-network/payload-builder": "~5.3.22",
-    "@xyo-network/payload-model": "~5.3.22",
-    "@xyo-network/payload-wrapper": "~5.3.22",
+    "@xylabs/sdk-js": "^5.0.93",
+    "@xylabs/ts-scripts-common": "~7.6.16",
+    "@xylabs/ts-scripts-pnpm": "~7.6.16",
+    "@xylabs/tsconfig": "~7.6.16",
+    "@xylabs/vitest-extended": "~5.0.93",
     "acorn": "^8.16.0",
     "async-mutex": "~0.5.0",
     "axios": "^1.14.0",
-    "cosmiconfig": "^9.0.1",
-    "esbuild": "^0.27.4",
-    "eslint": "^10.1.0",
+    "esbuild": "^0.28.0",
     "ethers": "^6.16.0",
-    "lru-cache": "^11.2.7",
-    "rollup": "^4.60.1",
+    "lru-cache": "~11.2.7",
     "tslib": "^2.8.1",
     "typescript": "~5.9.3",
     "vite": "^8.0.3",
     "vitest": "~4.1.2",
-    "zod": "^4.3.6"
+    "zod": "^4.3.6",
+    "@xyo-network/account": "~5.3.24",
+    "@xyo-network/archivist-memory": "~5.3.24",
+    "@xyo-network/archivist-model": "~5.3.24",
+    "@xyo-network/boundwitness-model": "~5.3.24",
+    "@xyo-network/bridge-model": "~5.3.24",
+    "@xyo-network/bridge-abstract": "~5.3.24",
+    "@xyo-network/config-payload-plugin": "~5.3.24",
+    "@xyo-network/diviner-boundwitness-memory": "~5.3.24",
+    "@xyo-network/diviner-model": "~5.3.24",
+    "@xyo-network/diviner-boundwitness-model": "~5.3.24",
+    "@xyo-network/node-model": "~5.3.24",
+    "@xyo-network/node-memory": "~5.3.24",
+    "@xyo-network/module-model": "~5.3.24",
+    "@xyo-network/module-abstract": "~5.3.24",
+    "@xyo-network/payload-builder": "~5.3.24",
+    "@xyo-network/payload-model": "~5.3.24",
+    "@xyo-network/payload-wrapper": "~5.3.24"
   },
   "peerDependencies": {
     "@xylabs/sdk-js": "^5",
@@ -101,4 +97,4 @@
   "publishConfig": {
     "access": "public"
   }
-}
+}

package/src/AbstractModuleHost/AbstractModuleHost.ts

@@ -1,12 +0,0 @@
-import type { BaseParams, Promisable } from '@xylabs/sdk-js'
-import { Base } from '@xylabs/sdk-js'
-import type { ModuleInstance } from '@xyo-network/module-model'
-
-export type ModuleHostParams<THostedInstance extends ModuleInstance = ModuleInstance> = BaseParams<{
-  mod: THostedInstance
-}>
-
-export abstract class AbstractModuleHost<TParams extends ModuleHostParams = ModuleHostParams> extends Base<TParams> {
-  abstract start(): Promisable<void>
-  abstract stop(): Promisable<void>
-}

package/src/AbstractModuleHost/index.ts

@@ -1 +0,0 @@
-export * from './AbstractModuleHost.ts'

package/src/AsyncQueryBus/AsyncQueryBusBase.ts

@@ -1,163 +0,0 @@
-import type { Address, TypeCheck } from '@xylabs/sdk-js'
-import { assertEx, Base } from '@xylabs/sdk-js'
-import type { ArchivistInstance } from '@xyo-network/archivist-model'
-import { isArchivistInstance } from '@xyo-network/archivist-model'
-import type { BoundWitness, QueryBoundWitness } from '@xyo-network/boundwitness-model'
-import type { BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'
-import type { DivinerInstance } from '@xyo-network/diviner-model'
-import { isDivinerInstance } from '@xyo-network/diviner-model'
-import type {
-  ModuleConfig, ModuleIdentifier, ModuleInstance,
-} from '@xyo-network/module-model'
-import { ResolveHelper } from '@xyo-network/module-model'
-import type { Sequence } from '@xyo-network/payload-model'
-import { SequenceConstants } from '@xyo-network/payload-model'
-import { Mutex } from 'async-mutex'
-import { LRUCache } from 'lru-cache'
-
-import type { AsyncQueryBusParams } from './model/index.ts'
-
-const POLLING_FREQUENCY_MIN = 100 as const
-const POLLING_FREQUENCY_MAX = 60_000 as const
-const POLLING_FREQUENCY_DEFAULT = 1000 as const
-
-export class AsyncQueryBusBase<TParams extends AsyncQueryBusParams = AsyncQueryBusParams> extends Base<TParams> {
-  protected _lastState?: LRUCache<Address, Sequence>
-  protected _targetConfigs: Record<Address, ModuleConfig> = {}
-  protected _targetQueries: Record<Address, string[]> = {}
-
-  private _lastResolveFailure: Record<ModuleIdentifier, number> = {}
-  private _queriesArchivist?: ArchivistInstance
-  private _queriesDiviner?: DivinerInstance<BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload, QueryBoundWitness>
-  private _reResolveDelay = 1000 * 5 // 5 seconds
-  private _resolveMutex = new Mutex()
-  private _responsesArchivist?: ArchivistInstance
-  private _responsesDiviner?: DivinerInstance<BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload, BoundWitness>
-
-  constructor(params: TParams) {
-    super(params)
-  }
-
-  get config(): TParams['config'] {
-    return this.params.config
-  }
-
-  get pollFrequency(): number {
-    const frequency = this.config?.pollFrequency ?? POLLING_FREQUENCY_DEFAULT
-    if (frequency < POLLING_FREQUENCY_MIN || frequency > POLLING_FREQUENCY_MAX) {
-      return POLLING_FREQUENCY_DEFAULT
-    }
-    return frequency
-  }
-
-  get rootModule() {
-    return this.params.rootModule
-  }
-
-  /**
-   * A cache of the last offset of the Diviner process per address
-   */
-  protected get lastState(): LRUCache<Address, Sequence> {
-    const requiredConfig = { max: 1000, ttl: 0 }
-    this._lastState = this._lastState ?? new LRUCache<Address, Sequence>(requiredConfig)
-    return this._lastState
-  }
-
-  async queriesArchivist() {
-    return await this._resolveMutex.runExclusive(async () => {
-      this._queriesArchivist
-        = this._queriesArchivist
-          ?? (await this.resolve(
-            assertEx(this.config?.intersect?.queries?.archivist, () => 'No queries Archivist defined'),
-            isArchivistInstance,
-          ))
-      return this._queriesArchivist
-    })
-  }
-
-  async queriesDiviner() {
-    return await this._resolveMutex.runExclusive(async () => {
-      this._queriesDiviner
-        = this._queriesDiviner
-          ?? ((await this.resolve(
-            assertEx(this.config?.intersect?.queries?.boundWitnessDiviner, () => 'No queries Diviner defined'),
-            isDivinerInstance,
-          )) as DivinerInstance<BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload, QueryBoundWitness>)
-      return this._queriesDiviner
-    })
-  }
-
-  async responsesArchivist() {
-    return await this._resolveMutex.runExclusive(async () => {
-      this._responsesArchivist
-        = this._responsesArchivist
-          ?? (await this.resolve(
-            assertEx(this.config?.intersect?.responses?.archivist, () => 'No responses Archivist defined'),
-            isArchivistInstance,
-          ))
-      return this._responsesArchivist
-    })
-  }
-
-  async responsesDiviner() {
-    return await this._resolveMutex.runExclusive(async () => {
-      this._responsesDiviner
-        = this._responsesDiviner
-          ?? ((await this.resolve(
-            assertEx(this.config?.intersect?.responses?.boundWitnessDiviner, () => 'No responses Diviner defined'),
-            isDivinerInstance,
-          )) as DivinerInstance<BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload, BoundWitness>)
-      return this._responsesDiviner
-    })
-  }
-
-  /**
-   * Commit the internal state of the process. This is similar
-   * to a transaction completion in a database and should only be called
-   * when results have been successfully persisted to the appropriate
-   * external stores.
-   * @param address The module address to commit the state for
-   * @param nextState The state to commit
-   */
-  protected async commitState(address: Address, nextState: Sequence) {
-    await Promise.resolve()
-    // TODO: Offload to Archivist/Diviner instead of in-memory
-    const lastState = this.lastState.get(address)
-    if (lastState && lastState >= nextState) return
-    this.lastState.set(address, nextState)
-  }
-
-  /**
-   * Retrieves the last state of the process. Used to recover state after
-   * preemptions, reboots, etc.
-   */
-  protected async retrieveState(address: Address): Promise<Sequence> {
-    await Promise.resolve()
-    const state = this.lastState.get(address)
-    if (state === undefined) {
-      const newState = SequenceConstants.minLocalSequence
-      this.lastState.set(address, newState)
-      return newState
-    } else {
-      return state
-    }
-  }
-
-  private async resolve<T extends ModuleInstance>(id: ModuleIdentifier, typeCheck: TypeCheck<T>): Promise<T | undefined> {
-    if (Date.now() - (this._lastResolveFailure[id] ?? 0) < this._reResolveDelay) {
-      return
-    }
-    this._lastResolveFailure[id] = Date.now()
-    const resolved = await ResolveHelper.resolveModuleIdentifier(this.rootModule, id)
-    if (resolved) {
-      if (typeCheck(resolved)) {
-        delete this._lastResolveFailure[id]
-        return resolved
-      } else {
-        this.logger?.warn(`Unable to resolve responsesDiviner as correct type [${id}][${resolved?.constructor?.name}]: ${resolved.id}`)
-      }
-    } else {
-      this.logger?.debug(`Unable to resolve queriesArchivist [${id}] [${await ResolveHelper.traceModuleIdentifier(this.rootModule, id)}]`)
-    }
-  }
-}

package/src/AsyncQueryBus/AsyncQueryBusClient.ts

@@ -1,190 +0,0 @@
-import type { Address, Hash } from '@xylabs/sdk-js'
-import {
-  assertEx, clearTimeoutEx, delay, forget, setTimeoutEx,
-} from '@xylabs/sdk-js'
-import type { QueryBoundWitness } from '@xyo-network/boundwitness-model'
-import { isBoundWitness } from '@xyo-network/boundwitness-model'
-import type { BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'
-import { BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'
-import type { CacheConfig, ModuleQueryResult } from '@xyo-network/module-model'
-import { PayloadBuilder } from '@xyo-network/payload-builder'
-import {
-  asSchema,
-  type ModuleError, type Payload, type WithSources,
-} from '@xyo-network/payload-model'
-import { LRUCache } from 'lru-cache'
-
-import { AsyncQueryBusBase } from './AsyncQueryBusBase.ts'
-import type { AsyncQueryBusClientParams } from './model/index.ts'
-import { Pending } from './model/index.ts'
-
-export class AsyncQueryBusClient<TParams extends AsyncQueryBusClientParams = AsyncQueryBusClientParams> extends AsyncQueryBusBase<TParams> {
-  protected _queryCache?: LRUCache<Hash, Pending | ModuleQueryResult>
-  private _pollCount = 0
-  private _pollId?: string
-
-  constructor(params: TParams) {
-    super(params)
-  }
-
-  get queryCacheConfig(): LRUCache.Options<Hash, Pending | ModuleQueryResult, unknown> {
-    const queryCacheConfig: CacheConfig | undefined = this.config?.queryCache === true ? {} : this.config?.queryCache
-    return {
-      max: 100, ttl: 1000 * 60, ...queryCacheConfig,
-    }
-  }
-
-  get started() {
-    return !!this._pollId
-  }
-
-  /**
-   * A cache of queries that have been issued
-   */
-  protected get queryCache(): LRUCache<Hash, Pending | ModuleQueryResult> {
-    const config = this.queryCacheConfig
-    const requiredConfig = { noUpdateTTL: false, ttlAutopurge: true }
-    this._queryCache = this._queryCache ?? new LRUCache<Hash, Pending | ModuleQueryResult>({ ...config, ...requiredConfig })
-    return this._queryCache
-  }
-
-  listeningAddresses() {
-    return this._queryCache?.keys()
-  }
-
-  async send(address: Address, query: QueryBoundWitness, payloads?: Payload[] | undefined): Promise<ModuleQueryResult> {
-    this.logger?.debug(`Begin issuing query to: ${address}`)
-    const routedQuery = { ...query, $destination: [address] }
-    // console.log('queryArchivist - calling')
-    const queryArchivist = assertEx(
-      await this.queriesArchivist(),
-      () => `Unable to contact queriesArchivist [${this.config?.intersect?.queries?.archivist}]`,
-    )
-    // console.log('queryArchivist')
-
-    // TODO: Should we always re-hash to true up timestamps?  We can't
-    // re-sign correctly so we would lose that information if we did and
-    // would also be replying to consumers with a different query hash than
-    // they sent us (which might be OK since it reflect the chain of custody)
-    // Revisit this once we have proxy module support as they are another
-    // intermediary to consider.
-    const routedQueryHash = await PayloadBuilder.dataHash(routedQuery)
-    this.logger?.debug(`Issuing query: ${routedQueryHash} to: ${address}`)
-    // If there was data associated with the query, add it to the insert
-    const data = payloads ? [routedQuery, ...payloads] : [routedQuery]
-    const insertResult = await queryArchivist.insert?.(data)
-    this.logger?.debug(`Issued query: ${routedQueryHash} to: ${address}`)
-    this.queryCache.set(routedQueryHash, Pending)
-    if (!insertResult) throw new Error('Unable to issue query to queryArchivist')
-    return new Promise<ModuleQueryResult>((resolve, reject) => {
-      this.logger?.debug(`Polling for response to query: ${routedQueryHash}`)
-      let nextDelay = 100
-      const pollForResponse = async () => {
-        try {
-          this.start()
-          let response = this.queryCache.get(routedQueryHash)
-          // Poll for response until cache key expires (response timed out)
-          while (response !== undefined) {
-            // console.log('polling...')
-            // Wait a bit
-            await delay(nextDelay)
-            // Check the status of the response
-            response = this.queryCache.get(routedQueryHash)
-            // If status is no longer pending that means we received a response
-            if (response && response !== Pending) {
-              this.logger?.log(`Returning response to query: ${routedQueryHash}`)
-              resolve(response)
-              return
-            }
-            // back off the polling frequency
-            nextDelay = Math.floor(nextDelay * 1.2)
-            // cap it at 1000ms
-            if (nextDelay > 1000) nextDelay = 1000
-          }
-          // If we got here waiting for a response timed out
-          this.logger?.error('Timeout waiting for query response')
-          // Resolve with error to match what a local module would do if it were to error
-          // TODO: BW Builder/Sign result as this module?
-          const error: WithSources<ModuleError> = {
-            message: 'Timeout waiting for query response',
-            query: asSchema('network.xyo.boundwitness', true),
-            schema: asSchema('network.xyo.error.module', true),
-            $sources: [routedQueryHash],
-          }
-          reject(error)
-          return
-        } finally {
-          this.stop()
-        }
-      }
-      forget(pollForResponse())
-    })
-  }
-
-  /**
-   * Runs the background divine process on a loop with a delay
-   * specified by the `config.pollFrequency`
-   */
-  private poll() {
-    this._pollId = setTimeoutEx(async () => {
-      try {
-        await this.processIncomingResponses()
-      } catch (e) {
-        this.logger?.error?.(`Error in main loop: ${e}`)
-      } finally {
-        if (this._pollId) clearTimeoutEx(this._pollId)
-        this._pollId = undefined
-        this.poll()
-      }
-    }, this.pollFrequency)
-  }
-
-  /**
-   * Background process for processing incoming responses to previously issued queries
-   */
-  private processIncomingResponses = async () => {
-    const responseArchivist = await this.responsesArchivist()
-    if (responseArchivist) {
-      const responseBoundWitnessDiviner = await this.responsesDiviner()
-      if (responseBoundWitnessDiviner) {
-        const pendingCommands = [...this.queryCache.entries()].filter(([_, status]) => status === Pending)
-        // TODO: Do in throttled batches
-        await Promise.allSettled(
-          pendingCommands.map(async ([sourceQuery, status]) => {
-            if (status === Pending) {
-              const divinerQuery: BoundWitnessDivinerQueryPayload = {
-                limit: 1, order: 'desc', schema: BoundWitnessDivinerQuerySchema, sourceQuery,
-              }
-              const result = await responseBoundWitnessDiviner.divine([divinerQuery])
-              if (result && result.length > 0) {
-                const response = result.find(isBoundWitness)
-                if (response && (response as unknown as { $sourceQuery: string })?.$sourceQuery === sourceQuery) {
-                  this.logger?.debug(`Found response to query: ${sourceQuery}`)
-                  // Get any payloads associated with the response
-                  const payloads: Payload[] = response.payload_hashes?.length > 0 ? await responseArchivist.get(response.payload_hashes) : []
-                  this.queryCache.set(sourceQuery, [response, payloads, []])
-                }
-              }
-            }
-          }),
-        )
-      }
-    }
-  }
-
-  private start() {
-    if (this._pollCount === 0) {
-      this.poll()
-    }
-    this._pollCount++
-  }
-
-  private stop() {
-    this._pollCount--
-    if (this._pollCount <= 0) {
-      if (this._pollId) clearTimeoutEx(this._pollId)
-      this._pollId = undefined
-      this._pollCount = 0
-    }
-  }
-}