@xyo-network/bridge-pub-sub 3.5.2 → 3.6.0-rc.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xyo-network/bridge-pub-sub",
-  "version": "3.5.2",
+  "version": "3.6.0-rc.10",
   "description": "Primary SDK for using XYO Protocol 2.0",
   "homepage": "https://xyo.network",
   "bugs": {
@@ -29,45 +29,46 @@
   "module": "dist/neutral/index.mjs",
   "types": "dist/neutral/index.d.ts",
   "dependencies": {
-    "@xylabs/array": "^4.4.9",
-    "@xylabs/assert": "^4.4.9",
-    "@xylabs/delay": "^4.4.9",
-    "@xylabs/exists": "^4.4.9",
-    "@xylabs/forget": "^4.4.9",
-    "@xylabs/hex": "^4.4.9",
-    "@xylabs/object": "^4.4.9",
-    "@xylabs/promise": "^4.4.9",
-    "@xylabs/timer": "^4.4.9",
-    "@xyo-network/account": "^3.5.2",
-    "@xyo-network/archivist-model": "^3.5.2",
-    "@xyo-network/boundwitness-model": "^3.5.2",
-    "@xyo-network/bridge-abstract": "^3.5.2",
-    "@xyo-network/bridge-model": "^3.5.2",
-    "@xyo-network/config-payload-plugin": "^3.5.2",
-    "@xyo-network/diviner-boundwitness-model": "^3.5.2",
-    "@xyo-network/diviner-model": "^3.5.2",
-    "@xyo-network/module-model": "^3.5.2",
-    "@xyo-network/node-model": "^3.5.2",
-    "@xyo-network/payload-builder": "^3.5.2",
-    "@xyo-network/payload-model": "^3.5.2",
+    "@xylabs/array": "^4.4.21",
+    "@xylabs/assert": "^4.4.21",
+    "@xylabs/delay": "^4.4.21",
+    "@xylabs/exists": "^4.4.21",
+    "@xylabs/forget": "^4.4.21",
+    "@xylabs/hex": "^4.4.21",
+    "@xylabs/object": "^4.4.21",
+    "@xylabs/promise": "^4.4.21",
+    "@xylabs/timer": "^4.4.21",
+    "@xyo-network/account": "^3.6.0-rc.10",
+    "@xyo-network/archivist-model": "^3.6.0-rc.10",
+    "@xyo-network/boundwitness-model": "^3.6.0-rc.10",
+    "@xyo-network/bridge-abstract": "^3.6.0-rc.10",
+    "@xyo-network/bridge-model": "^3.6.0-rc.10",
+    "@xyo-network/config-payload-plugin": "^3.6.0-rc.10",
+    "@xyo-network/diviner-boundwitness-model": "^3.6.0-rc.10",
+    "@xyo-network/diviner-model": "^3.6.0-rc.10",
+    "@xyo-network/module-model": "^3.6.0-rc.10",
+    "@xyo-network/node-model": "^3.6.0-rc.10",
+    "@xyo-network/payload-builder": "^3.6.0-rc.10",
+    "@xyo-network/payload-model": "^3.6.0-rc.10",
     "async-mutex": "^0.5.0",
     "lru-cache": "^11.0.2"
   },
   "devDependencies": {
-    "@xylabs/logger": "^4.4.9",
-    "@xylabs/ts-scripts-yarn3": "^4.2.4",
-    "@xylabs/tsconfig": "^4.2.4",
-    "@xylabs/vitest-extended": "^4.4.9",
-    "@xyo-network/archivist-memory": "^3.5.2",
-    "@xyo-network/diviner-boundwitness-memory": "^3.5.2",
-    "@xyo-network/module-abstract": "^3.5.2",
-    "@xyo-network/node-memory": "^3.5.2",
-    "@xyo-network/node-model": "^3.5.2",
-    "@xyo-network/payload-wrapper": "^3.5.2",
+    "@xylabs/logger": "^4.4.21",
+    "@xylabs/ts-scripts-yarn3": "^4.2.6",
+    "@xylabs/tsconfig": "^4.2.6",
+    "@xylabs/vitest-extended": "^4.4.21",
+    "@xyo-network/archivist-memory": "^3.6.0-rc.10",
+    "@xyo-network/diviner-boundwitness-memory": "^3.6.0-rc.10",
+    "@xyo-network/module-abstract": "^3.6.0-rc.10",
+    "@xyo-network/node-memory": "^3.6.0-rc.10",
+    "@xyo-network/node-model": "^3.6.0-rc.10",
+    "@xyo-network/payload-wrapper": "^3.6.0-rc.10",
     "typescript": "^5.7.2",
-    "vitest": "^2.1.5"
+    "vitest": "^2.1.8"
   },
   "publishConfig": {
     "access": "public"
-  }
+  },
+  "stableVersion": "3.5.2"
 }

package/src/AsyncQueryBus/AsyncQueryBusBase.ts

@@ -1,5 +1,5 @@
 import { assertEx } from '@xylabs/assert'
-import type { Address } from '@xylabs/hex'
+import type { Address, Hex } from '@xylabs/hex'
 import type { TypeCheck } from '@xylabs/object'
 import { Base } from '@xylabs/object'
 import type { ArchivistInstance } from '@xyo-network/archivist-model'
@@ -12,6 +12,7 @@ import type {
   ModuleConfig, ModuleIdentifier, ModuleInstance,
 } from '@xyo-network/module-model'
 import { ResolveHelper } from '@xyo-network/module-model'
+import { SequenceConstants } from '@xyo-network/payload-model'
 import { Mutex } from 'async-mutex'
 import { LRUCache } from 'lru-cache'
 
@@ -22,7 +23,7 @@ const POLLING_FREQUENCY_MAX = 60_000 as const
 const POLLING_FREQUENCY_DEFAULT = 1000 as const
 
 export class AsyncQueryBusBase<TParams extends AsyncQueryBusParams = AsyncQueryBusParams> extends Base<TParams> {
-  protected _lastState?: LRUCache<Address, number>
+  protected _lastState?: LRUCache<Address, Hex>
   protected _targetConfigs: Record<Address, ModuleConfig> = {}
   protected _targetQueries: Record<Address, string[]> = {}
 
@@ -57,9 +58,9 @@ export class AsyncQueryBusBase<TParams extends AsyncQueryBusParams = AsyncQueryB
   /**
    * A cache of the last offset of the Diviner process per address
    */
-  protected get lastState(): LRUCache<Address, number> {
+  protected get lastState(): LRUCache<Address, Hex> {
     const requiredConfig = { max: 1000, ttl: 0 }
-    this._lastState = this._lastState ?? new LRUCache<Address, number>(requiredConfig)
+    this._lastState = this._lastState ?? new LRUCache<Address, Hex>(requiredConfig)
     return this._lastState
   }
 
@@ -119,7 +120,7 @@ export class AsyncQueryBusBase<TParams extends AsyncQueryBusParams = AsyncQueryB
    * @param address The module address to commit the state for
    * @param nextState The state to commit
    */
-  protected async commitState(address: Address, nextState: number) {
+  protected async commitState(address: Address, nextState: Hex) {
     await Promise.resolve()
     // TODO: Offload to Archivist/Diviner instead of in-memory
     const lastState = this.lastState.get(address)
@@ -131,13 +132,11 @@ export class AsyncQueryBusBase<TParams extends AsyncQueryBusParams = AsyncQueryB
    * Retrieves the last state of the process. Used to recover state after
    * preemptions, reboots, etc.
    */
-  protected async retrieveState(address: Address): Promise<number> {
+  protected async retrieveState(address: Address): Promise<Hex> {
     await Promise.resolve()
     const state = this.lastState.get(address)
     if (state === undefined) {
-      // If this is a boot we can go back a bit in time
-      // and begin processing recent commands
-      const newState = Date.now() - 1000
+      const newState = SequenceConstants.minLocalSequence
       this.lastState.set(address, newState)
       return newState
     } else {

package/src/AsyncQueryBus/AsyncQueryBusClient.ts

@@ -4,13 +4,13 @@ import { forget } from '@xylabs/forget'
 import type { Address } from '@xylabs/hex'
 import { clearTimeoutEx, setTimeoutEx } from '@xylabs/timer'
 import type { QueryBoundWitness } from '@xyo-network/boundwitness-model'
-import { isBoundWitnessWithMeta } from '@xyo-network/boundwitness-model'
+import { isBoundWitness } from '@xyo-network/boundwitness-model'
 import type { BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'
 import { BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'
 import type { CacheConfig, ModuleQueryResult } from '@xyo-network/module-model'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
 import type {
-  ModuleError, Payload, PayloadWithMeta, WithMeta,
+  ModuleError, Payload, WithSources,
 } from '@xyo-network/payload-model'
 import { LRUCache } from 'lru-cache'
 
@@ -54,8 +54,7 @@ export class AsyncQueryBusClient<TParams extends AsyncQueryBusClientParams = Asy
 
   async send(address: Address, query: QueryBoundWitness, payloads?: Payload[] | undefined): Promise<ModuleQueryResult> {
     this.logger?.debug(`Begin issuing query to: ${address}`)
-    const $meta = { ...query?.$meta, destination: [address] }
-    const routedQuery = await PayloadBuilder.build({ ...query, $meta })
+    const routedQuery = { ...query, $destination: [address] }
     // console.log('queryArchivist - calling')
     const queryArchivist = assertEx(
       await this.queriesArchivist(),
@@ -69,11 +68,7 @@ export class AsyncQueryBusClient<TParams extends AsyncQueryBusClientParams = Asy
     // they sent us (which might be OK since it reflect the chain of custody)
     // Revisit this once we have proxy module support as they are another
     // intermediary to consider.
-    const routedQueryHash
-      // Trust the signed hash if it's there
-      = (routedQuery as WithMeta<QueryBoundWitness>)?.$hash
-      // Calculate the hash otherwise
-      ?? Object.keys(await PayloadBuilder.dataHash(routedQuery))
+    const routedQueryHash = await PayloadBuilder.dataHash(routedQuery)
     this.logger?.debug(`Issuing query: ${routedQueryHash} to: ${address}`)
     // If there was data associated with the query, add it to the insert
     const data = payloads ? [routedQuery, ...payloads] : [routedQuery]
@@ -110,11 +105,11 @@ export class AsyncQueryBusClient<TParams extends AsyncQueryBusClientParams = Asy
           this.logger?.error('Timeout waiting for query response')
           // Resolve with error to match what a local module would do if it were to error
           // TODO: BW Builder/Sign result as this module?
-          const error: ModuleError = {
+          const error: WithSources<ModuleError> = {
             message: 'Timeout waiting for query response',
             query: 'network.xyo.boundwitness',
             schema: 'network.xyo.error.module',
-            sources: [routedQueryHash],
+            $sources: [routedQueryHash],
           }
           reject(error)
           return
@@ -162,11 +157,11 @@ export class AsyncQueryBusClient<TParams extends AsyncQueryBusClientParams = Asy
               }
               const result = await responseBoundWitnessDiviner.divine([divinerQuery])
               if (result && result.length > 0) {
-                const response = result.find(isBoundWitnessWithMeta)
-                if (response && (response?.$meta as unknown as { sourceQuery: string })?.sourceQuery === sourceQuery) {
+                const response = result.find(isBoundWitness)
+                if (response && (response as unknown as { $sourceQuery: string })?.$sourceQuery === sourceQuery) {
                   this.logger?.debug(`Found response to query: ${sourceQuery}`)
                   // Get any payloads associated with the response
-                  const payloads: PayloadWithMeta[] = response.payload_hashes?.length > 0 ? await responseArchivist.get(response.payload_hashes) : []
+                  const payloads: Payload[] = response.payload_hashes?.length > 0 ? await responseArchivist.get(response.payload_hashes) : []
                   this.queryCache.set(sourceQuery, [response, payloads, []])
                 }
               }

package/src/AsyncQueryBus/AsyncQueryBusHost.ts

@@ -3,7 +3,7 @@ import { assertEx } from '@xylabs/assert'
 import type { Address } from '@xylabs/hex'
 import { clearTimeoutEx, setTimeoutEx } from '@xylabs/timer'
 import type { QueryBoundWitness } from '@xyo-network/boundwitness-model'
-import { isQueryBoundWitnessWithMeta } from '@xyo-network/boundwitness-model'
+import { isQueryBoundWitnessWithStorageMeta } from '@xyo-network/boundwitness-model'
 import { isBridgeInstance } from '@xyo-network/bridge-model'
 import type { BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'
 import { BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'
@@ -18,7 +18,9 @@ import {
   ResolveHelper,
 } from '@xyo-network/module-model'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
-import type { Schema, WithMeta } from '@xyo-network/payload-model'
+import {
+  type Schema, SequenceConstants, type WithStorageMeta,
+} from '@xyo-network/payload-model'
 
 import { AsyncQueryBusBase } from './AsyncQueryBusBase.ts'
 import type { AsyncQueryBusHostParams } from './model/index.ts'
@@ -132,7 +134,7 @@ export class AsyncQueryBusHost<TParams extends AsyncQueryBusHostParams = AsyncQu
   }
 
   // eslint-disable-next-line complexity
-  protected callLocalModule = async (localModule: ModuleInstance, query: WithMeta<QueryBoundWitness>) => {
+  protected callLocalModule = async (localModule: ModuleInstance, query: WithStorageMeta<QueryBoundWitness>) => {
     this._idle = false
     this._lastQueryTime = Date.now()
     const localModuleName = localModule.id
@@ -144,7 +146,7 @@ export class AsyncQueryBusHost<TParams extends AsyncQueryBusHostParams = AsyncQu
       await this.responsesArchivist(),
       () => `Unable to contact responsesArchivist [${this.config?.intersect?.queries?.archivist}]`,
     )
-    const queryDestination = (query.$meta as { destination?: string[] })?.destination
+    const queryDestination = (query as { $destination?: string[] })?.$destination
     if (queryDestination && queryDestination?.includes(localModule.address)) {
       // Find the query
       const queryIndex = query.payload_hashes.indexOf(query.query)
@@ -156,7 +158,7 @@ export class AsyncQueryBusHost<TParams extends AsyncQueryBusHostParams = AsyncQu
           const queryPayloads = await queryArchivist.get(query.payload_hashes)
           this.params.onQueryFulfillStarted?.({ payloads: queryPayloads, query })
           const queryPayloadsDict = await PayloadBuilder.toAllHashMap(queryPayloads)
-          const queryHash = (await PayloadBuilder.build(query)).$hash
+          const queryHash = await PayloadBuilder.dataHash(query)
           // Check that we have all the arguments for the command
           if (!containsAll(Object.keys(queryPayloadsDict), query.payload_hashes)) {
             this.logger?.error(`Error processing command ${queryHash} for module ${localModuleName}, missing payloads`)
@@ -179,11 +181,11 @@ export class AsyncQueryBusHost<TParams extends AsyncQueryBusHostParams = AsyncQu
             if (insertResult.length === 0) {
               this.logger?.error(`Error replying to query ${queryHash} addressed to module: ${localModuleName}`)
             }
-            if (query?.timestamp) {
+            if (query?._sequence) {
               // TODO: This needs to be thought through as we can't use a distributed timestamp
               // because of collisions. We need to ensure we are using the timestamp of the store
               // so there's no chance of multiple commands at the same time
-              await this.commitState(localModule.address, query.timestamp)
+              await this.commitState(localModule.address, query._sequence)
             }
             this.params.onQueryFulfillFinished?.({
               payloads: queryPayloads, query, result, status: 'success',
@@ -217,11 +219,13 @@ export class AsyncQueryBusHost<TParams extends AsyncQueryBusHostParams = AsyncQu
         limit,
         order: 'asc',
         schema: BoundWitnessDivinerQuerySchema,
-        timestamp: prevState + 1,
+        cursor: prevState,
       }
       const result = await queriesBoundWitnessDiviner.divine([divinerQuery])
-      const queries = result.filter(isQueryBoundWitnessWithMeta)
-      const nextState = queries.length > 0 ? Math.max(...queries.map(c => c.timestamp ?? prevState)) + 1 : Date.now()
+      const queries = result.filter(isQueryBoundWitnessWithStorageMeta)
+      // eslint-disable-next-line unicorn/no-array-reduce, unicorn/prefer-math-min-max
+      const highestQuerySequence = queries.reduce((acc, query) => acc = (query._sequence > acc ? query._sequence : acc), SequenceConstants.minLocalSequence)
+      const nextState = queries.length > 0 ? highestQuerySequence : SequenceConstants.minLocalSequence
       // TODO: This needs to be thought through as we can't use a distributed timestamp
       // because of collisions. We need to use the timestamp of the store so there's no
       // chance of multiple commands at the same time