@xyo-network/bridge-pub-sub 3.6.0-rc.1 → 3.6.0-rc.11

package/src/AsyncQueryBus/AsyncQueryBusBase.ts

@@ -1,5 +1,5 @@
 import { assertEx } from '@xylabs/assert'
-import type { Address } from '@xylabs/hex'
+import type { Address, Hex } from '@xylabs/hex'
 import type { TypeCheck } from '@xylabs/object'
 import { Base } from '@xylabs/object'
 import type { ArchivistInstance } from '@xyo-network/archivist-model'
@@ -12,6 +12,7 @@ import type {
   ModuleConfig, ModuleIdentifier, ModuleInstance,
 } from '@xyo-network/module-model'
 import { ResolveHelper } from '@xyo-network/module-model'
+import { SequenceConstants } from '@xyo-network/payload-model'
 import { Mutex } from 'async-mutex'
 import { LRUCache } from 'lru-cache'
 
@@ -22,7 +23,7 @@ const POLLING_FREQUENCY_MAX = 60_000 as const
 const POLLING_FREQUENCY_DEFAULT = 1000 as const
 
 export class AsyncQueryBusBase<TParams extends AsyncQueryBusParams = AsyncQueryBusParams> extends Base<TParams> {
-  protected _lastState?: LRUCache<Address, number>
+  protected _lastState?: LRUCache<Address, Hex>
   protected _targetConfigs: Record<Address, ModuleConfig> = {}
   protected _targetQueries: Record<Address, string[]> = {}
 
@@ -57,9 +58,9 @@ export class AsyncQueryBusBase<TParams extends AsyncQueryBusParams = AsyncQueryB
   /**
    * A cache of the last offset of the Diviner process per address
    */
-  protected get lastState(): LRUCache<Address, number> {
+  protected get lastState(): LRUCache<Address, Hex> {
     const requiredConfig = { max: 1000, ttl: 0 }
-    this._lastState = this._lastState ?? new LRUCache<Address, number>(requiredConfig)
+    this._lastState = this._lastState ?? new LRUCache<Address, Hex>(requiredConfig)
     return this._lastState
   }
 
@@ -119,7 +120,7 @@ export class AsyncQueryBusBase<TParams extends AsyncQueryBusParams = AsyncQueryB
    * @param address The module address to commit the state for
    * @param nextState The state to commit
    */
-  protected async commitState(address: Address, nextState: number) {
+  protected async commitState(address: Address, nextState: Hex) {
     await Promise.resolve()
     // TODO: Offload to Archivist/Diviner instead of in-memory
     const lastState = this.lastState.get(address)
@@ -131,13 +132,11 @@ export class AsyncQueryBusBase<TParams extends AsyncQueryBusParams = AsyncQueryB
    * Retrieves the last state of the process. Used to recover state after
    * preemptions, reboots, etc.
    */
-  protected async retrieveState(address: Address): Promise<number> {
+  protected async retrieveState(address: Address): Promise<Hex> {
     await Promise.resolve()
     const state = this.lastState.get(address)
     if (state === undefined) {
-      // If this is a boot we can go back a bit in time
-      // and begin processing recent commands
-      const newState = Date.now() - 1000
+      const newState = SequenceConstants.minLocalSequence
       this.lastState.set(address, newState)
       return newState
     } else {

package/src/AsyncQueryBus/AsyncQueryBusClient.ts

@@ -9,7 +9,9 @@ import type { BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-bound
 import { BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'
 import type { CacheConfig, ModuleQueryResult } from '@xyo-network/module-model'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
-import type { ModuleError, Payload } from '@xyo-network/payload-model'
+import type {
+  ModuleError, Payload, WithSources,
+} from '@xyo-network/payload-model'
 import { LRUCache } from 'lru-cache'
 
 import { AsyncQueryBusBase } from './AsyncQueryBusBase.ts'
@@ -103,11 +105,11 @@ export class AsyncQueryBusClient<TParams extends AsyncQueryBusClientParams = Asy
           this.logger?.error('Timeout waiting for query response')
           // Resolve with error to match what a local module would do if it were to error
           // TODO: BW Builder/Sign result as this module?
-          const error: ModuleError = {
+          const error: WithSources<ModuleError> = {
             message: 'Timeout waiting for query response',
             query: 'network.xyo.boundwitness',
             schema: 'network.xyo.error.module',
-            sources: [routedQueryHash],
+            $sources: [routedQueryHash],
           }
           reject(error)
           return

package/src/AsyncQueryBus/AsyncQueryBusHost.ts

@@ -3,7 +3,7 @@ import { assertEx } from '@xylabs/assert'
 import type { Address } from '@xylabs/hex'
 import { clearTimeoutEx, setTimeoutEx } from '@xylabs/timer'
 import type { QueryBoundWitness } from '@xyo-network/boundwitness-model'
-import { isQueryBoundWitness } from '@xyo-network/boundwitness-model'
+import { isQueryBoundWitnessWithStorageMeta } from '@xyo-network/boundwitness-model'
 import { isBridgeInstance } from '@xyo-network/bridge-model'
 import type { BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'
 import { BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'
@@ -18,7 +18,9 @@ import {
   ResolveHelper,
 } from '@xyo-network/module-model'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
-import type { Schema } from '@xyo-network/payload-model'
+import {
+  type Schema, SequenceConstants, type WithStorageMeta,
+} from '@xyo-network/payload-model'
 
 import { AsyncQueryBusBase } from './AsyncQueryBusBase.ts'
 import type { AsyncQueryBusHostParams } from './model/index.ts'
@@ -132,7 +134,7 @@ export class AsyncQueryBusHost<TParams extends AsyncQueryBusHostParams = AsyncQu
   }
 
   // eslint-disable-next-line complexity
-  protected callLocalModule = async (localModule: ModuleInstance, query: QueryBoundWitness) => {
+  protected callLocalModule = async (localModule: ModuleInstance, query: WithStorageMeta<QueryBoundWitness>) => {
     this._idle = false
     this._lastQueryTime = Date.now()
     const localModuleName = localModule.id
@@ -179,11 +181,11 @@ export class AsyncQueryBusHost<TParams extends AsyncQueryBusHostParams = AsyncQu
             if (insertResult.length === 0) {
               this.logger?.error(`Error replying to query ${queryHash} addressed to module: ${localModuleName}`)
             }
-            if (query?.timestamp) {
+            if (query?._sequence) {
               // TODO: This needs to be thought through as we can't use a distributed timestamp
               // because of collisions. We need to ensure we are using the timestamp of the store
               // so there's no chance of multiple commands at the same time
-              await this.commitState(localModule.address, query.timestamp)
+              await this.commitState(localModule.address, query._sequence)
             }
             this.params.onQueryFulfillFinished?.({
               payloads: queryPayloads, query, result, status: 'success',
@@ -217,11 +219,13 @@ export class AsyncQueryBusHost<TParams extends AsyncQueryBusHostParams = AsyncQu
         limit,
         order: 'asc',
         schema: BoundWitnessDivinerQuerySchema,
-        timestamp: prevState + 1,
+        cursor: prevState,
       }
       const result = await queriesBoundWitnessDiviner.divine([divinerQuery])
-      const queries = result.filter(isQueryBoundWitness)
-      const nextState = queries.length > 0 ? Math.max(...queries.map(c => c.timestamp ?? prevState)) + 1 : Date.now()
+      const queries = result.filter(isQueryBoundWitnessWithStorageMeta)
+      // eslint-disable-next-line unicorn/no-array-reduce, unicorn/prefer-math-min-max
+      const highestQuerySequence = queries.reduce((acc, query) => acc = (query._sequence > acc ? query._sequence : acc), SequenceConstants.minLocalSequence)
+      const nextState = queries.length > 0 ? highestQuerySequence : SequenceConstants.minLocalSequence
       // TODO: This needs to be thought through as we can't use a distributed timestamp
       // because of collisions. We need to use the timestamp of the store so there's no
       // chance of multiple commands at the same time