@xyo-network/sentinel 2.43.37

package/src/Sentinel.ts

@@ -0,0 +1,141 @@
+import { assertEx } from '@xylabs/assert'
+import { Account } from '@xyo-network/account'
+import { AbstractArchivist, ArchivingModule, ArchivingModuleConfig } from '@xyo-network/archivist'
+import { ArchivistWrapper } from '@xyo-network/archivist-wrapper'
+import { XyoBoundWitness } from '@xyo-network/boundwitness-model'
+import {
+  AbstractModuleConfig,
+  ModuleParams,
+  ModuleQueryResult,
+  QueryBoundWitnessWrapper,
+  XyoErrorBuilder,
+  XyoQueryBoundWitness,
+} from '@xyo-network/module'
+import { XyoPayload } from '@xyo-network/payload-model'
+import { AbstractWitness, WitnessWrapper } from '@xyo-network/witness'
+import compact from 'lodash/compact'
+import uniq from 'lodash/uniq'
+
+import { SentinelQuery, SentinelReportQuerySchema } from './Queries'
+import { SentinelModule } from './SentinelModel'
+
+export type SentinelConfigSchema = 'network.xyo.sentinel.config'
+export const SentinelConfigSchema: SentinelConfigSchema = 'network.xyo.sentinel.config'
+
+export type SentinelConfig = ArchivingModuleConfig<{
+  onReportEnd?: (boundWitness?: XyoBoundWitness, errors?: Error[]) => void
+  onReportStart?: () => void
+  onWitnessReportEnd?: (witness: WitnessWrapper, error?: Error) => void
+  onWitnessReportStart?: (witness: WitnessWrapper) => void
+  schema: SentinelConfigSchema
+  witnesses?: string[]
+}>
+
+export class Sentinel extends ArchivingModule<SentinelConfig> implements SentinelModule {
+  static override configSchema: SentinelConfigSchema
+
+  public history: XyoBoundWitness[] = []
+  private _archivists: ArchivistWrapper[] | undefined
+  private _witnesses: WitnessWrapper[] | undefined
+
+  static override async create(params?: Partial<ModuleParams<SentinelConfig>>): Promise<Sentinel> {
+    return (await super.create(params)) as Sentinel
+  }
+
+  public addWitness(address: string[]) {
+    this.config.witnesses = uniq([...address, ...(this.config.witnesses ?? [])])
+    this._witnesses = undefined
+  }
+
+  public async getArchivists() {
+    const addresses = this.config?.archivists ? (Array.isArray(this.config.archivists) ? this.config?.archivists : [this.config.archivists]) : []
+    this._archivists =
+      this._archivists ||
+      ((await this.resolver?.resolve({ address: addresses })) as AbstractArchivist[]).map((witness) => new ArchivistWrapper(witness))
+
+    return this._archivists
+  }
+
+  public async getWitnesses() {
+    const addresses = this.config?.witnesses ? (Array.isArray(this.config.witnesses) ? this.config?.witnesses : [this.config.witnesses]) : []
+    this._witnesses =
+      this._witnesses || ((await this.resolver?.resolve({ address: addresses })) as AbstractWitness[]).map((witness) => new WitnessWrapper(witness))
+
+    return this._witnesses
+  }
+
+  public override queries(): string[] {
+    return [SentinelReportQuerySchema, ...super.queries()]
+  }
+
+  override async query<T extends XyoQueryBoundWitness = XyoQueryBoundWitness, TConfig extends AbstractModuleConfig = AbstractModuleConfig>(
+    query: T,
+    payloads?: XyoPayload[],
+    queryConfig?: TConfig,
+  ): Promise<ModuleQueryResult> {
+    const wrapper = QueryBoundWitnessWrapper.parseQuery<SentinelQuery>(query, payloads)
+    const typedQuery = wrapper.query
+    assertEx(this.queryable(query, payloads, queryConfig))
+    const queryAccount = new Account()
+    const resultPayloads: XyoPayload[] = []
+    try {
+      switch (typedQuery.schemaName) {
+        case SentinelReportQuerySchema: {
+          const reportResult = await this.report(payloads)
+          resultPayloads.push(...(reportResult[0], reportResult[1]))
+          break
+        }
+        default:
+          return super.query(query, payloads)
+      }
+    } catch (ex) {
+      const error = ex as Error
+      resultPayloads.push(new XyoErrorBuilder([wrapper.hash], error.message).build())
+    }
+    return await this.bindResult(resultPayloads, queryAccount)
+  }
+
+  public removeArchivist(address: string[]) {
+    this.config.archivists = (this.config.archivists ?? []).filter((archivist) => !address.includes(archivist))
+    this._archivists = undefined
+  }
+
+  public removeWitness(address: string[]) {
+    this.config.witnesses = (this.config.witnesses ?? []).filter((witness) => !address.includes(witness))
+    this._witnesses = undefined
+  }
+
+  public async report(payloads: XyoPayload[] = []): Promise<[XyoBoundWitness, XyoPayload[]]> {
+    const errors: Error[] = []
+    this.config?.onReportStart?.()
+    const allWitnesses = [...(await this.getWitnesses())]
+    const allPayloads: XyoPayload[] = []
+
+    try {
+      const generatedPayloads = compact(await this.generatePayloads(allWitnesses))
+      const combinedPayloads = [...generatedPayloads, ...payloads]
+      allPayloads.push(...combinedPayloads)
+    } catch (e) {
+      errors.push(e as Error)
+    }
+
+    const [newBoundWitness] = await this.bindResult(allPayloads)
+    this.history.push(assertEx(newBoundWitness))
+    this.config?.onReportEnd?.(newBoundWitness, errors.length > 0 ? errors : undefined)
+    return [newBoundWitness, allPayloads]
+  }
+
+  public async tryReport(payloads: XyoPayload[] = []): Promise<[XyoBoundWitness | null, XyoPayload[]]> {
+    try {
+      return await this.report(payloads)
+    } catch (ex) {
+      const error = ex as Error
+      this.logger?.warn(`report failed [${error.message}]`)
+      return [null, []]
+    }
+  }
+
+  private async generatePayloads(witnesses: WitnessWrapper[]): Promise<XyoPayload[]> {
+    return (await Promise.all(witnesses?.map(async (witness) => await witness.observe()))).flat()
+  }
+}

package/src/SentinelIntervalAutomationWrapper.ts

@@ -0,0 +1,45 @@
+import { PayloadWrapper } from '@xyo-network/payload-wrapper'
+
+import { SentinelIntervalAutomationPayload } from './Automation'
+
+export class SentinelIntervalAutomationWrapper<
+  T extends SentinelIntervalAutomationPayload = SentinelIntervalAutomationPayload,
+> extends PayloadWrapper<T> {
+  protected get frequencyMillis() {
+    if (this.payload.frequency === undefined) return Infinity
+    switch (this.payload.frequencyUnits ?? 'hour') {
+      case 'minute':
+        return this.payload.frequency * 60 * 1000
+      case 'hour':
+        return this.payload.frequency * 60 * 60 * 1000
+      case 'day':
+        return this.payload.frequency * 24 * 60 * 60 * 1000
+    }
+  }
+
+  protected get remaining() {
+    //if remaining is not defined, we assume Infinity
+    return this.payload.remaining ?? Infinity
+  }
+
+  public next() {
+    this.payload.start = this.payload.start + this.frequencyMillis
+    this.consumeRemaining()
+    this.checkEnd()
+    return this
+  }
+
+  protected checkEnd() {
+    if (this.payload.start > (this.payload.end ?? Infinity)) {
+      this.payload.start = Infinity
+    }
+  }
+
+  protected consumeRemaining(count = 1) {
+    this.payload.remaining = this.remaining - count
+
+    if (this.payload.remaining <= 0) {
+      this.payload.start = Infinity
+    }
+  }
+}

package/src/SentinelModel.ts

@@ -0,0 +1,11 @@
+import { XyoBoundWitness } from '@xyo-network/boundwitness-model'
+import { Module } from '@xyo-network/module-model'
+import { XyoPayload } from '@xyo-network/payload-model'
+import { Promisable } from '@xyo-network/promise'
+
+export interface SentinelModel {
+  report: (payloads?: XyoPayload[]) => Promisable<[XyoBoundWitness, XyoPayload[]]>
+  tryReport: (payloads?: XyoPayload[]) => Promisable<[XyoBoundWitness | null, XyoPayload[]]>
+}
+
+export interface SentinelModule extends Module, SentinelModel {}

package/src/SentinelRunner.ts

@@ -0,0 +1,104 @@
+import { assertEx } from '@xylabs/assert'
+import { XyoBoundWitness } from '@xyo-network/boundwitness-model'
+import { XyoPayload } from '@xyo-network/payload-model'
+import { PayloadWrapper } from '@xyo-network/payload-wrapper'
+
+import { SentinelAutomationPayload, SentinelIntervalAutomationPayload } from './Automation'
+import { SentinelIntervalAutomationWrapper } from './SentinelIntervalAutomationWrapper'
+import { SentinelModel } from './SentinelModel'
+
+export type OnSentinelRunnerTriggerResult = (result: [XyoBoundWitness | null, XyoPayload[]]) => void
+
+export class SentinelRunner {
+  protected _automations: Record<string, SentinelAutomationPayload> = {}
+  protected onTriggerResult: OnSentinelRunnerTriggerResult | undefined
+  protected sentinel: SentinelModel
+  protected timeoutId?: NodeJS.Timer
+
+  constructor(sentinel: SentinelModel, automations?: SentinelAutomationPayload[], onTriggerResult?: OnSentinelRunnerTriggerResult) {
+    this.sentinel = sentinel
+    this.onTriggerResult = onTriggerResult
+    automations?.forEach((automation) => this.add(automation))
+  }
+
+  public get automations() {
+    return this._automations
+  }
+
+  private get next() {
+    return Object.values(this._automations).reduce<SentinelIntervalAutomationPayload | undefined>((previous, current) => {
+      if (current.type === 'interval') {
+        return current.start < (previous?.start ?? Infinity) ? current : previous
+      }
+    }, undefined)
+  }
+
+  public async add(automation: SentinelAutomationPayload, restart = true) {
+    const hash = new PayloadWrapper(automation).hash
+    this._automations[hash] = automation
+    if (restart) await this.restart()
+    return hash
+  }
+
+  public find(hash: string) {
+    Object.entries(this._automations).find(([key]) => key === hash)
+  }
+
+  public async remove(hash: string, restart = true) {
+    delete this._automations[hash]
+    if (restart) await this.restart()
+  }
+
+  public removeAll() {
+    this.stop()
+    this._automations = {}
+  }
+
+  public async restart() {
+    this.stop()
+    await this.start()
+  }
+
+  public async start() {
+    assertEx(this.timeoutId === undefined, 'Already started')
+    const automation = this.next
+    if (automation) {
+      const delay = automation.start - Date.now()
+      if (delay < 0) {
+        //automation is due, just do it
+        await this.trigger(automation)
+      } else {
+        this.timeoutId = setTimeout(
+          async () => {
+            this.timeoutId = undefined
+            await this.start()
+          },
+          delay > 0 ? delay : 0,
+        )
+      }
+    }
+  }
+
+  public stop() {
+    if (this.timeoutId) {
+      clearTimeout(this.timeoutId)
+      this.timeoutId = undefined
+    }
+  }
+
+  public async update(hash: string, automation: SentinelAutomationPayload, restart = true) {
+    await this.remove(hash, false)
+    await this.add(automation, false)
+    if (restart) await this.restart()
+  }
+
+  private async trigger(automation: SentinelIntervalAutomationPayload) {
+    const wrapper = new SentinelIntervalAutomationWrapper(automation)
+    await this.remove(wrapper.hash, false)
+    wrapper.next()
+    await this.add(wrapper.payload, false)
+    const triggerResult = await this.sentinel.tryReport()
+    this.onTriggerResult?.(triggerResult)
+    await this.start()
+  }
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+export * from './Automation'
+export * from './Sentinel'
+export * from './SentinelIntervalAutomationWrapper'
+export * from './SentinelModel'
+export * from './SentinelRunner'

package/src/spec/Sentinel.spec.ts

@@ -0,0 +1,215 @@
+import { AbstractArchivist, Archivist, MemoryArchivist } from '@xyo-network/archivist'
+import { XyoBoundWitness, XyoBoundWitnessSchema } from '@xyo-network/boundwitness-model'
+import { BoundWitnessValidator } from '@xyo-network/boundwitness-validator'
+import { BoundWitnessWrapper } from '@xyo-network/boundwitness-wrapper'
+import { Hasher } from '@xyo-network/core'
+import { IdWitness, IdWitnessConfigSchema } from '@xyo-network/id-plugin'
+import { ModuleParams, SimpleModuleResolver } from '@xyo-network/module'
+import { XyoNodeSystemInfoWitness, XyoNodeSystemInfoWitnessConfigSchema } from '@xyo-network/node-system-info-plugin'
+import { XyoPayload, XyoPayloadSchema } from '@xyo-network/payload-model'
+import { PayloadWrapper } from '@xyo-network/payload-wrapper'
+import { AbstractWitness } from '@xyo-network/witness'
+import { XyoAdhocWitness, XyoAdhocWitnessConfigSchema } from '@xyo-network/witnesses'
+
+import { Sentinel, SentinelConfig, SentinelConfigSchema } from '../Sentinel'
+
+describe('Sentinel', () => {
+  test('all [simple sentinel send]', async () => {
+    const archivist = await MemoryArchivist.create()
+
+    const witnesses: AbstractWitness[] = [
+      await IdWitness.create({ config: { salt: 'test', schema: IdWitnessConfigSchema } }),
+      await XyoNodeSystemInfoWitness.create({
+        config: {
+          nodeValues: {
+            osInfo: '*',
+          },
+          schema: XyoNodeSystemInfoWitnessConfigSchema,
+        },
+      }),
+    ]
+
+    const config: SentinelConfig = {
+      archivists: [archivist.address],
+      schema: SentinelConfigSchema,
+      witnesses: witnesses.map((witness) => witness.address),
+    }
+
+    const resolver = new SimpleModuleResolver()
+    resolver.add(archivist)
+    witnesses.forEach((witness) => resolver.add(witness))
+
+    const sentinel = await Sentinel.create({ config, resolver })
+    expect(await sentinel.getArchivists()).toBeArrayOfSize(1)
+    expect(await sentinel.getWitnesses()).toBeArrayOfSize(2)
+    const adhocWitness = await XyoAdhocWitness.create({
+      config: {
+        payload: {
+          schema: 'network.xyo.test.array',
+          testArray: [1, 2, 3],
+          testBoolean: true,
+          testNull: null,
+          testNullObject: { t: null, x: undefined },
+          testNumber: 5,
+          testObject: { t: 1 },
+          testSomeNullObject: { s: 1, t: null, x: undefined },
+          testString: 'hi',
+          testUndefined: undefined,
+        },
+        schema: XyoAdhocWitnessConfigSchema,
+      },
+    })
+
+    const adhocObserved = await adhocWitness.observe([adhocWitness.config.payload])
+
+    const report1Result = await sentinel.report(adhocObserved)
+    const report1 = BoundWitnessWrapper.parse(report1Result[0])
+    expect(report1.schemaName).toBe(XyoBoundWitnessSchema)
+    expect(report1.payloadHashes).toBeArrayOfSize(3)
+    const report2 = BoundWitnessWrapper.parse((await sentinel.report())[0])
+    expect(report2.schemaName).toBeDefined()
+    expect(report2.payloadHashes).toBeArrayOfSize(2)
+    expect(report2.hash !== report1.hash).toBe(true)
+    expect(report2.prev(sentinel.address)).toBeDefined()
+    expect(report2.prev(sentinel.address)).toBe(report1.hash)
+    expect(report1.valid).toBe(true)
+    expect(report2.valid).toBe(true)
+  })
+  describe('report', () => {
+    describe('reports witnesses when supplied in', () => {
+      let archivistA: AbstractArchivist
+      let archivistB: AbstractArchivist
+      let witnessA: AbstractWitness
+      let witnessB: AbstractWitness
+      const assertSentinelReport = (sentinelReport: [XyoBoundWitness, XyoPayload[]]) => {
+        expect(sentinelReport).toBeArrayOfSize(2)
+        const [bw, payloads] = sentinelReport
+        expect(new BoundWitnessValidator(bw).validate()).toBeArrayOfSize(0)
+        expect(payloads).toBeArrayOfSize(2)
+      }
+      const assertArchivistStateMatchesSentinelReport = async (sentinelReport: [XyoBoundWitness, XyoPayload[]], archivists: Archivist[]) => {
+        const [, payloads] = sentinelReport
+        for (const archivist of archivists) {
+          const archivistPayloads = await archivist.all?.()
+          expect(archivistPayloads).toBeArrayOfSize(payloads.length + 1)
+          const sentinelPayloads = payloads.map((payload) => {
+            const wrapped = new PayloadWrapper(payload)
+            return { ...payload, _hash: wrapped.hash, _timestamp: expect.toBeNumber() }
+          })
+          expect(archivistPayloads).toContainValues(sentinelPayloads)
+        }
+      }
+      beforeEach(async () => {
+        const paramsA = {
+          config: {
+            payload: { nonce: Math.floor(Math.random() * 9999999), schema: 'network.xyo.test' },
+            schema: XyoAdhocWitnessConfigSchema,
+            targetSchema: XyoPayloadSchema,
+          },
+        }
+        const paramsB = {
+          config: {
+            payload: { nonce: Math.floor(Math.random() * 9999999), schema: 'network.xyo.test' },
+            schema: XyoAdhocWitnessConfigSchema,
+            targetSchema: XyoPayloadSchema,
+          },
+        }
+        witnessA = await XyoAdhocWitness.create(paramsA)
+        witnessB = await XyoAdhocWitness.create(paramsB)
+        archivistA = await MemoryArchivist.create()
+        archivistB = await MemoryArchivist.create()
+      })
+      it('config', async () => {
+        const resolver = new SimpleModuleResolver()
+        resolver.add([witnessA, witnessB, archivistA, archivistB])
+        const params: ModuleParams<SentinelConfig> = {
+          config: {
+            archivists: [archivistA.address, archivistB.address],
+            schema: 'network.xyo.sentinel.config',
+            witnesses: [witnessA.address, witnessB.address],
+          },
+          resolver,
+        }
+        const sentinel = await Sentinel.create(params)
+        const result = await sentinel.report()
+        assertSentinelReport(result)
+        await assertArchivistStateMatchesSentinelReport(result, [archivistA, archivistB])
+      })
+      it('config & inline', async () => {
+        const resolver = new SimpleModuleResolver()
+        resolver.add([witnessA, archivistA, archivistB])
+        const params: ModuleParams<SentinelConfig> = {
+          config: {
+            archivists: [archivistA.address, archivistB.address],
+            schema: 'network.xyo.sentinel.config',
+            witnesses: [witnessA.address],
+          },
+          resolver,
+        }
+        const sentinel = await Sentinel.create(params)
+        const observed = await witnessB.observe()
+        expect(observed).toBeArrayOfSize(1)
+        const result = await sentinel.report(observed)
+        assertSentinelReport(result)
+        await assertArchivistStateMatchesSentinelReport(result, [archivistA, archivistB])
+      })
+      it('inline', async () => {
+        const resolver = new SimpleModuleResolver()
+        resolver.add([archivistA, archivistB])
+        const params: ModuleParams<SentinelConfig> = {
+          config: {
+            archivists: [archivistA.address, archivistB.address],
+            schema: 'network.xyo.sentinel.config',
+            witnesses: [],
+          },
+          resolver,
+        }
+        const sentinel = await Sentinel.create(params)
+        const observedA = await witnessA.observe()
+        expect(observedA).toBeArrayOfSize(1)
+        const observedB = await witnessB.observe()
+        expect(observedB).toBeArrayOfSize(1)
+        const result = await sentinel.report([...observedA, ...observedB])
+        assertSentinelReport(result)
+        expect((await archivistA.get([Hasher.hash(observedA)])).length).toBe(1)
+        expect((await archivistA.get([Hasher.hash(observedB)])).length).toBe(1)
+        expect((await archivistB.get([Hasher.hash(observedA)])).length).toBe(1)
+        expect((await archivistB.get([Hasher.hash(observedB)])).length).toBe(1)
+        await assertArchivistStateMatchesSentinelReport(result, [archivistA, archivistB])
+      })
+      it('reports errors', async () => {
+        const paramsA = {
+          config: {
+            payload: { nonce: Math.floor(Math.random() * 9999999), schema: 'network.xyo.test' },
+            schema: XyoAdhocWitnessConfigSchema,
+          },
+        }
+        class FailingWitness extends XyoAdhocWitness {
+          override async observe(): Promise<XyoPayload[]> {
+            await Promise.reject(Error('observation failed'))
+            return [{ schema: 'fake.result' }]
+          }
+        }
+        const witnessA = await FailingWitness.create(paramsA)
+
+        const resolver = new SimpleModuleResolver()
+        resolver.add([witnessA, witnessB, archivistA, archivistB])
+        const params: ModuleParams<SentinelConfig> = {
+          config: {
+            archivists: [archivistA.address, archivistB.address],
+            onReportEnd(_, errors) {
+              expect(errors?.length).toBe(1)
+              expect(errors?.[0]?.message).toBe('observation failed')
+            },
+            schema: 'network.xyo.sentinel.config',
+            witnesses: [witnessA.address, witnessB.address],
+          },
+          resolver,
+        }
+        const sentinel = await Sentinel.create(params)
+        await sentinel.report()
+        return
+      })
+    })
+  })
+})

package/src/spec/SentinelRunner.spec.ts

@@ -0,0 +1,47 @@
+import { XyoBoundWitnessSchema } from '@xyo-network/boundwitness-model'
+import { SimpleModuleResolver } from '@xyo-network/module'
+import { IdSchema, IdWitness, IdWitnessConfigSchema } from '@xyo-network/plugins'
+import { AbstractWitness } from '@xyo-network/witness'
+
+import { SentinelIntervalAutomationPayload, AutomationSchema } from '../Automation'
+import { Sentinel, SentinelConfig, SentinelConfigSchema } from '../Sentinel'
+import { OnSentinelRunnerTriggerResult, SentinelRunner } from '../SentinelRunner'
+
+describe('SentinelRunner', () => {
+  let sentinel: Sentinel
+  let config: SentinelConfig
+
+  beforeEach(async () => {
+    const witnesses: AbstractWitness[] = [await IdWitness.create({ config: { salt: 'test', schema: IdWitnessConfigSchema } })]
+    const resolver = new SimpleModuleResolver()
+    witnesses.forEach((witness) => resolver.add(witness))
+
+    config = {
+      schema: SentinelConfigSchema,
+      witnesses: witnesses.map((witness) => witness.address),
+    }
+
+    sentinel = await Sentinel.create({ config, resolver })
+  })
+
+  it('should output interval results', async () => {
+    const intervalAutomation: SentinelIntervalAutomationPayload = {
+      frequency: 1,
+      frequencyUnits: 'minute',
+      remaining: 1,
+      schema: AutomationSchema,
+      start: Date.now() - 1,
+      type: 'interval',
+      witnesses: config.witnesses,
+    }
+    const onTriggerResult: OnSentinelRunnerTriggerResult = (results) => {
+      expect(results.length).toBe(2)
+      expect(results[0]?.schema).toBe(XyoBoundWitnessSchema)
+      expect(results[1].length).toBe(1)
+      expect(results[1]?.[0].schema).toBe(IdSchema)
+    }
+
+    const runner = new SentinelRunner(sentinel, [intervalAutomation], onTriggerResult)
+    await runner.start()
+  })
+})

package/typedoc.json

@@ -0,0 +1,10 @@
+{
+  "intentionallyNotExported": [
+    "XyoBoundWitness",
+    "XyoAccount",
+    "XyoArchivistApi",
+    "XyoWitness",
+    "XyoPayloadFull",
+    "XyoWitnessConfig"
+  ]
+}