@xyo-network/diviner-temporal-indexing-memory 5.1.22 → 5.1.24

package/src/IndexQueryResponseToDivinerQueryResponseDiviner/spec/Diviner.spec.ts

@@ -1,103 +0,0 @@
-import '@xylabs/vitest-extended'
-
-import type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'
-import { PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'
-import type { Payload } from '@xyo-network/payload-model'
-import {
-  beforeAll,
-  describe, expect, it,
-} from 'vitest'
-
-import { TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner } from '../Diviner.ts'
-
-type QueryType = Payload<PayloadDivinerQueryPayload & Payload<{ status?: number; success?: boolean; url: string }>>
-
-describe('TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner', () => {
-  const url = 'https://xyo.network'
-  const queries: QueryType[] = [
-    {
-      schema: PayloadDivinerQuerySchema,
-      url,
-    },
-    {
-      schema: PayloadDivinerQuerySchema,
-      url,
-    },
-  ]
-  const indexes = [
-    [
-      {
-        schema: 'TODO',
-        sources: [],
-        status: 200,
-        success: true,
-        timestamp: 1_234_567_890,
-        url,
-      },
-    ],
-    [
-      {
-        schema: 'TODO',
-        sources: [],
-        status: 200,
-        success: true,
-        timestamp: 1_234_567_891,
-        url,
-      },
-      {
-        schema: 'TODO',
-        sources: [],
-        status: 500,
-        success: false,
-        timestamp: 1_234_567_892,
-        url,
-      },
-    ],
-  ]
-  let diviner: TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner
-  beforeAll(async () => {
-    diviner = await TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner.create({ account: 'random' })
-  })
-  const cases: [QueryType, Payload[]][] = queries.map((query, i) => [query, indexes[i]])
-  describe('divine', () => {
-    describe('with single url in index result', () => {
-      it.each(cases)('transforms single url index results', async (imageThumbnailDivinerQuery, imageThumbnailResultIndex) => {
-        const results = await diviner.divine([imageThumbnailDivinerQuery, ...imageThumbnailResultIndex])
-        expect(results).toBeArrayOfSize(imageThumbnailResultIndex.length)
-        expect(results).toBeArrayOfSize(imageThumbnailResultIndex.length)
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any, unicorn/no-array-for-each
-        results.forEach((result: any, i) => {
-          // eslint-disable-next-line @typescript-eslint/no-explicit-any
-          const index = imageThumbnailResultIndex[i] as any
-          expect(result.url).toBe(imageThumbnailDivinerQuery.url)
-          expect(result.success).toBe(index.success)
-          expect(result.timestamp).toBe(index.timestamp)
-          expect(result.status).toBe(index.status)
-          expect(result.schema).toBe('TODO')
-        })
-      })
-    })
-    describe('with multiple urls in index result', () => {
-      it('transforms multiple url index results', async () => {
-        const indexesLength = indexes.flat().length
-        const results = await diviner.divine([...queries, ...indexes.flat()])
-        expect(results).toBeArrayOfSize(indexesLength)
-        expect(results).toBeArrayOfSize(indexesLength)
-        let resultsIterator = 0
-        for (const [i, { url }] of queries.entries()) {
-          const indexSet = indexes[i]
-          for (const index of indexSet) {
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            const result = results[resultsIterator] as any
-            expect(result.url).toBe(url)
-            expect(result.success).toBe(index.success)
-            expect(result.timestamp).toBe(index.timestamp)
-            expect(result.status).toBe(index.status)
-            expect(result.schema).toBe('TODO')
-            resultsIterator = ++resultsIterator
-          }
-        }
-      })
-    })
-  })
-})

package/src/StateToIndexCandidateDiviner/spec/Diviner.spec.ts

@@ -1,167 +0,0 @@
-import '@xylabs/vitest-extended'
-
-import { filterAs } from '@xylabs/array'
-import { assertEx } from '@xylabs/assert'
-import { delay } from '@xylabs/delay'
-import type { MemoryArchivist } from '@xyo-network/archivist-memory'
-import { asArchivistInstance } from '@xyo-network/archivist-model'
-import { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'
-import { isBoundWitness } from '@xyo-network/boundwitness-model'
-import type { IndexingDivinerState } from '@xyo-network/diviner-indexing-model'
-import { asDivinerInstance } from '@xyo-network/diviner-model'
-import type { PackageManifestPayload } from '@xyo-network/manifest'
-import { ManifestWrapper } from '@xyo-network/manifest'
-import { ModuleFactoryLocator } from '@xyo-network/module-factory-locator'
-import type { ModuleState } from '@xyo-network/module-model'
-import { isModuleState, ModuleStateSchema } from '@xyo-network/module-model'
-import { PayloadBuilder } from '@xyo-network/payload-builder'
-import type { Payload, WithStorageMeta } from '@xyo-network/payload-model'
-import { asOptionalStorageMeta } from '@xyo-network/payload-model'
-import { HDWallet } from '@xyo-network/wallet'
-import type { TimeStamp } from '@xyo-network/witness-timestamp'
-import { isTimestamp, TimestampSchema } from '@xyo-network/witness-timestamp'
-import {
-  beforeAll, describe, expect, it,
-} from 'vitest'
-
-import { TemporalIndexingDivinerStateToIndexCandidateDiviner } from '../Diviner.ts'
-import TemporalStateToIndexCandidateDivinerManifest from './TemporalStateToIndexCandidateDiviner.json' with { type: 'json' }
-
-/**
- * @group slow
- */
-describe('TemporalStateToIndexCandidateDiviner', () => {
-  const sourceUrl = 'https://placekitten.com/200/300'
-  const thumbnailHttpSuccess = {
-    http: { status: 200 },
-    schema: 'network.xyo.image.thumbnail',
-    sourceHash: '7f39363514d9d9b958a5a993edeba35cb44f912c7072ed9ddd628728ac0fd681',
-    sourceUrl,
-    url: 'data:image/png;base64,===',
-  }
-
-  const thumbnailHttpFail = {
-    http: {
-
-      ipAddress: '104.17.96.13',
-      status: 429,
-    },
-    schema: 'network.xyo.image.thumbnail',
-    sourceUrl,
-  }
-
-  const thumbnailCodeFail = {
-    http: { code: 'FAILED' },
-    schema: 'network.xyo.image.thumbnail',
-    sourceUrl,
-  }
-
-  const thumbnailWitnessFail = {
-
-    http: { ipAddress: '104.17.96.13' },
-    schema: 'network.xyo.image.thumbnail',
-    sourceUrl,
-  }
-
-  let testCases: WithStorageMeta<Payload>[][] = []
-  let archivist: MemoryArchivist
-  let sut: TemporalIndexingDivinerStateToIndexCandidateDiviner
-
-  beforeAll(async () => {
-    const wallet = await HDWallet.random()
-    const locator = new ModuleFactoryLocator()
-    locator.register(TemporalIndexingDivinerStateToIndexCandidateDiviner.factory())
-    const manifest = TemporalStateToIndexCandidateDivinerManifest as PackageManifestPayload
-    const manifestWrapper = new ManifestWrapper(manifest, wallet, locator)
-    const node = await manifestWrapper.loadNodeFromIndex(0)
-    await node.start()
-
-    const privateModules = manifest.nodes[0].modules?.private ?? []
-    const publicModules = manifest.nodes[0].modules?.public ?? []
-    const mods = await node.resolve('*')
-    expect(mods.length).toBe(privateModules.length + publicModules.length + 1)
-
-    // Insert previously witnessed payloads into thumbnail archivist
-    const httpSuccessTimestamp: TimeStamp = { schema: TimestampSchema, timestamp: 1 }
-    const [httpSuccessBoundWitness, httpSuccessPayloads] = await new BoundWitnessBuilder()
-      .payloads([thumbnailHttpSuccess, httpSuccessTimestamp])
-      .build()
-    const httpFailTimestamp: TimeStamp = { schema: TimestampSchema, timestamp: 2 }
-    const [httpFailBoundWitness, httpFailPayloads] = await (new BoundWitnessBuilder().payloads([thumbnailHttpFail, httpFailTimestamp])).build()
-
-    const witnessFailTimestamp: TimeStamp = { schema: TimestampSchema, timestamp: 3 }
-    const [witnessFailBoundWitness, witnessFailPayloads] = await new BoundWitnessBuilder()
-      .payloads([thumbnailWitnessFail, witnessFailTimestamp])
-      .build()
-
-    const codeFailTimestamp: TimeStamp = { schema: TimestampSchema, timestamp: 4 }
-    const [codeFailBoundWitness, codeFailPayloads] = await (new BoundWitnessBuilder().payloads([thumbnailCodeFail, codeFailTimestamp])).build()
-
-    archivist = assertEx(asArchivistInstance<MemoryArchivist>(await node.resolve('ImageThumbnailArchivist')))
-    const testCasesToCreate = [
-      [httpSuccessBoundWitness, ...httpSuccessPayloads],
-      [httpFailBoundWitness, ...httpFailPayloads],
-      [witnessFailBoundWitness, ...witnessFailPayloads],
-      [codeFailBoundWitness, ...codeFailPayloads],
-    ]
-
-    for (const [bw, ...payloads] of testCasesToCreate) {
-      const createdTestCase = []
-      for (const payload of [bw, ...payloads]) {
-        await delay(2)
-        const [signedPayload] = await archivist.insert([payload])
-        createdTestCase.push(signedPayload)
-      }
-      testCases.push(createdTestCase)
-    }
-
-    sut = assertEx(
-      asDivinerInstance(await node.resolve('TemporalStateToIndexCandidateDiviner')),
-    ) as TemporalIndexingDivinerStateToIndexCandidateDiviner
-  })
-
-  describe('divine', () => {
-    describe('with no previous state', () => {
-      it('returns next state and batch results', async () => {
-        const results = await sut.divine()
-        expect(results.length).toBe(testCases.flat().length + 1)
-        const state = results.find(isModuleState<IndexingDivinerState>)
-        expect(state).toBeDefined()
-        const last = filterAs(results, asOptionalStorageMeta)
-          .map(p => p as WithStorageMeta<Payload>)
-          .toSorted(PayloadBuilder.compareStorageMeta)
-          .at(-1)
-        expect(last).toBeDefined()
-        expect(state?.state.cursor).toBe(last?._sequence)
-      })
-    })
-    describe('with previous state', () => {
-      it.each([1, 2, 3])('returns next state and batch results', async (batch) => {
-        const all = (await archivist.all()).toSorted(PayloadBuilder.compareStorageMeta)
-        const batchOffset = all.at((3 * batch) - 1)
-        expect(batchOffset).toBeDefined()
-        // Test across all offsets
-        const cursor = assertEx(batchOffset)._sequence
-        const lastState: ModuleState<IndexingDivinerState> = { schema: ModuleStateSchema, state: { cursor } }
-        const results = await sut.divine([lastState])
-
-        // Validate expected results length
-        // [BW, ImageThumbnail, TimeStamp] + 1 [ModuleState]
-        const expectedResults = testCases.slice(batch).flat().length + 1
-        expect(results.length).toBe(expectedResults)
-
-        // Validate expected state
-        const nextState = results.find(isModuleState<IndexingDivinerState>)
-        expect(nextState).toBeDefined()
-        expect(nextState?.state?.cursor).toBeDefined()
-        expect(nextState?.state.cursor).toBe(testCases?.at(-1)?.at(-1)?._sequence)
-
-        // Validate expected individual results
-        const bws = results.filter(isBoundWitness)
-        expect(bws.length).toBeGreaterThan(0)
-        const timestamps = results.filter(isTimestamp)
-        expect(timestamps.length).toBeGreaterThan(0)
-      })
-    })
-  })
-})

package/src/spec/Diviner.Multiple.spec.ts

@@ -1,214 +0,0 @@
-import '@xylabs/vitest-extended'
-
-import { filterAs } from '@xylabs/array'
-import { assertEx } from '@xylabs/assert'
-import { delay } from '@xylabs/delay'
-import { AsObjectFactory } from '@xylabs/object'
-import type { MemoryArchivist } from '@xyo-network/archivist-memory'
-import { asArchivistInstance } from '@xyo-network/archivist-model'
-import { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'
-import { asBoundWitness } from '@xyo-network/boundwitness-model'
-import { asDivinerInstance } from '@xyo-network/diviner-model'
-import type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'
-import { PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'
-import { isTemporalIndexingDivinerResultIndex } from '@xyo-network/diviner-temporal-indexing-model'
-import type { PackageManifestPayload } from '@xyo-network/manifest'
-import { ManifestWrapper } from '@xyo-network/manifest'
-import { ModuleFactoryLocator } from '@xyo-network/module-factory-locator'
-import type {
-  Labels, ModuleState, StateDictionary,
-} from '@xyo-network/module-model'
-import { isModuleState } from '@xyo-network/module-model'
-import type { MemoryNode } from '@xyo-network/node-memory'
-import { PayloadBuilder } from '@xyo-network/payload-builder'
-import type { Payload } from '@xyo-network/payload-model'
-import { HDWallet } from '@xyo-network/wallet'
-import type { TimeStamp } from '@xyo-network/witness-timestamp'
-import { TimestampSchema } from '@xyo-network/witness-timestamp'
-import {
-  beforeAll,
-  describe, expect, it,
-} from 'vitest'
-
-import { TemporalIndexingDiviner } from '../Diviner.ts'
-import { TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner } from '../DivinerQueryToIndexQueryDiviner/index.ts'
-import { TemporalIndexingDivinerIndexCandidateToIndexDiviner } from '../IndexCandidateToIndexDiviner/index.ts'
-import { TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner } from '../IndexQueryResponseToDivinerQueryResponseDiviner/index.ts'
-import { TemporalIndexingDivinerStateToIndexCandidateDiviner } from '../StateToIndexCandidateDiviner/index.ts'
-import imageThumbnailDivinerManifest from './TemporalDiviner.json' with { type: 'json' }
-
-type ImageThumbnail = Payload<{
-  http?: {
-    code?: string
-    ipAddress?: string
-    status?: number
-  }
-  // schema: 'network.xyo.image.thumbnail'
-  sourceHash?: string
-  sourceUrl: string
-  url?: string
-}>
-
-type Query = PayloadDivinerQueryPayload & { status?: number; success?: boolean; url?: string }
-
-/**
- * @group slow
- */
-describe('TemporalIndexingDiviner - Multiple', () => {
-  const sourceUrl = 'https://placekitten.com/200/300'
-  const thumbnailHttpSuccess: ImageThumbnail = {
-    http: { status: 200 },
-    schema: 'network.xyo.image.thumbnail',
-    sourceHash: '7f39363514d9d9b958a5a993edeba35cb44f912c7072ed9ddd628728ac0fd681',
-    sourceUrl,
-    url: 'data:image/png;base64,===',
-  }
-
-  const thumbnailHttpFail: ImageThumbnail = {
-    http: {
-
-      ipAddress: '104.17.96.13',
-      status: 429,
-    },
-    schema: 'network.xyo.image.thumbnail',
-    sourceUrl,
-  }
-
-  const thumbnailCodeFail: ImageThumbnail = {
-    http: { code: 'FAILED' },
-    schema: 'network.xyo.image.thumbnail',
-    sourceUrl,
-  }
-
-  const thumbnailWitnessFail: ImageThumbnail = {
-
-    http: { ipAddress: '104.17.96.13' },
-    schema: 'network.xyo.image.thumbnail',
-    sourceUrl,
-  }
-  const witnessedThumbnails = [thumbnailHttpSuccess, thumbnailHttpFail, thumbnailCodeFail, thumbnailWitnessFail]
-
-  let sut: TemporalIndexingDiviner
-  let node: MemoryNode
-
-  beforeAll(async () => {
-    const labels: Labels = { 'network.xyo.image.thumbnail': 'diviner' }
-    const wallet = await HDWallet.random()
-    const locator = new ModuleFactoryLocator()
-    locator.register(TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner.factory(), labels)
-    locator.register(TemporalIndexingDivinerIndexCandidateToIndexDiviner.factory(), labels)
-    locator.register(TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner.factory(), labels)
-    locator.register(TemporalIndexingDivinerStateToIndexCandidateDiviner.factory(), labels)
-    locator.register(TemporalIndexingDiviner.factory(), labels)
-    const manifest = imageThumbnailDivinerManifest as PackageManifestPayload
-    const manifestWrapper = new ManifestWrapper(manifest, wallet, locator)
-    node = await manifestWrapper.loadNodeFromIndex(0)
-    await node.start()
-
-    // Insert previously witnessed payloads into thumbnail archivist
-    const timestamp: TimeStamp = { schema: TimestampSchema, timestamp: Date.now() }
-    const [boundWitness, payloads] = await new BoundWitnessBuilder().payloads([timestamp, ...witnessedThumbnails]).build()
-
-    const thumbnailArchivist = assertEx(asArchivistInstance<MemoryArchivist>(await node.resolve('ImageThumbnailArchivist')))
-    await thumbnailArchivist.insert([boundWitness, ...payloads])
-
-    sut = assertEx(asDivinerInstance(await node.resolve('ImageThumbnailDiviner'))) as TemporalIndexingDiviner
-
-    // Allow enough time for diviner to divine
-    await delay(1000)
-  }, 40_000)
-  describe('diviner state', () => {
-    let stateArchivist: MemoryArchivist
-    beforeAll(async () => {
-      const mod = await node.resolve('AddressStateArchivist')
-      stateArchivist = assertEx(asArchivistInstance<MemoryArchivist>(mod))
-    })
-    it('has expected bound witnesses', async () => {
-      const payloads = await stateArchivist.all()
-      const stateBoundWitnesses = filterAs(payloads, asBoundWitness)
-      expect(stateBoundWitnesses).toBeArrayOfSize(1)
-      for (const stateBoundWitness of stateBoundWitnesses) {
-        expect(stateBoundWitness).toBeObject()
-        expect(stateBoundWitness.addresses).toBeArrayOfSize(1)
-        expect(stateBoundWitness.addresses).toContain(sut.address)
-      }
-    })
-    it('has expected state', async () => {
-      const payloads = await stateArchivist.all()
-      try {
-        const asModuleState = AsObjectFactory.create<ModuleState<StateDictionary>>(isModuleState)
-        const statePayloads = filterAs(payloads, asModuleState)
-        expect(statePayloads).toBeArrayOfSize(1)
-        expect(statePayloads.at(-1)).toBeObject()
-        const statePayload = assertEx(statePayloads.at(-1))
-        expect(statePayload.state).toBeObject()
-        expect(statePayload.state?.cursor).toBeDefined()
-      } catch (ex) {
-        console.error('State payloads:', payloads)
-        throw ex
-      }
-    })
-  })
-  describe('diviner index', () => {
-    let indexArchivist: MemoryArchivist
-    beforeAll(async () => {
-      const mod = await node.resolve('ImageThumbnailDivinerIndexArchivist')
-      indexArchivist = assertEx(asArchivistInstance<MemoryArchivist>(mod))
-    })
-    // NOTE: We're not signing indexes for performance reasons
-    it.skip('has expected bound witnesses', async () => {
-      const payloads = await indexArchivist.all()
-      const indexBoundWitnesses = filterAs(payloads, asBoundWitness)
-      expect(indexBoundWitnesses).toBeArrayOfSize(1)
-      const indexBoundWitness = indexBoundWitnesses[0]
-      expect(indexBoundWitness).toBeObject()
-      expect(indexBoundWitness.addresses).toBeArrayOfSize(1)
-      expect(indexBoundWitness.addresses).toContain(sut.address)
-    })
-    it('has expected index', async () => {
-      const payloads = await indexArchivist.all()
-      const indexPayloads = payloads.filter(isTemporalIndexingDivinerResultIndex)
-      expect(indexPayloads).toBeArrayOfSize((witnessedThumbnails).length)
-    })
-  })
-  describe('with no thumbnail for the provided URL', () => {
-    const url = 'https://does.not.exist.io'
-    const schema = PayloadDivinerQuerySchema
-    it('returns nothing', async () => {
-      const query: Query = { schema, url }
-      const result = await sut.divine([query])
-      expect(result).toBeArrayOfSize(0)
-    })
-  })
-  describe('with thumbnails for the provided URL', () => {
-    const url = sourceUrl
-    const schema = PayloadDivinerQuerySchema
-    describe('with no filter criteria', () => {
-      it('returns the most recent result', async () => {
-        const query: Query = { schema, url }
-        const results = await sut.divine([query])
-        const result = results.find(isTemporalIndexingDivinerResultIndex)
-        expect(result).toBeDefined()
-        const payload = assertEx((witnessedThumbnails).at(-1))
-        const expected = await PayloadBuilder.dataHash(payload)
-        expect(result?.$sources).toContain(expected)
-      })
-    })
-    describe('with filter criteria', () => {
-      describe('for status code', () => {
-        const cases: ImageThumbnail[] = [thumbnailHttpSuccess, thumbnailHttpFail]
-        it.each(cases)('returns the most recent instance of that status code', async (payload) => {
-          const { status } = payload.http ?? {}
-          const query: Query = {
-            schema, status, url,
-          }
-          const results = await sut.divine([query])
-          const result = results.find(isTemporalIndexingDivinerResultIndex)
-          expect(result).toBeDefined()
-          const expected = await PayloadBuilder.dataHash(payload)
-          expect(result?.$sources).toContain(expected)
-        })
-      })
-    })
-  })
-})