@xyo-network/diviner-temporal-indexing-memory 5.1.22 → 5.1.23

package/src/IndexCandidateToIndexDiviner/spec/Diviner.spec.ts

@@ -1,247 +0,0 @@
-import '@xylabs/vitest-extended'
-
-import { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'
-import type { BoundWitness } from '@xyo-network/boundwitness-model'
-import type { SchemaToJsonPathTransformExpressionsDictionary } from '@xyo-network/diviner-jsonpath-model'
-import { isTemporalIndexingDivinerResultIndex } from '@xyo-network/diviner-temporal-indexing-model'
-import { PayloadBuilder } from '@xyo-network/payload-builder'
-import type { Payload } from '@xyo-network/payload-model'
-import type { TimeStamp } from '@xyo-network/witness-timestamp'
-import { TimestampSchema } from '@xyo-network/witness-timestamp'
-import {
-  beforeAll,
-  describe, expect, it,
-} from 'vitest'
-
-import { TemporalIndexingDivinerIndexCandidateToIndexDiviner } from '../Diviner.ts'
-
-type ImageThumbnail = Payload<{
-  http?: {
-    ipAddress?: string
-    status?: number
-  }
-  // schema: 'network.xyo.image.thumbnail'
-  sourceUrl: string
-  url?: string
-}>
-
-describe('TemporalIndexCandidateToImageThumbnailIndexDiviner', () => {
-  describe('divine', () => {
-    const timestampA = 1_234_567_890
-    const timestampPayloadA: TimeStamp = { schema: TimestampSchema, timestamp: timestampA }
-    const imageThumbnailPayloadA: ImageThumbnail = {
-      http: { status: 200 },
-      schema: 'network.xyo.image.thumbnail',
-      sourceUrl: 'https://xyo.network',
-      url: 'data',
-    }
-    const timestampB = 1_234_567_891
-    const timestampPayloadB: TimeStamp = { schema: TimestampSchema, timestamp: timestampB }
-    const imageThumbnailPayloadB: ImageThumbnail = {
-      http: { status: 500 },
-      schema: 'network.xyo.image.thumbnail',
-      sourceUrl: 'https://xyo.network',
-    }
-    const timestampC = 1_234_567_892
-    const timestampPayloadC: TimeStamp = { schema: TimestampSchema, timestamp: timestampC }
-    const imageThumbnailPayloadC: ImageThumbnail = {
-
-      http: { ipAddress: '192.169.1.1' },
-      schema: 'network.xyo.image.thumbnail',
-      sourceUrl: 'https://www.google.com',
-    }
-    describe('with single schema transform', () => {
-      const validateSingleResult = async (
-        input: [boundWitness: BoundWitness, timestamp: TimeStamp, thumbnail: ImageThumbnail],
-        result: Payload[],
-      ) => {
-        const [boundWitness, timestamp, thumbnail] = input
-        const payloadDictionary = await PayloadBuilder.toHashMap([boundWitness, timestamp, thumbnail])
-        expect(result).toBeArrayOfSize(1)
-        expect(result.filter(isTemporalIndexingDivinerResultIndex)).toBeArrayOfSize(1)
-        const index = result.find(isTemporalIndexingDivinerResultIndex)
-        // eslint-disable-next-line sonarjs/no-alphabetical-sort
-        expect(index?.$sources.toSorted()).toEqual(Object.keys(payloadDictionary).toSorted())
-        expect(index?.timestamp).toBe(timestamp.timestamp)
-        expect((index as { url?: string })?.url).toBe(thumbnail.sourceUrl)
-        expect((index as { status?: number })?.status).toBe(thumbnail.http?.status)
-      }
-      beforeAll(async () => {
-        diviner = await TemporalIndexingDivinerIndexCandidateToIndexDiviner.create({
-          account: 'random',
-          config: {
-            schema: TemporalIndexingDivinerIndexCandidateToIndexDiviner.defaultConfigSchema,
-            schemaTransforms,
-          },
-        })
-      })
-      const schemaTransforms: SchemaToJsonPathTransformExpressionsDictionary = {
-        'network.xyo.image.thumbnail': [
-          { destinationField: 'url', sourcePathExpression: '$.sourceUrl' },
-          { destinationField: 'status', sourcePathExpression: '$.http.status' },
-        ],
-        'network.xyo.timestamp': [{ destinationField: 'timestamp', sourcePathExpression: '$.timestamp' }],
-      }
-      let diviner: TemporalIndexingDivinerIndexCandidateToIndexDiviner
-
-      const cases: [TimeStamp, ImageThumbnail][] = [
-        [timestampPayloadA, imageThumbnailPayloadA],
-        [timestampPayloadB, imageThumbnailPayloadB],
-        [timestampPayloadC, imageThumbnailPayloadC],
-      ]
-      describe('with single result', () => {
-        it.each(cases)('transforms single result', async (timestamp, thumbnail) => {
-          const [boundWitness] = await new BoundWitnessBuilder().payloads([timestamp, thumbnail]).build()
-          const result = await diviner.divine([boundWitness, timestamp, thumbnail])
-          await validateSingleResult([boundWitness, timestamp, thumbnail], result)
-        })
-        it('transforms BW with multiple results inside', async () => {
-          const payloads = cases.flat()
-          const [boundWitness] = await new BoundWitnessBuilder().payloads(payloads).build()
-          const results = await diviner.divine([boundWitness, ...payloads])
-          expect(results).toBeArrayOfSize(Math.pow(cases.length, cases[0].length))
-          let resultIndex = 0
-          for (let i = 0; i < cases.length; i++) {
-            const thumbnail = cases[i][1]
-            // eslint-disable-next-line unicorn/no-for-loop
-            for (let j = 0; j < cases.length; j++) {
-              const timestamp = cases[j][0]
-              const result = results[resultIndex]
-              await validateSingleResult([boundWitness, timestamp, thumbnail], [result])
-              resultIndex++
-            }
-          }
-        })
-        it.each(cases)('handles sparse inputs', async (thumbnail, timestamp) => {
-          const [boundWitness] = await new BoundWitnessBuilder().payloads([timestamp, thumbnail]).build()
-          expect(await diviner.divine([thumbnail, timestamp])).toBeArrayOfSize(0)
-          expect(await diviner.divine([boundWitness, timestamp])).toBeArrayOfSize(0)
-          expect(await diviner.divine([boundWitness, thumbnail])).toBeArrayOfSize(0)
-        })
-      })
-      describe('with multiple results', () => {
-        it('transforms multiple results', async () => {
-          const data: [BoundWitness, TimeStamp, ImageThumbnail][] = await Promise.all(
-            cases.map(async (payloads) => {
-              const [bw] = await new BoundWitnessBuilder().payloads(payloads).build()
-              return [bw, ...payloads]
-            }),
-          )
-          const results = await diviner.divine(data.flat())
-          expect(results).toBeArrayOfSize(cases.length)
-
-          await Promise.all(
-            data.map(async (input, i) => {
-              const result = results[i]
-              await validateSingleResult(input, [result])
-            }),
-          )
-        })
-        it('handles sparse inputs', async () => {
-          const [bw] = await new BoundWitnessBuilder().payloads(cases[0]).build()
-          const results = await diviner.divine([bw, ...cases.flat()])
-          expect(results).toBeArrayOfSize(1)
-          await validateSingleResult([bw, ...cases[0]], results)
-        })
-        it('handles missing inputs', async () => {
-          const [bw] = await new BoundWitnessBuilder().payloads([...cases[0]]).build()
-          const results = await diviner.divine([bw, ...cases[0].slice(0, -1)])
-          expect(results).toBeArrayOfSize(0)
-        })
-      })
-    })
-    describe('with multiple schema transforms', () => {
-      const validateMultiResult = async (
-        input: [boundWitness: BoundWitness, timestamp: TimeStamp, thumbnail: ImageThumbnail, payload: Payload],
-        result: Payload[],
-      ) => {
-        const [boundWitness, timestamp, thumbnail, payload] = input
-        const payloadDictionary = await PayloadBuilder.toHashMap([boundWitness, timestamp, thumbnail, payload])
-        expect(result).toBeArrayOfSize(1)
-        expect(result.filter(isTemporalIndexingDivinerResultIndex)).toBeArrayOfSize(1)
-        const index = result.find(isTemporalIndexingDivinerResultIndex)
-        // eslint-disable-next-line sonarjs/no-alphabetical-sort
-        expect(index?.$sources.toSorted()).toEqual(Object.keys(payloadDictionary).toSorted())
-        expect(index?.timestamp).toBe(timestamp.timestamp)
-        expect((index as { url?: string })?.url).toBe((payload as { sourceUrl?: string }).sourceUrl)
-        expect((index as { status?: number })?.status).toBe(thumbnail.http?.status)
-      }
-      beforeAll(async () => {
-        diviner = await TemporalIndexingDivinerIndexCandidateToIndexDiviner.create({
-          account: 'random',
-          config: {
-            schema: TemporalIndexingDivinerIndexCandidateToIndexDiviner.defaultConfigSchema,
-            schemaTransforms,
-          },
-        })
-      })
-      const schemaTransforms: SchemaToJsonPathTransformExpressionsDictionary = {
-        'network.xyo.image.thumbnail': [{ destinationField: 'status', sourcePathExpression: '$.http.status' }],
-        'network.xyo.image.thumbnail.other': [{ destinationField: 'url', sourcePathExpression: '$.sourceUrl' }],
-        'network.xyo.timestamp': [{ destinationField: 'timestamp', sourcePathExpression: '$.timestamp' }],
-      }
-      let diviner: TemporalIndexingDivinerIndexCandidateToIndexDiviner
-
-      const cases: [TimeStamp, ImageThumbnail, Payload<{ sourceUrl: string }>][] = [
-        [
-          timestampPayloadA,
-          { ...imageThumbnailPayloadA, sourceUrl: '' },
-          { schema: 'network.xyo.image.thumbnail.other', sourceUrl: imageThumbnailPayloadA.sourceUrl },
-        ],
-        [
-          timestampPayloadB,
-          { ...imageThumbnailPayloadB, sourceUrl: '' },
-          { schema: 'network.xyo.image.thumbnail.other', sourceUrl: imageThumbnailPayloadB.sourceUrl },
-        ],
-        [
-          timestampPayloadC,
-          { ...imageThumbnailPayloadC, sourceUrl: '' },
-          { schema: 'network.xyo.image.thumbnail.other', sourceUrl: imageThumbnailPayloadC.sourceUrl },
-        ],
-      ]
-      describe('with single result', () => {
-        it.each(cases)('transforms single result', async (timestamp, thumbnail, payload) => {
-          const [boundWitness] = await new BoundWitnessBuilder().payloads([timestamp, thumbnail, payload]).build()
-          const result = await diviner.divine([boundWitness, timestamp, thumbnail, payload])
-          await validateMultiResult([boundWitness, timestamp, thumbnail, payload], result)
-        })
-        it.each(cases)('handles sparse inputs', async (thumbnail, timestamp, payload) => {
-          const [boundWitness] = await new BoundWitnessBuilder().payloads([timestamp, thumbnail, payload]).build()
-          expect(await diviner.divine([thumbnail, timestamp])).toBeArrayOfSize(0)
-          expect(await diviner.divine([boundWitness, timestamp])).toBeArrayOfSize(0)
-          expect(await diviner.divine([boundWitness, thumbnail])).toBeArrayOfSize(0)
-        })
-      })
-      describe('with multiple results', () => {
-        it('transforms multiple results', async () => {
-          const data: [BoundWitness, TimeStamp, ImageThumbnail, Payload][] = await Promise.all(
-            cases.map(async (payloads) => {
-              const [bw] = await new BoundWitnessBuilder().payloads(payloads).build()
-              return [bw, ...payloads]
-            }),
-          )
-          const results = await diviner.divine(data.flat())
-          expect(results).toBeArrayOfSize(cases.length)
-
-          await Promise.all(
-            data.map(async (input, i) => {
-              const result = results[i]
-              await validateMultiResult(input, [result])
-            }),
-          )
-        })
-        it('handles sparse inputs', async () => {
-          const [bw] = await new BoundWitnessBuilder().payloads(cases[0]).build()
-          const results = await diviner.divine([bw, ...cases.flat()])
-          expect(results).toBeArrayOfSize(1)
-          await validateMultiResult([bw, ...cases[0]], results)
-        })
-        it('handles missing inputs', async () => {
-          const [bw] = await new BoundWitnessBuilder().payloads([...cases[0]]).build()
-          const results = await diviner.divine([bw, ...cases[0].slice(0, -1)])
-          expect(results).toBeArrayOfSize(0)
-        })
-      })
-    })
-  })
-})

package/src/IndexQueryResponseToDivinerQueryResponseDiviner/spec/Diviner.spec.ts

@@ -1,103 +0,0 @@
-import '@xylabs/vitest-extended'
-
-import type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'
-import { PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'
-import type { Payload } from '@xyo-network/payload-model'
-import {
-  beforeAll,
-  describe, expect, it,
-} from 'vitest'
-
-import { TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner } from '../Diviner.ts'
-
-type QueryType = Payload<PayloadDivinerQueryPayload & Payload<{ status?: number; success?: boolean; url: string }>>
-
-describe('TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner', () => {
-  const url = 'https://xyo.network'
-  const queries: QueryType[] = [
-    {
-      schema: PayloadDivinerQuerySchema,
-      url,
-    },
-    {
-      schema: PayloadDivinerQuerySchema,
-      url,
-    },
-  ]
-  const indexes = [
-    [
-      {
-        schema: 'TODO',
-        sources: [],
-        status: 200,
-        success: true,
-        timestamp: 1_234_567_890,
-        url,
-      },
-    ],
-    [
-      {
-        schema: 'TODO',
-        sources: [],
-        status: 200,
-        success: true,
-        timestamp: 1_234_567_891,
-        url,
-      },
-      {
-        schema: 'TODO',
-        sources: [],
-        status: 500,
-        success: false,
-        timestamp: 1_234_567_892,
-        url,
-      },
-    ],
-  ]
-  let diviner: TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner
-  beforeAll(async () => {
-    diviner = await TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner.create({ account: 'random' })
-  })
-  const cases: [QueryType, Payload[]][] = queries.map((query, i) => [query, indexes[i]])
-  describe('divine', () => {
-    describe('with single url in index result', () => {
-      it.each(cases)('transforms single url index results', async (imageThumbnailDivinerQuery, imageThumbnailResultIndex) => {
-        const results = await diviner.divine([imageThumbnailDivinerQuery, ...imageThumbnailResultIndex])
-        expect(results).toBeArrayOfSize(imageThumbnailResultIndex.length)
-        expect(results).toBeArrayOfSize(imageThumbnailResultIndex.length)
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any, unicorn/no-array-for-each
-        results.forEach((result: any, i) => {
-          // eslint-disable-next-line @typescript-eslint/no-explicit-any
-          const index = imageThumbnailResultIndex[i] as any
-          expect(result.url).toBe(imageThumbnailDivinerQuery.url)
-          expect(result.success).toBe(index.success)
-          expect(result.timestamp).toBe(index.timestamp)
-          expect(result.status).toBe(index.status)
-          expect(result.schema).toBe('TODO')
-        })
-      })
-    })
-    describe('with multiple urls in index result', () => {
-      it('transforms multiple url index results', async () => {
-        const indexesLength = indexes.flat().length
-        const results = await diviner.divine([...queries, ...indexes.flat()])
-        expect(results).toBeArrayOfSize(indexesLength)
-        expect(results).toBeArrayOfSize(indexesLength)
-        let resultsIterator = 0
-        for (const [i, { url }] of queries.entries()) {
-          const indexSet = indexes[i]
-          for (const index of indexSet) {
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            const result = results[resultsIterator] as any
-            expect(result.url).toBe(url)
-            expect(result.success).toBe(index.success)
-            expect(result.timestamp).toBe(index.timestamp)
-            expect(result.status).toBe(index.status)
-            expect(result.schema).toBe('TODO')
-            resultsIterator = ++resultsIterator
-          }
-        }
-      })
-    })
-  })
-})

package/src/StateToIndexCandidateDiviner/spec/Diviner.spec.ts

@@ -1,167 +0,0 @@
-import '@xylabs/vitest-extended'
-
-import { filterAs } from '@xylabs/array'
-import { assertEx } from '@xylabs/assert'
-import { delay } from '@xylabs/delay'
-import type { MemoryArchivist } from '@xyo-network/archivist-memory'
-import { asArchivistInstance } from '@xyo-network/archivist-model'
-import { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'
-import { isBoundWitness } from '@xyo-network/boundwitness-model'
-import type { IndexingDivinerState } from '@xyo-network/diviner-indexing-model'
-import { asDivinerInstance } from '@xyo-network/diviner-model'
-import type { PackageManifestPayload } from '@xyo-network/manifest'
-import { ManifestWrapper } from '@xyo-network/manifest'
-import { ModuleFactoryLocator } from '@xyo-network/module-factory-locator'
-import type { ModuleState } from '@xyo-network/module-model'
-import { isModuleState, ModuleStateSchema } from '@xyo-network/module-model'
-import { PayloadBuilder } from '@xyo-network/payload-builder'
-import type { Payload, WithStorageMeta } from '@xyo-network/payload-model'
-import { asOptionalStorageMeta } from '@xyo-network/payload-model'
-import { HDWallet } from '@xyo-network/wallet'
-import type { TimeStamp } from '@xyo-network/witness-timestamp'
-import { isTimestamp, TimestampSchema } from '@xyo-network/witness-timestamp'
-import {
-  beforeAll, describe, expect, it,
-} from 'vitest'
-
-import { TemporalIndexingDivinerStateToIndexCandidateDiviner } from '../Diviner.ts'
-import TemporalStateToIndexCandidateDivinerManifest from './TemporalStateToIndexCandidateDiviner.json' with { type: 'json' }
-
-/**
- * @group slow
- */
-describe('TemporalStateToIndexCandidateDiviner', () => {
-  const sourceUrl = 'https://placekitten.com/200/300'
-  const thumbnailHttpSuccess = {
-    http: { status: 200 },
-    schema: 'network.xyo.image.thumbnail',
-    sourceHash: '7f39363514d9d9b958a5a993edeba35cb44f912c7072ed9ddd628728ac0fd681',
-    sourceUrl,
-    url: 'data:image/png;base64,===',
-  }
-
-  const thumbnailHttpFail = {
-    http: {
-
-      ipAddress: '104.17.96.13',
-      status: 429,
-    },
-    schema: 'network.xyo.image.thumbnail',
-    sourceUrl,
-  }
-
-  const thumbnailCodeFail = {
-    http: { code: 'FAILED' },
-    schema: 'network.xyo.image.thumbnail',
-    sourceUrl,
-  }
-
-  const thumbnailWitnessFail = {
-
-    http: { ipAddress: '104.17.96.13' },
-    schema: 'network.xyo.image.thumbnail',
-    sourceUrl,
-  }
-
-  let testCases: WithStorageMeta<Payload>[][] = []
-  let archivist: MemoryArchivist
-  let sut: TemporalIndexingDivinerStateToIndexCandidateDiviner
-
-  beforeAll(async () => {
-    const wallet = await HDWallet.random()
-    const locator = new ModuleFactoryLocator()
-    locator.register(TemporalIndexingDivinerStateToIndexCandidateDiviner.factory())
-    const manifest = TemporalStateToIndexCandidateDivinerManifest as PackageManifestPayload
-    const manifestWrapper = new ManifestWrapper(manifest, wallet, locator)
-    const node = await manifestWrapper.loadNodeFromIndex(0)
-    await node.start()
-
-    const privateModules = manifest.nodes[0].modules?.private ?? []
-    const publicModules = manifest.nodes[0].modules?.public ?? []
-    const mods = await node.resolve('*')
-    expect(mods.length).toBe(privateModules.length + publicModules.length + 1)
-
-    // Insert previously witnessed payloads into thumbnail archivist
-    const httpSuccessTimestamp: TimeStamp = { schema: TimestampSchema, timestamp: 1 }
-    const [httpSuccessBoundWitness, httpSuccessPayloads] = await new BoundWitnessBuilder()
-      .payloads([thumbnailHttpSuccess, httpSuccessTimestamp])
-      .build()
-    const httpFailTimestamp: TimeStamp = { schema: TimestampSchema, timestamp: 2 }
-    const [httpFailBoundWitness, httpFailPayloads] = await (new BoundWitnessBuilder().payloads([thumbnailHttpFail, httpFailTimestamp])).build()
-
-    const witnessFailTimestamp: TimeStamp = { schema: TimestampSchema, timestamp: 3 }
-    const [witnessFailBoundWitness, witnessFailPayloads] = await new BoundWitnessBuilder()
-      .payloads([thumbnailWitnessFail, witnessFailTimestamp])
-      .build()
-
-    const codeFailTimestamp: TimeStamp = { schema: TimestampSchema, timestamp: 4 }
-    const [codeFailBoundWitness, codeFailPayloads] = await (new BoundWitnessBuilder().payloads([thumbnailCodeFail, codeFailTimestamp])).build()
-
-    archivist = assertEx(asArchivistInstance<MemoryArchivist>(await node.resolve('ImageThumbnailArchivist')))
-    const testCasesToCreate = [
-      [httpSuccessBoundWitness, ...httpSuccessPayloads],
-      [httpFailBoundWitness, ...httpFailPayloads],
-      [witnessFailBoundWitness, ...witnessFailPayloads],
-      [codeFailBoundWitness, ...codeFailPayloads],
-    ]
-
-    for (const [bw, ...payloads] of testCasesToCreate) {
-      const createdTestCase = []
-      for (const payload of [bw, ...payloads]) {
-        await delay(2)
-        const [signedPayload] = await archivist.insert([payload])
-        createdTestCase.push(signedPayload)
-      }
-      testCases.push(createdTestCase)
-    }
-
-    sut = assertEx(
-      asDivinerInstance(await node.resolve('TemporalStateToIndexCandidateDiviner')),
-    ) as TemporalIndexingDivinerStateToIndexCandidateDiviner
-  })
-
-  describe('divine', () => {
-    describe('with no previous state', () => {
-      it('returns next state and batch results', async () => {
-        const results = await sut.divine()
-        expect(results.length).toBe(testCases.flat().length + 1)
-        const state = results.find(isModuleState<IndexingDivinerState>)
-        expect(state).toBeDefined()
-        const last = filterAs(results, asOptionalStorageMeta)
-          .map(p => p as WithStorageMeta<Payload>)
-          .toSorted(PayloadBuilder.compareStorageMeta)
-          .at(-1)
-        expect(last).toBeDefined()
-        expect(state?.state.cursor).toBe(last?._sequence)
-      })
-    })
-    describe('with previous state', () => {
-      it.each([1, 2, 3])('returns next state and batch results', async (batch) => {
-        const all = (await archivist.all()).toSorted(PayloadBuilder.compareStorageMeta)
-        const batchOffset = all.at((3 * batch) - 1)
-        expect(batchOffset).toBeDefined()
-        // Test across all offsets
-        const cursor = assertEx(batchOffset)._sequence
-        const lastState: ModuleState<IndexingDivinerState> = { schema: ModuleStateSchema, state: { cursor } }
-        const results = await sut.divine([lastState])
-
-        // Validate expected results length
-        // [BW, ImageThumbnail, TimeStamp] + 1 [ModuleState]
-        const expectedResults = testCases.slice(batch).flat().length + 1
-        expect(results.length).toBe(expectedResults)
-
-        // Validate expected state
-        const nextState = results.find(isModuleState<IndexingDivinerState>)
-        expect(nextState).toBeDefined()
-        expect(nextState?.state?.cursor).toBeDefined()
-        expect(nextState?.state.cursor).toBe(testCases?.at(-1)?.at(-1)?._sequence)
-
-        // Validate expected individual results
-        const bws = results.filter(isBoundWitness)
-        expect(bws.length).toBeGreaterThan(0)
-        const timestamps = results.filter(isTimestamp)
-        expect(timestamps.length).toBeGreaterThan(0)
-      })
-    })
-  })
-})