@highstate/backend 0.9.15 → 0.9.18

package/src/state/memory/backend.ts

@@ -0,0 +1,70 @@
+import type { TestTracer } from "../../common"
+import {
+  type StateBackend,
+  type StateBatch,
+  type StateSnapshot,
+  type StateCollection,
+  type CollectionMap,
+} from "../abstractions"
+import { MemoryStateCollection, MemoryStateBatch, MemoryStateSnapshot } from "./collection"
+
+/**
+ * A memory-based state backend implementation for testing purposes.
+ */
+export class MemoryStateBackend implements StateBackend {
+  private collections = new Map<string, MemoryStateCollection>()
+  private batchIdCounter = 0
+  private snapshotIdCounter = 0
+
+  constructor(private readonly tracer: TestTracer) {}
+
+  getEncryptedBackendMasterKey(): Promise<string | undefined> {
+    throw new Error("Method not implemented.")
+  }
+
+  setEncryptedBackendMasterKey(): Promise<void> {
+    throw new Error("Method not implemented.")
+  }
+
+  getEncryptedBackendNamespace(): Promise<Uint8Array | undefined> {
+    throw new Error("Method not implemented.")
+  }
+
+  setEncryptedBackendNamespace(): Promise<void> {
+    throw new Error("Method not implemented.")
+  }
+
+  createStateBatch(): StateBatch {
+    this.batchIdCounter += 1
+    return new MemoryStateBatch(this.batchIdCounter, this.tracer)
+  }
+
+  createStateSnapshot(): StateSnapshot {
+    this.snapshotIdCounter += 1
+    // Create a snapshot of all existing collections
+    const allCollections = Array.from(this.collections.values())
+    return new MemoryStateSnapshot(allCollections, this.snapshotIdCounter, this.tracer)
+  }
+
+  createCollection<TName extends keyof CollectionMap>(
+    name: TName,
+    params: CollectionMap[TName],
+  ): StateCollection {
+    const fullName = [name, ...Object.values(params)].join(":")
+
+    const existingCollection = this.collections.get(fullName)
+    if (existingCollection) {
+      return existingCollection
+    }
+
+    const collection = new MemoryStateCollection(
+      name,
+      params,
+      Promise.resolve(fullName), // this should be a UUIDv5 namespace, but for testing we will use the full name
+      this.tracer,
+    )
+    this.collections.set(fullName, collection)
+
+    return collection
+  }
+}

package/src/state/memory/collection.ts

@@ -0,0 +1,270 @@
+import type { CollectionQueryResult } from "../../shared"
+import { linkTraceEntry, type TestTracer, type TraceEntry } from "../../common"
+import {
+  CollectionBackend,
+  type StateCollection,
+  type StateBatch,
+  type StateSnapshot,
+  type CollectionQuery,
+} from "../abstractions"
+
+/**
+ * A memory-based state collection implementation for testing purposes.
+ */
+export class MemoryStateCollection implements StateCollection {
+  private data = new Map<string, Uint8Array>()
+
+  constructor(
+    readonly name: string,
+    readonly params: Record<string, string>,
+    readonly namespace: Promise<string>,
+    readonly tracer: TestTracer,
+  ) {}
+
+  get(id: string, snapshot?: StateSnapshot): Promise<Uint8Array | undefined> {
+    if (snapshot) {
+      const memorySnapshot = snapshot as unknown as MemoryStateSnapshot
+      const snapshotData = memorySnapshot.getCollection(this)
+
+      return Promise.resolve(snapshotData.get(id))
+    }
+
+    return Promise.resolve(this.data.get(id))
+  }
+
+  getMany(ids: string[], snapshot?: StateSnapshot): Promise<(Uint8Array | undefined)[]> {
+    const source = snapshot
+      ? (snapshot as unknown as MemoryStateSnapshot).getCollection(this)
+      : this.data
+
+    return Promise.resolve(ids.map(id => source.get(id)))
+  }
+
+  getAll(snapshot?: StateSnapshot): Promise<[string, Uint8Array][]> {
+    const source = snapshot
+      ? (snapshot as unknown as MemoryStateSnapshot).getCollection(this)
+      : this.data
+
+    return Promise.resolve(Array.from(source.entries()))
+  }
+
+  // eslint-disable-next-line @typescript-eslint/require-await
+  async query(
+    query: CollectionQuery,
+    snapshot?: StateSnapshot,
+  ): Promise<CollectionQueryResult<[string, Uint8Array]>> {
+    const source = snapshot
+      ? (snapshot as unknown as MemoryStateSnapshot).getCollection(this)
+      : this.data
+
+    let entries = Array.from(source.entries())
+
+    // apply cursor filtering
+    if (query.cursor) {
+      if (query.sort === "desc") {
+        entries = entries.filter(([id]) => id < query.cursor!)
+      } else {
+        entries = entries.filter(([id]) => id > query.cursor!)
+      }
+    }
+
+    // apply sorting
+    entries.sort(([a], [b]) => (query.sort === "desc" ? b.localeCompare(a) : a.localeCompare(b)))
+
+    // apply limit
+    const count = Math.min(query.count ?? 20, 100)
+    const limitedEntries = entries.slice(0, count)
+
+    return {
+      items: limitedEntries,
+      total: source.size,
+      nextCursor:
+        limitedEntries.length > 0 ? limitedEntries[limitedEntries.length - 1][0] : undefined,
+    }
+  }
+
+  getTotalCount(snapshot?: StateSnapshot): Promise<number> {
+    const source = snapshot
+      ? (snapshot as unknown as MemoryStateSnapshot).getCollection(this)
+      : this.data
+
+    return Promise.resolve(source.size)
+  }
+
+  // eslint-disable-next-line @typescript-eslint/require-await
+  async *iterate(snapshot?: StateSnapshot): AsyncIterable<[string, Uint8Array]> {
+    const source = snapshot
+      ? (snapshot as unknown as MemoryStateSnapshot).getCollection(this)
+      : this.data
+
+    for (const entry of source.entries()) {
+      yield entry
+    }
+  }
+
+  put(id: string, value: Uint8Array, batch?: StateBatch): Promise<void> {
+    if (batch) {
+      const memoryBatch = batch as MemoryStateBatch
+      memoryBatch[CollectionBackend].push({ type: "put", collection: this, id, value })
+      return Promise.resolve()
+    }
+
+    this.data.set(id, value)
+    return Promise.resolve()
+  }
+
+  putMany(entries: Array<[string, Uint8Array]>, batch?: StateBatch): Promise<void> {
+    if (batch) {
+      const memoryBatch = batch as MemoryStateBatch
+      for (const [id, value] of entries) {
+        memoryBatch[CollectionBackend].push({ type: "put", collection: this, id, value })
+      }
+      return Promise.resolve()
+    }
+
+    for (const [id, value] of entries) {
+      this.data.set(id, value)
+    }
+    return Promise.resolve()
+  }
+
+  delete(id: string, batch?: StateBatch): Promise<void> {
+    if (batch) {
+      const memoryBatch = batch as MemoryStateBatch
+      memoryBatch[CollectionBackend].push({ type: "delete", collection: this, id })
+      return Promise.resolve()
+    }
+
+    this.data.delete(id)
+    return Promise.resolve()
+  }
+
+  deleteMany(ids: string[], batch?: StateBatch): Promise<void> {
+    if (batch) {
+      const memoryBatch = batch as MemoryStateBatch
+      for (const id of ids) {
+        memoryBatch[CollectionBackend].push({ type: "delete", collection: this, id })
+      }
+      return Promise.resolve()
+    }
+
+    for (const id of ids) {
+      this.data.delete(id)
+    }
+    return Promise.resolve()
+  }
+
+  clear(): Promise<void> {
+    this.data.clear()
+    return Promise.resolve()
+  }
+}
+
+export interface MemoryBatchOperation {
+  type: "put" | "delete"
+  collection: MemoryStateCollection
+  id: string
+  value?: Uint8Array
+}
+
+class StateBatchEntry implements TraceEntry {
+  constructor(
+    readonly id: number,
+    private readonly opEntries: TraceEntry[],
+  ) {}
+
+  render(): string {
+    return `💾 write ` + this.opEntries.map(linkTraceEntry).join(", ")
+  }
+}
+
+class StateSnapshotEntry implements TraceEntry {
+  constructor(
+    readonly id: number,
+    readonly usageEntries: TraceEntry[],
+  ) {}
+
+  render(): string {
+    if (this.usageEntries.length === 0) {
+      return `📸 snapshot`
+    }
+
+    return `📸 snapshot, used by ` + this.usageEntries.map(linkTraceEntry).join(", ")
+  }
+}
+
+export class MemoryStateBatch implements StateBatch {
+  [CollectionBackend]: MemoryBatchOperation[] = []
+
+  constructor(
+    readonly id: number,
+    private readonly tracer: TestTracer,
+  ) {}
+
+  // for testing purposes
+  readonly opEntries: TraceEntry[] = []
+  traceEntry?: TraceEntry
+
+  disposed = false
+
+  write(): Promise<void> {
+    if (this.disposed) {
+      return Promise.resolve()
+    }
+
+    for (const operation of this[CollectionBackend]) {
+      if (operation.type === "put") {
+        operation.collection["data"].set(operation.id, operation.value!)
+      } else if (operation.type === "delete") {
+        operation.collection["data"].delete(operation.id)
+      }
+    }
+
+    this.traceEntry = new StateBatchEntry(this.tracer.nextEntryId(), this.opEntries)
+    this.tracer.addEntry(this.traceEntry)
+
+    this.disposed = true
+    return Promise.resolve()
+  }
+
+  [Symbol.asyncDispose](): Promise<void> {
+    if (this.disposed) {
+      return Promise.resolve()
+    }
+
+    this.disposed = true
+    return Promise.resolve()
+  }
+}
+
+export class MemoryStateSnapshot implements StateSnapshot {
+  [CollectionBackend] = new Map<MemoryStateCollection, Map<string, Uint8Array>>()
+
+  readonly traceEntry: TraceEntry
+  readonly usageEntries: TraceEntry[] = []
+
+  constructor(
+    collections: MemoryStateCollection[],
+    readonly id: number,
+    readonly tracer: TestTracer,
+  ) {
+    // create snapshots of all collections at this point in time
+    for (const collection of collections) {
+      this[CollectionBackend].set(collection, new Map(collection["data"]))
+    }
+
+    this.traceEntry = new StateSnapshotEntry(tracer.nextEntryId(), this.usageEntries)
+    tracer.addEntry(this.traceEntry)
+  }
+
+  getCollection(collection: MemoryStateCollection): Map<string, Uint8Array> {
+    const snapshot = this[CollectionBackend].get(collection)
+
+    return snapshot ?? new Map<string, Uint8Array>()
+  }
+
+  [Symbol.asyncDispose](): Promise<void> {
+    this[CollectionBackend].clear()
+    return Promise.resolve()
+  }
+}

package/src/state/memory/index.ts

@@ -0,0 +1,2 @@
+export * from "./backend"
+export * from "./collection"

package/src/state/repository/index.ts

@@ -0,0 +1,2 @@
+export * from "./repository"
+export * from "./repository.index"

package/src/state/repository/repository.index.ts

@@ -0,0 +1,193 @@
+import type { Logger } from "pino"
+import type { StateRepository } from "./repository"
+import type { CollectionQuery, StateSnapshot } from "../abstractions"
+import type { CollectionQueryResult } from "../../shared"
+import type { StateManager } from "../manager"
+import { type z } from "zod"
+
+/**
+ * The index value that indicates an object key is the same as the index key.
+ *
+ * NOTE: DO NOT use this feature when the index repository has key hashing enabled.
+ */
+export const SAME_KEY = ""
+
+/**
+ * StateIndexRepository provides read-only operations that combine two StateRepository instances.
+ *
+ * The index repository maps index keys to object keys, allowing indirect access to objects in the main repository.
+ */
+export class StateIndexRepository<TSchema extends z.ZodType> {
+  constructor(
+    private readonly stateManager: StateManager,
+    public readonly repository: StateRepository<TSchema>,
+    public readonly indexRepository: StateRepository<z.ZodString>,
+    private readonly logger: Logger,
+  ) {}
+
+  /**
+   * Retrieves a single item by its index key.
+   */
+  async get(indexKey: string): Promise<z.infer<TSchema> | undefined> {
+    const snapshot = this.stateManager.snapshot()
+
+    // fetch the object key from the index repository
+    const objectKey = await this.indexRepository.get(indexKey, snapshot)
+    if (!objectKey) {
+      return undefined
+    }
+
+    // if index value is empty string, use the index key itself
+    const resolvedObjectId = objectKey === SAME_KEY ? indexKey : objectKey
+
+    const item = await this.repository.get(resolvedObjectId, snapshot)
+    if (item === undefined) {
+      this.logger.warn({ indexKey, resolvedObjectId }, "index points to non-existent object")
+      return undefined
+    }
+
+    return item
+  }
+
+  /**
+   * Retrieves multiple items by their index keys, returning an array of records.
+   *
+   * NOTE: the keys of the records are the index keys, not the object keys.
+   */
+  async getMany(indexKeys: string[]): Promise<[string, z.infer<TSchema>][]> {
+    if (indexKeys.length === 0) return []
+
+    const snapshot = this.stateManager.snapshot()
+
+    // get all index records for the provided keys
+    const indexRecords = await this.indexRepository.getMany(indexKeys, snapshot)
+    if (indexRecords.length === 0) return []
+
+    // resolve index records to get the actual objects
+    return this.resolveIndexEntries(indexRecords, snapshot)
+  }
+
+  /**
+   * Retrieves multiple items by their index keys, returning a record of objects.
+   *
+   * NOTE: the keys of the record are the index keys, not the object keys.
+   */
+  async getManyRecord(indexKeys: string[]): Promise<Record<string, z.infer<TSchema> | undefined>> {
+    const records = await this.getMany(indexKeys)
+
+    return Object.fromEntries(records)
+  }
+
+  /**
+   * Retrieves multiple items by their index keys, returning an array of objects.
+   */
+  async getManyItems(indexKeys: string[]): Promise<z.infer<TSchema>[]> {
+    const records = await this.getMany(indexKeys)
+
+    return Object.values(records)
+      .filter(item => item !== null)
+      .map(([, item]) => item)
+  }
+
+  /**
+   * Retrieves all items that have index entries, returning an array of records.
+   *
+   * NOTE: the keys of the records are the index keys, not the object keys.
+   */
+  async getAll(): Promise<Array<[string, z.infer<TSchema>]>> {
+    const snapshot = this.stateManager.snapshot()
+
+    // get all index records
+    const indexEntries = await this.indexRepository.getAll(snapshot)
+    if (indexEntries.length === 0) return []
+
+    // resolve index entries to get the actual objects
+    return this.resolveIndexEntries(indexEntries, snapshot)
+  }
+
+  /**
+   * Retrieves all items that have index entries, returning a record of objects.
+   *
+   * NOTE: the keys of the record are the index keys, not the object keys.
+   */
+  async getAllRecord(): Promise<Record<string, z.infer<TSchema> | null>> {
+    const allEntries = await this.getAll()
+
+    // convert the array of entries to a record
+    return Object.fromEntries(allEntries)
+  }
+
+  /**
+   * Retrieves all items that have index entries.
+   * Iterates through the index and returns all matched objects.
+   */
+  async getAllItems(): Promise<z.infer<TSchema>[]> {
+    const allEntries = await this.getAll()
+
+    return allEntries.map(([, item]) => item)
+  }
+
+  /**
+   * Queries the index repository and retrieves items from the main repository.
+   */
+  async query(query: CollectionQuery): Promise<CollectionQueryResult<[string, z.infer<TSchema>]>> {
+    const snapshot = this.stateManager.snapshot()
+
+    // first, resolve the query to get the index keys
+    const result = await this.indexRepository.query(query, snapshot)
+
+    // then resolve the index entries to get the actual objects
+    const items = await this.resolveIndexEntries(result.items, snapshot)
+
+    return {
+      total: result.total,
+      items,
+      nextCursor: result.nextCursor,
+    }
+  }
+
+  /**
+   * Queries the index repository and retrieves items from the main repository, returning an array of objects.
+   */
+  async queryItems(query: CollectionQuery): Promise<CollectionQueryResult<z.infer<TSchema>>> {
+    const result = await this.query(query)
+
+    // extract the items from the result
+    return {
+      total: result.total,
+      items: result.items.map(([, item]) => item),
+      nextCursor: result.nextCursor,
+    }
+  }
+
+  private async resolveIndexEntries(
+    indexEntries: [string, string][],
+    snapshot: StateSnapshot,
+  ): Promise<[string, z.infer<TSchema>][]> {
+    // collect all object keys that were found
+    const objectKeys: string[] = []
+    for (const [indexKey, objectId] of indexEntries) {
+      // if index value is empty string, use the index key itself
+      const resolvedObjectId = objectId === SAME_KEY ? indexKey : objectId
+      objectKeys.push(resolvedObjectId)
+    }
+
+    // get objects from main repository
+    const objectRecord = await this.repository.getManyRecord(objectKeys, snapshot)
+    const result: [string, z.infer<TSchema>][] = []
+
+    for (const [indexKey, objectKey] of indexEntries) {
+      const resolvedObjectId = objectKey === SAME_KEY ? indexKey : objectKey
+      const item = objectRecord[resolvedObjectId]
+
+      if (item === undefined) {
+        this.logger.warn({ indexKey, resolvedObjectId }, "index points to non-existent object")
+        continue
+      }
+
+      result.push([indexKey, item])
+    }
+
+    return result
+  }
+}