@xyo-network/archivist-leveldb 5.3.22 → 5.3.25

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xyo-network/archivist-leveldb",
-  "version": "5.3.22",
+  "version": "5.3.25",
   "description": "Primary SDK for using XYO Protocol 2.0",
   "homepage": "https://xyo.network",
   "bugs": {
@@ -30,57 +30,53 @@
   "types": "dist/browser/index.d.ts",
   "files": [
     "dist",
-    "src",
     "!**/*.bench.*",
     "!**/*.spec.*",
     "!**/*.test.*",
     "README.md"
   ],
   "dependencies": {
-    "@xyo-network/archivist-abstract": "~5.3.22",
-    "@xyo-network/archivist-model": "~5.3.22",
-    "@xyo-network/boundwitness-model": "~5.3.22",
-    "@xyo-network/module-model": "~5.3.22",
-    "@xyo-network/payload-builder": "~5.3.22",
-    "@xyo-network/payload-model": "~5.3.22",
-    "async-mutex": "~0.5.0"
+    "async-mutex": "~0.5.0",
+    "@xyo-network/archivist-abstract": "~5.3.25",
+    "@xyo-network/archivist-model": "~5.3.25",
+    "@xyo-network/boundwitness-model": "~5.3.25",
+    "@xyo-network/payload-builder": "~5.3.25",
+    "@xyo-network/module-model": "~5.3.25",
+    "@xyo-network/payload-model": "~5.3.25"
   },
   "devDependencies": {
     "@opentelemetry/api": "^1.9.1",
     "@types/node": "^25.5.0",
-    "@xylabs/sdk-js": "^5.0.91",
-    "@xylabs/ts-scripts-common": "~7.6.8",
-    "@xylabs/ts-scripts-yarn3": "~7.6.8",
-    "@xylabs/tsconfig": "~7.6.8",
-    "@xylabs/vitest-extended": "~5.0.91",
-    "@xyo-network/account": "~5.3.22",
-    "@xyo-network/account-model": "~5.3.22",
-    "@xyo-network/archivist-abstract": "~5.3.22",
-    "@xyo-network/archivist-acceptance-tests": "~5.3.22",
-    "@xyo-network/archivist-model": "~5.3.22",
-    "@xyo-network/boundwitness-model": "~5.3.22",
-    "@xyo-network/id-payload-plugin": "~5.3.22",
-    "@xyo-network/module-model": "~5.3.22",
-    "@xyo-network/payload-builder": "~5.3.22",
-    "@xyo-network/payload-model": "~5.3.22",
-    "@xyo-network/payload-wrapper": "~5.3.22",
-    "@xyo-network/wallet": "~5.3.22",
+    "@xylabs/sdk-js": "^5.0.93",
+    "@xylabs/ts-scripts-common": "~7.6.16",
+    "@xylabs/ts-scripts-pnpm": "~7.6.16",
+    "@xylabs/tsconfig": "~7.6.16",
+    "@xylabs/vitest-extended": "~5.0.93",
     "abstract-level": "^3.1.1",
     "acorn": "^8.16.0",
     "async-mutex": "~0.5.0",
     "axios": "^1.14.0",
-    "cosmiconfig": "^9.0.1",
-    "esbuild": "^0.27.4",
-    "eslint": "^10.1.0",
+    "esbuild": "^0.28.0",
     "ethers": "^6.16.0",
     "level": "^10.0.0",
-    "rollup": "^4.60.1",
     "tslib": "^2.8.1",
     "typescript": "~5.9.3",
     "uuid": "~13.0.0",
     "vite": "^8.0.3",
     "vitest": "~4.1.2",
-    "zod": "^4.3.6"
+    "zod": "^4.3.6",
+    "@xyo-network/account": "~5.3.25",
+    "@xyo-network/archivist-abstract": "~5.3.25",
+    "@xyo-network/account-model": "~5.3.25",
+    "@xyo-network/archivist-model": "~5.3.25",
+    "@xyo-network/boundwitness-model": "~5.3.25",
+    "@xyo-network/id-payload-plugin": "~5.3.25",
+    "@xyo-network/archivist-acceptance-tests": "~5.3.25",
+    "@xyo-network/module-model": "~5.3.25",
+    "@xyo-network/payload-builder": "~5.3.25",
+    "@xyo-network/payload-model": "~5.3.25",
+    "@xyo-network/payload-wrapper": "~5.3.25",
+    "@xyo-network/wallet": "~5.3.25"
   },
   "peerDependencies": {
     "@xylabs/sdk-js": "^5",
@@ -93,4 +89,4 @@
   "publishConfig": {
     "access": "public"
   }
-}
+}

package/src/Archivist.ts

@@ -1,301 +0,0 @@
-import {
-  assertEx,
-  exists,
-  fulfilled, Hash, Hex, isDefined, Promisable,
-} from '@xylabs/sdk-js'
-import { AbstractArchivist, StorageClassLabel } from '@xyo-network/archivist-abstract'
-import {
-  ArchivistAllQuerySchema,
-  ArchivistClearQuerySchema,
-  ArchivistCommitQuerySchema,
-  ArchivistDeleteQuerySchema,
-  ArchivistInsertQuery,
-  ArchivistInsertQuerySchema,
-  ArchivistModuleEventData,
-  ArchivistNextOptions,
-  ArchivistNextQuerySchema,
-  buildStandardIndexName,
-  IndexDescription,
-} from '@xyo-network/archivist-model'
-import { BoundWitness } from '@xyo-network/boundwitness-model'
-import { creatableModule } from '@xyo-network/module-model'
-import { PayloadBuilder } from '@xyo-network/payload-builder'
-import {
-  Payload, Schema, WithStorageMeta,
-} from '@xyo-network/payload-model'
-import {
-  AbstractBatchOperation, AbstractLevel, AbstractSublevel,
-} from 'abstract-level'
-import { Mutex } from 'async-mutex'
-import { Level } from 'level'
-
-import { LevelDbArchivistConfigSchema } from './Config.ts'
-import { LevelDbArchivistParams } from './Params.ts'
-
-/** We have indexes as top level sublevels since making them a sublevel of a store, getting all the values of that store includes the sublevels  */
-
-export interface PayloadStore {
-  [s: string]: WithStorageMeta
-}
-
-export type AbstractPayloadLevel = AbstractLevel<string | Buffer | Uint8Array, Hash, WithStorageMeta<Payload>>
-export type AbstractPayloadSubLevel = AbstractSublevel<AbstractPayloadLevel, string | Buffer | Uint8Array, Hash, WithStorageMeta<Payload>>
-export type AbstractIndexSubLevel<T> = AbstractSublevel<AbstractPayloadLevel, string | Buffer | Uint8Array, T, Hash>
-
-const indexSubLevelName = (storeName: string, indexName: string) => {
-  return `_${storeName}|${indexName}`
-}
-
-export abstract class AbstractLevelDbArchivist<
-  TParams extends LevelDbArchivistParams = LevelDbArchivistParams,
-  TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,
-> extends AbstractArchivist<TParams, TEventData> {
-  static override readonly configSchemas: Schema[] = [...super.configSchemas, LevelDbArchivistConfigSchema]
-  static override readonly defaultConfigSchema: Schema = LevelDbArchivistConfigSchema
-  static override readonly labels = { ...super.labels, [StorageClassLabel]: 'disk' }
-
-  private static readonly dataHashIndex: IndexDescription = {
-    key: { _dataHash: 1 }, multiEntry: false, unique: false,
-  }
-
-  private static readonly sequenceIndex: IndexDescription = {
-    key: { _sequence: 1 }, multiEntry: false, unique: true,
-  }
-
-  // eslint-disable-next-line @typescript-eslint/member-ordering
-  static readonly dataHashIndexName = buildStandardIndexName(AbstractLevelDbArchivist.dataHashIndex)
-  // eslint-disable-next-line @typescript-eslint/member-ordering
-  static readonly sequenceIndexName = buildStandardIndexName(AbstractLevelDbArchivist.sequenceIndex)
-
-  get dbName() {
-    return assertEx(this.config.dbName, () => 'No dbName specified')
-  }
-
-  get folderPath() {
-    return `${this.location}/${this.config.dbName}/${this.storeName}`
-  }
-
-  get location() {
-    return assertEx(this.config.location, () => 'No location specified')
-  }
-
-  override get queries() {
-    return [
-      ArchivistAllQuerySchema,
-      ArchivistDeleteQuerySchema,
-      ArchivistClearQuerySchema,
-      ArchivistInsertQuerySchema,
-      ArchivistCommitQuerySchema,
-      ArchivistNextQuerySchema,
-      ...super.queries,
-    ]
-  }
-
-  get storeName() {
-    return assertEx(this.config.storeName, () => 'No storeName specified')
-  }
-
-  private static findIndexFromCursor(payloads: WithStorageMeta[], cursor: Hex) {
-    const index = payloads.findIndex(({ _sequence }) => _sequence === cursor)
-    if (index === -1) {
-      return Infinity // move to the end
-    }
-    return index
-  }
-
-  protected override async allHandler(): Promise<WithStorageMeta<Payload>[]> {
-    return await this.withStore(async (db) => {
-      const values = [...(await db.values().all())]
-      return values.filter(exists).toSorted(PayloadBuilder.compareStorageMeta)
-    })
-  }
-
-  protected override async clearHandler(): Promise<void> {
-    await this.withDb(async (db) => {
-      await db.clear()
-    })
-    await this.withDataHashIndex(async (index) => {
-      await index.clear()
-    })
-    await this.withSequenceIndex(async (index) => {
-      await index.clear()
-    })
-    return this.emit('cleared', { mod: this })
-  }
-
-  protected override async commitHandler(): Promise<BoundWitness[]> {
-    const payloads = assertEx(await this.allHandler(), () => 'Nothing to commit')
-    const settled = await Promise.allSettled(
-      Object.values((await this.parentArchivists()).commit ?? [])?.map(async (parent) => {
-        const queryPayload: ArchivistInsertQuery = { schema: ArchivistInsertQuerySchema }
-        const query = await this.bindQuery(queryPayload, payloads)
-        return (await parent?.query(query[0], query[1]))?.[0]
-      }).filter(exists),
-    )
-    await this.clearHandler()
-    return settled.filter(fulfilled).map(result => result.value).filter(exists)
-  }
-
-  protected override async deleteHandler(hashes: Hash[]): Promise<WithStorageMeta<Payload>[]> {
-    // not using the getHandler since duplicate data hashes are not handled
-    const payloadsWithMeta = (await this.allHandler()).filter(({ _hash, _dataHash }) => hashes.includes(_hash) || hashes.includes(_dataHash))
-    // Delete the payloads
-    const batchCommands: Array<AbstractBatchOperation<AbstractPayloadSubLevel, Hash, WithStorageMeta<Payload>>> = payloadsWithMeta.map(payload => ({
-      type: 'del',
-      key: payload._hash,
-    }))
-
-    await this.withStore(async (store) => {
-      await store.batch(batchCommands)
-    })
-
-    // Delete the dataHash indexes
-    const batchDataHashIndexCommands: Array<AbstractBatchOperation<AbstractPayloadSubLevel, string, Hash>> = payloadsWithMeta.map(payload => ({
-      type: 'del',
-      key: payload._dataHash,
-    }))
-
-    await this.withDataHashIndex(async (index) => {
-      await index.batch(batchDataHashIndexCommands)
-    })
-
-    // Delete the sequence indexes
-    const batchSequenceIndexCommands: Array<AbstractBatchOperation<AbstractPayloadSubLevel, Hex, Hash>> = payloadsWithMeta.map(payload => ({
-      type: 'del',
-      key: payload._sequence,
-    }))
-
-    await this.withSequenceIndex(async (index) => {
-      await index.batch(batchSequenceIndexCommands)
-    })
-
-    return payloadsWithMeta
-  }
-
-  protected override async getHandler(hashes: Hash[]): Promise<WithStorageMeta<Payload>[]> {
-    const foundByHash = await this.withStore(async (store) => {
-      return (await store.getMany(hashes)).filter(exists)
-    })
-    const remainingHashes = hashes.filter(hash => !foundByHash.some(({ _hash }) => _hash === hash))
-    const hashesFromDataHashes = await this.withDataHashIndex(async (index) => {
-      return (await index.getMany(remainingHashes)).filter(exists)
-    })
-    const foundByDataHash = hashesFromDataHashes.length > 0
-      ? await this.withStore(async (store) => {
-          return (await store.getMany(hashesFromDataHashes)).filter(exists)
-        })
-      : []
-    const result = [...foundByHash, ...foundByDataHash].toSorted(PayloadBuilder.compareStorageMeta)
-    return result
-  }
-
-  protected override async insertHandler(payloads: WithStorageMeta<Payload>[]): Promise<WithStorageMeta<Payload>[]> {
-    // Insert the payloads
-    const payloadsWithMeta = payloads.toSorted(PayloadBuilder.compareStorageMeta)
-    const batchCommands: Array<AbstractBatchOperation<AbstractPayloadSubLevel, Hash, WithStorageMeta<Payload>>> = payloadsWithMeta.map(payload => ({
-      type: 'put', key: payload._hash, value: payload, keyEncoding: 'utf8', valueEncoding: 'json',
-    }))
-    await this.withStore(async (store) => {
-      await store.batch(batchCommands)
-    })
-
-    // Insert the dataHash indexes
-    // We use the dataHash|hash for the key to allow for multiple entries
-    const batchDataHashIndexCommands: Array<AbstractBatchOperation<AbstractPayloadLevel, string, Hash>> = payloadsWithMeta.map(payload => ({
-      type: 'put', key: payload._dataHash, value: payload._hash, keyEncoding: 'utf8', valueEncoding: 'utf8',
-    }))
-    await this.withDataHashIndex(async (index) => {
-      await index.batch(batchDataHashIndexCommands)
-    })
-
-    // Insert the sequence indexes
-    // We use the dataHash|hash for the key to allow for multiple entries
-    const batchSequenceIndexCommands: Array<AbstractBatchOperation<AbstractPayloadLevel, Hex, Hash>> = payloadsWithMeta.map(payload => ({
-      type: 'put', key: payload._sequence, value: payload._hash, keyEncoding: 'utf8', valueEncoding: 'utf8',
-    }))
-    await this.withSequenceIndex(async (index) => {
-      await index.batch(batchSequenceIndexCommands)
-    })
-
-    return payloadsWithMeta
-  }
-
-  protected override async nextHandler(options?: ArchivistNextOptions): Promise<WithStorageMeta<Payload>[]> {
-    const {
-      limit = 100, cursor, order, open = true,
-    } = options ?? {}
-    let all = await this.allHandler()
-    if (order === 'desc') {
-      all = all.toReversed()
-    }
-    const startIndex = isDefined(cursor)
-      ? AbstractLevelDbArchivist.findIndexFromCursor(all, cursor) + (open ? 1 : 0)
-      : 0
-    const result = all.slice(startIndex, startIndex + limit)
-    return result
-  }
-
-  protected override async startHandler() {
-    await super.startHandler()
-    // We could defer this creation to first access but we
-    // want to fail fast here in case something is wrong
-    await this.withStore(() => {})
-    if (this.config.clearStoreOnStart) {
-      await this.clearHandler()
-    }
-  }
-
-  protected withDataHashIndex<T>(func: (index: AbstractIndexSubLevel<string>) => Promisable<T>): Promisable<T> {
-    return this.withDb(async (db) => {
-      const index = db.sublevel<string, Hash>(
-        indexSubLevelName(this.storeName, AbstractLevelDbArchivist.dataHashIndexName),
-        { keyEncoding: 'utf8', valueEncoding: 'utf8' },
-      )
-      return await func(index)
-    })
-  }
-
-  protected withSequenceIndex<T>(func: (index: AbstractIndexSubLevel<Hex>) => Promisable<T>): Promisable<T> {
-    return this.withDb(async (db) => {
-      const index = db.sublevel<Hex, Hash>(
-        indexSubLevelName(this.storeName, AbstractLevelDbArchivist.sequenceIndexName),
-        { keyEncoding: 'utf8', valueEncoding: 'utf8' },
-      )
-      return await func(index)
-    })
-  }
-
-  protected async withStore<T>(func: (store: AbstractPayloadSubLevel) => Promisable<T>): Promise<T> {
-    return await this.withDb(async (db) => {
-      const subLevel: AbstractPayloadSubLevel = db.sublevel<Hash, WithStorageMeta<Payload>>(this.storeName, { keyEncoding: 'utf8', valueEncoding: 'json' })
-      return await func(subLevel)
-    })
-  }
-
-  protected abstract withDb<T>(func: (db: AbstractPayloadLevel) => Promisable<T>): Promisable<T>
-}
-
-@creatableModule()
-export class LevelDbArchivist extends AbstractLevelDbArchivist {
-  private _db: Level<Hash, WithStorageMeta<Payload>> | undefined
-  private _dbMutex = new Mutex()
-
-  protected override async stopHandler() {
-    await this._dbMutex.runExclusive(async () => {
-      await this._db?.close()
-      this._db = undefined
-    })
-    await super.stopHandler()
-  }
-
-  protected override async withDb<T>(func: (db: AbstractPayloadLevel) => Promisable<T>): Promise<T> {
-    const db = await this._dbMutex.runExclusive(async () => {
-      if (!this._db) {
-        this._db = new Level<Hash, WithStorageMeta<Payload>>(this.folderPath, { keyEncoding: 'utf8', valueEncoding: 'json' })
-        await this._db.open()
-      }
-      return this._db
-    })
-    return await func(db)
-  }
-}

package/src/Config.ts

@@ -1,27 +0,0 @@
-import type { ArchivistConfig } from '@xyo-network/archivist-model'
-import { asSchema } from '@xyo-network/payload-model'
-
-import { LevelDbArchivistSchema } from './Schema.ts'
-
-export const LevelDbArchivistConfigSchema = asSchema(`${LevelDbArchivistSchema}.config`, true)
-export type LevelDbArchivistConfigSchema = typeof LevelDbArchivistConfigSchema
-
-export type LevelDbArchivistConfig<TStoreName extends string = string> = ArchivistConfig<{
-  /**
-   * If true, the store will be cleared on start
-   */
-  clearStoreOnStart?: boolean
-  /**
-   * The database name - also used as the filename for the db
-   */
-  dbName?: string
-  /**
-   * The location where the folder for the db will be created
-   */
-  location?: string
-  schema: LevelDbArchivistConfigSchema
-  /**
-   * The name of the object store - becomes a sub-level
-   */
-  storeName?: TStoreName
-}>

package/src/Params.ts

@@ -1,8 +0,0 @@
-import type { ArchivistParams } from '@xyo-network/archivist-model'
-import type { AnyConfigSchema } from '@xyo-network/module-model'
-
-import type { LevelDbArchivistConfig } from './Config.ts'
-
-export interface LevelDbArchivistParams extends ArchivistParams<AnyConfigSchema<LevelDbArchivistConfig>> {
-
-}

package/src/Schema.ts

@@ -1,4 +0,0 @@
-import { asSchema } from '@xyo-network/payload-model'
-
-export const LevelDbArchivistSchema = asSchema('network.xyo.archivist.leveldb', true)
-export type LevelDbArchivistSchema = typeof LevelDbArchivistSchema

package/src/index.ts

@@ -1,4 +0,0 @@
-export * from './Archivist.ts'
-export * from './Config.ts'
-export * from './Params.ts'
-export * from './Schema.ts'