@xyo-network/archivist-leveldb 5.1.23 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -6,7 +6,7 @@ import { BoundWitness } from '@xyo-network/boundwitness-model';
|
|
|
6
6
|
import { Payload, Schema, WithStorageMeta } from '@xyo-network/payload-model';
|
|
7
7
|
import { AbstractLevel, AbstractSublevel } from 'abstract-level';
|
|
8
8
|
import { LevelDbArchivistParams } from './Params.ts';
|
|
9
|
-
/**
|
|
9
|
+
/** We have indexes as top level sublevels since making them a sublevel of a store, getting all the values of that store includes the sublevels */
|
|
10
10
|
export interface PayloadStore {
|
|
11
11
|
[s: string]: WithStorageMeta;
|
|
12
12
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Archivist.d.ts","sourceRoot":"","sources":["../../src/Archivist.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,aAAa,CAAA;AACvC,OAAO,EAAa,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAEvD,OAAO,EAAE,iBAAiB,EAAqB,MAAM,iCAAiC,CAAA;AACtF,OAAO,EAOL,wBAAwB,EACxB,oBAAoB,EAIrB,MAAM,8BAA8B,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AAG9D,OAAO,EACL,OAAO,EAAE,MAAM,EAAE,eAAe,EACjC,MAAM,4BAA4B,CAAA;AACnC,OAAO,EACmB,aAAa,EAAE,gBAAgB,EACxD,MAAM,gBAAgB,CAAA;AAKvB,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAA;AAEpD,
|
|
1
|
+
{"version":3,"file":"Archivist.d.ts","sourceRoot":"","sources":["../../src/Archivist.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,aAAa,CAAA;AACvC,OAAO,EAAa,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAEvD,OAAO,EAAE,iBAAiB,EAAqB,MAAM,iCAAiC,CAAA;AACtF,OAAO,EAOL,wBAAwB,EACxB,oBAAoB,EAIrB,MAAM,8BAA8B,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AAG9D,OAAO,EACL,OAAO,EAAE,MAAM,EAAE,eAAe,EACjC,MAAM,4BAA4B,CAAA;AACnC,OAAO,EACmB,aAAa,EAAE,gBAAgB,EACxD,MAAM,gBAAgB,CAAA;AAKvB,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAA;AAEpD,mJAAmJ;AAEnJ,MAAM,WAAW,YAAY;IAC3B,CAAC,CAAC,EAAE,MAAM,GAAG,eAAe,CAAA;CAC7B;AAED,MAAM,MAAM,oBAAoB,GAAG,aAAa,CAAC,MAAM,GAAG,MAAM,GAAG,UAAU,EAAE,IAAI,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC,CAAA;AAC9G,MAAM,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,oBAAoB,EAAE,MAAM,GAAG,MAAM,GAAG,UAAU,EAAE,IAAI,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC,CAAA;AAC1I,MAAM,MAAM,qBAAqB,CAAC,CAAC,IAAI,gBAAgB,CAAC,oBAAoB,EAAE,MAAM,GAAG,MAAM,GAAG,UAAU,EAAE,CAAC,EAAE,IAAI,CAAC,CAAA;AAMpH,8BAAsB,wBAAwB,CAC5C,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,EAC/D,UAAU,SAAS,wBAAwB,GAAG,wBAAwB,CACtE,SAAQ,iBAAiB,CAAC,OAAO,EAAE,UAAU,CAAC;IAC9C,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAyD;IACzG,gBAAyB,mBAAmB,EAAE,MAAM,CAA+B;IACnF,gBAAyB,MAAM;;MAAmD;IAElF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAEpC;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAEpC;IAGD,MAAM,CAAC,QAAQ,CAAC,iBAAiB,SAAiE;IAElG,MAAM,CAAC,QAAQ,CAAC,iBAAiB,SAAiE;IAElG,IAAI,MAAM,WAET;IAED,IAAI,UAAU,WAEb;IAED,IAAI,QAAQ,WAEX;IAED,IAAa,OAAO,aAUnB;IAED,IAAI,SAAS,WAEZ;IAED,OAAO,CAAC,MAAM,CAAC,mBAAmB;cAQT,UAAU,IAAI,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cAOjD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;cAa7B,aAAa,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;cAaxC,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cAoClE,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cAiB/D,aAAa,CAAC,QAAQ,EAAE,eAAe,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cA+BxF,WAAW,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cAehF,YAAY;IAUrC,SAAS,CAAC,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,qBAAqB,CAAC,MAAM,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC;IAU5G,SAAS,CAAC,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,qBAAqB,CAAC,GAAG,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC;cAUzF,SAAS,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,uBAAuB,KAAK,UAAU,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAOjG,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,oBAAoB,KAAK,UAAU,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC;CAC/F;AAED,qBACa,gBAAiB,SAAQ,wBAAwB;IAC5D,OAAO,CAAC,GAAG,CAAmD;IAC9D,OAAO,CAAC,QAAQ,CAAc;cAEL,WAAW;cAQX,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,oBAAoB,KAAK,UAAU,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;CAUlG"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/Archivist.ts","../../src/Schema.ts","../../src/Config.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { Hash, Hex } from '@xylabs/hex'\nimport { fulfilled, Promisable } from '@xylabs/promise'\nimport { isDefined } from '@xylabs/typeof'\nimport { AbstractArchivist, StorageClassLabel } from '@xyo-network/archivist-abstract'\nimport {\n ArchivistAllQuerySchema,\n ArchivistClearQuerySchema,\n ArchivistCommitQuerySchema,\n ArchivistDeleteQuerySchema,\n ArchivistInsertQuery,\n ArchivistInsertQuerySchema,\n ArchivistModuleEventData,\n ArchivistNextOptions,\n ArchivistNextQuerySchema,\n buildStandardIndexName,\n IndexDescription,\n} from '@xyo-network/archivist-model'\nimport { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { creatableModule } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport {\n Payload, Schema, WithStorageMeta,\n} from '@xyo-network/payload-model'\nimport {\n AbstractBatchOperation, AbstractLevel, AbstractSublevel,\n} from 'abstract-level'\nimport { Mutex } from 'async-mutex'\nimport { Level } from 'level'\n\nimport { LevelDbArchivistConfigSchema } from './Config.ts'\nimport { LevelDbArchivistParams } from './Params.ts'\n\n/** Note: We have indexes as top level sublevels since making them a sublevel of a store, getting all the values of that store includes the sublevels */\n\nexport interface PayloadStore {\n [s: string]: WithStorageMeta\n}\n\nexport type AbstractPayloadLevel = AbstractLevel<string | Buffer | Uint8Array, Hash, WithStorageMeta<Payload>>\nexport type AbstractPayloadSubLevel = AbstractSublevel<AbstractPayloadLevel, string | Buffer | Uint8Array, Hash, WithStorageMeta<Payload>>\nexport type AbstractIndexSubLevel<T> = AbstractSublevel<AbstractPayloadLevel, string | Buffer | Uint8Array, T, Hash>\n\nconst indexSubLevelName = (storeName: string, indexName: string) => {\n return `_${storeName}|${indexName}`\n}\n\nexport abstract class AbstractLevelDbArchivist<\n TParams extends LevelDbArchivistParams = LevelDbArchivistParams,\n TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,\n> extends AbstractArchivist<TParams, TEventData> {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, LevelDbArchivistConfigSchema]\n static override readonly defaultConfigSchema: Schema = LevelDbArchivistConfigSchema\n static override readonly labels = { ...super.labels, [StorageClassLabel]: 'disk' }\n\n private static readonly dataHashIndex: IndexDescription = {\n key: { _dataHash: 1 }, multiEntry: false, unique: false,\n }\n\n private static readonly sequenceIndex: IndexDescription = {\n key: { _sequence: 1 }, multiEntry: false, unique: true,\n }\n\n // eslint-disable-next-line @typescript-eslint/member-ordering\n static readonly dataHashIndexName = buildStandardIndexName(AbstractLevelDbArchivist.dataHashIndex)\n // eslint-disable-next-line @typescript-eslint/member-ordering\n static readonly sequenceIndexName = buildStandardIndexName(AbstractLevelDbArchivist.sequenceIndex)\n\n get dbName() {\n return assertEx(this.config.dbName, () => 'No dbName specified')\n }\n\n get folderPath() {\n return `${this.location}/${this.config.dbName}/${this.storeName}`\n }\n\n get location() {\n return assertEx(this.config.location, () => 'No location specified')\n }\n\n override get queries() {\n return [\n ArchivistAllQuerySchema,\n ArchivistDeleteQuerySchema,\n ArchivistClearQuerySchema,\n ArchivistInsertQuerySchema,\n ArchivistCommitQuerySchema,\n ArchivistNextQuerySchema,\n ...super.queries,\n ]\n }\n\n get storeName() {\n return assertEx(this.config.storeName, () => 'No storeName specified')\n }\n\n private static findIndexFromCursor(payloads: WithStorageMeta[], cursor: Hex) {\n const index = payloads.findIndex(({ _sequence }) => _sequence === cursor)\n if (index === -1) {\n return Infinity // move to the end\n }\n return index\n }\n\n protected override async allHandler(): Promise<WithStorageMeta<Payload>[]> {\n return await this.withStore(async (db) => {\n const values = [...(await db.values().all())]\n return values.filter(exists).toSorted(PayloadBuilder.compareStorageMeta)\n })\n }\n\n protected override async clearHandler(): Promise<void> {\n await this.withDb(async (db) => {\n await db.clear()\n })\n await this.withDataHashIndex(async (index) => {\n await index.clear()\n })\n await this.withSequenceIndex(async (index) => {\n await index.clear()\n })\n return this.emit('cleared', { mod: this })\n }\n\n protected override async commitHandler(): Promise<BoundWitness[]> {\n const payloads = assertEx(await this.allHandler(), () => 'Nothing to commit')\n const settled = await Promise.allSettled(\n Object.values((await this.parentArchivists()).commit ?? [])?.map(async (parent) => {\n const queryPayload: ArchivistInsertQuery = { schema: ArchivistInsertQuerySchema }\n const query = await this.bindQuery(queryPayload, payloads)\n return (await parent?.query(query[0], query[1]))?.[0]\n }).filter(exists),\n )\n await this.clearHandler()\n return settled.filter(fulfilled).map(result => result.value).filter(exists)\n }\n\n protected override async deleteHandler(hashes: Hash[]): Promise<WithStorageMeta<Payload>[]> {\n // not using the getHandler since duplicate data hashes are not handled\n const payloadsWithMeta = (await this.allHandler()).filter(({ _hash, _dataHash }) => hashes.includes(_hash) || hashes.includes(_dataHash))\n // Delete the payloads\n const batchCommands: Array<AbstractBatchOperation<AbstractPayloadSubLevel, Hash, WithStorageMeta<Payload>>> = payloadsWithMeta.map(payload => ({\n type: 'del',\n key: payload._hash,\n }))\n\n await this.withStore(async (store) => {\n await store.batch(batchCommands)\n })\n\n // Delete the dataHash indexes\n const batchDataHashIndexCommands: Array<AbstractBatchOperation<AbstractPayloadSubLevel, string, Hash>> = payloadsWithMeta.map(payload => ({\n type: 'del',\n key: payload._dataHash,\n }))\n\n await this.withDataHashIndex(async (index) => {\n await index.batch(batchDataHashIndexCommands)\n })\n\n // Delete the sequence indexes\n const batchSequenceIndexCommands: Array<AbstractBatchOperation<AbstractPayloadSubLevel, Hex, Hash>> = payloadsWithMeta.map(payload => ({\n type: 'del',\n key: payload._sequence,\n }))\n\n await this.withSequenceIndex(async (index) => {\n await index.batch(batchSequenceIndexCommands)\n })\n\n return payloadsWithMeta\n }\n\n protected override async getHandler(hashes: Hash[]): Promise<WithStorageMeta<Payload>[]> {\n const foundByHash = await this.withStore(async (store) => {\n return (await store.getMany(hashes)).filter(exists)\n })\n const remainingHashes = hashes.filter(hash => !foundByHash.some(({ _hash }) => _hash === hash))\n const hashesFromDataHashes = await this.withDataHashIndex(async (index) => {\n return (await index.getMany(remainingHashes)).filter(exists)\n })\n const foundByDataHash = hashesFromDataHashes.length > 0\n ? await this.withStore(async (store) => {\n return (await store.getMany(hashesFromDataHashes)).filter(exists)\n })\n : []\n const result = [...foundByHash, ...foundByDataHash].toSorted(PayloadBuilder.compareStorageMeta)\n return result\n }\n\n protected override async insertHandler(payloads: WithStorageMeta<Payload>[]): Promise<WithStorageMeta<Payload>[]> {\n // Insert the payloads\n const payloadsWithMeta = payloads.toSorted(PayloadBuilder.compareStorageMeta)\n const batchCommands: Array<AbstractBatchOperation<AbstractPayloadSubLevel, Hash, WithStorageMeta<Payload>>> = payloadsWithMeta.map(payload => ({\n type: 'put', key: payload._hash, value: payload, keyEncoding: 'utf8', valueEncoding: 'json',\n }))\n await this.withStore(async (store) => {\n await store.batch(batchCommands)\n })\n\n // Insert the dataHash indexes\n // Note: We use the dataHash|hash for the key to allow for multiple entries\n const batchDataHashIndexCommands: Array<AbstractBatchOperation<AbstractPayloadLevel, string, Hash>> = payloadsWithMeta.map(payload => ({\n type: 'put', key: payload._dataHash, value: payload._hash, keyEncoding: 'utf8', valueEncoding: 'utf8',\n }))\n await this.withDataHashIndex(async (index) => {\n await index.batch(batchDataHashIndexCommands)\n })\n\n // Insert the sequence indexes\n // Note: We use the dataHash|hash for the key to allow for multiple entries\n const batchSequenceIndexCommands: Array<AbstractBatchOperation<AbstractPayloadLevel, Hex, Hash>> = payloadsWithMeta.map(payload => ({\n type: 'put', key: payload._sequence, value: payload._hash, keyEncoding: 'utf8', valueEncoding: 'utf8',\n }))\n await this.withSequenceIndex(async (index) => {\n await index.batch(batchSequenceIndexCommands)\n })\n\n return payloadsWithMeta\n }\n\n protected override async nextHandler(options?: ArchivistNextOptions): Promise<WithStorageMeta<Payload>[]> {\n const {\n limit = 100, cursor, order, open = true,\n } = options ?? {}\n let all = await this.allHandler()\n if (order === 'desc') {\n all = all.toReversed()\n }\n const startIndex = isDefined(cursor)\n ? AbstractLevelDbArchivist.findIndexFromCursor(all, cursor) + (open ? 1 : 0)\n : 0\n const result = all.slice(startIndex, startIndex + limit)\n return result\n }\n\n protected override async startHandler() {\n await super.startHandler()\n // NOTE: We could defer this creation to first access but we\n // want to fail fast here in case something is wrong\n await this.withStore(() => {})\n if (this.config.clearStoreOnStart) {\n await this.clearHandler()\n }\n }\n\n protected withDataHashIndex<T>(func: (index: AbstractIndexSubLevel<string>) => Promisable<T>): Promisable<T> {\n return this.withDb(async (db) => {\n const index = db.sublevel<string, Hash>(\n indexSubLevelName(this.storeName, AbstractLevelDbArchivist.dataHashIndexName),\n { keyEncoding: 'utf8', valueEncoding: 'utf8' },\n )\n return await func(index)\n })\n }\n\n protected withSequenceIndex<T>(func: (index: AbstractIndexSubLevel<Hex>) => Promisable<T>): Promisable<T> {\n return this.withDb(async (db) => {\n const index = db.sublevel<Hex, Hash>(\n indexSubLevelName(this.storeName, AbstractLevelDbArchivist.sequenceIndexName),\n { keyEncoding: 'utf8', valueEncoding: 'utf8' },\n )\n return await func(index)\n })\n }\n\n protected async withStore<T>(func: (store: AbstractPayloadSubLevel) => Promisable<T>): Promise<T> {\n return await this.withDb(async (db) => {\n const subLevel: AbstractPayloadSubLevel = db.sublevel<Hash, WithStorageMeta<Payload>>(this.storeName, { keyEncoding: 'utf8', valueEncoding: 'json' })\n return await func(subLevel)\n })\n }\n\n protected abstract withDb<T>(func: (db: AbstractPayloadLevel) => Promisable<T>): Promisable<T>\n}\n\n@creatableModule()\nexport class LevelDbArchivist extends AbstractLevelDbArchivist {\n private _db: Level<Hash, WithStorageMeta<Payload>> | undefined\n private _dbMutex = new Mutex()\n\n protected override async stopHandler() {\n await this._dbMutex.runExclusive(async () => {\n await this._db?.close()\n this._db = undefined\n })\n await super.stopHandler()\n }\n\n protected override async withDb<T>(func: (db: AbstractPayloadLevel) => Promisable<T>): Promise<T> {\n const db = await this._dbMutex.runExclusive(async () => {\n if (!this._db) {\n this._db = new Level<Hash, WithStorageMeta<Payload>>(this.folderPath, { keyEncoding: 'utf8', valueEncoding: 'json' })\n await this._db.open()\n }\n return this._db\n })\n return await func(db)\n }\n}\n","export const LevelDbArchivistSchema = 'network.xyo.archivist.leveldb' as const\nexport type LevelDbArchivistSchema = typeof LevelDbArchivistSchema\n","import type { ArchivistConfig } from '@xyo-network/archivist-model'\n\nimport { LevelDbArchivistSchema } from './Schema.ts'\n\nexport type LevelDbArchivistConfigSchema = `${LevelDbArchivistSchema}.config`\nexport const LevelDbArchivistConfigSchema: LevelDbArchivistConfigSchema = `${LevelDbArchivistSchema}.config`\n\nexport type LevelDbArchivistConfig<TStoreName extends string = string> = ArchivistConfig<{\n /**\n * If true, the store will be cleared on start\n */\n clearStoreOnStart?: boolean\n /**\n * The database name - also used as the filename for the db\n */\n dbName?: string\n /**\n * The location where the folder for the db will be created\n */\n location?: string\n schema: LevelDbArchivistConfigSchema\n /**\n * The name of the object store - becomes a sub-level\n */\n storeName?: TStoreName\n}>\n"],"mappings":";;;;;;;;;;;;AAAA,SAAS,gBAAgB;AACzB,SAAS,cAAc;AAEvB,SAAS,iBAA6B;AACtC,SAAS,iBAAiB;AAC1B,SAAS,mBAAmB,yBAAyB;AACrD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EAGA;AAAA,EACA;AAAA,OAEK;AAEP,SAAS,uBAAuB;AAChC,SAAS,sBAAsB;AAO/B,SAAS,aAAa;AACtB,SAAS,aAAa;;;AC7Bf,IAAM,yBAAyB;;;ACK/B,IAAM,+BAA6D,GAAG,sBAAsB;;;AFuCnG,IAAM,oBAAoB,CAAC,WAAmB,cAAsB;AAClE,SAAO,IAAI,SAAS,IAAI,SAAS;AACnC;AAEO,IAAe,2BAAf,MAAe,kCAGZ,kBAAuC;AAAA,EAC/C,OAAyB,gBAA0B,CAAC,GAAG,MAAM,eAAe,4BAA4B;AAAA,EACxG,OAAyB,sBAA8B;AAAA,EACvD,OAAyB,SAAS,EAAE,GAAG,MAAM,QAAQ,CAAC,iBAAiB,GAAG,OAAO;AAAA,EAEjF,OAAwB,gBAAkC;AAAA,IACxD,KAAK,EAAE,WAAW,EAAE;AAAA,IAAG,YAAY;AAAA,IAAO,QAAQ;AAAA,EACpD;AAAA,EAEA,OAAwB,gBAAkC;AAAA,IACxD,KAAK,EAAE,WAAW,EAAE;AAAA,IAAG,YAAY;AAAA,IAAO,QAAQ;AAAA,EACpD;AAAA;AAAA,EAGA,OAAgB,oBAAoB,uBAAuB,0BAAyB,aAAa;AAAA;AAAA,EAEjG,OAAgB,oBAAoB,uBAAuB,0BAAyB,aAAa;AAAA,EAEjG,IAAI,SAAS;AACX,WAAO,SAAS,KAAK,OAAO,QAAQ,MAAM,qBAAqB;AAAA,EACjE;AAAA,EAEA,IAAI,aAAa;AACf,WAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,OAAO,MAAM,IAAI,KAAK,SAAS;AAAA,EACjE;AAAA,EAEA,IAAI,WAAW;AACb,WAAO,SAAS,KAAK,OAAO,UAAU,MAAM,uBAAuB;AAAA,EACrE;AAAA,EAEA,IAAa,UAAU;AACrB,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,GAAG,MAAM;AAAA,IACX;AAAA,EACF;AAAA,EAEA,IAAI,YAAY;AACd,WAAO,SAAS,KAAK,OAAO,WAAW,MAAM,wBAAwB;AAAA,EACvE;AAAA,EAEA,OAAe,oBAAoB,UAA6B,QAAa;AAC3E,UAAM,QAAQ,SAAS,UAAU,CAAC,EAAE,UAAU,MAAM,cAAc,MAAM;AACxE,QAAI,UAAU,IAAI;AAChB,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,aAAkD;AACzE,WAAO,MAAM,KAAK,UAAU,OAAO,OAAO;AACxC,YAAM,SAAS,CAAC,GAAI,MAAM,GAAG,OAAO,EAAE,IAAI,CAAE;AAC5C,aAAO,OAAO,OAAO,MAAM,EAAE,SAAS,eAAe,kBAAkB;AAAA,IACzE,CAAC;AAAA,EACH;AAAA,EAEA,MAAyB,eAA8B;AACrD,UAAM,KAAK,OAAO,OAAO,OAAO;AAC9B,YAAM,GAAG,MAAM;AAAA,IACjB,CAAC;AACD,UAAM,KAAK,kBAAkB,OAAO,UAAU;AAC5C,YAAM,MAAM,MAAM;AAAA,IACpB,CAAC;AACD,UAAM,KAAK,kBAAkB,OAAO,UAAU;AAC5C,YAAM,MAAM,MAAM;AAAA,IACpB,CAAC;AACD,WAAO,KAAK,KAAK,WAAW,EAAE,KAAK,KAAK,CAAC;AAAA,EAC3C;AAAA,EAEA,MAAyB,gBAAyC;AAChE,UAAM,WAAW,SAAS,MAAM,KAAK,WAAW,GAAG,MAAM,mBAAmB;AAC5E,UAAM,UAAU,MAAM,QAAQ;AAAA,MAC5B,OAAO,QAAQ,MAAM,KAAK,iBAAiB,GAAG,UAAU,CAAC,CAAC,GAAG,IAAI,OAAO,WAAW;AACjF,cAAM,eAAqC,EAAE,QAAQ,2BAA2B;AAChF,cAAM,QAAQ,MAAM,KAAK,UAAU,cAAc,QAAQ;AACzD,gBAAQ,MAAM,QAAQ,MAAM,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC;AAAA,MACtD,CAAC,EAAE,OAAO,MAAM;AAAA,IAClB;AACA,UAAM,KAAK,aAAa;AACxB,WAAO,QAAQ,OAAO,SAAS,EAAE,IAAI,YAAU,OAAO,KAAK,EAAE,OAAO,MAAM;AAAA,EAC5E;AAAA,EAEA,MAAyB,cAAc,QAAqD;AAE1F,UAAM,oBAAoB,MAAM,KAAK,WAAW,GAAG,OAAO,CAAC,EAAE,OAAO,UAAU,MAAM,OAAO,SAAS,KAAK,KAAK,OAAO,SAAS,SAAS,CAAC;AAExI,UAAM,gBAAwG,iBAAiB,IAAI,cAAY;AAAA,MAC7I,MAAM;AAAA,MACN,KAAK,QAAQ;AAAA,IACf,EAAE;AAEF,UAAM,KAAK,UAAU,OAAO,UAAU;AACpC,YAAM,MAAM,MAAM,aAAa;AAAA,IACjC,CAAC;AAGD,UAAM,6BAAmG,iBAAiB,IAAI,cAAY;AAAA,MACxI,MAAM;AAAA,MACN,KAAK,QAAQ;AAAA,IACf,EAAE;AAEF,UAAM,KAAK,kBAAkB,OAAO,UAAU;AAC5C,YAAM,MAAM,MAAM,0BAA0B;AAAA,IAC9C,CAAC;AAGD,UAAM,6BAAgG,iBAAiB,IAAI,cAAY;AAAA,MACrI,MAAM;AAAA,MACN,KAAK,QAAQ;AAAA,IACf,EAAE;AAEF,UAAM,KAAK,kBAAkB,OAAO,UAAU;AAC5C,YAAM,MAAM,MAAM,0BAA0B;AAAA,IAC9C,CAAC;AAED,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,WAAW,QAAqD;AACvF,UAAM,cAAc,MAAM,KAAK,UAAU,OAAO,UAAU;AACxD,cAAQ,MAAM,MAAM,QAAQ,MAAM,GAAG,OAAO,MAAM;AAAA,IACpD,CAAC;AACD,UAAM,kBAAkB,OAAO,OAAO,UAAQ,CAAC,YAAY,KAAK,CAAC,EAAE,MAAM,MAAM,UAAU,IAAI,CAAC;AAC9F,UAAM,uBAAuB,MAAM,KAAK,kBAAkB,OAAO,UAAU;AACzE,cAAQ,MAAM,MAAM,QAAQ,eAAe,GAAG,OAAO,MAAM;AAAA,IAC7D,CAAC;AACD,UAAM,kBAAkB,qBAAqB,SAAS,IAClD,MAAM,KAAK,UAAU,OAAO,UAAU;AACpC,cAAQ,MAAM,MAAM,QAAQ,oBAAoB,GAAG,OAAO,MAAM;AAAA,IAClE,CAAC,IACD,CAAC;AACL,UAAM,SAAS,CAAC,GAAG,aAAa,GAAG,eAAe,EAAE,SAAS,eAAe,kBAAkB;AAC9F,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,cAAc,UAA2E;AAEhH,UAAM,mBAAmB,SAAS,SAAS,eAAe,kBAAkB;AAC5E,UAAM,gBAAwG,iBAAiB,IAAI,cAAY;AAAA,MAC7I,MAAM;AAAA,MAAO,KAAK,QAAQ;AAAA,MAAO,OAAO;AAAA,MAAS,aAAa;AAAA,MAAQ,eAAe;AAAA,IACvF,EAAE;AACF,UAAM,KAAK,UAAU,OAAO,UAAU;AACpC,YAAM,MAAM,MAAM,aAAa;AAAA,IACjC,CAAC;AAID,UAAM,6BAAgG,iBAAiB,IAAI,cAAY;AAAA,MACrI,MAAM;AAAA,MAAO,KAAK,QAAQ;AAAA,MAAW,OAAO,QAAQ;AAAA,MAAO,aAAa;AAAA,MAAQ,eAAe;AAAA,IACjG,EAAE;AACF,UAAM,KAAK,kBAAkB,OAAO,UAAU;AAC5C,YAAM,MAAM,MAAM,0BAA0B;AAAA,IAC9C,CAAC;AAID,UAAM,6BAA6F,iBAAiB,IAAI,cAAY;AAAA,MAClI,MAAM;AAAA,MAAO,KAAK,QAAQ;AAAA,MAAW,OAAO,QAAQ;AAAA,MAAO,aAAa;AAAA,MAAQ,eAAe;AAAA,IACjG,EAAE;AACF,UAAM,KAAK,kBAAkB,OAAO,UAAU;AAC5C,YAAM,MAAM,MAAM,0BAA0B;AAAA,IAC9C,CAAC;AAED,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,SAAqE;AACxG,UAAM;AAAA,MACJ,QAAQ;AAAA,MAAK;AAAA,MAAQ;AAAA,MAAO,OAAO;AAAA,IACrC,IAAI,WAAW,CAAC;AAChB,QAAI,MAAM,MAAM,KAAK,WAAW;AAChC,QAAI,UAAU,QAAQ;AACpB,YAAM,IAAI,WAAW;AAAA,IACvB;AACA,UAAM,aAAa,UAAU,MAAM,IAC/B,0BAAyB,oBAAoB,KAAK,MAAM,KAAK,OAAO,IAAI,KACxE;AACJ,UAAM,SAAS,IAAI,MAAM,YAAY,aAAa,KAAK;AACvD,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAe;AACtC,UAAM,MAAM,aAAa;AAGzB,UAAM,KAAK,UAAU,MAAM;AAAA,IAAC,CAAC;AAC7B,QAAI,KAAK,OAAO,mBAAmB;AACjC,YAAM,KAAK,aAAa;AAAA,IAC1B;AAAA,EACF;AAAA,EAEU,kBAAqB,MAA8E;AAC3G,WAAO,KAAK,OAAO,OAAO,OAAO;AAC/B,YAAM,QAAQ,GAAG;AAAA,QACf,kBAAkB,KAAK,WAAW,0BAAyB,iBAAiB;AAAA,QAC5E,EAAE,aAAa,QAAQ,eAAe,OAAO;AAAA,MAC/C;AACA,aAAO,MAAM,KAAK,KAAK;AAAA,IACzB,CAAC;AAAA,EACH;AAAA,EAEU,kBAAqB,MAA2E;AACxG,WAAO,KAAK,OAAO,OAAO,OAAO;AAC/B,YAAM,QAAQ,GAAG;AAAA,QACf,kBAAkB,KAAK,WAAW,0BAAyB,iBAAiB;AAAA,QAC5E,EAAE,aAAa,QAAQ,eAAe,OAAO;AAAA,MAC/C;AACA,aAAO,MAAM,KAAK,KAAK;AAAA,IACzB,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,UAAa,MAAqE;AAChG,WAAO,MAAM,KAAK,OAAO,OAAO,OAAO;AACrC,YAAM,WAAoC,GAAG,SAAyC,KAAK,WAAW,EAAE,aAAa,QAAQ,eAAe,OAAO,CAAC;AACpJ,aAAO,MAAM,KAAK,QAAQ;AAAA,IAC5B,CAAC;AAAA,EACH;AAGF;AAGO,IAAM,mBAAN,cAA+B,yBAAyB;AAAA,EACrD;AAAA,EACA,WAAW,IAAI,MAAM;AAAA,EAE7B,MAAyB,cAAc;AACrC,UAAM,KAAK,SAAS,aAAa,YAAY;AAC3C,YAAM,KAAK,KAAK,MAAM;AACtB,WAAK,MAAM;AAAA,IACb,CAAC;AACD,UAAM,MAAM,YAAY;AAAA,EAC1B;AAAA,EAEA,MAAyB,OAAU,MAA+D;AAChG,UAAM,KAAK,MAAM,KAAK,SAAS,aAAa,YAAY;AACtD,UAAI,CAAC,KAAK,KAAK;AACb,aAAK,MAAM,IAAI,MAAsC,KAAK,YAAY,EAAE,aAAa,QAAQ,eAAe,OAAO,CAAC;AACpH,cAAM,KAAK,IAAI,KAAK;AAAA,MACtB;AACA,aAAO,KAAK;AAAA,IACd,CAAC;AACD,WAAO,MAAM,KAAK,EAAE;AAAA,EACtB;AACF;AAtBa,mBAAN;AAAA,EADN,gBAAgB;AAAA,GACJ;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../src/Archivist.ts","../../src/Schema.ts","../../src/Config.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { Hash, Hex } from '@xylabs/hex'\nimport { fulfilled, Promisable } from '@xylabs/promise'\nimport { isDefined } from '@xylabs/typeof'\nimport { AbstractArchivist, StorageClassLabel } from '@xyo-network/archivist-abstract'\nimport {\n ArchivistAllQuerySchema,\n ArchivistClearQuerySchema,\n ArchivistCommitQuerySchema,\n ArchivistDeleteQuerySchema,\n ArchivistInsertQuery,\n ArchivistInsertQuerySchema,\n ArchivistModuleEventData,\n ArchivistNextOptions,\n ArchivistNextQuerySchema,\n buildStandardIndexName,\n IndexDescription,\n} from '@xyo-network/archivist-model'\nimport { BoundWitness } from '@xyo-network/boundwitness-model'\nimport { creatableModule } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport {\n Payload, Schema, WithStorageMeta,\n} from '@xyo-network/payload-model'\nimport {\n AbstractBatchOperation, AbstractLevel, AbstractSublevel,\n} from 'abstract-level'\nimport { Mutex } from 'async-mutex'\nimport { Level } from 'level'\n\nimport { LevelDbArchivistConfigSchema } from './Config.ts'\nimport { LevelDbArchivistParams } from './Params.ts'\n\n/** We have indexes as top level sublevels since making them a sublevel of a store, getting all the values of that store includes the sublevels */\n\nexport interface PayloadStore {\n [s: string]: WithStorageMeta\n}\n\nexport type AbstractPayloadLevel = AbstractLevel<string | Buffer | Uint8Array, Hash, WithStorageMeta<Payload>>\nexport type AbstractPayloadSubLevel = AbstractSublevel<AbstractPayloadLevel, string | Buffer | Uint8Array, Hash, WithStorageMeta<Payload>>\nexport type AbstractIndexSubLevel<T> = AbstractSublevel<AbstractPayloadLevel, string | Buffer | Uint8Array, T, Hash>\n\nconst indexSubLevelName = (storeName: string, indexName: string) => {\n return `_${storeName}|${indexName}`\n}\n\nexport abstract class AbstractLevelDbArchivist<\n TParams extends LevelDbArchivistParams = LevelDbArchivistParams,\n TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,\n> extends AbstractArchivist<TParams, TEventData> {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, LevelDbArchivistConfigSchema]\n static override readonly defaultConfigSchema: Schema = LevelDbArchivistConfigSchema\n static override readonly labels = { ...super.labels, [StorageClassLabel]: 'disk' }\n\n private static readonly dataHashIndex: IndexDescription = {\n key: { _dataHash: 1 }, multiEntry: false, unique: false,\n }\n\n private static readonly sequenceIndex: IndexDescription = {\n key: { _sequence: 1 }, multiEntry: false, unique: true,\n }\n\n // eslint-disable-next-line @typescript-eslint/member-ordering\n static readonly dataHashIndexName = buildStandardIndexName(AbstractLevelDbArchivist.dataHashIndex)\n // eslint-disable-next-line @typescript-eslint/member-ordering\n static readonly sequenceIndexName = buildStandardIndexName(AbstractLevelDbArchivist.sequenceIndex)\n\n get dbName() {\n return assertEx(this.config.dbName, () => 'No dbName specified')\n }\n\n get folderPath() {\n return `${this.location}/${this.config.dbName}/${this.storeName}`\n }\n\n get location() {\n return assertEx(this.config.location, () => 'No location specified')\n }\n\n override get queries() {\n return [\n ArchivistAllQuerySchema,\n ArchivistDeleteQuerySchema,\n ArchivistClearQuerySchema,\n ArchivistInsertQuerySchema,\n ArchivistCommitQuerySchema,\n ArchivistNextQuerySchema,\n ...super.queries,\n ]\n }\n\n get storeName() {\n return assertEx(this.config.storeName, () => 'No storeName specified')\n }\n\n private static findIndexFromCursor(payloads: WithStorageMeta[], cursor: Hex) {\n const index = payloads.findIndex(({ _sequence }) => _sequence === cursor)\n if (index === -1) {\n return Infinity // move to the end\n }\n return index\n }\n\n protected override async allHandler(): Promise<WithStorageMeta<Payload>[]> {\n return await this.withStore(async (db) => {\n const values = [...(await db.values().all())]\n return values.filter(exists).toSorted(PayloadBuilder.compareStorageMeta)\n })\n }\n\n protected override async clearHandler(): Promise<void> {\n await this.withDb(async (db) => {\n await db.clear()\n })\n await this.withDataHashIndex(async (index) => {\n await index.clear()\n })\n await this.withSequenceIndex(async (index) => {\n await index.clear()\n })\n return this.emit('cleared', { mod: this })\n }\n\n protected override async commitHandler(): Promise<BoundWitness[]> {\n const payloads = assertEx(await this.allHandler(), () => 'Nothing to commit')\n const settled = await Promise.allSettled(\n Object.values((await this.parentArchivists()).commit ?? [])?.map(async (parent) => {\n const queryPayload: ArchivistInsertQuery = { schema: ArchivistInsertQuerySchema }\n const query = await this.bindQuery(queryPayload, payloads)\n return (await parent?.query(query[0], query[1]))?.[0]\n }).filter(exists),\n )\n await this.clearHandler()\n return settled.filter(fulfilled).map(result => result.value).filter(exists)\n }\n\n protected override async deleteHandler(hashes: Hash[]): Promise<WithStorageMeta<Payload>[]> {\n // not using the getHandler since duplicate data hashes are not handled\n const payloadsWithMeta = (await this.allHandler()).filter(({ _hash, _dataHash }) => hashes.includes(_hash) || hashes.includes(_dataHash))\n // Delete the payloads\n const batchCommands: Array<AbstractBatchOperation<AbstractPayloadSubLevel, Hash, WithStorageMeta<Payload>>> = payloadsWithMeta.map(payload => ({\n type: 'del',\n key: payload._hash,\n }))\n\n await this.withStore(async (store) => {\n await store.batch(batchCommands)\n })\n\n // Delete the dataHash indexes\n const batchDataHashIndexCommands: Array<AbstractBatchOperation<AbstractPayloadSubLevel, string, Hash>> = payloadsWithMeta.map(payload => ({\n type: 'del',\n key: payload._dataHash,\n }))\n\n await this.withDataHashIndex(async (index) => {\n await index.batch(batchDataHashIndexCommands)\n })\n\n // Delete the sequence indexes\n const batchSequenceIndexCommands: Array<AbstractBatchOperation<AbstractPayloadSubLevel, Hex, Hash>> = payloadsWithMeta.map(payload => ({\n type: 'del',\n key: payload._sequence,\n }))\n\n await this.withSequenceIndex(async (index) => {\n await index.batch(batchSequenceIndexCommands)\n })\n\n return payloadsWithMeta\n }\n\n protected override async getHandler(hashes: Hash[]): Promise<WithStorageMeta<Payload>[]> {\n const foundByHash = await this.withStore(async (store) => {\n return (await store.getMany(hashes)).filter(exists)\n })\n const remainingHashes = hashes.filter(hash => !foundByHash.some(({ _hash }) => _hash === hash))\n const hashesFromDataHashes = await this.withDataHashIndex(async (index) => {\n return (await index.getMany(remainingHashes)).filter(exists)\n })\n const foundByDataHash = hashesFromDataHashes.length > 0\n ? await this.withStore(async (store) => {\n return (await store.getMany(hashesFromDataHashes)).filter(exists)\n })\n : []\n const result = [...foundByHash, ...foundByDataHash].toSorted(PayloadBuilder.compareStorageMeta)\n return result\n }\n\n protected override async insertHandler(payloads: WithStorageMeta<Payload>[]): Promise<WithStorageMeta<Payload>[]> {\n // Insert the payloads\n const payloadsWithMeta = payloads.toSorted(PayloadBuilder.compareStorageMeta)\n const batchCommands: Array<AbstractBatchOperation<AbstractPayloadSubLevel, Hash, WithStorageMeta<Payload>>> = payloadsWithMeta.map(payload => ({\n type: 'put', key: payload._hash, value: payload, keyEncoding: 'utf8', valueEncoding: 'json',\n }))\n await this.withStore(async (store) => {\n await store.batch(batchCommands)\n })\n\n // Insert the dataHash indexes\n // We use the dataHash|hash for the key to allow for multiple entries\n const batchDataHashIndexCommands: Array<AbstractBatchOperation<AbstractPayloadLevel, string, Hash>> = payloadsWithMeta.map(payload => ({\n type: 'put', key: payload._dataHash, value: payload._hash, keyEncoding: 'utf8', valueEncoding: 'utf8',\n }))\n await this.withDataHashIndex(async (index) => {\n await index.batch(batchDataHashIndexCommands)\n })\n\n // Insert the sequence indexes\n // We use the dataHash|hash for the key to allow for multiple entries\n const batchSequenceIndexCommands: Array<AbstractBatchOperation<AbstractPayloadLevel, Hex, Hash>> = payloadsWithMeta.map(payload => ({\n type: 'put', key: payload._sequence, value: payload._hash, keyEncoding: 'utf8', valueEncoding: 'utf8',\n }))\n await this.withSequenceIndex(async (index) => {\n await index.batch(batchSequenceIndexCommands)\n })\n\n return payloadsWithMeta\n }\n\n protected override async nextHandler(options?: ArchivistNextOptions): Promise<WithStorageMeta<Payload>[]> {\n const {\n limit = 100, cursor, order, open = true,\n } = options ?? {}\n let all = await this.allHandler()\n if (order === 'desc') {\n all = all.toReversed()\n }\n const startIndex = isDefined(cursor)\n ? AbstractLevelDbArchivist.findIndexFromCursor(all, cursor) + (open ? 1 : 0)\n : 0\n const result = all.slice(startIndex, startIndex + limit)\n return result\n }\n\n protected override async startHandler() {\n await super.startHandler()\n // We could defer this creation to first access but we\n // want to fail fast here in case something is wrong\n await this.withStore(() => {})\n if (this.config.clearStoreOnStart) {\n await this.clearHandler()\n }\n }\n\n protected withDataHashIndex<T>(func: (index: AbstractIndexSubLevel<string>) => Promisable<T>): Promisable<T> {\n return this.withDb(async (db) => {\n const index = db.sublevel<string, Hash>(\n indexSubLevelName(this.storeName, AbstractLevelDbArchivist.dataHashIndexName),\n { keyEncoding: 'utf8', valueEncoding: 'utf8' },\n )\n return await func(index)\n })\n }\n\n protected withSequenceIndex<T>(func: (index: AbstractIndexSubLevel<Hex>) => Promisable<T>): Promisable<T> {\n return this.withDb(async (db) => {\n const index = db.sublevel<Hex, Hash>(\n indexSubLevelName(this.storeName, AbstractLevelDbArchivist.sequenceIndexName),\n { keyEncoding: 'utf8', valueEncoding: 'utf8' },\n )\n return await func(index)\n })\n }\n\n protected async withStore<T>(func: (store: AbstractPayloadSubLevel) => Promisable<T>): Promise<T> {\n return await this.withDb(async (db) => {\n const subLevel: AbstractPayloadSubLevel = db.sublevel<Hash, WithStorageMeta<Payload>>(this.storeName, { keyEncoding: 'utf8', valueEncoding: 'json' })\n return await func(subLevel)\n })\n }\n\n protected abstract withDb<T>(func: (db: AbstractPayloadLevel) => Promisable<T>): Promisable<T>\n}\n\n@creatableModule()\nexport class LevelDbArchivist extends AbstractLevelDbArchivist {\n private _db: Level<Hash, WithStorageMeta<Payload>> | undefined\n private _dbMutex = new Mutex()\n\n protected override async stopHandler() {\n await this._dbMutex.runExclusive(async () => {\n await this._db?.close()\n this._db = undefined\n })\n await super.stopHandler()\n }\n\n protected override async withDb<T>(func: (db: AbstractPayloadLevel) => Promisable<T>): Promise<T> {\n const db = await this._dbMutex.runExclusive(async () => {\n if (!this._db) {\n this._db = new Level<Hash, WithStorageMeta<Payload>>(this.folderPath, { keyEncoding: 'utf8', valueEncoding: 'json' })\n await this._db.open()\n }\n return this._db\n })\n return await func(db)\n }\n}\n","export const LevelDbArchivistSchema = 'network.xyo.archivist.leveldb' as const\nexport type LevelDbArchivistSchema = typeof LevelDbArchivistSchema\n","import type { ArchivistConfig } from '@xyo-network/archivist-model'\n\nimport { LevelDbArchivistSchema } from './Schema.ts'\n\nexport type LevelDbArchivistConfigSchema = `${LevelDbArchivistSchema}.config`\nexport const LevelDbArchivistConfigSchema: LevelDbArchivistConfigSchema = `${LevelDbArchivistSchema}.config`\n\nexport type LevelDbArchivistConfig<TStoreName extends string = string> = ArchivistConfig<{\n /**\n * If true, the store will be cleared on start\n */\n clearStoreOnStart?: boolean\n /**\n * The database name - also used as the filename for the db\n */\n dbName?: string\n /**\n * The location where the folder for the db will be created\n */\n location?: string\n schema: LevelDbArchivistConfigSchema\n /**\n * The name of the object store - becomes a sub-level\n */\n storeName?: TStoreName\n}>\n"],"mappings":";;;;;;;;;;;;AAAA,SAAS,gBAAgB;AACzB,SAAS,cAAc;AAEvB,SAAS,iBAA6B;AACtC,SAAS,iBAAiB;AAC1B,SAAS,mBAAmB,yBAAyB;AACrD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EAGA;AAAA,EACA;AAAA,OAEK;AAEP,SAAS,uBAAuB;AAChC,SAAS,sBAAsB;AAO/B,SAAS,aAAa;AACtB,SAAS,aAAa;;;AC7Bf,IAAM,yBAAyB;;;ACK/B,IAAM,+BAA6D,GAAG,sBAAsB;;;AFuCnG,IAAM,oBAAoB,CAAC,WAAmB,cAAsB;AAClE,SAAO,IAAI,SAAS,IAAI,SAAS;AACnC;AAEO,IAAe,2BAAf,MAAe,kCAGZ,kBAAuC;AAAA,EAC/C,OAAyB,gBAA0B,CAAC,GAAG,MAAM,eAAe,4BAA4B;AAAA,EACxG,OAAyB,sBAA8B;AAAA,EACvD,OAAyB,SAAS,EAAE,GAAG,MAAM,QAAQ,CAAC,iBAAiB,GAAG,OAAO;AAAA,EAEjF,OAAwB,gBAAkC;AAAA,IACxD,KAAK,EAAE,WAAW,EAAE;AAAA,IAAG,YAAY;AAAA,IAAO,QAAQ;AAAA,EACpD;AAAA,EAEA,OAAwB,gBAAkC;AAAA,IACxD,KAAK,EAAE,WAAW,EAAE;AAAA,IAAG,YAAY;AAAA,IAAO,QAAQ;AAAA,EACpD;AAAA;AAAA,EAGA,OAAgB,oBAAoB,uBAAuB,0BAAyB,aAAa;AAAA;AAAA,EAEjG,OAAgB,oBAAoB,uBAAuB,0BAAyB,aAAa;AAAA,EAEjG,IAAI,SAAS;AACX,WAAO,SAAS,KAAK,OAAO,QAAQ,MAAM,qBAAqB;AAAA,EACjE;AAAA,EAEA,IAAI,aAAa;AACf,WAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,OAAO,MAAM,IAAI,KAAK,SAAS;AAAA,EACjE;AAAA,EAEA,IAAI,WAAW;AACb,WAAO,SAAS,KAAK,OAAO,UAAU,MAAM,uBAAuB;AAAA,EACrE;AAAA,EAEA,IAAa,UAAU;AACrB,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,GAAG,MAAM;AAAA,IACX;AAAA,EACF;AAAA,EAEA,IAAI,YAAY;AACd,WAAO,SAAS,KAAK,OAAO,WAAW,MAAM,wBAAwB;AAAA,EACvE;AAAA,EAEA,OAAe,oBAAoB,UAA6B,QAAa;AAC3E,UAAM,QAAQ,SAAS,UAAU,CAAC,EAAE,UAAU,MAAM,cAAc,MAAM;AACxE,QAAI,UAAU,IAAI;AAChB,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,aAAkD;AACzE,WAAO,MAAM,KAAK,UAAU,OAAO,OAAO;AACxC,YAAM,SAAS,CAAC,GAAI,MAAM,GAAG,OAAO,EAAE,IAAI,CAAE;AAC5C,aAAO,OAAO,OAAO,MAAM,EAAE,SAAS,eAAe,kBAAkB;AAAA,IACzE,CAAC;AAAA,EACH;AAAA,EAEA,MAAyB,eAA8B;AACrD,UAAM,KAAK,OAAO,OAAO,OAAO;AAC9B,YAAM,GAAG,MAAM;AAAA,IACjB,CAAC;AACD,UAAM,KAAK,kBAAkB,OAAO,UAAU;AAC5C,YAAM,MAAM,MAAM;AAAA,IACpB,CAAC;AACD,UAAM,KAAK,kBAAkB,OAAO,UAAU;AAC5C,YAAM,MAAM,MAAM;AAAA,IACpB,CAAC;AACD,WAAO,KAAK,KAAK,WAAW,EAAE,KAAK,KAAK,CAAC;AAAA,EAC3C;AAAA,EAEA,MAAyB,gBAAyC;AAChE,UAAM,WAAW,SAAS,MAAM,KAAK,WAAW,GAAG,MAAM,mBAAmB;AAC5E,UAAM,UAAU,MAAM,QAAQ;AAAA,MAC5B,OAAO,QAAQ,MAAM,KAAK,iBAAiB,GAAG,UAAU,CAAC,CAAC,GAAG,IAAI,OAAO,WAAW;AACjF,cAAM,eAAqC,EAAE,QAAQ,2BAA2B;AAChF,cAAM,QAAQ,MAAM,KAAK,UAAU,cAAc,QAAQ;AACzD,gBAAQ,MAAM,QAAQ,MAAM,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC;AAAA,MACtD,CAAC,EAAE,OAAO,MAAM;AAAA,IAClB;AACA,UAAM,KAAK,aAAa;AACxB,WAAO,QAAQ,OAAO,SAAS,EAAE,IAAI,YAAU,OAAO,KAAK,EAAE,OAAO,MAAM;AAAA,EAC5E;AAAA,EAEA,MAAyB,cAAc,QAAqD;AAE1F,UAAM,oBAAoB,MAAM,KAAK,WAAW,GAAG,OAAO,CAAC,EAAE,OAAO,UAAU,MAAM,OAAO,SAAS,KAAK,KAAK,OAAO,SAAS,SAAS,CAAC;AAExI,UAAM,gBAAwG,iBAAiB,IAAI,cAAY;AAAA,MAC7I,MAAM;AAAA,MACN,KAAK,QAAQ;AAAA,IACf,EAAE;AAEF,UAAM,KAAK,UAAU,OAAO,UAAU;AACpC,YAAM,MAAM,MAAM,aAAa;AAAA,IACjC,CAAC;AAGD,UAAM,6BAAmG,iBAAiB,IAAI,cAAY;AAAA,MACxI,MAAM;AAAA,MACN,KAAK,QAAQ;AAAA,IACf,EAAE;AAEF,UAAM,KAAK,kBAAkB,OAAO,UAAU;AAC5C,YAAM,MAAM,MAAM,0BAA0B;AAAA,IAC9C,CAAC;AAGD,UAAM,6BAAgG,iBAAiB,IAAI,cAAY;AAAA,MACrI,MAAM;AAAA,MACN,KAAK,QAAQ;AAAA,IACf,EAAE;AAEF,UAAM,KAAK,kBAAkB,OAAO,UAAU;AAC5C,YAAM,MAAM,MAAM,0BAA0B;AAAA,IAC9C,CAAC;AAED,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,WAAW,QAAqD;AACvF,UAAM,cAAc,MAAM,KAAK,UAAU,OAAO,UAAU;AACxD,cAAQ,MAAM,MAAM,QAAQ,MAAM,GAAG,OAAO,MAAM;AAAA,IACpD,CAAC;AACD,UAAM,kBAAkB,OAAO,OAAO,UAAQ,CAAC,YAAY,KAAK,CAAC,EAAE,MAAM,MAAM,UAAU,IAAI,CAAC;AAC9F,UAAM,uBAAuB,MAAM,KAAK,kBAAkB,OAAO,UAAU;AACzE,cAAQ,MAAM,MAAM,QAAQ,eAAe,GAAG,OAAO,MAAM;AAAA,IAC7D,CAAC;AACD,UAAM,kBAAkB,qBAAqB,SAAS,IAClD,MAAM,KAAK,UAAU,OAAO,UAAU;AACpC,cAAQ,MAAM,MAAM,QAAQ,oBAAoB,GAAG,OAAO,MAAM;AAAA,IAClE,CAAC,IACD,CAAC;AACL,UAAM,SAAS,CAAC,GAAG,aAAa,GAAG,eAAe,EAAE,SAAS,eAAe,kBAAkB;AAC9F,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,cAAc,UAA2E;AAEhH,UAAM,mBAAmB,SAAS,SAAS,eAAe,kBAAkB;AAC5E,UAAM,gBAAwG,iBAAiB,IAAI,cAAY;AAAA,MAC7I,MAAM;AAAA,MAAO,KAAK,QAAQ;AAAA,MAAO,OAAO;AAAA,MAAS,aAAa;AAAA,MAAQ,eAAe;AAAA,IACvF,EAAE;AACF,UAAM,KAAK,UAAU,OAAO,UAAU;AACpC,YAAM,MAAM,MAAM,aAAa;AAAA,IACjC,CAAC;AAID,UAAM,6BAAgG,iBAAiB,IAAI,cAAY;AAAA,MACrI,MAAM;AAAA,MAAO,KAAK,QAAQ;AAAA,MAAW,OAAO,QAAQ;AAAA,MAAO,aAAa;AAAA,MAAQ,eAAe;AAAA,IACjG,EAAE;AACF,UAAM,KAAK,kBAAkB,OAAO,UAAU;AAC5C,YAAM,MAAM,MAAM,0BAA0B;AAAA,IAC9C,CAAC;AAID,UAAM,6BAA6F,iBAAiB,IAAI,cAAY;AAAA,MAClI,MAAM;AAAA,MAAO,KAAK,QAAQ;AAAA,MAAW,OAAO,QAAQ;AAAA,MAAO,aAAa;AAAA,MAAQ,eAAe;AAAA,IACjG,EAAE;AACF,UAAM,KAAK,kBAAkB,OAAO,UAAU;AAC5C,YAAM,MAAM,MAAM,0BAA0B;AAAA,IAC9C,CAAC;AAED,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,SAAqE;AACxG,UAAM;AAAA,MACJ,QAAQ;AAAA,MAAK;AAAA,MAAQ;AAAA,MAAO,OAAO;AAAA,IACrC,IAAI,WAAW,CAAC;AAChB,QAAI,MAAM,MAAM,KAAK,WAAW;AAChC,QAAI,UAAU,QAAQ;AACpB,YAAM,IAAI,WAAW;AAAA,IACvB;AACA,UAAM,aAAa,UAAU,MAAM,IAC/B,0BAAyB,oBAAoB,KAAK,MAAM,KAAK,OAAO,IAAI,KACxE;AACJ,UAAM,SAAS,IAAI,MAAM,YAAY,aAAa,KAAK;AACvD,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAe;AACtC,UAAM,MAAM,aAAa;AAGzB,UAAM,KAAK,UAAU,MAAM;AAAA,IAAC,CAAC;AAC7B,QAAI,KAAK,OAAO,mBAAmB;AACjC,YAAM,KAAK,aAAa;AAAA,IAC1B;AAAA,EACF;AAAA,EAEU,kBAAqB,MAA8E;AAC3G,WAAO,KAAK,OAAO,OAAO,OAAO;AAC/B,YAAM,QAAQ,GAAG;AAAA,QACf,kBAAkB,KAAK,WAAW,0BAAyB,iBAAiB;AAAA,QAC5E,EAAE,aAAa,QAAQ,eAAe,OAAO;AAAA,MAC/C;AACA,aAAO,MAAM,KAAK,KAAK;AAAA,IACzB,CAAC;AAAA,EACH;AAAA,EAEU,kBAAqB,MAA2E;AACxG,WAAO,KAAK,OAAO,OAAO,OAAO;AAC/B,YAAM,QAAQ,GAAG;AAAA,QACf,kBAAkB,KAAK,WAAW,0BAAyB,iBAAiB;AAAA,QAC5E,EAAE,aAAa,QAAQ,eAAe,OAAO;AAAA,MAC/C;AACA,aAAO,MAAM,KAAK,KAAK;AAAA,IACzB,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,UAAa,MAAqE;AAChG,WAAO,MAAM,KAAK,OAAO,OAAO,OAAO;AACrC,YAAM,WAAoC,GAAG,SAAyC,KAAK,WAAW,EAAE,aAAa,QAAQ,eAAe,OAAO,CAAC;AACpJ,aAAO,MAAM,KAAK,QAAQ;AAAA,IAC5B,CAAC;AAAA,EACH;AAGF;AAGO,IAAM,mBAAN,cAA+B,yBAAyB;AAAA,EACrD;AAAA,EACA,WAAW,IAAI,MAAM;AAAA,EAE7B,MAAyB,cAAc;AACrC,UAAM,KAAK,SAAS,aAAa,YAAY;AAC3C,YAAM,KAAK,KAAK,MAAM;AACtB,WAAK,MAAM;AAAA,IACb,CAAC;AACD,UAAM,MAAM,YAAY;AAAA,EAC1B;AAAA,EAEA,MAAyB,OAAU,MAA+D;AAChG,UAAM,KAAK,MAAM,KAAK,SAAS,aAAa,YAAY;AACtD,UAAI,CAAC,KAAK,KAAK;AACb,aAAK,MAAM,IAAI,MAAsC,KAAK,YAAY,EAAE,aAAa,QAAQ,eAAe,OAAO,CAAC;AACpH,cAAM,KAAK,IAAI,KAAK;AAAA,MACtB;AACA,aAAO,KAAK;AAAA,IACd,CAAC;AACD,WAAO,MAAM,KAAK,EAAE;AAAA,EACtB;AACF;AAtBa,mBAAN;AAAA,EADN,gBAAgB;AAAA,GACJ;","names":[]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xyo-network/archivist-leveldb",
|
|
3
|
-
"version": "5.
|
|
3
|
+
"version": "5.2.0",
|
|
4
4
|
"description": "Primary SDK for using XYO Protocol 2.0",
|
|
5
5
|
"homepage": "https://xyo.network",
|
|
6
6
|
"bugs": {
|
|
@@ -36,36 +36,36 @@
|
|
|
36
36
|
"!**/*.test.*"
|
|
37
37
|
],
|
|
38
38
|
"dependencies": {
|
|
39
|
-
"@xylabs/assert": "~5.0.
|
|
40
|
-
"@xylabs/exists": "~5.0.
|
|
41
|
-
"@xylabs/hex": "~5.0.
|
|
42
|
-
"@xylabs/promise": "~5.0.
|
|
43
|
-
"@xylabs/typeof": "~5.0.
|
|
44
|
-
"@xyo-network/archivist-abstract": "~5.
|
|
45
|
-
"@xyo-network/archivist-model": "~5.
|
|
46
|
-
"@xyo-network/boundwitness-model": "~5.
|
|
47
|
-
"@xyo-network/module-model": "~5.
|
|
48
|
-
"@xyo-network/payload-builder": "~5.
|
|
49
|
-
"@xyo-network/payload-model": "~5.
|
|
39
|
+
"@xylabs/assert": "~5.0.39",
|
|
40
|
+
"@xylabs/exists": "~5.0.39",
|
|
41
|
+
"@xylabs/hex": "~5.0.39",
|
|
42
|
+
"@xylabs/promise": "~5.0.39",
|
|
43
|
+
"@xylabs/typeof": "~5.0.39",
|
|
44
|
+
"@xyo-network/archivist-abstract": "~5.2.0",
|
|
45
|
+
"@xyo-network/archivist-model": "~5.2.0",
|
|
46
|
+
"@xyo-network/boundwitness-model": "~5.2.0",
|
|
47
|
+
"@xyo-network/module-model": "~5.2.0",
|
|
48
|
+
"@xyo-network/payload-builder": "~5.2.0",
|
|
49
|
+
"@xyo-network/payload-model": "~5.2.0",
|
|
50
50
|
"abstract-level": "~3.1.1",
|
|
51
51
|
"async-mutex": "~0.5.0",
|
|
52
52
|
"level": "~10.0.0"
|
|
53
53
|
},
|
|
54
54
|
"devDependencies": {
|
|
55
|
-
"@xylabs/delay": "~5.0.
|
|
56
|
-
"@xylabs/object": "~5.0.
|
|
55
|
+
"@xylabs/delay": "~5.0.39",
|
|
56
|
+
"@xylabs/object": "~5.0.39",
|
|
57
57
|
"@xylabs/ts-scripts-yarn3": "~7.2.8",
|
|
58
58
|
"@xylabs/tsconfig": "~7.2.8",
|
|
59
|
-
"@xylabs/vitest-extended": "~5.0.
|
|
60
|
-
"@xyo-network/account": "~5.
|
|
61
|
-
"@xyo-network/account-model": "~5.
|
|
62
|
-
"@xyo-network/archivist-acceptance-tests": "~5.
|
|
63
|
-
"@xyo-network/id-payload-plugin": "~5.
|
|
64
|
-
"@xyo-network/payload-wrapper": "~5.
|
|
65
|
-
"@xyo-network/wallet": "~5.
|
|
59
|
+
"@xylabs/vitest-extended": "~5.0.39",
|
|
60
|
+
"@xyo-network/account": "~5.2.0",
|
|
61
|
+
"@xyo-network/account-model": "~5.2.0",
|
|
62
|
+
"@xyo-network/archivist-acceptance-tests": "~5.2.0",
|
|
63
|
+
"@xyo-network/id-payload-plugin": "~5.2.0",
|
|
64
|
+
"@xyo-network/payload-wrapper": "~5.2.0",
|
|
65
|
+
"@xyo-network/wallet": "~5.2.0",
|
|
66
66
|
"typescript": "~5.9.3",
|
|
67
67
|
"uuid": "~13.0.0",
|
|
68
|
-
"vitest": "~4.0.
|
|
68
|
+
"vitest": "~4.0.10"
|
|
69
69
|
},
|
|
70
70
|
"publishConfig": {
|
|
71
71
|
"access": "public"
|
package/src/Archivist.ts
CHANGED
|
@@ -32,7 +32,7 @@ import { Level } from 'level'
|
|
|
32
32
|
import { LevelDbArchivistConfigSchema } from './Config.ts'
|
|
33
33
|
import { LevelDbArchivistParams } from './Params.ts'
|
|
34
34
|
|
|
35
|
-
/**
|
|
35
|
+
/** We have indexes as top level sublevels since making them a sublevel of a store, getting all the values of that store includes the sublevels */
|
|
36
36
|
|
|
37
37
|
export interface PayloadStore {
|
|
38
38
|
[s: string]: WithStorageMeta
|
|
@@ -200,7 +200,7 @@ export abstract class AbstractLevelDbArchivist<
|
|
|
200
200
|
})
|
|
201
201
|
|
|
202
202
|
// Insert the dataHash indexes
|
|
203
|
-
//
|
|
203
|
+
// We use the dataHash|hash for the key to allow for multiple entries
|
|
204
204
|
const batchDataHashIndexCommands: Array<AbstractBatchOperation<AbstractPayloadLevel, string, Hash>> = payloadsWithMeta.map(payload => ({
|
|
205
205
|
type: 'put', key: payload._dataHash, value: payload._hash, keyEncoding: 'utf8', valueEncoding: 'utf8',
|
|
206
206
|
}))
|
|
@@ -209,7 +209,7 @@ export abstract class AbstractLevelDbArchivist<
|
|
|
209
209
|
})
|
|
210
210
|
|
|
211
211
|
// Insert the sequence indexes
|
|
212
|
-
//
|
|
212
|
+
// We use the dataHash|hash for the key to allow for multiple entries
|
|
213
213
|
const batchSequenceIndexCommands: Array<AbstractBatchOperation<AbstractPayloadLevel, Hex, Hash>> = payloadsWithMeta.map(payload => ({
|
|
214
214
|
type: 'put', key: payload._sequence, value: payload._hash, keyEncoding: 'utf8', valueEncoding: 'utf8',
|
|
215
215
|
}))
|
|
@@ -237,7 +237,7 @@ export abstract class AbstractLevelDbArchivist<
|
|
|
237
237
|
|
|
238
238
|
protected override async startHandler() {
|
|
239
239
|
await super.startHandler()
|
|
240
|
-
//
|
|
240
|
+
// We could defer this creation to first access but we
|
|
241
241
|
// want to fail fast here in case something is wrong
|
|
242
242
|
await this.withStore(() => {})
|
|
243
243
|
if (this.config.clearStoreOnStart) {
|