@xyo-network/archivist-memory 3.6.0-rc.3 → 3.6.0-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -24,6 +24,7 @@ export declare class MemoryArchivist<TParams extends MemoryArchivistParams<AnyCo
|
|
|
24
24
|
} & import("@xyo-network/payload-model").PayloadFields>, unknown>;
|
|
25
25
|
protected get dataHashIndex(): LRUCache<Lowercase<string>, Lowercase<string>, unknown>;
|
|
26
26
|
protected get max(): number;
|
|
27
|
+
private static findIndexFromCursor;
|
|
27
28
|
protected allHandler(): Promisable<WithStorageMeta<Payload>[]>;
|
|
28
29
|
protected clearHandler(): void | Promise<void>;
|
|
29
30
|
protected commitHandler(): Promise<BoundWitness[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MemoryArchivist.d.ts","sourceRoot":"","sources":["../../src/MemoryArchivist.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,
|
|
1
|
+
{"version":3,"file":"MemoryArchivist.d.ts","sourceRoot":"","sources":["../../src/MemoryArchivist.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAO,MAAM,aAAa,CAAA;AACvC,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC5D,OAAO,EAAa,UAAU,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAA;AACnE,OAAO,EAIL,eAAe,EAIf,wBAAwB,EACxB,oBAAoB,EAEpB,2BAA2B,EAC5B,MAAM,8BAA8B,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AAC9D,OAAO,EACL,eAAe,EAAmB,cAAc,EAAE,YAAY,EAC/D,MAAM,2BAA2B,CAAA;AAElC,OAAO,EACL,OAAO,EAAE,MAAM,EAAE,eAAe,EACjC,MAAM,4BAA4B,CAAA;AACnC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAEpC,MAAM,MAAM,2BAA2B,GAAG,qCAAqC,CAAA;AAC/E,eAAO,MAAM,2BAA2B,EAAE,2BAAmE,CAAA;AAE7G,MAAM,MAAM,qBAAqB,CAAC,OAAO,SAAS,OAAO,GAAG,WAAW,GAAG,IAAI,GAAG,IAAI,EAAE,OAAO,SAAS,MAAM,GAAG,IAAI,GAAG,IAAI,IAAI,eAAe,CAC5I,cAAc,CACZ;IACE,GAAG,CAAC,EAAE,MAAM,CAAA;CACb,EACD,OAAO,CACR,EACD,OAAO,SAAS,MAAM,GAAG,OAAO,GAAG,2BAA2B,GAAG,eAAe,CAAC,QAAQ,CAAC,CAC3F,CAAA;AAED,MAAM,MAAM,qBAAqB,CAAC,OAAO,SAAS,eAAe,CAAC,qBAAqB,CAAC,GAAG,eAAe,CAAC,qBAAqB,CAAC,IAC/H,YAAY,CAAC,OAAO,CAAC,CAAA;AACvB,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,CAAC,eAAe,CAAC,qBAAqB,CAAC,CAAC,GAAG,qBAAqB,EACrG,UAAU,SAAS,wBAAwB,GAAG,wBAAwB,CAEtE,SAAQ,iBAAiB,CAAC,OAAO,EAAE,UAAU,CAC7C,YAAW,2BAA2B,EAAE,cAAc;IACtD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAwD;IACxG,gBAAyB,mBAAmB,EAAE,MAAM,CAA8B;IAElF,OAAO,CAAC,MAAM,CAAC,CAA0C;IACzD,OAAO,CAAC,cAAc,CAAC,CAAsB;IAE7C,IAAa,OAAO,aAUnB;IAED,SAAS,KAAK,KAAK;;sEAGlB;IAED,SAAS,KAAK,aAAa,4DAG1B;IAED,SAAS,KAAK,GAAG,WAEhB;IAED,OAAO,CAAC,MAAM,CAAC,mBAAmB;cAQf,UAAU,IAAI,UAAU,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cAKpD,YAAY,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;cAM9B,aAAa,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;cAaxC,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;cAepD,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,UAAU,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cAW5D,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cASvE,WAAW,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;YAkB3F,sBAAsB;YAOtB,oBAAoB;CAQnC"}
|
package/dist/neutral/index.mjs
CHANGED
|
@@ -18,7 +18,7 @@ function _ts_decorate(decorators, target, key, desc) {
|
|
|
18
18
|
}
|
|
19
19
|
__name(_ts_decorate, "_ts_decorate");
|
|
20
20
|
var MemoryArchivistConfigSchema = "network.xyo.archivist.memory.config";
|
|
21
|
-
var MemoryArchivist = class extends AbstractArchivist {
|
|
21
|
+
var MemoryArchivist = class _MemoryArchivist extends AbstractArchivist {
|
|
22
22
|
static {
|
|
23
23
|
__name(this, "MemoryArchivist");
|
|
24
24
|
}
|
|
@@ -55,6 +55,13 @@ var MemoryArchivist = class extends AbstractArchivist {
|
|
|
55
55
|
get max() {
|
|
56
56
|
return this.config?.max ?? 1e4;
|
|
57
57
|
}
|
|
58
|
+
static findIndexFromCursor(payloads, cursor) {
|
|
59
|
+
const index = payloads.findIndex(({ _sequence }) => _sequence === cursor);
|
|
60
|
+
if (index === -1) {
|
|
61
|
+
return Infinity;
|
|
62
|
+
}
|
|
63
|
+
return index;
|
|
64
|
+
}
|
|
58
65
|
allHandler() {
|
|
59
66
|
const all = this.cache.dump().map(([, item]) => item.value).filter(exists);
|
|
60
67
|
return PayloadBuilder.sortByStorageMeta(all);
|
|
@@ -107,13 +114,17 @@ var MemoryArchivist = class extends AbstractArchivist {
|
|
|
107
114
|
return insertedPayloads;
|
|
108
115
|
}
|
|
109
116
|
async nextHandler(options) {
|
|
117
|
+
this.logger.warn("nextHandler:start", options);
|
|
110
118
|
const { limit, cursor, order } = options ?? {};
|
|
111
119
|
let all = await this.allHandler();
|
|
112
120
|
if (order === "desc") {
|
|
113
121
|
all = all.reverse();
|
|
114
122
|
}
|
|
115
|
-
const startIndex = cursor ?
|
|
116
|
-
|
|
123
|
+
const startIndex = cursor ? _MemoryArchivist.findIndexFromCursor(all, cursor) + 1 : 0;
|
|
124
|
+
this.logger.warn("nextHandler:startIndex", startIndex);
|
|
125
|
+
const result = all.slice(startIndex, limit ? startIndex + limit : void 0);
|
|
126
|
+
this.logger.warn("nextHandler:result", result);
|
|
127
|
+
return result;
|
|
117
128
|
}
|
|
118
129
|
async insertPayloadIntoCache(payload) {
|
|
119
130
|
const withMeta = await PayloadBuilder.addStorageMeta(payload);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/MemoryArchivist.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject, WithAdditional } from '@xylabs/object'\nimport { fulfilled, Promisable } from '@xylabs/promise'\nimport { AbstractArchivist } from '@xyo-network/archivist-abstract'\nimport {\n ArchivistAllQuerySchema,\n ArchivistClearQuerySchema,\n ArchivistCommitQuerySchema,\n ArchivistConfig,\n ArchivistDeleteQuerySchema,\n ArchivistInsertQuery,\n ArchivistInsertQuerySchema,\n ArchivistModuleEventData,\n ArchivistNextOptions,\n ArchivistNextQuerySchema,\n AttachableArchivistInstance,\n} from '@xyo-network/archivist-model'\nimport { BoundWitness } from '@xyo-network/boundwitness-model'\nimport {\n AnyConfigSchema, creatableModule, ModuleInstance, ModuleParams,\n} from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport {\n Payload, Schema, WithStorageMeta,\n} from '@xyo-network/payload-model'\nimport { LRUCache } from 'lru-cache'\n\nexport type MemoryArchivistConfigSchema = 'network.xyo.archivist.memory.config'\nexport const MemoryArchivistConfigSchema: MemoryArchivistConfigSchema = 'network.xyo.archivist.memory.config'\n\nexport type MemoryArchivistConfig<TConfig extends Payload | EmptyObject | void = void, TSchema extends Schema | void = void> = ArchivistConfig<\n WithAdditional<\n {\n max?: number\n },\n TConfig\n >,\n TSchema extends Schema ? TSchema : MemoryArchivistConfigSchema | ArchivistConfig['schema']\n>\n\nexport type MemoryArchivistParams<TConfig extends AnyConfigSchema<MemoryArchivistConfig> = AnyConfigSchema<MemoryArchivistConfig>> =\n ModuleParams<TConfig>\n@creatableModule()\nexport class MemoryArchivist<\n TParams extends MemoryArchivistParams<AnyConfigSchema<MemoryArchivistConfig>> = MemoryArchivistParams,\n TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,\n>\n extends AbstractArchivist<TParams, TEventData>\n implements AttachableArchivistInstance, ModuleInstance {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, MemoryArchivistConfigSchema]\n static override readonly defaultConfigSchema: Schema = MemoryArchivistConfigSchema\n\n private _cache?: LRUCache<Hash, WithStorageMeta<Payload>>\n private _dataHashIndex?: LRUCache<Hash, Hash>\n\n override get queries() {\n return [\n ArchivistAllQuerySchema,\n ArchivistDeleteQuerySchema,\n ArchivistClearQuerySchema,\n ArchivistInsertQuerySchema,\n ArchivistCommitQuerySchema,\n ArchivistNextQuerySchema,\n ...super.queries,\n ]\n }\n\n protected get cache() {\n this._cache = this._cache ?? new LRUCache<Hash, WithStorageMeta<Payload>>({ max: this.max })\n return this._cache\n }\n\n protected get dataHashIndex() {\n this._dataHashIndex = this._dataHashIndex ?? new LRUCache<Hash, Hash>({ max: this.max })\n return this._dataHashIndex\n }\n\n protected get max() {\n return this.config?.max ?? 10_000\n }\n\n protected override allHandler(): Promisable<WithStorageMeta<Payload>[]> {\n const all = this.cache.dump().map(([, item]) => item.value).filter(exists)\n return PayloadBuilder.sortByStorageMeta(all)\n }\n\n protected override clearHandler(): void | Promise<void> {\n this.cache.clear()\n this.dataHashIndex.clear()\n return this.emit('cleared', { mod: this })\n }\n\n protected override async commitHandler(): Promise<BoundWitness[]> {\n const payloads = assertEx(await this.allHandler(), () => 'Nothing to commit')\n const settled = await Promise.allSettled(\n Object.values((await this.parentArchivists()).commit ?? [])?.map(async (parent) => {\n const queryPayload: ArchivistInsertQuery = { schema: ArchivistInsertQuerySchema }\n const query = await this.bindQuery(queryPayload, payloads)\n return (await parent?.query(query[0], query[1]))?.[0]\n }).filter(exists),\n )\n await this.clearHandler()\n return settled.filter(fulfilled).map(result => result.value).filter(exists)\n }\n\n protected override async deleteHandler(hashes: Hash[]): Promise<Hash[]> {\n const deletedHashes: Hash[] = (await Promise.all(this.cache\n .dump()\n .map(async ([key, item]) => {\n const itemValueDataHash = await PayloadBuilder.dataHash(item.value)\n if (hashes.includes(key) || hashes.includes(itemValueDataHash)) {\n this.cache.delete(key)\n return key\n }\n })))\n .filter(exists)\n await this.rebuildDataHashIndex()\n return deletedHashes\n }\n\n protected override getHandler(hashes: Hash[]): Promisable<WithStorageMeta<Payload>[]> {\n return hashes.map((hash) => {\n const resolvedHash = this.dataHashIndex.get(hash) ?? hash\n const result = this.cache.get(resolvedHash)\n if (resolvedHash !== hash && !result) {\n throw new Error('Missing referenced payload')\n }\n return result\n }).filter(exists)\n }\n\n protected override async insertHandler(payloads: Payload[]): Promise<WithStorageMeta<Payload>[]> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n const insertedPayloads = await Promise.all(pairs.map(async ([payload, hash]) => {\n return this.cache.get(hash) ?? await this.insertPayloadIntoCache(payload)\n }))\n\n return insertedPayloads\n }\n\n protected override async nextHandler(options?: ArchivistNextOptions): Promise<WithStorageMeta<Payload>[]> {\n const {\n limit, cursor, order,\n } = options ?? {}\n let all = await this.allHandler()\n if (order === 'desc') {\n all = all.reverse()\n }\n const startIndex = cursor ? all.findIndex(({ _sequence }) => _sequence === cursor) + 1 : 0\n return all.slice(startIndex, limit ? startIndex + limit : undefined)\n }\n\n private async insertPayloadIntoCache(payload: Payload): Promise<WithStorageMeta<Payload>> {\n const withMeta = await PayloadBuilder.addStorageMeta(payload)\n this.cache.set(withMeta._hash, withMeta)\n this.dataHashIndex.set(withMeta._dataHash, withMeta._hash)\n return withMeta\n }\n\n private async rebuildDataHashIndex() {\n this._dataHashIndex = new LRUCache<Hash, Hash>({ max: this.max })\n const pairs = this.cache.dump()\n for (const [hash, payload] of pairs) {\n const dataHash = await PayloadBuilder.dataHash(payload.value)\n this.dataHashIndex.set(dataHash, hash)\n }\n }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAGvB,SAASC,iBAA6B;AACtC,SAASC,yBAAyB;AAClC,SACEC,yBACAC,2BACAC,4BAEAC,4BAEAC,4BAGAC,gCAEK;AAEP,SACmBC,uBACZ;AACP,SAASC,sBAAsB;AAI/B,SAASC,gBAAgB;;;;;;;;AAGlB,IAAMC,8BAA2D;AAejE,IAAMC,kBAAN,cAIGC,kBAAAA;SAAAA;;;EAER,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE/CK;EACAC;EAER,IAAaC,UAAU;AACrB,WAAO;MACLC;MACAC;MACAC;MACAC;MACAC;MACAC;SACG,MAAMN;;EAEb;EAEA,IAAcO,QAAQ;AACpB,SAAKT,SAAS,KAAKA,UAAU,IAAIU,SAAyC;MAAEC,KAAK,KAAKA;IAAI,CAAA;AAC1F,WAAO,KAAKX;EACd;EAEA,IAAcY,gBAAgB;AAC5B,SAAKX,iBAAiB,KAAKA,kBAAkB,IAAIS,SAAqB;MAAEC,KAAK,KAAKA;IAAI,CAAA;AACtF,WAAO,KAAKV;EACd;EAEA,IAAcU,MAAM;AAClB,WAAO,KAAKE,QAAQF,OAAO;EAC7B;EAEmBG,aAAqD;AACtE,UAAMC,MAAM,KAAKN,MAAMO,KAAI,EAAGC,IAAI,CAAC,CAAA,EAAGC,IAAAA,MAAUA,KAAKC,KAAK,EAAEC,OAAOC,MAAAA;AACnE,WAAOC,eAAeC,kBAAkBR,GAAAA;EAC1C;EAEmBS,eAAqC;AACtD,SAAKf,MAAMgB,MAAK;AAChB,SAAKb,cAAca,MAAK;AACxB,WAAO,KAAKC,KAAK,WAAW;MAAEC,KAAK;IAAK,CAAA;EAC1C;EAEA,MAAyBC,gBAAyC;AAChE,UAAMC,WAAWC,SAAS,MAAM,KAAKhB,WAAU,GAAI,MAAM,mBAAA;AACzD,UAAMiB,UAAU,MAAMC,QAAQC,WAC5BC,OAAOC,QAAQ,MAAM,KAAKC,iBAAgB,GAAIC,UAAU,CAAA,CAAE,GAAGpB,IAAI,OAAOqB,WAAAA;AACtE,YAAMC,eAAqC;QAAEC,QAAQlC;MAA2B;AAChF,YAAMmC,QAAQ,MAAM,KAAKC,UAAUH,cAAcV,QAAAA;AACjD,cAAQ,MAAMS,QAAQG,MAAMA,MAAM,CAAA,GAAIA,MAAM,CAAA,CAAE,KAAK,CAAA;IACrD,CAAA,EAAGrB,OAAOC,MAAAA,CAAAA;AAEZ,UAAM,KAAKG,aAAY;AACvB,WAAOO,QAAQX,OAAOuB,SAAAA,EAAW1B,IAAI2B,CAAAA,WAAUA,OAAOzB,KAAK,EAAEC,OAAOC,MAAAA;EACtE;EAEA,MAAyBwB,cAAcC,QAAiC;AACtE,UAAMC,iBAAyB,MAAMf,QAAQjB,IAAI,KAAKN,MACnDO,KAAI,EACJC,IAAI,OAAO,CAAC+B,KAAK9B,IAAAA,MAAK;AACrB,YAAM+B,oBAAoB,MAAM3B,eAAe4B,SAAShC,KAAKC,KAAK;AAClE,UAAI2B,OAAOK,SAASH,GAAAA,KAAQF,OAAOK,SAASF,iBAAAA,GAAoB;AAC9D,aAAKxC,MAAM2C,OAAOJ,GAAAA;AAClB,eAAOA;MACT;IACF,CAAA,CAAA,GACC5B,OAAOC,MAAAA;AACV,UAAM,KAAKgC,qBAAoB;AAC/B,WAAON;EACT;EAEmBO,WAAWR,QAAwD;AACpF,WAAOA,OAAO7B,IAAI,CAACsC,SAAAA;AACjB,YAAMC,eAAe,KAAK5C,cAAc6C,IAAIF,IAAAA,KAASA;AACrD,YAAMX,SAAS,KAAKnC,MAAMgD,IAAID,YAAAA;AAC9B,UAAIA,iBAAiBD,QAAQ,CAACX,QAAQ;AACpC,cAAM,IAAIc,MAAM,4BAAA;MAClB;AACA,aAAOd;IACT,CAAA,EAAGxB,OAAOC,MAAAA;EACZ;EAEA,MAAyBsC,cAAc9B,UAA0D;AAC/F,UAAM+B,QAAQ,MAAMtC,eAAeuC,UAAUhC,QAAAA;AAC7C,UAAMiC,mBAAmB,MAAM9B,QAAQjB,IAAI6C,MAAM3C,IAAI,OAAO,CAAC8C,SAASR,IAAAA,MAAK;AACzE,aAAO,KAAK9C,MAAMgD,IAAIF,IAAAA,KAAS,MAAM,KAAKS,uBAAuBD,OAAAA;IACnE,CAAA,CAAA;AAEA,WAAOD;EACT;EAEA,MAAyBG,YAAYC,SAAqE;AACxG,UAAM,EACJC,OAAOC,QAAQC,MAAK,IAClBH,WAAW,CAAC;AAChB,QAAInD,MAAM,MAAM,KAAKD,WAAU;AAC/B,QAAIuD,UAAU,QAAQ;AACpBtD,YAAMA,IAAIuD,QAAO;IACnB;AACA,UAAMC,aAAaH,SAASrD,IAAIyD,UAAU,CAAC,EAAEC,UAAS,MAAOA,cAAcL,MAAAA,IAAU,IAAI;AACzF,WAAOrD,IAAI2D,MAAMH,YAAYJ,QAAQI,aAAaJ,QAAQQ,MAAAA;EAC5D;EAEA,MAAcX,uBAAuBD,SAAqD;AACxF,UAAMa,WAAW,MAAMtD,eAAeuD,eAAed,OAAAA;AACrD,SAAKtD,MAAMqE,IAAIF,SAASG,OAAOH,QAAAA;AAC/B,SAAKhE,cAAckE,IAAIF,SAASI,WAAWJ,SAASG,KAAK;AACzD,WAAOH;EACT;EAEA,MAAcvB,uBAAuB;AACnC,SAAKpD,iBAAiB,IAAIS,SAAqB;MAAEC,KAAK,KAAKA;IAAI,CAAA;AAC/D,UAAMiD,QAAQ,KAAKnD,MAAMO,KAAI;AAC7B,eAAW,CAACuC,MAAMQ,OAAAA,KAAYH,OAAO;AACnC,YAAMV,WAAW,MAAM5B,eAAe4B,SAASa,QAAQ5C,KAAK;AAC5D,WAAKP,cAAckE,IAAI5B,UAAUK,IAAAA;IACnC;EACF;AACF;;;;","names":["assertEx","exists","fulfilled","AbstractArchivist","ArchivistAllQuerySchema","ArchivistClearQuerySchema","ArchivistCommitQuerySchema","ArchivistDeleteQuerySchema","ArchivistInsertQuerySchema","ArchivistNextQuerySchema","creatableModule","PayloadBuilder","LRUCache","MemoryArchivistConfigSchema","MemoryArchivist","AbstractArchivist","configSchemas","defaultConfigSchema","_cache","_dataHashIndex","queries","ArchivistAllQuerySchema","ArchivistDeleteQuerySchema","ArchivistClearQuerySchema","ArchivistInsertQuerySchema","ArchivistCommitQuerySchema","ArchivistNextQuerySchema","cache","LRUCache","max","dataHashIndex","config","allHandler","all","dump","map","item","value","filter","exists","PayloadBuilder","sortByStorageMeta","clearHandler","clear","emit","mod","commitHandler","payloads","assertEx","settled","Promise","allSettled","Object","values","parentArchivists","commit","parent","queryPayload","schema","query","bindQuery","fulfilled","result","deleteHandler","hashes","deletedHashes","key","itemValueDataHash","dataHash","includes","delete","rebuildDataHashIndex","getHandler","hash","resolvedHash","get","Error","insertHandler","pairs","hashPairs","insertedPayloads","payload","insertPayloadIntoCache","nextHandler","options","limit","cursor","order","reverse","startIndex","findIndex","_sequence","slice","undefined","withMeta","addStorageMeta","set","_hash","_dataHash"]}
|
|
1
|
+
{"version":3,"sources":["../../src/MemoryArchivist.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { Hash, Hex } from '@xylabs/hex'\nimport { EmptyObject, WithAdditional } from '@xylabs/object'\nimport { fulfilled, Promisable } from '@xylabs/promise'\nimport { AbstractArchivist } from '@xyo-network/archivist-abstract'\nimport {\n ArchivistAllQuerySchema,\n ArchivistClearQuerySchema,\n ArchivistCommitQuerySchema,\n ArchivistConfig,\n ArchivistDeleteQuerySchema,\n ArchivistInsertQuery,\n ArchivistInsertQuerySchema,\n ArchivistModuleEventData,\n ArchivistNextOptions,\n ArchivistNextQuerySchema,\n AttachableArchivistInstance,\n} from '@xyo-network/archivist-model'\nimport { BoundWitness } from '@xyo-network/boundwitness-model'\nimport {\n AnyConfigSchema, creatableModule, ModuleInstance, ModuleParams,\n} from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport {\n Payload, Schema, WithStorageMeta,\n} from '@xyo-network/payload-model'\nimport { LRUCache } from 'lru-cache'\n\nexport type MemoryArchivistConfigSchema = 'network.xyo.archivist.memory.config'\nexport const MemoryArchivistConfigSchema: MemoryArchivistConfigSchema = 'network.xyo.archivist.memory.config'\n\nexport type MemoryArchivistConfig<TConfig extends Payload | EmptyObject | void = void, TSchema extends Schema | void = void> = ArchivistConfig<\n WithAdditional<\n {\n max?: number\n },\n TConfig\n >,\n TSchema extends Schema ? TSchema : MemoryArchivistConfigSchema | ArchivistConfig['schema']\n>\n\nexport type MemoryArchivistParams<TConfig extends AnyConfigSchema<MemoryArchivistConfig> = AnyConfigSchema<MemoryArchivistConfig>> =\n ModuleParams<TConfig>\n@creatableModule()\nexport class MemoryArchivist<\n TParams extends MemoryArchivistParams<AnyConfigSchema<MemoryArchivistConfig>> = MemoryArchivistParams,\n TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,\n>\n extends AbstractArchivist<TParams, TEventData>\n implements AttachableArchivistInstance, ModuleInstance {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, MemoryArchivistConfigSchema]\n static override readonly defaultConfigSchema: Schema = MemoryArchivistConfigSchema\n\n private _cache?: LRUCache<Hash, WithStorageMeta<Payload>>\n private _dataHashIndex?: LRUCache<Hash, Hash>\n\n override get queries() {\n return [\n ArchivistAllQuerySchema,\n ArchivistDeleteQuerySchema,\n ArchivistClearQuerySchema,\n ArchivistInsertQuerySchema,\n ArchivistCommitQuerySchema,\n ArchivistNextQuerySchema,\n ...super.queries,\n ]\n }\n\n protected get cache() {\n this._cache = this._cache ?? new LRUCache<Hash, WithStorageMeta<Payload>>({ max: this.max })\n return this._cache\n }\n\n protected get dataHashIndex() {\n this._dataHashIndex = this._dataHashIndex ?? new LRUCache<Hash, Hash>({ max: this.max })\n return this._dataHashIndex\n }\n\n protected get max() {\n return this.config?.max ?? 10_000\n }\n\n private static findIndexFromCursor(payloads: WithStorageMeta[], cursor: Hex) {\n const index = payloads.findIndex(({ _sequence }) => _sequence === cursor)\n if (index === -1) {\n return Infinity // move to the end\n }\n return index\n }\n\n protected override allHandler(): Promisable<WithStorageMeta<Payload>[]> {\n const all = this.cache.dump().map(([, item]) => item.value).filter(exists)\n return PayloadBuilder.sortByStorageMeta(all)\n }\n\n protected override clearHandler(): void | Promise<void> {\n this.cache.clear()\n this.dataHashIndex.clear()\n return this.emit('cleared', { mod: this })\n }\n\n protected override async commitHandler(): Promise<BoundWitness[]> {\n const payloads = assertEx(await this.allHandler(), () => 'Nothing to commit')\n const settled = await Promise.allSettled(\n Object.values((await this.parentArchivists()).commit ?? [])?.map(async (parent) => {\n const queryPayload: ArchivistInsertQuery = { schema: ArchivistInsertQuerySchema }\n const query = await this.bindQuery(queryPayload, payloads)\n return (await parent?.query(query[0], query[1]))?.[0]\n }).filter(exists),\n )\n await this.clearHandler()\n return settled.filter(fulfilled).map(result => result.value).filter(exists)\n }\n\n protected override async deleteHandler(hashes: Hash[]): Promise<Hash[]> {\n const deletedHashes: Hash[] = (await Promise.all(this.cache\n .dump()\n .map(async ([key, item]) => {\n const itemValueDataHash = await PayloadBuilder.dataHash(item.value)\n if (hashes.includes(key) || hashes.includes(itemValueDataHash)) {\n this.cache.delete(key)\n return key\n }\n })))\n .filter(exists)\n await this.rebuildDataHashIndex()\n return deletedHashes\n }\n\n protected override getHandler(hashes: Hash[]): Promisable<WithStorageMeta<Payload>[]> {\n return hashes.map((hash) => {\n const resolvedHash = this.dataHashIndex.get(hash) ?? hash\n const result = this.cache.get(resolvedHash)\n if (resolvedHash !== hash && !result) {\n throw new Error('Missing referenced payload')\n }\n return result\n }).filter(exists)\n }\n\n protected override async insertHandler(payloads: Payload[]): Promise<WithStorageMeta<Payload>[]> {\n const pairs = await PayloadBuilder.hashPairs(payloads)\n const insertedPayloads = await Promise.all(pairs.map(async ([payload, hash]) => {\n return this.cache.get(hash) ?? await this.insertPayloadIntoCache(payload)\n }))\n\n return insertedPayloads\n }\n\n protected override async nextHandler(options?: ArchivistNextOptions): Promise<WithStorageMeta<Payload>[]> {\n this.logger.warn('nextHandler:start', options)\n const {\n limit, cursor, order,\n } = options ?? {}\n let all = await this.allHandler()\n if (order === 'desc') {\n all = all.reverse()\n }\n const startIndex = cursor\n ? MemoryArchivist.findIndexFromCursor(all, cursor) + 1\n : 0\n this.logger.warn('nextHandler:startIndex', startIndex)\n const result = all.slice(startIndex, limit ? startIndex + limit : undefined)\n this.logger.warn('nextHandler:result', result)\n return result\n }\n\n private async insertPayloadIntoCache(payload: Payload): Promise<WithStorageMeta<Payload>> {\n const withMeta = await PayloadBuilder.addStorageMeta(payload)\n this.cache.set(withMeta._hash, withMeta)\n this.dataHashIndex.set(withMeta._dataHash, withMeta._hash)\n return withMeta\n }\n\n private async rebuildDataHashIndex() {\n this._dataHashIndex = new LRUCache<Hash, Hash>({ max: this.max })\n const pairs = this.cache.dump()\n for (const [hash, payload] of pairs) {\n const dataHash = await PayloadBuilder.dataHash(payload.value)\n this.dataHashIndex.set(dataHash, hash)\n }\n }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAGvB,SAASC,iBAA6B;AACtC,SAASC,yBAAyB;AAClC,SACEC,yBACAC,2BACAC,4BAEAC,4BAEAC,4BAGAC,gCAEK;AAEP,SACmBC,uBACZ;AACP,SAASC,sBAAsB;AAI/B,SAASC,gBAAgB;;;;;;;;AAGlB,IAAMC,8BAA2D;AAejE,IAAMC,kBAAN,MAAMA,yBAIHC,kBAAAA;SAAAA;;;EAER,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE/CK;EACAC;EAER,IAAaC,UAAU;AACrB,WAAO;MACLC;MACAC;MACAC;MACAC;MACAC;MACAC;SACG,MAAMN;;EAEb;EAEA,IAAcO,QAAQ;AACpB,SAAKT,SAAS,KAAKA,UAAU,IAAIU,SAAyC;MAAEC,KAAK,KAAKA;IAAI,CAAA;AAC1F,WAAO,KAAKX;EACd;EAEA,IAAcY,gBAAgB;AAC5B,SAAKX,iBAAiB,KAAKA,kBAAkB,IAAIS,SAAqB;MAAEC,KAAK,KAAKA;IAAI,CAAA;AACtF,WAAO,KAAKV;EACd;EAEA,IAAcU,MAAM;AAClB,WAAO,KAAKE,QAAQF,OAAO;EAC7B;EAEA,OAAeG,oBAAoBC,UAA6BC,QAAa;AAC3E,UAAMC,QAAQF,SAASG,UAAU,CAAC,EAAEC,UAAS,MAAOA,cAAcH,MAAAA;AAClE,QAAIC,UAAU,IAAI;AAChB,aAAOG;IACT;AACA,WAAOH;EACT;EAEmBI,aAAqD;AACtE,UAAMC,MAAM,KAAKb,MAAMc,KAAI,EAAGC,IAAI,CAAC,CAAA,EAAGC,IAAAA,MAAUA,KAAKC,KAAK,EAAEC,OAAOC,MAAAA;AACnE,WAAOC,eAAeC,kBAAkBR,GAAAA;EAC1C;EAEmBS,eAAqC;AACtD,SAAKtB,MAAMuB,MAAK;AAChB,SAAKpB,cAAcoB,MAAK;AACxB,WAAO,KAAKC,KAAK,WAAW;MAAEC,KAAK;IAAK,CAAA;EAC1C;EAEA,MAAyBC,gBAAyC;AAChE,UAAMpB,WAAWqB,SAAS,MAAM,KAAKf,WAAU,GAAI,MAAM,mBAAA;AACzD,UAAMgB,UAAU,MAAMC,QAAQC,WAC5BC,OAAOC,QAAQ,MAAM,KAAKC,iBAAgB,GAAIC,UAAU,CAAA,CAAE,GAAGnB,IAAI,OAAOoB,WAAAA;AACtE,YAAMC,eAAqC;QAAEC,QAAQxC;MAA2B;AAChF,YAAMyC,QAAQ,MAAM,KAAKC,UAAUH,cAAc9B,QAAAA;AACjD,cAAQ,MAAM6B,QAAQG,MAAMA,MAAM,CAAA,GAAIA,MAAM,CAAA,CAAE,KAAK,CAAA;IACrD,CAAA,EAAGpB,OAAOC,MAAAA,CAAAA;AAEZ,UAAM,KAAKG,aAAY;AACvB,WAAOM,QAAQV,OAAOsB,SAAAA,EAAWzB,IAAI0B,CAAAA,WAAUA,OAAOxB,KAAK,EAAEC,OAAOC,MAAAA;EACtE;EAEA,MAAyBuB,cAAcC,QAAiC;AACtE,UAAMC,iBAAyB,MAAMf,QAAQhB,IAAI,KAAKb,MACnDc,KAAI,EACJC,IAAI,OAAO,CAAC8B,KAAK7B,IAAAA,MAAK;AACrB,YAAM8B,oBAAoB,MAAM1B,eAAe2B,SAAS/B,KAAKC,KAAK;AAClE,UAAI0B,OAAOK,SAASH,GAAAA,KAAQF,OAAOK,SAASF,iBAAAA,GAAoB;AAC9D,aAAK9C,MAAMiD,OAAOJ,GAAAA;AAClB,eAAOA;MACT;IACF,CAAA,CAAA,GACC3B,OAAOC,MAAAA;AACV,UAAM,KAAK+B,qBAAoB;AAC/B,WAAON;EACT;EAEmBO,WAAWR,QAAwD;AACpF,WAAOA,OAAO5B,IAAI,CAACqC,SAAAA;AACjB,YAAMC,eAAe,KAAKlD,cAAcmD,IAAIF,IAAAA,KAASA;AACrD,YAAMX,SAAS,KAAKzC,MAAMsD,IAAID,YAAAA;AAC9B,UAAIA,iBAAiBD,QAAQ,CAACX,QAAQ;AACpC,cAAM,IAAIc,MAAM,4BAAA;MAClB;AACA,aAAOd;IACT,CAAA,EAAGvB,OAAOC,MAAAA;EACZ;EAEA,MAAyBqC,cAAclD,UAA0D;AAC/F,UAAMmD,QAAQ,MAAMrC,eAAesC,UAAUpD,QAAAA;AAC7C,UAAMqD,mBAAmB,MAAM9B,QAAQhB,IAAI4C,MAAM1C,IAAI,OAAO,CAAC6C,SAASR,IAAAA,MAAK;AACzE,aAAO,KAAKpD,MAAMsD,IAAIF,IAAAA,KAAS,MAAM,KAAKS,uBAAuBD,OAAAA;IACnE,CAAA,CAAA;AAEA,WAAOD;EACT;EAEA,MAAyBG,YAAYC,SAAqE;AACxG,SAAKC,OAAOC,KAAK,qBAAqBF,OAAAA;AACtC,UAAM,EACJG,OAAO3D,QAAQ4D,MAAK,IAClBJ,WAAW,CAAC;AAChB,QAAIlD,MAAM,MAAM,KAAKD,WAAU;AAC/B,QAAIuD,UAAU,QAAQ;AACpBtD,YAAMA,IAAIuD,QAAO;IACnB;AACA,UAAMC,aAAa9D,SACfpB,iBAAgBkB,oBAAoBQ,KAAKN,MAAAA,IAAU,IACnD;AACJ,SAAKyD,OAAOC,KAAK,0BAA0BI,UAAAA;AAC3C,UAAM5B,SAAS5B,IAAIyD,MAAMD,YAAYH,QAAQG,aAAaH,QAAQK,MAAAA;AAClE,SAAKP,OAAOC,KAAK,sBAAsBxB,MAAAA;AACvC,WAAOA;EACT;EAEA,MAAcoB,uBAAuBD,SAAqD;AACxF,UAAMY,WAAW,MAAMpD,eAAeqD,eAAeb,OAAAA;AACrD,SAAK5D,MAAM0E,IAAIF,SAASG,OAAOH,QAAAA;AAC/B,SAAKrE,cAAcuE,IAAIF,SAASI,WAAWJ,SAASG,KAAK;AACzD,WAAOH;EACT;EAEA,MAActB,uBAAuB;AACnC,SAAK1D,iBAAiB,IAAIS,SAAqB;MAAEC,KAAK,KAAKA;IAAI,CAAA;AAC/D,UAAMuD,QAAQ,KAAKzD,MAAMc,KAAI;AAC7B,eAAW,CAACsC,MAAMQ,OAAAA,KAAYH,OAAO;AACnC,YAAMV,WAAW,MAAM3B,eAAe2B,SAASa,QAAQ3C,KAAK;AAC5D,WAAKd,cAAcuE,IAAI3B,UAAUK,IAAAA;IACnC;EACF;AACF;;;;","names":["assertEx","exists","fulfilled","AbstractArchivist","ArchivistAllQuerySchema","ArchivistClearQuerySchema","ArchivistCommitQuerySchema","ArchivistDeleteQuerySchema","ArchivistInsertQuerySchema","ArchivistNextQuerySchema","creatableModule","PayloadBuilder","LRUCache","MemoryArchivistConfigSchema","MemoryArchivist","AbstractArchivist","configSchemas","defaultConfigSchema","_cache","_dataHashIndex","queries","ArchivistAllQuerySchema","ArchivistDeleteQuerySchema","ArchivistClearQuerySchema","ArchivistInsertQuerySchema","ArchivistCommitQuerySchema","ArchivistNextQuerySchema","cache","LRUCache","max","dataHashIndex","config","findIndexFromCursor","payloads","cursor","index","findIndex","_sequence","Infinity","allHandler","all","dump","map","item","value","filter","exists","PayloadBuilder","sortByStorageMeta","clearHandler","clear","emit","mod","commitHandler","assertEx","settled","Promise","allSettled","Object","values","parentArchivists","commit","parent","queryPayload","schema","query","bindQuery","fulfilled","result","deleteHandler","hashes","deletedHashes","key","itemValueDataHash","dataHash","includes","delete","rebuildDataHashIndex","getHandler","hash","resolvedHash","get","Error","insertHandler","pairs","hashPairs","insertedPayloads","payload","insertPayloadIntoCache","nextHandler","options","logger","warn","limit","order","reverse","startIndex","slice","undefined","withMeta","addStorageMeta","set","_hash","_dataHash"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xyo-network/archivist-memory",
|
|
3
|
-
"version": "3.6.0-rc.
|
|
3
|
+
"version": "3.6.0-rc.4",
|
|
4
4
|
"description": "Primary SDK for using XYO Protocol 2.0",
|
|
5
5
|
"homepage": "https://xyo.network",
|
|
6
6
|
"bugs": {
|
|
@@ -29,24 +29,24 @@
|
|
|
29
29
|
"module": "dist/neutral/index.mjs",
|
|
30
30
|
"types": "dist/neutral/index.d.ts",
|
|
31
31
|
"dependencies": {
|
|
32
|
-
"@xylabs/assert": "^4.4.
|
|
33
|
-
"@xylabs/exists": "^4.4.
|
|
34
|
-
"@xylabs/hex": "^4.4.
|
|
35
|
-
"@xylabs/object": "^4.4.
|
|
36
|
-
"@xylabs/promise": "^4.4.
|
|
37
|
-
"@xyo-network/archivist-abstract": "^3.6.0-rc.
|
|
38
|
-
"@xyo-network/archivist-model": "^3.6.0-rc.
|
|
39
|
-
"@xyo-network/boundwitness-model": "^3.6.0-rc.
|
|
40
|
-
"@xyo-network/module-model": "^3.6.0-rc.
|
|
41
|
-
"@xyo-network/payload-builder": "^3.6.0-rc.
|
|
42
|
-
"@xyo-network/payload-model": "^3.6.0-rc.
|
|
32
|
+
"@xylabs/assert": "^4.4.13",
|
|
33
|
+
"@xylabs/exists": "^4.4.13",
|
|
34
|
+
"@xylabs/hex": "^4.4.13",
|
|
35
|
+
"@xylabs/object": "^4.4.13",
|
|
36
|
+
"@xylabs/promise": "^4.4.13",
|
|
37
|
+
"@xyo-network/archivist-abstract": "^3.6.0-rc.4",
|
|
38
|
+
"@xyo-network/archivist-model": "^3.6.0-rc.4",
|
|
39
|
+
"@xyo-network/boundwitness-model": "^3.6.0-rc.4",
|
|
40
|
+
"@xyo-network/module-model": "^3.6.0-rc.4",
|
|
41
|
+
"@xyo-network/payload-builder": "^3.6.0-rc.4",
|
|
42
|
+
"@xyo-network/payload-model": "^3.6.0-rc.4",
|
|
43
43
|
"lru-cache": "^11.0.2"
|
|
44
44
|
},
|
|
45
45
|
"devDependencies": {
|
|
46
46
|
"@xylabs/ts-scripts-yarn3": "^4.2.4",
|
|
47
47
|
"@xylabs/tsconfig": "^4.2.4",
|
|
48
|
-
"@xylabs/vitest-extended": "^4.4.
|
|
49
|
-
"@xyo-network/account": "^3.6.0-rc.
|
|
48
|
+
"@xylabs/vitest-extended": "^4.4.13",
|
|
49
|
+
"@xyo-network/account": "^3.6.0-rc.4",
|
|
50
50
|
"typescript": "^5.7.2",
|
|
51
51
|
"vitest": "^2.1.8"
|
|
52
52
|
},
|
package/src/MemoryArchivist.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { assertEx } from '@xylabs/assert'
|
|
2
2
|
import { exists } from '@xylabs/exists'
|
|
3
|
-
import { Hash } from '@xylabs/hex'
|
|
3
|
+
import { Hash, Hex } from '@xylabs/hex'
|
|
4
4
|
import { EmptyObject, WithAdditional } from '@xylabs/object'
|
|
5
5
|
import { fulfilled, Promisable } from '@xylabs/promise'
|
|
6
6
|
import { AbstractArchivist } from '@xyo-network/archivist-abstract'
|
|
@@ -81,6 +81,14 @@ export class MemoryArchivist<
|
|
|
81
81
|
return this.config?.max ?? 10_000
|
|
82
82
|
}
|
|
83
83
|
|
|
84
|
+
private static findIndexFromCursor(payloads: WithStorageMeta[], cursor: Hex) {
|
|
85
|
+
const index = payloads.findIndex(({ _sequence }) => _sequence === cursor)
|
|
86
|
+
if (index === -1) {
|
|
87
|
+
return Infinity // move to the end
|
|
88
|
+
}
|
|
89
|
+
return index
|
|
90
|
+
}
|
|
91
|
+
|
|
84
92
|
protected override allHandler(): Promisable<WithStorageMeta<Payload>[]> {
|
|
85
93
|
const all = this.cache.dump().map(([, item]) => item.value).filter(exists)
|
|
86
94
|
return PayloadBuilder.sortByStorageMeta(all)
|
|
@@ -141,6 +149,7 @@ export class MemoryArchivist<
|
|
|
141
149
|
}
|
|
142
150
|
|
|
143
151
|
protected override async nextHandler(options?: ArchivistNextOptions): Promise<WithStorageMeta<Payload>[]> {
|
|
152
|
+
this.logger.warn('nextHandler:start', options)
|
|
144
153
|
const {
|
|
145
154
|
limit, cursor, order,
|
|
146
155
|
} = options ?? {}
|
|
@@ -148,8 +157,13 @@ export class MemoryArchivist<
|
|
|
148
157
|
if (order === 'desc') {
|
|
149
158
|
all = all.reverse()
|
|
150
159
|
}
|
|
151
|
-
const startIndex = cursor
|
|
152
|
-
|
|
160
|
+
const startIndex = cursor
|
|
161
|
+
? MemoryArchivist.findIndexFromCursor(all, cursor) + 1
|
|
162
|
+
: 0
|
|
163
|
+
this.logger.warn('nextHandler:startIndex', startIndex)
|
|
164
|
+
const result = all.slice(startIndex, limit ? startIndex + limit : undefined)
|
|
165
|
+
this.logger.warn('nextHandler:result', result)
|
|
166
|
+
return result
|
|
153
167
|
}
|
|
154
168
|
|
|
155
169
|
private async insertPayloadIntoCache(payload: Payload): Promise<WithStorageMeta<Payload>> {
|