@xyo-network/archivist-storage 3.6.9 → 3.6.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -5,8 +5,8 @@ import type { ArchivistConfig, ArchivistInstance, ArchivistModuleEventData, Arch
|
|
|
5
5
|
import type { BoundWitness } from '@xyo-network/boundwitness-model';
|
|
6
6
|
import type { AnyConfigSchema } from '@xyo-network/module-model';
|
|
7
7
|
import type { Payload, Schema, WithStorageMeta } from '@xyo-network/payload-model';
|
|
8
|
-
export
|
|
9
|
-
export
|
|
8
|
+
export declare const StorageArchivistConfigSchema: "network.xyo.archivist.storage.config";
|
|
9
|
+
export type StorageArchivistConfigSchema = typeof StorageArchivistConfigSchema;
|
|
10
10
|
export type StorageArchivistConfig = ArchivistConfig<{
|
|
11
11
|
maxEntries?: number;
|
|
12
12
|
maxEntrySize?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"StorageArchivist.d.ts","sourceRoot":"","sources":["../../src/StorageArchivist.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AAElE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAA;AACnE,OAAO,KAAK,EACV,eAAe,EAEf,iBAAiB,EACjB,wBAAwB,EACxB,oBAAoB,EACpB,eAAe,EAChB,MAAM,8BAA8B,CAAA;AAQrC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AACnE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAEhE,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EAAE,eAAe,EACjC,MAAM,4BAA4B,CAAA;AAMnC,
|
|
1
|
+
{"version":3,"file":"StorageArchivist.d.ts","sourceRoot":"","sources":["../../src/StorageArchivist.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AAElE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAA;AACnE,OAAO,KAAK,EACV,eAAe,EAEf,iBAAiB,EACjB,wBAAwB,EACxB,oBAAoB,EACpB,eAAe,EAChB,MAAM,8BAA8B,CAAA;AAQrC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AACnE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAEhE,OAAO,KAAK,EACV,OAAO,EAAE,MAAM,EAAE,eAAe,EACjC,MAAM,4BAA4B,CAAA;AAMnC,eAAO,MAAM,4BAA4B,EAAG,sCAA+C,CAAA;AAC3F,MAAM,MAAM,4BAA4B,GAAG,OAAO,4BAA4B,CAAA;AAE9E,MAAM,MAAM,sBAAsB,GAAG,eAAe,CAAC;IACnD,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,4BAA4B,CAAA;IACpC,IAAI,CAAC,EAAE,OAAO,GAAG,SAAS,GAAG,MAAM,CAAA;CACpC,CAAC,CAAA;AAEF,MAAM,MAAM,sBAAsB,GAAG,eAAe,CAAC,eAAe,CAAC,sBAAsB,CAAC,CAAC,CAAA;AAC7F,qBAAa,gBAAgB,CAC3B,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,EAC/D,UAAU,SAAS,wBAAwB,GAAG,wBAAwB,CAEtE,SAAQ,iBAAiB,CAAC,OAAO,EAAE,UAAU,CAC7C,YAAW,iBAAiB;IAC5B,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAyD;IACzG,gBAAyB,mBAAmB,EAAE,MAAM,CAA+B;IAEnF,OAAO,CAAC,eAAe,CAAuB;IAC9C,OAAO,CAAC,QAAQ,CAAuB;IAEvC,IAAI,UAAU,WAEb;IAED,IAAI,YAAY,WAEf;IAED,IAAI,SAAS,WAEZ;IAED,IAAa,OAAO,IAAI,MAAM,EAAE,CAS/B;IAED,IAAI,IAAI,iCAEP;IAGD,OAAO,KAAK,cAAc,GAGzB;IAGD,OAAO,KAAK,OAAO,GAGlB;cAEkB,UAAU,IAAI,eAAe,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;cAiBvD,YAAY,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;cAM9B,aAAa,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;cAgBxC,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAWvE,SAAS,CAAC,aAAa,CACrB,KAAK,GAAE,KAAK,GAAG,MAAc,EAC7B,KAAK,GAAE,MAAW,EAClB,MAAM,CAAC,EAAE,GAAG,GACX,eAAe,EAAE;cAcD,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cAiB9D,aAAa,CAAC,QAAQ,EAAE,eAAe,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cAW9F,WAAW,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,UAAU,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cAO7E,YAAY;CAItC"}
|
package/dist/neutral/index.mjs
CHANGED
|
@@ -1,24 +1,21 @@
|
|
|
1
|
-
var __defProp = Object.defineProperty;
|
|
2
|
-
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
3
|
-
|
|
4
1
|
// src/StorageArchivist.ts
|
|
5
2
|
import { assertEx } from "@xylabs/assert";
|
|
6
3
|
import { exists } from "@xylabs/exists";
|
|
7
4
|
import { fulfilled } from "@xylabs/promise";
|
|
8
5
|
import { AbstractArchivist } from "@xyo-network/archivist-abstract";
|
|
9
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
ArchivistAllQuerySchema,
|
|
8
|
+
ArchivistClearQuerySchema,
|
|
9
|
+
ArchivistCommitQuerySchema,
|
|
10
|
+
ArchivistDeleteQuerySchema,
|
|
11
|
+
ArchivistInsertQuerySchema
|
|
12
|
+
} from "@xyo-network/archivist-model";
|
|
10
13
|
import { PayloadBuilder } from "@xyo-network/payload-builder";
|
|
11
14
|
import store from "store2";
|
|
12
15
|
var storeTypes = store;
|
|
13
16
|
var StorageArchivistConfigSchema = "network.xyo.archivist.storage.config";
|
|
14
17
|
var StorageArchivist = class extends AbstractArchivist {
|
|
15
|
-
static
|
|
16
|
-
__name(this, "StorageArchivist");
|
|
17
|
-
}
|
|
18
|
-
static configSchemas = [
|
|
19
|
-
...super.configSchemas,
|
|
20
|
-
StorageArchivistConfigSchema
|
|
21
|
-
];
|
|
18
|
+
static configSchemas = [...super.configSchemas, StorageArchivistConfigSchema];
|
|
22
19
|
static defaultConfigSchema = StorageArchivistConfigSchema;
|
|
23
20
|
_privateStorage;
|
|
24
21
|
_storage;
|
|
@@ -69,29 +66,29 @@ var StorageArchivist = class extends AbstractArchivist {
|
|
|
69
66
|
clearHandler() {
|
|
70
67
|
this.logger?.log(`this.storage.length: ${this.storage.length}`);
|
|
71
68
|
this.storage.clear();
|
|
72
|
-
return this.emit("cleared", {
|
|
73
|
-
mod: this
|
|
74
|
-
});
|
|
69
|
+
return this.emit("cleared", { mod: this });
|
|
75
70
|
}
|
|
76
71
|
async commitHandler() {
|
|
77
72
|
this.logger?.log(`this.storage.length: ${this.storage.length}`);
|
|
78
73
|
const payloads = await this.all();
|
|
79
74
|
assertEx(payloads.length > 0, () => "Nothing to commit");
|
|
80
|
-
const settled = (await Promise.allSettled(
|
|
81
|
-
|
|
82
|
-
schema: ArchivistInsertQuerySchema
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
75
|
+
const settled = (await Promise.allSettled(
|
|
76
|
+
Object.values((await this.parentArchivists()).commit ?? [])?.map(async (parent) => {
|
|
77
|
+
const queryPayload = { schema: ArchivistInsertQuerySchema };
|
|
78
|
+
const query = await this.bindQuery(queryPayload, payloads);
|
|
79
|
+
return (await parent?.query(query[0], query[1]))?.[0];
|
|
80
|
+
})
|
|
81
|
+
)).filter(exists);
|
|
87
82
|
await this.clear();
|
|
88
83
|
return settled.filter(fulfilled).map((result) => result.value).filter(exists);
|
|
89
84
|
}
|
|
90
85
|
async deleteHandler(hashes) {
|
|
91
|
-
return (await Promise.all(
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
86
|
+
return (await Promise.all(
|
|
87
|
+
hashes.map((hash) => {
|
|
88
|
+
this.storage.remove(hash);
|
|
89
|
+
return hash;
|
|
90
|
+
})
|
|
91
|
+
)).filter(exists);
|
|
95
92
|
}
|
|
96
93
|
getFromCursor(order = "asc", limit = 10, cursor) {
|
|
97
94
|
const all = Object.values(this.storage.getAll());
|
|
@@ -127,7 +124,11 @@ var StorageArchivist = class extends AbstractArchivist {
|
|
|
127
124
|
}));
|
|
128
125
|
}
|
|
129
126
|
nextHandler(options) {
|
|
130
|
-
const {
|
|
127
|
+
const {
|
|
128
|
+
limit,
|
|
129
|
+
cursor,
|
|
130
|
+
order
|
|
131
|
+
} = options ?? {};
|
|
131
132
|
return this.getFromCursor(order, limit ?? 10, cursor);
|
|
132
133
|
}
|
|
133
134
|
async startHandler() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/StorageArchivist.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { Hash, Hex } from '@xylabs/hex'\nimport type { Promisable, PromisableArray } from '@xylabs/promise'\nimport { fulfilled } from '@xylabs/promise'\nimport { AbstractArchivist } from '@xyo-network/archivist-abstract'\nimport type {\n ArchivistConfig,\n ArchivistInsertQuery,\n ArchivistInstance,\n ArchivistModuleEventData,\n ArchivistNextOptions,\n ArchivistParams,\n} from '@xyo-network/archivist-model'\nimport {\n ArchivistAllQuerySchema,\n ArchivistClearQuerySchema,\n ArchivistCommitQuerySchema,\n ArchivistDeleteQuerySchema,\n ArchivistInsertQuerySchema,\n} from '@xyo-network/archivist-model'\nimport type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport type { AnyConfigSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type {\n Payload, Schema, WithStorageMeta,\n} from '@xyo-network/payload-model'\nimport type { StoreBase, StoreType } from 'store2'\nimport store from 'store2'\n\nconst storeTypes = store as unknown as StoreType\n\nexport type StorageArchivistConfigSchema = 'network.xyo.archivist.storage.config'\nexport const StorageArchivistConfigSchema: StorageArchivistConfigSchema = 'network.xyo.archivist.storage.config'\n\nexport type StorageArchivistConfig = ArchivistConfig<{\n maxEntries?: number\n maxEntrySize?: number\n namespace?: string\n schema: StorageArchivistConfigSchema\n type?: 'local' | 'session' | 'page'\n}>\n\nexport type StorageArchivistParams = ArchivistParams<AnyConfigSchema<StorageArchivistConfig>>\nexport class StorageArchivist<\n TParams extends StorageArchivistParams = StorageArchivistParams,\n TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,\n>\n extends AbstractArchivist<TParams, TEventData>\n implements ArchivistInstance {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, StorageArchivistConfigSchema]\n static override readonly defaultConfigSchema: Schema = StorageArchivistConfigSchema\n\n private _privateStorage: StoreBase | undefined\n private _storage: StoreBase | undefined\n\n get maxEntries() {\n return this.config?.maxEntries ?? 1000\n }\n\n get maxEntrySize() {\n return this.config?.maxEntrySize ?? 16_000\n }\n\n get namespace() {\n return this.config?.namespace ?? 'xyo-archivist'\n }\n\n override get queries(): string[] {\n return [\n ArchivistAllQuerySchema,\n ArchivistDeleteQuerySchema,\n ArchivistClearQuerySchema,\n ArchivistInsertQuerySchema,\n ArchivistCommitQuerySchema,\n ...super.queries,\n ]\n }\n\n get type() {\n return this.config?.type ?? 'local'\n }\n\n /* This has to be a getter so that it can access it during construction */\n private get privateStorage(): StoreBase {\n this._privateStorage = this._storage ?? storeTypes[this.type].namespace(`${this.namespace}|private`)\n return this._privateStorage\n }\n\n /* This has to be a getter so that it can access it during construction */\n private get storage(): StoreBase {\n this._storage = this._storage ?? storeTypes[this.type].namespace(this.namespace)\n return this._storage\n }\n\n protected override allHandler(): PromisableArray<WithStorageMeta<Payload>> {\n const found = new Set<string>()\n this.logger?.log(`this.storage.length: ${this.storage.length}`)\n return Object.entries(this.storage.getAll())\n .map(([, value]) => value)\n .filter((payload) => {\n if (found.has(payload._dataHash)) {\n return false\n } else {\n found.add(payload._dataHash)\n return true\n }\n })\n .sort(PayloadBuilder.compareStorageMeta)\n .map(payload => PayloadBuilder.omitStorageMeta(payload))\n }\n\n protected override clearHandler(): void | Promise<void> {\n this.logger?.log(`this.storage.length: ${this.storage.length}`)\n this.storage.clear()\n return this.emit('cleared', { mod: this })\n }\n\n protected override async commitHandler(): Promise<BoundWitness[]> {\n this.logger?.log(`this.storage.length: ${this.storage.length}`)\n const payloads = await this.all()\n assertEx(payloads.length > 0, () => 'Nothing to commit')\n const settled = (await Promise.allSettled(\n Object.values((await this.parentArchivists()).commit ?? [])?.map(async (parent) => {\n const queryPayload: ArchivistInsertQuery = { schema: ArchivistInsertQuerySchema }\n const query = await this.bindQuery(queryPayload, payloads)\n return (await parent?.query(query[0], query[1]))?.[0]\n }),\n )).filter(exists)\n // TODO - rather than clear, delete the payloads that come back as successfully inserted\n await this.clear()\n return (settled.filter(fulfilled).map(result => result.value)).filter(exists)\n }\n\n protected override async deleteHandler(hashes: Hash[]): Promise<Hash[]> {\n return (\n await Promise.all(\n hashes.map((hash) => {\n this.storage.remove(hash)\n return hash\n }),\n )\n ).filter(exists)\n }\n\n protected getFromCursor(\n order: 'asc' | 'desc' = 'asc',\n limit: number = 10,\n cursor?: Hex,\n ): WithStorageMeta[] {\n const all = Object.values(this.storage.getAll()) as WithStorageMeta[]\n const payloads: WithStorageMeta[] = all\n .map(value => value)\n .sort((a, b) => {\n return order === 'asc' ? PayloadBuilder.compareStorageMeta(a, b) : PayloadBuilder.compareStorageMeta(b, a)\n })\n const index = payloads.findIndex(payload => payload._sequence === cursor)\n if (index !== -1) {\n return payloads.slice(index + 1, index + 1 + limit)\n }\n return payloads.slice(0, limit)\n }\n\n protected override getHandler(hashes: string[]): Promisable<WithStorageMeta<Payload>[]> {\n const found = new Set<string>()\n return (\n hashes.map((hash) => {\n return this.storage.get(hash)\n })\n ).filter(exists)\n .filter((payload) => {\n if (found.has(payload._dataHash)) {\n return false\n } else {\n found.add(payload._dataHash)\n return true\n }\n }).map(payload => PayloadBuilder.omitStorageMeta(payload))\n }\n\n protected override async insertHandler(payloads: WithStorageMeta<Payload>[]): Promise<WithStorageMeta<Payload>[]> {\n return await Promise.all(payloads.map((payload) => {\n const value = JSON.stringify(payload)\n // console.log('insert.storagePayloads:', storagePayload)\n assertEx(value.length < this.maxEntrySize, () => `Payload too large [${payload._hash}, ${value.length}]`)\n this.storage.set(payload._hash, payload)\n this.storage.set(payload._dataHash, payload)\n return payload\n }))\n }\n\n protected override nextHandler(options?: ArchivistNextOptions): Promisable<WithStorageMeta<Payload>[]> {\n const {\n limit, cursor, order,\n } = options ?? {}\n return this.getFromCursor(order, limit ?? 10, cursor)\n }\n\n protected override async startHandler() {\n await super.startHandler()\n return true\n }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAGvB,SAASC,iBAAiB;AAC1B,SAASC,yBAAyB;AASlC,SACEC,yBACAC,2BACAC,4BACAC,4BACAC,kCACK;AAGP,SAASC,sBAAsB;AAK/B,OAAOC,WAAW;AAElB,IAAMC,aAAaC;AAGZ,IAAMC,+BAA6D;AAWnE,IAAMC,mBAAN,cAIGC,kBAAAA;EAhDV,OAgDUA;;;EAER,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE/CK;EACAC;EAER,IAAIC,aAAa;AACf,WAAO,KAAKC,QAAQD,cAAc;EACpC;EAEA,IAAIE,eAAe;AACjB,WAAO,KAAKD,QAAQC,gBAAgB;EACtC;EAEA,IAAIC,YAAY;AACd,WAAO,KAAKF,QAAQE,aAAa;EACnC;EAEA,IAAaC,UAAoB;AAC/B,WAAO;MACLC;MACAC;MACAC;MACAC;MACAC;SACG,MAAML;;EAEb;EAEA,IAAIM,OAAO;AACT,WAAO,KAAKT,QAAQS,QAAQ;EAC9B;;EAGA,IAAYC,iBAA4B;AACtC,SAAKb,kBAAkB,KAAKC,YAAYR,WAAW,KAAKmB,IAAI,EAAEP,UAAU,GAAG,KAAKA,SAAS,UAAU;AACnG,WAAO,KAAKL;EACd;;EAGA,IAAYc,UAAqB;AAC/B,SAAKb,WAAW,KAAKA,YAAYR,WAAW,KAAKmB,IAAI,EAAEP,UAAU,KAAKA,SAAS;AAC/E,WAAO,KAAKJ;EACd;EAEmBc,aAAwD;AACzE,UAAMC,QAAQ,oBAAIC,IAAAA;AAClB,SAAKC,QAAQC,IAAI,wBAAwB,KAAKL,QAAQM,MAAM,EAAE;AAC9D,WAAOC,OAAOC,QAAQ,KAAKR,QAAQS,OAAM,CAAA,EACtCC,IAAI,CAAC,CAAA,EAAGC,KAAAA,MAAWA,KAAAA,EACnBC,OAAO,CAACC,YAAAA;AACP,UAAIX,MAAMY,IAAID,QAAQE,SAAS,GAAG;AAChC,eAAO;MACT,OAAO;AACLb,cAAMc,IAAIH,QAAQE,SAAS;AAC3B,eAAO;MACT;IACF,CAAA,EACCE,KAAKC,eAAeC,kBAAkB,EACtCT,IAAIG,CAAAA,YAAWK,eAAeE,gBAAgBP,OAAAA,CAAAA;EACnD;EAEmBQ,eAAqC;AACtD,SAAKjB,QAAQC,IAAI,wBAAwB,KAAKL,QAAQM,MAAM,EAAE;AAC9D,SAAKN,QAAQsB,MAAK;AAClB,WAAO,KAAKC,KAAK,WAAW;MAAEC,KAAK;IAAK,CAAA;EAC1C;EAEA,MAAyBC,gBAAyC;AAChE,SAAKrB,QAAQC,IAAI,wBAAwB,KAAKL,QAAQM,MAAM,EAAE;AAC9D,UAAMoB,WAAW,MAAM,KAAKC,IAAG;AAC/BC,aAASF,SAASpB,SAAS,GAAG,MAAM,mBAAA;AACpC,UAAMuB,WAAW,MAAMC,QAAQC,WAC7BxB,OAAOyB,QAAQ,MAAM,KAAKC,iBAAgB,GAAIC,UAAU,CAAA,CAAE,GAAGxB,IAAI,OAAOyB,WAAAA;AACtE,YAAMC,eAAqC;QAAEC,QAAQzC;MAA2B;AAChF,YAAM0C,QAAQ,MAAM,KAAKC,UAAUH,cAAcV,QAAAA;AACjD,cAAQ,MAAMS,QAAQG,MAAMA,MAAM,CAAA,GAAIA,MAAM,CAAA,CAAE,KAAK,CAAA;IACrD,CAAA,CAAA,GACC1B,OAAO4B,MAAAA;AAEV,UAAM,KAAKlB,MAAK;AAChB,WAAQO,QAAQjB,OAAO6B,SAAAA,EAAW/B,IAAIgC,CAAAA,WAAUA,OAAO/B,KAAK,EAAGC,OAAO4B,MAAAA;EACxE;EAEA,MAAyBG,cAAcC,QAAiC;AACtE,YACE,MAAMd,QAAQH,IACZiB,OAAOlC,IAAI,CAACmC,SAAAA;AACV,WAAK7C,QAAQ8C,OAAOD,IAAAA;AACpB,aAAOA;IACT,CAAA,CAAA,GAEFjC,OAAO4B,MAAAA;EACX;EAEUO,cACRC,QAAwB,OACxBC,QAAgB,IAChBC,QACmB;AACnB,UAAMvB,MAAMpB,OAAOyB,OAAO,KAAKhC,QAAQS,OAAM,CAAA;AAC7C,UAAMiB,WAA8BC,IACjCjB,IAAIC,CAAAA,UAASA,KAAAA,EACbM,KAAK,CAACkC,GAAGC,MAAAA;AACR,aAAOJ,UAAU,QAAQ9B,eAAeC,mBAAmBgC,GAAGC,CAAAA,IAAKlC,eAAeC,mBAAmBiC,GAAGD,CAAAA;IAC1G,CAAA;AACF,UAAME,QAAQ3B,SAAS4B,UAAUzC,CAAAA,YAAWA,QAAQ0C,cAAcL,MAAAA;AAClE,QAAIG,UAAU,IAAI;AAChB,aAAO3B,SAAS8B,MAAMH,QAAQ,GAAGA,QAAQ,IAAIJ,KAAAA;IAC/C;AACA,WAAOvB,SAAS8B,MAAM,GAAGP,KAAAA;EAC3B;EAEmBQ,WAAWb,QAA0D;AACtF,UAAM1C,QAAQ,oBAAIC,IAAAA;AAClB,WACEyC,OAAOlC,IAAI,CAACmC,SAAAA;AACV,aAAO,KAAK7C,QAAQ0D,IAAIb,IAAAA;IAC1B,CAAA,EACAjC,OAAO4B,MAAAA,EACN5B,OAAO,CAACC,YAAAA;AACP,UAAIX,MAAMY,IAAID,QAAQE,SAAS,GAAG;AAChC,eAAO;MACT,OAAO;AACLb,cAAMc,IAAIH,QAAQE,SAAS;AAC3B,eAAO;MACT;IACF,CAAA,EAAGL,IAAIG,CAAAA,YAAWK,eAAeE,gBAAgBP,OAAAA,CAAAA;EACrD;EAEA,MAAyB8C,cAAcjC,UAA2E;AAChH,WAAO,MAAMI,QAAQH,IAAID,SAAShB,IAAI,CAACG,YAAAA;AACrC,YAAMF,QAAQiD,KAAKC,UAAUhD,OAAAA;AAE7Be,eAASjB,MAAML,SAAS,KAAKhB,cAAc,MAAM,sBAAsBuB,QAAQiD,KAAK,KAAKnD,MAAML,MAAM,GAAG;AACxG,WAAKN,QAAQ+D,IAAIlD,QAAQiD,OAAOjD,OAAAA;AAChC,WAAKb,QAAQ+D,IAAIlD,QAAQE,WAAWF,OAAAA;AACpC,aAAOA;IACT,CAAA,CAAA;EACF;EAEmBmD,YAAYC,SAAwE;AACrG,UAAM,EACJhB,OAAOC,QAAQF,MAAK,IAClBiB,WAAW,CAAC;AAChB,WAAO,KAAKlB,cAAcC,OAAOC,SAAS,IAAIC,MAAAA;EAChD;EAEA,MAAyBgB,eAAe;AACtC,UAAM,MAAMA,aAAAA;AACZ,WAAO;EACT;AACF;","names":["assertEx","exists","fulfilled","AbstractArchivist","ArchivistAllQuerySchema","ArchivistClearQuerySchema","ArchivistCommitQuerySchema","ArchivistDeleteQuerySchema","ArchivistInsertQuerySchema","PayloadBuilder","store","storeTypes","store","StorageArchivistConfigSchema","StorageArchivist","AbstractArchivist","configSchemas","defaultConfigSchema","_privateStorage","_storage","maxEntries","config","maxEntrySize","namespace","queries","ArchivistAllQuerySchema","ArchivistDeleteQuerySchema","ArchivistClearQuerySchema","ArchivistInsertQuerySchema","ArchivistCommitQuerySchema","type","privateStorage","storage","allHandler","found","Set","logger","log","length","Object","entries","getAll","map","value","filter","payload","has","_dataHash","add","sort","PayloadBuilder","compareStorageMeta","omitStorageMeta","clearHandler","clear","emit","mod","commitHandler","payloads","all","assertEx","settled","Promise","allSettled","values","parentArchivists","commit","parent","queryPayload","schema","query","bindQuery","exists","fulfilled","result","deleteHandler","hashes","hash","remove","getFromCursor","order","limit","cursor","a","b","index","findIndex","_sequence","slice","getHandler","get","insertHandler","JSON","stringify","_hash","set","nextHandler","options","startHandler"]}
|
|
1
|
+
{"version":3,"sources":["../../src/StorageArchivist.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { Hash, Hex } from '@xylabs/hex'\nimport type { Promisable, PromisableArray } from '@xylabs/promise'\nimport { fulfilled } from '@xylabs/promise'\nimport { AbstractArchivist } from '@xyo-network/archivist-abstract'\nimport type {\n ArchivistConfig,\n ArchivistInsertQuery,\n ArchivistInstance,\n ArchivistModuleEventData,\n ArchivistNextOptions,\n ArchivistParams,\n} from '@xyo-network/archivist-model'\nimport {\n ArchivistAllQuerySchema,\n ArchivistClearQuerySchema,\n ArchivistCommitQuerySchema,\n ArchivistDeleteQuerySchema,\n ArchivistInsertQuerySchema,\n} from '@xyo-network/archivist-model'\nimport type { BoundWitness } from '@xyo-network/boundwitness-model'\nimport type { AnyConfigSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport type {\n Payload, Schema, WithStorageMeta,\n} from '@xyo-network/payload-model'\nimport type { StoreBase, StoreType } from 'store2'\nimport store from 'store2'\n\nconst storeTypes = store as unknown as StoreType\n\nexport const StorageArchivistConfigSchema = 'network.xyo.archivist.storage.config' as const\nexport type StorageArchivistConfigSchema = typeof StorageArchivistConfigSchema\n\nexport type StorageArchivistConfig = ArchivistConfig<{\n maxEntries?: number\n maxEntrySize?: number\n namespace?: string\n schema: StorageArchivistConfigSchema\n type?: 'local' | 'session' | 'page'\n}>\n\nexport type StorageArchivistParams = ArchivistParams<AnyConfigSchema<StorageArchivistConfig>>\nexport class StorageArchivist<\n TParams extends StorageArchivistParams = StorageArchivistParams,\n TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,\n>\n extends AbstractArchivist<TParams, TEventData>\n implements ArchivistInstance {\n static override readonly configSchemas: Schema[] = [...super.configSchemas, StorageArchivistConfigSchema]\n static override readonly defaultConfigSchema: Schema = StorageArchivistConfigSchema\n\n private _privateStorage: StoreBase | undefined\n private _storage: StoreBase | undefined\n\n get maxEntries() {\n return this.config?.maxEntries ?? 1000\n }\n\n get maxEntrySize() {\n return this.config?.maxEntrySize ?? 16_000\n }\n\n get namespace() {\n return this.config?.namespace ?? 'xyo-archivist'\n }\n\n override get queries(): string[] {\n return [\n ArchivistAllQuerySchema,\n ArchivistDeleteQuerySchema,\n ArchivistClearQuerySchema,\n ArchivistInsertQuerySchema,\n ArchivistCommitQuerySchema,\n ...super.queries,\n ]\n }\n\n get type() {\n return this.config?.type ?? 'local'\n }\n\n /* This has to be a getter so that it can access it during construction */\n private get privateStorage(): StoreBase {\n this._privateStorage = this._storage ?? storeTypes[this.type].namespace(`${this.namespace}|private`)\n return this._privateStorage\n }\n\n /* This has to be a getter so that it can access it during construction */\n private get storage(): StoreBase {\n this._storage = this._storage ?? storeTypes[this.type].namespace(this.namespace)\n return this._storage\n }\n\n protected override allHandler(): PromisableArray<WithStorageMeta<Payload>> {\n const found = new Set<string>()\n this.logger?.log(`this.storage.length: ${this.storage.length}`)\n return Object.entries(this.storage.getAll())\n .map(([, value]) => value)\n .filter((payload) => {\n if (found.has(payload._dataHash)) {\n return false\n } else {\n found.add(payload._dataHash)\n return true\n }\n })\n .sort(PayloadBuilder.compareStorageMeta)\n .map(payload => PayloadBuilder.omitStorageMeta(payload))\n }\n\n protected override clearHandler(): void | Promise<void> {\n this.logger?.log(`this.storage.length: ${this.storage.length}`)\n this.storage.clear()\n return this.emit('cleared', { mod: this })\n }\n\n protected override async commitHandler(): Promise<BoundWitness[]> {\n this.logger?.log(`this.storage.length: ${this.storage.length}`)\n const payloads = await this.all()\n assertEx(payloads.length > 0, () => 'Nothing to commit')\n const settled = (await Promise.allSettled(\n Object.values((await this.parentArchivists()).commit ?? [])?.map(async (parent) => {\n const queryPayload: ArchivistInsertQuery = { schema: ArchivistInsertQuerySchema }\n const query = await this.bindQuery(queryPayload, payloads)\n return (await parent?.query(query[0], query[1]))?.[0]\n }),\n )).filter(exists)\n // TODO - rather than clear, delete the payloads that come back as successfully inserted\n await this.clear()\n return (settled.filter(fulfilled).map(result => result.value)).filter(exists)\n }\n\n protected override async deleteHandler(hashes: Hash[]): Promise<Hash[]> {\n return (\n await Promise.all(\n hashes.map((hash) => {\n this.storage.remove(hash)\n return hash\n }),\n )\n ).filter(exists)\n }\n\n protected getFromCursor(\n order: 'asc' | 'desc' = 'asc',\n limit: number = 10,\n cursor?: Hex,\n ): WithStorageMeta[] {\n const all = Object.values(this.storage.getAll()) as WithStorageMeta[]\n const payloads: WithStorageMeta[] = all\n .map(value => value)\n .sort((a, b) => {\n return order === 'asc' ? PayloadBuilder.compareStorageMeta(a, b) : PayloadBuilder.compareStorageMeta(b, a)\n })\n const index = payloads.findIndex(payload => payload._sequence === cursor)\n if (index !== -1) {\n return payloads.slice(index + 1, index + 1 + limit)\n }\n return payloads.slice(0, limit)\n }\n\n protected override getHandler(hashes: string[]): Promisable<WithStorageMeta<Payload>[]> {\n const found = new Set<string>()\n return (\n hashes.map((hash) => {\n return this.storage.get(hash)\n })\n ).filter(exists)\n .filter((payload) => {\n if (found.has(payload._dataHash)) {\n return false\n } else {\n found.add(payload._dataHash)\n return true\n }\n }).map(payload => PayloadBuilder.omitStorageMeta(payload))\n }\n\n protected override async insertHandler(payloads: WithStorageMeta<Payload>[]): Promise<WithStorageMeta<Payload>[]> {\n return await Promise.all(payloads.map((payload) => {\n const value = JSON.stringify(payload)\n // console.log('insert.storagePayloads:', storagePayload)\n assertEx(value.length < this.maxEntrySize, () => `Payload too large [${payload._hash}, ${value.length}]`)\n this.storage.set(payload._hash, payload)\n this.storage.set(payload._dataHash, payload)\n return payload\n }))\n }\n\n protected override nextHandler(options?: ArchivistNextOptions): Promisable<WithStorageMeta<Payload>[]> {\n const {\n limit, cursor, order,\n } = options ?? {}\n return this.getFromCursor(order, limit ?? 10, cursor)\n }\n\n protected override async startHandler() {\n await super.startHandler()\n return true\n }\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,cAAc;AAGvB,SAAS,iBAAiB;AAC1B,SAAS,yBAAyB;AASlC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAGP,SAAS,sBAAsB;AAK/B,OAAO,WAAW;AAElB,IAAM,aAAa;AAEZ,IAAM,+BAA+B;AAYrC,IAAM,mBAAN,cAIG,kBACqB;AAAA,EAC7B,OAAyB,gBAA0B,CAAC,GAAG,MAAM,eAAe,4BAA4B;AAAA,EACxG,OAAyB,sBAA8B;AAAA,EAE/C;AAAA,EACA;AAAA,EAER,IAAI,aAAa;AACf,WAAO,KAAK,QAAQ,cAAc;AAAA,EACpC;AAAA,EAEA,IAAI,eAAe;AACjB,WAAO,KAAK,QAAQ,gBAAgB;AAAA,EACtC;AAAA,EAEA,IAAI,YAAY;AACd,WAAO,KAAK,QAAQ,aAAa;AAAA,EACnC;AAAA,EAEA,IAAa,UAAoB;AAC/B,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,GAAG,MAAM;AAAA,IACX;AAAA,EACF;AAAA,EAEA,IAAI,OAAO;AACT,WAAO,KAAK,QAAQ,QAAQ;AAAA,EAC9B;AAAA;AAAA,EAGA,IAAY,iBAA4B;AACtC,SAAK,kBAAkB,KAAK,YAAY,WAAW,KAAK,IAAI,EAAE,UAAU,GAAG,KAAK,SAAS,UAAU;AACnG,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAY,UAAqB;AAC/B,SAAK,WAAW,KAAK,YAAY,WAAW,KAAK,IAAI,EAAE,UAAU,KAAK,SAAS;AAC/E,WAAO,KAAK;AAAA,EACd;AAAA,EAEmB,aAAwD;AACzE,UAAM,QAAQ,oBAAI,IAAY;AAC9B,SAAK,QAAQ,IAAI,wBAAwB,KAAK,QAAQ,MAAM,EAAE;AAC9D,WAAO,OAAO,QAAQ,KAAK,QAAQ,OAAO,CAAC,EACxC,IAAI,CAAC,CAAC,EAAE,KAAK,MAAM,KAAK,EACxB,OAAO,CAAC,YAAY;AACnB,UAAI,MAAM,IAAI,QAAQ,SAAS,GAAG;AAChC,eAAO;AAAA,MACT,OAAO;AACL,cAAM,IAAI,QAAQ,SAAS;AAC3B,eAAO;AAAA,MACT;AAAA,IACF,CAAC,EACA,KAAK,eAAe,kBAAkB,EACtC,IAAI,aAAW,eAAe,gBAAgB,OAAO,CAAC;AAAA,EAC3D;AAAA,EAEmB,eAAqC;AACtD,SAAK,QAAQ,IAAI,wBAAwB,KAAK,QAAQ,MAAM,EAAE;AAC9D,SAAK,QAAQ,MAAM;AACnB,WAAO,KAAK,KAAK,WAAW,EAAE,KAAK,KAAK,CAAC;AAAA,EAC3C;AAAA,EAEA,MAAyB,gBAAyC;AAChE,SAAK,QAAQ,IAAI,wBAAwB,KAAK,QAAQ,MAAM,EAAE;AAC9D,UAAM,WAAW,MAAM,KAAK,IAAI;AAChC,aAAS,SAAS,SAAS,GAAG,MAAM,mBAAmB;AACvD,UAAM,WAAW,MAAM,QAAQ;AAAA,MAC7B,OAAO,QAAQ,MAAM,KAAK,iBAAiB,GAAG,UAAU,CAAC,CAAC,GAAG,IAAI,OAAO,WAAW;AACjF,cAAM,eAAqC,EAAE,QAAQ,2BAA2B;AAChF,cAAM,QAAQ,MAAM,KAAK,UAAU,cAAc,QAAQ;AACzD,gBAAQ,MAAM,QAAQ,MAAM,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC;AAAA,MACtD,CAAC;AAAA,IACH,GAAG,OAAO,MAAM;AAEhB,UAAM,KAAK,MAAM;AACjB,WAAQ,QAAQ,OAAO,SAAS,EAAE,IAAI,YAAU,OAAO,KAAK,EAAG,OAAO,MAAM;AAAA,EAC9E;AAAA,EAEA,MAAyB,cAAc,QAAiC;AACtE,YACE,MAAM,QAAQ;AAAA,MACZ,OAAO,IAAI,CAAC,SAAS;AACnB,aAAK,QAAQ,OAAO,IAAI;AACxB,eAAO;AAAA,MACT,CAAC;AAAA,IACH,GACA,OAAO,MAAM;AAAA,EACjB;AAAA,EAEU,cACR,QAAwB,OACxB,QAAgB,IAChB,QACmB;AACnB,UAAM,MAAM,OAAO,OAAO,KAAK,QAAQ,OAAO,CAAC;AAC/C,UAAM,WAA8B,IACjC,IAAI,WAAS,KAAK,EAClB,KAAK,CAAC,GAAG,MAAM;AACd,aAAO,UAAU,QAAQ,eAAe,mBAAmB,GAAG,CAAC,IAAI,eAAe,mBAAmB,GAAG,CAAC;AAAA,IAC3G,CAAC;AACH,UAAM,QAAQ,SAAS,UAAU,aAAW,QAAQ,cAAc,MAAM;AACxE,QAAI,UAAU,IAAI;AAChB,aAAO,SAAS,MAAM,QAAQ,GAAG,QAAQ,IAAI,KAAK;AAAA,IACpD;AACA,WAAO,SAAS,MAAM,GAAG,KAAK;AAAA,EAChC;AAAA,EAEmB,WAAW,QAA0D;AACtF,UAAM,QAAQ,oBAAI,IAAY;AAC9B,WACE,OAAO,IAAI,CAAC,SAAS;AACnB,aAAO,KAAK,QAAQ,IAAI,IAAI;AAAA,IAC9B,CAAC,EACD,OAAO,MAAM,EACZ,OAAO,CAAC,YAAY;AACnB,UAAI,MAAM,IAAI,QAAQ,SAAS,GAAG;AAChC,eAAO;AAAA,MACT,OAAO;AACL,cAAM,IAAI,QAAQ,SAAS;AAC3B,eAAO;AAAA,MACT;AAAA,IACF,CAAC,EAAE,IAAI,aAAW,eAAe,gBAAgB,OAAO,CAAC;AAAA,EAC7D;AAAA,EAEA,MAAyB,cAAc,UAA2E;AAChH,WAAO,MAAM,QAAQ,IAAI,SAAS,IAAI,CAAC,YAAY;AACjD,YAAM,QAAQ,KAAK,UAAU,OAAO;AAEpC,eAAS,MAAM,SAAS,KAAK,cAAc,MAAM,sBAAsB,QAAQ,KAAK,KAAK,MAAM,MAAM,GAAG;AACxG,WAAK,QAAQ,IAAI,QAAQ,OAAO,OAAO;AACvC,WAAK,QAAQ,IAAI,QAAQ,WAAW,OAAO;AAC3C,aAAO;AAAA,IACT,CAAC,CAAC;AAAA,EACJ;AAAA,EAEmB,YAAY,SAAwE;AACrG,UAAM;AAAA,MACJ;AAAA,MAAO;AAAA,MAAQ;AAAA,IACjB,IAAI,WAAW,CAAC;AAChB,WAAO,KAAK,cAAc,OAAO,SAAS,IAAI,MAAM;AAAA,EACtD;AAAA,EAEA,MAAyB,eAAe;AACtC,UAAM,MAAM,aAAa;AACzB,WAAO;AAAA,EACT;AACF;","names":[]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xyo-network/archivist-storage",
|
|
3
|
-
"version": "3.6.
|
|
3
|
+
"version": "3.6.11",
|
|
4
4
|
"description": "Primary SDK for using XYO Protocol 2.0",
|
|
5
5
|
"homepage": "https://xyo.network",
|
|
6
6
|
"bugs": {
|
|
@@ -29,31 +29,31 @@
|
|
|
29
29
|
"module": "dist/neutral/index.mjs",
|
|
30
30
|
"types": "dist/neutral/index.d.ts",
|
|
31
31
|
"dependencies": {
|
|
32
|
-
"@xylabs/assert": "^4.
|
|
33
|
-
"@xylabs/exists": "^4.
|
|
34
|
-
"@xylabs/hex": "^4.
|
|
35
|
-
"@xylabs/promise": "^4.
|
|
36
|
-
"@xyo-network/archivist-abstract": "^3.6.
|
|
37
|
-
"@xyo-network/archivist-model": "^3.6.
|
|
38
|
-
"@xyo-network/boundwitness-model": "^3.6.
|
|
39
|
-
"@xyo-network/module-model": "^3.6.
|
|
40
|
-
"@xyo-network/payload-builder": "^3.6.
|
|
41
|
-
"@xyo-network/payload-model": "^3.6.
|
|
32
|
+
"@xylabs/assert": "^4.5.1",
|
|
33
|
+
"@xylabs/exists": "^4.5.1",
|
|
34
|
+
"@xylabs/hex": "^4.5.1",
|
|
35
|
+
"@xylabs/promise": "^4.5.1",
|
|
36
|
+
"@xyo-network/archivist-abstract": "^3.6.11",
|
|
37
|
+
"@xyo-network/archivist-model": "^3.6.11",
|
|
38
|
+
"@xyo-network/boundwitness-model": "^3.6.11",
|
|
39
|
+
"@xyo-network/module-model": "^3.6.11",
|
|
40
|
+
"@xyo-network/payload-builder": "^3.6.11",
|
|
41
|
+
"@xyo-network/payload-model": "^3.6.11",
|
|
42
42
|
"store2": "^2.14.4"
|
|
43
43
|
},
|
|
44
44
|
"devDependencies": {
|
|
45
|
-
"@xylabs/delay": "^4.
|
|
45
|
+
"@xylabs/delay": "^4.5.1",
|
|
46
46
|
"@xylabs/ts-scripts-yarn3": "^4.2.6",
|
|
47
47
|
"@xylabs/tsconfig": "^4.2.6",
|
|
48
|
-
"@xylabs/vitest-extended": "^4.
|
|
49
|
-
"@xyo-network/account": "^3.6.
|
|
50
|
-
"@xyo-network/archivist-memory": "^3.6.
|
|
51
|
-
"@xyo-network/boundwitness-wrapper": "^3.6.
|
|
52
|
-
"@xyo-network/id-payload-plugin": "^3.6.
|
|
53
|
-
"@xyo-network/node-memory": "^3.6.
|
|
54
|
-
"@xyo-network/payload-wrapper": "^3.6.
|
|
55
|
-
"typescript": "^5.7.
|
|
56
|
-
"vitest": "^
|
|
48
|
+
"@xylabs/vitest-extended": "^4.5.1",
|
|
49
|
+
"@xyo-network/account": "^3.6.11",
|
|
50
|
+
"@xyo-network/archivist-memory": "^3.6.11",
|
|
51
|
+
"@xyo-network/boundwitness-wrapper": "^3.6.11",
|
|
52
|
+
"@xyo-network/id-payload-plugin": "^3.6.11",
|
|
53
|
+
"@xyo-network/node-memory": "^3.6.11",
|
|
54
|
+
"@xyo-network/payload-wrapper": "^3.6.11",
|
|
55
|
+
"typescript": "^5.7.3",
|
|
56
|
+
"vitest": "^3.0.4"
|
|
57
57
|
},
|
|
58
58
|
"publishConfig": {
|
|
59
59
|
"access": "public"
|
package/src/StorageArchivist.ts
CHANGED
|
@@ -30,8 +30,8 @@ import store from 'store2'
|
|
|
30
30
|
|
|
31
31
|
const storeTypes = store as unknown as StoreType
|
|
32
32
|
|
|
33
|
-
export
|
|
34
|
-
export
|
|
33
|
+
export const StorageArchivistConfigSchema = 'network.xyo.archivist.storage.config' as const
|
|
34
|
+
export type StorageArchivistConfigSchema = typeof StorageArchivistConfigSchema
|
|
35
35
|
|
|
36
36
|
export type StorageArchivistConfig = ArchivistConfig<{
|
|
37
37
|
maxEntries?: number
|