@web3-storage/pail 0.6.0 → 0.6.2

package/dist/test/del.test.js

@@ -0,0 +1,21 @@
+import { put, get, del } from '../src/index.js';
+import { ShardBlock } from '../src/shard.js';
+import { Blockstore, randomCID } from './helpers.js';
+describe('del', () => {
+    it('deletes a value', async () => {
+        const empty = await ShardBlock.create();
+        const blocks = new Blockstore();
+        await blocks.put(empty.cid, empty.bytes);
+        const dataCID = await randomCID(32);
+        const { root: root0, additions: additions0 } = await put(blocks, empty.cid, 'test', dataCID);
+        for (const b of additions0) {
+            await blocks.put(b.cid, b.bytes);
+        }
+        const { root: root1, additions: additions1 } = await del(blocks, root0, 'test');
+        for (const b of additions1) {
+            await blocks.put(b.cid, b.bytes);
+        }
+        const res = await get(blocks, root1, 'test');
+        assert.strictEqual(res, undefined);
+    });
+});

package/dist/test/diff.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=diff.test.d.ts.map

package/dist/test/diff.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"diff.test.d.ts","sourceRoot":"","sources":["../../test/diff.test.js"],"names":[],"mappings":""}

package/dist/test/diff.test.js

@@ -0,0 +1,38 @@
+// eslint-disable-next-line no-unused-vars
+import * as API from '../src/api.js';
+import { put } from '../src/index.js';
+import { ShardBlock } from '../src/shard.js';
+import { difference } from '../src/diff.js';
+import { Blockstore, randomCID } from './helpers.js';
+describe('diff', () => {
+    it('diffs a non-sharded addition', async () => {
+        const empty = await ShardBlock.create();
+        const blocks = new Blockstore();
+        await blocks.put(empty.cid, empty.bytes);
+        /** @type {Array<[string, API.UnknownLink]>} */
+        const testdata = [
+            ['a', await randomCID(32)]
+        ];
+        /** @type {API.ShardLink} */
+        let root = empty.cid;
+        for (const [k, v] of testdata) {
+            const res = await put(blocks, root, k, v);
+            for (const b of res.additions) {
+                await blocks.put(b.cid, b.bytes);
+            }
+            root = res.root;
+        }
+        const diff = await difference(blocks, empty.cid, root);
+        assert.equal(diff.shards.removals.length, 1);
+        assert.equal(diff.shards.removals[0].cid.toString(), empty.cid.toString());
+        assert.equal(diff.shards.additions.length, 1);
+        assert.equal(diff.shards.additions[0].cid.toString(), root.toString());
+        assert.equal(diff.keys.length, testdata.length);
+        for (const [k, v] of testdata) {
+            const d = diff.keys.find(p => p[0] === k);
+            assert(d);
+            assert.equal(d[1][0], null);
+            assert.equal(d[1][1]?.toString(), v.toString());
+        }
+    });
+});

package/dist/test/entries.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=entries.test.d.ts.map

package/dist/test/entries.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"entries.test.d.ts","sourceRoot":"","sources":["../../test/entries.test.js"],"names":[],"mappings":""}

package/dist/test/entries.test.js

@@ -0,0 +1,210 @@
+// eslint-disable-next-line no-unused-vars
+import * as API from '../src/api.js';
+import { put, entries } from '../src/index.js';
+import { ShardBlock } from '../src/shard.js';
+import { Blockstore, randomCID } from './helpers.js';
+describe('entries', () => {
+    it('lists entries in lexicographical order', async () => {
+        const empty = await ShardBlock.create();
+        const blocks = new Blockstore();
+        await blocks.put(empty.cid, empty.bytes);
+        /** @type {Array<[string, API.UnknownLink]>} */
+        const testdata = [
+            ['c', await randomCID(32)],
+            ['d', await randomCID(32)],
+            ['a', await randomCID(32)],
+            ['b', await randomCID(32)]
+        ];
+        /** @type {API.ShardLink} */
+        let root = empty.cid;
+        for (const [k, v] of testdata) {
+            const res = await put(blocks, root, k, v);
+            for (const b of res.additions) {
+                await blocks.put(b.cid, b.bytes);
+            }
+            root = res.root;
+        }
+        const results = [];
+        for await (const entry of entries(blocks, root)) {
+            results.push(entry);
+        }
+        for (const [i, key] of testdata.map(d => d[0]).sort().entries()) {
+            assert.equal(results[i][0], key);
+        }
+    });
+    it('lists entries by prefix', async () => {
+        const empty = await ShardBlock.create();
+        const blocks = new Blockstore();
+        await blocks.put(empty.cid, empty.bytes);
+        /** @type {Array<[string, API.UnknownLink]>} */
+        const testdata = [
+            ['cccc', await randomCID(32)],
+            ['deee', await randomCID(32)],
+            ['dooo', await randomCID(32)],
+            ['beee', await randomCID(32)]
+        ];
+        /** @type {API.ShardLink} */
+        let root = empty.cid;
+        for (const [k, v] of testdata) {
+            const res = await put(blocks, root, k, v);
+            for (const b of res.additions) {
+                await blocks.put(b.cid, b.bytes);
+            }
+            root = res.root;
+        }
+        const prefix = 'd';
+        const results = [];
+        for await (const entry of entries(blocks, root, { prefix })) {
+            results.push(entry);
+        }
+        for (const [i, key] of testdata.map(d => d[0]).filter(k => k.startsWith(prefix)).sort().entries()) {
+            assert.equal(results[i][0], key);
+        }
+    });
+    it('lists entries by key greater than string', async () => {
+        const empty = await ShardBlock.create();
+        const blocks = new Blockstore();
+        await blocks.put(empty.cid, empty.bytes);
+        /** @type {Array<[string, API.UnknownLink]>} */
+        const testdata = [
+            ['cccc', await randomCID(32)],
+            ['deee', await randomCID(32)],
+            ['dooo', await randomCID(32)],
+            ['beee', await randomCID(32)]
+        ];
+        /** @type {API.ShardLink} */
+        let root = empty.cid;
+        for (const [k, v] of testdata) {
+            const res = await put(blocks, root, k, v);
+            for (const b of res.additions) {
+                await blocks.put(b.cid, b.bytes);
+            }
+            root = res.root;
+        }
+        const gt = 'beee';
+        const results = [];
+        for await (const entry of entries(blocks, root, { gt })) {
+            results.push(entry);
+        }
+        for (const [i, key] of testdata.map(d => d[0]).filter(k => k > gt).sort().entries()) {
+            assert.equal(results[i][0], key);
+        }
+    });
+    it('lists entries by key greater than or equal to string', async () => {
+        const empty = await ShardBlock.create();
+        const blocks = new Blockstore();
+        await blocks.put(empty.cid, empty.bytes);
+        /** @type {Array<[string, API.UnknownLink]>} */
+        const testdata = [
+            ['cccc', await randomCID(32)],
+            ['deee', await randomCID(32)],
+            ['dooo', await randomCID(32)],
+            ['beee', await randomCID(32)]
+        ];
+        /** @type {API.ShardLink} */
+        let root = empty.cid;
+        for (const [k, v] of testdata) {
+            const res = await put(blocks, root, k, v);
+            for (const b of res.additions) {
+                await blocks.put(b.cid, b.bytes);
+            }
+            root = res.root;
+        }
+        const gte = 'beee';
+        const results = [];
+        for await (const entry of entries(blocks, root, { gte })) {
+            results.push(entry);
+        }
+        for (const [i, key] of testdata.map(d => d[0]).filter(k => k >= gte).sort().entries()) {
+            assert.equal(results[i][0], key);
+        }
+    });
+    it('lists entries by key less than string', async () => {
+        const empty = await ShardBlock.create();
+        const blocks = new Blockstore();
+        await blocks.put(empty.cid, empty.bytes);
+        /** @type {Array<[string, API.UnknownLink]>} */
+        const testdata = [
+            ['cccc', await randomCID(32)],
+            ['deee', await randomCID(32)],
+            ['dooo', await randomCID(32)],
+            ['beee', await randomCID(32)]
+        ];
+        /** @type {API.ShardLink} */
+        let root = empty.cid;
+        for (const [k, v] of testdata) {
+            const res = await put(blocks, root, k, v);
+            for (const b of res.additions) {
+                await blocks.put(b.cid, b.bytes);
+            }
+            root = res.root;
+        }
+        const lt = 'doo';
+        const results = [];
+        for await (const entry of entries(blocks, root, { lt })) {
+            results.push(entry);
+        }
+        for (const [i, key] of testdata.map(d => d[0]).filter(k => k < lt).sort().entries()) {
+            assert.equal(results[i][0], key);
+        }
+    });
+    it('lists entries by key less than or equal to string', async () => {
+        const empty = await ShardBlock.create();
+        const blocks = new Blockstore();
+        await blocks.put(empty.cid, empty.bytes);
+        /** @type {Array<[string, API.UnknownLink]>} */
+        const testdata = [
+            ['cccc', await randomCID(32)],
+            ['deee', await randomCID(32)],
+            ['dooo', await randomCID(32)],
+            ['beee', await randomCID(32)]
+        ];
+        /** @type {API.ShardLink} */
+        let root = empty.cid;
+        for (const [k, v] of testdata) {
+            const res = await put(blocks, root, k, v);
+            for (const b of res.additions) {
+                await blocks.put(b.cid, b.bytes);
+            }
+            root = res.root;
+        }
+        const lte = 'dooo';
+        const results = [];
+        for await (const entry of entries(blocks, root, { lte })) {
+            results.push(entry);
+        }
+        for (const [i, key] of testdata.map(d => d[0]).filter(k => k <= lte).sort().entries()) {
+            assert.equal(results[i][0], key);
+        }
+    });
+    it('lists entries by key greater than and less than or equal to string', async () => {
+        const empty = await ShardBlock.create();
+        const blocks = new Blockstore();
+        await blocks.put(empty.cid, empty.bytes);
+        /** @type {Array<[string, API.UnknownLink]>} */
+        const testdata = [
+            ['cccc', await randomCID(32)],
+            ['deee', await randomCID(32)],
+            ['dooo', await randomCID(32)],
+            ['beee', await randomCID(32)]
+        ];
+        /** @type {API.ShardLink} */
+        let root = empty.cid;
+        for (const [k, v] of testdata) {
+            const res = await put(blocks, root, k, v);
+            for (const b of res.additions) {
+                await blocks.put(b.cid, b.bytes);
+            }
+            root = res.root;
+        }
+        const gt = 'c';
+        const lte = 'deee';
+        const results = [];
+        for await (const entry of entries(blocks, root, { gt, lte })) {
+            results.push(entry);
+        }
+        for (const [i, key] of testdata.map(d => d[0]).filter(k => k > gt && k <= lte).sort().entries()) {
+            assert.equal(results[i][0], key);
+        }
+    });
+});

package/dist/test/get.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=get.test.d.ts.map

package/dist/test/get.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get.test.d.ts","sourceRoot":"","sources":["../../test/get.test.js"],"names":[],"mappings":""}

package/dist/test/get.test.js

@@ -0,0 +1,26 @@
+import { put, get } from '../src/index.js';
+import { ShardBlock } from '../src/shard.js';
+import { Blockstore, randomCID } from './helpers.js';
+describe('get', () => {
+    it('get from root shard', async () => {
+        const empty = await ShardBlock.create();
+        const blocks = new Blockstore();
+        await blocks.put(empty.cid, empty.bytes);
+        const dataCID = await randomCID(32);
+        const { root, additions } = await put(blocks, empty.cid, 'test', dataCID);
+        for (const b of additions) {
+            await blocks.put(b.cid, b.bytes);
+        }
+        const res = await get(blocks, root, 'test');
+        assert(res);
+        assert(res.toString(), dataCID.toString());
+    });
+    it('returns undefined when not found', async () => {
+        const empty = await ShardBlock.create();
+        const blocks = new Blockstore();
+        await blocks.put(empty.cid, empty.bytes);
+        const res = await get(blocks, empty.cid, 'test');
+        assert.strictEqual(res, undefined);
+    });
+    // TODO: test get when key is also shard link
+});

package/dist/test/helpers.d.ts

@@ -0,0 +1,35 @@
+/**
+ * @param {number} min
+ * @param {number} max
+ */
+export function randomInteger(min: number, max: number): number;
+/**
+ * @param {number} size
+ */
+export function randomString(size: number, alphabet?: string): string;
+/** @param {number} [size] Number of random bytes to hash. */
+export function randomCID(size?: number): Promise<Link.Link<any, 85, 18, 1>>;
+/**
+ * @param {number} size
+ */
+export function randomBytes(size: number): Promise<Uint8Array<ArrayBuffer>>;
+export class Blockstore extends MemoryBlockstore {
+    /**
+     * @param {import('../src/api.js').ShardLink} cid
+     */
+    getShardBlock(cid: import("../src/api.js").ShardLink): Promise<API.ShardBlockView>;
+}
+export function vis(blocks: API.BlockFetcher, root: API.ShardLink): Promise<void>;
+export function materialize(blocks: API.BlockFetcher, root: API.ShardLink): Promise<any[]>;
+export function putAll(blocks: Blockstore, root: API.ShardLink, items: Array<[string, API.UnknownLink]>): Promise<{
+    root: API.ShardLink;
+}>;
+export function verify(blocks: MemoryBlockstore, root: API.ShardLink, data: Map<string, API.UnknownLink>): Promise<void>;
+export function clockVis<T>(blocks: API.BlockFetcher, head: ClockAPI.EventLink<T>[], options?: {
+    renderNodeLabel?: ((b: ClockAPI.EventBlockView<T>) => string) | undefined;
+}): Promise<void>;
+import * as Link from 'multiformats/link';
+import { MemoryBlockstore } from '../src/block.js';
+import * as API from '../src/api.js';
+import * as ClockAPI from '../src/clock/api.js';
+//# sourceMappingURL=helpers.d.ts.map

package/dist/test/helpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../test/helpers.js"],"names":[],"mappings":"AAcA;;;GAGG;AACH,mCAHW,MAAM,OACN,MAAM,UAMhB;AAID;;GAEG;AACH,mCAFW,MAAM,6BAQhB;AAED,6DAA6D;AAC7D,iCADY,MAAM,sCAIjB;AAyGD;;GAEG;AACH,kCAFW,MAAM,oCAahB;AArHD;IACE;;OAEG;IACH,mBAFW,OAAO,eAAe,EAAE,SAAS,+BAM3C;CACF;AAMM,4BAHI,GAAG,CAAC,YAAY,QAChB,GAAG,CAAC,SAAS,iBA8BvB;AAMM,oCAHI,GAAG,CAAC,YAAY,QAChB,GAAG,CAAC,SAAS,kBAkBvB;AAOM,+BAJI,UAAU,QACV,GAAG,CAAC,SAAS,SACb,KAAK,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;;GAc1C;AAOM,+BAJI,gBAAgB,QAChB,GAAG,CAAC,SAAS,QACb,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC,iBAYtC;AAyBM,yBANM,CAAC,UACH,GAAG,CAAC,YAAY,QAChB,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,YAE/B;IAA4D,eAAe,QAA/D,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,KAAK,MAAM;CAA2B,iBAQ9E;sBA/KqB,mBAAmB;iCAUR,iBAAiB;qBAJ7B,eAAe;0BAEV,qBAAqB"}

package/dist/test/helpers.js

@@ -0,0 +1,168 @@
+import * as Link from 'multiformats/link';
+import * as raw from 'multiformats/codecs/raw';
+import { sha256 } from 'multiformats/hashes/sha2';
+import clc from 'cli-color';
+import archy from 'archy';
+// eslint-disable-next-line no-unused-vars
+import * as API from '../src/api.js';
+// eslint-disable-next-line no-unused-vars
+import * as ClockAPI from '../src/clock/api.js';
+import { ShardFetcher, decodeBlock } from '../src/shard.js';
+import { MemoryBlockstore } from '../src/block.js';
+import { entries, get, put } from '../src/index.js';
+import * as Clock from '../src/clock/index.js';
+/**
+ * @param {number} min
+ * @param {number} max
+ */
+export function randomInteger(min, max) {
+    min = Math.ceil(min);
+    max = Math.floor(max);
+    return Math.floor(Math.random() * (max - min) + min);
+}
+const Alphabet = 'abcdefghijklmnopqrstuvwxyz-/_';
+/**
+ * @param {number} size
+ */
+export function randomString(size, alphabet = Alphabet) {
+    let str = '';
+    while (str.length < size) {
+        str += alphabet[randomInteger(0, alphabet.length)];
+    }
+    return str;
+}
+/** @param {number} [size] Number of random bytes to hash. */
+export async function randomCID(size = 32) {
+    const hash = await sha256.digest(await randomBytes(size));
+    return Link.create(raw.code, hash);
+}
+export class Blockstore extends MemoryBlockstore {
+    /**
+     * @param {import('../src/api.js').ShardLink} cid
+     */
+    async getShardBlock(cid) {
+        const blk = await this.get(cid);
+        assert(blk);
+        return decodeBlock(blk.bytes);
+    }
+}
+/**
+ * @param {API.BlockFetcher} blocks Block storage.
+ * @param {API.ShardLink} root
+ */
+export const vis = async (blocks, root) => {
+    const shards = new ShardFetcher(blocks);
+    const rshard = await shards.get(root);
+    /** @type {archy.Data} */
+    const archyRoot = { label: `Shard(${clc.yellow(rshard.cid.toString())}) ${rshard.bytes.length + 'b'}`, nodes: [] };
+    /** @param {API.ShardEntry} entry */
+    const getData = async ([k, v]) => {
+        if (!Array.isArray(v)) {
+            return { label: `Key(${clc.magenta(k)})`, nodes: [{ label: `Value(${clc.cyan(v)})` }] };
+        }
+        /** @type {archy.Data} */
+        const data = { label: `Key(${clc.magenta(k)})`, nodes: [] };
+        if (v[1])
+            data.nodes?.push({ label: `Value(${clc.cyan(v[1])})` });
+        const blk = await shards.get(v[0]);
+        data.nodes?.push({
+            label: `Shard(${clc.yellow(v[0])}) ${blk.bytes.length + 'b'}`,
+            nodes: await Promise.all(blk.value.entries.map(e => getData(e)))
+        });
+        return data;
+    };
+    for (const entry of rshard.value.entries) {
+        archyRoot.nodes?.push(await getData(entry));
+    }
+    console.log(archy(archyRoot));
+};
+/**
+ * @param {API.BlockFetcher} blocks
+ * @param {API.ShardLink} root
+ */
+export const materialize = async (blocks, root) => {
+    const shards = new ShardFetcher(blocks);
+    const shard = await shards.get(root);
+    /** @type {any[]} */
+    const entries = [];
+    for (const e of shard.value.entries) {
+        if (Array.isArray(e[1])) {
+            const v = [...e[1]];
+            // @ts-expect-error
+            v[0] = await materialize(blocks, e[1][0]);
+            entries.push([e[0], v]);
+        }
+        else {
+            entries.push([...e]);
+        }
+    }
+    return entries;
+};
+/**
+ * @param {Blockstore} blocks
+ * @param {API.ShardLink} root
+ * @param {Array<[string, API.UnknownLink]>} items
+ */
+export const putAll = async (blocks, root, items) => {
+    for (const [k, v] of items) {
+        const res = await put(blocks, root, k, v);
+        for (const b of res.additions) {
+            blocks.putSync(b.cid, b.bytes);
+        }
+        for (const b of res.removals) {
+            blocks.deleteSync(b.cid);
+        }
+        root = res.root;
+    }
+    return { root };
+};
+/**
+ * @param {MemoryBlockstore} blocks
+ * @param {API.ShardLink} root
+ * @param {Map<string, API.UnknownLink>} data
+ */
+export const verify = async (blocks, root, data) => {
+    for (const [k, v] of data) {
+        const result = await get(blocks, root, k);
+        if (!result)
+            throw new Error(`missing item: "${k}": ${v}`);
+        if (result.toString() !== v.toString())
+            throw new Error(`incorrect value for ${k}: ${result} !== ${v}`);
+    }
+    let total = 0;
+    // eslint-disable-next-line no-unused-vars
+    for await (const _ of entries(blocks, root))
+        total++;
+    if (data.size !== total)
+        throw new Error(`incorrect entry count: ${total} !== ${data.size}`);
+};
+/**
+ * @param {number} size
+ */
+export async function randomBytes(size) {
+    /** @type {{getRandomValues: (u: Uint8Array) => Uint8Array}} */
+    let myCrypto;
+    if ('crypto' in globalThis && 'getRandomValues' in globalThis.crypto) {
+        myCrypto = globalThis.crypto;
+    }
+    else {
+        myCrypto = await import('node:crypto');
+    }
+    const bytes = new Uint8Array(size);
+    myCrypto.getRandomValues(bytes);
+    return bytes;
+}
+/**
+ * @template T
+ * @param {API.BlockFetcher} blocks Block storage.
+ * @param {ClockAPI.EventLink<T>[]} head
+ * @param {object} [options]
+ * @param {(b: ClockAPI.EventBlockView<T>) => string} [options.renderNodeLabel]
+ */
+export const clockVis = async (blocks, head, options) => {
+    const lines = [];
+    for await (const line of Clock.vis(blocks, head, options)) {
+        lines.push(line);
+    }
+    console.log(lines.join('\n'));
+};

package/dist/test/put.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=put.test.d.ts.map

package/dist/test/put.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"put.test.d.ts","sourceRoot":"","sources":["../../test/put.test.js"],"names":[],"mappings":""}