@web3-storage/pail 0.6.0 → 0.6.2

package/dist/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../cli.js"],"names":[],"mappings":""}

package/dist/cli.js

@@ -0,0 +1,208 @@
+#!/usr/bin/env node
+import fs from 'fs';
+import os from 'os';
+import { join } from 'path';
+import { Readable, Writable } from 'stream';
+import sade from 'sade';
+import { CID } from 'multiformats/cid';
+import { CARReaderStream, CARWriterStream } from 'carstream';
+import clc from 'cli-color';
+import archy from 'archy';
+// eslint-disable-next-line no-unused-vars
+import * as API from './src/api.js';
+import { put, get, del, entries } from './src/index.js';
+import { ShardFetcher, ShardBlock, isShardLink } from './src/shard.js';
+import { difference } from './src/diff.js';
+import { merge } from './src/merge.js';
+import { MemoryBlockstore, MultiBlockFetcher } from './src/block.js';
+const cli = sade('pail')
+    .option('--path', 'Path to data store.', './pail.car');
+cli.command('put <key> <value>')
+    .describe('Put a value (a CID) for the given key. If the key exists it\'s value is overwritten.')
+    .alias('set')
+    .action(async (key, value, opts) => {
+    const { root: prevRoot, blocks } = await openPail(opts.path);
+    const { root, additions, removals } = await put(blocks, prevRoot, key, CID.parse(value));
+    await updatePail(opts.path, blocks, root, { additions, removals });
+    console.log(clc.red(`--- ${prevRoot}`));
+    console.log(clc.green(`+++ ${root}`));
+    console.log(clc.magenta('@@ -1 +1 @@'));
+    additions.forEach(b => console.log(clc.green(`+${b.cid}`)));
+    removals.forEach(b => console.log(clc.red(`-${b.cid}`)));
+});
+cli.command('get <key>')
+    .describe('Get the stored value for the given key from the pail. If the key is not found, `undefined` is returned.')
+    .action(async (key, opts) => {
+    const { root, blocks } = await openPail(opts.path);
+    const value = await get(blocks, root, key);
+    if (value)
+        console.log(value.toString());
+});
+cli.command('del <key>')
+    .describe('Delete the value for the given key from the pail. If the key is not found no operation occurs.')
+    .alias('delete', 'rm', 'remove')
+    .action(async (key, opts) => {
+    const { root: prevRoot, blocks } = await openPail(opts.path);
+    const { root, additions, removals } = await del(blocks, prevRoot, key);
+    await updatePail(opts.path, blocks, root, { additions, removals });
+    console.log(clc.red(`--- ${prevRoot}`));
+    console.log(clc.green(`+++ ${root}`));
+    console.log(clc.magenta('@@ -1 +1 @@'));
+    additions.forEach(b => console.log(clc.green(`+ ${b.cid}`)));
+    removals.forEach(b => console.log(clc.red(`- ${b.cid}`)));
+});
+cli.command('ls')
+    .describe('List entries in the pail.')
+    .alias('list')
+    .option('-p, --prefix', 'Key prefix to filter by.')
+    .option('--gt', 'Filter results by keys greater than this string.')
+    .option('--lt', 'Filter results by keys less than this string.')
+    .option('--json', 'Format output as newline delimted JSON.')
+    .action(async (opts) => {
+    const { root, blocks } = await openPail(opts.path);
+    let n = 0;
+    for await (const [k, v] of entries(blocks, root, { prefix: opts.prefix, gt: opts.gt, lt: opts.lt })) {
+        console.log(opts.json ? JSON.stringify({ key: k, value: v.toString() }) : `${k}\t${v}`);
+        n++;
+    }
+    if (!opts.json)
+        console.log(`total ${n}`);
+});
+cli.command('tree')
+    .describe('Visualise the pail.')
+    .alias('vis')
+    .action(async (opts) => {
+    const { root, blocks } = await openPail(opts.path);
+    const shards = new ShardFetcher(blocks);
+    const rshard = await shards.get(root);
+    /** @type {archy.Data} */
+    const archyRoot = { label: `Shard(${clc.yellow(rshard.cid.toString())}) ${rshard.bytes.length + 'b'}`, nodes: [] };
+    /** @param {API.ShardEntry} entry */
+    const getData = async ([k, v]) => {
+        if (!Array.isArray(v)) {
+            return { label: `Key(${clc.magenta(k)})`, nodes: [{ label: `Value(${clc.cyan(v)})` }] };
+        }
+        /** @type {archy.Data} */
+        const data = { label: `Key(${clc.magenta(k)})`, nodes: [] };
+        if (v[1])
+            data.nodes?.push({ label: `Value(${clc.cyan(v[1])})` });
+        const blk = await shards.get(v[0]);
+        data.nodes?.push({
+            label: `Shard(${clc.yellow(v[0])}) ${blk.bytes.length + 'b'}`,
+            nodes: await Promise.all(blk.value.entries.map(e => getData(e)))
+        });
+        return data;
+    };
+    for (const entry of rshard.value.entries) {
+        archyRoot.nodes?.push(await getData(entry));
+    }
+    console.log(archy(archyRoot));
+});
+cli.command('diff <path>')
+    .describe('Find the differences between this pail and the passed pail.')
+    .option('-k, --keys', 'Output key/value diff.')
+    .action(async (path, opts) => {
+    const [{ root: aroot, blocks: ablocks }, { root: broot, blocks: bblocks }] = await Promise.all([openPail(opts.path), openPail(path)]);
+    if (aroot.toString() === broot.toString())
+        return;
+    const fetcher = new MultiBlockFetcher(ablocks, bblocks);
+    const { shards: { additions, removals }, keys } = await difference(fetcher, aroot, broot);
+    console.log(clc.red(`--- ${aroot}`));
+    console.log(clc.green(`+++ ${broot}`));
+    console.log(clc.magenta('@@ -1 +1 @@'));
+    if (opts.keys) {
+        keys.forEach(([k, v]) => {
+            if (v[0] != null)
+                console.log(clc.red(`- ${k}\t${v[0]}`));
+            if (v[1] != null)
+                console.log(clc.green(`+ ${k}\t${v[1]}`));
+        });
+    }
+    else {
+        additions.forEach(b => console.log(clc.green(`+ ${b.cid}`)));
+        removals.forEach(b => console.log(clc.red(`- ${b.cid}`)));
+    }
+});
+cli.command('merge <path>')
+    .describe('Merge the passed pail into this pail.')
+    .action(async (path, opts) => {
+    const [{ root: aroot, blocks: ablocks }, { root: broot, blocks: bblocks }] = await Promise.all([openPail(opts.path), openPail(path)]);
+    if (aroot.toString() === broot.toString())
+        return;
+    const fetcher = new MultiBlockFetcher(ablocks, bblocks);
+    const { root, additions, removals } = await merge(fetcher, aroot, [broot]);
+    await updatePail(opts.path, ablocks, root, { additions, removals });
+    console.log(clc.red(`--- ${aroot}`));
+    console.log(clc.green(`+++ ${root}`));
+    console.log(clc.magenta('@@ -1 +1 @@'));
+    additions.forEach(b => console.log(clc.green(`+ ${b.cid}`)));
+    removals.forEach(b => console.log(clc.red(`- ${b.cid}`)));
+});
+cli.parse(process.argv);
+/**
+ * @param {string} path
+ * @returns {Promise<{ root: API.ShardLink, blocks: MemoryBlockstore }>}
+ */
+async function openPail(path) {
+    const blocks = new MemoryBlockstore();
+    try {
+        const carReader = new CARReaderStream();
+        const readable = /** @type {ReadableStream<Uint8Array>} */ (Readable.toWeb(fs.createReadStream(path)));
+        await readable.pipeThrough(carReader).pipeTo(new WritableStream({ write: b => blocks.put(b.cid, b.bytes) }));
+        const header = await carReader.getHeader();
+        if (!isShardLink(header.roots[0]))
+            throw new Error(`not a shard: ${header.roots[0]}`);
+        return { root: header.roots[0], blocks };
+    }
+    catch (err) {
+        // @ts-ignore
+        if (err.code !== 'ENOENT')
+            throw new Error('failed to open bucket', { cause: err });
+        const rootblk = await ShardBlock.create();
+        blocks.put(rootblk.cid, rootblk.bytes);
+        return { root: rootblk.cid, blocks };
+    }
+}
+/**
+ * @param {string} path
+ * @param {MemoryBlockstore} blocks
+ * @param {API.ShardLink} root
+ * @param {API.ShardDiff} diff
+ */
+async function updatePail(path, blocks, root, { additions, removals }) {
+    const tmp = join(os.tmpdir(), `pail${Date.now()}.car`);
+    const iterator = blocks.entries();
+    const readable = new ReadableStream({
+        start(controller) {
+            for (const b of additions)
+                controller.enqueue(b);
+        },
+        pull(controller) {
+            for (const b of iterator) {
+                if (removals.some(r => b.cid.toString() === r.cid.toString()))
+                    continue;
+                return controller.enqueue(b);
+            }
+            controller.close();
+        }
+    });
+    await readable.pipeThrough(new CARWriterStream([root])).pipeTo(Writable.toWeb(fs.createWriteStream(tmp)));
+    const old = `${path}-${new Date().toISOString()}`;
+    try {
+        await fs.promises.rename(path, old);
+    }
+    catch (err) {
+        // @ts-ignore
+        if (err.code !== 'ENOENT')
+            throw err;
+    }
+    await fs.promises.rename(tmp, path);
+    try {
+        await fs.promises.rm(old);
+    }
+    catch (err) {
+        // @ts-ignore
+        if (err.code !== 'ENOENT')
+            throw err;
+    }
+}

package/dist/scripts/randomcid.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=randomcid.d.ts.map

package/dist/scripts/randomcid.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"randomcid.d.ts","sourceRoot":"","sources":["../../scripts/randomcid.js"],"names":[],"mappings":""}

package/dist/scripts/randomcid.js

@@ -0,0 +1,10 @@
+import { CID } from 'multiformats/cid';
+import * as raw from 'multiformats/codecs/raw';
+import { sha256 } from 'multiformats/hashes/sha2';
+import { randomBytes } from '../test/helpers.js';
+async function main() {
+    const bytes = await randomBytes(32);
+    const hash = await sha256.digest(bytes);
+    process.stdout.write(CID.create(1, raw.code, hash).toString());
+}
+main();

package/dist/scripts/words/gen.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=gen.d.ts.map

package/dist/scripts/words/gen.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gen.d.ts","sourceRoot":"","sources":["../../../scripts/words/gen.js"],"names":[],"mappings":""}

package/dist/scripts/words/gen.js

@@ -0,0 +1,51 @@
+import fs from 'node:fs';
+import { Readable } from 'node:stream';
+import { CarWriter } from '@ipld/car';
+import { CID } from 'multiformats/cid';
+import * as raw from 'multiformats/codecs/raw';
+import { sha256 } from 'multiformats/hashes/sha2';
+import { put } from '../../src/index.js';
+import { ShardBlock } from '../../src/shard.js';
+import { MemoryBlockstore } from '../../src/block.js';
+/** @param {string} str */
+async function stringToCID(str) {
+    const hash = await sha256.digest(new TextEncoder().encode(str));
+    return CID.create(1, raw.code, hash);
+}
+async function main() {
+    const data = await fs.promises.readFile('/usr/share/dict/words', 'utf8');
+    const words = data.split(/\n/);
+    const cids = await Promise.all(words.map(stringToCID));
+    const blocks = new MemoryBlockstore();
+    const rootblk = await ShardBlock.create();
+    blocks.putSync(rootblk.cid, rootblk.bytes);
+    console.time(`put x${words.length}`);
+    /** @type {import('../../src/api.ts').ShardLink} */
+    let root = rootblk.cid;
+    for (const [i, word] of words.entries()) {
+        const res = await put(blocks, root, word, cids[i]);
+        root = res.root;
+        for (const b of res.additions) {
+            blocks.putSync(b.cid, b.bytes);
+        }
+        for (const b of res.removals) {
+            blocks.deleteSync(b.cid);
+        }
+        if (i % 1000 === 0) {
+            console.log(`${Math.floor(i / words.length * 100)}%`);
+        }
+    }
+    console.timeEnd(`put x${words.length}`);
+    // @ts-expect-error
+    const { writer, out } = CarWriter.create(root);
+    const finishPromise = new Promise(resolve => {
+        Readable.from(out).pipe(fs.createWriteStream('./pail.car')).on('finish', () => resolve(true));
+    });
+    for (const b of blocks.entries()) {
+        // @ts-expect-error
+        await writer.put(b);
+    }
+    await writer.close();
+    await finishPromise;
+}
+main();

package/dist/src/api.d.ts

@@ -1,4 +1,6 @@
-import { Link, UnknownLink, BlockView, Block, Version } from 'multiformats';
+import type { Version } from 'multiformats';
+import type { Link, UnknownLink } from 'multiformats/link/interface';
+import type { Block, BlockView } from 'multiformats/block/interface';
 import { sha256 } from 'multiformats/hashes/sha2';
 import * as dagCBOR from '@ipld/dag-cbor';
 export { Link, UnknownLink, BlockView, Block, Version };
@@ -20,7 +22,7 @@ export interface ShardDiff {
     removals: ShardBlockView[];
 }
 export interface BlockFetcher {
-    get<T = unknown, C extends number = number, A extends number = number, V extends Version = 1>(link: Link<T, C, A, V>): Promise<Block<T, C, A, V> | undefined>;
+    get: <T = unknown, C extends number = number, A extends number = number, V extends Version = 1>(link: Link<T, C, A, V>) => Promise<Block<T, C, A, V> | undefined>;
 }
 export interface ShardConfig {
     /** Shard compatibility version. */

package/dist/src/api.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAC3E,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAA;AACjD,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AAEzC,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAA;AAEvD,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAA;AAE9C,MAAM,MAAM,mBAAmB,GAAG,CAAC,SAAS,CAAC,CAAA;AAE7C,MAAM,MAAM,2BAA2B,GAAG,CAAC,SAAS,EAAE,WAAW,CAAC,CAAA;AAElE,MAAM,MAAM,eAAe,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,oBAAoB,CAAC,CAAA;AAExE,MAAM,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,GAAG,2BAA2B,CAAC,CAAA;AAEpG,6CAA6C;AAC7C,MAAM,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,oBAAoB,GAAG,mBAAmB,GAAG,2BAA2B,CAAC,CAAA;AAEvH,MAAM,WAAW,KAAM,SAAQ,WAAW;IACxC,OAAO,EAAE,UAAU,EAAE,CAAA;CACtB;AAED,MAAM,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,OAAO,CAAC,IAAI,EAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;AAE/E,MAAM,WAAW,cAAe,SAAQ,SAAS,CAAC,KAAK,EAAE,OAAO,OAAO,CAAC,IAAI,EAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;CAAG;AAEvG,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,cAAc,EAAE,CAAA;IAC3B,QAAQ,EAAE,cAAc,EAAE,CAAA;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,SAAS,MAAM,GAAG,MAAM,EAAE,CAAC,SAAS,MAAM,GAAG,MAAM,EAAE,CAAC,SAAS,OAAO,GAAG,CAAC,EAAG,IAAI,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GACnH,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,CAAA;CACzC;AAED,MAAM,WAAW,WAAW;IAC1B,mCAAmC;IACnC,OAAO,EAAE,MAAM,CAAA;IACf;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAA;IAClB,kDAAkD;IAClD,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,MAAM,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAAA;AAE/C,MAAM,WAAW,eAAe;IAC9B,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,MAAM,cAAc,GACtB,wBAAwB,GACxB,wBAAwB,GACxB,CAAC,wBAAwB,GAAG,wBAAwB,CAAC,CAAA;AAEzD,MAAM,MAAM,wBAAwB,GAChC,iCAAiC,GACjC,iCAAiC,CAAA;AAErC,MAAM,WAAW,iCAAiC;IAChD,EAAE,EAAE,MAAM,CAAA;CACX;AAED,MAAM,WAAW,iCAAiC;IAChD,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,MAAM,wBAAwB,GAChC,iCAAiC,GACjC,iCAAiC,CAAA;AAErC,MAAM,WAAW,iCAAiC;IAChD,EAAE,EAAE,MAAM,CAAA;CACX;AAED,MAAM,WAAW,iCAAiC;IAChD,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,MAAM,cAAc,GACtB,eAAe,GACf,cAAc,CAAA"}
+{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAC3C,OAAO,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAA;AACpE,OAAO,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAA;AACpE,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAA;AACjD,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AAEzC,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAA;AAEvD,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAA;AAE9C,MAAM,MAAM,mBAAmB,GAAG,CAAC,SAAS,CAAC,CAAA;AAE7C,MAAM,MAAM,2BAA2B,GAAG,CAAC,SAAS,EAAE,WAAW,CAAC,CAAA;AAElE,MAAM,MAAM,eAAe,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,oBAAoB,CAAC,CAAA;AAExE,MAAM,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,GAAG,2BAA2B,CAAC,CAAA;AAEpG,6CAA6C;AAC7C,MAAM,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,oBAAoB,GAAG,mBAAmB,GAAG,2BAA2B,CAAC,CAAA;AAEvH,MAAM,WAAW,KAAM,SAAQ,WAAW;IACxC,OAAO,EAAE,UAAU,EAAE,CAAA;CACtB;AAED,MAAM,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,OAAO,CAAC,IAAI,EAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;AAE/E,MAAM,WAAW,cAAe,SAAQ,SAAS,CAAC,KAAK,EAAE,OAAO,OAAO,CAAC,IAAI,EAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;CAAG;AAEvG,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,cAAc,EAAE,CAAA;IAC3B,QAAQ,EAAE,cAAc,EAAE,CAAA;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,SAAS,MAAM,GAAG,MAAM,EAAE,CAAC,SAAS,MAAM,GAAG,MAAM,EAAE,CAAC,SAAS,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,CAAA;CAClK;AAED,MAAM,WAAW,WAAW;IAC1B,mCAAmC;IACnC,OAAO,EAAE,MAAM,CAAA;IACf;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAA;IAClB,kDAAkD;IAClD,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,MAAM,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAAA;AAE/C,MAAM,WAAW,eAAe;IAC9B,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,MAAM,cAAc,GACtB,wBAAwB,GACxB,wBAAwB,GACxB,CAAC,wBAAwB,GAAG,wBAAwB,CAAC,CAAA;AAEzD,MAAM,MAAM,wBAAwB,GAChC,iCAAiC,GACjC,iCAAiC,CAAA;AAErC,MAAM,WAAW,iCAAiC;IAChD,EAAE,EAAE,MAAM,CAAA;CACX;AAED,MAAM,WAAW,iCAAiC;IAChD,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,MAAM,wBAAwB,GAChC,iCAAiC,GACjC,iCAAiC,CAAA;AAErC,MAAM,WAAW,iCAAiC;IAChD,EAAE,EAAE,MAAM,CAAA;CACX;AAED,MAAM,WAAW,iCAAiC;IAChD,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,MAAM,cAAc,GACtB,eAAe,GACf,cAAc,CAAA"}

package/dist/src/api.js

@@ -0,0 +1 @@
+export {};

package/dist/src/batch/api.d.ts

@@ -19,12 +19,12 @@ export interface Batcher {
      * Put a value (a CID) for the given key. If the key exists it's value is
      * overwritten.
      */
-    put(key: string, value: UnknownLink): Promise<void>;
+    put: (key: string, value: UnknownLink) => Promise<void>;
     /**
      * Encode all altered shards in the batch and return the new root CID and
      * difference blocks.
      */
-    commit(): Promise<{
+    commit: () => Promise<{
         root: ShardLink;
     } & ShardDiff>;
 }

package/dist/src/batch/api.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../../../src/batch/api.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,UAAU,EACV,oBAAoB,EACpB,mBAAmB,EACnB,2BAA2B,EAC3B,WAAW,EACX,YAAY,EACZ,cAAc,EACd,YAAY,EACb,MAAM,WAAW,CAAA;AAElB,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,UAAU,EACV,oBAAoB,EACpB,mBAAmB,EACnB,2BAA2B,EAC3B,WAAW,EACX,YAAY,EACZ,cAAc,EACd,YAAY,EACb,CAAA;AAED,MAAM,WAAW,YAAa,SAAQ,WAAW;IAC/C,IAAI,CAAC,EAAE,cAAc,CAAA;IACrB,OAAO,EAAE,iBAAiB,EAAE,CAAA;CAC7B;AAED,MAAM,WAAW,gBAAiB,SAAQ,YAAY;IACpD,IAAI,CAAC,EAAE,cAAc,CAAA;IACrB,OAAO,CAAC,EAAE,iBAAiB,EAAE,CAAA;CAC9B;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,GAAG,EAAE,MAAM;IACX,KAAK,EAAE,oBAAoB,GAAG,mBAAmB,GAAG,2BAA2B,GAAG,oBAAoB,GAAG,4BAA4B;CACtI,CAAA;AAED,MAAM,MAAM,oBAAoB,GAAG,CAAC,YAAY,CAAC,CAAA;AAEjD,MAAM,MAAM,4BAA4B,GAAG,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;AAEtE,MAAM,WAAW,OAAO;IACtB;;;OAGG;IACH,GAAG,CAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACpD;;;OAGG;IACH,MAAM,IAAK,OAAO,CAAC;QAAE,IAAI,EAAE,SAAS,CAAA;KAAE,GAAG,SAAS,CAAC,CAAA;CACpD"}
+{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../../../src/batch/api.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,UAAU,EACV,oBAAoB,EACpB,mBAAmB,EACnB,2BAA2B,EAC3B,WAAW,EACX,YAAY,EACZ,cAAc,EACd,YAAY,EACb,MAAM,WAAW,CAAA;AAElB,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,UAAU,EACV,oBAAoB,EACpB,mBAAmB,EACnB,2BAA2B,EAC3B,WAAW,EACX,YAAY,EACZ,cAAc,EACd,YAAY,EACb,CAAA;AAED,MAAM,WAAW,YAAa,SAAQ,WAAW;IAC/C,IAAI,CAAC,EAAE,cAAc,CAAA;IACrB,OAAO,EAAE,iBAAiB,EAAE,CAAA;CAC7B;AAED,MAAM,WAAW,gBAAiB,SAAQ,YAAY;IACpD,IAAI,CAAC,EAAE,cAAc,CAAA;IACrB,OAAO,CAAC,EAAE,iBAAiB,EAAE,CAAA;CAC9B;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,GAAG,EAAE,MAAM;IACX,KAAK,EAAE,oBAAoB,GAAG,mBAAmB,GAAG,2BAA2B,GAAG,oBAAoB,GAAG,4BAA4B;CACtI,CAAA;AAED,MAAM,MAAM,oBAAoB,GAAG,CAAC,YAAY,CAAC,CAAA;AAEjD,MAAM,MAAM,4BAA4B,GAAG,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;AAEtE,MAAM,WAAW,OAAO;IACtB;;;OAGG;IACH,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IACvD;;;OAGG;IACH,MAAM,EAAE,MAAM,OAAO,CAAC;QAAE,IAAI,EAAE,SAAS,CAAA;KAAE,GAAG,SAAS,CAAC,CAAA;CACvD"}

package/dist/src/batch/api.js

@@ -0,0 +1 @@
+export {};

package/dist/src/batch/index.d.ts

@@ -15,61 +15,9 @@ export class BatchCommittedError extends Error {
      * @param {string} [message]
      * @param {ErrorOptions} [options]
      */
-    constructor(message?: string | undefined, options?: ErrorOptions | undefined);
+    constructor(message?: string, options?: ErrorOptions);
     code: string;
 }
 import * as API from './api.js';
-import * as BatcherShard from './shard.js';
 import { ShardFetcher } from '../shard.js';
-/** @implements {API.Batcher} */
-declare class Batcher implements API.Batcher {
-    /**
-     * @param {object} init
-     * @param {API.BlockFetcher} init.blocks Block storage.
-     * @param {API.ShardLink} init.link CID of the shard block.
-     */
-    static create({ blocks, link }: {
-        blocks: API.BlockFetcher;
-        link: API.ShardLink;
-    }): Promise<Batcher>;
-    /**
-     * @param {object} init
-     * @param {API.BlockFetcher} init.blocks Block storage.
-     * @param {API.BatcherShardEntry[]} init.entries The entries in this shard.
-     * @param {string} init.prefix Key prefix.
-     * @param {number} init.version Shard compatibility version.
-     * @param {string} init.keyChars Characters allowed in keys, referring to a known character set.
-     * @param {number} init.maxKeySize Max key size in bytes.
-     * @param {API.ShardBlockView} init.base Original shard this batcher is based on.
-     */
-    constructor({ blocks, entries, prefix, version, keyChars, maxKeySize, base }: {
-        blocks: API.BlockFetcher;
-        entries: API.BatcherShardEntry[];
-        prefix: string;
-        version: number;
-        keyChars: string;
-        maxKeySize: number;
-        base: API.ShardBlockView;
-    });
-    blocks: API.BlockFetcher;
-    prefix: string;
-    entries: API.BatcherShardEntry[];
-    base: API.ShardBlockView;
-    version: number;
-    keyChars: string;
-    maxKeySize: number;
-    /**
-     * @param {string} key The key of the value to put.
-     * @param {API.UnknownLink} value The value to put.
-     * @returns {Promise<void>}
-     */
-    put(key: string, value: API.UnknownLink): Promise<void>;
-    commit(): Promise<{
-        root: import("multiformats").CID<import("../api.js").Shard, 113, 18, 1>;
-        additions: API.ShardBlockView[];
-        removals: API.ShardBlockView[];
-    }>;
-    #private;
-}
-export {};
 //# sourceMappingURL=index.d.ts.map

package/dist/src/batch/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/batch/index.js"],"names":[],"mappings":"AAiEO,4BANI,IAAI,YAAY,SAChB,gBAAgB,OAChB,MAAM,SACN,IAAI,WAAW,GACb,QAAQ,IAAI,CAAC,CAsGzB;AAWM,iCALI,YAAY,SACZ,gBAAgB,OAChB,MAAM,GACJ,QAAQ;IAAE,KAAK,EAAE,gBAAgB,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,CAiB7D;AAQM,8BAFI,gBAAgB;;;;GAkC1B;AAaM,+BAJI,IAAI,YAAY,QAChB,IAAI,SAAS,GACX,QAAQ,WAAW,CAAC,CAE6C;AAE9E;IAUE,oBAAmC;IATnC;;;OAGG;IACH,8EAGC;IADC,aAAoC;CAIvC;qBAhQoB,UAAU;8BAGD,YAAY;6BAFK,aAAa;AAI5D,gCAAgC;AAChC,iCADiB,GAAG,CAAC,OAAO;IAwC1B;;;;OAIG;IACH;QAHkC,MAAM,EAA7B,IAAI,YAAY;QACI,IAAI,EAAxB,IAAI,SAAS;yBAMvB;IA7CD;;;;;;;;;OASG;IACH;QARkC,MAAM,EAA7B,IAAI,YAAY;QACc,OAAO,EAArC,uBAAuB;QACV,MAAM,EAAnB,MAAM;QACO,OAAO,EAApB,MAAM;QACO,QAAQ,EAArB,MAAM;QACO,UAAU,EAAvB,MAAM;QACmB,IAAI,EAA7B,IAAI,cAAc;OAU5B;IAPC,yBAAoB;IACpB,eAAoB;IACpB,iCAA2B;IAC3B,yBAAgB;IAChB,gBAAsB;IACtB,iBAAwB;IACxB,mBAA4B;IAG9B;;;;OAIG;IACH,SAJW,MAAM,SACN,IAAI,WAAW,GACb,QAAQ,IAAI,CAAC,CAKzB;IAED;;;;OAIC;;CAYF"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/batch/index.js"],"names":[],"mappings":"AAiEO,4BANI,GAAG,CAAC,YAAY,SAChB,GAAG,CAAC,YAAY,OAChB,MAAM,SACN,GAAG,CAAC,WAAW,GACb,OAAO,CAAC,IAAI,CAAC,CAsGzB;AAWM,iCALI,YAAY,SACZ,GAAG,CAAC,YAAY,OAChB,MAAM,GACJ,OAAO,CAAC;IAAE,KAAK,EAAE,GAAG,CAAC,YAAY,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,CAiB7D;AAQM,8BAFI,GAAG,CAAC,YAAY;;;;GAkC1B;AAaM,+BAJI,GAAG,CAAC,YAAY,QAChB,GAAG,CAAC,SAAS,GACX,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAE6C;AAE9E;IAUE,oBAAmC;IATnC;;;OAGG;IACH,sBAHW,MAAM,YACN,YAAY,EAKtB;IADC,aAAoC;CAIvC;qBAhQoB,UAAU;6BACgB,aAAa"}

package/dist/src/batch/index.js

@@ -0,0 +1,241 @@
+// eslint-disable-next-line no-unused-vars
+import * as API from './api.js';
+import { ShardFetcher, isPrintableASCII } from '../shard.js';
+import * as Shard from '../shard.js';
+import * as BatcherShard from './shard.js';
+/** @implements {API.Batcher} */
+class Batcher {
+    #committed = false;
+    /**
+     * @param {object} init
+     * @param {API.BlockFetcher} init.blocks Block storage.
+     * @param {API.BatcherShardEntry[]} init.entries The entries in this shard.
+     * @param {string} init.prefix Key prefix.
+     * @param {number} init.version Shard compatibility version.
+     * @param {string} init.keyChars Characters allowed in keys, referring to a known character set.
+     * @param {number} init.maxKeySize Max key size in bytes.
+     * @param {API.ShardBlockView} init.base Original shard this batcher is based on.
+     */
+    constructor({ blocks, entries, prefix, version, keyChars, maxKeySize, base }) {
+        this.blocks = blocks;
+        this.prefix = prefix;
+        this.entries = [...entries];
+        this.base = base;
+        this.version = version;
+        this.keyChars = keyChars;
+        this.maxKeySize = maxKeySize;
+    }
+    /**
+     * @param {string} key The key of the value to put.
+     * @param {API.UnknownLink} value The value to put.
+     * @returns {Promise<void>}
+     */
+    async put(key, value) {
+        if (this.#committed)
+            throw new BatchCommittedError();
+        return put(this.blocks, this, key, value);
+    }
+    async commit() {
+        if (this.#committed)
+            throw new BatchCommittedError();
+        this.#committed = true;
+        return commit(this);
+    }
+    /**
+     * @param {object} init
+     * @param {API.BlockFetcher} init.blocks Block storage.
+     * @param {API.ShardLink} init.link CID of the shard block.
+     */
+    static async create({ blocks, link }) {
+        const shards = new ShardFetcher(blocks);
+        const base = await shards.get(link);
+        return new Batcher({ blocks, base, ...base.value });
+    }
+}
+/**
+ * @param {API.BlockFetcher} blocks
+ * @param {API.BatcherShard} shard
+ * @param {string} key The key of the value to put.
+ * @param {API.UnknownLink} value The value to put.
+ * @returns {Promise<void>}
+ */
+export const put = async (blocks, shard, key, value) => {
+    if (shard.keyChars !== Shard.KeyCharsASCII) {
+        throw new Error(`unsupported key character set: ${shard.keyChars}`);
+    }
+    if (!isPrintableASCII(key)) {
+        throw new Error('key contains non-ASCII characters');
+    }
+    // ensure utf8 encoded key is smaller than max
+    if (new TextEncoder().encode(key).length > shard.maxKeySize) {
+        throw new Error(`UTF-8 encoded key exceeds max size of ${shard.maxKeySize} bytes`);
+    }
+    const shards = new ShardFetcher(blocks);
+    const dest = await traverse(shards, shard, key);
+    if (dest.shard !== shard) {
+        shard = dest.shard;
+        key = dest.key;
+    }
+    /** @type {API.BatcherShardEntry} */
+    let entry = [dest.key, value];
+    const targetEntries = [...dest.shard.entries];
+    for (const [i, e] of targetEntries.entries()) {
+        const [k, v] = e;
+        // is this just a replace?
+        if (k === dest.key)
+            break;
+        // do we need to shard this entry?
+        const shortest = k.length < dest.key.length ? k : dest.key;
+        const other = shortest === k ? dest.key : k;
+        let common = '';
+        for (const char of shortest) {
+            const next = common + char;
+            if (!other.startsWith(next))
+                break;
+            common = next;
+        }
+        if (common.length) {
+            /** @type {API.ShardEntry[]} */
+            let entries = [];
+            // if the existing entry key or new key is equal to the common prefix,
+            // then the existing value / new value needs to persist in the parent
+            // shard. Otherwise they persist in this new shard.
+            if (common !== dest.key) {
+                entries = Shard.putEntry(entries, [dest.key.slice(common.length), value]);
+            }
+            if (common !== k) {
+                entries = Shard.putEntry(entries, asShardEntry([k.slice(common.length), v]));
+            }
+            let child = BatcherShard.create({
+                ...Shard.configure(dest.shard),
+                prefix: dest.shard.prefix + common,
+                entries
+            });
+            // need to spread as access by index does not consider utf-16 surrogates
+            const commonChars = [...common];
+            // create parent shards for each character of the common prefix
+            for (let i = commonChars.length - 1; i > 0; i--) {
+                /** @type {API.ShardEntryShardValue | API.ShardEntryShardAndValueValue} */
+                let parentValue;
+                // if the first iteration and the existing entry key is equal to the
+                // common prefix, then existing value needs to persist in this parent
+                if (i === commonChars.length - 1 && common === k) {
+                    if (Array.isArray(v))
+                        throw new Error('found a shard link when expecting a value');
+                    parentValue = [child, v];
+                }
+                else if (i === commonChars.length - 1 && common === dest.key) {
+                    parentValue = [child, value];
+                }
+                else {
+                    parentValue = [child];
+                }
+                const parent = BatcherShard.create({
+                    ...Shard.configure(dest.shard),
+                    prefix: dest.shard.prefix + commonChars.slice(0, i).join(''),
+                    entries: [[commonChars[i], parentValue]]
+                });
+                child = parent;
+            }
+            // remove the sharded entry
+            targetEntries.splice(i, 1);
+            // create the entry that will be added to target
+            if (commonChars.length === 1 && common === k) {
+                if (Array.isArray(v))
+                    throw new Error('found a shard link when expecting a value');
+                entry = [commonChars[0], [child, v]];
+            }
+            else if (commonChars.length === 1 && common === dest.key) {
+                entry = [commonChars[0], [child, value]];
+            }
+            else {
+                entry = [commonChars[0], [child]];
+            }
+            break;
+        }
+    }
+    shard.entries = Shard.putEntry(asShardEntries(targetEntries), asShardEntry(entry));
+};
+/**
+ * Traverse from the passed shard through to the correct shard for the passed
+ * key.
+ *
+ * @param {ShardFetcher} shards
+ * @param {API.BatcherShard} shard
+ * @param {string} key
+ * @returns {Promise<{ shard: API.BatcherShard, key: string }>}
+ */
+export const traverse = async (shards, shard, key) => {
+    for (let i = 0; i < shard.entries.length; i++) {
+        const [k, v] = shard.entries[i];
+        if (key <= k)
+            break;
+        if (key.startsWith(k) && Array.isArray(v)) {
+            if (Shard.isShardLink(v[0])) {
+                const blk = await shards.get(v[0]);
+                const batcher = BatcherShard.create({ base: blk, ...blk.value });
+                shard.entries[i] = [k, v[1] == null ? [batcher] : [batcher, v[1]]];
+                return traverse(shards, batcher, key.slice(k.length));
+            }
+            return traverse(shards, v[0], key.slice(k.length));
+        }
+    }
+    return { shard, key };
+};
+/**
+ * Encode all altered shards in the batch and return the new root CID and
+ * difference blocks.
+ *
+ * @param {API.BatcherShard} shard
+ */
+export const commit = async (shard) => {
+    /** @type {API.ShardBlockView[]} */
+    const additions = [];
+    /** @type {API.ShardBlockView[]} */
+    const removals = [];
+    /** @type {API.ShardEntry[]} */
+    const entries = [];
+    for (const entry of shard.entries) {
+        if (Array.isArray(entry[1]) && !Shard.isShardLink(entry[1][0])) {
+            const result = await commit(entry[1][0]);
+            entries.push([
+                entry[0],
+                entry[1][1] == null ? [result.root] : [result.root, entry[1][1]]
+            ]);
+            additions.push(...result.additions);
+            removals.push(...result.removals);
+        }
+        else {
+            entries.push(asShardEntry(entry));
+        }
+    }
+    const block = await Shard.encodeBlock(Shard.withEntries(entries, shard));
+    additions.push(block);
+    if (shard.base && shard.base.cid.toString() === block.cid.toString()) {
+        return { root: block.cid, additions: [], removals: [] };
+    }
+    if (shard.base)
+        removals.push(shard.base);
+    return { root: block.cid, additions, removals };
+};
+/** @param {API.BatcherShardEntry[]} entries */
+const asShardEntries = entries => /** @type {API.ShardEntry[]} */ (entries);
+/** @param {API.BatcherShardEntry} entry */
+const asShardEntry = entry => /** @type {API.ShardEntry} */ (entry);
+/**
+ * @param {API.BlockFetcher} blocks Bucket block storage.
+ * @param {API.ShardLink} root CID of the root shard block.
+ * @returns {Promise<API.Batcher>}
+ */
+export const create = (blocks, root) => Batcher.create({ blocks, link: root });
+export class BatchCommittedError extends Error {
+    /**
+     * @param {string} [message]
+     * @param {ErrorOptions} [options]
+     */
+    constructor(message, options) {
+        super(message ?? 'batch already committed', options);
+        this.code = BatchCommittedError.code;
+    }
+    static code = 'ERR_BATCH_COMMITTED';
+}