@web3-storage/pail 0.6.0 → 0.6.2

package/package.json

@@ -1,132 +1,73 @@
 {
   "name": "@web3-storage/pail",
-  "version": "0.6.0",
+  "version": "0.6.2",
   "description": "DAG based key value store.",
-  "main": "src/index.js",
+  "main": "./dist/src/index.js",
   "type": "module",
   "types": "./dist/src/index.d.ts",
-  "typesVersions": {
-    "*": {
-      "*": [
-        "dist/*"
-      ],
-      "dist/src/index.d.ts": [
-        "dist/src/index.d.ts"
-      ],
-      "api": [
-        "dist/src/api.d.ts"
-      ],
-      "batch": [
-        "dist/src/batch/index.d.ts"
-      ],
-      "batch/api": [
-        "dist/src/batch/api.d.ts"
-      ],
-      "block": [
-        "dist/src/block.d.ts"
-      ],
-      "clock": [
-        "dist/src/clock/index.d.ts"
-      ],
-      "clock/api": [
-        "dist/src/clock/api.d.ts"
-      ],
-      "crdt": [
-        "dist/src/crdt/index.d.ts"
-      ],
-      "crdt/api": [
-        "dist/src/crdt/api.d.ts"
-      ],
-      "crdt/batch": [
-        "dist/src/crdt/batch/index.d.ts"
-      ],
-      "crdt/batch/api": [
-        "dist/src/crdt/batch/api.d.ts"
-      ],
-      "diff": [
-        "dist/src/diff.d.ts"
-      ],
-      "link": [
-        "dist/src/link.d.ts"
-      ],
-      "merge": [
-        "dist/src/merge.d.ts"
-      ],
-      "shard": [
-        "dist/src/shard.d.ts"
-      ]
-    }
-  },
   "exports": {
     ".": {
       "types": "./dist/src/index.d.ts",
-      "import": "./src/index.js"
+      "import": "./dist/src/index.js"
     },
     "./api": {
       "types": "./dist/src/api.d.ts",
-      "import": "./src/api.js"
+      "import": "./dist/src/api.js"
     },
     "./batch": {
       "types": "./dist/src/batch/index.d.ts",
-      "import": "./src/batch/index.js"
+      "import": "./dist/src/batch/index.js"
     },
     "./batch/api": {
       "types": "./dist/src/batch/api.d.ts",
-      "import": "./src/batch/api.js"
+      "import": "./dist/src/batch/api.js"
     },
     "./block": {
       "types": "./dist/src/block.d.ts",
-      "import": "./src/block.js"
+      "import": "./dist/src/block.js"
     },
     "./clock": {
       "types": "./dist/src/clock/index.d.ts",
-      "import": "./src/clock/index.js"
+      "import": "./dist/src/clock/index.js"
     },
     "./clock/api": {
       "types": "./dist/src/clock/api.d.ts",
-      "import": "./src/clock/api.js"
+      "import": "./dist/src/clock/api.js"
     },
     "./crdt": {
       "types": "./dist/src/crdt/index.d.ts",
-      "import": "./src/crdt/index.js"
+      "import": "./dist/src/crdt/index.js"
     },
     "./crdt/api": {
       "types": "./dist/src/crdt/api.d.ts",
-      "import": "./src/crdt/api.js"
+      "import": "./dist/src/crdt/api.js"
     },
     "./crdt/batch": {
       "types": "./dist/src/crdt/batch/index.d.ts",
-      "import": "./src/crdt/batch/index.js"
+      "import": "./dist/src/crdt/batch/index.js"
     },
     "./crdt/batch/api": {
       "types": "./dist/src/crdt/batch/api.d.ts",
-      "import": "./src/crdt/batch/api.js"
+      "import": "./dist/src/crdt/batch/api.js"
     },
     "./diff": {
       "types": "./dist/src/diff.d.ts",
-      "import": "./src/diff.js"
-    },
-    "./link": {
-      "types": "./dist/src/link.d.ts",
-      "import": "./src/link.js"
+      "import": "./dist/src/diff.js"
     },
     "./merge": {
       "types": "./dist/src/merge.d.ts",
-      "import": "./src/merge.js"
+      "import": "./dist/src/merge.js"
     },
     "./shard": {
       "types": "./dist/src/shard.d.ts",
-      "import": "./src/shard.js"
+      "import": "./dist/src/shard.js"
     }
   },
   "bin": {
-    "pail": "./cli.js"
-  },
-  "scripts": {
-    "build": "tsc --build",
-    "test": "mocha test/*.test.js",
-    "coverage": "c8 -r html -r text npm test",
-    "lint": "standard"
+    "pail": "./dist/cli.js",
+    "pail-propernames": "./scripts/propernames/gen.sh",
+    "pail-gen": "./dist/scripts/words/gen.js",
+    "pail-randomcid": "./dist/scripts/randomcid.js"
   },
   "keywords": [
     "bucket",
@@ -139,23 +80,25 @@
   "author": "Alan Shaw",
   "license": "Apache-2.0 OR MIT",
   "files": [
-    "src",
     "dist"
   ],
   "dependencies": {
-    "@ipld/dag-cbor": "^9.2.0",
-    "multiformats": "^13.1.0"
-  },
-  "devDependencies": {
+    "@ipld/dag-cbor": "^9.2.2",
+    "cli-color": "^2.0.3",
+    "@ipld/car": "^5.4.0",
     "archy": "^1.0.0",
-    "c8": "^8.0.1",
     "carstream": "^2.0.0",
-    "cli-color": "^2.0.3",
-    "mocha": "^10.2.0",
-    "nanoid": "^4.0.0",
-    "sade": "^1.8.1",
-    "standard": "^17.0.0",
-    "typescript": "^5.0.2"
+    "multiformats": "^13.3.2",
+    "sade": "^1.8.1"
+  },
+  "devDependencies": {
+    "@types/archy": "^0.0.36",
+    "@types/cli-color": "^2.0.6",
+    "@types/node": "^22.13.17",
+    "@vitest/coverage-v8": "^3.1.1",
+    "standard": "^17.1.2",
+    "typescript": "^5.8.2",
+    "vitest": "^3.1.1"
   },
   "repository": {
     "type": "git",
@@ -166,8 +109,20 @@
   },
   "homepage": "https://github.com/web3-storage/pail#readme",
   "standard": {
-    "ignore": [
-      "*.ts"
+    "globals": [
+      "describe",
+      "before",
+      "after",
+      "beforeEach",
+      "afterEach",
+      "it",
+      "assert"
     ]
+  },
+  "scripts": {
+    "build": "tsc --build",
+    "test": "vitest --run",
+    "coverage": "vitest --run --coverage",
+    "lint": "standard"
   }
-}
+}

package/scripts/propernames/gen.sh

@@ -0,0 +1,3 @@
+while read p; do
+  pail put $p $(pail-randomcid) --max-shard-size=10000
+done </usr/share/dict/propernames

package/cli.js

@@ -1,218 +0,0 @@
-#!/usr/bin/env node
-import fs from 'fs'
-import os from 'os'
-import { join } from 'path'
-import { Readable, Writable } from 'stream'
-import sade from 'sade'
-import { CID } from 'multiformats/cid'
-import { CARReaderStream, CARWriterStream } from 'carstream'
-import clc from 'cli-color'
-import archy from 'archy'
-// eslint-disable-next-line no-unused-vars
-import * as API from './src/api.js'
-import { put, get, del, entries } from './src/index.js'
-import { ShardFetcher, ShardBlock, isShardLink } from './src/shard.js'
-import { difference } from './src/diff.js'
-import { merge } from './src/merge.js'
-import { MemoryBlockstore, MultiBlockFetcher } from './src/block.js'
-
-const cli = sade('pail')
-  .option('--path', 'Path to data store.', './pail.car')
-
-cli.command('put <key> <value>')
-  .describe('Put a value (a CID) for the given key. If the key exists it\'s value is overwritten.')
-  .alias('set')
-  .action(async (key, value, opts) => {
-    const { root: prevRoot, blocks } = await openPail(opts.path)
-    const { root, additions, removals } = await put(blocks, prevRoot, key, CID.parse(value))
-    await updatePail(opts.path, blocks, root, { additions, removals })
-
-    console.log(clc.red(`--- ${prevRoot}`))
-    console.log(clc.green(`+++ ${root}`))
-    console.log(clc.magenta('@@ -1 +1 @@'))
-    additions.forEach(b => console.log(clc.green(`+${b.cid}`)))
-    removals.forEach(b => console.log(clc.red(`-${b.cid}`)))
-  })
-
-cli.command('get <key>')
-  .describe('Get the stored value for the given key from the pail. If the key is not found, `undefined` is returned.')
-  .action(async (key, opts) => {
-    const { root, blocks } = await openPail(opts.path)
-    const value = await get(blocks, root, key)
-    if (value) console.log(value.toString())
-  })
-
-cli.command('del <key>')
-  .describe('Delete the value for the given key from the pail. If the key is not found no operation occurs.')
-  .alias('delete', 'rm', 'remove')
-  .action(async (key, opts) => {
-    const { root: prevRoot, blocks } = await openPail(opts.path)
-    const { root, additions, removals } = await del(blocks, prevRoot, key)
-    await updatePail(opts.path, blocks, root, { additions, removals })
-
-    console.log(clc.red(`--- ${prevRoot}`))
-    console.log(clc.green(`+++ ${root}`))
-    console.log(clc.magenta('@@ -1 +1 @@'))
-    additions.forEach(b => console.log(clc.green(`+ ${b.cid}`)))
-    removals.forEach(b => console.log(clc.red(`- ${b.cid}`)))
-  })
-
-cli.command('ls')
-  .describe('List entries in the pail.')
-  .alias('list')
-  .option('-p, --prefix', 'Key prefix to filter by.')
-  .option('--gt', 'Filter results by keys greater than this string.')
-  .option('--lt', 'Filter results by keys less than this string.')
-  .option('--json', 'Format output as newline delimted JSON.')
-  .action(async (opts) => {
-    const { root, blocks } = await openPail(opts.path)
-    let n = 0
-    for await (const [k, v] of entries(blocks, root, { prefix: opts.prefix, gt: opts.gt, lt: opts.lt })) {
-      console.log(opts.json ? JSON.stringify({ key: k, value: v.toString() }) : `${k}\t${v}`)
-      n++
-    }
-    if (!opts.json) console.log(`total ${n}`)
-  })
-
-cli.command('tree')
-  .describe('Visualise the pail.')
-  .alias('vis')
-  .action(async (opts) => {
-    const { root, blocks } = await openPail(opts.path)
-    const shards = new ShardFetcher(blocks)
-    const rshard = await shards.get(root)
-
-    /** @type {archy.Data} */
-    const archyRoot = { label: `Shard(${clc.yellow(rshard.cid.toString())}) ${rshard.bytes.length + 'b'}`, nodes: [] }
-
-    /** @param {API.ShardEntry} entry */
-    const getData = async ([k, v]) => {
-      if (!Array.isArray(v)) {
-        return { label: `Key(${clc.magenta(k)})`, nodes: [{ label: `Value(${clc.cyan(v)})` }] }
-      }
-      /** @type {archy.Data} */
-      const data = { label: `Key(${clc.magenta(k)})`, nodes: [] }
-      if (v[1]) data.nodes?.push({ label: `Value(${clc.cyan(v[1])})` })
-      const blk = await shards.get(v[0])
-      data.nodes?.push({
-        label: `Shard(${clc.yellow(v[0])}) ${blk.bytes.length + 'b'}`,
-        nodes: await Promise.all(blk.value.entries.map(e => getData(e)))
-      })
-      return data
-    }
-
-    for (const entry of rshard.value.entries) {
-      archyRoot.nodes?.push(await getData(entry))
-    }
-
-    console.log(archy(archyRoot))
-  })
-
-cli.command('diff <path>')
-  .describe('Find the differences between this pail and the passed pail.')
-  .option('-k, --keys', 'Output key/value diff.')
-  .action(async (path, opts) => {
-    const [
-      { root: aroot, blocks: ablocks },
-      { root: broot, blocks: bblocks }
-    ] = await Promise.all([openPail(opts.path), openPail(path)])
-    if (aroot.toString() === broot.toString()) return
-
-    const fetcher = new MultiBlockFetcher(ablocks, bblocks)
-    const { shards: { additions, removals }, keys } = await difference(fetcher, aroot, broot)
-
-    console.log(clc.red(`--- ${aroot}`))
-    console.log(clc.green(`+++ ${broot}`))
-    console.log(clc.magenta('@@ -1 +1 @@'))
-
-    if (opts.keys) {
-      keys.forEach(([k, v]) => {
-        if (v[0] != null) console.log(clc.red(`- ${k}\t${v[0]}`))
-        if (v[1] != null) console.log(clc.green(`+ ${k}\t${v[1]}`))
-      })
-    } else {
-      additions.forEach(b => console.log(clc.green(`+ ${b.cid}`)))
-      removals.forEach(b => console.log(clc.red(`- ${b.cid}`)))
-    }
-  })
-
-cli.command('merge <path>')
-  .describe('Merge the passed pail into this pail.')
-  .action(async (path, opts) => {
-    const [
-      { root: aroot, blocks: ablocks },
-      { root: broot, blocks: bblocks }
-    ] = await Promise.all([openPail(opts.path), openPail(path)])
-    if (aroot.toString() === broot.toString()) return
-
-    const fetcher = new MultiBlockFetcher(ablocks, bblocks)
-    const { root, additions, removals } = await merge(fetcher, aroot, [broot])
-
-    await updatePail(opts.path, ablocks, root, { additions, removals })
-
-    console.log(clc.red(`--- ${aroot}`))
-    console.log(clc.green(`+++ ${root}`))
-    console.log(clc.magenta('@@ -1 +1 @@'))
-    additions.forEach(b => console.log(clc.green(`+ ${b.cid}`)))
-    removals.forEach(b => console.log(clc.red(`- ${b.cid}`)))
-  })
-
-cli.parse(process.argv)
-
-/**
- * @param {string} path
- * @returns {Promise<{ root: API.ShardLink, blocks: MemoryBlockstore }>}
- */
-async function openPail (path) {
-  const blocks = new MemoryBlockstore()
-  try {
-    const carReader = new CARReaderStream()
-    const readable = /** @type {ReadableStream<Uint8Array>} */ (Readable.toWeb(fs.createReadStream(path)))
-    await readable.pipeThrough(carReader).pipeTo(new WritableStream({ write: b => blocks.put(b.cid, b.bytes) }))
-    const header = await carReader.getHeader()
-    if (!isShardLink(header.roots[0])) throw new Error(`not a shard: ${header.roots[0]}`)
-    return { root: header.roots[0], blocks }
-  } catch (err) {
-    if (err.code !== 'ENOENT') throw new Error('failed to open bucket', { cause: err })
-    const rootblk = await ShardBlock.create()
-    blocks.put(rootblk.cid, rootblk.bytes)
-    return { root: rootblk.cid, blocks }
-  }
-}
-
-/**
- * @param {string} path
- * @param {MemoryBlockstore} blocks
- * @param {API.ShardLink} root
- * @param {API.ShardDiff} diff
- */
-async function updatePail (path, blocks, root, { additions, removals }) {
-  const tmp = join(os.tmpdir(), `pail${Date.now()}.car`)
-  const iterator = blocks.entries()
-  const readable = new ReadableStream({
-    start (controller) {
-      for (const b of additions) controller.enqueue(b)
-    },
-    pull (controller) {
-      for (const b of iterator) {
-        if (removals.some(r => b.cid.toString() === r.cid.toString())) continue
-        return controller.enqueue(b)
-      }
-      controller.close()
-    }
-  })
-  await readable.pipeThrough(new CARWriterStream([root])).pipeTo(Writable.toWeb(fs.createWriteStream(tmp)))
-
-  const old = `${path}-${new Date().toISOString()}`
-  try {
-    await fs.promises.rename(path, old)
-  } catch (err) {
-    if (err.code !== 'ENOENT') throw err
-  }
-  await fs.promises.rename(tmp, path)
-  try {
-    await fs.promises.rm(old)
-  } catch (err) {
-    if (err.code !== 'ENOENT') throw err
-  }
-}

package/src/api.js

@@ -1 +0,0 @@
-export {}

package/src/api.ts

@@ -1,90 +0,0 @@
-import { Link, UnknownLink, BlockView, Block, Version } from 'multiformats'
-import { sha256 } from 'multiformats/hashes/sha2'
-import * as dagCBOR from '@ipld/dag-cbor'
-
-export { Link, UnknownLink, BlockView, Block, Version }
-
-export type ShardEntryValueValue = UnknownLink
-
-export type ShardEntryLinkValue = [ShardLink]
-
-export type ShardEntryLinkAndValueValue = [ShardLink, UnknownLink]
-
-export type ShardValueEntry = [key: string, value: ShardEntryValueValue]
-
-export type ShardLinkEntry = [key: string, value: ShardEntryLinkValue | ShardEntryLinkAndValueValue]
-
-/** Single key/value entry within a shard. */
-export type ShardEntry = [key: string, value: ShardEntryValueValue | ShardEntryLinkValue | ShardEntryLinkAndValueValue]
-
-export interface Shard extends ShardConfig {
-  entries: ShardEntry[]
-}
-
-export type ShardLink = Link<Shard, typeof dagCBOR.code, typeof sha256.code, 1>
-
-export interface ShardBlockView extends BlockView<Shard, typeof dagCBOR.code, typeof sha256.code, 1> {}
-
-export interface ShardDiff {
-  additions: ShardBlockView[]
-  removals: ShardBlockView[]
-}
-
-export interface BlockFetcher {
-  get<T = unknown, C extends number = number, A extends number = number, V extends Version = 1> (link: Link<T, C, A, V>):
-    Promise<Block<T, C, A, V> | undefined>
-}
-
-export interface ShardConfig {
-  /** Shard compatibility version. */
-  version: number
-  /**
-   * Characters allowed in keys, referring to a known character set.
-   * e.g. "ascii" refers to the printable ASCII characters in the code range 32-126.
-   */
-  keyChars: string
-  /** Max key size in bytes - default 4096 bytes. */
-  maxKeySize: number
-  /** The key prefix from the root to this shard. */
-  prefix: string
-}
-
-export type ShardOptions = Partial<ShardConfig>
-
-export interface KeyPrefixOption {
-  /** Filter results to entries with keys prefixed with this string. */
-  prefix: string
-}
-
-export type KeyRangeOption =
-  | KeyLowerBoundRangeOption
-  | KeyUpperBoundRangeOption
-  | (KeyLowerBoundRangeOption & KeyUpperBoundRangeOption)
-
-export type KeyLowerBoundRangeOption =
-  | KeyLowerBoundRangeExclusiveOption
-  | KeyLowerBoundRangeInclusiveOption
-
-export interface KeyLowerBoundRangeExclusiveOption {
-  gt: string
-}
-
-export interface KeyLowerBoundRangeInclusiveOption {
-  gte: string
-}
-
-export type KeyUpperBoundRangeOption =
-  | KeyUpperBoundRangeExclusiveOption
-  | KeyUpperBoundRangeInclusiveOption
-
-export interface KeyUpperBoundRangeExclusiveOption {
-  lt: string
-}
-
-export interface KeyUpperBoundRangeInclusiveOption {
-  lte: string
-}
-
-export type EntriesOptions =
-  | KeyPrefixOption
-  | KeyRangeOption

package/src/batch/api.js

@@ -1 +0,0 @@
-export {}

package/src/batch/api.ts

@@ -1,59 +0,0 @@
-import {
-  UnknownLink,
-  ShardLink,
-  ShardDiff,
-  ShardEntry,
-  ShardEntryValueValue,
-  ShardEntryLinkValue,
-  ShardEntryLinkAndValueValue,
-  ShardConfig,
-  ShardOptions,
-  ShardBlockView,
-  BlockFetcher
-} from '../api.js'
-
-export {
-  UnknownLink,
-  ShardLink,
-  ShardDiff,
-  ShardEntry,
-  ShardEntryValueValue,
-  ShardEntryLinkValue,
-  ShardEntryLinkAndValueValue,
-  ShardConfig,
-  ShardOptions,
-  ShardBlockView,
-  BlockFetcher
-}
-
-export interface BatcherShard extends ShardConfig {
-  base?: ShardBlockView
-  entries: BatcherShardEntry[]
-}
-
-export interface BatcherShardInit extends ShardOptions {
-  base?: ShardBlockView
-  entries?: BatcherShardEntry[]
-}
-
-export type BatcherShardEntry = [
-  key: string,
-  value: ShardEntryValueValue | ShardEntryLinkValue | ShardEntryLinkAndValueValue | ShardEntryShardValue | ShardEntryShardAndValueValue
-]
-
-export type ShardEntryShardValue = [BatcherShard]
-
-export type ShardEntryShardAndValueValue = [BatcherShard, UnknownLink]
-
-export interface Batcher {
-  /**
-   * Put a value (a CID) for the given key. If the key exists it's value is
-   * overwritten.
-   */
-  put (key: string, value: UnknownLink): Promise<void>
-  /**
-   * Encode all altered shards in the batch and return the new root CID and
-   * difference blocks.
-   */
-  commit (): Promise<{ root: ShardLink } & ShardDiff>
-}