@fireproof/core 0.0.1

package/index.js

@@ -0,0 +1,5 @@
+import Fireproof from './src/fireproof'
+import Index from './src/db-index'
+import Listener from './src/listener'
+
+export { Fireproof, Index, Listener }

package/package.json

@@ -0,0 +1,70 @@
+{
+  "name": "@fireproof/core",
+  "version": "0.0.1",
+  "description": "Realtime database for IPFS",
+  "main": "index.js",
+  "type": "module",
+  "scripts": {
+    "test": "standard && mocha test/*.test.js",
+    "coverage": "c8 -r html -r text npm test",
+    "lint": "standard",
+    "lint:fix": "standard --fix"
+  },
+  "keywords": [
+    "database",
+    "JSON",
+    "document",
+    "IPLD",
+    "CID",
+    "IPFS"
+  ],
+  "contributors": [
+    "J Chris Anderson",
+    "Alan Shaw",
+    "Travis Vachon",
+    "Mikeal Rogers"
+  ],
+  "license": "Apache-2.0 OR MIT",
+  "dependencies": {
+    "@ipld/car": "^5.1.0",
+    "@ipld/dag-cbor": "^9.0.0",
+    "archy": "^1.0.0",
+    "car-transaction": "^1.0.1",
+    "charwise": "^3.0.1",
+    "cli-color": "^2.0.3",
+    "idb": "^7.1.1",
+    "multiformats": "^11.0.1",
+    "prolly-trees": "^0.2.2",
+    "sade": "^1.8.1"
+  },
+  "devDependencies": {
+    "c8": "^7.12.0",
+    "fake-indexeddb": "^4.0.1",
+    "mocha": "^10.2.0",
+    "nanoid": "^4.0.0",
+    "standard": "^17.0.0"
+  },
+  "mocha": {
+    "require": [
+      "fake-indexeddb/auto"
+    ]
+  },
+  "standard": {
+    "ignore": [
+      "examples/**/*.tsx",
+      "examples/**/dist"
+    ]
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jchris/fireproof.git"
+  },
+  "bugs": {
+    "url": "https://github.com/jchris/fireproof/issues"
+  },
+  "homepage": "https://github.com/jchris/fireproof#readme",
+  "workspaces": [
+    "examples/todomvc"
+  ]
+
+}

package/scripts/propernames/gen.sh

@@ -0,0 +1,3 @@
+while read p; do
+  pail put $p $(node ../randomcid.js) --max-shard-size=10000
+done </usr/share/dict/propernames

package/scripts/randomcid.js

@@ -0,0 +1,12 @@
+import crypto from 'node:crypto'
+import { CID } from 'multiformats/cid'
+import * as raw from 'multiformats/codecs/raw'
+import { sha256 } from 'multiformats/hashes/sha2'
+
+async function main () {
+  const bytes = crypto.webcrypto.getRandomValues(new Uint8Array(32))
+  const hash = await sha256.digest(bytes)
+  process.stdout.write(CID.create(1, raw.code, hash).toString())
+}
+
+main()

package/scripts/words/gen.js

@@ -0,0 +1,55 @@
+import fs from 'node:fs'
+import { Readable } from 'node:stream'
+import { CarWriter } from '@ipld/car'
+import { CID } from 'multiformats/cid'
+import * as raw from 'multiformats/codecs/raw'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { ShardBlock, put } from '../../index.js'
+import { MemoryBlockstore } from '../../block.js'
+
+/** @param {string} str */
+async function stringToCID (str) {
+  const hash = await sha256.digest(new TextEncoder().encode(str))
+  return CID.create(1, raw.code, hash)
+}
+
+async function main () {
+  const data = await fs.promises.readFile('/usr/share/dict/words', 'utf8')
+  const words = data.split(/\n/)
+  const cids = await Promise.all(words.map(stringToCID))
+  const blocks = new MemoryBlockstore()
+  const rootblk = await ShardBlock.create()
+  blocks.putSync(rootblk.cid, rootblk.bytes)
+
+  console.time(`put x${words.length}`)
+  /** @type {import('../../shard').ShardLink} */
+  let root = rootblk.cid
+  for (const [i, word] of words.entries()) {
+    const res = await put(blocks, root, word, cids[i])
+    root = res.root
+    for (const b of res.additions) {
+      blocks.putSync(b.cid, b.bytes)
+    }
+    for (const b of res.removals) {
+      blocks.deleteSync(b.cid)
+    }
+    if (i % 1000 === 0) {
+      console.log(`${Math.floor(i / words.length * 100)}%`)
+    }
+  }
+  console.timeEnd(`put x${words.length}`)
+
+  // @ts-expect-error
+  const { writer, out } = CarWriter.create(root)
+  const finishPromise = new Promise(resolve => {
+    Readable.from(out).pipe(fs.createWriteStream('./pail.car')).on('finish', resolve)
+  })
+
+  for (const b of blocks.entries()) {
+    await writer.put(b)
+  }
+  await writer.close()
+  await finishPromise
+}
+
+main()

package/src/block.js

@@ -0,0 +1,75 @@
+import { parse } from 'multiformats/link'
+
+/**
+ * @typedef {{ cid: import('./link').AnyLink, bytes: Uint8Array }} AnyBlock
+ * @typedef {{ get: (link: import('./link').AnyLink) => Promise<AnyBlock | undefined> }} BlockFetcher
+ */
+
+/** @implements {BlockFetcher} */
+export class MemoryBlockstore {
+  /** @type {Map<string, Uint8Array>} */
+  #blocks = new Map()
+
+  /**
+   * @param {import('./link').AnyLink} cid
+   * @returns {Promise<AnyBlock | undefined>}
+   */
+  async get (cid) {
+    const bytes = this.#blocks.get(cid.toString())
+    if (!bytes) return
+    return { cid, bytes }
+  }
+
+  /**
+   * @param {import('./link').AnyLink} cid
+   * @param {Uint8Array} bytes
+   */
+  async put (cid, bytes) {
+    // console.log('put', cid)
+    this.#blocks.set(cid.toString(), bytes)
+  }
+
+  /**
+   * @param {import('./link').AnyLink} cid
+   * @param {Uint8Array} bytes
+   */
+  putSync (cid, bytes) {
+    this.#blocks.set(cid.toString(), bytes)
+  }
+
+  /** @param {import('./link').AnyLink} cid */
+  async delete (cid) {
+    this.#blocks.delete(cid.toString())
+  }
+
+  /** @param {import('./link').AnyLink} cid */
+  deleteSync (cid) {
+    this.#blocks.delete(cid.toString())
+  }
+
+  * entries () {
+    for (const [str, bytes] of this.#blocks) {
+      yield { cid: parse(str), bytes }
+    }
+  }
+}
+
+export class MultiBlockFetcher {
+  /** @type {BlockFetcher[]} */
+  #fetchers
+
+  /** @param {BlockFetcher[]} fetchers */
+  constructor (...fetchers) {
+    this.#fetchers = fetchers
+  }
+
+  /** @param {import('./link').AnyLink} link */
+  async get (link) {
+    for (const f of this.#fetchers) {
+      const v = await f.get(link)
+      if (v) {
+        return v
+      }
+    }
+  }
+}

package/src/blockstore.js

@@ -0,0 +1,246 @@
+import { parse } from 'multiformats/link'
+import * as raw from 'multiformats/codecs/raw'
+import { sha256 } from 'multiformats/hashes/sha2'
+import * as Block from 'multiformats/block'
+import * as CBW from '@ipld/car/buffer-writer'
+
+import Valet from './valet.js'
+
+// const sleep = ms => new Promise(r => setTimeout(r, ms))
+
+/**
+ * @typedef {Object} AnyBlock
+ * @property {import('./link').AnyLink} cid - The CID of the block
+ * @property {Uint8Array} bytes - The block's data
+ *
+ * @typedef {Object} Blockstore
+ * @property {function(import('./link').AnyLink): Promise<AnyBlock|undefined>} get - A function to retrieve a block by CID
+ * @property {function(import('./link').AnyLink, Uint8Array): Promise<void>} put - A function to store a block's data and CID
+ *
+ * A blockstore that caches writes to a transaction and only persists them when committed.
+ * @implements {Blockstore}
+ */
+export default class TransactionBlockstore {
+  /** @type {Map<string, Uint8Array>} */
+  #oldBlocks = new Map()
+
+  #valet = new Valet() // cars by cid
+
+  #instanceId = 'blkz.' + Math.random().toString(36).substring(2, 4)
+  #inflightTransactions = new Set()
+
+  /**
+   * Get a block from the store.
+   *
+   * @param {import('./link').AnyLink} cid
+   * @returns {Promise<AnyBlock | undefined>}
+   */
+  async get (cid) {
+    const key = cid.toString()
+    // it is safe to read from the in-flight transactions becauase they are immutable
+    // const bytes = this.#oldBlocks.get(key) || await this.#valet.getBlock(key)
+    const bytes = await this.#transactionsGet(key) || await this.commitedGet(key)
+    // const bytes = this.#blocks.get(key) || await this.#valet.getBlock(key)
+    // console.log('bytes', typeof bytes)
+    if (!bytes) throw new Error('Missing block: ' + key)
+    return { cid, bytes }
+  }
+
+  // this iterates over the in-flight transactions
+  // and returns the first matching block it finds
+  async #transactionsGet (key) {
+    for (const transaction of this.#inflightTransactions) {
+      const got = await transaction.get(key)
+      if (got && got.bytes) return got.bytes
+    }
+  }
+
+  async commitedGet (key) {
+    return this.#oldBlocks.get(key) || await this.#valet.getBlock(key)
+    // return await this.#valet.getBlock(key) // todo this is just for testing
+  }
+
+  /**
+   * Add a block to the store. Usually bound to a transaction by a closure.
+   * It sets the lastCid property to the CID of the block that was put.
+   * This is used by the transaction as the head of the car when written to the valet.
+   * We don't have to worry about which transaction we are when we are here because
+   * we are the transactionBlockstore.
+   *
+   * @param {import('./link').AnyLink} cid
+   * @param {Uint8Array} bytes
+   */
+  put (cid, bytes) {
+    throw new Error('use a transaction to put')
+  }
+
+  /**
+   * Iterate over all blocks in the store.
+   *
+   * @yields {AnyBlock}
+   * @returns {AsyncGenerator<AnyBlock>}
+   */
+  * entries () {
+    // todo needs transaction blocks?
+    // for (const [str, bytes] of this.#blocks) {
+    //   yield { cid: parse(str), bytes }
+    // }
+    for (const [str, bytes] of this.#oldBlocks) {
+      yield { cid: parse(str), bytes }
+    }
+  }
+
+  /**
+     * Begin a transaction. Ensures the uncommited blocks are empty at the begining.
+     * Returns the blocks to read and write during the transaction.
+     * @returns {InnerBlockstore}
+     * @memberof TransactionBlockstore
+     */
+  begin (label = '') {
+    const innerTransactionBlockstore = new InnerBlockstore(label, this)
+    this.#inflightTransactions.add(innerTransactionBlockstore)
+    return innerTransactionBlockstore
+  }
+
+  /**
+     * Commit the transaction. Writes the blocks to the store.
+     * @returns {Promise<void>}
+     * @memberof TransactionBlockstore
+     */
+  async commit (innerBlockstore) {
+    await this.#doCommit(innerBlockstore)
+  }
+
+  // first get the transaction blockstore from the map of transaction blockstores
+  // then copy it to oldBlocks
+  // then write the transaction blockstore to a car
+  // then write the car to the valet
+  // then remove the transaction blockstore from the map of transaction blockstores
+  #doCommit = async (innerBlockstore) => {
+    const cids = new Set()
+    for (const { cid, bytes } of innerBlockstore.entries()) {
+      const stringCid = cid.toString() // unnecessary string conversion, can we fix upstream?
+      if (this.#oldBlocks.has(stringCid)) {
+        // console.log('Duplicate block: ' + stringCid)
+      } else {
+        this.#oldBlocks.set(stringCid, bytes)
+        cids.add(stringCid)
+      }
+    }
+    if (cids.size > 0) {
+      console.log(innerBlockstore.label, 'committing', cids.size, 'blocks')
+      await this.#valetWriteTransaction(innerBlockstore, cids)
+    }
+  }
+
+  /**
+   * Group the blocks into a car and write it to the valet.
+   * @param {InnerBlockstore} innerBlockstore
+   * @param {Set<string>} cids
+   * @returns {Promise<void>}
+   * @memberof TransactionBlockstore
+   * @private
+   */
+  #valetWriteTransaction = async (innerBlockstore, cids) => {
+    if (innerBlockstore.lastCid) {
+      const newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
+      await this.#valet.parkCar(newCar.cid.toString(), newCar.bytes, cids)
+    }
+  }
+
+  /**
+     * Retire the transaction. Clears the uncommited blocks.
+     * @returns {void}
+     * @memberof TransactionBlockstore
+     */
+  retire (innerBlockstore) {
+    this.#inflightTransactions.delete(innerBlockstore)
+  }
+}
+
+/**
+ * Runs a function on an inner blockstore, then persists the change to a car writer
+ * or other outer blockstore.
+ * @param {string} label
+ * @param {TransactionBlockstore} blockstore
+ * @param {(innerBlockstore: Blockstore) => Promise<any>} doFun
+ * @returns {Promise<any>}
+ * @memberof TransactionBlockstore
+ */
+export const doTransaction = async (label, blockstore, doFun) => {
+  if (!blockstore.commit) return await doFun(blockstore)
+  const innerBlockstore = blockstore.begin(label)
+  try {
+    const result = await doFun(innerBlockstore)
+    await blockstore.commit(innerBlockstore)
+    return result
+  } catch (e) {
+    console.error(`Transaction ${label} failed`, e, e.stack)
+    throw e
+  } finally {
+    blockstore.retire(innerBlockstore)
+  }
+}
+
+const blocksToCarBlock = async (lastCid, blocks) => {
+  let size = 0
+  const headerSize = CBW.headerLength({ roots: [lastCid] })
+  size += headerSize
+  for (const { cid, bytes } of blocks.entries()) {
+    size += CBW.blockLength({ cid, bytes })
+  }
+  const buffer = new Uint8Array(size)
+  const writer = await CBW.createWriter(buffer, { headerSize })
+
+  writer.addRoot(lastCid)
+
+  for (const { cid, bytes } of blocks.entries()) {
+    writer.write({ cid, bytes })
+  }
+  await writer.close()
+  return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
+}
+
+/** @implements {BlockFetcher} */
+export class InnerBlockstore {
+  /** @type {Map<string, Uint8Array>} */
+  #blocks = new Map()
+  lastCid = null
+  label = ''
+  parentBlockstore = null
+
+  constructor (label, parentBlockstore) {
+    this.label = label
+    this.parentBlockstore = parentBlockstore
+  }
+
+  /**
+   * @param {import('./link').AnyLink} cid
+   * @returns {Promise<AnyBlock | undefined>}
+   */
+  async get (cid) {
+    const key = cid.toString()
+    let bytes = this.#blocks.get(key)
+    if (bytes) { return { cid, bytes } }
+    bytes = await this.parentBlockstore.commitedGet(key)
+    if (bytes) {
+      return { cid, bytes }
+    }
+  }
+
+  /**
+   * @param {import('./link').AnyLink} cid
+   * @param {Uint8Array} bytes
+   */
+  put (cid, bytes) {
+    // console.log('put', cid)
+    this.#blocks.set(cid.toString(), bytes)
+    this.lastCid = cid
+  }
+
+  * entries () {
+    for (const [str, bytes] of this.#blocks) {
+      yield { cid: parse(str), bytes }
+    }
+  }
+}