@fireproof/core 0.0.1 → 0.0.2-dev-test

package/src/blockstore.js

@@ -1,246 +0,0 @@
-import { parse } from 'multiformats/link'
-import * as raw from 'multiformats/codecs/raw'
-import { sha256 } from 'multiformats/hashes/sha2'
-import * as Block from 'multiformats/block'
-import * as CBW from '@ipld/car/buffer-writer'
-
-import Valet from './valet.js'
-
-// const sleep = ms => new Promise(r => setTimeout(r, ms))
-
-/**
- * @typedef {Object} AnyBlock
- * @property {import('./link').AnyLink} cid - The CID of the block
- * @property {Uint8Array} bytes - The block's data
- *
- * @typedef {Object} Blockstore
- * @property {function(import('./link').AnyLink): Promise<AnyBlock|undefined>} get - A function to retrieve a block by CID
- * @property {function(import('./link').AnyLink, Uint8Array): Promise<void>} put - A function to store a block's data and CID
- *
- * A blockstore that caches writes to a transaction and only persists them when committed.
- * @implements {Blockstore}
- */
-export default class TransactionBlockstore {
-  /** @type {Map<string, Uint8Array>} */
-  #oldBlocks = new Map()
-
-  #valet = new Valet() // cars by cid
-
-  #instanceId = 'blkz.' + Math.random().toString(36).substring(2, 4)
-  #inflightTransactions = new Set()
-
-  /**
-   * Get a block from the store.
-   *
-   * @param {import('./link').AnyLink} cid
-   * @returns {Promise<AnyBlock | undefined>}
-   */
-  async get (cid) {
-    const key = cid.toString()
-    // it is safe to read from the in-flight transactions becauase they are immutable
-    // const bytes = this.#oldBlocks.get(key) || await this.#valet.getBlock(key)
-    const bytes = await this.#transactionsGet(key) || await this.commitedGet(key)
-    // const bytes = this.#blocks.get(key) || await this.#valet.getBlock(key)
-    // console.log('bytes', typeof bytes)
-    if (!bytes) throw new Error('Missing block: ' + key)
-    return { cid, bytes }
-  }
-
-  // this iterates over the in-flight transactions
-  // and returns the first matching block it finds
-  async #transactionsGet (key) {
-    for (const transaction of this.#inflightTransactions) {
-      const got = await transaction.get(key)
-      if (got && got.bytes) return got.bytes
-    }
-  }
-
-  async commitedGet (key) {
-    return this.#oldBlocks.get(key) || await this.#valet.getBlock(key)
-    // return await this.#valet.getBlock(key) // todo this is just for testing
-  }
-
-  /**
-   * Add a block to the store. Usually bound to a transaction by a closure.
-   * It sets the lastCid property to the CID of the block that was put.
-   * This is used by the transaction as the head of the car when written to the valet.
-   * We don't have to worry about which transaction we are when we are here because
-   * we are the transactionBlockstore.
-   *
-   * @param {import('./link').AnyLink} cid
-   * @param {Uint8Array} bytes
-   */
-  put (cid, bytes) {
-    throw new Error('use a transaction to put')
-  }
-
-  /**
-   * Iterate over all blocks in the store.
-   *
-   * @yields {AnyBlock}
-   * @returns {AsyncGenerator<AnyBlock>}
-   */
-  * entries () {
-    // todo needs transaction blocks?
-    // for (const [str, bytes] of this.#blocks) {
-    //   yield { cid: parse(str), bytes }
-    // }
-    for (const [str, bytes] of this.#oldBlocks) {
-      yield { cid: parse(str), bytes }
-    }
-  }
-
-  /**
-     * Begin a transaction. Ensures the uncommited blocks are empty at the begining.
-     * Returns the blocks to read and write during the transaction.
-     * @returns {InnerBlockstore}
-     * @memberof TransactionBlockstore
-     */
-  begin (label = '') {
-    const innerTransactionBlockstore = new InnerBlockstore(label, this)
-    this.#inflightTransactions.add(innerTransactionBlockstore)
-    return innerTransactionBlockstore
-  }
-
-  /**
-     * Commit the transaction. Writes the blocks to the store.
-     * @returns {Promise<void>}
-     * @memberof TransactionBlockstore
-     */
-  async commit (innerBlockstore) {
-    await this.#doCommit(innerBlockstore)
-  }
-
-  // first get the transaction blockstore from the map of transaction blockstores
-  // then copy it to oldBlocks
-  // then write the transaction blockstore to a car
-  // then write the car to the valet
-  // then remove the transaction blockstore from the map of transaction blockstores
-  #doCommit = async (innerBlockstore) => {
-    const cids = new Set()
-    for (const { cid, bytes } of innerBlockstore.entries()) {
-      const stringCid = cid.toString() // unnecessary string conversion, can we fix upstream?
-      if (this.#oldBlocks.has(stringCid)) {
-        // console.log('Duplicate block: ' + stringCid)
-      } else {
-        this.#oldBlocks.set(stringCid, bytes)
-        cids.add(stringCid)
-      }
-    }
-    if (cids.size > 0) {
-      console.log(innerBlockstore.label, 'committing', cids.size, 'blocks')
-      await this.#valetWriteTransaction(innerBlockstore, cids)
-    }
-  }
-
-  /**
-   * Group the blocks into a car and write it to the valet.
-   * @param {InnerBlockstore} innerBlockstore
-   * @param {Set<string>} cids
-   * @returns {Promise<void>}
-   * @memberof TransactionBlockstore
-   * @private
-   */
-  #valetWriteTransaction = async (innerBlockstore, cids) => {
-    if (innerBlockstore.lastCid) {
-      const newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
-      await this.#valet.parkCar(newCar.cid.toString(), newCar.bytes, cids)
-    }
-  }
-
-  /**
-     * Retire the transaction. Clears the uncommited blocks.
-     * @returns {void}
-     * @memberof TransactionBlockstore
-     */
-  retire (innerBlockstore) {
-    this.#inflightTransactions.delete(innerBlockstore)
-  }
-}
-
-/**
- * Runs a function on an inner blockstore, then persists the change to a car writer
- * or other outer blockstore.
- * @param {string} label
- * @param {TransactionBlockstore} blockstore
- * @param {(innerBlockstore: Blockstore) => Promise<any>} doFun
- * @returns {Promise<any>}
- * @memberof TransactionBlockstore
- */
-export const doTransaction = async (label, blockstore, doFun) => {
-  if (!blockstore.commit) return await doFun(blockstore)
-  const innerBlockstore = blockstore.begin(label)
-  try {
-    const result = await doFun(innerBlockstore)
-    await blockstore.commit(innerBlockstore)
-    return result
-  } catch (e) {
-    console.error(`Transaction ${label} failed`, e, e.stack)
-    throw e
-  } finally {
-    blockstore.retire(innerBlockstore)
-  }
-}
-
-const blocksToCarBlock = async (lastCid, blocks) => {
-  let size = 0
-  const headerSize = CBW.headerLength({ roots: [lastCid] })
-  size += headerSize
-  for (const { cid, bytes } of blocks.entries()) {
-    size += CBW.blockLength({ cid, bytes })
-  }
-  const buffer = new Uint8Array(size)
-  const writer = await CBW.createWriter(buffer, { headerSize })
-
-  writer.addRoot(lastCid)
-
-  for (const { cid, bytes } of blocks.entries()) {
-    writer.write({ cid, bytes })
-  }
-  await writer.close()
-  return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
-}
-
-/** @implements {BlockFetcher} */
-export class InnerBlockstore {
-  /** @type {Map<string, Uint8Array>} */
-  #blocks = new Map()
-  lastCid = null
-  label = ''
-  parentBlockstore = null
-
-  constructor (label, parentBlockstore) {
-    this.label = label
-    this.parentBlockstore = parentBlockstore
-  }
-
-  /**
-   * @param {import('./link').AnyLink} cid
-   * @returns {Promise<AnyBlock | undefined>}
-   */
-  async get (cid) {
-    const key = cid.toString()
-    let bytes = this.#blocks.get(key)
-    if (bytes) { return { cid, bytes } }
-    bytes = await this.parentBlockstore.commitedGet(key)
-    if (bytes) {
-      return { cid, bytes }
-    }
-  }
-
-  /**
-   * @param {import('./link').AnyLink} cid
-   * @param {Uint8Array} bytes
-   */
-  put (cid, bytes) {
-    // console.log('put', cid)
-    this.#blocks.set(cid.toString(), bytes)
-    this.lastCid = cid
-  }
-
-  * entries () {
-    for (const [str, bytes] of this.#blocks) {
-      yield { cid: parse(str), bytes }
-    }
-  }
-}

package/src/clock.js

@@ -1,352 +0,0 @@
-import { Block, encode, decode } from 'multiformats/block'
-import { sha256 } from 'multiformats/hashes/sha2'
-import * as cbor from '@ipld/dag-cbor'
-
-/**
- * @template T
- * @typedef {{ parents: EventLink<T>[], data: T }} EventView
- */
-
-/**
- * @template T
- * @typedef {import('multiformats').BlockView<EventView<T>>} EventBlockView
- */
-
-/**
- * @template T
- * @typedef {import('multiformats').Link<EventView<T>>} EventLink
- */
-
-/**
- * Advance the clock by adding an event.
- *
- * @template T
- * @param {import('./block').BlockFetcher} blocks Block storage.
- * @param {EventLink<T>[]} head The head of the clock.
- * @param {EventLink<T>} event The event to add.
- * @returns {Promise<EventLink<T>[]>} The new head of the clock.
- */
-export async function advance (blocks, head, event) {
-  /** @type {EventFetcher<T>} */
-  const events = new EventFetcher(blocks)
-  const headmap = new Map(head.map(cid => [cid.toString(), cid]))
-
-  // Check if the headmap already includes the event, return head if it does
-  if (headmap.has(event.toString())) return head
-
-  // Does event contain the clock?
-  let changed = false
-  for (const cid of head) {
-    if (await contains(events, event, cid)) {
-      headmap.delete(cid.toString())
-      headmap.set(event.toString(), event)
-      changed = true
-    }
-  }
-
-  // If the headmap has been changed, return the new headmap values
-  if (changed) {
-    return [...headmap.values()]
-  }
-
-  // Does clock contain the event?
-  for (const p of head) {
-    if (await contains(events, p, event)) {
-      return head
-    }
-  }
-
-  // Return the head concatenated with the new event if it passes both checks
-  return head.concat(event)
-}
-
-/**
- * @template T
- * @implements {EventBlockView<T>}
- */
-export class EventBlock extends Block {
-  /**
-   * @param {object} config
-   * @param {EventLink<T>} config.cid
-   * @param {Event} config.value
-   * @param {Uint8Array} config.bytes
-   */
-  constructor ({ cid, value, bytes }) {
-    // @ts-expect-error
-    super({ cid, value, bytes })
-  }
-
-  /**
-   * @template T
-   * @param {T} data
-   * @param {EventLink<T>[]} [parents]
-   */
-  static create (data, parents) {
-    return encodeEventBlock({ data, parents: parents ?? [] })
-  }
-}
-
-/** @template T */
-export class EventFetcher {
-  /** @param {import('./block').BlockFetcher} blocks */
-  constructor (blocks) {
-    /** @private */
-    this._blocks = blocks
-  }
-
-  /**
-   * @param {EventLink<T>} link
-   * @returns {Promise<EventBlockView<T>>}
-   */
-  async get (link) {
-    const block = await this._blocks.get(link)
-    if (!block) throw new Error(`missing block: ${link}`)
-    return decodeEventBlock(block.bytes)
-  }
-}
-
-/**
- * @template T
- * @param {EventView<T>} value
- * @returns {Promise<EventBlockView<T>>}
- */
-export async function encodeEventBlock (value) {
-  // TODO: sort parents
-  const { cid, bytes } = await encode({ value, codec: cbor, hasher: sha256 })
-  // @ts-expect-error
-  return new Block({ cid, value, bytes })
-}
-
-/**
- * @template T
- * @param {Uint8Array} bytes
- * @returns {Promise<EventBlockView<T>>}
- */
-export async function decodeEventBlock (bytes) {
-  const { cid, value } = await decode({ bytes, codec: cbor, hasher: sha256 })
-  // @ts-expect-error
-  return new Block({ cid, value, bytes })
-}
-
-/**
- * Returns true if event "a" contains event "b". Breadth first search.
- * @template T
- * @param {EventFetcher} events
- * @param {EventLink<T>} a
- * @param {EventLink<T>} b
- */
-async function contains (events, a, b) {
-  if (a.toString() === b.toString()) return true
-  const [{ value: aevent }, { value: bevent }] = await Promise.all([events.get(a), events.get(b)])
-  const links = [...aevent.parents]
-  while (links.length) {
-    const link = links.shift()
-    if (!link) break
-    if (link.toString() === b.toString()) return true
-    // if any of b's parents are this link, then b cannot exist in any of the
-    // tree below, since that would create a cycle.
-    if (bevent.parents.some(p => link.toString() === p.toString())) continue
-    const { value: event } = await events.get(link)
-    links.push(...event.parents)
-  }
-  return false
-}
-
-/**
- * @template T
- * @param {import('./block').BlockFetcher} blocks Block storage.
- * @param {EventLink<T>[]} head
- * @param {object} [options]
- * @param {(b: EventBlockView<T>) => string} [options.renderNodeLabel]
- */
-export async function * vis (blocks, head, options = {}) {
-  const renderNodeLabel = options.renderNodeLabel ?? (b => (b.value.data.value))
-  const events = new EventFetcher(blocks)
-  yield 'digraph clock {'
-  yield '  node [shape=point fontname="Courier"]; head;'
-  const hevents = await Promise.all(head.map(link => events.get(link)))
-  const links = []
-  const nodes = new Set()
-  for (const e of hevents) {
-    nodes.add(e.cid.toString())
-    yield `  node [shape=oval fontname="Courier"]; ${e.cid} [label="${renderNodeLabel(e)}"];`
-    yield `  head -> ${e.cid};`
-    for (const p of e.value.parents) {
-      yield `  ${e.cid} -> ${p};`
-    }
-    links.push(...e.value.parents)
-  }
-  while (links.length) {
-    const link = links.shift()
-    if (!link) break
-    if (nodes.has(link.toString())) continue
-    nodes.add(link.toString())
-    const block = await events.get(link)
-    yield `  node [shape=oval]; ${link} [label="${renderNodeLabel(block)}" fontname="Courier"];`
-    for (const p of block.value.parents) {
-      yield `  ${link} -> ${p};`
-    }
-    links.push(...block.value.parents)
-  }
-  yield '}'
-}
-
-export async function findEventsToSync (blocks, head) {
-  const toSync = await findUnknownSortedEvents(blocks, head, await findCommonAncestorWithSortedEvents(blocks, head))
-  return toSync
-}
-
-export async function findUnknownSortedEvents (blocks, children, { ancestor, sorted }) {
-  const events = new EventFetcher(blocks)
-  // const childrenCids = children.map(c => c.toString())
-  // const lowerEvent = sorted.find(({ cid }) => childrenCids.includes(cid.toString()))
-  // const knownAncestor = await findCommonAncestor(events, [lowerEvent.cid]) // should this be [lowerEvent.cid] ?
-  // const knownAncestor = await findCommonAncestor(events, [...children]) // should this be [lowerEvent.cid] ?
-  // console.x('already knownAncestor', knownAncestor.toString() === ancestor.toString(),
-  //   (await (await decodeEventBlock((await blocks.get(knownAncestor)).bytes)).value.data?.value), knownAncestor
-  // )
-
-  const matchHead = [ancestor]
-  const unknownSorted = await asyncFilter(sorted, async (uks) => {
-    for (const ev of matchHead) {
-      const isIn = await contains(events, ev, uks.cid)
-      if (isIn) return false
-    }
-    return true
-  })
-  // console.x('unknownSorted contains', unknownSorted.length, sorted.length)
-  return unknownSorted
-}
-
-const asyncFilter = async (arr, predicate) => Promise.all(arr.map(predicate))
-  .then((results) => arr.filter((_v, index) => results[index]))
-
-export async function findCommonAncestorWithSortedEvents (blocks, children) {
-  const events = new EventFetcher(blocks)
-
-  const ancestor = await findCommonAncestor(events, children)
-  if (!ancestor) {
-    throw new Error('failed to find common ancestor event')
-  }
-  // Sort the events by their sequence number
-  const sorted = await findSortedEvents(events, children, ancestor)
-  // console.x('ancstor', ancestor, (await decodeEventBlock((await blocks.get(ancestor)).bytes)).value.data?.value)
-  // sorted.forEach(({ cid, value }) => console.x('xsorted', cid, value.data.value))
-  return { ancestor, sorted }
-}
-
-/**
- * Find the common ancestor event of the passed children. A common ancestor is
- * the first single event in the DAG that _all_ paths from children lead to.
- *
- * @param {import('./clock').EventFetcher} events
- * @param  {import('./clock').EventLink<EventData>[]} children
- */
-async function findCommonAncestor (events, children) {
-  if (!children.length) return
-  const candidates = children.map((c) => [c])
-  while (true) {
-    let changed = false
-    for (const c of candidates) {
-      const candidate = await findAncestorCandidate(events, c[c.length - 1])
-      if (!candidate) continue
-      changed = true
-      c.push(candidate)
-      const ancestor = findCommonString(candidates)
-      if (ancestor) return ancestor
-    }
-    if (!changed) return
-  }
-}
-
-/**
- * @param {import('./clock').EventFetcher} events
- * @param {import('./clock').EventLink<EventData>} root
- */
-async function findAncestorCandidate (events, root) {
-  const { value: event } = await events.get(root)
-  if (!event.parents.length) return root
-  return event.parents.length === 1
-    ? event.parents[0]
-    : findCommonAncestor(events, event.parents)
-}
-
-/**
- * @template {{ toString: () => string }} T
- * @param  {Array<T[]>} arrays
- */
-function findCommonString (arrays) {
-  arrays = arrays.map((a) => [...a])
-  for (const arr of arrays) {
-    for (const item of arr) {
-      let matched = true
-      for (const other of arrays) {
-        if (arr === other) continue
-        matched = other.some((i) => String(i) === String(item))
-        if (!matched) break
-      }
-      if (matched) return item
-    }
-  }
-}
-
-/**
- * Find and sort events between the head(s) and the tail.
- * @param {import('./clock').EventFetcher} events
- * @param {import('./clock').EventLink<EventData>[]} head
- * @param {import('./clock').EventLink<EventData>} tail
- */
-async function findSortedEvents (events, head, tail) {
-  // get weighted events - heavier events happened first
-  /** @type {Map<string, { event: import('./clock').EventBlockView<EventData>, weight: number }>} */
-  const weights = new Map()
-  const all = await Promise.all(head.map((h) => findEvents(events, h, tail)))
-  for (const arr of all) {
-    for (const { event, depth } of arr) {
-      const info = weights.get(event.cid.toString())
-      if (info) {
-        info.weight += depth
-      } else {
-        weights.set(event.cid.toString(), { event, weight: depth })
-      }
-    }
-  }
-
-  // group events into buckets by weight
-  /** @type {Map<number, import('./clock').EventBlockView<EventData>[]>} */
-  const buckets = new Map()
-  for (const { event, weight } of weights.values()) {
-    const bucket = buckets.get(weight)
-    if (bucket) {
-      bucket.push(event)
-    } else {
-      buckets.set(weight, [event])
-    }
-  }
-
-  // sort by weight, and by CID within weight
-  const sorted = Array.from(buckets)
-    .sort((a, b) => b[0] - a[0])
-    .flatMap(([, es]) =>
-      es.sort((a, b) => (String(a.cid) < String(b.cid) ? -1 : 1))
-    )
-  // console.log('sorted', sorted.map(s => s.value.data.value))
-  return sorted
-}
-
-/**
- * @param {import('./clock').EventFetcher} events
- * @param {import('./clock').EventLink<EventData>} start
- * @param {import('./clock').EventLink<EventData>} end
- * @returns {Promise<Array<{ event: import('./clock').EventBlockView<EventData>, depth: number }>>}
- */
-async function findEvents (events, start, end, depth = 0) {
-  const event = await events.get(start)
-  const acc = [{ event, depth }]
-  const { parents } = event.value
-  if (parents.length === 1 && String(parents[0]) === String(end)) return acc
-  const rest = await Promise.all(
-    parents.map((p) => findEvents(events, p, end, depth + 1))
-  )
-  return acc.concat(...rest)
-}