@fireproof/core 0.0.1

package/src/clock.js

@@ -0,0 +1,352 @@
+import { Block, encode, decode } from 'multiformats/block'
+import { sha256 } from 'multiformats/hashes/sha2'
+import * as cbor from '@ipld/dag-cbor'
+
+/**
+ * @template T
+ * @typedef {{ parents: EventLink<T>[], data: T }} EventView
+ */
+
+/**
+ * @template T
+ * @typedef {import('multiformats').BlockView<EventView<T>>} EventBlockView
+ */
+
+/**
+ * @template T
+ * @typedef {import('multiformats').Link<EventView<T>>} EventLink
+ */
+
+/**
+ * Advance the clock by adding an event.
+ *
+ * @template T
+ * @param {import('./block').BlockFetcher} blocks Block storage.
+ * @param {EventLink<T>[]} head The head of the clock.
+ * @param {EventLink<T>} event The event to add.
+ * @returns {Promise<EventLink<T>[]>} The new head of the clock.
+ */
+export async function advance (blocks, head, event) {
+  /** @type {EventFetcher<T>} */
+  const events = new EventFetcher(blocks)
+  const headmap = new Map(head.map(cid => [cid.toString(), cid]))
+
+  // Check if the headmap already includes the event, return head if it does
+  if (headmap.has(event.toString())) return head
+
+  // Does event contain the clock?
+  let changed = false
+  for (const cid of head) {
+    if (await contains(events, event, cid)) {
+      headmap.delete(cid.toString())
+      headmap.set(event.toString(), event)
+      changed = true
+    }
+  }
+
+  // If the headmap has been changed, return the new headmap values
+  if (changed) {
+    return [...headmap.values()]
+  }
+
+  // Does clock contain the event?
+  for (const p of head) {
+    if (await contains(events, p, event)) {
+      return head
+    }
+  }
+
+  // Return the head concatenated with the new event if it passes both checks
+  return head.concat(event)
+}
+
+/**
+ * @template T
+ * @implements {EventBlockView<T>}
+ */
+export class EventBlock extends Block {
+  /**
+   * @param {object} config
+   * @param {EventLink<T>} config.cid
+   * @param {Event} config.value
+   * @param {Uint8Array} config.bytes
+   */
+  constructor ({ cid, value, bytes }) {
+    // @ts-expect-error
+    super({ cid, value, bytes })
+  }
+
+  /**
+   * @template T
+   * @param {T} data
+   * @param {EventLink<T>[]} [parents]
+   */
+  static create (data, parents) {
+    return encodeEventBlock({ data, parents: parents ?? [] })
+  }
+}
+
+/** @template T */
+export class EventFetcher {
+  /** @param {import('./block').BlockFetcher} blocks */
+  constructor (blocks) {
+    /** @private */
+    this._blocks = blocks
+  }
+
+  /**
+   * @param {EventLink<T>} link
+   * @returns {Promise<EventBlockView<T>>}
+   */
+  async get (link) {
+    const block = await this._blocks.get(link)
+    if (!block) throw new Error(`missing block: ${link}`)
+    return decodeEventBlock(block.bytes)
+  }
+}
+
+/**
+ * @template T
+ * @param {EventView<T>} value
+ * @returns {Promise<EventBlockView<T>>}
+ */
+export async function encodeEventBlock (value) {
+  // TODO: sort parents
+  const { cid, bytes } = await encode({ value, codec: cbor, hasher: sha256 })
+  // @ts-expect-error
+  return new Block({ cid, value, bytes })
+}
+
+/**
+ * @template T
+ * @param {Uint8Array} bytes
+ * @returns {Promise<EventBlockView<T>>}
+ */
+export async function decodeEventBlock (bytes) {
+  const { cid, value } = await decode({ bytes, codec: cbor, hasher: sha256 })
+  // @ts-expect-error
+  return new Block({ cid, value, bytes })
+}
+
+/**
+ * Returns true if event "a" contains event "b". Breadth first search.
+ * @template T
+ * @param {EventFetcher} events
+ * @param {EventLink<T>} a
+ * @param {EventLink<T>} b
+ */
+async function contains (events, a, b) {
+  if (a.toString() === b.toString()) return true
+  const [{ value: aevent }, { value: bevent }] = await Promise.all([events.get(a), events.get(b)])
+  const links = [...aevent.parents]
+  while (links.length) {
+    const link = links.shift()
+    if (!link) break
+    if (link.toString() === b.toString()) return true
+    // if any of b's parents are this link, then b cannot exist in any of the
+    // tree below, since that would create a cycle.
+    if (bevent.parents.some(p => link.toString() === p.toString())) continue
+    const { value: event } = await events.get(link)
+    links.push(...event.parents)
+  }
+  return false
+}
+
+/**
+ * @template T
+ * @param {import('./block').BlockFetcher} blocks Block storage.
+ * @param {EventLink<T>[]} head
+ * @param {object} [options]
+ * @param {(b: EventBlockView<T>) => string} [options.renderNodeLabel]
+ */
+export async function * vis (blocks, head, options = {}) {
+  const renderNodeLabel = options.renderNodeLabel ?? (b => (b.value.data.value))
+  const events = new EventFetcher(blocks)
+  yield 'digraph clock {'
+  yield '  node [shape=point fontname="Courier"]; head;'
+  const hevents = await Promise.all(head.map(link => events.get(link)))
+  const links = []
+  const nodes = new Set()
+  for (const e of hevents) {
+    nodes.add(e.cid.toString())
+    yield `  node [shape=oval fontname="Courier"]; ${e.cid} [label="${renderNodeLabel(e)}"];`
+    yield `  head -> ${e.cid};`
+    for (const p of e.value.parents) {
+      yield `  ${e.cid} -> ${p};`
+    }
+    links.push(...e.value.parents)
+  }
+  while (links.length) {
+    const link = links.shift()
+    if (!link) break
+    if (nodes.has(link.toString())) continue
+    nodes.add(link.toString())
+    const block = await events.get(link)
+    yield `  node [shape=oval]; ${link} [label="${renderNodeLabel(block)}" fontname="Courier"];`
+    for (const p of block.value.parents) {
+      yield `  ${link} -> ${p};`
+    }
+    links.push(...block.value.parents)
+  }
+  yield '}'
+}
+
+export async function findEventsToSync (blocks, head) {
+  const toSync = await findUnknownSortedEvents(blocks, head, await findCommonAncestorWithSortedEvents(blocks, head))
+  return toSync
+}
+
+export async function findUnknownSortedEvents (blocks, children, { ancestor, sorted }) {
+  const events = new EventFetcher(blocks)
+  // const childrenCids = children.map(c => c.toString())
+  // const lowerEvent = sorted.find(({ cid }) => childrenCids.includes(cid.toString()))
+  // const knownAncestor = await findCommonAncestor(events, [lowerEvent.cid]) // should this be [lowerEvent.cid] ?
+  // const knownAncestor = await findCommonAncestor(events, [...children]) // should this be [lowerEvent.cid] ?
+  // console.x('already knownAncestor', knownAncestor.toString() === ancestor.toString(),
+  //   (await (await decodeEventBlock((await blocks.get(knownAncestor)).bytes)).value.data?.value), knownAncestor
+  // )
+
+  const matchHead = [ancestor]
+  const unknownSorted = await asyncFilter(sorted, async (uks) => {
+    for (const ev of matchHead) {
+      const isIn = await contains(events, ev, uks.cid)
+      if (isIn) return false
+    }
+    return true
+  })
+  // console.x('unknownSorted contains', unknownSorted.length, sorted.length)
+  return unknownSorted
+}
+
+const asyncFilter = async (arr, predicate) => Promise.all(arr.map(predicate))
+  .then((results) => arr.filter((_v, index) => results[index]))
+
+export async function findCommonAncestorWithSortedEvents (blocks, children) {
+  const events = new EventFetcher(blocks)
+
+  const ancestor = await findCommonAncestor(events, children)
+  if (!ancestor) {
+    throw new Error('failed to find common ancestor event')
+  }
+  // Sort the events by their sequence number
+  const sorted = await findSortedEvents(events, children, ancestor)
+  // console.x('ancstor', ancestor, (await decodeEventBlock((await blocks.get(ancestor)).bytes)).value.data?.value)
+  // sorted.forEach(({ cid, value }) => console.x('xsorted', cid, value.data.value))
+  return { ancestor, sorted }
+}
+
+/**
+ * Find the common ancestor event of the passed children. A common ancestor is
+ * the first single event in the DAG that _all_ paths from children lead to.
+ *
+ * @param {import('./clock').EventFetcher} events
+ * @param  {import('./clock').EventLink<EventData>[]} children
+ */
+async function findCommonAncestor (events, children) {
+  if (!children.length) return
+  const candidates = children.map((c) => [c])
+  while (true) {
+    let changed = false
+    for (const c of candidates) {
+      const candidate = await findAncestorCandidate(events, c[c.length - 1])
+      if (!candidate) continue
+      changed = true
+      c.push(candidate)
+      const ancestor = findCommonString(candidates)
+      if (ancestor) return ancestor
+    }
+    if (!changed) return
+  }
+}
+
+/**
+ * @param {import('./clock').EventFetcher} events
+ * @param {import('./clock').EventLink<EventData>} root
+ */
+async function findAncestorCandidate (events, root) {
+  const { value: event } = await events.get(root)
+  if (!event.parents.length) return root
+  return event.parents.length === 1
+    ? event.parents[0]
+    : findCommonAncestor(events, event.parents)
+}
+
+/**
+ * @template {{ toString: () => string }} T
+ * @param  {Array<T[]>} arrays
+ */
+function findCommonString (arrays) {
+  arrays = arrays.map((a) => [...a])
+  for (const arr of arrays) {
+    for (const item of arr) {
+      let matched = true
+      for (const other of arrays) {
+        if (arr === other) continue
+        matched = other.some((i) => String(i) === String(item))
+        if (!matched) break
+      }
+      if (matched) return item
+    }
+  }
+}
+
+/**
+ * Find and sort events between the head(s) and the tail.
+ * @param {import('./clock').EventFetcher} events
+ * @param {import('./clock').EventLink<EventData>[]} head
+ * @param {import('./clock').EventLink<EventData>} tail
+ */
+async function findSortedEvents (events, head, tail) {
+  // get weighted events - heavier events happened first
+  /** @type {Map<string, { event: import('./clock').EventBlockView<EventData>, weight: number }>} */
+  const weights = new Map()
+  const all = await Promise.all(head.map((h) => findEvents(events, h, tail)))
+  for (const arr of all) {
+    for (const { event, depth } of arr) {
+      const info = weights.get(event.cid.toString())
+      if (info) {
+        info.weight += depth
+      } else {
+        weights.set(event.cid.toString(), { event, weight: depth })
+      }
+    }
+  }
+
+  // group events into buckets by weight
+  /** @type {Map<number, import('./clock').EventBlockView<EventData>[]>} */
+  const buckets = new Map()
+  for (const { event, weight } of weights.values()) {
+    const bucket = buckets.get(weight)
+    if (bucket) {
+      bucket.push(event)
+    } else {
+      buckets.set(weight, [event])
+    }
+  }
+
+  // sort by weight, and by CID within weight
+  const sorted = Array.from(buckets)
+    .sort((a, b) => b[0] - a[0])
+    .flatMap(([, es]) =>
+      es.sort((a, b) => (String(a.cid) < String(b.cid) ? -1 : 1))
+    )
+  // console.log('sorted', sorted.map(s => s.value.data.value))
+  return sorted
+}
+
+/**
+ * @param {import('./clock').EventFetcher} events
+ * @param {import('./clock').EventLink<EventData>} start
+ * @param {import('./clock').EventLink<EventData>} end
+ * @returns {Promise<Array<{ event: import('./clock').EventBlockView<EventData>, depth: number }>>}
+ */
+async function findEvents (events, start, end, depth = 0) {
+  const event = await events.get(start)
+  const acc = [{ event, depth }]
+  const { parents } = event.value
+  if (parents.length === 1 && String(parents[0]) === String(end)) return acc
+  const rest = await Promise.all(
+    parents.map((p) => findEvents(events, p, end, depth + 1))
+  )
+  return acc.concat(...rest)
+}

package/src/db-index.js

@@ -0,0 +1,196 @@
+import { create, load } from 'prolly-trees/db-index'
+import { sha256 as hasher } from 'multiformats/hashes/sha2'
+import { nocache as cache } from 'prolly-trees/cache'
+import { bf, simpleCompare as compare } from 'prolly-trees/utils'
+import * as codec from '@ipld/dag-cbor'
+import { create as createBlock } from 'multiformats/block'
+import { doTransaction } from './blockstore.js'
+import charwise from 'charwise'
+const opts = { cache, chunker: bf(3), codec, hasher, compare }
+
+const ALWAYS_REBUILD = true // todo: remove this
+
+const makeGetBlock = (blocks) => async (address) => {
+  const { cid, bytes } = await blocks.get(address)
+  return createBlock({ cid, bytes, hasher, codec })
+}
+const makeDoc = ({ key, value }) => ({ _id: key, ...value })
+
+console.x = function () {}
+
+/**
+ * Transforms a set of changes to index entries using a map function.
+ *
+ * @param {Array<{ key: string, value: import('./link').AnyLink, del?: boolean }>} changes
+ * @param {Function} mapFun
+ * @returns {Array<{ key: [string, string], value: any }>} The index entries generated by the map function.
+ */
+
+const indexEntriesForChanges = (changes, mapFun) => {
+  const indexEntries = []
+  changes.forEach(({ key, value, del }) => {
+    if (del || !value) return
+    mapFun(makeDoc({ key, value }), (k, v) => {
+      indexEntries.push({
+        key: [charwise.encode(k), key],
+        value: v
+      })
+    })
+  })
+  return indexEntries
+}
+
+const indexEntriesForOldChanges = async (blocks, byIDindexRoot, ids, mapFun) => {
+  const getBlock = makeGetBlock(blocks)
+  const byIDindex = await load({ cid: byIDindexRoot.cid, get: getBlock, ...opts })
+  // console.trace('ids', ids)
+  const result = await byIDindex.getMany(ids)
+  return result.result
+}
+
+/**
+ * Represents an index for a Fireproof database.
+ *
+ * @class
+ * @classdesc An index can be used to order and filter the documents in a Fireproof database.
+ *
+ * @param {import('./fireproof').Fireproof} database - The Fireproof database instance to index.
+ * @param {Function} mapFun - The map function to apply to each entry in the database.
+ *
+ */
+export default class Index {
+  /**
+   * Creates a new index with the given map function and database.
+   * @param {import('./fireproof').Fireproof} database - The Fireproof database instance to index.
+   * @param {Function} mapFun - The map function to apply to each entry in the database.
+   */
+  constructor (database, mapFun) {
+    /**
+     * The database instance to index.
+     * @type {import('./fireproof').Fireproof}
+     */
+    this.database = database
+    /**
+     * The map function to apply to each entry in the database.
+     * @type {Function}
+     */
+    this.mapFun = mapFun
+    this.indexRoot = null
+    this.byIDindexRoot = null
+    this.dbHead = null
+  }
+
+  /**
+   * Query object can have {range}
+   *
+   */
+  async query (query, root = null) {
+    if (!root) { // pass a root to query a snapshot
+      await doTransaction('#updateIndex', this.database.blocks, async (blocks) => {
+        await this.#updateIndex(blocks)
+      })
+    }
+    const response = await doIndexQuery(this.database.blocks, root || this.indexRoot, query)
+    return {
+      // TODO fix this naming upstream in prolly/db-index
+      // todo maybe this is a hint about why deletes arent working?
+      rows: response.result.map(({ id, key, row }) => ({ id: key, key: charwise.decode(id), value: row }))
+    }
+  }
+
+  /**
+   * Update the index with the latest changes
+   * @private
+   * @returns {Promise<void>}
+   */
+  async #updateIndex (blocks) {
+    // todo remove this hack
+    if (ALWAYS_REBUILD) {
+      this.dbHead = null // hack
+      this.indexRoot = null // hack
+    }
+    const result = await this.database.changesSince(this.dbHead) // {key, value, del}
+    if (this.dbHead) {
+      const oldIndexEntries = (await indexEntriesForOldChanges(blocks, this.byIDindexRoot, result.rows.map(({ key }) => key), this.mapFun))
+        // .map((key) => ({ key, value: null })) // tombstone just adds more rows...
+        .map((key) => ({ key, del: true })) // should be this
+        // .map((key) => ({ key: undefined, del: true })) // todo why does this work?
+
+      this.indexRoot = await bulkIndex(blocks, this.indexRoot, oldIndexEntries, opts)
+      // console.x('oldIndexEntries', oldIndexEntries)
+      // [ { key: ['b', 1], del: true } ]
+      // [ { key: [ 5, 'x' ], del: true } ]
+      // for now we just let the by id index grow and then don't use the results...
+      // const removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }))
+      // this.byIDindexRoot = await bulkIndex(blocks, this.byIDindexRoot, removeByIdIndexEntries, opts)
+    }
+    const indexEntries = indexEntriesForChanges(result.rows, this.mapFun)
+    const byIdIndexEntries = indexEntries.map(({ key }) => ({ key: key[1], value: key }))
+    // [{key:  'xxxx-3c3a-4b5e-9c1c-8c5c0c5c0c5c', value : [ 53, 'xxxx-3c3a-4b5e-9c1c-8c5c0c5c0c5c' ]}]
+    this.byIDindexRoot = await bulkIndex(blocks, this.byIDindexRoot, byIdIndexEntries, opts)
+    // console.log('indexEntries', indexEntries)
+    this.indexRoot = await bulkIndex(blocks, this.indexRoot, indexEntries, opts)
+    // console.log('did index', this.indexRoot)
+    this.dbHead = result.clock
+  }
+
+  // todo use the index from other peers?
+  // we might need to add CRDT logic to it for that
+  // it would only be a performance improvement, but might add a lot of complexity
+  //   advanceIndex ()) {}
+}
+
+/**
+ * Update the index with the given entries
+ * @param {Blockstore} blocks
+ * @param {import('multiformats/block').Block} inRoot
+ * @param {import('prolly-trees/db-index').IndexEntry[]} indexEntries
+ */
+async function bulkIndex (blocks, inRoot, indexEntries) {
+  if (!indexEntries.length) return inRoot
+  const putBlock = blocks.put.bind(blocks)
+  const getBlock = makeGetBlock(blocks)
+  if (!inRoot) {
+    // make a new index
+
+    for await (const node of await create({ get: getBlock, list: indexEntries, ...opts })) {
+      const block = await node.block
+      await putBlock(block.cid, block.bytes)
+      inRoot = block
+    }
+    // console.x('created index', inRoot.cid)
+    return inRoot
+  } else {
+    // load existing index
+    // console.x('loading index', inRoot.cid)
+    const index = await load({ cid: inRoot.cid, get: getBlock, ...opts })
+    // console.log('new indexEntries', indexEntries)
+    const { root, blocks } = await index.bulk(indexEntries)
+    for await (const block of blocks) {
+      await putBlock(block.cid, block.bytes)
+    }
+    // console.x('updated index', root.block.cid)
+    return await root.block // if we hold the root we won't have to load every time
+  }
+}
+
+/**
+ * Query the index for the given range
+ * @param {Blockstore} blocks
+ * @param {import('multiformats/block').Block} inRoot
+ * @param {import('prolly-trees/db-index').Query} query
+ * @returns {Promise<import('prolly-trees/db-index').QueryResult>}
+ **/
+async function doIndexQuery (blocks, root, query) {
+  const cid = root && root.cid
+  if (!cid) return { result: [] }
+  const getBlock = makeGetBlock(blocks)
+  const index = await load({ cid, get: getBlock, ...opts })
+  if (query.range) {
+    const encodedRange = query.range.map((key) => charwise.encode(key))
+    return index.range(...encodedRange)
+  } else if (query.key) {
+    const encodedKey = charwise.encode(query.key)
+    return index.get(encodedKey)
+  }
+}