@fireproof/core 0.8.0 → 0.10.1-dev

package/src/storage/utils.js

@@ -1,152 +0,0 @@
-import * as CBW from '@ipld/car/buffer-writer'
-import * as raw from 'multiformats/codecs/raw'
-import { encrypt, decrypt } from '../crypto.js'
-import { parse } from 'multiformats/link'
-import { sha256 } from 'multiformats/hashes/sha2'
-import * as Block from 'multiformats/block'
-import { Buffer } from 'buffer'
-// @ts-ignore
-import { bf } from 'prolly-trees/utils'
-// @ts-ignore
-import { nocache as cache } from 'prolly-trees/cache'
-const chunker = bf(30)
-
-export async function getEmptyLoader () { // unused?
-  const theseWriteableBlocks = new VMemoryBlockstore()
-  return {
-    blocks: theseWriteableBlocks,
-    put: async (cid, bytes) => {
-      return await theseWriteableBlocks.put(cid, bytes)
-    },
-    get: async (cid) => {
-      const got = await theseWriteableBlocks.get(cid)
-      return got.bytes
-    }
-  }
-}
-
-export class VMemoryBlockstore {
-  /** @type {Map<string, Uint8Array>} */
-  blocks = new Map()
-  instanceId = Math.random().toString(36).slice(2)
-
-  async get (cid) {
-    const bytes = this.blocks.get(cid.toString())
-    if (!bytes) throw new Error('block not found ' + cid.toString())
-    return { cid, bytes }
-  }
-
-  /**
-   * @param {any} cid
-   * @param {Uint8Array} bytes
-   */
-  async put (cid, bytes) {
-    this.blocks.set(cid.toString(), bytes)
-  }
-
-  * entries () {
-    for (const [str, bytes] of this.blocks) {
-      yield { cid: parse(str), bytes }
-    }
-  }
-}
-
-export const blocksToCarBlock = async (rootCids, blocks) => {
-  // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
-  let size = 0
-  if (!Array.isArray(rootCids)) {
-    rootCids = [rootCids]
-  }
-  const headerSize = CBW.headerLength({ roots: rootCids })
-  size += headerSize
-  if (!Array.isArray(blocks)) {
-    blocks = Array.from(blocks.entries())
-  }
-  for (const { cid, bytes } of blocks) {
-    // console.log(cid, bytes)
-    size += CBW.blockLength({ cid, bytes })
-  }
-  const buffer = new Uint8Array(size)
-  const writer = await CBW.createWriter(buffer, { headerSize })
-
-  for (const cid of rootCids) {
-    writer.addRoot(cid)
-  }
-
-  for (const { cid, bytes } of blocks) {
-    writer.write({ cid, bytes })
-  }
-  await writer.close()
-  return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
-}
-
-export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
-  const encryptionKey = Buffer.from(keyMaterial, 'hex')
-  const encryptedBlocks = []
-  const theCids = cids
-  // console.trace('blocksToEncryptedCarBlock', blocks)
-  // for (const { cid } of blocks.entries()) {
-  //   theCids.push(cid.toString())
-  // }
-  // console.log(
-  //   'encrypting',
-  //   theCids.length,
-  //   'blocks',
-  //   theCids.includes(innerBlockStoreClockRootCid.toString()),
-  //   keyMaterial
-  // )
-  // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
-  let last
-  for await (const block of encrypt({
-    cids: theCids,
-    get: async (cid) => {
-      // console.log('getencrypt', cid)
-      const got = blocks.get(cid)
-      // console.log('got', got)
-      return got.block ? ({ cid, bytes: got.block }) : got
-    },
-    key: encryptionKey,
-    hasher: sha256,
-    chunker,
-    cache,
-    // codec: dagcbor, // should be crypto?
-    root: innerBlockStoreClockRootCid
-  })) {
-    encryptedBlocks.push(block)
-    last = block
-  }
-  // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
-  const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks)
-  return encryptedCar
-}
-
-// { root, get, key, cache, chunker, hasher }
-const memoizeDecryptedCarBlocks = new Map()
-export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
-  if (memoizeDecryptedCarBlocks.has(cid.toString())) {
-    return memoizeDecryptedCarBlocks.get(cid.toString())
-  } else {
-    const blocksPromise = (async () => {
-      const decryptionKey = Buffer.from(keyMaterial, 'hex')
-      // console.log('decrypting', keyMaterial, cid.toString())
-      const cids = new Set()
-      const decryptedBlocks = []
-      for await (const block of decrypt({
-        root: cid,
-        get,
-        key: decryptionKey,
-        chunker,
-        hasher: sha256,
-        cache
-        // codec: dagcbor
-      })) {
-        // console.log('decrypted', block.cid.toString())
-        decryptedBlocks.push(block)
-        cids.add(block.cid.toString())
-      }
-      return { blocks: decryptedBlocks, cids }
-    })()
-    memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise)
-    return blocksPromise
-  }
-}

package/src/sync.js

@@ -1,237 +0,0 @@
-import SimplePeer from 'simple-peer'
-import { parseCID } from './database.js'
-import { decodeEventBlock } from './clock.js'
-import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/utils.js'
-import { CarReader } from '@ipld/car'
-
-/**
- * @typedef {import('./database.js').Database} Database
- */
-export class Sync {
-  /**
-   * @param {Database} database
-   * @param {typeof SimplePeer} [PeerClass]
-   * @memberof Sync
-   * @static
-   */
-  status = 'new'
-  constructor (database, PeerClass = SimplePeer) {
-    this.database = database
-    this.database.blocks.syncs.add(this) // should this happen during setup?
-    this.PeerClass = PeerClass
-    this.pushBacklog = new Promise((resolve, reject) => {
-      this.pushBacklogResolve = resolve
-      this.pushBacklogReject = reject
-    })
-    this.isReady = false
-    // this.pushBacklog.then(() => {
-    //   // console.log('sync backlog resolved')
-    //   this.database.notifyReset()
-    // })
-    // this.connected = new Promise((resolve, reject) => {
-    //   this.readyResolve = resolve
-    //   this.readyReject = reject
-    // })
-  }
-
-  async offer () {
-    this.status = 'offering'
-    return this.setupPeer(true)
-  }
-
-  async accept (base64offer) {
-    const offer = JSON.parse(atob(base64offer))
-    const p = this.setupPeer(false)
-    this.peer.signal(offer)
-    this.status = 'accepting'
-    return p
-  }
-
-  connect (base64accept) {
-    const accept = JSON.parse(atob(base64accept))
-    this.status = 'connecting'
-    this.peer.signal(accept)
-  }
-
-  async setupPeer (initiator = false) {
-    this.peer = new this.PeerClass({
-      initiator,
-      trickle: false
-    })
-    this.peer.on('connect', () => this.startSync())
-    this.peer.on('data', data => this.gotData(data))
-    const p = new Promise((resolve, reject) => {
-      this.peer.on('signal', resolve)
-      this.peer.on('error', reject)
-    })
-    return p.then(signal => btoa(JSON.stringify(signal)))
-  }
-
-  async backlog () {
-    return this.pushBacklog
-  }
-
-  async gotData (data) {
-    // console.log('got data', data.toString())
-    let reader = null
-    try {
-      reader = await CarReader.fromBytes(data)
-    } catch (e) {
-      // console.log('not a car', data.toString())
-    }
-    if (reader) {
-      // console.log('got car')
-      this.status = 'parking car'
-      const blz = new Set()
-      for await (const block of reader.blocks()) {
-        blz.add(block)
-      }
-      const roots = await reader.getRoots()
-      // console.log(
-      //   'got car',
-      //   roots.map(c => c.toString()),
-      //   this.database.clock.map(c => c.toString())
-      // )
-      // console.log(
-      //   'got blocks!',
-      //   [...blz].map(({ cid }) => cid.toString())
-      // )
-      // @ts-ignore
-      reader.entries = reader.blocks
-      await this.database.blocks.commit(
-        {
-          label: 'sync',
-          entries: () => [...blz],
-          get: async cid => await reader.get(cid),
-          lastCid: [...blz][0].cid // doesn't matter
-        },
-        false
-      )
-      // first arg could be the roots parents?
-      // get the roots parents
-      const parents = await Promise.all(
-        roots.map(async cid => {
-          const rbl = await reader.get(cid)
-          if (!rbl) {
-            console.log('missing root block', cid.toString(), reader)
-            throw new Error('missing root block')
-          }
-          const block = await decodeEventBlock(rbl.bytes)
-          return block.value.parents
-        })
-      )
-      this.database.applyClock(parents.flat(), roots)
-      this.database.notifyReset()
-      // console.log('after', this.database.clockToJSON())
-      this.pushBacklogResolve({ ok: true })
-    } else {
-      // data is a json string, parse it
-      const message = JSON.parse(data.toString())
-      // console.log('got message', message)
-      if (message.ok) {
-        this.status = 'ok'
-        this.pushBacklogResolve({ ok: true })
-      } else if (message.clock) {
-        const reqCidDiff = message
-        // this might be a CID diff
-        // console.log('got diff', reqCidDiff)
-        const carBlock = await Sync.makeCar(this.database, null, reqCidDiff.cids)
-        if (!carBlock) {
-          // we are full synced
-          // console.log('we are full synced')
-          this.status = 'full synced'
-          this.peer.send(JSON.stringify({ ok: true }))
-          // this.pushBacklogResolve({ ok: true })
-        } else {
-          // console.log('do send diff', carBlock.bytes.length)
-          this.status = 'sending diff car'
-          this.peer.send(carBlock.bytes)
-          // console.log('sent diff car')
-          // this.pushBacklogResolve({ ok: true })
-        }
-      }
-    }
-  }
-
-  destroy () {
-    this.database.blocks.syncs.delete(this)
-    this.status = 'destroyed'
-    // this.peer.destroy()  todo
-  }
-
-  async sendUpdate (blockstore) {
-    if (!this.peer || !this.isReady) return
-    // console.log('send update from', this.database.instanceId)
-    // todo should send updates since last sync (currently sends each transaction)
-    const newCar = await blocksToCarBlock(blockstore.lastCid, blockstore)
-    this.status = 'sending update car'
-    this.peer.send(newCar.bytes)
-  }
-
-  async startSync () {
-    // console.log('start sync', this.peer.initiator)
-    this.isReady = true
-    const allCIDs = await this.database.allStoredCIDs()
-    // console.log('allCIDs', allCIDs)
-    const reqCidDiff = {
-      clock: this.database.clockToJSON(),
-      cids: allCIDs.map(cid => cid.toString())
-    }
-    // console.log('send diff', reqCidDiff)
-    this.status = 'sending cid diff'
-    this.peer.send(JSON.stringify(reqCidDiff))
-  }
-
-  // get all the cids
-  // tell valet to make a file
-  /**
-   * @param {import("./database.js").Database} database
-   * @param {string} key
-   */
-  static async makeCar (database, key, skip = []) {
-    const allCIDs = await database.allCIDs()
-    const blocks = database.blocks
-    const rootCIDs = database.clock
-
-    const newCIDs = [...new Set([...rootCIDs, ...allCIDs])].filter(cid => !skip.includes(cid.toString()))
-    const syncCIDs = [...new Set([...rootCIDs, ...allCIDs.filter(cid => !skip.includes(cid.toString()))])]
-    // console.log(
-    //   'makeCar',
-    //   rootCIDs.map(c => c.toString()),
-    //   syncCIDs.map(c => c.toString()),
-    //   allCIDs.map(c => c.toString())
-    // )
-    if (newCIDs.length === 0) {
-      return null
-    }
-
-    if (typeof key === 'undefined') {
-      key = blocks.valet?.primary.keyMaterial
-    }
-    if (key) {
-      return blocksToEncryptedCarBlock(
-        rootCIDs,
-        {
-          entries: () => syncCIDs.map(cid => ({ cid })),
-          get: async cid => await blocks.get(cid)
-        },
-        key,
-        syncCIDs.map(c => c.toString())
-      )
-    } else {
-      const carBlocks = await Promise.all(
-        syncCIDs.map(async c => {
-          const b = await blocks.get(c)
-          if (typeof b.cid === 'string') {
-            b.cid = parseCID(b.cid)
-          }
-          return b
-        })
-      )
-      // console.log('carblock')
-      return blocksToCarBlock(rootCIDs, {
-        entries: () => carBlocks
-      })
-    }
-  }
-}

package/src/valet.js

@@ -1,105 +0,0 @@
-
-import { Loader } from './loader.js'
-
-export class Valet {
-  idb = null
-  name = null
-  uploadQueue = null
-  alreadyEnqueued = new Set()
-
-  instanceId = Math.random().toString(36).slice(2)
-
-  constructor (name = 'default', config = {}) {
-    this.name = name
-    // console.log('new Valet', name, config.primary)
-    this.primary = Loader.appropriate(name, config.primary)
-    this.secondary = config.secondary ? Loader.appropriate(name, config.secondary) : null
-    // set up a promise listener that applies all the headers to the clock
-    // when they resolve
-    const readyP = [this.primary.ready]
-    if (this.secondary) readyP.push(this.secondary.ready)
-
-    this.ready = Promise.all(readyP).then((blocksReady) => {
-      // console.log('blocksReady valet', this.name, blocksReady)
-      return blocksReady
-    })
-  }
-
-  async saveHeader (header) {
-    // each storage needs to add its own carCidMapCarCid to the header
-    if (this.secondary) { this.secondary.saveHeader(header) } // todo: await?
-    return await this.primary.saveHeader(header)
-  }
-
-  async compact (clock) {
-    await this.primary.compact(clock)
-    if (this.secondary) await this.secondary.compact(clock)
-  }
-
-  /**
-   * Group the blocks into a car and write it to the valet.
-   * @param {import('./blockstore.js').InnerBlockstore} innerBlockstore
-   * @param {Set<string>} cids
-   * @returns {Promise<void>}
-   * @memberof Valet
-   */
-  async writeTransaction (innerBlockstore, cids) {
-    if (innerBlockstore.lastCid) {
-      await this.primary.parkCar(innerBlockstore, cids)
-      if (this.secondary) await this.secondary.parkCar(innerBlockstore, cids)
-    } else {
-      throw new Error('missing lastCid for car header')
-    }
-  }
-
-  /**
-   * Iterate over all blocks in the store.
-   *
-   * @yields {{cid: string, value: Uint8Array}}
-   * @returns {AsyncGenerator<any, any, any>}
-   */
-  async * cids () {
-    // console.log('valet cids')
-    // todo use cidMap
-    // while (cursor) {
-    // yield { cid: cursor.key, car: cursor.value.car }
-    // cursor = await cursor.continue()
-    // }
-  }
-
-  remoteBlockFunction = null
-
-  async getValetBlock (dataCID) {
-    // console.log('getValetBlock primary', dataCID)
-    try {
-      const { block } = await this.primary.getLoaderBlock(dataCID)
-      return block
-    } catch (e) {
-      // console.log('getValetBlock error', e)
-      if (this.secondary) {
-        // console.log('getValetBlock secondary', dataCID)
-        try {
-          // eslint-disable-next-line
-          const { block, reader } = await this.secondary.getLoaderBlock(dataCID)
-          const writeableCarReader = await this.primary.getWriteableCarReader(reader)
-          // console.log('getValetBlock secondary', dataCID, block.length)
-          // eslint-disable-next-line
-          const cids = new Set()
-          for await (const { cid } of reader.entries()) {
-            // console.log(cid, bytes)
-            cids.add(cid.toString())
-          }
-          // reader.get = reader.gat // some consumers prefer get
-          // console.log('replicating', reader.root)
-          writeableCarReader.lastCid = reader.root.cid
-          writeableCarReader.head = []
-          await this.primary.parkCar(writeableCarReader, cids).catch(e => console.error('parkCar error', e))
-          // console.log('FIX THIS', did)
-          return block
-        } catch (e) {
-          // console.log('getValetBlock secondary error', e)
-        }
-      }
-    }
-  }
-}