@fireproof/core 0.8.0 → 0.10.1-dev

package/src/prolly.js

@@ -1,445 +0,0 @@
-import {
-  advance,
-  EventFetcher,
-  EventBlock,
-  findCommonAncestorWithSortedEvents,
-  findEventsToSync,
-  vis as visClock
-} from './clock.js'
-// import { create, load } from '../../../../prolly-trees/src/map.js'
-// @ts-ignore
-import { create, load } from 'prolly-trees/map'
-// @ts-ignore
-import { nocache as cache } from 'prolly-trees/cache'
-// @ts-ignore
-import { CIDCounter, bf, simpleCompare as compare } from 'prolly-trees/utils'
-import * as codec from '@ipld/dag-cbor'
-import { sha256 as hasher } from 'multiformats/hashes/sha2'
-// import { blake2b256 as hasher } from '@multiformats/blake2/blake2b'
-
-import { doTransaction } from './blockstore.js'
-import { create as createBlock } from 'multiformats/block'
-const blockOpts = { cache, chunker: bf(30), codec, hasher, compare }
-
-// const SYNC_ROOT = 'fireproof' // change this if you want to break sync
-
-/**
- * @typedef {import('./blockstore.js').TransactionBlockstore} TransactionBlockstore
- */
-
-// const withLog = async (label, fn) => {
-//   const resp = await fn()
-//   // console.log('withLog', label, !!resp)
-//   return resp
-// }
-
-// should also return a CIDCounter
-export const makeGetBlock = blocks => {
-  // const cids = new CIDCounter() // this could be used for proofs of mutations
-  const getBlockFn = async address => {
-    // const { cid, bytes } = await withLog(address, () => blocks.get(address))
-    const { cid, bytes } = await blocks.get(address)
-    // cids.add({ address: cid })
-    return createBlock({ cid, bytes, hasher, codec })
-  }
-  return {
-    // cids,
-    getBlock: getBlockFn
-  }
-}
-
-/**
- *
- * @param {*} param0
- * @returns
- */
-async function createAndSaveNewEvent ({ inBlocks, bigPut, root, event: inEvent, head, additions, removals = [] }) {
-  let cids
-  const { key, value, del } = inEvent
-  // console.log('createAndSaveNewEvent', root.constructor.name, root.entryList)
-  // root = await root.block
-  const data = {
-    root: root
-      ? (await root.address)
-      : null,
-    key
-  }
-  // import('./clock').EventLink<import('./clock').EventData>
-  if (del) {
-    data.value = null
-    data.type = 'del'
-  } else {
-    data.value = value
-    data.type = 'put'
-  }
-  // console.log('head', head)
-  // if (head.length === 0) {
-  //   // create an empty prolly root
-  //   let emptyRoot
-
-  //   for await (const node of create({ get: getBlock, list: [{ key: '_sync', value: SYNC_ROOT }], ...blockOpts })) {
-  //     emptyRoot = await node.block
-  //     bigPut(emptyRoot)
-  //   }
-  //   console.log('emptyRoot', emptyRoot)
-  //   const first = await EventBlock.create(
-  //     {
-  //       root: emptyRoot.cid,
-  //       key: null,
-  //       value: null,
-  //       type: 'del'
-  //     },
-  //     []
-  //   )
-  //   bigPut(first)
-  //   head = [first.cid]
-  // }
-
-  /** @type {import('./clock').EventData} */
-  // @ts-ignore
-  const event = await EventBlock.create(data, head)
-  bigPut(event)
-  // @ts-ignore
-  ;({ head, cids } = await advance(inBlocks, head, event.cid))
-
-  return {
-    root,
-    additions,
-    removals,
-    head,
-    clockCIDs: cids,
-    event
-  }
-}
-
-const makeGetAndPutBlock = inBlocks => {
-  // const mblocks = new MemoryBlockstore()
-  // const blocks = new MultiBlockFetcher(mblocks, inBlocks)
-  const { getBlock, cids } = makeGetBlock(inBlocks)
-  // const put = inBlocks.put.bind(inBlocks)
-  const bigPut = async (block, additions) => {
-    // console.log('bigPut', block.cid.toString())
-    const { cid, bytes } = block
-    inBlocks.put(cid, bytes)
-    // mblocks.putSync(cid, bytes)
-    if (additions) {
-      additions.set(cid.toString(), block)
-    }
-  }
-  return { getBlock, bigPut, blocks: inBlocks, cids }
-}
-
-const bulkFromEvents = (sorted, event) => {
-  if (event) {
-    const update = { value: { data: { key: event.key } } }
-    if (event.del) {
-      update.value.data.type = 'del'
-    } else {
-      update.value.data.type = 'put'
-      update.value.data.value = event.value
-    }
-    sorted.push(update)
-  }
-  const bulk = new Map()
-  for (const { value: event } of sorted) {
-    const {
-      data: { type, value, key }
-    } = event
-    if (!key) {
-      throw new Error('key is required')
-    }
-    const bulkEvent = type === 'put' ? { key, value } : { key, del: true }
-    bulk.set(bulkEvent.key, bulkEvent) // last wins
-  }
-  return Array.from(bulk.values())
-}
-
-// Get the value of the root from the ancestor event
-/**
- *
- * @param {EventFetcher} events
- * @param {import('./clock').EventLink<import('./clock').EventData>} ancestor
- * @param {*} getBlock
- * @returns
- */
-const prollyRootFromAncestor = async (events, ancestor, getBlock) => {
-  // console.log('prollyRootFromAncestor', ancestor)
-  const event = await events.get(ancestor)
-  const { root } = event.value.data
-  if (root) {
-    return load({ cid: root, get: getBlock, ...blockOpts })
-  } else {
-    // console.log('no root', root) // false means no common ancestor. null means empty database.
-    return root
-  }
-}
-
-const doProllyBulk = async (inBlocks, head, event, doFull = false) => {
-  const { getBlock, blocks } = makeGetAndPutBlock(inBlocks) // this is doubled with eventfetcher
-  let bulkSorted = []
-  let prollyRootNode = null
-  const events = new EventFetcher(blocks)
-  if (head.length) {
-    if (!doFull && head.length === 1) {
-      prollyRootNode = await prollyRootFromAncestor(events, head[0], getBlock)
-    } else {
-      // Otherwise, we find the common ancestor and update the root and other blocks
-      // todo this is returning more events than necessary, lets define the desired semantics from the top down
-      // good semantics mean we can cache the results of this call
-      // const {cids, events : bulkSorted } = await findEventsToSync(blocks, head)
-      const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head, doFull)
-
-      bulkSorted = sorted
-      // console.log('sorted', !!ancestor, JSON.stringify(sorted.map(({ value: { data: { key, value } } }) => ({ key, value }))))
-      if (ancestor) {
-        prollyRootNode = await prollyRootFromAncestor(events, ancestor, getBlock)
-        // if (!prollyRootNode) {
-        //   prollyRootNode = await bigMerge(events, head, getBlock)
-        //   // throw new Error('no common ancestor')
-        // }
-      }
-      // console.log('event', event)
-    }
-  }
-
-  const bulkOperations = bulkFromEvents(bulkSorted, event)
-
-  // if prolly root node is null, we need to create a new one
-  if (!prollyRootNode) {
-    // console.log('make new root', bulkOperations.length)
-    let root
-    // let rootNode
-    const newBlocks = []
-    // if all operations are deletes, we can just return an empty root
-    if (bulkOperations.every(op => op.del)) {
-      return { root: null, blocks: [], clockCIDs: await events.all() }
-    }
-    for await (const node of create({ get: getBlock, list: bulkOperations, ...blockOpts })) {
-      // root = await node.block
-      root = node
-      newBlocks.push(await node.block)
-    }
-    // throw new Error('not root time')
-    // root.isThisOne = 'yes'
-    // console.log('made new root', root.constructor.name, root.block.cid.toString())
-
-    return { root, blocks: newBlocks, clockCIDs: await events.all() }
-  } else {
-    const writeResp = await prollyRootNode.bulk(bulkOperations) // { root: newProllyRootNode, blocks: newBlocks }
-    writeResp.clockCIDs = await events.all()
-    return writeResp
-  }
-}
-
-/**
- * Put a value (a CID) for the given key. If the key exists it's value is overwritten.
- *
- * @param {import('./blockstore.js').Blockstore} inBlocks Bucket block storage.
- * @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
- * @param {{key: string, value: import('./clock').EventLink<import('./clock').EventData>}} event The key of the value to put.
- * @param {object} [options]
- * @returns {Promise<any>}
- */
-export async function put (inBlocks, head, event, options) {
-  const { bigPut } = makeGetAndPutBlock(inBlocks)
-  // console.log('major put')
-  // If the head is empty, we create a new event and return the root and addition blocks
-  if (!head.length) {
-    const additions = new Map()
-    const { root, blocks } = await doProllyBulk(inBlocks, head, event)
-    for (const b of blocks) {
-      bigPut(b, additions)
-    }
-    return createAndSaveNewEvent({ inBlocks, bigPut, root, event, head, additions: Array.from(additions.values()) })
-  }
-  const { root: newProllyRootNode, blocks: newBlocks } = await doProllyBulk(inBlocks, head, event)
-
-  if (!newProllyRootNode) {
-    return createAndSaveNewEvent({
-      inBlocks,
-      bigPut,
-      root: null,
-      event,
-      head,
-      additions: []
-    })
-  } else {
-    const prollyRootBlock = await newProllyRootNode.block
-    const additions = new Map() // ; const removals = new Map()
-    bigPut(prollyRootBlock, additions)
-    for (const nb of newBlocks) {
-      bigPut(nb, additions)
-    }
-    // additions are new blocks
-    return createAndSaveNewEvent({
-      inBlocks,
-      bigPut,
-      root: newProllyRootNode, // Block
-      event,
-      head,
-      additions: Array.from(additions.values()) /*, todo? Array.from(removals.values()) */
-    })
-  }
-}
-
-/**
- * Determine the effective prolly root given the current merkle clock head.
- *
- * @param {TransactionBlockstore} inBlocks Bucket block storage.
- * @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
- */
-export async function root (inBlocks, head, doFull = false) {
-  if (!head.length) {
-    throw new Error('no head')
-  }
-  // console.log('root', head.map(h => h.toString()))
-  // todo maybe these should go to a temp blockstore?
-  return await doTransaction(
-    'root',
-    inBlocks,
-    async transactionBlocks => {
-      const { bigPut } = makeGetAndPutBlock(transactionBlocks)
-      const { root: newProllyRootNode, blocks: newBlocks, clockCIDs } = await doProllyBulk(inBlocks, head, null, doFull)
-      //
-      // const rootBlock = await newProllyRootNode.block
-      // bigPut(rootBlock)
-      for (const nb of newBlocks) {
-        bigPut(nb)
-      }
-      // console.log('root root', newProllyRootNode.constructor.name, newProllyRootNode)
-      return { clockCIDs, node: newProllyRootNode, head }
-    },
-    false
-  )
-  // return { clockCIDs, node: newProllyRootNode }
-}
-
-/**
- * Get the list of events not known by the `since` event
- * @param {TransactionBlockstore} blocks Bucket block storage.
- * @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
- * @param {import('./clock').EventLink<import('./clock').EventData>} since Event to compare against.
- * @returns {Promise<{clockCIDs: CIDCounter, result: import('./clock').EventData[]}>}
- */
-export async function eventsSince (blocks, head, since) {
-  if (!head.length) {
-    // throw new Error('no head')
-    return { clockCIDs: [], result: [] }
-  }
-  // @ts-ignore
-  const sinceHead = [...since, ...head].map(h => h.toString()) // ?
-  // console.log('eventsSince', sinceHead.map(h => h.toString()))
-  const { cids, events: unknownSorted3 } = await findEventsToSync(blocks, sinceHead)
-  return { clockCIDs: cids, result: unknownSorted3.map(({ value: { data } }) => data) }
-}
-
-/**
- *
- * @param {TransactionBlockstore} blocks Bucket block storage.
- * @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
- *
- * @returns {Promise<{root: any, cids: CIDCounter, clockCIDs: CIDCounter, result: import('./clock').EventData[]}>}
- *
- */
-export async function getAll (blocks, head, rootCache = null, doFull = false) {
-  if (!head.length) {
-    return { root: null, clockCIDs: new CIDCounter(), cids: new CIDCounter(), result: [] }
-  }
-  const { node: prollyRootNode, clockCIDs } = await rootOrCache(blocks, head, rootCache, doFull)
-
-  if (!prollyRootNode) {
-    return { root: null, clockCIDs, cids: new CIDCounter(), result: [] }
-  }
-  const { result, cids } = await prollyRootNode.getAllEntries() // todo params
-  return { root: prollyRootNode, clockCIDs, cids, result: result.map(({ key, value }) => ({ key, value })) }
-}
-
-async function rootOrCache (blocks, head, rootCache, doFull = false) {
-  let node
-  let clockCIDs
-  if (!doFull && rootCache && rootCache.root) {
-    // console.log('get root from cache', rootCache)
-    node = rootCache.root
-    clockCIDs = rootCache.clockCIDs
-  } else {
-    // console.log('finding root')
-    // const callTag = Math.random().toString(36).substring(7)
-    // console.time(callTag + '.root')
-    //
-    // const prevClock = [...this.clock]
-
-    ;({ node, clockCIDs } = await root(blocks, head, doFull))
-
-    // this.applyClock(prevClock, result.head)
-    // await this.notifyListeners([decodedEvent])
-
-    // console.timeEnd(callTag + '.root')
-    // console.log('found root', node.entryList)
-  }
-  return { node, clockCIDs }
-}
-
-/**
- * @param {TransactionBlockstore} blocks Bucket block storage.
- * @param {import('./clock').EventLink<import('./clock').EventData>[]} head Merkle clock head.
- * @param {string} key The key of the value to retrieve.
- */
-export async function get (blocks, head, key, rootCache = null) {
-  // instead pass root from db? and always update on change
-  if (!head.length) {
-    return { cids: new CIDCounter(), result: null }
-  }
-
-  const { node: prollyRootNode, clockCIDs } = await rootOrCache(blocks, head, rootCache)
-
-  if (!prollyRootNode) {
-    return { clockCIDs, cids: new CIDCounter(), result: null }
-  }
-  const { result, cids } = await prollyRootNode.get(key)
-  return { result, cids, clockCIDs, root: prollyRootNode }
-}
-
-export async function * vis (blocks, head) {
-  if (!head.length) {
-    return { cids: new CIDCounter(), result: null }
-  }
-  const { node: prollyRootNode, cids } = await root(blocks, head)
-  const lines = []
-  for await (const line of prollyRootNode.vis()) {
-    yield line
-    lines.push(line)
-  }
-  return { vis: lines.join('\n'), cids }
-}
-
-export async function visMerkleTree (blocks, head) {
-  // if (!head) return
-  if (head && !Array.isArray(head)) {
-    const getBl = makeGetBlock(blocks)
-    const prollyRootNode = await load({
-      cid: head,
-      get: getBl.getBlock,
-      ...blockOpts
-    })
-    const lines = []
-    for await (const line of prollyRootNode.vis()) {
-      lines.push(line)
-    }
-    return { vis: lines.join('\n'), cids: new CIDCounter() }
-  }
-  const { node: prollyRootNode, cids } = await root(blocks, head)
-  const lines = []
-  for await (const line of prollyRootNode.vis()) {
-    lines.push(line)
-  }
-  return { vis: lines.join('\n'), cids }
-}
-
-export async function visMerkleClock (blocks, head) {
-  const lines = []
-  for await (const line of visClock(blocks, head)) {
-    // yield line
-    lines.push(line)
-  }
-  return { vis: lines.join('\n') }
-}

package/src/remote.js

@@ -1,113 +0,0 @@
-// when you call database.connect(email)
-// it will return a promise that resolves when the user is logged in
-// and sends you an email
-
-import { create } from '@web3-storage/w3up-client'
-import * as w3clock from '@web3-storage/clock/client'
-import { CID } from 'multiformats'
-
-export class Remote {
-  client = null
-  name = 'unset'
-  config = {}
-
-  constructor (database, name, config) {
-    this.name = name
-    this.config = config
-    this.database = database
-  }
-
-  async clock (cid) {
-    // const did = this.client.currentSpace()
-    const agent = this.client.agent()
-    const head = await w3clock.head({ issuer: agent, with: agent.did(), proofs: [] })
-    console.log('head', head, JSON.stringify(head.root.data.ocm.out))
-    const headCids = head.root.data.ocm.out.ok.head
-    const blocks = await Promise.all([this.database.blocks.get(CID.parse(cid)),
-      ...headCids.map(c => this.database.blocks.get(c))])
-
-    console.log('blocks', blocks)
-    const adv = await w3clock.advance({ issuer: agent, with: agent.did(), proofs: [] }, CID.parse(cid)
-      , { blocks }
-    )
-    console.log('adv', adv, JSON.stringify(adv.root.data.ocm.out))
-    return { head, adv }
-  }
-
-  async sync (cid) {
-    // fetch the remote clock headCids using w3clock.head
-    const agent = this.client.agent()
-    const head = await w3clock.head({ issuer: agent, with: agent.did(), proofs: [] })
-    console.log('head', head, JSON.stringify(head.root.data.ocm.out))
-    const headCids = head.root.data.ocm.out.ok.head
-    const lastSyncHead = await this.database.blocks.valet.primary.getLastSynced()
-    console.log('lastSyncHead', lastSyncHead)
-    const headSet = new Set(headCids.map(c => c.toString()))
-    const lastSyncSet = new Set(lastSyncHead.map(c => c.toString()))
-
-    // are they the same?
-    const same = headSet.size === lastSyncSet.size && [...headSet].every(value => lastSyncSet.has(value))
-
-    // if the headCids and the lastSyncHead are the same, we are in sync and can push
-    if (same) {
-      const currentHead = this.database.clock
-      const currentHeadSet = new Set(currentHead.map(c => c.toString()))
-
-      console.log('synced with cloud', headSet, lastSyncSet)
-
-      // are they the same?
-      const currentSame = headSet.size === currentHeadSet.size && [...headSet].every(value => currentHeadSet.has(value))
-      if (currentSame) {
-        // we are in sync, do nothing
-        return true
-      } else {
-        console.log('push to cloud', headSet, currentHeadSet)
-        // we are ahead of the remote, push our clock
-        // const lastCompact = this.database.blocks.valet.primary.getLastCompact()
-        // get a compact since the last sync
-        console.log('we are ahead of the remote, push our clock')
-        // const compact = this.database.blocks.valet.primary.getCompactSince(lastSyncHead)
-      }
-    } else {
-      // we are behind, fetch the remote
-      console.log('we are behind, fetch the remote')
-    }
-
-    // if it is the same as the local (current metadata carcid? `newValetCidCar` / sync clock), do nothing, we are in sync
-    // if it is the same as our previously pushed clock event, but our local clock is ahead of it, we need to push our clock
-    //               - we can store the previous clock event cid in the metadata
-    // - sending our updates:
-    //   - get the _last_sync and _last_compact values from our metadata
-    //   - if last sync is after last compact
-    //     - make a merged car file for the syncs
-    //   - else
-    //     - upload the car file for the last compact
-    //     - make a merge car file for any uncompacted car files since the last compact, it should base its cidMap on the compact car file (as we go the sync stream will need to track it's own cidMap)
-    //        - if there is only one car file, it is the merge car file (already based on last compact)
-    //     - upload the merge car file
-    //     -  create a new clock block with the current w3clock.head as parent and the merge car file cid as the data
-    //     - update the remote clock with the new clock block (it doesn't need to fetch the car file, and we dont need to store the clock blocks locally, just the most recent one)
-    //
-    // else if the remote head is not contained by our clock, it is is ahead of the local sync clock.
-    // -  get the car file it points to from its data field
-    // - merge to the local clock (park that car so we have both carcid indexes)
-    // - calculate a new root from the merged head, and update the local clock
-  }
-
-  async connect (email) {
-    try {
-      const client = await create()
-      await client.authorize(email)
-      const claims = await client.capability.access.claim()
-      console.log('claims', claims)
-      const space = await client.createSpace('fp.' + this.name)
-      console.log('space', space)
-      await client.setCurrentSpace(space.did())
-      await client.registerSpace(email)
-      this.client = client
-      console.log('client', client)
-    } catch (err) {
-      console.error('registration failed: ', err)
-    }
-  }
-}

package/src/sha1.js

@@ -1,83 +0,0 @@
-// @ts-nocheck
-// from https://github.com/duzun/sync-sha1/blob/master/rawSha1.js
-// MIT License Copyright (c) 2020 Dumitru Uzun
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-// import {
-//   isLittleEndian, switchEndianness32
-// } from 'string-encode'
-
-/**
-* SHA1 on binary array
-*
-* @param   {Uint8Array}  b  Data to hash
-*
-* @return  {Uint8Array}  sha1 hash
-*/
-export function rawSha1 (b) {
-  let i = b.byteLength
-  let bs = 0
-  let A; let B; let C; let D; let G
-  const H = Uint32Array.from([A = 0x67452301, B = 0xEFCDAB89, ~A, ~B, 0xC3D2E1F0])
-  const W = new Uint32Array(80)
-  const nrWords = (i / 4 + 2) | 15
-  const words = new Uint32Array(nrWords + 1)
-  let j
-
-  words[nrWords] = i * 8
-  words[i >> 2] |= 0x80 << (~i << 3)
-  for (;i--;) {
-    words[i >> 2] |= b[i] << (~i << 3)
-  }
-
-  for (A = H.slice(); bs < nrWords; bs += 16, A.set(H)) {
-    for (i = 0; i < 80;
-      A[0] = (
-        G = ((b = A[0]) << 5 | b >>> 27) +
-                  A[4] +
-                  (W[i] = (i < 16) ? words[bs + i] : G << 1 | G >>> 31) +
-                  0x5A827999,
-        B = A[1],
-        C = A[2],
-        D = A[3],
-        G + ((j = i / 5 >> 2) // eslint-disable-line no-cond-assign
-          ? j !== 2
-            ? (B ^ C ^ D) + (j & 2 ? 0x6FE0483D : 0x14577208)
-            : (B & C | B & D | C & D) + 0x34994343
-          : B & C | ~B & D
-        )
-      )
-      , A[1] = b
-      , A[2] = B << 30 | B >>> 2
-      , A[3] = C
-      , A[4] = D
-      , ++i
-    ) {
-      G = W[i - 3] ^ W[i - 8] ^ W[i - 14] ^ W[i - 16]
-    }
-
-    for (i = 5; i;) H[--i] = H[i] + A[i]
-  }
-
-  // if (isLittleEndian()) {
-  //   H = H.map(switchEndianness32)
-  // }
-
-  return new Uint8Array(H.buffer, H.byteOffset, H.byteLength)
-}