@fireproof/core 0.8.0-dev → 0.8.0-dev.3

package/src/storage/filesystem.js

@@ -45,11 +45,10 @@ export class Filesystem extends Base {
   }
 
   async writeHeader (branch, header) {
-    // console.log('saveHeader', this.isBrowser)
+    // console.log('saveHeader fs', header)
     if (this.config.readonly) return
-    const pHeader = this.prepareHeader(header)
     // console.log('writeHeader fs', branch, pHeader)
-    await writeSync(this.headerFilename(branch), pHeader)
+    await writeSync(this.headerFilename(branch), header)
   }
 
   headerFilename (branch = 'main') {

package/src/storage/rest.js

@@ -47,12 +47,11 @@ export class Rest extends Base {
 
   async writeHeader (branch, header) {
     if (this.config.readonly) return
-    const pHeader = this.prepareHeader(header)
     // console.log('writeHeader rt', branch, pHeader)
 
     const response = await fetch(this.headerURL(branch), {
       method: 'PUT',
-      body: pHeader,
+      body: header,
       headers: { 'Content-Type': 'application/json' }
     })
     if (!response.ok) throw new Error(`An error occurred: ${response.statusText}`)

package/src/storage/utils.js

@@ -0,0 +1,152 @@
+import * as CBW from '@ipld/car/buffer-writer'
+import * as raw from 'multiformats/codecs/raw'
+import { encrypt, decrypt } from '../crypto.js'
+import { parse } from 'multiformats/link'
+import { sha256 } from 'multiformats/hashes/sha2'
+import * as Block from 'multiformats/block'
+import { Buffer } from 'buffer'
+// @ts-ignore
+import { bf } from 'prolly-trees/utils'
+// @ts-ignore
+import { nocache as cache } from 'prolly-trees/cache'
+const chunker = bf(30)
+
+export async function getEmptyLoader () { // unused?
+  const theseWriteableBlocks = new VMemoryBlockstore()
+  return {
+    blocks: theseWriteableBlocks,
+    put: async (cid, bytes) => {
+      return await theseWriteableBlocks.put(cid, bytes)
+    },
+    get: async (cid) => {
+      const got = await theseWriteableBlocks.get(cid)
+      return got.bytes
+    }
+  }
+}
+
+export class VMemoryBlockstore {
+  /** @type {Map<string, Uint8Array>} */
+  blocks = new Map()
+  instanceId = Math.random().toString(36).slice(2)
+
+  async get (cid) {
+    const bytes = this.blocks.get(cid.toString())
+    if (!bytes) throw new Error('block not found ' + cid.toString())
+    return { cid, bytes }
+  }
+
+  /**
+   * @param {any} cid
+   * @param {Uint8Array} bytes
+   */
+  async put (cid, bytes) {
+    this.blocks.set(cid.toString(), bytes)
+  }
+
+  * entries () {
+    for (const [str, bytes] of this.blocks) {
+      yield { cid: parse(str), bytes }
+    }
+  }
+}
+
+export const blocksToCarBlock = async (rootCids, blocks) => {
+  // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
+  let size = 0
+  if (!Array.isArray(rootCids)) {
+    rootCids = [rootCids]
+  }
+  const headerSize = CBW.headerLength({ roots: rootCids })
+  size += headerSize
+  if (!Array.isArray(blocks)) {
+    blocks = Array.from(blocks.entries())
+  }
+  for (const { cid, bytes } of blocks) {
+    // console.log(cid, bytes)
+    size += CBW.blockLength({ cid, bytes })
+  }
+  const buffer = new Uint8Array(size)
+  const writer = await CBW.createWriter(buffer, { headerSize })
+
+  for (const cid of rootCids) {
+    writer.addRoot(cid)
+  }
+
+  for (const { cid, bytes } of blocks) {
+    writer.write({ cid, bytes })
+  }
+  await writer.close()
+  return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
+}
+
+export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
+  const encryptionKey = Buffer.from(keyMaterial, 'hex')
+  const encryptedBlocks = []
+  const theCids = cids
+  // console.trace('blocksToEncryptedCarBlock', blocks)
+  // for (const { cid } of blocks.entries()) {
+  //   theCids.push(cid.toString())
+  // }
+  // console.log(
+  //   'encrypting',
+  //   theCids.length,
+  //   'blocks',
+  //   theCids.includes(innerBlockStoreClockRootCid.toString()),
+  //   keyMaterial
+  // )
+  // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
+  let last
+  for await (const block of encrypt({
+    cids: theCids,
+    get: async (cid) => {
+      // console.log('getencrypt', cid)
+      const got = blocks.get(cid)
+      // console.log('got', got)
+      return got.block ? ({ cid, bytes: got.block }) : got
+    },
+    key: encryptionKey,
+    hasher: sha256,
+    chunker,
+    cache,
+    // codec: dagcbor, // should be crypto?
+    root: innerBlockStoreClockRootCid
+  })) {
+    encryptedBlocks.push(block)
+    last = block
+  }
+  // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
+  const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks)
+  return encryptedCar
+}
+
+// { root, get, key, cache, chunker, hasher }
+const memoizeDecryptedCarBlocks = new Map()
+export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
+  if (memoizeDecryptedCarBlocks.has(cid.toString())) {
+    return memoizeDecryptedCarBlocks.get(cid.toString())
+  } else {
+    const blocksPromise = (async () => {
+      const decryptionKey = Buffer.from(keyMaterial, 'hex')
+      // console.log('decrypting', keyMaterial, cid.toString())
+      const cids = new Set()
+      const decryptedBlocks = []
+      for await (const block of decrypt({
+        root: cid,
+        get,
+        key: decryptionKey,
+        chunker,
+        hasher: sha256,
+        cache
+        // codec: dagcbor
+      })) {
+        // console.log('decrypted', block.cid.toString())
+        decryptedBlocks.push(block)
+        cids.add(block.cid.toString())
+      }
+      return { blocks: decryptedBlocks, cids }
+    })()
+    memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise)
+    return blocksPromise
+  }
+}

package/src/sync.js

@@ -1,7 +1,7 @@
 import SimplePeer from 'simple-peer'
 import { parseCID } from './database.js'
 import { decodeEventBlock } from './clock.js'
-import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/base.js'
+import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/utils.js'
 import { CarReader } from '@ipld/car'
 
 /**

package/src/valet.js

@@ -79,16 +79,22 @@ export class Valet {
       if (this.secondary) {
         // console.log('getValetBlock secondary', dataCID)
         try {
+          // eslint-disable-next-line
           const { block, reader } = await this.secondary.getLoaderBlock(dataCID)
+          const writeableCarReader = await this.primary.getWriteableCarReader(reader)
+          // console.log('getValetBlock secondary', dataCID, block.length)
+          // eslint-disable-next-line
           const cids = new Set()
           for await (const { cid } of reader.entries()) {
             // console.log(cid, bytes)
             cids.add(cid.toString())
           }
-          reader.get = reader.gat // some consumers prefer get
+          // reader.get = reader.gat // some consumers prefer get
           // console.log('replicating', reader.root)
-          reader.lastCid = reader.root.cid
-          await this.primary.parkCar(reader, [...cids])
+          writeableCarReader.lastCid = reader.root.cid
+          writeableCarReader.head = []
+          await this.primary.parkCar(writeableCarReader, cids).catch(e => console.error('parkCar error', e))
+          // console.log('FIX THIS', did)
           return block
         } catch (e) {
           // console.log('getValetBlock secondary error', e)