@fireproof/core 0.7.2-dev.4 → 0.7.2-dev.6

package/dist/storage/base.js

@@ -1,3 +1,14 @@
+// import { sha256 } from 'multiformats/hashes/sha2'
+import * as CBW from '@ipld/car/buffer-writer';
+import * as raw from 'multiformats/codecs/raw';
+// import * as Block from 'multiformats/block'
+// @ts-ignore
+// import { bf } from 'prolly-trees/utils'
+// @ts-ignore
+// import { nocache as cache } from 'prolly-trees/cache'
+import { encrypt, decrypt } from '../crypto.js';
+// import { Buffer } from 'buffer'
+// const chunker = bf(30)
 import randomBytes from 'randombytes';
 // import { randomBytes } from 'crypto'
 import { create, load } from 'ipld-hashmap';
@@ -15,7 +26,6 @@ import { Buffer } from 'buffer';
 import { rawSha1 as sha1sync } from '../sha1.js';
 // @ts-ignore
 import * as codec from '../encrypted-block.js';
-import { blocksToCarBlock, blocksToEncryptedCarBlock, blocksFromEncryptedCarBlock } from '../valet.js';
 const chunker = bf(30);
 const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare };
 const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT;
@@ -24,6 +34,7 @@ export class Base {
     valetRootCarCid = null; // used on initial hydrate, if you change this, set this.valetCarCidMap = null
     keyMaterial = null;
     keyId = 'null';
+    readonly = false;
     constructor(name, config = {}) {
         this.instanceId = Math.random().toString(36).slice(2);
         this.name = name;
@@ -65,6 +76,42 @@ export class Base {
             this.keyId = 'null';
         }
     }
+    async compact(clock) {
+        if (this.readonly)
+            return;
+        if (clock.length !== 1) {
+            throw new Error(`Compacting with clock length ${clock.length} instead of 1. To merge the clock, apply an update to the database first.`);
+        }
+        const cidMap = await this.getCidCarMap();
+        const dataCids = [...cidMap.keys()];
+        const allBlocks = new Map();
+        for (const cid of dataCids) {
+            const block = await this.getLoaderBlock(cid);
+            allBlocks.set(cid, block);
+        }
+        cidMap.clear();
+        const blocks = {
+            lastCid: clock[0],
+            get: cid => allBlocks.get(cid.toString())
+        };
+        await this.parkCar(blocks, dataCids);
+    }
+    async parkCar(innerBlockstore, cids) {
+        // console.log('parkCar', this.instanceId, this.name, this.readonly)
+        if (this.readonly)
+            return;
+        let newCar;
+        if (this.keyMaterial) {
+            // console.log('encrypting car', innerBlockstore.label)
+            newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial, [...cids]);
+        }
+        else {
+            // todo should we pass cids in instead of iterating innerBlockstore?
+            newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
+        }
+        // console.log('new car', newCar.cid.toString())
+        return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids);
+    }
     async saveCar(carCid, value, cids) {
         const newValetCidCar = await this.updateCarCidMap(carCid, cids);
         // console.log('writeCars', carCid.toString(), newValetCidCar.cid.toString())
@@ -83,6 +130,7 @@ export class Base {
         ];
         await this.writeCars(carList);
         this.valetRootCarCid = newValetCidCar.cid;
+        // console.trace('saved car', this.instanceId, this.name, newValetCidCar.cid.toString())
         return newValetCidCar;
     }
     applyHeaders(headers) {
@@ -125,7 +173,7 @@ export class Base {
     }
     async saveHeader(header) {
         // for each branch, save the header
-        // console.log('saveHeader', this.config.branches)
+        // console.log('saveHeader', this.config.branches, header)
         //  for (const branch of this.branches) {
         //    await this.saveBranchHeader(branch)
         //  }
@@ -139,7 +187,7 @@ export class Base {
     prepareHeader(header, json = true) {
         header.key = this.keyMaterial;
         header.car = this.valetRootCarCid.toString();
-        // console.log('prepareHeader', this.instanceId, this.name, header.key, this.valetRootCarCid.toString())
+        // console.log('prepareHeader', this.instanceId, this.name, header)
         return json ? JSON.stringify(header) : header;
     }
     writeHeader(branch, header) {
@@ -305,7 +353,9 @@ export class Base {
         }
         let newValetCidCar;
         if (this.keyMaterial) {
-            newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial);
+            const cids = [...ipldLoader.blocks.blocks.keys()];
+            // console.log('persistCarMap', cids)
+            newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial, cids);
         }
         else {
             newValetCidCar = await blocksToCarBlock(indexNode.cid, ipldLoader.blocks);
@@ -349,3 +399,98 @@ export class VMemoryBlockstore {
         }
     }
 }
+export const blocksToCarBlock = async (rootCids, blocks) => {
+    // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
+    let size = 0;
+    if (!Array.isArray(rootCids)) {
+        rootCids = [rootCids];
+    }
+    const headerSize = CBW.headerLength({ roots: rootCids });
+    size += headerSize;
+    if (!Array.isArray(blocks)) {
+        blocks = Array.from(blocks.entries());
+    }
+    for (const { cid, bytes } of blocks) {
+        // console.log(cid, bytes)
+        size += CBW.blockLength({ cid, bytes });
+    }
+    const buffer = new Uint8Array(size);
+    const writer = await CBW.createWriter(buffer, { headerSize });
+    for (const cid of rootCids) {
+        writer.addRoot(cid);
+    }
+    for (const { cid, bytes } of blocks) {
+        writer.write({ cid, bytes });
+    }
+    await writer.close();
+    return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
+};
+export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
+    const encryptionKey = Buffer.from(keyMaterial, 'hex');
+    const encryptedBlocks = [];
+    const theCids = cids;
+    // console.trace('blocksToEncryptedCarBlock', blocks)
+    // for (const { cid } of blocks.entries()) {
+    //   theCids.push(cid.toString())
+    // }
+    // console.log(
+    //   'encrypting',
+    //   theCids.length,
+    //   'blocks',
+    //   theCids.includes(innerBlockStoreClockRootCid.toString()),
+    //   keyMaterial
+    // )
+    // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
+    let last;
+    for await (const block of encrypt({
+        cids: theCids,
+        get: async (cid) => {
+            // console.log('getencrypt', cid)
+            const got = blocks.get(cid);
+            // console.log('got', got)
+            return got.block ? ({ cid, bytes: got.block }) : got;
+        },
+        key: encryptionKey,
+        hasher: sha256,
+        chunker,
+        cache,
+        // codec: dagcbor, // should be crypto?
+        root: innerBlockStoreClockRootCid
+    })) {
+        encryptedBlocks.push(block);
+        last = block;
+    }
+    // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
+    const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
+    return encryptedCar;
+};
+// { root, get, key, cache, chunker, hasher }
+const memoizeDecryptedCarBlocks = new Map();
+export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
+    if (memoizeDecryptedCarBlocks.has(cid.toString())) {
+        return memoizeDecryptedCarBlocks.get(cid.toString());
+    }
+    else {
+        const blocksPromise = (async () => {
+            const decryptionKey = Buffer.from(keyMaterial, 'hex');
+            // console.log('decrypting', keyMaterial, cid.toString())
+            const cids = new Set();
+            const decryptedBlocks = [];
+            for await (const block of decrypt({
+                root: cid,
+                get,
+                key: decryptionKey,
+                chunker,
+                hasher: sha256,
+                cache
+                // codec: dagcbor
+            })) {
+                decryptedBlocks.push(block);
+                cids.add(block.cid.toString());
+            }
+            return { blocks: decryptedBlocks, cids };
+        })();
+        memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
+        return blocksPromise;
+    }
+};

package/dist/sync.js

@@ -1,7 +1,7 @@
 import SimplePeer from 'simple-peer';
 import { parseCID } from './database.js';
 import { decodeEventBlock } from './clock.js';
-import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js';
+import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/base.js';
 import { CarReader } from '@ipld/car';
 /**
  * @typedef {import('./database.js').Database} Database
@@ -199,7 +199,7 @@ export class Sync {
             return blocksToEncryptedCarBlock(rootCIDs, {
                 entries: () => syncCIDs.map(cid => ({ cid })),
                 get: async (cid) => await blocks.get(cid)
-            }, key);
+            }, key, syncCIDs.map(c => c.toString()));
         }
         else {
             const carBlocks = await Promise.all(syncCIDs.map(async (c) => {

package/dist/valet.js

@@ -1,15 +1,4 @@
-import { sha256 } from 'multiformats/hashes/sha2';
-import * as CBW from '@ipld/car/buffer-writer';
-import * as raw from 'multiformats/codecs/raw';
-import * as Block from 'multiformats/block';
 import { Loader } from './loader.js';
-// @ts-ignore
-import { bf } from 'prolly-trees/utils';
-// @ts-ignore
-import { nocache as cache } from 'prolly-trees/cache';
-import { encrypt, decrypt } from './crypto.js';
-import { Buffer } from 'buffer';
-const chunker = bf(30);
 export class Valet {
     idb = null;
     name = null;
@@ -38,6 +27,11 @@ export class Valet {
         } // todo: await?
         return await this.primary.saveHeader(header);
     }
+    async compact(clock) {
+        await this.primary.compact(clock);
+        if (this.secondary)
+            await this.secondary.compact(clock);
+    }
     /**
      * Group the blocks into a car and write it to the valet.
      * @param {import('./blockstore.js').InnerBlockstore} innerBlockstore
@@ -47,14 +41,26 @@ export class Valet {
      */
     async writeTransaction(innerBlockstore, cids) {
         if (innerBlockstore.lastCid) {
-            await parkCar(this.primary, innerBlockstore, cids);
+            await this.primary.parkCar(innerBlockstore, cids);
             if (this.secondary)
-                await parkCar(this.secondary, innerBlockstore, cids);
+                await this.secondary.parkCar(innerBlockstore, cids);
         }
         else {
             throw new Error('missing lastCid for car header');
         }
     }
+    // async compact() {
+    //   const carCids = []
+    //   // for await (const { cid } of this.valet.cids()) {
+    //   //   yield { cid }
+    //   // }
+    //   // create a blockstore with all data
+    //   {
+    //     entries: () => syncCIDs.map(cid => ({ cid })),
+    //     get: async cid => await blocks.get(cid)
+    //   }
+    //   // park it
+    // }
     /**
      * Iterate over all blocks in the store.
      *
@@ -90,7 +96,7 @@ export class Valet {
                     reader.get = reader.gat; // some consumers prefer get
                     // console.log('replicating', reader.root)
                     reader.lastCid = reader.root.cid;
-                    await parkCar(this.primary, reader, [...cids]);
+                    await this.primary.parkCar(reader, [...cids]);
                     return block;
                 }
                 catch (e) {
@@ -100,103 +106,3 @@ export class Valet {
         }
     }
 }
-async function parkCar(storage, innerBlockstore, cids) {
-    // console.log('parkCar', this.instanceId, this.name, carCid, cids)
-    if (storage.readonly)
-        return;
-    let newCar;
-    if (storage.keyMaterial) {
-        // console.log('encrypting car', innerBlockstore.label)
-        newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, storage.keyMaterial);
-    }
-    else {
-        // todo should we pass cids in instead of iterating innerBlockstore?
-        newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
-    }
-    // console.log('new car', newCar.cid.toString())
-    return await storage.saveCar(newCar.cid.toString(), newCar.bytes, cids);
-}
-export const blocksToCarBlock = async (rootCids, blocks) => {
-    // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
-    let size = 0;
-    if (!Array.isArray(rootCids)) {
-        rootCids = [rootCids];
-    }
-    const headerSize = CBW.headerLength({ roots: rootCids });
-    size += headerSize;
-    if (!Array.isArray(blocks)) {
-        blocks = Array.from(blocks.entries());
-    }
-    for (const { cid, bytes } of blocks) {
-        // console.log(cid, bytes)
-        size += CBW.blockLength({ cid, bytes });
-    }
-    const buffer = new Uint8Array(size);
-    const writer = await CBW.createWriter(buffer, { headerSize });
-    for (const cid of rootCids) {
-        writer.addRoot(cid);
-    }
-    for (const { cid, bytes } of blocks) {
-        writer.write({ cid, bytes });
-    }
-    await writer.close();
-    return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
-};
-export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
-    const encryptionKey = Buffer.from(keyMaterial, 'hex');
-    const encryptedBlocks = [];
-    const theCids = [];
-    // console.trace('blocksToEncryptedCarBlock', blocks)
-    for (const { cid } of blocks.entries()) {
-        theCids.push(cid.toString());
-    }
-    // console.log('encrypting', theCids.length, 'blocks', theCids.includes(innerBlockStoreClockRootCid.toString()), keyMaterial)
-    // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
-    let last;
-    for await (const block of encrypt({
-        cids: theCids,
-        get: async (cid) => blocks.get(cid),
-        key: encryptionKey,
-        hasher: sha256,
-        chunker,
-        cache,
-        // codec: dagcbor, // should be crypto?
-        root: innerBlockStoreClockRootCid
-    })) {
-        encryptedBlocks.push(block);
-        last = block;
-    }
-    // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
-    const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
-    return encryptedCar;
-};
-// { root, get, key, cache, chunker, hasher }
-const memoizeDecryptedCarBlocks = new Map();
-export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
-    if (memoizeDecryptedCarBlocks.has(cid.toString())) {
-        return memoizeDecryptedCarBlocks.get(cid.toString());
-    }
-    else {
-        const blocksPromise = (async () => {
-            const decryptionKey = Buffer.from(keyMaterial, 'hex');
-            // console.log('decrypting', keyMaterial, cid.toString())
-            const cids = new Set();
-            const decryptedBlocks = [];
-            for await (const block of decrypt({
-                root: cid,
-                get,
-                key: decryptionKey,
-                chunker,
-                hasher: sha256,
-                cache
-                // codec: dagcbor
-            })) {
-                decryptedBlocks.push(block);
-                cids.add(block.cid.toString());
-            }
-            return { blocks: decryptedBlocks, cids };
-        })();
-        memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
-        return blocksPromise;
-    }
-};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.7.2-dev.4",
+  "version": "0.7.2-dev.6",
   "description": "Live data for React, accelerated by proofs, powered by IPFS",
   "main": "dist/src/fireproof.js",
   "module": "dist/src/fireproof.mjs",

package/src/database.js

@@ -66,7 +66,7 @@ export class Database {
                 clock: {
                   byId: byId ? parseCID(byId) : null,
                   byKey: byKey ? parseCID(byKey) : null,
-                  db: (db && db.length > 0) ? db.map(c => parseCID(c)) : null
+                  db: db && db.length > 0 ? db.map(c => parseCID(c)) : null
                 },
                 code,
                 name
@@ -138,6 +138,13 @@ export class Database {
     await this.notifyListeners({ _reset: true, _clock: this.clockToJSON() })
   }
 
+  async compact () {
+    if (this.name && this.blocks.valet) {
+      await this.blocks.valet.compact(this.clock)
+      await this.blocks.valet.saveHeader(this.toHeader())
+    }
+  }
+
   /**
    * Returns the changes made to the Fireproof instance since the specified event.
    * @function changesSince

package/src/prolly.js

@@ -116,11 +116,11 @@ const makeGetAndPutBlock = inBlocks => {
   // const mblocks = new MemoryBlockstore()
   // const blocks = new MultiBlockFetcher(mblocks, inBlocks)
   const { getBlock, cids } = makeGetBlock(inBlocks)
-  const put = inBlocks.put.bind(inBlocks)
+  // const put = inBlocks.put.bind(inBlocks)
   const bigPut = async (block, additions) => {
     // console.log('bigPut', block.cid.toString())
     const { cid, bytes } = block
-    put(cid, bytes)
+    inBlocks.put(cid, bytes)
     // mblocks.putSync(cid, bytes)
     if (additions) {
       additions.set(cid.toString(), block)

package/src/storage/base.js

@@ -1,3 +1,16 @@
+// import { sha256 } from 'multiformats/hashes/sha2'
+import * as CBW from '@ipld/car/buffer-writer'
+import * as raw from 'multiformats/codecs/raw'
+// import * as Block from 'multiformats/block'
+// @ts-ignore
+// import { bf } from 'prolly-trees/utils'
+// @ts-ignore
+// import { nocache as cache } from 'prolly-trees/cache'
+import { encrypt, decrypt } from '../crypto.js'
+// import { Buffer } from 'buffer'
+
+// const chunker = bf(30)
+
 import randomBytes from 'randombytes'
 // import { randomBytes } from 'crypto'
 import { create, load } from 'ipld-hashmap'
@@ -15,7 +28,6 @@ import { Buffer } from 'buffer'
 import { rawSha1 as sha1sync } from '../sha1.js'
 // @ts-ignore
 import * as codec from '../encrypted-block.js'
-import { blocksToCarBlock, blocksToEncryptedCarBlock, blocksFromEncryptedCarBlock } from '../valet.js'
 
 const chunker = bf(30)
 const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare }
@@ -27,6 +39,7 @@ export class Base {
   valetRootCarCid = null // used on initial hydrate, if you change this, set this.valetCarCidMap = null
   keyMaterial = null
   keyId = 'null'
+  readonly = false
 
   constructor (name, config = {}) {
     this.instanceId = Math.random().toString(36).slice(2)
@@ -72,6 +85,43 @@ export class Base {
     }
   }
 
+  async compact (clock) {
+    if (this.readonly) return
+    if (clock.length !== 1) {
+      throw new Error(
+        `Compacting with clock length ${clock.length} instead of 1. To merge the clock, apply an update to the database first.`
+      )
+    }
+    const cidMap = await this.getCidCarMap()
+    const dataCids = [...cidMap.keys()]
+    const allBlocks = new Map()
+    for (const cid of dataCids) {
+      const block = await this.getLoaderBlock(cid)
+      allBlocks.set(cid, block)
+    }
+    cidMap.clear()
+    const blocks = {
+      lastCid: clock[0],
+      get: cid => allBlocks.get(cid.toString())
+    }
+    await this.parkCar(blocks, dataCids)
+  }
+
+  async parkCar (innerBlockstore, cids) {
+    // console.log('parkCar', this.instanceId, this.name, this.readonly)
+    if (this.readonly) return
+    let newCar
+    if (this.keyMaterial) {
+      // console.log('encrypting car', innerBlockstore.label)
+      newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial, [...cids])
+    } else {
+      // todo should we pass cids in instead of iterating innerBlockstore?
+      newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
+    }
+    // console.log('new car', newCar.cid.toString())
+    return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids)
+  }
+
   async saveCar (carCid, value, cids) {
     const newValetCidCar = await this.updateCarCidMap(carCid, cids)
     // console.log('writeCars', carCid.toString(), newValetCidCar.cid.toString())
@@ -91,6 +141,7 @@ export class Base {
 
     await this.writeCars(carList)
     this.valetRootCarCid = newValetCidCar.cid
+    // console.trace('saved car', this.instanceId, this.name, newValetCidCar.cid.toString())
     return newValetCidCar
   }
 
@@ -136,7 +187,7 @@ export class Base {
 
   async saveHeader (header) {
     // for each branch, save the header
-    // console.log('saveHeader', this.config.branches)
+    // console.log('saveHeader', this.config.branches, header)
     //  for (const branch of this.branches) {
     //    await this.saveBranchHeader(branch)
     //  }
@@ -150,7 +201,7 @@ export class Base {
   prepareHeader (header, json = true) {
     header.key = this.keyMaterial
     header.car = this.valetRootCarCid.toString()
-    // console.log('prepareHeader', this.instanceId, this.name, header.key, this.valetRootCarCid.toString())
+    // console.log('prepareHeader', this.instanceId, this.name, header)
     return json ? JSON.stringify(header) : header
   }
 
@@ -325,7 +376,9 @@ export class Base {
 
     let newValetCidCar
     if (this.keyMaterial) {
-      newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial)
+      const cids = [...ipldLoader.blocks.blocks.keys()]
+      // console.log('persistCarMap', cids)
+      newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial, cids)
     } else {
       newValetCidCar = await blocksToCarBlock(indexNode.cid, ipldLoader.blocks)
     }
@@ -372,3 +425,102 @@ export class VMemoryBlockstore {
     }
   }
 }
+
+export const blocksToCarBlock = async (rootCids, blocks) => {
+  // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
+  let size = 0
+  if (!Array.isArray(rootCids)) {
+    rootCids = [rootCids]
+  }
+  const headerSize = CBW.headerLength({ roots: rootCids })
+  size += headerSize
+  if (!Array.isArray(blocks)) {
+    blocks = Array.from(blocks.entries())
+  }
+  for (const { cid, bytes } of blocks) {
+    // console.log(cid, bytes)
+    size += CBW.blockLength({ cid, bytes })
+  }
+  const buffer = new Uint8Array(size)
+  const writer = await CBW.createWriter(buffer, { headerSize })
+
+  for (const cid of rootCids) {
+    writer.addRoot(cid)
+  }
+
+  for (const { cid, bytes } of blocks) {
+    writer.write({ cid, bytes })
+  }
+  await writer.close()
+  return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
+}
+
+export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
+  const encryptionKey = Buffer.from(keyMaterial, 'hex')
+  const encryptedBlocks = []
+  const theCids = cids
+  // console.trace('blocksToEncryptedCarBlock', blocks)
+  // for (const { cid } of blocks.entries()) {
+  //   theCids.push(cid.toString())
+  // }
+  // console.log(
+  //   'encrypting',
+  //   theCids.length,
+  //   'blocks',
+  //   theCids.includes(innerBlockStoreClockRootCid.toString()),
+  //   keyMaterial
+  // )
+  // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
+  let last
+  for await (const block of encrypt({
+    cids: theCids,
+    get: async cid => {
+      // console.log('getencrypt', cid)
+      const got = blocks.get(cid)
+      // console.log('got', got)
+      return got.block ? ({ cid, bytes: got.block }) : got
+    }, // maybe we can just use blocks.get
+    key: encryptionKey,
+    hasher: sha256,
+    chunker,
+    cache,
+    // codec: dagcbor, // should be crypto?
+    root: innerBlockStoreClockRootCid
+  })) {
+    encryptedBlocks.push(block)
+    last = block
+  }
+  // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
+  const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks)
+  return encryptedCar
+}
+// { root, get, key, cache, chunker, hasher }
+
+const memoizeDecryptedCarBlocks = new Map()
+export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
+  if (memoizeDecryptedCarBlocks.has(cid.toString())) {
+    return memoizeDecryptedCarBlocks.get(cid.toString())
+  } else {
+    const blocksPromise = (async () => {
+      const decryptionKey = Buffer.from(keyMaterial, 'hex')
+      // console.log('decrypting', keyMaterial, cid.toString())
+      const cids = new Set()
+      const decryptedBlocks = []
+      for await (const block of decrypt({
+        root: cid,
+        get,
+        key: decryptionKey,
+        chunker,
+        hasher: sha256,
+        cache
+        // codec: dagcbor
+      })) {
+        decryptedBlocks.push(block)
+        cids.add(block.cid.toString())
+      }
+      return { blocks: decryptedBlocks, cids }
+    })()
+    memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise)
+    return blocksPromise
+  }
+}

package/src/sync.js

@@ -1,7 +1,7 @@
 import SimplePeer from 'simple-peer'
 import { parseCID } from './database.js'
 import { decodeEventBlock } from './clock.js'
-import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js'
+import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/base.js'
 import { CarReader } from '@ipld/car'
 
 /**
@@ -215,7 +215,8 @@ export class Sync {
           entries: () => syncCIDs.map(cid => ({ cid })),
           get: async cid => await blocks.get(cid)
         },
-        key
+        key,
+        syncCIDs.map(c => c.toString())
       )
     } else {
       const carBlocks = await Promise.all(