@fireproof/core 0.8.0-dev → 0.8.0-dev.3

package/dist/storage/blocksToEncryptedCarBlock.js

@@ -0,0 +1,144 @@
+import * as CBW from '@ipld/car/buffer-writer';
+import * as raw from 'multiformats/codecs/raw';
+import { encrypt, decrypt } from '../crypto.js';
+import { parse } from 'multiformats/link';
+import { sha256 } from 'multiformats/hashes/sha2';
+import * as Block from 'multiformats/block';
+import { Buffer } from 'buffer';
+// @ts-ignore
+import { bf } from 'prolly-trees/utils';
+// @ts-ignore
+import { nocache as cache } from 'prolly-trees/cache';
+const chunker = bf(30);
+export async function getEmptyLoader() {
+    const theseWriteableBlocks = new VMemoryBlockstore();
+    return {
+        blocks: theseWriteableBlocks,
+        put: async (cid, bytes) => {
+            return await theseWriteableBlocks.put(cid, bytes);
+        },
+        get: async (cid) => {
+            const got = await theseWriteableBlocks.get(cid);
+            return got.bytes;
+        }
+    };
+}
+export class VMemoryBlockstore {
+    /** @type {Map<string, Uint8Array>} */
+    blocks = new Map();
+    instanceId = Math.random().toString(36).slice(2);
+    async get(cid) {
+        const bytes = this.blocks.get(cid.toString());
+        if (!bytes)
+            throw new Error('block not found ' + cid.toString());
+        return { cid, bytes };
+    }
+    /**
+     * @param {any} cid
+     * @param {Uint8Array} bytes
+     */
+    async put(cid, bytes) {
+        this.blocks.set(cid.toString(), bytes);
+    }
+    *entries() {
+        for (const [str, bytes] of this.blocks) {
+            yield { cid: parse(str), bytes };
+        }
+    }
+}
+export const blocksToCarBlock = async (rootCids, blocks) => {
+    // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
+    let size = 0;
+    if (!Array.isArray(rootCids)) {
+        rootCids = [rootCids];
+    }
+    const headerSize = CBW.headerLength({ roots: rootCids });
+    size += headerSize;
+    if (!Array.isArray(blocks)) {
+        blocks = Array.from(blocks.entries());
+    }
+    for (const { cid, bytes } of blocks) {
+        // console.log(cid, bytes)
+        size += CBW.blockLength({ cid, bytes });
+    }
+    const buffer = new Uint8Array(size);
+    const writer = await CBW.createWriter(buffer, { headerSize });
+    for (const cid of rootCids) {
+        writer.addRoot(cid);
+    }
+    for (const { cid, bytes } of blocks) {
+        writer.write({ cid, bytes });
+    }
+    await writer.close();
+    return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
+};
+export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
+    const encryptionKey = Buffer.from(keyMaterial, 'hex');
+    const encryptedBlocks = [];
+    const theCids = cids;
+    // console.trace('blocksToEncryptedCarBlock', blocks)
+    // for (const { cid } of blocks.entries()) {
+    //   theCids.push(cid.toString())
+    // }
+    // console.log(
+    //   'encrypting',
+    //   theCids.length,
+    //   'blocks',
+    //   theCids.includes(innerBlockStoreClockRootCid.toString()),
+    //   keyMaterial
+    // )
+    // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
+    let last;
+    for await (const block of encrypt({
+        cids: theCids,
+        get: async (cid) => {
+            // console.log('getencrypt', cid)
+            const got = blocks.get(cid);
+            // console.log('got', got)
+            return got.block ? ({ cid, bytes: got.block }) : got;
+        },
+        key: encryptionKey,
+        hasher: sha256,
+        chunker,
+        cache,
+        // codec: dagcbor, // should be crypto?
+        root: innerBlockStoreClockRootCid
+    })) {
+        encryptedBlocks.push(block);
+        last = block;
+    }
+    // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
+    const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
+    return encryptedCar;
+};
+// { root, get, key, cache, chunker, hasher }
+const memoizeDecryptedCarBlocks = new Map();
+export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
+    if (memoizeDecryptedCarBlocks.has(cid.toString())) {
+        return memoizeDecryptedCarBlocks.get(cid.toString());
+    }
+    else {
+        const blocksPromise = (async () => {
+            const decryptionKey = Buffer.from(keyMaterial, 'hex');
+            // console.log('decrypting', keyMaterial, cid.toString())
+            const cids = new Set();
+            const decryptedBlocks = [];
+            for await (const block of decrypt({
+                root: cid,
+                get,
+                key: decryptionKey,
+                chunker,
+                hasher: sha256,
+                cache
+                // codec: dagcbor
+            })) {
+                // console.log('decrypted', block.cid.toString())
+                decryptedBlocks.push(block);
+                cids.add(block.cid.toString());
+            }
+            return { blocks: decryptedBlocks, cids };
+        })();
+        memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
+        return blocksPromise;
+    }
+};

package/dist/storage/browser.js

@@ -52,7 +52,7 @@ export class Browser extends Base {
         if (this.config.readonly)
             return;
         try {
-            return localStorage.setItem(this.headerKey(branch), this.prepareHeader(header));
+            return localStorage.setItem(this.headerKey(branch), header);
         }
         catch (e) { }
     }

package/dist/storage/filesystem.js

@@ -40,12 +40,11 @@ export class Filesystem extends Base {
         return JSON.parse(header);
     }
     async writeHeader(branch, header) {
-        // console.log('saveHeader', this.isBrowser)
+        // console.log('saveHeader fs', header)
         if (this.config.readonly)
             return;
-        const pHeader = this.prepareHeader(header);
         // console.log('writeHeader fs', branch, pHeader)
-        await writeSync(this.headerFilename(branch), pHeader);
+        await writeSync(this.headerFilename(branch), header);
     }
     headerFilename(branch = 'main') {
         // console.log('headerFilename', this.config.dataDir, this.name)

package/dist/storage/rest.js

@@ -45,11 +45,10 @@ export class Rest extends Base {
     async writeHeader(branch, header) {
         if (this.config.readonly)
             return;
-        const pHeader = this.prepareHeader(header);
         // console.log('writeHeader rt', branch, pHeader)
         const response = await fetch(this.headerURL(branch), {
             method: 'PUT',
-            body: pHeader,
+            body: header,
             headers: { 'Content-Type': 'application/json' }
         });
         if (!response.ok)

package/dist/storage/utils.js

@@ -0,0 +1,144 @@
+import * as CBW from '@ipld/car/buffer-writer';
+import * as raw from 'multiformats/codecs/raw';
+import { encrypt, decrypt } from '../crypto.js';
+import { parse } from 'multiformats/link';
+import { sha256 } from 'multiformats/hashes/sha2';
+import * as Block from 'multiformats/block';
+import { Buffer } from 'buffer';
+// @ts-ignore
+import { bf } from 'prolly-trees/utils';
+// @ts-ignore
+import { nocache as cache } from 'prolly-trees/cache';
+const chunker = bf(30);
+export async function getEmptyLoader() {
+    const theseWriteableBlocks = new VMemoryBlockstore();
+    return {
+        blocks: theseWriteableBlocks,
+        put: async (cid, bytes) => {
+            return await theseWriteableBlocks.put(cid, bytes);
+        },
+        get: async (cid) => {
+            const got = await theseWriteableBlocks.get(cid);
+            return got.bytes;
+        }
+    };
+}
+export class VMemoryBlockstore {
+    /** @type {Map<string, Uint8Array>} */
+    blocks = new Map();
+    instanceId = Math.random().toString(36).slice(2);
+    async get(cid) {
+        const bytes = this.blocks.get(cid.toString());
+        if (!bytes)
+            throw new Error('block not found ' + cid.toString());
+        return { cid, bytes };
+    }
+    /**
+     * @param {any} cid
+     * @param {Uint8Array} bytes
+     */
+    async put(cid, bytes) {
+        this.blocks.set(cid.toString(), bytes);
+    }
+    *entries() {
+        for (const [str, bytes] of this.blocks) {
+            yield { cid: parse(str), bytes };
+        }
+    }
+}
+export const blocksToCarBlock = async (rootCids, blocks) => {
+    // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
+    let size = 0;
+    if (!Array.isArray(rootCids)) {
+        rootCids = [rootCids];
+    }
+    const headerSize = CBW.headerLength({ roots: rootCids });
+    size += headerSize;
+    if (!Array.isArray(blocks)) {
+        blocks = Array.from(blocks.entries());
+    }
+    for (const { cid, bytes } of blocks) {
+        // console.log(cid, bytes)
+        size += CBW.blockLength({ cid, bytes });
+    }
+    const buffer = new Uint8Array(size);
+    const writer = await CBW.createWriter(buffer, { headerSize });
+    for (const cid of rootCids) {
+        writer.addRoot(cid);
+    }
+    for (const { cid, bytes } of blocks) {
+        writer.write({ cid, bytes });
+    }
+    await writer.close();
+    return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
+};
+export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
+    const encryptionKey = Buffer.from(keyMaterial, 'hex');
+    const encryptedBlocks = [];
+    const theCids = cids;
+    // console.trace('blocksToEncryptedCarBlock', blocks)
+    // for (const { cid } of blocks.entries()) {
+    //   theCids.push(cid.toString())
+    // }
+    // console.log(
+    //   'encrypting',
+    //   theCids.length,
+    //   'blocks',
+    //   theCids.includes(innerBlockStoreClockRootCid.toString()),
+    //   keyMaterial
+    // )
+    // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
+    let last;
+    for await (const block of encrypt({
+        cids: theCids,
+        get: async (cid) => {
+            // console.log('getencrypt', cid)
+            const got = blocks.get(cid);
+            // console.log('got', got)
+            return got.block ? ({ cid, bytes: got.block }) : got;
+        },
+        key: encryptionKey,
+        hasher: sha256,
+        chunker,
+        cache,
+        // codec: dagcbor, // should be crypto?
+        root: innerBlockStoreClockRootCid
+    })) {
+        encryptedBlocks.push(block);
+        last = block;
+    }
+    // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
+    const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
+    return encryptedCar;
+};
+// { root, get, key, cache, chunker, hasher }
+const memoizeDecryptedCarBlocks = new Map();
+export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
+    if (memoizeDecryptedCarBlocks.has(cid.toString())) {
+        return memoizeDecryptedCarBlocks.get(cid.toString());
+    }
+    else {
+        const blocksPromise = (async () => {
+            const decryptionKey = Buffer.from(keyMaterial, 'hex');
+            // console.log('decrypting', keyMaterial, cid.toString())
+            const cids = new Set();
+            const decryptedBlocks = [];
+            for await (const block of decrypt({
+                root: cid,
+                get,
+                key: decryptionKey,
+                chunker,
+                hasher: sha256,
+                cache
+                // codec: dagcbor
+            })) {
+                // console.log('decrypted', block.cid.toString())
+                decryptedBlocks.push(block);
+                cids.add(block.cid.toString());
+            }
+            return { blocks: decryptedBlocks, cids };
+        })();
+        memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
+        return blocksPromise;
+    }
+};

package/dist/sync.js

@@ -1,7 +1,7 @@
 import SimplePeer from 'simple-peer';
 import { parseCID } from './database.js';
 import { decodeEventBlock } from './clock.js';
-import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/base.js';
+import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/utils.js';
 import { CarReader } from '@ipld/car';
 /**
  * @typedef {import('./database.js').Database} Database

package/dist/valet.js

@@ -75,16 +75,22 @@ export class Valet {
             if (this.secondary) {
                 // console.log('getValetBlock secondary', dataCID)
                 try {
+                    // eslint-disable-next-line
                     const { block, reader } = await this.secondary.getLoaderBlock(dataCID);
+                    const writeableCarReader = await this.primary.getWriteableCarReader(reader);
+                    // console.log('getValetBlock secondary', dataCID, block.length)
+                    // eslint-disable-next-line
                     const cids = new Set();
                     for await (const { cid } of reader.entries()) {
                         // console.log(cid, bytes)
                         cids.add(cid.toString());
                     }
-                    reader.get = reader.gat; // some consumers prefer get
+                    // reader.get = reader.gat // some consumers prefer get
                     // console.log('replicating', reader.root)
-                    reader.lastCid = reader.root.cid;
-                    await this.primary.parkCar(reader, [...cids]);
+                    writeableCarReader.lastCid = reader.root.cid;
+                    writeableCarReader.head = [];
+                    await this.primary.parkCar(writeableCarReader, cids).catch(e => console.error('parkCar error', e));
+                    // console.log('FIX THIS', did)
                     return block;
                 }
                 catch (e) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.8.0-dev",
+  "version": "0.8.0-dev.3",
   "description": "Live data for React, accelerated by proofs, powered by IPFS",
   "main": "dist/src/fireproof.js",
   "module": "dist/src/fireproof.mjs",
@@ -42,6 +42,8 @@
     "@ipld/car": "^5.1.0",
     "@ipld/dag-cbor": "^9.0.0",
     "@jsonlines/core": "^1.0.2",
+    "@web3-storage/clock": "^0.3.0",
+    "@web3-storage/w3up-client": "^7.0.0",
     "async": "^3.2.4",
     "charwise": "^3.0.1",
     "cross-fetch": "^3.1.6",

package/src/crypto.js

@@ -16,7 +16,11 @@ const encrypt = async function * ({ get, cids, hasher, key, cache, chunker, root
   let eroot
   for (const string of cids) {
     const cid = CID.parse(string)
-    const unencrypted = await get(cid)
+    let unencrypted = await get(cid)
+    if (!unencrypted.cid) {
+      unencrypted = { cid, bytes: unencrypted }
+    }
+    // console.log('unencrypted', unencrypted)
     const block = await encode({ ...await codec.encrypt({ ...unencrypted, key }), codec, hasher })
     // console.log(`encrypting ${string} as ${block.cid}`)
     yield block

package/src/database.js

@@ -5,6 +5,8 @@ import charwise from 'charwise'
 import { CID } from 'multiformats'
 import { DbIndex as Index } from './db-index.js'
 
+import { Remote } from './remote.js'
+
 // TypeScript Types
 // eslint-disable-next-line no-unused-vars
 // import { CID } from 'multiformats/dist/types/src/cid.js'
@@ -29,14 +31,15 @@ export class Database {
   indexes = new Map()
   rootCache = null
   eventsCache = new Map()
-
+  remote = null
+  name = ''
   constructor (name, config = {}) {
     this.name = name
     this.clock = []
     this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`
     this.blocks = new TransactionBlockstore(name, config)
     this.indexBlocks = new TransactionBlockstore(name ? name + '.indexes' : null, { primary: config.index })
-
+    this.remote = new Remote(this, name, config)
     this.config = config
     // todo we can wait for index blocks elsewhere
     this.ready = Promise.all([this.blocks.ready, this.indexBlocks.ready]).then(([blocksReady, indexReady]) => {
@@ -52,7 +55,7 @@ export class Database {
             clock.add(cid)
           }
           if (header.index) {
-            this.indexBlocks.valet.primary.setCarCidMapCarCid(header.index.car)
+            this.indexBlocks.valet.primary.setLastCar(header.index.car)
             this.indexBlocks.valet.primary.setKeyMaterial(header.index.key)
           }
           if (header.indexes) {
@@ -96,7 +99,7 @@ export class Database {
       name: this.name,
       index: {
         key: this.indexBlocks.valet?.primary.keyMaterial,
-        car: this.indexBlocks.valet?.primary.valetRootCarCid?.toString()
+        car: this.indexBlocks.valet?.primary.lastCar?.toString()
       },
       indexes: [...this.indexes.values()].map(index => index.toJSON())
     }
@@ -112,9 +115,9 @@ export class Database {
     return (clock || this.clock).map(cid => cid.toString())
   }
 
-  maybeSaveClock () {
+  async maybeSaveClock () {
     if (this.name && this.blocks.valet) {
-      this.blocks.valet.saveHeader(this.toHeader())
+      await this.blocks.valet.saveHeader(this.toHeader())
     }
   }
 

package/src/remote.js

@@ -0,0 +1,113 @@
+// when you call database.connect(email)
+// it will return a promise that resolves when the user is logged in
+// and sends you an email
+
+import { create } from '@web3-storage/w3up-client'
+import * as w3clock from '@web3-storage/clock/client'
+import { CID } from 'multiformats'
+
+export class Remote {
+  client = null
+  name = 'unset'
+  config = {}
+
+  constructor (database, name, config) {
+    this.name = name
+    this.config = config
+    this.database = database
+  }
+
+  async clock (cid) {
+    // const did = this.client.currentSpace()
+    const agent = this.client.agent()
+    const head = await w3clock.head({ issuer: agent, with: agent.did(), proofs: [] })
+    console.log('head', head, JSON.stringify(head.root.data.ocm.out))
+    const headCids = head.root.data.ocm.out.ok.head
+    const blocks = await Promise.all([this.database.blocks.get(CID.parse(cid)),
+      ...headCids.map(c => this.database.blocks.get(c))])
+
+    console.log('blocks', blocks)
+    const adv = await w3clock.advance({ issuer: agent, with: agent.did(), proofs: [] }, CID.parse(cid)
+      , { blocks }
+    )
+    console.log('adv', adv, JSON.stringify(adv.root.data.ocm.out))
+    return { head, adv }
+  }
+
+  async sync (cid) {
+    // fetch the remote clock headCids using w3clock.head
+    const agent = this.client.agent()
+    const head = await w3clock.head({ issuer: agent, with: agent.did(), proofs: [] })
+    console.log('head', head, JSON.stringify(head.root.data.ocm.out))
+    const headCids = head.root.data.ocm.out.ok.head
+    const lastSyncHead = await this.database.blocks.valet.primary.getLastSynced()
+    console.log('lastSyncHead', lastSyncHead)
+    const headSet = new Set(headCids.map(c => c.toString()))
+    const lastSyncSet = new Set(lastSyncHead.map(c => c.toString()))
+
+    // are they the same?
+    const same = headSet.size === lastSyncSet.size && [...headSet].every(value => lastSyncSet.has(value))
+
+    // if the headCids and the lastSyncHead are the same, we are in sync and can push
+    if (same) {
+      const currentHead = this.database.clock
+      const currentHeadSet = new Set(currentHead.map(c => c.toString()))
+
+      console.log('synced with cloud', headSet, lastSyncSet)
+
+      // are they the same?
+      const currentSame = headSet.size === currentHeadSet.size && [...headSet].every(value => currentHeadSet.has(value))
+      if (currentSame) {
+        // we are in sync, do nothing
+        return true
+      } else {
+        console.log('push to cloud', headSet, currentHeadSet)
+        // we are ahead of the remote, push our clock
+        // const lastCompact = this.database.blocks.valet.primary.getLastCompact()
+        // get a compact since the last sync
+        console.log('we are ahead of the remote, push our clock')
+        // const compact = this.database.blocks.valet.primary.getCompactSince(lastSyncHead)
+      }
+    } else {
+      // we are behind, fetch the remote
+      console.log('we are behind, fetch the remote')
+    }
+
+    // if it is the same as the local (current metadata carcid? `newValetCidCar` / sync clock), do nothing, we are in sync
+    // if it is the same as our previously pushed clock event, but our local clock is ahead of it, we need to push our clock
+    //               - we can store the previous clock event cid in the metadata
+    // - sending our updates:
+    //   - get the _last_sync and _last_compact values from our metadata
+    //   - if last sync is after last compact
+    //     - make a merged car file for the syncs
+    //   - else
+    //     - upload the car file for the last compact
+    //     - make a merge car file for any uncompacted car files since the last compact, it should base its cidMap on the compact car file (as we go the sync stream will need to track it's own cidMap)
+    //        - if there is only one car file, it is the merge car file (already based on last compact)
+    //     - upload the merge car file
+    //     -  create a new clock block with the current w3clock.head as parent and the merge car file cid as the data
+    //     - update the remote clock with the new clock block (it doesn't need to fetch the car file, and we dont need to store the clock blocks locally, just the most recent one)
+    //
+    // else if the remote head is not contained by our clock, it is is ahead of the local sync clock.
+    // -  get the car file it points to from its data field
+    // - merge to the local clock (park that car so we have both carcid indexes)
+    // - calculate a new root from the merged head, and update the local clock
+  }
+
+  async connect (email) {
+    try {
+      const client = await create()
+      await client.authorize(email)
+      const claims = await client.capability.access.claim()
+      console.log('claims', claims)
+      const space = await client.createSpace('fp.' + this.name)
+      console.log('space', space)
+      await client.setCurrentSpace(space.did())
+      await client.registerSpace(email)
+      this.client = client
+      console.log('client', client)
+    } catch (err) {
+      console.error('registration failed: ', err)
+    }
+  }
+}