@fireproof/core 0.7.3-dev.2 → 0.8.0-dev.2

package/dist/storage/blocksToEncryptedCarBlock.js

@@ -0,0 +1,144 @@
+import * as CBW from '@ipld/car/buffer-writer';
+import * as raw from 'multiformats/codecs/raw';
+import { encrypt, decrypt } from '../crypto.js';
+import { parse } from 'multiformats/link';
+import { sha256 } from 'multiformats/hashes/sha2';
+import * as Block from 'multiformats/block';
+import { Buffer } from 'buffer';
+// @ts-ignore
+import { bf } from 'prolly-trees/utils';
+// @ts-ignore
+import { nocache as cache } from 'prolly-trees/cache';
+const chunker = bf(30);
+export async function getEmptyLoader() {
+    const theseWriteableBlocks = new VMemoryBlockstore();
+    return {
+        blocks: theseWriteableBlocks,
+        put: async (cid, bytes) => {
+            return await theseWriteableBlocks.put(cid, bytes);
+        },
+        get: async (cid) => {
+            const got = await theseWriteableBlocks.get(cid);
+            return got.bytes;
+        }
+    };
+}
+export class VMemoryBlockstore {
+    /** @type {Map<string, Uint8Array>} */
+    blocks = new Map();
+    instanceId = Math.random().toString(36).slice(2);
+    async get(cid) {
+        const bytes = this.blocks.get(cid.toString());
+        if (!bytes)
+            throw new Error('block not found ' + cid.toString());
+        return { cid, bytes };
+    }
+    /**
+     * @param {any} cid
+     * @param {Uint8Array} bytes
+     */
+    async put(cid, bytes) {
+        this.blocks.set(cid.toString(), bytes);
+    }
+    *entries() {
+        for (const [str, bytes] of this.blocks) {
+            yield { cid: parse(str), bytes };
+        }
+    }
+}
+export const blocksToCarBlock = async (rootCids, blocks) => {
+    // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
+    let size = 0;
+    if (!Array.isArray(rootCids)) {
+        rootCids = [rootCids];
+    }
+    const headerSize = CBW.headerLength({ roots: rootCids });
+    size += headerSize;
+    if (!Array.isArray(blocks)) {
+        blocks = Array.from(blocks.entries());
+    }
+    for (const { cid, bytes } of blocks) {
+        // console.log(cid, bytes)
+        size += CBW.blockLength({ cid, bytes });
+    }
+    const buffer = new Uint8Array(size);
+    const writer = await CBW.createWriter(buffer, { headerSize });
+    for (const cid of rootCids) {
+        writer.addRoot(cid);
+    }
+    for (const { cid, bytes } of blocks) {
+        writer.write({ cid, bytes });
+    }
+    await writer.close();
+    return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
+};
+export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
+    const encryptionKey = Buffer.from(keyMaterial, 'hex');
+    const encryptedBlocks = [];
+    const theCids = cids;
+    // console.trace('blocksToEncryptedCarBlock', blocks)
+    // for (const { cid } of blocks.entries()) {
+    //   theCids.push(cid.toString())
+    // }
+    // console.log(
+    //   'encrypting',
+    //   theCids.length,
+    //   'blocks',
+    //   theCids.includes(innerBlockStoreClockRootCid.toString()),
+    //   keyMaterial
+    // )
+    // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
+    let last;
+    for await (const block of encrypt({
+        cids: theCids,
+        get: async (cid) => {
+            // console.log('getencrypt', cid)
+            const got = blocks.get(cid);
+            // console.log('got', got)
+            return got.block ? ({ cid, bytes: got.block }) : got;
+        },
+        key: encryptionKey,
+        hasher: sha256,
+        chunker,
+        cache,
+        // codec: dagcbor, // should be crypto?
+        root: innerBlockStoreClockRootCid
+    })) {
+        encryptedBlocks.push(block);
+        last = block;
+    }
+    // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
+    const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
+    return encryptedCar;
+};
+// { root, get, key, cache, chunker, hasher }
+const memoizeDecryptedCarBlocks = new Map();
+export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
+    if (memoizeDecryptedCarBlocks.has(cid.toString())) {
+        return memoizeDecryptedCarBlocks.get(cid.toString());
+    }
+    else {
+        const blocksPromise = (async () => {
+            const decryptionKey = Buffer.from(keyMaterial, 'hex');
+            // console.log('decrypting', keyMaterial, cid.toString())
+            const cids = new Set();
+            const decryptedBlocks = [];
+            for await (const block of decrypt({
+                root: cid,
+                get,
+                key: decryptionKey,
+                chunker,
+                hasher: sha256,
+                cache
+                // codec: dagcbor
+            })) {
+                // console.log('decrypted', block.cid.toString())
+                decryptedBlocks.push(block);
+                cids.add(block.cid.toString());
+            }
+            return { blocks: decryptedBlocks, cids };
+        })();
+        memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
+        return blocksPromise;
+    }
+};

package/dist/storage/browser.js

@@ -52,7 +52,7 @@ export class Browser extends Base {
         if (this.config.readonly)
             return;
         try {
-            return localStorage.setItem(this.headerKey(branch), this.prepareHeader(header));
+            return localStorage.setItem(this.headerKey(branch), header);
         }
         catch (e) { }
     }

package/dist/storage/filesystem.js

@@ -40,12 +40,11 @@ export class Filesystem extends Base {
         return JSON.parse(header);
     }
     async writeHeader(branch, header) {
-        // console.log('saveHeader', this.isBrowser)
+        // console.log('saveHeader fs', header)
         if (this.config.readonly)
             return;
-        const pHeader = this.prepareHeader(header);
         // console.log('writeHeader fs', branch, pHeader)
-        await writeSync(this.headerFilename(branch), pHeader);
+        await writeSync(this.headerFilename(branch), header);
     }
     headerFilename(branch = 'main') {
         // console.log('headerFilename', this.config.dataDir, this.name)

package/dist/storage/rest.js

@@ -45,11 +45,10 @@ export class Rest extends Base {
     async writeHeader(branch, header) {
         if (this.config.readonly)
             return;
-        const pHeader = this.prepareHeader(header);
         // console.log('writeHeader rt', branch, pHeader)
         const response = await fetch(this.headerURL(branch), {
             method: 'PUT',
-            body: pHeader,
+            body: header,
             headers: { 'Content-Type': 'application/json' }
         });
         if (!response.ok)

package/dist/storage/ucan.js

@@ -1,40 +0,0 @@
-import fetch from 'cross-fetch';
-import { Base } from './base.js';
-const defaultConfig = {
-    upload: () => { },
-    url: (cid) => `https://${cid}.ipfs.w3s.link/`
-};
-export class UCAN extends Base {
-    constructor(name, config = {}) {
-        super(name, Object.assign({}, defaultConfig, config));
-    }
-    async writeCars(cars) {
-        if (this.config.readonly)
-            return;
-        for (const { cid, bytes } of cars) {
-            console.log(`write UCAN ${cid}, ${bytes.length} bytes`);
-            const upCid = await this.config.upload(bytes);
-            console.log(`wrote UCAN ${cid}, ${upCid}`);
-            // if (!response.ok) throw new Error(`An error occurred: ${response.statusText}`)
-        }
-    }
-    async readCar(carCid) {
-        const carURL = this.config.url(carCid);
-        const response = await fetch(carURL);
-        if (!response.ok)
-            throw new Error(`An error occurred: ${response.statusText}`);
-        const got = await response.arrayBuffer();
-        return new Uint8Array(got);
-    }
-    async loadHeader(branch = 'main') {
-        return headerMock.get(branch);
-    }
-    async writeHeader(branch, header) {
-        if (this.config.readonly)
-            return;
-        const pHeader = this.prepareHeader(header);
-        // console.log('writeHeader rt', branch, pHeader)
-        headerMock.set(branch, pHeader);
-    }
-}
-const headerMock = new Map();

package/dist/storage/utils.js

@@ -0,0 +1,144 @@
+import * as CBW from '@ipld/car/buffer-writer';
+import * as raw from 'multiformats/codecs/raw';
+import { encrypt, decrypt } from '../crypto.js';
+import { parse } from 'multiformats/link';
+import { sha256 } from 'multiformats/hashes/sha2';
+import * as Block from 'multiformats/block';
+import { Buffer } from 'buffer';
+// @ts-ignore
+import { bf } from 'prolly-trees/utils';
+// @ts-ignore
+import { nocache as cache } from 'prolly-trees/cache';
+const chunker = bf(30);
+export async function getEmptyLoader() {
+    const theseWriteableBlocks = new VMemoryBlockstore();
+    return {
+        blocks: theseWriteableBlocks,
+        put: async (cid, bytes) => {
+            return await theseWriteableBlocks.put(cid, bytes);
+        },
+        get: async (cid) => {
+            const got = await theseWriteableBlocks.get(cid);
+            return got.bytes;
+        }
+    };
+}
+export class VMemoryBlockstore {
+    /** @type {Map<string, Uint8Array>} */
+    blocks = new Map();
+    instanceId = Math.random().toString(36).slice(2);
+    async get(cid) {
+        const bytes = this.blocks.get(cid.toString());
+        if (!bytes)
+            throw new Error('block not found ' + cid.toString());
+        return { cid, bytes };
+    }
+    /**
+     * @param {any} cid
+     * @param {Uint8Array} bytes
+     */
+    async put(cid, bytes) {
+        this.blocks.set(cid.toString(), bytes);
+    }
+    *entries() {
+        for (const [str, bytes] of this.blocks) {
+            yield { cid: parse(str), bytes };
+        }
+    }
+}
+export const blocksToCarBlock = async (rootCids, blocks) => {
+    // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
+    let size = 0;
+    if (!Array.isArray(rootCids)) {
+        rootCids = [rootCids];
+    }
+    const headerSize = CBW.headerLength({ roots: rootCids });
+    size += headerSize;
+    if (!Array.isArray(blocks)) {
+        blocks = Array.from(blocks.entries());
+    }
+    for (const { cid, bytes } of blocks) {
+        // console.log(cid, bytes)
+        size += CBW.blockLength({ cid, bytes });
+    }
+    const buffer = new Uint8Array(size);
+    const writer = await CBW.createWriter(buffer, { headerSize });
+    for (const cid of rootCids) {
+        writer.addRoot(cid);
+    }
+    for (const { cid, bytes } of blocks) {
+        writer.write({ cid, bytes });
+    }
+    await writer.close();
+    return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
+};
+export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
+    const encryptionKey = Buffer.from(keyMaterial, 'hex');
+    const encryptedBlocks = [];
+    const theCids = cids;
+    // console.trace('blocksToEncryptedCarBlock', blocks)
+    // for (const { cid } of blocks.entries()) {
+    //   theCids.push(cid.toString())
+    // }
+    // console.log(
+    //   'encrypting',
+    //   theCids.length,
+    //   'blocks',
+    //   theCids.includes(innerBlockStoreClockRootCid.toString()),
+    //   keyMaterial
+    // )
+    // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
+    let last;
+    for await (const block of encrypt({
+        cids: theCids,
+        get: async (cid) => {
+            // console.log('getencrypt', cid)
+            const got = blocks.get(cid);
+            // console.log('got', got)
+            return got.block ? ({ cid, bytes: got.block }) : got;
+        },
+        key: encryptionKey,
+        hasher: sha256,
+        chunker,
+        cache,
+        // codec: dagcbor, // should be crypto?
+        root: innerBlockStoreClockRootCid
+    })) {
+        encryptedBlocks.push(block);
+        last = block;
+    }
+    // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
+    const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
+    return encryptedCar;
+};
+// { root, get, key, cache, chunker, hasher }
+const memoizeDecryptedCarBlocks = new Map();
+export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
+    if (memoizeDecryptedCarBlocks.has(cid.toString())) {
+        return memoizeDecryptedCarBlocks.get(cid.toString());
+    }
+    else {
+        const blocksPromise = (async () => {
+            const decryptionKey = Buffer.from(keyMaterial, 'hex');
+            // console.log('decrypting', keyMaterial, cid.toString())
+            const cids = new Set();
+            const decryptedBlocks = [];
+            for await (const block of decrypt({
+                root: cid,
+                get,
+                key: decryptionKey,
+                chunker,
+                hasher: sha256,
+                cache
+                // codec: dagcbor
+            })) {
+                // console.log('decrypted', block.cid.toString())
+                decryptedBlocks.push(block);
+                cids.add(block.cid.toString());
+            }
+            return { blocks: decryptedBlocks, cids };
+        })();
+        memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
+        return blocksPromise;
+    }
+};

package/dist/sync.js

@@ -1,7 +1,7 @@
 import SimplePeer from 'simple-peer';
 import { parseCID } from './database.js';
 import { decodeEventBlock } from './clock.js';
-import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/base.js';
+import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/utils.js';
 import { CarReader } from '@ipld/car';
 /**
  * @typedef {import('./database.js').Database} Database

package/dist/valet.js

@@ -75,16 +75,22 @@ export class Valet {
             if (this.secondary) {
                 // console.log('getValetBlock secondary', dataCID)
                 try {
+                    // eslint-disable-next-line
                     const { block, reader } = await this.secondary.getLoaderBlock(dataCID);
+                    const writeableCarReader = await this.primary.getWriteableCarReader(reader);
+                    // console.log('getValetBlock secondary', dataCID, block.length)
+                    // eslint-disable-next-line
                     const cids = new Set();
                     for await (const { cid } of reader.entries()) {
                         // console.log(cid, bytes)
                         cids.add(cid.toString());
                     }
-                    reader.get = reader.gat; // some consumers prefer get
+                    // reader.get = reader.gat // some consumers prefer get
                     // console.log('replicating', reader.root)
-                    reader.lastCid = reader.root.cid;
-                    await this.primary.parkCar(reader, [...cids]);
+                    writeableCarReader.lastCid = reader.root.cid;
+                    writeableCarReader.head = [];
+                    await this.primary.parkCar(writeableCarReader, cids).catch(e => console.error('parkCar error', e));
+                    // console.log('FIX THIS', did)
                     return block;
                 }
                 catch (e) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.7.3-dev.2",
+  "version": "0.8.0-dev.2",
   "description": "Live data for React, accelerated by proofs, powered by IPFS",
   "main": "dist/src/fireproof.js",
   "module": "dist/src/fireproof.mjs",
@@ -42,10 +42,8 @@
     "@ipld/car": "^5.1.0",
     "@ipld/dag-cbor": "^9.0.0",
     "@jsonlines/core": "^1.0.2",
-    "@ucanto/principal": "^8.0.0",
-    "@ucanto/server": "^8.0.1",
-    "@ucanto/transport": "^8.0.0",
-    "@ucanto/validator": "^8.0.0",
+    "@web3-storage/clock": "^0.3.0",
+    "@web3-storage/w3up-client": "^7.0.0",
     "async": "^3.2.4",
     "charwise": "^3.0.1",
     "cross-fetch": "^3.1.6",

package/src/blockstore.js

@@ -221,6 +221,9 @@ export const doTransaction = async (label, blockstore, doFun, doSync = true) =>
   const innerBlockstore = blockstore.begin(label)
   try {
     const result = await doFun(innerBlockstore)
+    // console.log('doTransaction', label, 'result', result.head)
+    if (result && result.head) { innerBlockstore.head = result.head }
+    // pass the latest clock head for writing to the valet
     // @ts-ignore
     await blockstore.commit(innerBlockstore, doSync)
     return result
@@ -236,6 +239,7 @@ export const doTransaction = async (label, blockstore, doFun, doSync = true) =>
 export class InnerBlockstore {
   /** @type {Map<string, Uint8Array>} */
   blocks = new Map()
+  head = []
   lastCid = null
   label = ''
   parentBlockstore = null

package/src/crypto.js

@@ -16,7 +16,11 @@ const encrypt = async function * ({ get, cids, hasher, key, cache, chunker, root
   let eroot
   for (const string of cids) {
     const cid = CID.parse(string)
-    const unencrypted = await get(cid)
+    let unencrypted = await get(cid)
+    if (!unencrypted.cid) {
+      unencrypted = { cid, bytes: unencrypted }
+    }
+    // console.log('unencrypted', unencrypted)
     const block = await encode({ ...await codec.encrypt({ ...unencrypted, key }), codec, hasher })
     // console.log(`encrypting ${string} as ${block.cid}`)
     yield block

package/src/database.js

@@ -5,6 +5,8 @@ import charwise from 'charwise'
 import { CID } from 'multiformats'
 import { DbIndex as Index } from './db-index.js'
 
+import { Remote } from './remote.js'
+
 // TypeScript Types
 // eslint-disable-next-line no-unused-vars
 // import { CID } from 'multiformats/dist/types/src/cid.js'
@@ -29,14 +31,15 @@ export class Database {
   indexes = new Map()
   rootCache = null
   eventsCache = new Map()
-
+  remote = null
+  name = ''
   constructor (name, config = {}) {
     this.name = name
     this.clock = []
     this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`
     this.blocks = new TransactionBlockstore(name, config)
     this.indexBlocks = new TransactionBlockstore(name ? name + '.indexes' : null, { primary: config.index })
-
+    this.remote = new Remote(this, name, config)
     this.config = config
     // todo we can wait for index blocks elsewhere
     this.ready = Promise.all([this.blocks.ready, this.indexBlocks.ready]).then(([blocksReady, indexReady]) => {
@@ -52,7 +55,7 @@ export class Database {
             clock.add(cid)
           }
           if (header.index) {
-            this.indexBlocks.valet.primary.setCarCidMapCarCid(header.index.car)
+            this.indexBlocks.valet.primary.setLastCar(header.index.car)
             this.indexBlocks.valet.primary.setKeyMaterial(header.index.key)
           }
           if (header.indexes) {
@@ -92,11 +95,11 @@ export class Database {
 
   toHeader () {
     return {
-      clock: this.clockToJSON(),
+      // clock: this.clockToJSON(),
       name: this.name,
       index: {
         key: this.indexBlocks.valet?.primary.keyMaterial,
-        car: this.indexBlocks.valet?.primary.valetRootCarCid?.toString()
+        car: this.indexBlocks.valet?.primary.lastCar?.toString()
       },
       indexes: [...this.indexes.values()].map(index => index.toJSON())
     }
@@ -112,9 +115,9 @@ export class Database {
     return (clock || this.clock).map(cid => cid.toString())
   }
 
-  maybeSaveClock () {
+  async maybeSaveClock () {
     if (this.name && this.blocks.valet) {
-      this.blocks.valet.saveHeader(this.toHeader())
+      await this.blocks.valet.saveHeader(this.toHeader())
     }
   }
 
@@ -277,11 +280,13 @@ export class Database {
    * @memberof Fireproof
    * @instance
    */
-  async put ({ _id, _proof, ...doc }) {
+  async put ({ _id, _proof, _clock, ...doc }) {
     await this.ready
     const id = _id || 'f' + Math.random().toString(36).slice(2)
+    doc = JSON.parse(JSON.stringify(doc))
+    if (_clock) doc._clock = _clock
     await this.runValidation({ _id: id, ...doc })
-    return await this.putToProllyTree({ key: id, value: doc }, doc._clock)
+    return await this.putToProllyTree({ key: id, value: doc }, _clock)
   }
 
   /**

package/src/fireproof.js

@@ -62,6 +62,7 @@ export class Fireproof {
 
   static snapshot (database, clock) {
     const definition = database.toJSON()
+    definition.clock = database.clockToJSON()
     if (clock) {
       definition.clock = clock.map(c => parseCID(c))
       definition.indexes.forEach(index => {

package/src/loader.js

@@ -1,6 +1,5 @@
 import { Browser } from './storage/browser.js'
 import { Rest } from './storage/rest.js'
-import { UCAN } from './storage/ucan.js'
 
 export const Loader = {
   appropriate: (name, config = {}) => {
@@ -12,10 +11,6 @@ export const Loader = {
       return new Rest(name, config)
     }
 
-    if (config.type === 'ucan') {
-      return new UCAN(name, config)
-    }
-
     return new Browser(name, config)
   }
 }

package/src/prolly.js

@@ -307,7 +307,7 @@ export async function root (inBlocks, head, doFull = false) {
         bigPut(nb)
       }
       // console.log('root root', newProllyRootNode.constructor.name, newProllyRootNode)
-      return { clockCIDs, node: newProllyRootNode }
+      return { clockCIDs, node: newProllyRootNode, head }
     },
     false
   )