@fireproof/core 0.7.3-dev.2 → 0.8.0-dev

package/dist/storage/base.js

@@ -91,10 +91,11 @@ export class Base {
         }
         cidMap.clear();
         const blocks = {
+            head: clock,
             lastCid: clock[0],
             get: cid => allBlocks.get(cid.toString())
         };
-        console.log('compact', this.instanceId, this.name, blocks.lastCid.toString(), dataCids.length);
+        // console.log('compact', this.instanceId, this.name, blocks.lastCid.toString(), dataCids.length)
         await this.parkCar(blocks, dataCids);
     }
     async parkCar(innerBlockstore, cids) {
@@ -111,10 +112,10 @@ export class Base {
             newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore);
         }
         // console.log('new car', newCar.cid.toString())
-        return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids);
+        return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids, innerBlockstore.head);
     }
-    async saveCar(carCid, value, cids) {
-        const newValetCidCar = await this.updateCarCidMap(carCid, cids);
+    async saveCar(carCid, value, cids, head = null) {
+        const newValetCidCar = await this.updateCarCidMap(carCid, cids, head);
         // console.log('writeCars', carCid.toString(), newValetCidCar.cid.toString())
         const carList = [
             {
@@ -134,7 +135,7 @@ export class Base {
         // console.trace('saved car', this.instanceId, this.name, newValetCidCar.cid.toString())
         return newValetCidCar;
     }
-    applyHeaders(headers) {
+    async applyHeaders(headers) {
         // console.log('applyHeaders', headers.index)
         this.headers = headers;
         // console.log('before applied', this.instanceId, this.name, this.keyMaterial, this.valetRootCarCid)
@@ -143,6 +144,9 @@ export class Base {
                 // console.log('applyHeaders', this.instanceId, this.name, header.key, header.car)
                 header.key && this.setKeyMaterial(header.key);
                 this.setCarCidMapCarCid(header.car);
+                const { clock } = await this.readHeaderCar(header.car);
+                // console.log('stored clock', this.name, branch, clock, header)
+                header.clock = clock.map(c => c.toString());
             }
         }
         if (!this.valetRootCarCid) {
@@ -163,18 +167,28 @@ export class Base {
         const headers = {};
         for (const [branch] of Object.entries(this.config.branches)) {
             const got = await this.loadHeader(branch);
+            // const carCid = got.car
             // console.log('getHeaders', this.name, branch, got)
+            // if (got && got.car) {
+            // const { clock } = await this.readHeaderCar(got.car)
+            // console.log('stored clock', this.name, branch, clock)
+            // }
             headers[branch] = got;
         }
-        this.applyHeaders(headers);
+        await this.applyHeaders(headers);
         return headers;
     }
     loadHeader(branch = 'main') {
-        throw new Error('not implemented');
+        if (NOT_IMPL)
+            throw new Error('not implemented');
+        return {};
+    }
+    async getStoredClock(carCid) {
     }
     async saveHeader(header) {
+        // this.clock = header.clock
         // for each branch, save the header
-        // console.log('saveHeader', this.config.branches, header)
+        // console.log('saveHeader', header.clock)
         //  for (const branch of this.branches) {
         //    await this.saveBranchHeader(branch)
         //  }
@@ -233,8 +247,10 @@ export class Base {
     async mapForIPLDHashmapCarCid(carCid) {
         // console.log('mapForIPLDHashmapCarCid', carCid)
         // todo why is this writeable?
-        const carMapReader = await this.getWriteableCarReader(carCid);
-        const indexNode = await load(carMapReader, carMapReader.root.cid, {
+        const { cars, reader: carMapReader } = await this.readHeaderCar(carCid);
+        // this.clock = clock
+        // console.log('mapForIPLDHashmapCarCid', cars)
+        const indexNode = await load(carMapReader, cars, {
             blockHasher: blockOpts.hasher,
             blockCodec: blockOpts.codec
         });
@@ -245,9 +261,20 @@ export class Base {
         }
         return theCarMap;
     }
+    async readHeaderCar(carCid) {
+        const carMapReader = await this.getWriteableCarReader(carCid);
+        // console.log('readHeaderCar', carCid, carMapReader)
+        // now when we load the root cid from the car, we get our new custom root node
+        const bytes = await carMapReader.get(carMapReader.root.cid);
+        const decoded = await Block.decode({ bytes, hasher: blockOpts.hasher, codec: blockOpts.codec });
+        // @ts-ignore
+        const { fp: { cars, clock } } = decoded.value;
+        return { cars, clock, reader: carMapReader };
+    }
     async getWriteableCarReader(carCid) {
         // console.log('getWriteableCarReader', carCid)
         const carMapReader = await this.getCarReader(carCid);
+        // console.log('getWriteableCarReader', carCid, carMapReader)
         const theseWriteableBlocks = new VMemoryBlockstore();
         const combinedReader = {
             blocks: theseWriteableBlocks,
@@ -283,12 +310,16 @@ export class Base {
     }
     async getCarReaderImpl(carCid) {
         carCid = carCid.toString();
+        // console.log('getCarReaderImpl', carCid)
         const carBytes = await this.readCar(carCid);
         // console.log('getCarReader', this.constructor.name, carCid, carBytes.length)
         const reader = await CarReader.fromBytes(carBytes);
+        // console.log('getCarReader', carCid, reader._header)
         if (this.keyMaterial) {
             const roots = await reader.getRoots();
+            // let count = 0
             const readerGetWithCodec = async (cid) => {
+                // console.log('readerGetWithCodec', count++, cid)
                 const got = await reader.get(cid);
                 let useCodec = codec;
                 if (cid.toString().indexOf('bafy') === 0) {
@@ -343,16 +374,16 @@ export class Base {
         }
     }
     writeCars(cars) { }
-    async updateCarCidMap(carCid, cids) {
+    async updateCarCidMap(carCid, cids, head) {
         // this hydrates the map if it has not been hydrated
         const theCarMap = await this.getCidCarMap();
         for (const cid of cids) {
             theCarMap.set(cid, carCid);
         }
         // todo can we debounce this? -- maybe put it into a queue so we can batch it
-        return await this.persistCarMap(theCarMap);
+        return await this.persistCarMap(theCarMap, head);
     }
-    async persistCarMap(theCarMap) {
+    async persistCarMap(theCarMap, head) {
         const ipldLoader = await getEmptyLoader();
         const indexNode = await create(ipldLoader, {
             bitWidth: 4,
@@ -363,14 +394,20 @@ export class Base {
         for (const [key, value] of theCarMap.entries()) {
             await indexNode.set(key, value);
         }
+        // console.log('persistCarMap', indexNode.cid, head)
+        const value = { fp: { cars: indexNode.cid, clock: head } };
+        const header = await Block.encode({ value, hasher: blockOpts.hasher, codec: blockOpts.codec });
+        ipldLoader.blocks.put(header.cid, header.bytes);
         let newValetCidCar;
         if (this.keyMaterial) {
             const cids = [...ipldLoader.blocks.blocks.keys()];
             // console.log('persistCarMap', cids)
-            newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial, cids);
+            // store the clock head and a link to the indexNode.cid in a custom root?
+            newValetCidCar = await blocksToEncryptedCarBlock(header.cid, ipldLoader.blocks, this.keyMaterial, cids);
+            // then put this carcid into the header / w3clock
         }
         else {
-            newValetCidCar = await blocksToCarBlock(indexNode.cid, ipldLoader.blocks);
+            newValetCidCar = await blocksToCarBlock(header.cid, ipldLoader.blocks);
         }
         return newValetCidCar;
     }
@@ -497,6 +534,7 @@ export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
                 cache
                 // codec: dagcbor
             })) {
+                // console.log('decrypted', block.cid.toString())
                 decryptedBlocks.push(block);
                 cids.add(block.cid.toString());
             }

package/dist/storage/ucan.js

@@ -1,40 +0,0 @@
-import fetch from 'cross-fetch';
-import { Base } from './base.js';
-const defaultConfig = {
-    upload: () => { },
-    url: (cid) => `https://${cid}.ipfs.w3s.link/`
-};
-export class UCAN extends Base {
-    constructor(name, config = {}) {
-        super(name, Object.assign({}, defaultConfig, config));
-    }
-    async writeCars(cars) {
-        if (this.config.readonly)
-            return;
-        for (const { cid, bytes } of cars) {
-            console.log(`write UCAN ${cid}, ${bytes.length} bytes`);
-            const upCid = await this.config.upload(bytes);
-            console.log(`wrote UCAN ${cid}, ${upCid}`);
-            // if (!response.ok) throw new Error(`An error occurred: ${response.statusText}`)
-        }
-    }
-    async readCar(carCid) {
-        const carURL = this.config.url(carCid);
-        const response = await fetch(carURL);
-        if (!response.ok)
-            throw new Error(`An error occurred: ${response.statusText}`);
-        const got = await response.arrayBuffer();
-        return new Uint8Array(got);
-    }
-    async loadHeader(branch = 'main') {
-        return headerMock.get(branch);
-    }
-    async writeHeader(branch, header) {
-        if (this.config.readonly)
-            return;
-        const pHeader = this.prepareHeader(header);
-        // console.log('writeHeader rt', branch, pHeader)
-        headerMock.set(branch, pHeader);
-    }
-}
-const headerMock = new Map();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.7.3-dev.2",
+  "version": "0.8.0-dev",
   "description": "Live data for React, accelerated by proofs, powered by IPFS",
   "main": "dist/src/fireproof.js",
   "module": "dist/src/fireproof.mjs",
@@ -42,10 +42,6 @@
     "@ipld/car": "^5.1.0",
     "@ipld/dag-cbor": "^9.0.0",
     "@jsonlines/core": "^1.0.2",
-    "@ucanto/principal": "^8.0.0",
-    "@ucanto/server": "^8.0.1",
-    "@ucanto/transport": "^8.0.0",
-    "@ucanto/validator": "^8.0.0",
     "async": "^3.2.4",
     "charwise": "^3.0.1",
     "cross-fetch": "^3.1.6",

package/src/blockstore.js

@@ -221,6 +221,9 @@ export const doTransaction = async (label, blockstore, doFun, doSync = true) =>
   const innerBlockstore = blockstore.begin(label)
   try {
     const result = await doFun(innerBlockstore)
+    // console.log('doTransaction', label, 'result', result.head)
+    if (result && result.head) { innerBlockstore.head = result.head }
+    // pass the latest clock head for writing to the valet
     // @ts-ignore
     await blockstore.commit(innerBlockstore, doSync)
     return result
@@ -236,6 +239,7 @@ export const doTransaction = async (label, blockstore, doFun, doSync = true) =>
 export class InnerBlockstore {
   /** @type {Map<string, Uint8Array>} */
   blocks = new Map()
+  head = []
   lastCid = null
   label = ''
   parentBlockstore = null

package/src/database.js

@@ -92,7 +92,7 @@ export class Database {
 
   toHeader () {
     return {
-      clock: this.clockToJSON(),
+      // clock: this.clockToJSON(),
       name: this.name,
       index: {
         key: this.indexBlocks.valet?.primary.keyMaterial,
@@ -277,11 +277,13 @@ export class Database {
    * @memberof Fireproof
    * @instance
    */
-  async put ({ _id, _proof, ...doc }) {
+  async put ({ _id, _proof, _clock, ...doc }) {
     await this.ready
     const id = _id || 'f' + Math.random().toString(36).slice(2)
+    doc = JSON.parse(JSON.stringify(doc))
+    if (_clock) doc._clock = _clock
     await this.runValidation({ _id: id, ...doc })
-    return await this.putToProllyTree({ key: id, value: doc }, doc._clock)
+    return await this.putToProllyTree({ key: id, value: doc }, _clock)
   }
 
   /**

package/src/fireproof.js

@@ -62,6 +62,7 @@ export class Fireproof {
 
   static snapshot (database, clock) {
     const definition = database.toJSON()
+    definition.clock = database.clockToJSON()
     if (clock) {
       definition.clock = clock.map(c => parseCID(c))
       definition.indexes.forEach(index => {

package/src/loader.js

@@ -1,6 +1,5 @@
 import { Browser } from './storage/browser.js'
 import { Rest } from './storage/rest.js'
-import { UCAN } from './storage/ucan.js'
 
 export const Loader = {
   appropriate: (name, config = {}) => {
@@ -12,10 +11,6 @@ export const Loader = {
       return new Rest(name, config)
     }
 
-    if (config.type === 'ucan') {
-      return new UCAN(name, config)
-    }
-
     return new Browser(name, config)
   }
 }

package/src/prolly.js

@@ -307,7 +307,7 @@ export async function root (inBlocks, head, doFull = false) {
         bigPut(nb)
       }
       // console.log('root root', newProllyRootNode.constructor.name, newProllyRootNode)
-      return { clockCIDs, node: newProllyRootNode }
+      return { clockCIDs, node: newProllyRootNode, head }
     },
     false
   )

package/src/storage/base.js

@@ -101,10 +101,11 @@ export class Base {
     }
     cidMap.clear()
     const blocks = {
+      head: clock,
       lastCid: clock[0],
       get: cid => allBlocks.get(cid.toString())
     }
-    console.log('compact', this.instanceId, this.name, blocks.lastCid.toString(), dataCids.length)
+    // console.log('compact', this.instanceId, this.name, blocks.lastCid.toString(), dataCids.length)
     await this.parkCar(blocks, dataCids)
   }
 
@@ -120,11 +121,11 @@ export class Base {
       newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
     }
     // console.log('new car', newCar.cid.toString())
-    return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids)
+    return await this.saveCar(newCar.cid.toString(), newCar.bytes, cids, innerBlockstore.head)
   }
 
-  async saveCar (carCid, value, cids) {
-    const newValetCidCar = await this.updateCarCidMap(carCid, cids)
+  async saveCar (carCid, value, cids, head = null) {
+    const newValetCidCar = await this.updateCarCidMap(carCid, cids, head)
     // console.log('writeCars', carCid.toString(), newValetCidCar.cid.toString())
     const carList = [
       {
@@ -146,7 +147,7 @@ export class Base {
     return newValetCidCar
   }
 
-  applyHeaders (headers) {
+  async applyHeaders (headers) {
     // console.log('applyHeaders', headers.index)
     this.headers = headers
     // console.log('before applied', this.instanceId, this.name, this.keyMaterial, this.valetRootCarCid)
@@ -155,6 +156,9 @@ export class Base {
         // console.log('applyHeaders', this.instanceId, this.name, header.key, header.car)
         header.key && this.setKeyMaterial(header.key)
         this.setCarCidMapCarCid(header.car)
+        const { clock } = await this.readHeaderCar(header.car)
+        // console.log('stored clock', this.name, branch, clock, header)
+        header.clock = clock.map(c => c.toString())
       }
     }
     if (!this.valetRootCarCid) {
@@ -175,20 +179,32 @@ export class Base {
     const headers = {}
     for (const [branch] of Object.entries(this.config.branches)) {
       const got = await this.loadHeader(branch)
+      // const carCid = got.car
       // console.log('getHeaders', this.name, branch, got)
+      // if (got && got.car) {
+      // const { clock } = await this.readHeaderCar(got.car)
+      // console.log('stored clock', this.name, branch, clock)
+      // }
+
       headers[branch] = got
     }
-    this.applyHeaders(headers)
+    await this.applyHeaders(headers)
     return headers
   }
 
   loadHeader (branch = 'main') {
-    throw new Error('not implemented')
+    if (NOT_IMPL) throw new Error('not implemented')
+    return {}
+  }
+
+  async getStoredClock (carCid) {
+
   }
 
   async saveHeader (header) {
+    // this.clock = header.clock
     // for each branch, save the header
-    // console.log('saveHeader', this.config.branches, header)
+    // console.log('saveHeader', header.clock)
     //  for (const branch of this.branches) {
     //    await this.saveBranchHeader(branch)
     //  }
@@ -251,8 +267,13 @@ export class Base {
   async mapForIPLDHashmapCarCid (carCid) {
     // console.log('mapForIPLDHashmapCarCid', carCid)
     // todo why is this writeable?
-    const carMapReader = await this.getWriteableCarReader(carCid)
-    const indexNode = await load(carMapReader, carMapReader.root.cid, {
+    const { cars, reader: carMapReader } = await this.readHeaderCar(carCid)
+
+    // this.clock = clock
+
+    // console.log('mapForIPLDHashmapCarCid', cars)
+
+    const indexNode = await load(carMapReader, cars, {
       blockHasher: blockOpts.hasher,
       blockCodec: blockOpts.codec
     })
@@ -264,9 +285,21 @@ export class Base {
     return theCarMap
   }
 
+  async readHeaderCar (carCid) {
+    const carMapReader = await this.getWriteableCarReader(carCid)
+    // console.log('readHeaderCar', carCid, carMapReader)
+    // now when we load the root cid from the car, we get our new custom root node
+    const bytes = await carMapReader.get(carMapReader.root.cid)
+    const decoded = await Block.decode({ bytes, hasher: blockOpts.hasher, codec: blockOpts.codec })
+    // @ts-ignore
+    const { fp: { cars, clock } } = decoded.value
+    return { cars, clock, reader: carMapReader }
+  }
+
   async getWriteableCarReader (carCid) {
     // console.log('getWriteableCarReader', carCid)
     const carMapReader = await this.getCarReader(carCid)
+    // console.log('getWriteableCarReader', carCid, carMapReader)
     const theseWriteableBlocks = new VMemoryBlockstore()
     const combinedReader = {
       blocks: theseWriteableBlocks,
@@ -303,12 +336,16 @@ export class Base {
 
   async getCarReaderImpl (carCid) {
     carCid = carCid.toString()
+    // console.log('getCarReaderImpl', carCid)
     const carBytes = await this.readCar(carCid)
     // console.log('getCarReader', this.constructor.name, carCid, carBytes.length)
     const reader = await CarReader.fromBytes(carBytes)
+    // console.log('getCarReader', carCid, reader._header)
     if (this.keyMaterial) {
       const roots = await reader.getRoots()
+      // let count = 0
       const readerGetWithCodec = async cid => {
+        // console.log('readerGetWithCodec', count++, cid)
         const got = await reader.get(cid)
         let useCodec = codec
         if (cid.toString().indexOf('bafy') === 0) {
@@ -365,17 +402,17 @@ export class Base {
 
   writeCars (cars) {}
 
-  async updateCarCidMap (carCid, cids) {
+  async updateCarCidMap (carCid, cids, head) {
     // this hydrates the map if it has not been hydrated
     const theCarMap = await this.getCidCarMap()
     for (const cid of cids) {
       theCarMap.set(cid, carCid)
     }
     // todo can we debounce this? -- maybe put it into a queue so we can batch it
-    return await this.persistCarMap(theCarMap)
+    return await this.persistCarMap(theCarMap, head)
   }
 
-  async persistCarMap (theCarMap) {
+  async persistCarMap (theCarMap, head) {
     const ipldLoader = await getEmptyLoader()
     const indexNode = await create(ipldLoader, {
       bitWidth: 4,
@@ -388,13 +425,21 @@ export class Base {
       await indexNode.set(key, value)
     }
 
+    // console.log('persistCarMap', indexNode.cid, head)
+    const value = { fp: { cars: indexNode.cid, clock: head } }
+    const header = await Block.encode({ value, hasher: blockOpts.hasher, codec: blockOpts.codec })
+    ipldLoader.blocks.put(header.cid, header.bytes)
+
     let newValetCidCar
     if (this.keyMaterial) {
       const cids = [...ipldLoader.blocks.blocks.keys()]
       // console.log('persistCarMap', cids)
-      newValetCidCar = await blocksToEncryptedCarBlock(indexNode.cid, ipldLoader.blocks, this.keyMaterial, cids)
+      // store the clock head and a link to the indexNode.cid in a custom root?
+
+      newValetCidCar = await blocksToEncryptedCarBlock(header.cid, ipldLoader.blocks, this.keyMaterial, cids)
+      // then put this carcid into the header / w3clock
     } else {
-      newValetCidCar = await blocksToCarBlock(indexNode.cid, ipldLoader.blocks)
+      newValetCidCar = await blocksToCarBlock(header.cid, ipldLoader.blocks)
     }
     return newValetCidCar
   }
@@ -529,6 +574,7 @@ export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
         cache
         // codec: dagcbor
       })) {
+        // console.log('decrypted', block.cid.toString())
         decryptedBlocks.push(block)
         cids.add(block.cid.toString())
       }

package/src/storage/ucan.js

@@ -1,45 +0,0 @@
-import fetch from 'cross-fetch'
-import { Base } from './base.js'
-
-const defaultConfig = {
-  upload: () => {},
-  url: (cid) => `https://${cid}.ipfs.w3s.link/`
-}
-
-export class UCAN extends Base {
-  constructor (name, config = {}) {
-    super(name, Object.assign({}, defaultConfig, config))
-  }
-
-  async writeCars (cars) {
-    if (this.config.readonly) return
-    for (const { cid, bytes } of cars) {
-      console.log(`write UCAN ${cid}, ${bytes.length} bytes`)
-      const upCid = await this.config.upload(bytes)
-      console.log(`wrote UCAN ${cid}, ${upCid}`)
-      // if (!response.ok) throw new Error(`An error occurred: ${response.statusText}`)
-    }
-  }
-
-  async readCar (carCid) {
-    const carURL = this.config.url(carCid)
-    const response = await fetch(carURL)
-    if (!response.ok) throw new Error(`An error occurred: ${response.statusText}`)
-    const got = await response.arrayBuffer()
-    return new Uint8Array(got)
-  }
-
-  async loadHeader (branch = 'main') {
-    return headerMock.get(branch)
-  }
-
-  async writeHeader (branch, header) {
-    if (this.config.readonly) return
-    const pHeader = this.prepareHeader(header)
-    // console.log('writeHeader rt', branch, pHeader)
-
-    headerMock.set(branch, pHeader)
-  }
-}
-
-const headerMock = new Map()