@fireproof/core 0.4.0 → 0.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Realtime database for IPFS",
   "main": "dist/src/fireproof.js",
   "module": "dist/src/fireproof.mjs",
@@ -11,7 +11,7 @@
     "keygen": "node scripts/keygen.js",
     "test": "standard && npm run test:unencrypted && npm run test:mocha",
     "test:unencrypted": "NO_ENCRYPT=true npm run test:mocha",
-    "test:mocha": "mocha --reporter list test/*.test.js",
+    "test:mocha": "mocha test/*.test.js",
     "test:watch": "npm run test:mocha -- -w --parallel",
     "coverage": "c8 -r html -r text npm test",
     "prepublishOnly": "cp ../../README.md . && npm run build",

package/src/database.js

@@ -117,7 +117,7 @@ export class Database {
       // console.log('change rows', this.instanceId, rows)
     } else {
       const allResp = await getAll(this.blocks, this.clock)
-      rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value })))
+      rows = allResp.result.map(({ key, value }) => decodeEvent({ key, value }))
       dataCIDs = allResp.cids
       // console.log('dbdoc rows', this.instanceId, rows)
     }
@@ -130,7 +130,9 @@ export class Database {
 
   async allDocuments () {
     const allResp = await getAll(this.blocks, this.clock)
-    const rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value }))).map(({ key, value }) => ({ key, value: { _id: key, ...value } }))
+    const rows = allResp.result
+      .map(({ key, value }) => decodeEvent({ key, value }))
+      .map(({ key, value }) => ({ key, value: { _id: key, ...value } }))
     return {
       rows,
       clock: this.clockToJSON(),
@@ -138,6 +140,15 @@ export class Database {
     }
   }
 
+  async allCIDs () {
+    const allResp = await getAll(this.blocks, this.clock)
+    const cids = await cidsToProof(allResp.cids)
+    const clockCids = await cidsToProof(allResp.clockCIDs)
+    // console.log('allcids', cids, clockCids)
+    // todo we need to put the clock head as the last block in the encrypted car
+    return [...cids, ...clockCids] // need a single block version of clock head, maybe an encoded block for it
+  }
+
   /**
    * Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
    *
@@ -150,7 +161,7 @@ export class Database {
   async runValidation (doc) {
     if (this.config && this.config.validateChange) {
       const oldDoc = await this.get(doc._id)
-        .then((doc) => doc)
+        .then(doc => doc)
         .catch(() => ({}))
       this.config.validateChange(doc, oldDoc, this.authCtx)
     }
@@ -185,12 +196,13 @@ export class Database {
     return doc
   }
   /**
- * @typedef {any} Document
- * @property {string} _id - The ID of the document (required)
- * @property {string} [_proof] - The proof of the document (optional)
- * @property {string} [_clock] - The clock of the document (optional)
- * @property {any} [key: string] - Index signature notation to allow any other unknown fields
- */
+   * @typedef {Object} Document
+   * @property {string} _id - The ID of the document (required)
+   * @property {string} [_proof] - The proof of the document (optional)
+   * @property {string} [_clock] - The clock of the document (optional)
+   * @property {any} [key: string] - Index signature notation to allow any other unknown fields
+   *  * @property {Object.<string, any>} [otherProperties] - Any other unknown properties (optional)
+   */
 
   /**
    * Adds a new document to the database, or updates an existing document. Returns the ID of the document and the new clock head.
@@ -245,17 +257,17 @@ export class Database {
         throw new Error('MVCC conflict, document is changed, please reload the document and try again.')
       }
     }
+    const prevClock = [...this.clock]
     const result = await doTransaction(
       'putToProllyTree',
       this.blocks,
-      async (blocks) => await put(blocks, this.clock, event)
+      async blocks => await put(blocks, this.clock, event)
     )
     if (!result) {
       console.error('failed', event)
       throw new Error('failed to put at storage layer')
     }
-    // console.log('new clock head', this.instanceId, result.head.toString())
-    this.clock = result.head // do we want to do this as a finally block
+    this.applyClock(prevClock, result.head)
     await this.notifyListeners([decodedEvent]) // this type is odd
     return {
       id: decodedEvent.key,
@@ -265,6 +277,12 @@ export class Database {
     // todo should include additions (or split clock)
   }
 
+  applyClock (prevClock, newClock) {
+    // console.log('applyClock', prevClock, newClock, this.clock)
+    const removedprevCIDs = this.clock.filter(cid => prevClock.indexOf(cid) === -1)
+    this.clock = removedprevCIDs.concat(newClock)
+  }
+
   //   /**
   //    * Advances the clock to the specified event and updates the root CID
   //    *   Will be used by replication
@@ -323,7 +341,7 @@ export class Database {
 export async function cidsToProof (cids) {
   if (!cids || !cids.all) return []
   const all = await cids.all()
-  return [...all].map((cid) => cid.toString())
+  return [...all].map(cid => cid.toString())
 }
 
 function decodeEvent (event) {

package/src/db-index.js

@@ -72,10 +72,10 @@ const indexEntriesForChanges = (changes, mapFn) => {
   changes.forEach(({ key, value, del }) => {
     if (del || !value) return
     mapFn(makeDoc({ key, value }), (k, v) => {
-      if (typeof v === 'undefined' || typeof k === 'undefined') return
+      if (typeof k === 'undefined') return
       indexEntries.push({
         key: [charwise.encode(k), key],
-        value: v
+        value: v || null
       })
     })
   })
@@ -93,7 +93,7 @@ const indexEntriesForChanges = (changes, mapFn) => {
  *
  */
 export class DbIndex {
-  constructor (database, mapFn, clock, opts = {}) {
+  constructor (database, name, mapFn, clock = null, opts = {}) {
     this.database = database
     if (!database.indexBlocks) {
       database.indexBlocks = new TransactionBlockstore(database?.name + '.indexes', database.blocks.valet?.getKeyMaterial())
@@ -109,7 +109,7 @@ export class DbIndex {
       this.mapFn = mapFn
       this.mapFnString = mapFn.toString()
     }
-    this.name = opts.name || this.makeName()
+    this.name = name || this.makeName()
     this.indexById = { root: null, cid: null }
     this.indexByKey = { root: null, cid: null }
     this.dbHead = null
@@ -159,7 +159,7 @@ export class DbIndex {
 
   static fromJSON (database, { code, clock, name }) {
     // console.log('DbIndex.fromJSON', database.constructor.name, code, clock)
-    return new DbIndex(database, code, clock, { name })
+    return new DbIndex(database, name, code, clock)
   }
 
   /**
@@ -176,7 +176,7 @@ export class DbIndex {
    * @memberof DbIndex
    * @instance
    */
-  async query (query, update = true) {
+  async query (query = {}, update = true) {
     // const callId = Math.random().toString(36).substring(2, 7)
     // todo pass a root to query a snapshot
     // console.time(callId + '.updateIndex')
@@ -203,7 +203,12 @@ export class DbIndex {
   async updateIndex (blocks) {
     // todo this could enqueue the request and give fresh ones to all second comers -- right now it gives out stale promises while working
     // what would it do in a world where all indexes provide a database snapshot to query?
-    if (this.updateIndexPromise) return this.updateIndexPromise
+    if (this.updateIndexPromise) {
+      return this.updateIndexPromise.then(() => {
+        this.updateIndexPromise = null
+        return this.updateIndex(blocks)
+      })
+    }
     this.updateIndexPromise = this.innerUpdateIndex(blocks)
     this.updateIndexPromise.finally(() => { this.updateIndexPromise = null })
     return this.updateIndexPromise
@@ -232,7 +237,7 @@ export class DbIndex {
       this.dbHead = result.clock
       return
     }
-    await doTransaction('updateIndex', inBlocks, async (blocks) => {
+    const didT = await doTransaction('updateIndex', inBlocks, async (blocks) => {
       let oldIndexEntries = []
       let removeByIdIndexEntries = []
       await loadIndex(blocks, this.indexById, idIndexOpts)
@@ -254,6 +259,7 @@ export class DbIndex {
     this.database.notifyExternal('dbIndex')
     // console.timeEnd(callTag + '.doTransactionupdateIndex')
     // console.log(`updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
+    return didT
   }
 }
 
@@ -296,7 +302,11 @@ async function bulkIndex (blocks, inIndex, indexEntries, opts) {
 async function loadIndex (blocks, index, indexOpts) {
   if (!index.root) {
     const cid = index.cid
-    if (!cid) return
+    if (!cid) {
+      // console.log('no cid', index)
+      // throw new Error('cannot load index')
+      return null
+    }
     const { getBlock } = makeGetBlock(blocks)
     index.root = await load({ cid, get: getBlock, ...indexOpts })
   }

package/src/fireproof.js

@@ -6,10 +6,11 @@ import { DbIndex as Index } from './db-index.js'
 import { CID } from 'multiformats'
 import { TransactionBlockstore } from './blockstore.js'
 import { localGet } from './utils.js'
+import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js'
 
-export { Index, Listener }
+export { Index, Listener, Database }
 
-const parseCID = cid => typeof cid === 'string' ? CID.parse(cid) : cid
+const parseCID = cid => (typeof cid === 'string' ? CID.parse(cid) : cid)
 
 export class Fireproof {
   /**
@@ -40,7 +41,11 @@ export class Fireproof {
   static fromJSON (json, database) {
     database.hydrate({ clock: json.clock.map(c => parseCID(c)), name: json.name, key: json.key })
     if (json.indexes) {
-      for (const { name, code, clock: { byId, byKey, db } } of json.indexes) {
+      for (const {
+        name,
+        code,
+        clock: { byId, byKey, db }
+      } of json.indexes) {
         Index.fromJSON(database, {
           clock: {
             byId: byId ? parseCID(byId) : null,
@@ -67,14 +72,14 @@ export class Fireproof {
       })
     }
     const snappedDb = this.fromJSON(definition, withBlocks)
-    ;([...database.indexes.values()]).forEach(index => {
+    ;[...database.indexes.values()].forEach(index => {
       snappedDb.indexes.get(index.mapFnString).mapFn = index.mapFn
     })
     return snappedDb
   }
 
   static async zoom (database, clock) {
-    ;([...database.indexes.values()]).forEach(index => {
+    ;[...database.indexes.values()].forEach(index => {
       index.indexById = { root: null, cid: null }
       index.indexByKey = { root: null, cid: null }
       index.dbHead = null
@@ -83,4 +88,39 @@ export class Fireproof {
     await database.notifyReset() // hmm... indexes should listen to this? might be more complex than worth it. so far this is the only caller
     return database
   }
+
+  // get all the cids
+  // tell valet to make a file
+  static async makeCar (database, key) {
+    const allCIDs = await database.allCIDs()
+    const blocks = database.blocks
+
+    const rootCid = CID.parse(allCIDs[allCIDs.length - 1])
+    if (typeof key === 'undefined') {
+      key = blocks.valet?.getKeyMaterial()
+    }
+    if (key) {
+      return blocksToEncryptedCarBlock(
+        rootCid,
+        {
+          entries: () => allCIDs.map(cid => ({ cid })),
+          get: async cid => await blocks.get(cid)
+        },
+        key
+      )
+    } else {
+      const carBlocks = await Promise.all(
+        allCIDs.map(async c => {
+          const b = await blocks.get(c)
+          // console.log('block', b)
+          if (typeof b.cid === 'string') { b.cid = CID.parse(b.cid) }
+          // if (b.bytes.constructor.name === 'Buffer') console.log('conver vbuff')
+          return b
+        })
+      )
+      return blocksToCarBlock(rootCid, {
+        entries: () => carBlocks
+      })
+    }
+  }
 }

package/src/prolly.js

@@ -270,7 +270,8 @@ export async function root (inBlocks, head) {
  */
 export async function eventsSince (blocks, head, since) {
   if (!head.length) {
-    throw new Error('no head')
+    // throw new Error('no head')
+    return { clockCIDs: [], result: [] }
   }
   // @ts-ignore
   const sinceHead = [...since, ...head] // ?

package/src/valet.js

@@ -200,7 +200,7 @@ export class Valet {
   }
 }
 
-const blocksToCarBlock = async (lastCid, blocks) => {
+export const blocksToCarBlock = async (lastCid, blocks) => {
   let size = 0
   const headerSize = CBW.headerLength({ roots: [lastCid] })
   size += headerSize
@@ -208,6 +208,7 @@ const blocksToCarBlock = async (lastCid, blocks) => {
     blocks = Array.from(blocks.entries())
   }
   for (const { cid, bytes } of blocks) {
+    // console.log(cid, bytes)
     size += CBW.blockLength({ cid, bytes })
   }
   const buffer = new Uint8Array(size)
@@ -222,7 +223,7 @@ const blocksToCarBlock = async (lastCid, blocks) => {
   return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
 }
 
-const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
+export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
   const encryptionKey = Buffer.from(keyMaterial, 'hex')
   const encryptedBlocks = []
   const theCids = []

package/dist/blockstore.js

@@ -1,242 +0,0 @@
-import { parse } from 'multiformats/link';
-import { CID } from 'multiformats';
-import { Valet } from './valet.js';
-// const sleep = ms => new Promise(r => setTimeout(r, ms))
-const husherMap = new Map();
-const husher = (id, workFn) => {
-    if (!husherMap.has(id)) {
-        husherMap.set(id, workFn().finally(() => setTimeout(() => husherMap.delete(id), 100)));
-    }
-    return husherMap.get(id);
-};
-/**
- * @typedef {{ get: (link: import('../src/link').AnyLink) => Promise<AnyBlock | undefined> }} BlockFetcher
- */
-/**
- * @typedef {Object} AnyBlock
- * @property {import('./link').AnyLink} cid - The CID of the block
- * @property {Uint8Array} bytes - The block's data
- *
- * @typedef {Object} Blockstore
- * @property {function(import('./link').AnyLink): Promise<AnyBlock|undefined>} get - A function to retrieve a block by CID
- * @property {function(import('./link').AnyLink, Uint8Array): Promise<void>} put - A function to store a block's data and CID
- *
- * A blockstore that caches writes to a transaction and only persists them when committed.
-*/
-export class TransactionBlockstore {
-    /** @type {Map<string, Uint8Array>} */
-    committedBlocks = new Map();
-    valet = null;
-    instanceId = 'blkz.' + Math.random().toString(36).substring(2, 4);
-    inflightTransactions = new Set();
-    constructor(name, encryptionKey) {
-        if (name) {
-            this.valet = new Valet(name, encryptionKey);
-        }
-        this.remoteBlockFunction = null;
-    }
-    /**
-     * Get a block from the store.
-     *
-     * @param {import('./link').AnyLink} cid
-     * @returns {Promise<AnyBlock | undefined>}
-     */
-    async get(cid) {
-        const key = cid.toString();
-        // it is safe to read from the in-flight transactions becauase they are immutable
-        const bytes = await Promise.any([this.transactionsGet(key), this.committedGet(key)]).catch(e => {
-            // console.log('networkGet', cid.toString(), e)
-            return this.networkGet(key);
-        });
-        if (!bytes)
-            throw new Error('Missing block: ' + key);
-        return { cid, bytes };
-    }
-    // this iterates over the in-flight transactions
-    // and returns the first matching block it finds
-    async transactionsGet(key) {
-        for (const transaction of this.inflightTransactions) {
-            const got = await transaction.get(key);
-            if (got && got.bytes)
-                return got.bytes;
-        }
-        throw new Error('Missing block: ' + key);
-    }
-    async committedGet(key) {
-        const old = this.committedBlocks.get(key);
-        if (old)
-            return old;
-        if (!this.valet)
-            throw new Error('Missing block: ' + key);
-        const got = await this.valet.getBlock(key);
-        // console.log('committedGet: ' + key)
-        this.committedBlocks.set(key, got);
-        return got;
-    }
-    async clearCommittedCache() {
-        this.committedBlocks.clear();
-    }
-    async networkGet(key) {
-        if (this.remoteBlockFunction) {
-            // todo why is this on valet?
-            const value = await husher(key, async () => await this.remoteBlockFunction(key));
-            if (value) {
-                // console.log('networkGot: ' + key, value.length)
-                doTransaction('networkGot: ' + key, this, async (innerBlockstore) => {
-                    await innerBlockstore.put(CID.parse(key), value);
-                });
-                return value;
-            }
-        }
-        else {
-            return false;
-        }
-    }
-    /**
-     * Add a block to the store. Usually bound to a transaction by a closure.
-     * It sets the lastCid property to the CID of the block that was put.
-     * This is used by the transaction as the head of the car when written to the valet.
-     * We don't have to worry about which transaction we are when we are here because
-     * we are the transactionBlockstore.
-     *
-     * @param {import('./link').AnyLink} cid
-     * @param {Uint8Array} bytes
-     */
-    put(cid, bytes) {
-        throw new Error('use a transaction to put');
-    }
-    /**
-     * Iterate over all blocks in the store.
-     *
-     * @yields {AnyBlock}
-     * @returns {AsyncGenerator<AnyBlock>}
-     */
-    // * entries () {
-    //   // needs transaction blocks?
-    //   // for (const [str, bytes] of this.blocks) {
-    //   //   yield { cid: parse(str), bytes }
-    //   // }
-    //   for (const [str, bytes] of this.committedBlocks) {
-    //     yield { cid: parse(str), bytes }
-    //   }
-    // }
-    /**
-     * Begin a transaction. Ensures the uncommited blocks are empty at the begining.
-     * Returns the blocks to read and write during the transaction.
-     * @returns {InnerBlockstore}
-     * @memberof TransactionBlockstore
-     */
-    begin(label = '') {
-        const innerTransactionBlockstore = new InnerBlockstore(label, this);
-        this.inflightTransactions.add(innerTransactionBlockstore);
-        return innerTransactionBlockstore;
-    }
-    /**
-     * Commit the transaction. Writes the blocks to the store.
-     * @returns {Promise<void>}
-     * @memberof TransactionBlockstore
-     */
-    async commit(innerBlockstore) {
-        await this.doCommit(innerBlockstore);
-    }
-    // first get the transaction blockstore from the map of transaction blockstores
-    // then copy it to committedBlocks
-    // then write the transaction blockstore to a car
-    // then write the car to the valet
-    // then remove the transaction blockstore from the map of transaction blockstores
-    doCommit = async (innerBlockstore) => {
-        const cids = new Set();
-        for (const { cid, bytes } of innerBlockstore.entries()) {
-            const stringCid = cid.toString(); // unnecessary string conversion, can we fix upstream?
-            if (this.committedBlocks.has(stringCid)) {
-                // console.log('Duplicate block: ' + stringCid) // todo some of this can be avoided, cost is extra size on car files
-            }
-            else {
-                this.committedBlocks.set(stringCid, bytes);
-                cids.add(stringCid);
-            }
-        }
-        if (cids.size > 0 && this.valet) {
-            // console.log(innerBlockstore.label, 'committing', cids.size, 'blocks')
-            await this.valet.writeTransaction(innerBlockstore, cids);
-        }
-    };
-    /**
-     * Retire the transaction. Clears the uncommited blocks.
-     * @returns {void}
-     * @memberof TransactionBlockstore
-     */
-    retire(innerBlockstore) {
-        this.inflightTransactions.delete(innerBlockstore);
-    }
-}
-/**
- * Runs a function on an inner blockstore, then persists the change to a car writer
- * or other outer blockstore.
- * @param {string} label
- * @param {TransactionBlockstore} blockstore
- * @param {(innerBlockstore: Blockstore) => Promise<any>} doFun
- * @returns {Promise<any>}
- * @memberof TransactionBlockstore
- */
-export const doTransaction = async (label, blockstore, doFun) => {
-    // @ts-ignore
-    if (!blockstore.commit)
-        return await doFun(blockstore);
-    // @ts-ignore
-    const innerBlockstore = blockstore.begin(label);
-    try {
-        const result = await doFun(innerBlockstore);
-        // @ts-ignore
-        await blockstore.commit(innerBlockstore);
-        return result;
-    }
-    catch (e) {
-        console.error(`Transaction ${label} failed`, e, e.stack);
-        throw e;
-    }
-    finally {
-        // @ts-ignore
-        blockstore.retire(innerBlockstore);
-    }
-};
-export class InnerBlockstore {
-    /** @type {Map<string, Uint8Array>} */
-    blocks = new Map();
-    lastCid = null;
-    label = '';
-    parentBlockstore = null;
-    constructor(label, parentBlockstore) {
-        this.label = label;
-        this.parentBlockstore = parentBlockstore;
-    }
-    /**
-     * @param {import('./link').AnyLink} cid
-     * @returns {Promise<AnyBlock | undefined>}
-     */
-    async get(cid) {
-        const key = cid.toString();
-        let bytes = this.blocks.get(key);
-        if (bytes) {
-            return { cid, bytes };
-        }
-        bytes = await this.parentBlockstore.committedGet(key);
-        if (bytes) {
-            return { cid, bytes };
-        }
-    }
-    /**
-     * @param {import('./link').AnyLink} cid
-     * @param {Uint8Array} bytes
-     */
-    async put(cid, bytes) {
-        // console.log('put', cid)
-        this.blocks.set(cid.toString(), bytes);
-        this.lastCid = cid;
-    }
-    *entries() {
-        for (const [str, bytes] of this.blocks) {
-            yield { cid: parse(str), bytes };
-        }
-    }
-}