@fireproof/core 0.4.1 → 0.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.4.1",
+  "version": "0.5.0",
   "description": "Realtime database for IPFS",
   "main": "dist/src/fireproof.js",
   "module": "dist/src/fireproof.mjs",
@@ -11,7 +11,7 @@
     "keygen": "node scripts/keygen.js",
     "test": "standard && npm run test:unencrypted && npm run test:mocha",
     "test:unencrypted": "NO_ENCRYPT=true npm run test:mocha",
-    "test:mocha": "mocha --reporter list test/*.test.js",
+    "test:mocha": "mocha test/*.test.js",
     "test:watch": "npm run test:mocha -- -w --parallel",
     "coverage": "c8 -r html -r text npm test",
     "prepublishOnly": "cp ../../README.md . && npm run build",

package/src/database.js

@@ -117,7 +117,7 @@ export class Database {
       // console.log('change rows', this.instanceId, rows)
     } else {
       const allResp = await getAll(this.blocks, this.clock)
-      rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value })))
+      rows = allResp.result.map(({ key, value }) => decodeEvent({ key, value }))
       dataCIDs = allResp.cids
       // console.log('dbdoc rows', this.instanceId, rows)
     }
@@ -130,7 +130,9 @@ export class Database {
 
   async allDocuments () {
     const allResp = await getAll(this.blocks, this.clock)
-    const rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value }))).map(({ key, value }) => ({ key, value: { _id: key, ...value } }))
+    const rows = allResp.result
+      .map(({ key, value }) => decodeEvent({ key, value }))
+      .map(({ key, value }) => ({ key, value: { _id: key, ...value } }))
     return {
       rows,
       clock: this.clockToJSON(),
@@ -138,6 +140,15 @@ export class Database {
     }
   }
 
+  async allCIDs () {
+    const allResp = await getAll(this.blocks, this.clock)
+    const cids = await cidsToProof(allResp.cids)
+    const clockCids = await cidsToProof(allResp.clockCIDs)
+    // console.log('allcids', cids, clockCids)
+    // todo we need to put the clock head as the last block in the encrypted car
+    return [...cids, ...clockCids] // need a single block version of clock head, maybe an encoded block for it
+  }
+
   /**
    * Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
    *
@@ -150,7 +161,7 @@ export class Database {
   async runValidation (doc) {
     if (this.config && this.config.validateChange) {
       const oldDoc = await this.get(doc._id)
-        .then((doc) => doc)
+        .then(doc => doc)
         .catch(() => ({}))
       this.config.validateChange(doc, oldDoc, this.authCtx)
     }
@@ -185,13 +196,13 @@ export class Database {
     return doc
   }
   /**
- * @typedef {Object} Document
- * @property {string} _id - The ID of the document (required)
- * @property {string} [_proof] - The proof of the document (optional)
- * @property {string} [_clock] - The clock of the document (optional)
- * @property {any} [key: string] - Index signature notation to allow any other unknown fields
- *  * @property {Object.<string, any>} [otherProperties] - Any other unknown properties (optional)
- */
+   * @typedef {Object} Document
+   * @property {string} _id - The ID of the document (required)
+   * @property {string} [_proof] - The proof of the document (optional)
+   * @property {string} [_clock] - The clock of the document (optional)
+   * @property {any} [key: string] - Index signature notation to allow any other unknown fields
+   *  * @property {Object.<string, any>} [otherProperties] - Any other unknown properties (optional)
+   */
 
   /**
    * Adds a new document to the database, or updates an existing document. Returns the ID of the document and the new clock head.
@@ -246,17 +257,17 @@ export class Database {
         throw new Error('MVCC conflict, document is changed, please reload the document and try again.')
       }
     }
+    const prevClock = [...this.clock]
     const result = await doTransaction(
       'putToProllyTree',
       this.blocks,
-      async (blocks) => await put(blocks, this.clock, event)
+      async blocks => await put(blocks, this.clock, event)
     )
     if (!result) {
       console.error('failed', event)
       throw new Error('failed to put at storage layer')
     }
-    // console.log('new clock head', this.instanceId, result.head.toString())
-    this.clock = result.head // do we want to do this as a finally block
+    this.applyClock(prevClock, result.head)
     await this.notifyListeners([decodedEvent]) // this type is odd
     return {
       id: decodedEvent.key,
@@ -266,6 +277,12 @@ export class Database {
     // todo should include additions (or split clock)
   }
 
+  applyClock (prevClock, newClock) {
+    // console.log('applyClock', prevClock, newClock, this.clock)
+    const removedprevCIDs = this.clock.filter(cid => prevClock.indexOf(cid) === -1)
+    this.clock = removedprevCIDs.concat(newClock)
+  }
+
   //   /**
   //    * Advances the clock to the specified event and updates the root CID
   //    *   Will be used by replication
@@ -324,7 +341,7 @@ export class Database {
 export async function cidsToProof (cids) {
   if (!cids || !cids.all) return []
   const all = await cids.all()
-  return [...all].map((cid) => cid.toString())
+  return [...all].map(cid => cid.toString())
 }
 
 function decodeEvent (event) {

package/src/db-index.js

@@ -72,10 +72,10 @@ const indexEntriesForChanges = (changes, mapFn) => {
   changes.forEach(({ key, value, del }) => {
     if (del || !value) return
     mapFn(makeDoc({ key, value }), (k, v) => {
-      if (typeof v === 'undefined' || typeof k === 'undefined') return
+      if (typeof k === 'undefined') return
       indexEntries.push({
         key: [charwise.encode(k), key],
-        value: v
+        value: v || null
       })
     })
   })
@@ -93,7 +93,7 @@ const indexEntriesForChanges = (changes, mapFn) => {
  *
  */
 export class DbIndex {
-  constructor (database, mapFn, clock, opts = {}) {
+  constructor (database, name, mapFn, clock = null, opts = {}) {
     this.database = database
     if (!database.indexBlocks) {
       database.indexBlocks = new TransactionBlockstore(database?.name + '.indexes', database.blocks.valet?.getKeyMaterial())
@@ -109,7 +109,7 @@ export class DbIndex {
       this.mapFn = mapFn
       this.mapFnString = mapFn.toString()
     }
-    this.name = opts.name || this.makeName()
+    this.name = name || this.makeName()
     this.indexById = { root: null, cid: null }
     this.indexByKey = { root: null, cid: null }
     this.dbHead = null
@@ -159,7 +159,7 @@ export class DbIndex {
 
   static fromJSON (database, { code, clock, name }) {
     // console.log('DbIndex.fromJSON', database.constructor.name, code, clock)
-    return new DbIndex(database, code, clock, { name })
+    return new DbIndex(database, name, code, clock)
   }
 
   /**
@@ -176,7 +176,7 @@ export class DbIndex {
    * @memberof DbIndex
    * @instance
    */
-  async query (query, update = true) {
+  async query (query = {}, update = true) {
     // const callId = Math.random().toString(36).substring(2, 7)
     // todo pass a root to query a snapshot
     // console.time(callId + '.updateIndex')
@@ -203,7 +203,12 @@ export class DbIndex {
   async updateIndex (blocks) {
     // todo this could enqueue the request and give fresh ones to all second comers -- right now it gives out stale promises while working
     // what would it do in a world where all indexes provide a database snapshot to query?
-    if (this.updateIndexPromise) return this.updateIndexPromise
+    if (this.updateIndexPromise) {
+      return this.updateIndexPromise.then(() => {
+        this.updateIndexPromise = null
+        return this.updateIndex(blocks)
+      })
+    }
     this.updateIndexPromise = this.innerUpdateIndex(blocks)
     this.updateIndexPromise.finally(() => { this.updateIndexPromise = null })
     return this.updateIndexPromise
@@ -232,7 +237,7 @@ export class DbIndex {
       this.dbHead = result.clock
       return
     }
-    await doTransaction('updateIndex', inBlocks, async (blocks) => {
+    const didT = await doTransaction('updateIndex', inBlocks, async (blocks) => {
       let oldIndexEntries = []
       let removeByIdIndexEntries = []
       await loadIndex(blocks, this.indexById, idIndexOpts)
@@ -254,6 +259,7 @@ export class DbIndex {
     this.database.notifyExternal('dbIndex')
     // console.timeEnd(callTag + '.doTransactionupdateIndex')
     // console.log(`updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
+    return didT
   }
 }
 
@@ -296,7 +302,11 @@ async function bulkIndex (blocks, inIndex, indexEntries, opts) {
 async function loadIndex (blocks, index, indexOpts) {
   if (!index.root) {
     const cid = index.cid
-    if (!cid) return
+    if (!cid) {
+      // console.log('no cid', index)
+      // throw new Error('cannot load index')
+      return null
+    }
     const { getBlock } = makeGetBlock(blocks)
     index.root = await load({ cid, get: getBlock, ...indexOpts })
   }

package/src/fireproof.js

@@ -6,10 +6,11 @@ import { DbIndex as Index } from './db-index.js'
 import { CID } from 'multiformats'
 import { TransactionBlockstore } from './blockstore.js'
 import { localGet } from './utils.js'
+import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js'
 
-export { Index, Listener }
+export { Index, Listener, Database }
 
-const parseCID = cid => typeof cid === 'string' ? CID.parse(cid) : cid
+const parseCID = cid => (typeof cid === 'string' ? CID.parse(cid) : cid)
 
 export class Fireproof {
   /**
@@ -40,7 +41,11 @@ export class Fireproof {
   static fromJSON (json, database) {
     database.hydrate({ clock: json.clock.map(c => parseCID(c)), name: json.name, key: json.key })
     if (json.indexes) {
-      for (const { name, code, clock: { byId, byKey, db } } of json.indexes) {
+      for (const {
+        name,
+        code,
+        clock: { byId, byKey, db }
+      } of json.indexes) {
         Index.fromJSON(database, {
           clock: {
             byId: byId ? parseCID(byId) : null,
@@ -67,14 +72,14 @@ export class Fireproof {
       })
     }
     const snappedDb = this.fromJSON(definition, withBlocks)
-    ;([...database.indexes.values()]).forEach(index => {
+    ;[...database.indexes.values()].forEach(index => {
       snappedDb.indexes.get(index.mapFnString).mapFn = index.mapFn
     })
     return snappedDb
   }
 
   static async zoom (database, clock) {
-    ;([...database.indexes.values()]).forEach(index => {
+    ;[...database.indexes.values()].forEach(index => {
       index.indexById = { root: null, cid: null }
       index.indexByKey = { root: null, cid: null }
       index.dbHead = null
@@ -83,4 +88,39 @@ export class Fireproof {
     await database.notifyReset() // hmm... indexes should listen to this? might be more complex than worth it. so far this is the only caller
     return database
   }
+
+  // get all the cids
+  // tell valet to make a file
+  static async makeCar (database, key) {
+    const allCIDs = await database.allCIDs()
+    const blocks = database.blocks
+
+    const rootCid = CID.parse(allCIDs[allCIDs.length - 1])
+    if (typeof key === 'undefined') {
+      key = blocks.valet?.getKeyMaterial()
+    }
+    if (key) {
+      return blocksToEncryptedCarBlock(
+        rootCid,
+        {
+          entries: () => allCIDs.map(cid => ({ cid })),
+          get: async cid => await blocks.get(cid)
+        },
+        key
+      )
+    } else {
+      const carBlocks = await Promise.all(
+        allCIDs.map(async c => {
+          const b = await blocks.get(c)
+          // console.log('block', b)
+          if (typeof b.cid === 'string') { b.cid = CID.parse(b.cid) }
+          // if (b.bytes.constructor.name === 'Buffer') console.log('conver vbuff')
+          return b
+        })
+      )
+      return blocksToCarBlock(rootCid, {
+        entries: () => carBlocks
+      })
+    }
+  }
 }

package/src/prolly.js

@@ -270,7 +270,8 @@ export async function root (inBlocks, head) {
  */
 export async function eventsSince (blocks, head, since) {
   if (!head.length) {
-    throw new Error('no head')
+    // throw new Error('no head')
+    return { clockCIDs: [], result: [] }
   }
   // @ts-ignore
   const sinceHead = [...since, ...head] // ?

package/src/valet.js

@@ -200,7 +200,7 @@ export class Valet {
   }
 }
 
-const blocksToCarBlock = async (lastCid, blocks) => {
+export const blocksToCarBlock = async (lastCid, blocks) => {
   let size = 0
   const headerSize = CBW.headerLength({ roots: [lastCid] })
   size += headerSize
@@ -208,6 +208,7 @@ const blocksToCarBlock = async (lastCid, blocks) => {
     blocks = Array.from(blocks.entries())
   }
   for (const { cid, bytes } of blocks) {
+    // console.log(cid, bytes)
     size += CBW.blockLength({ cid, bytes })
   }
   const buffer = new Uint8Array(size)
@@ -222,7 +223,7 @@ const blocksToCarBlock = async (lastCid, blocks) => {
   return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
 }
 
-const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
+export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
   const encryptionKey = Buffer.from(keyMaterial, 'hex')
   const encryptedBlocks = []
   const theCids = []