@fireproof/core 0.5.8 → 0.5.10

package/src/database.js

@@ -8,6 +8,7 @@ import { CID } from 'multiformats'
 // TypeScript Types
 // eslint-disable-next-line no-unused-vars
 // import { CID } from 'multiformats/dist/types/src/cid.js'
+
 // eslint-disable-next-line no-unused-vars
 class Proof {}
 export const parseCID = cid => (typeof cid === 'string' ? CID.parse(cid) : cid)
@@ -89,9 +90,9 @@ export class Database {
   }
 
   // used be indexes etc to notify database listeners of new availability
-  async notifyExternal (source = 'unknown') {
-    await this.notifyListeners({ _external: source, _clock: this.clockToJSON() })
-  }
+  // async notifyExternal (source = 'unknown') {
+  //   // await this.notifyListeners({ _external: source, _clock: this.clockToJSON() })
+  // }
 
   /**
    * Returns the changes made to the Fireproof instance since the specified event.
@@ -351,13 +352,23 @@ export class Database {
    * @returns {Function} - A function that can be called to unregister the listener.
    * @memberof Fireproof
    */
-  registerListener (listener) {
+  subscribe (listener) {
     this.listeners.add(listener)
     return () => {
       this.listeners.delete(listener)
     }
   }
 
+  /**
+   * @deprecated 0.7.0 - renamed subscribe(listener)
+   * @param {Function} listener - The listener to be called when the clock is updated.
+   * @returns {Function} - A function that can be called to unregister the listener.
+   * @memberof Fireproof
+   */
+  registerListener (listener) {
+    return this.subscribe(listener)
+  }
+
   async notifyListeners (changes) {
     // await sleep(10)
     await this.maybeSaveClock()

package/src/db-index.js

@@ -70,14 +70,23 @@ const makeDoc = ({ key, value }) => ({ _id: key, ...value })
 const indexEntriesForChanges = (changes, mapFn) => {
   const indexEntries = []
   changes.forEach(({ key, value, del }) => {
+    // key is _id, value is the document
     if (del || !value) return
-    mapFn(makeDoc({ key, value }), (k, v) => {
+    let mapCalled = false
+    const mapReturn = mapFn(makeDoc({ key, value }), (k, v) => {
+      mapCalled = true
       if (typeof k === 'undefined') return
       indexEntries.push({
         key: [charwise.encode(k), key],
         value: v || null
       })
     })
+    if (!mapCalled && mapReturn) {
+      indexEntries.push({
+        key: [charwise.encode(mapReturn), key],
+        value: null
+      })
+    }
   })
   return indexEntries
 }
@@ -99,7 +108,10 @@ export class DbIndex {
   constructor (database, name, mapFn, clock = null, opts = {}) {
     this.database = database
     if (!database.indexBlocks) {
-      database.indexBlocks = new TransactionBlockstore(database?.name + '.indexes', database.blocks.valet?.getKeyMaterial())
+      database.indexBlocks = new TransactionBlockstore(
+        database?.name + '.indexes',
+        database.blocks.valet?.getKeyMaterial()
+      )
     }
     if (typeof name === 'function') {
       // app is using deprecated API, remove in 0.7
@@ -108,13 +120,8 @@ export class DbIndex {
       mapFn = name
       name = null
     }
-    if (typeof mapFn === 'string') {
-      this.mapFnString = mapFn
-    } else {
-      this.mapFn = mapFn
-      this.mapFnString = mapFn.toString()
-    }
-    this.name = name || this.makeName()
+    this.applyMapFn(mapFn, name)
+
     this.indexById = { root: null, cid: null }
     this.indexByKey = { root: null, cid: null }
     this.dbHead = null
@@ -125,13 +132,34 @@ export class DbIndex {
     }
     this.instanceId = this.database.instanceId + `.DbIndex.${Math.random().toString(36).substring(2, 7)}`
     this.updateIndexPromise = null
-    if (!opts.temporary) { DbIndex.registerWithDatabase(this, this.database) }
+    if (!opts.temporary) {
+      DbIndex.registerWithDatabase(this, this.database)
+    }
+  }
+
+  applyMapFn (mapFn, name) {
+    if (typeof mapFn === 'string') {
+      this.mapFnString = mapFn
+    } else {
+      this.mapFn = mapFn
+      this.mapFnString = mapFn.toString()
+    }
+    this.name = name || this.makeName()
   }
 
   makeName () {
     const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g
-    const matches = Array.from(this.mapFnString.matchAll(regex), match => match[1].trim())
-    return matches[1]
+    let matches = Array.from(this.mapFnString.matchAll(regex), match => match[1].trim())
+    if (matches.length === 0) {
+      matches = /=>\s*(.*)/.exec(this.mapFnString)
+    }
+    if (matches.length === 0) {
+      return this.mapFnString
+    } else {
+      // it's a consise arrow function, match everythign after the arrow
+      this.includeDocsDefault = true
+      return matches[1]
+    }
   }
 
   static registerWithDatabase (inIndex, database) {
@@ -141,7 +169,8 @@ export class DbIndex {
       // merge our inIndex code with the inIndex clock or vice versa
       const existingIndex = database.indexes.get(inIndex.mapFnString)
       // keep the code instance, discard the clock instance
-      if (existingIndex.mapFn) { // this one also has other config
+      if (existingIndex.mapFn) {
+        // this one also has other config
         existingIndex.dbHead = inIndex.dbHead
         existingIndex.indexById.cid = inIndex.indexById.cid
         existingIndex.indexByKey.cid = inIndex.indexByKey.cid
@@ -185,7 +214,7 @@ export class DbIndex {
   /**
    * Query object can have {range}
    * @param {DbQuery} query - the query range to use
-   * @returns {Promise<{proof: {}, rows: Array<{id: string, key: string, value: any}>}>}
+   * @returns {Promise<{proof: {}, rows: Array<{id: string, key: string, value: any, doc?: any}>}>}
    * @memberof DbIndex
    * @instance
    */
@@ -193,20 +222,63 @@ export class DbIndex {
     // const callId = Math.random().toString(36).substring(2, 7)
     // todo pass a root to query a snapshot
     // console.time(callId + '.updateIndex')
-    update && await this.updateIndex(this.database.indexBlocks)
+    update && (await this.updateIndex(this.database.indexBlocks))
     // console.timeEnd(callId + '.updateIndex')
     // console.time(callId + '.doIndexQuery')
     // console.log('query', query)
-    const response = await doIndexQuery(this.database.indexBlocks, this.indexByKey, query)
+    const response = await this.doIndexQuery(query)
     // console.timeEnd(callId + '.doIndexQuery')
     return {
       proof: { index: await cidsToProof(response.cids) },
-      rows: response.result.map(({ id, key, row }) => {
-        return ({ id, key: charwise.decode(key), value: row })
+      rows: response.result.map(({ id, key, row, doc }) => {
+        return { id, key: charwise.decode(key), value: row, doc }
       })
     }
   }
 
+  /**
+   *
+   * @param {any} resp
+   * @param {any} query
+   * @returns
+   */
+  async applyQuery (resp, query) {
+    if (query.descending) {
+      resp.result = resp.result.reverse()
+    }
+    if (query.limit) {
+      resp.result = resp.result.slice(0, query.limit)
+    }
+    if (query.includeDocs) {
+      resp.result = await Promise.all(
+        resp.result.map(async row => {
+          const doc = await this.database.get(row.id)
+          return { ...row, doc }
+        })
+      )
+    }
+    return resp
+  }
+
+  async doIndexQuery (query = {}) {
+    await loadIndex(this.database.indexBlocks, this.indexByKey, dbIndexOpts)
+    if (!this.indexByKey.root) return { result: [] }
+    if (query.includeDocs === undefined) query.includeDocs = this.includeDocsDefault
+    if (query.range) {
+      const encodedRange = query.range.map(key => charwise.encode(key))
+      return await this.applyQuery(await this.indexByKey.root.range(...encodedRange), query)
+    } else if (query.key) {
+      const encodedKey = charwise.encode(query.key)
+      return await this.applyQuery(this.indexByKey.root.get(encodedKey), query)
+    } else {
+      const { result, ...all } = await this.indexByKey.root.getAllEntries()
+      return await this.applyQuery(
+        { result: result.map(({ key: [k, id], value }) => ({ key: k, id, row: value })), ...all },
+        query
+      )
+    }
+  }
+
   /**
    * Update the DbIndex with the latest changes
    * @private
@@ -223,7 +295,9 @@ export class DbIndex {
       })
     }
     this.updateIndexPromise = this.innerUpdateIndex(blocks)
-    this.updateIndexPromise.finally(() => { this.updateIndexPromise = null })
+    this.updateIndexPromise.finally(() => {
+      this.updateIndexPromise = null
+    })
     return this.updateIndexPromise
   }
 
@@ -250,7 +324,7 @@ export class DbIndex {
       this.dbHead = result.clock
       return
     }
-    const didT = await doTransaction('updateIndex', inBlocks, async (blocks) => {
+    const didT = await doTransaction('updateIndex', inBlocks, async blocks => {
       let oldIndexEntries = []
       let removeByIdIndexEntries = []
       await loadIndex(blocks, this.indexById, idIndexOpts)
@@ -258,19 +332,28 @@ export class DbIndex {
       // console.log('head', this.dbHead, this.indexById)
       if (this.indexById.root) {
         const oldChangeEntries = await this.indexById.root.getMany(result.rows.map(({ key }) => key))
-        oldIndexEntries = oldChangeEntries.result.map((key) => ({ key, del: true }))
+        oldIndexEntries = oldChangeEntries.result.map(key => ({ key, del: true }))
         removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }))
       }
       if (!this.mapFn) {
-        throw new Error('No live map function installed for index, cannot update. Make sure your index definition runs before any queries.' + (this.mapFnString ? ' Your code should match the stored map function source:\n' + this.mapFnString : ''))
+        throw new Error(
+          'No live map function installed for index, cannot update. Make sure your index definition runs before any queries.' +
+            (this.mapFnString ? ' Your code should match the stored map function source:\n' + this.mapFnString : '')
+        )
       }
       const indexEntries = indexEntriesForChanges(result.rows, this.mapFn)
       const byIdIndexEntries = indexEntries.map(({ key }) => ({ key: key[1], value: key }))
-      this.indexById = await bulkIndex(blocks, this.indexById, removeByIdIndexEntries.concat(byIdIndexEntries), idIndexOpts)
+      this.indexById = await bulkIndex(
+        blocks,
+        this.indexById,
+        removeByIdIndexEntries.concat(byIdIndexEntries),
+        idIndexOpts
+      )
       this.indexByKey = await bulkIndex(blocks, this.indexByKey, oldIndexEntries.concat(indexEntries), dbIndexOpts)
       this.dbHead = result.clock
     })
-    this.database.notifyExternal('dbIndex')
+    // todo index subscriptions
+    // this.database.notifyExternal('dbIndex')
     // console.timeEnd(callTag + '.doTransactionupdateIndex')
     // console.log(`updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
     return didT
@@ -331,23 +414,3 @@ async function loadIndex (blocks, index, indexOpts) {
   }
   return index.root
 }
-
-async function applyLimit (results, limit) {
-  results.result = results.result.slice(0, limit)
-  return results
-}
-
-async function doIndexQuery (blocks, indexByKey, query = {}) {
-  await loadIndex(blocks, indexByKey, dbIndexOpts)
-  if (!indexByKey.root) return { result: [] }
-  if (query.range) {
-    const encodedRange = query.range.map((key) => charwise.encode(key))
-    return applyLimit(await indexByKey.root.range(...encodedRange), query.limit)
-  } else if (query.key) {
-    const encodedKey = charwise.encode(query.key)
-    return indexByKey.root.get(encodedKey)
-  } else {
-    const { result, ...all } = await indexByKey.root.getAllEntries()
-    return applyLimit({ result: result.map(({ key: [k, id], value }) => ({ key: k, id, row: value })), ...all }, query.limit)
-  }
-}

package/src/fireproof.js

@@ -24,7 +24,7 @@ export class Fireproof {
       if (existing) {
         const existingConfig = JSON.parse(existing)
         const fp = new Database(new TransactionBlockstore(name, existingConfig.key), [], opts)
-        return this.fromJSON(existingConfig, fp)
+        return Fireproof.fromJSON(existingConfig, fp)
       } else {
         const instanceKey = randomBytes(32).toString('hex') // pass null to disable encryption
         return new Database(new TransactionBlockstore(name, instanceKey), [], opts)
@@ -67,7 +67,7 @@ export class Fireproof {
         index.clock.db = null
       })
     }
-    const snappedDb = this.fromJSON(definition, withBlocks)
+    const snappedDb = Fireproof.fromJSON(definition, withBlocks)
     ;[...database.indexes.values()].forEach(index => {
       snappedDb.indexes.get(index.mapFnString).mapFn = index.mapFn
     })

package/src/listener.js

@@ -8,7 +8,9 @@
  * @param {Function} routingFn - The routing function to apply to each entry in the database.
  */
 // import { ChangeEvent } from './db-index'
-
+/**
+ * @deprecated since version 0.7.0
+ */
 export class Listener {
   subcribers = new Map()
   doStopListening = null

package/src/sync.js

@@ -97,6 +97,10 @@ export class Sync {
       // get the roots parents
       const parents = await Promise.all(roots.map(async (cid) => {
         const rbl = await reader.get(cid)
+        if (!rbl) {
+          console.log('missing root block', cid.toString(), reader)
+          throw new Error('missing root block')
+        }
         const block = await decodeEventBlock(rbl.bytes)
         return block.value.parents
       }))
@@ -113,7 +117,7 @@ export class Sync {
       } else if (message.clock) {
         const reqCidDiff = message
         // this might be a CID diff
-        // console.log('got diff', reqCidDiff)
+        console.log('got diff', reqCidDiff)
         const carBlock = await Sync.makeCar(this.database, null, reqCidDiff.cids)
         if (!carBlock) {
         // we are full synced