@fireproof/core 0.0.7 → 0.0.8

package/src/db-index.js

@@ -1,15 +1,19 @@
-import { create, load } from 'prolly-trees/db-index'
+// import { create, load } from 'prolly-trees/db-index'
+import { create, load } from '../../../../prolly-trees/src/db-index.js'
+
 import { sha256 as hasher } from 'multiformats/hashes/sha2'
 import { nocache as cache } from 'prolly-trees/cache'
 import { bf, simpleCompare } from 'prolly-trees/utils'
 import { makeGetBlock } from './prolly.js'
 import { cidsToProof } from './fireproof.js'
+import { CID } from 'multiformats'
+
 import * as codec from '@ipld/dag-cbor'
 // import { create as createBlock } from 'multiformats/block'
 import { doTransaction } from './blockstore.js'
 import charwise from 'charwise'
 
-const ALWAYS_REBUILD = true // todo: remove this
+const ALWAYS_REBUILD = false // todo: make false
 
 // const arrayCompare = (a, b) => {
 //   if (Array.isArray(a) && Array.isArray(b)) {
@@ -37,12 +41,13 @@ const compare = (a, b) => {
 const refCompare = (aRef, bRef) => {
   if (Number.isNaN(aRef)) return -1
   if (Number.isNaN(bRef)) throw new Error('ref may not be Infinity or NaN')
-  if (!Number.isFinite(aRef)) return 1
+  if (aRef === Infinity) return 1 // need to test this on equal docids!
   // if (!Number.isFinite(bRef)) throw new Error('ref may not be Infinity or NaN')
   return simpleCompare(aRef, bRef)
 }
 
-const opts = { cache, chunker: bf(3), codec, hasher, compare }
+const dbIndexOpts = { cache, chunker: bf(3), codec, hasher, compare }
+const idIndexOpts = { cache, chunker: bf(3), codec, hasher, compare: simpleCompare }
 
 const makeDoc = ({ key, value }) => ({ _id: key, ...value })
 
@@ -87,14 +92,6 @@ const indexEntriesForChanges = (changes, mapFun) => {
   return indexEntries
 }
 
-const indexEntriesForOldChanges = async (blocks, byIDindexRoot, ids, mapFun) => {
-  const { getBlock } = makeGetBlock(blocks)
-  const byIDindex = await load({ cid: byIDindexRoot.cid, get: getBlock, ...opts })
-
-  const result = await byIDindex.getMany(ids)
-  return result
-}
-
 /**
  * Represents an DbIndex for a Fireproof database.
  *
@@ -118,11 +115,31 @@ export default class DbIndex {
      */
     this.mapFun = mapFun
 
-    this.dbIndexRoot = null
-    this.dbIndex = null
+    this.database.indexes.set(mapFun.toString(), this)
+
+    this.indexById = { root: null, cid: null }
+    this.indexByKey = { root: null, cid: null }
 
-    this.byIDindexRoot = null
     this.dbHead = null
+
+    this.instanceId = this.database.instanceId + `.DbIndex.${Math.random().toString(36).substring(2, 7)}`
+
+    this.updateIndexPromise = null
+  }
+
+  toJSON () {
+    return { code: this.mapFun?.toString(), clock: { db: this.dbHead?.map(cid => cid.toString()), byId: this.indexById.cid?.toString(), byKey: this.indexByKey.cid?.toString() } }
+  }
+
+  static fromJSON (database, { code, clock: { byId, byKey, db } }) {
+    let mapFun
+    // eslint-disable-next-line
+    eval("mapFun = "+ code)
+    const index = new DbIndex(database, mapFun)
+    index.indexById.cid = CID.parse(byId)
+    index.indexByKey.cid = CID.parse(byKey)
+    index.dbHead = db.map(cid => CID.parse(cid))
+    return index
   }
 
   /**
@@ -140,13 +157,18 @@ export default class DbIndex {
    * @instance
    */
   async query (query) {
+    // const callId = Math.random().toString(36).substring(2, 7)
     // if (!root) {
     // pass a root to query a snapshot
-    await doTransaction('#updateIndex', this.database.blocks, async (blocks) => {
-      await this.#updateIndex(blocks)
-    })
+    // console.time(callId + '.#updateIndex')
+    await this.#updateIndex(this.database.blocks)
+    // console.timeEnd(callId + '.#updateIndex')
+
     // }
-    const response = await doIndexQuery(this.database.blocks, this.dbIndexRoot, this.dbIndex, query)
+    // console.time(callId + '.doIndexQuery')
+    const response = await doIndexQuery(this.database.blocks, this.indexByKey, query)
+    // console.timeEnd(callId + '.doIndexQuery')
+
     return {
       proof: { index: await cidsToProof(response.cids) },
       // TODO fix this naming upstream in prolly/db-DbIndex?
@@ -162,50 +184,54 @@ export default class DbIndex {
    * @private
    * @returns {Promise<void>}
    */
+
   async #updateIndex (blocks) {
+    // todo this could enqueue the request and give fresh ones to all second comers -- right now it gives out stale promises while working
+    // what would it do in a world where all indexes provide a database snapshot to query?
+    if (this.updateIndexPromise) return this.updateIndexPromise
+    this.updateIndexPromise = this.#innerUpdateIndex(blocks)
+    this.updateIndexPromise.finally(() => { this.updateIndexPromise = null })
+    return this.updateIndexPromise
+  }
+
+  async #innerUpdateIndex (inBlocks) {
+    // const callTag = Math.random().toString(36).substring(4)
+    // console.log(`#updateIndex ${callTag} >`, this.instanceId, this.dbHead?.toString(), this.dbIndexRoot?.cid.toString(), this.indexByIdRoot?.cid.toString())
     // todo remove this hack
     if (ALWAYS_REBUILD) {
       this.dbHead = null // hack
-      this.dbIndex = null // hack
+      this.indexByKey = null // hack
       this.dbIndexRoot = null
     }
+    // console.log('dbHead', this.dbHead)
+    // console.time(callTag + '.changesSince')
     const result = await this.database.changesSince(this.dbHead) // {key, value, del}
-    if (this.dbHead) {
-      const oldChangeEntries = await indexEntriesForOldChanges(
-        blocks,
-        this.byIDindexRoot,
-        result.rows.map(({ key }) => key),
-        this.mapFun
-      )
-      const oldIndexEntries = oldChangeEntries.result.map((key) => ({ key, del: true }))
-      const removalResult = await bulkIndex(blocks, this.dbIndexRoot, this.dbIndex, oldIndexEntries, opts)
-      this.dbIndexRoot = removalResult.root
-      this.dbIndex = removalResult.dbIndex
+    // console.timeEnd(callTag + '.changesSince')
+    // console.log('result.rows.length', result.rows.length)
 
-      const removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }))
-      const purgedRemovalResults = await bulkIndex(
-        blocks,
-        this.byIDindexRoot,
-        this.byIDIndex,
-        removeByIdIndexEntries,
-        opts
-      )
-      this.byIDindexRoot = purgedRemovalResults.root
-      this.byIDIndex = purgedRemovalResults.dbIndex
-    }
-    const indexEntries = indexEntriesForChanges(result.rows, this.mapFun)
-    const byIdIndexEntries = indexEntries.map(({ key }) => ({ key: key[1], value: key }))
-    const addFutureRemovalsResult = await bulkIndex(blocks, this.byIDindexRoot, this.byIDIndex, byIdIndexEntries, opts)
-    this.byIDindexRoot = addFutureRemovalsResult.root
-    this.byIDIndex = addFutureRemovalsResult.dbIndex
+    // console.time(callTag + '.doTransaction#updateIndex')
 
-    // console.log('indexEntries', indexEntries)
-
-    const updateIndexResult = await bulkIndex(blocks, this.dbIndexRoot, this.dbIndex, indexEntries, opts)
-    this.dbIndexRoot = updateIndexResult.root
-    this.dbIndex = updateIndexResult.dbIndex
-
-    this.dbHead = result.clock
+    if (result.rows.length === 0) {
+      // console.log('#updateIndex < no changes')
+      this.dbHead = result.clock
+      return
+    }
+    await doTransaction('#updateIndex', inBlocks, async (blocks) => {
+      let oldIndexEntries = []
+      let removeByIdIndexEntries = []
+      if (this.dbHead) { // need a maybe load
+        const oldChangeEntries = await this.indexById.root.getMany(result.rows.map(({ key }) => key))
+        oldIndexEntries = oldChangeEntries.result.map((key) => ({ key, del: true }))
+        removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }))
+      }
+      const indexEntries = indexEntriesForChanges(result.rows, this.mapFun)
+      const byIdIndexEntries = indexEntries.map(({ key }) => ({ key: key[1], value: key }))
+      this.indexById = await bulkIndex(blocks, this.indexById, removeByIdIndexEntries.concat(byIdIndexEntries), idIndexOpts)
+      this.indexByKey = await bulkIndex(blocks, this.indexByKey, oldIndexEntries.concat(indexEntries), dbIndexOpts)
+      this.dbHead = result.clock
+    })
+    // console.timeEnd(callTag + '.doTransaction#updateIndex')
+    // console.log(`#updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.dbIndexRoot?.cid.toString(), this.indexByIdRoot?.cid.toString())
   }
 }
 
@@ -216,45 +242,47 @@ export default class DbIndex {
  * @param {DbIndexEntry[]} indexEntries
  * @private
  */
-async function bulkIndex (blocks, inRoot, inDBindex, indexEntries) {
-  if (!indexEntries.length) return { dbIndex: inDBindex, root: inRoot }
+async function bulkIndex (blocks, inIndex, indexEntries, opts) {
+  if (!indexEntries.length) return inIndex
   const putBlock = blocks.put.bind(blocks)
   const { getBlock } = makeGetBlock(blocks)
   let returnRootBlock
   let returnNode
-  if (!inDBindex) {
-    for await (const node of await create({ get: getBlock, list: indexEntries, ...opts })) {
-      const block = await node.block
-      await putBlock(block.cid, block.bytes)
-      returnRootBlock = block
-      returnNode = node
-    }
-  } else {
-    // const dbIndex = await load({ cid: inRoot.cid, get: getBlock, ...opts }) // todo load from root on refresh
-    const { root, blocks } = await inDBindex.bulk(indexEntries)
-    returnRootBlock = await root.block
-    returnNode = root
-    for await (const block of blocks) {
-      await putBlock(block.cid, block.bytes)
+  if (!inIndex.root) {
+    const cid = inIndex.cid
+    if (!cid) {
+      for await (const node of await create({ get: getBlock, list: indexEntries, ...opts })) {
+        const block = await node.block
+        await putBlock(block.cid, block.bytes)
+        returnRootBlock = block
+        returnNode = node
+      }
+      return { root: returnNode, cid: returnRootBlock.cid }
     }
-    await putBlock(returnRootBlock.cid, returnRootBlock.bytes)
+    inIndex.root = await load({ cid, get: getBlock, ...dbIndexOpts })
+  }
+  const { root, blocks: newBlocks } = await inIndex.root.bulk(indexEntries)
+  returnRootBlock = await root.block
+  returnNode = root
+  for await (const block of newBlocks) {
+    await putBlock(block.cid, block.bytes)
   }
-  return { dbIndex: returnNode, root: returnRootBlock }
+  await putBlock(returnRootBlock.cid, returnRootBlock.bytes)
+  return { root: returnNode, cid: returnRootBlock.cid }
 }
 
-async function doIndexQuery (blocks, dbIndexRoot, dbIndex, query) {
-  if (!dbIndex) {
-    const cid = dbIndexRoot && dbIndexRoot.cid
+async function doIndexQuery (blocks, indexByKey, query) {
+  if (!indexByKey.root) {
+    const cid = indexByKey.cid
     if (!cid) return { result: [] }
     const { getBlock } = makeGetBlock(blocks)
-    dbIndex = await load({ cid, get: getBlock, ...opts })
+    indexByKey.root = await load({ cid, get: getBlock, ...dbIndexOpts })
   }
   if (query.range) {
     const encodedRange = query.range.map((key) => charwise.encode(key))
-    return dbIndex.range(...encodedRange)
+    return indexByKey.root.range(...encodedRange)
   } else if (query.key) {
     const encodedKey = charwise.encode(query.key)
-    console.log('getting key', encodedKey)
-    return dbIndex.get(encodedKey)
+    return indexByKey.root.get(encodedKey)
   }
 }

package/src/fireproof.js

@@ -38,6 +38,7 @@ export default class Fireproof {
     this.clock = clock
     this.config = config
     this.authCtx = authCtx
+    this.indexes = new Map()
   }
 
   /**
@@ -78,7 +79,11 @@ export default class Fireproof {
    */
   toJSON () {
     // todo this also needs to return the index roots...
-    return { clock: this.clock }
+    return {
+      clock: this.clock.map(cid => cid.toString()),
+      name: this.name,
+      indexes: [...this.indexes.values()].map((index) => index.toJSON())
+    }
   }
 
   /**
@@ -133,7 +138,7 @@ export default class Fireproof {
   }
 
   async #notifyListeners (changes) {
-    // await sleep(0)
+    // await sleep(10)
     for (const listener of this.#listeners) {
       await listener(changes)
     }
@@ -221,6 +226,7 @@ export default class Fireproof {
       console.error('failed', event)
       throw new Error('failed to put at storage layer')
     }
+    // console.log('new clock head', this.instanceId, result.head.toString())
     this.clock = result.head // do we want to do this as a finally block
     await this.#notifyListeners([decodedEvent]) // this type is odd
     return {

package/src/hydrator.js

@@ -0,0 +1,10 @@
+import Fireproof from './fireproof.js'
+import DbIndex from './db-index.js'
+
+export function fromJSON (json, blocks) {
+  const fp = new Fireproof(blocks, json.clock, { name: json.name })
+  for (const index of json.indexes) {
+    DbIndex.fromJSON(fp, index)
+  }
+  return fp
+}

package/src/prolly.js

@@ -164,7 +164,8 @@ export async function put (inBlocks, head, event, options) {
 
   // Otherwise, we find the common ancestor and update the root and other blocks
   const events = new EventFetcher(blocks)
-  // this is returning more events than necessary
+  // todo this is returning more events than necessary, lets define the desired semantics from the top down
+  // good semantics mean we can cache the results of this call
   const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head)
   // console.log('sorted', JSON.stringify(sorted.map(({ value: { data: { key, value } } }) => ({ key, value }))))
   const prollyRootNode = await prollyRootFromAncestor(events, ancestor, getBlock)

package/test/db-index.test.js

@@ -113,7 +113,7 @@ describe('DbIndex query', () => {
     await index.query({ range: [51, 54] })
 
     console.x('--- make Xander 53')
-    const DOCID = 'xxxx-3c3a-4b5e-9c1c-8c5c0c5c0c5c'
+    const DOCID = 'xander-doc'
     const r1 = await database.put({ _id: DOCID, name: 'Xander', age: 53 })
     assert(r1.id, 'should have id')
 

package/test/fireproof.test.js

@@ -2,7 +2,7 @@ import { describe, it, beforeEach } from 'mocha'
 import assert from 'node:assert'
 import Blockstore from '../src/blockstore.js'
 import Fireproof from '../src/fireproof.js'
-import * as codec from '@ipld/dag-cbor'
+// import * as codec from '@ipld/dag-cbor'
 
 let database, resp0
 
@@ -335,21 +335,21 @@ describe('Fireproof', () => {
     assert.equal(res9.rows.length, 0)
   })
 
-  it.skip('docs since repeated changes', async () => {
+  it('docs since repeated changes', async () => {
     assert.equal((await database.changesSince()).rows.length, 1)
     let resp, doc, changes
-    for (let index = 0; index < 200; index++) {
+    for (let index = 0; index < 30; index++) {
       const id = '1' + (301 - index).toString()
-      console.log(`Putting id: ${id}, index: ${index}`)
+      // console.log(`Putting id: ${id}, index: ${index}`)
       resp = await database.put({ index, _id: id }).catch(e => {
         assert.fail(`put failed on _id: ${id}, error: ${e.message}`)
       })
       assert(resp.id, `Failed to obtain resp.id for _id: ${id}`)
 
-      console.log(`vis for update id: ${id}, index:`, index)
-      for await (const line of database.vis()) {
-        console.log(line)
-      }
+      // console.log(`vis for update id: ${id}, index:`, index)
+      // for await (const line of database.vis()) {
+      //   console.log(line)
+      // }
 
       doc = await database.get(resp.id).catch(e => {
         console.log('failed', e)
@@ -366,20 +366,11 @@ describe('Fireproof', () => {
           assert(!/^bafy/.test(value), `Unexpected "bafy..." value found at index ${index} in row ${JSON.stringify(row)}`)
         }
       })
-      if (index > 3) {
-        const stored = await database.blocks.get('bafyreicumn7tvssch4xslbe4jjq55c6w3jt4yxyjagkr2tengsudato7vi').catch((e) => {
-          console.log(`Error getting block for index ${index}: ${e.message}`)
-        })
-        if (stored) {
-          const doc = codec.decode(await stored.bytes)
-          // console.log('stored', JSON.stringify(dec))
-          assert.equal(doc.closed, false)
-        }
-      }
-      console.log('changes: ', index, changes.rows.length, JSON.stringify(changes.rows))
+
+      // console.log('changes: ', index, changes.rows.length, JSON.stringify(changes.rows))
       assert.equal(changes.rows.length, index + 2, `failed on ${index}, with ${changes.rows.length} ${id}`)
     }
-  }).timeout(20000)
+  }).timeout(30000)
 
   it('concurrent transactions', async () => {
     assert.equal((await database.changesSince()).rows.length, 1)
@@ -413,4 +404,11 @@ describe('Fireproof', () => {
     // await sleep(1000)
     assert.equal((await database.changesSince()).rows.length, 2)
   }).timeout(20000)
+  it('serialize database', async () => {
+    await database.put({ _id: 'rehy', name: 'drate' })
+    assert.equal((await database.changesSince()).rows.length, 2)
+    const serialized = JSON.parse(JSON.stringify(database))
+    assert.equal(serialized.name, 'helloName')
+    assert.equal(serialized.clock.length, 1)
+  })
 })

package/test/hydrator.test.js

@@ -0,0 +1,75 @@
+import { describe, it, beforeEach } from 'mocha'
+import assert from 'node:assert'
+import Fireproof from '../src/fireproof.js'
+import DbIndex from '../src/db-index.js'
+import { fromJSON } from '../src/hydrator.js'
+console.x = function () {}
+
+describe('DbIndex query', () => {
+  let database, index
+  beforeEach(async () => {
+    database = Fireproof.storage()
+    const docs = [
+      { _id: 'a1s3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c', name: 'alice', age: 40 },
+      { _id: 'b2s3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c', name: 'bob', age: 40 },
+      { _id: 'c3s3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c', name: 'carol', age: 43 },
+      { _id: 'd4s3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c', name: 'dave', age: 48 },
+      { _id: 'e4s3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c', name: 'emily', age: 4 },
+      { _id: 'f4s3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c', name: 'frank', age: 7 }
+    ]
+    for (const doc of docs) {
+      const id = doc._id
+      const response = await database.put(doc)
+      assert(response)
+      assert(response.id, 'should have id')
+      assert.equal(response.id, id)
+    }
+    index = new DbIndex(database, function (doc, map) {
+      map(doc.age, doc.name)
+    })
+  })
+  it('serialize database with index', async () => {
+    await database.put({ _id: 'rehy', name: 'drate', age: 1 })
+    assert.equal((await database.changesSince()).rows.length, 7)
+    const result = await index.query({ range: [0, 54] })
+    assert.equal(result.rows[0].value, 'drate')
+    const serialized = database.toJSON()
+    // console.log('serialized', serialized)
+    assert.equal(serialized.name, 'global')
+    assert.equal(serialized.clock.length, 1)
+    assert.equal(serialized.clock[0].constructor.name, 'String')
+    assert.equal(serialized.indexes.length, 1)
+    assert.equal(serialized.indexes[0].code, `function (doc, map) {
+      map(doc.age, doc.name)
+    }`)
+    assert.equal(serialized.indexes[0].clock.byId.constructor.name, 'String')
+    assert.equal(serialized.indexes[0].clock.byKey.constructor.name, 'String')
+    assert.equal(serialized.indexes[0].clock.db[0].constructor.name, 'String')
+  })
+  it('rehydrate database', async () => {
+    await database.put({ _id: 'rehy', name: 'drate', age: 1 })
+    assert.equal((await database.changesSince()).rows.length, 7)
+    const result = await index.query({ range: [0, 54] })
+    assert.equal(result.rows[0].value, 'drate')
+
+    const serialized = JSON.parse(JSON.stringify(database))
+    // console.log('serialized', JSON.stringify(serialized))
+    // connect it to the same blockstore for testing
+    const newDb = fromJSON(serialized, database.blocks)
+    assert.equal(newDb.name, 'global')
+    assert.equal(newDb.clock.length, 1)
+    assert.equal((await newDb.changesSince()).rows.length, 7)
+    const newIndex = [...newDb.indexes.values()][0]
+    assert.equal(newIndex.mapFun, `function (doc, map) {
+      map(doc.age, doc.name)
+    }`)
+    assert.equal(newIndex.indexById.cid, 'bafyreifuz54ugnq77fur47vwv3dwab7p3gpnf5to6hlnbhv5p4kwo7auoi')
+    assert.equal(newIndex.indexById.root, null)
+
+    assert.equal(newIndex.indexByKey.cid, 'bafyreicr5rpvsxnqchcwk5rxlmdvd3fah2vexmbsp2dvr4cfdxd2q2ycgu')
+    assert.equal(newIndex.indexByKey.root, null)
+
+    const newResult = await newIndex.query({ range: [0, 54] })
+    assert.equal(newResult.rows[0].value, 'drate')
+  })
+})