@fireproof/core 0.0.2 → 0.0.4

package/src/valet.js

@@ -1,27 +1,71 @@
 import { CarReader } from '@ipld/car'
 import { CID } from 'multiformats/cid'
 import { openDB } from 'idb'
+import cargoQueue from 'async/cargoQueue.js'
 
 // const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
-let storageSupported = false
-try {
-  storageSupported = (window.localStorage && true)
-} catch (e) { }
+// let storageSupported = false
+// try {
+//   storageSupported = window.localStorage && true
+// } catch (e) {}
+// const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
 
 export default class Valet {
   #cars = new Map() // cars by cid
   #cidToCar = new Map() // cid to car
-
   #db = null
+  #uploadQueue = null
+  #alreadyEnqueued = new Set()
+
+  /**
+   * Function installed by the database to upload car files
+   * @type {null|function(string, Uint8Array):Promise<void>}
+   */
+  uploadFunction = null
+
+  constructor () {
+    this.#uploadQueue = cargoQueue(async (tasks, callback) => {
+      console.log(
+        'queue worker',
+        tasks.length,
+        tasks.reduce((acc, t) => acc + t.value.length, 0)
+      )
+      if (this.uploadFunction) {
+        // todo we can coalesce these into a single car file
+        for (const task of tasks) {
+          await this.uploadFunction(task.carCid, task.value)
+        }
+      }
+      callback()
+    })
+
+    this.#uploadQueue.drain(async () => {
+      return await this.withDB(async (db) => {
+        const carKeys = (await db.getAllFromIndex('cidToCar', 'pending')).map((c) => c.car)
+        for (const carKey of carKeys) {
+          await this.uploadFunction(carKey, await db.get('cars', carKey))
+          const carMeta = await db.get('cidToCar', carKey)
+          delete carMeta.pending
+          await db.put('cidToCar', carMeta)
+        }
+      })
+    })
+  }
 
   withDB = async (dbWorkFun) => {
-    if (!storageSupported) return
+    // if (!storageSupported) return
     if (!this.#db) {
-      this.#db = await openDB('valet', 1, {
-        upgrade (db) {
-          db.createObjectStore('cars') // todo use database name
-          const cidToCar = db.createObjectStore('cidToCar', { keyPath: 'car' })
-          cidToCar.createIndex('cids', 'cids', { multiEntry: true })
+      this.#db = await openDB('valet', 2, {
+        upgrade (db, oldVersion, newVersion, transaction) {
+          if (oldVersion < 1) {
+            db.createObjectStore('cars') // todo use database name
+            const cidToCar = db.createObjectStore('cidToCar', { keyPath: 'car' })
+            cidToCar.createIndex('cids', 'cids', { multiEntry: true })
+          }
+          if (oldVersion < 2) {
+            const cidToCar = transaction.objectStore('cidToCar')
+            cidToCar.createIndex('pending', 'pending')
+          }
         }
       })
     }
@@ -29,44 +73,47 @@ export default class Valet {
   }
 
   /**
-     *
-     * @param {string} carCid
-     * @param {*} value
-     */
+   *
+   * @param {string} carCid
+   * @param {*} value
+   */
   async parkCar (carCid, value, cids) {
-    this.#cars.set(carCid, value)
-    for (const cid of cids) {
-      this.#cidToCar.set(cid, carCid)
-    }
+    // this.#cars.set(carCid, value)
+    // for (const cid of cids) {
+    //   this.#cidToCar.set(cid, carCid)
+    // }
 
     await this.withDB(async (db) => {
       const tx = db.transaction(['cars', 'cidToCar'], 'readwrite')
       await tx.objectStore('cars').put(value, carCid)
-      await tx.objectStore('cidToCar').put({ car: carCid, cids: Array.from(cids) })
+      await tx.objectStore('cidToCar').put({ pending: 'y', car: carCid, cids: Array.from(cids) })
       return await tx.done
     })
-  }
 
-  async getBlock (dataCID) {
-    // return await this.#valetGet(dataCID)
-    // const MEMcarCid = this.#cidToCar.get(dataCID)
+    // upload to web3.storage if we have credentials
+    if (this.uploadFunction) {
+      if (this.#alreadyEnqueued.has(carCid)) {
+        // console.log('already enqueued', carCid)
+        return
+      }
+      // don't await this, it will be done in the queue
+      // console.log('add to queue', carCid, value.length)
+      this.#uploadQueue.push({ carCid, value })
+      this.#alreadyEnqueued.add(carCid)
+    } else {
+      // console.log('no upload function', carCid, value.length, this.uploadFunction)
+    }
+  }
 
-    // return await this.withDB(async (db) => {
-    //   const tx = db.transaction(['cars', 'cidToCar'], 'readonly')
-    //   const carBytes = await tx.objectStore('cars').get(MEMcarCid)
-    //   const reader = await CarReader.fromBytes(carBytes)
-    //   const gotBlock = await reader.get(CID.parse(dataCID))
-    //   if (gotBlock) {
-    //     return gotBlock.bytes
-    //   }
-    // })
+  remoteBlockFunction = null
 
+  async getBlock (dataCID) {
     return await this.withDB(async (db) => {
       const tx = db.transaction(['cars', 'cidToCar'], 'readonly')
       const indexResp = await tx.objectStore('cidToCar').index('cids').get(dataCID)
       const carCid = indexResp?.car
       if (!carCid) {
-        return
+        throw new Error('Missing block: ' + dataCID)
       }
       const carBytes = await tx.objectStore('cars').get(carCid)
       const reader = await CarReader.fromBytes(carBytes)
@@ -76,67 +123,46 @@ export default class Valet {
       }
     })
   }
-
-  /**
-   * Internal function to load blocks from persistent storage.
-   * Currently it just searches all the cars for the block, but in the future
-   * we need to index the block CIDs to the cars, and reference that to find the block.
-   * This index will also allow us to use accelerator links for the gateway when needed.
-   * It can itself be a prolly tree...
-   * @param {string} cid
-   * @returns {Promise<Uint8Array|undefined>}
-   */
-  #valetGet = async (cid) => {
-    const carCid = this.#cidToCar.get(cid)
-    if (carCid) {
-      const carBytes = this.#cars.get(carCid)
-      const reader = await CarReader.fromBytes(carBytes)
-      const gotBlock = await reader.get(CID.parse(cid))
-      if (gotBlock) {
-        return gotBlock.bytes
-      }
-    }
-  }
 }
 
-export class MemoryValet {
-  #cars = new Map() // cars by cid
-  #cidToCar = new Map() // cid to car
+// export class MemoryValet {
+//   #cars = new Map() // cars by cid
+//   #cidToCar = new Map() // cid to car
 
-  /**
-       *
-       * @param {string} carCid
-       * @param {*} value
-       */
-  async parkCar (carCid, value, cids) {
-    this.#cars.set(carCid, value)
-    for (const cid of cids) {
-      this.#cidToCar.set(cid, carCid)
-    }
-  }
+//   /**
+//    *
+//    * @param {string} carCid
+//    * @param {*} value
+//    */
+//   async parkCar (carCid, value, cids) {
+//     this.#cars.set(carCid, value)
+//     for (const cid of cids) {
+//       this.#cidToCar.set(cid, carCid)
+//     }
+//   }
 
-  async getBlock (dataCID) {
-    return await this.#valetGet(dataCID)
-  }
+//   async getBlock (dataCID) {
+//     return await this.#valetGet(dataCID)
+//   }
 
-  /**
-     * Internal function to load blocks from persistent storage.
-     * Currently it just searches all the cars for the block, but in the future
-     * we need to index the block CIDs to the cars, and reference that to find the block.
-     * This index will also allow us to use accelerator links for the gateway when needed.
-     * It can itself be a prolly tree...
-     * @param {string} cid
-     * @returns {Promise<Uint8Array|undefined>}
-     */
-  #valetGet = async (cid) => {
-    const carCid = this.#cidToCar.get(cid)
-    if (carCid) {
-      const carBytes = this.#cars.get(carCid)
-      const reader = await CarReader.fromBytes(carBytes)
-      const gotBlock = await reader.get(CID.parse(cid))
-      if (gotBlock) {
-        return gotBlock.bytes
-      }
-    }
-  }
-}
+//   /**
+//    * Internal function to load blocks from persistent storage.
+//    * Currently it just searches all the cars for the block, but in the future
+//    * we need to index the block CIDs to the cars, and reference that to find the block.
+//    * This index will also allow us to use accelerator links for the gateway when needed.
+//    * It can itself be a prolly tree...
+//    * @param {string} cid
+//    * @returns {Promise<Uint8Array|undefined>}
+//    */
+//   #valetGet = async (cid) => {
+//     const carCid = this.#cidToCar.get(cid)
+//     if (carCid) {
+//       const carBytes = this.#cars.get(carCid)
+//       const reader = await CarReader.fromBytes(carBytes)
+//       const gotBlock = await reader.get(CID.parse(cid))
+//       if (gotBlock) {
+//         return gotBlock.bytes
+//       }
+//     }
+//   }
+// }

package/test/clock.test.js

@@ -1,6 +1,14 @@
-import { describe, it } from 'mocha'
+/* global describe, it */
+// import { describe, it } from 'mocha'
 import assert from 'node:assert'
-import { advance, EventBlock, findCommonAncestorWithSortedEvents, findUnknownSortedEvents, decodeEventBlock, findEventsToSync } from '../src/clock.js'
+import {
+  advance,
+  EventBlock,
+  findCommonAncestorWithSortedEvents,
+  findUnknownSortedEvents,
+  decodeEventBlock,
+  findEventsToSync
+} from '../src/clock.js'
 // import { vis } from '../src/clock.js'
 import { Blockstore, seqEventData, setSeq } from './helpers.js'
 
@@ -34,7 +42,11 @@ describe('Clock', () => {
     assert.equal(head[0].toString(), event.cid.toString())
 
     const sinceHead = head
-    const toSync = await findUnknownSortedEvents(blocks, sinceHead, await findCommonAncestorWithSortedEvents(blocks, sinceHead))
+    const toSync = await findUnknownSortedEvents(
+      blocks,
+      sinceHead,
+      await findCommonAncestorWithSortedEvents(blocks, sinceHead)
+    )
     assert.equal(toSync.length, 0)
   })
 
@@ -418,7 +430,11 @@ describe('Clock', () => {
     assert.equal(head[1].toString(), event1.cid.toString())
 
     let sinceHead = head1
-    let toSync = await findUnknownSortedEvents(blocks, sinceHead, await findCommonAncestorWithSortedEvents(blocks, sinceHead))
+    let toSync = await findUnknownSortedEvents(
+      blocks,
+      sinceHead,
+      await findCommonAncestorWithSortedEvents(blocks, sinceHead)
+    )
     // assert.equal(toSync.length, 1) // 0
     // assert.equal(toSync[0].cid.toString(), event0.cid.toString())
 
@@ -430,12 +446,20 @@ describe('Clock', () => {
     assert.equal(head.length, 1)
 
     sinceHead = head2
-    toSync = await findUnknownSortedEvents(blocks, sinceHead, await findCommonAncestorWithSortedEvents(blocks, sinceHead))
+    toSync = await findUnknownSortedEvents(
+      blocks,
+      sinceHead,
+      await findCommonAncestorWithSortedEvents(blocks, sinceHead)
+    )
     assert.equal(toSync.length, 0)
 
     // todo do these since heads make sense?
     sinceHead = [...head0, ...head2]
-    toSync = await findUnknownSortedEvents(blocks, sinceHead, await findCommonAncestorWithSortedEvents(blocks, sinceHead))
+    toSync = await findUnknownSortedEvents(
+      blocks,
+      sinceHead,
+      await findCommonAncestorWithSortedEvents(blocks, sinceHead)
+    )
     // console.log('need', toSync.map(b => b.value.data))
     // assert.equal(toSync.length, 2) // 0
     // assert.equal(toSync[0].cid.toString(), event1.cid.toString())
@@ -668,7 +692,11 @@ describe('Clock', () => {
     const roothead = head
     // db root
     let sinceHead = [...roothead]
-    let toSync = await findUnknownSortedEvents(blocks, sinceHead, await findCommonAncestorWithSortedEvents(blocks, sinceHead))
+    let toSync = await findUnknownSortedEvents(
+      blocks,
+      sinceHead,
+      await findCommonAncestorWithSortedEvents(blocks, sinceHead)
+    )
     assert.equal(toSync.length, 0) // we use all docs for first query in Fireproof
 
     // create bob
@@ -680,11 +708,19 @@ describe('Clock', () => {
 
     const event0head = head
     sinceHead = event0head
-    toSync = await findUnknownSortedEvents(blocks, sinceHead, await findCommonAncestorWithSortedEvents(blocks, sinceHead))
+    toSync = await findUnknownSortedEvents(
+      blocks,
+      sinceHead,
+      await findCommonAncestorWithSortedEvents(blocks, sinceHead)
+    )
     assert.equal(toSync.length, 0)
 
     sinceHead = [...roothead, ...event0head]
-    toSync = await findUnknownSortedEvents(blocks, sinceHead, await findCommonAncestorWithSortedEvents(blocks, sinceHead))
+    toSync = await findUnknownSortedEvents(
+      blocks,
+      sinceHead,
+      await findCommonAncestorWithSortedEvents(blocks, sinceHead)
+    )
     assert.equal(toSync.length, 1)
 
     // create carol
@@ -705,7 +741,11 @@ describe('Clock', () => {
     // for await (const line of vis(blocks, head)) console.log(line)
 
     sinceHead = [...event1head, ...roothead]
-    toSync = await findUnknownSortedEvents(blocks, sinceHead, await findCommonAncestorWithSortedEvents(blocks, sinceHead))
+    toSync = await findUnknownSortedEvents(
+      blocks,
+      sinceHead,
+      await findCommonAncestorWithSortedEvents(blocks, sinceHead)
+    )
 
     assert.equal(toSync.length, 2)
 
@@ -715,11 +755,19 @@ describe('Clock', () => {
     const event2head = head
 
     sinceHead = [...event2head, ...event0head]
-    toSync = await findUnknownSortedEvents(blocks, sinceHead, await findCommonAncestorWithSortedEvents(blocks, sinceHead))
+    toSync = await findUnknownSortedEvents(
+      blocks,
+      sinceHead,
+      await findCommonAncestorWithSortedEvents(blocks, sinceHead)
+    )
     assert.equal(toSync.length, 2)
 
     sinceHead = [...event2head, ...event1head]
-    toSync = await findUnknownSortedEvents(blocks, sinceHead, await findCommonAncestorWithSortedEvents(blocks, sinceHead))
+    toSync = await findUnknownSortedEvents(
+      blocks,
+      sinceHead,
+      await findCommonAncestorWithSortedEvents(blocks, sinceHead)
+    )
     assert.equal(toSync.length, 1)
   })
 })

package/test/db-index.test.js

@@ -207,8 +207,11 @@ describe('Index query with bad index definition', () => {
     })
   })
   it('query index range', async () => {
+    const oldErrFn = console.error
+    console.error = () => {}
     await index.query({ range: [41, 44] }).catch((e) => {
       assert(/missingField/.test(e.message))
+      console.error = oldErrFn
     })
   })
 })

package/test/listener.test.js

@@ -32,7 +32,6 @@ describe('Listener', () => {
   it('all listeners get the reset event', (done) => {
     let count = 0
     const check = () => {
-      console.log('increment check count')
       count++
       if (count === 3) done()
     }
@@ -76,7 +75,7 @@ describe('Listener', () => {
     // it's safe to make this number longer if it start failing
     await sleep(50)
     assert.equal(people, 6)
-  }).timeout(200)
+  }).timeout(2000)
   it('shares events since db.clock', (done) => {
     const clock = database.clock
     const afterEvent = () => {
@@ -96,7 +95,7 @@ describe('Listener', () => {
         if (people === 1) afterEvent()
       }, clock)
     })
-  }).timeout(200)
+  }).timeout(2000)
 })
 
 const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))

package/test/valet.test.js

@@ -0,0 +1,59 @@
+import { before, describe, it } from 'mocha'
+import assert from 'node:assert'
+import Valet from '../src/valet.js'
+
+describe('new Valet', () => {
+  let val
+  const calls = []
+
+  before(async () => {
+    val = new Valet()
+    val.uploadFunction = async (carCid, value) => {
+      calls.push({ carCid, value })
+    }
+  })
+  it('has default attributes', async () => {
+    assert(val.getBlock)
+    assert(val.parkCar)
+  })
+  it('can park a car and serve the blocks', async () => {
+    await val
+      .parkCar('carCid', carBytes, [
+        'bafyreifwghknmzabvgearl72url3v5leqqhtafvybcsrceelc3psqkocoi',
+        'bafyreieth2ckopwivda5mf6vu76xwqvox3q5wsaxgbmxy2dgrd4hfuzmma'
+      ])
+      .then((car) => {
+        assert('car parked')
+        val.getBlock('bafyreieth2ckopwivda5mf6vu76xwqvox3q5wsaxgbmxy2dgrd4hfuzmma').then((block) => {
+          assert.equal(block.length, carBytes.length)
+        })
+      })
+  })
+  it('calls the upload function', () => {
+    assert.equal(calls.length, 1)
+    assert.equal(calls[0].carCid, 'carCid')
+  })
+})
+
+const carBytes = new Uint8Array([
+  58, 162, 101, 114, 111, 111, 116, 115, 129, 216, 42, 88, 37, 0, 1, 113, 18, 32, 147, 62, 132, 167, 62, 200, 168, 193,
+  214, 23, 213, 167, 253, 123, 66, 174, 190, 225, 219, 72, 23, 48, 89, 124, 104, 102, 136, 248, 114, 211, 44, 96, 103,
+  118, 101, 114, 115, 105, 111, 110, 1, 108, 1, 113, 18, 32, 182, 49, 212, 214, 100, 1, 169, 136, 8, 175, 250, 164, 87,
+  186, 245, 100, 132, 15, 48, 22, 184, 8, 165, 17, 16, 139, 22, 223, 40, 41, 194, 114, 162, 100, 108, 101, 97, 102, 129,
+  130, 120, 36, 49, 101, 102, 51, 98, 51, 50, 97, 45, 51, 99, 51, 97, 45, 52, 98, 53, 101, 45, 57, 99, 49, 99, 45, 56,
+  99, 53, 99, 48, 99, 53, 99, 48, 99, 53, 99, 162, 99, 97, 103, 101, 24, 42, 100, 110, 97, 109, 101, 101, 97, 108, 105,
+  99, 101, 102, 99, 108, 111, 115, 101, 100, 244, 208, 2, 1, 113, 18, 32, 147, 62, 132, 167, 62, 200, 168, 193, 214, 23,
+  213, 167, 253, 123, 66, 174, 190, 225, 219, 72, 23, 48, 89, 124, 104, 102, 136, 248, 114, 211, 44, 96, 162, 100, 100,
+  97, 116, 97, 164, 99, 107, 101, 121, 120, 36, 49, 101, 102, 51, 98, 51, 50, 97, 45, 51, 99, 51, 97, 45, 52, 98, 53,
+  101, 45, 57, 99, 49, 99, 45, 56, 99, 53, 99, 48, 99, 53, 99, 48, 99, 53, 99, 100, 114, 111, 111, 116, 163, 99, 99,
+  105, 100, 216, 42, 88, 37, 0, 1, 113, 18, 32, 182, 49, 212, 214, 100, 1, 169, 136, 8, 175, 250, 164, 87, 186, 245,
+  100, 132, 15, 48, 22, 184, 8, 165, 17, 16, 139, 22, 223, 40, 41, 194, 114, 101, 98, 121, 116, 101, 115, 88, 72, 162,
+  100, 108, 101, 97, 102, 129, 130, 120, 36, 49, 101, 102, 51, 98, 51, 50, 97, 45, 51, 99, 51, 97, 45, 52, 98, 53, 101,
+  45, 57, 99, 49, 99, 45, 56, 99, 53, 99, 48, 99, 53, 99, 48, 99, 53, 99, 162, 99, 97, 103, 101, 24, 42, 100, 110, 97,
+  109, 101, 101, 97, 108, 105, 99, 101, 102, 99, 108, 111, 115, 101, 100, 244, 101, 118, 97, 108, 117, 101, 162, 100,
+  108, 101, 97, 102, 129, 130, 120, 36, 49, 101, 102, 51, 98, 51, 50, 97, 45, 51, 99, 51, 97, 45, 52, 98, 53, 101, 45,
+  57, 99, 49, 99, 45, 56, 99, 53, 99, 48, 99, 53, 99, 48, 99, 53, 99, 162, 99, 97, 103, 101, 24, 42, 100, 110, 97, 109,
+  101, 101, 97, 108, 105, 99, 101, 102, 99, 108, 111, 115, 101, 100, 244, 100, 116, 121, 112, 101, 99, 112, 117, 116,
+  101, 118, 97, 108, 117, 101, 162, 99, 97, 103, 101, 24, 42, 100, 110, 97, 109, 101, 101, 97, 108, 105, 99, 101, 103,
+  112, 97, 114, 101, 110, 116, 115, 128
+])