@fireproof/core 0.0.6 → 0.0.7

package/hooks/use-fireproof.ts

@@ -15,8 +15,6 @@ export const FireproofCtx = createContext<FireproofCtxValue>({
   ready: false,
 })
 
-
-
 const inboundSubscriberQueue = new Map()
 const database = Fireproof.storage()
 const listener = new Listener(database)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "description": "Realtime database for IPFS",
   "main": "index.js",
   "type": "module",
@@ -37,7 +37,7 @@
     "cli-color": "^2.0.3",
     "idb": "^7.1.1",
     "multiformats": "^11.0.1",
-    "prolly-trees": "^0.2.5",
+    "prolly-trees": "1.0.2",
     "sade": "^1.8.1"
   },
   "devDependencies": {
@@ -62,12 +62,12 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/jchris/fireproof.git"
+    "url": "git+https://github.com/fireproof-storage/fireproof.git"
   },
   "bugs": {
-    "url": "https://github.com/jchris/fireproof/issues"
+    "url": "https://github.com/fireproof-storage/fireproof/issues"
   },
-  "homepage": "https://github.com/jchris/fireproof#readme",
+  "homepage": "https://github.com/fireproof-storage/fireproof#readme",
   "workspaces": [
     "examples/todomvc"
   ]

package/src/blockstore.js

@@ -36,11 +36,15 @@ export default class TransactionBlockstore {
   /** @type {Map<string, Uint8Array>} */
   #oldBlocks = new Map()
 
-  valet = new Valet() // cars by cid
+  valet = null
 
   #instanceId = 'blkz.' + Math.random().toString(36).substring(2, 4)
   #inflightTransactions = new Set()
 
+  constructor (name) {
+    this.valet = new Valet(name)
+  }
+
   /**
    * Get a block from the store.
    *
@@ -111,15 +115,15 @@ export default class TransactionBlockstore {
    * @yields {AnyBlock}
    * @returns {AsyncGenerator<AnyBlock>}
    */
-  * entries () {
-    // todo needs transaction blocks?
-    // for (const [str, bytes] of this.#blocks) {
-    //   yield { cid: parse(str), bytes }
-    // }
-    for (const [str, bytes] of this.#oldBlocks) {
-      yield { cid: parse(str), bytes }
-    }
-  }
+  // * entries () {
+  //   // needs transaction blocks?
+  //   // for (const [str, bytes] of this.#blocks) {
+  //   //   yield { cid: parse(str), bytes }
+  //   // }
+  //   for (const [str, bytes] of this.#oldBlocks) {
+  //     yield { cid: parse(str), bytes }
+  //   }
+  // }
 
   /**
    * Begin a transaction. Ensures the uncommited blocks are empty at the begining.

package/src/clock.js

@@ -22,7 +22,7 @@ import { CIDCounter } from 'prolly-trees/utils'
  * Advance the clock by adding an event.
  *
  * @template T
- * @param {import('./block').BlockFetcher} blocks Block storage.
+ * @param {import('../test/block').BlockFetcher} blocks Block storage.
  * @param {EventLink<T>[]} head The head of the clock.
  * @param {EventLink<T>} event The event to add.
  * @returns {Promise<EventLink<T>[]>} The new head of the clock.
@@ -89,7 +89,7 @@ export class EventBlock extends Block {
 
 /** @template T */
 export class EventFetcher {
-  /** @param {import('./block').BlockFetcher} blocks */
+  /** @param {import('../test/block').BlockFetcher} blocks */
   constructor (blocks) {
     /** @private */
     this._blocks = blocks
@@ -162,7 +162,7 @@ async function contains (events, a, b) {
 
 /**
  * @template T
- * @param {import('./block').BlockFetcher} blocks Block storage.
+ * @param {import('../test/block').BlockFetcher} blocks Block storage.
  * @param {EventLink<T>[]} head
  * @param {object} [options]
  * @param {(b: EventBlockView<T>) => string} [options.renderNodeLabel]
@@ -285,6 +285,7 @@ async function findSortedEvents (events, head, tail) {
   const all = await Promise.all(head.map((h) => findEvents(events, h, tail)))
   for (const arr of all) {
     for (const { event, depth } of arr) {
+      // console.log('event value', event.value.data.value)
       const info = weights.get(event.cid.toString())
       if (info) {
         info.weight += depth

package/src/db-index.js

@@ -11,22 +11,38 @@ import charwise from 'charwise'
 
 const ALWAYS_REBUILD = true // todo: remove this
 
-const arrayCompare = (a, b) => {
-  if (Array.isArray(a) && Array.isArray(b)) {
-    const len = Math.min(a.length, b.length)
-    for (let i = 0; i < len; i++) {
-      const comp = simpleCompare(a[i], b[i])
-      if (comp !== 0) {
-        return comp
-      }
-    }
-    return simpleCompare(a.length, b.length)
-  } else {
-    return simpleCompare(a, b)
-  }
+// const arrayCompare = (a, b) => {
+//   if (Array.isArray(a) && Array.isArray(b)) {
+//     const len = Math.min(a.length, b.length)
+//     for (let i = 0; i < len; i++) {
+//       const comp = simpleCompare(a[i], b[i])
+//       if (comp !== 0) {
+//         return comp
+//       }
+//     }
+//     return simpleCompare(a.length, b.length)
+//   } else {
+//     return simpleCompare(a, b)
+//   }
+// }
+
+const compare = (a, b) => {
+  const [aKey, aRef] = a
+  const [bKey, bRef] = b
+  const comp = simpleCompare(aKey, bKey)
+  if (comp !== 0) return comp
+  return refCompare(aRef, bRef)
 }
 
-const opts = { cache, chunker: bf(3), codec, hasher, compare: arrayCompare }
+const refCompare = (aRef, bRef) => {
+  if (Number.isNaN(aRef)) return -1
+  if (Number.isNaN(bRef)) throw new Error('ref may not be Infinity or NaN')
+  if (!Number.isFinite(aRef)) return 1
+  // if (!Number.isFinite(bRef)) throw new Error('ref may not be Infinity or NaN')
+  return simpleCompare(aRef, bRef)
+}
+
+const opts = { cache, chunker: bf(3), codec, hasher, compare }
 
 const makeDoc = ({ key, value }) => ({ _id: key, ...value })
 
@@ -134,7 +150,10 @@ export default class DbIndex {
     return {
       proof: { index: await cidsToProof(response.cids) },
       // TODO fix this naming upstream in prolly/db-DbIndex?
-      rows: response.result.map(({ id, key, row }) => ({ id: key, key: charwise.decode(id), value: row }))
+      rows: response.result.map(({ id, key, row }) => {
+        // console.log('query', id, key, row)
+        return ({ id, key: charwise.decode(key), value: row })
+      })
     }
   }
 
@@ -188,11 +207,6 @@ export default class DbIndex {
 
     this.dbHead = result.clock
   }
-
-  // todo use the DbIndex from other peers?
-  // we might need to add CRDT logic to it for that
-  // it would only be a performance improvement, but might add a lot of complexity
-  //   advanceIndex ()) {}
 }
 
 /**
@@ -240,6 +254,7 @@ async function doIndexQuery (blocks, dbIndexRoot, dbIndex, query) {
     return dbIndex.range(...encodedRange)
   } else if (query.key) {
     const encodedKey = charwise.encode(query.key)
+    console.log('getting key', encodedKey)
     return dbIndex.get(encodedKey)
   }
 }

package/src/fireproof.js

@@ -1,5 +1,6 @@
-import { put, get, getAll, eventsSince } from './prolly.js'
-import Blockstore, { doTransaction } from './blockstore.js'
+import { vis, put, get, getAll, eventsSince } from './prolly.js'
+import TransactionBlockstore, { doTransaction } from './blockstore.js'
+import charwise from 'charwise'
 
 // const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
 
@@ -26,16 +27,17 @@ export default class Fireproof {
    * @static
    * @returns {Fireproof} - a new Fireproof instance
    */
-  static storage = () => {
-    return new Fireproof(new Blockstore(), [])
+  static storage = (name) => {
+    return new Fireproof(new TransactionBlockstore(name), [], { name })
   }
 
-  constructor (blocks, clock, config = {}, authCtx = {}) {
+  constructor (blocks, clock, config, authCtx = {}) {
+    this.name = config?.name || 'global'
+    this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`
     this.blocks = blocks
     this.clock = clock
     this.config = config
     this.authCtx = authCtx
-    this.instanceId = 'fp.' + Math.random().toString(36).substring(2, 7)
   }
 
   /**
@@ -49,7 +51,7 @@ export default class Fireproof {
    */
   snapshot (clock) {
     // how to handle listeners, views, and config?
-    // todo needs a test for that
+    // todo needs a test for listeners, views, and config
     return new Fireproof(this.blocks, clock || this.clock)
   }
 
@@ -93,7 +95,7 @@ export default class Fireproof {
     if (event) {
       const resp = await eventsSince(this.blocks, this.clock, event)
       const docsMap = new Map()
-      for (const { key, type, value } of resp.result) {
+      for (const { key, type, value } of resp.result.map(decodeEvent)) {
         if (type === 'del') {
           docsMap.set(key, { key, del: true })
         } else {
@@ -105,11 +107,15 @@ export default class Fireproof {
       // console.log('change rows', this.instanceId, rows)
     } else {
       const allResp = await getAll(this.blocks, this.clock)
-      rows = allResp.result.map(({ key, value }) => ({ key, value }))
+      rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value })))
       dataCIDs = allResp.cids
       // console.log('dbdoc rows', this.instanceId, rows)
     }
-    return { rows, clock: this.clock, proof: { data: await cidsToProof(dataCIDs), clock: await cidsToProof(clockCIDs) } }
+    return {
+      rows,
+      clock: this.clock,
+      proof: { data: await cidsToProof(dataCIDs), clock: await cidsToProof(clockCIDs) }
+    }
   }
 
   /**
@@ -195,7 +201,8 @@ export default class Fireproof {
    * @param {Object<{key : string, value: any}>} event - the event to add
    * @returns {Object<{ id: string, clock: CID[] }>} - The result of adding the event to storage
    */
-  async #putToProllyTree (event, clock = null) {
+  async #putToProllyTree (decodedEvent, clock = null) {
+    const event = encodeEvent(decodedEvent)
     if (clock && JSON.stringify(clock) !== JSON.stringify(this.clock)) {
       // we need to check and see what version of the document exists at the clock specified
       // if it is the same as the one we are trying to put, then we can proceed
@@ -215,8 +222,12 @@ export default class Fireproof {
       throw new Error('failed to put at storage layer')
     }
     this.clock = result.head // do we want to do this as a finally block
-    await this.#notifyListeners([event])
-    return { id: event.key, clock: this.clock, proof: { data: await cidsToProof(result.cids), clock: await cidsToProof(result.clockCIDs) } }
+    await this.#notifyListeners([decodedEvent]) // this type is odd
+    return {
+      id: decodedEvent.key,
+      clock: this.clock,
+      proof: { data: await cidsToProof(result.cids), clock: await cidsToProof(result.clockCIDs) }
+    }
     // todo should include additions (or split clock)
   }
 
@@ -255,7 +266,7 @@ export default class Fireproof {
    */
   async get (key, opts = {}) {
     const clock = opts.clock || this.clock
-    const resp = await get(this.blocks, clock, key)
+    const resp = await get(this.blocks, clock, charwise.encode(key))
 
     // this tombstone is temporary until we can get the prolly tree to delete
     if (!resp || resp.result === null) {
@@ -273,6 +284,10 @@ export default class Fireproof {
     return doc
   }
 
+  async * vis () {
+    return yield * vis(this.blocks, this.clock)
+  }
+
   setCarUploader (carUploaderFn) {
     console.log('registering car uploader')
     // https://en.wikipedia.org/wiki/Law_of_Demeter - this is a violation of the law of demeter
@@ -290,3 +305,14 @@ export async function cidsToProof (cids) {
   const all = await cids.all()
   return [...all].map((cid) => cid.toString())
 }
+
+function decodeEvent (event) {
+  const decodedKey = charwise.decode(event.key)
+  return { ...event, key: decodedKey }
+}
+
+function encodeEvent (event) {
+  if (!(event && event.key)) return
+  const encodedKey = charwise.encode(event.key)
+  return { ...event, key: encodedKey }
+}

package/src/listener.js

@@ -11,12 +11,6 @@
 
 export default class Listener {
   #subcribers = new Map()
-
-  // todo code review if there is a better way that doesn't create a circular reference
-  // because otherwise we need to document that the user must call stopListening
-  // or else the listener will never be garbage collected
-  // maybe we can use WeakRef on the db side
-  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WeakRef
   #doStopListening = null
 
   constructor (database, routingFn) {

package/src/prolly.js

@@ -6,13 +6,12 @@ import {
   findEventsToSync
 } from './clock.js'
 import { create, load } from 'prolly-trees/map'
+// import { create, load } from '../../../../prolly-trees/src/map.js'
+import { nocache as cache } from 'prolly-trees/cache'
+import { CIDCounter, bf, simpleCompare as compare } from 'prolly-trees/utils'
 import * as codec from '@ipld/dag-cbor'
 import { sha256 as hasher } from 'multiformats/hashes/sha2'
-import { MemoryBlockstore, MultiBlockFetcher } from './block.js'
 import { doTransaction } from './blockstore.js'
-
-import { nocache as cache } from 'prolly-trees/cache'
-import { CIDCounter, bf, simpleCompare as compare } from 'prolly-trees/utils'
 import { create as createBlock } from 'multiformats/block'
 const opts = { cache, chunker: bf(3), codec, hasher, compare }
 
@@ -22,9 +21,9 @@ const withLog = async (label, fn) => {
   return resp
 }
 
-// todo should also return a CIDCounter
+// should also return a CIDCounter
 export const makeGetBlock = (blocks) => {
-  // const cids = new CIDCounter() // todo this could be used for proofs of mutations
+  // const cids = new CIDCounter() // this could be used for proofs of mutations
   const getBlockFn = async (address) => {
     const { cid, bytes } = await withLog(address, () => blocks.get(address))
     // cids.add({ address: cid })
@@ -55,18 +54,17 @@ export const makeGetBlock = (blocks) => {
  *   event: CID[]
  * }>}
  */
-async function createAndSaveNewEvent (
+async function createAndSaveNewEvent ({
   inBlocks,
-  mblocks,
-  getBlock,
   bigPut,
   root,
-  { key, value, del },
+  event: inEvent,
   head,
   additions,
   removals = []
-) {
+}) {
   let cids
+  const { key, value, del } = inEvent
   const data = {
     type: 'put',
     root: {
@@ -100,20 +98,20 @@ async function createAndSaveNewEvent (
 }
 
 const makeGetAndPutBlock = (inBlocks) => {
-  const mblocks = new MemoryBlockstore()
-  const blocks = new MultiBlockFetcher(mblocks, inBlocks)
-  const { getBlock, cids } = makeGetBlock(blocks)
+  // const mblocks = new MemoryBlockstore()
+  // const blocks = new MultiBlockFetcher(mblocks, inBlocks)
+  const { getBlock, cids } = makeGetBlock(inBlocks)
   const put = inBlocks.put.bind(inBlocks)
   const bigPut = async (block, additions) => {
     // console.log('bigPut', block.cid.toString())
     const { cid, bytes } = block
     put(cid, bytes)
-    mblocks.putSync(cid, bytes)
+    // mblocks.putSync(cid, bytes)
     if (additions) {
       additions.set(cid.toString(), block)
     }
   }
-  return { getBlock, bigPut, mblocks, blocks, cids }
+  return { getBlock, bigPut, blocks: inBlocks, cids }
 }
 
 const bulkFromEvents = (sorted) =>
@@ -143,7 +141,7 @@ const prollyRootFromAncestor = async (events, ancestor, getBlock) => {
 /**
  * Put a value (a CID) for the given key. If the key exists it's value is overwritten.
  *
- * @param {import('./block').BlockFetcher} blocks Bucket block storage.
+ * @param {import('../test/block.js').BlockFetcher} blocks Bucket block storage.
  * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
  * @param {string} key The key of the value to put.
  * @param {CID} value The value to put.
@@ -151,7 +149,7 @@ const prollyRootFromAncestor = async (events, ancestor, getBlock) => {
  * @returns {Promise<Result>}
  */
 export async function put (inBlocks, head, event, options) {
-  const { getBlock, bigPut, mblocks, blocks } = makeGetAndPutBlock(inBlocks)
+  const { getBlock, bigPut, blocks } = makeGetAndPutBlock(inBlocks)
 
   // If the head is empty, we create a new event and return the root and addition blocks
   if (!head.length) {
@@ -161,12 +159,14 @@ export async function put (inBlocks, head, event, options) {
       root = await node.block
       bigPut(root, additions)
     }
-    return createAndSaveNewEvent(inBlocks, mblocks, getBlock, bigPut, root, event, head, Array.from(additions.values()))
+    return createAndSaveNewEvent({ inBlocks, bigPut, root, event, head, additions: Array.from(additions.values()) })
   }
 
   // Otherwise, we find the common ancestor and update the root and other blocks
   const events = new EventFetcher(blocks)
+  // this is returning more events than necessary
   const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head)
+  // console.log('sorted', JSON.stringify(sorted.map(({ value: { data: { key, value } } }) => ({ key, value }))))
   const prollyRootNode = await prollyRootFromAncestor(events, ancestor, getBlock)
 
   const bulkOperations = bulkFromEvents(sorted)
@@ -178,22 +178,20 @@ export async function put (inBlocks, head, event, options) {
     bigPut(nb, additions)
   }
   // additions are new blocks
-  return createAndSaveNewEvent(
+  return createAndSaveNewEvent({
     inBlocks,
-    mblocks,
-    getBlock,
     bigPut,
-    prollyRootBlock,
+    root: prollyRootBlock,
     event,
     head,
-    Array.from(additions.values()) /*, todo? Array.from(removals.values()) */
-  )
+    additions: Array.from(additions.values()) /*, todo? Array.from(removals.values()) */
+  })
 }
 
 /**
  * Determine the effective prolly root given the current merkle clock head.
  *
- * @param {import('./block').BlockFetcher} blocks Bucket block storage.
+ * @param {import('../test/block.js').BlockFetcher} blocks Bucket block storage.
  * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
  */
 export async function root (inBlocks, head) {
@@ -208,22 +206,22 @@ export async function root (inBlocks, head) {
   // Perform bulk operations (put or delete) for each event in the sorted array
   const bulkOperations = bulkFromEvents(sorted)
   const { root: newProllyRootNode, blocks: newBlocks } = await prollyRootNode.bulk(bulkOperations)
-  const prollyRootBlock = await newProllyRootNode.block
-  // console.log('emphemeral blocks', newBlocks.map((nb) => nb.cid.toString()))
+  // const prollyRootBlock = await newProllyRootNode.block
+  // console.log('newBlocks', newBlocks.map((nb) => nb.cid.toString()))
   // todo maybe these should go to a temp blockstore?
   await doTransaction('root', inBlocks, async (transactionBlockstore) => {
     const { bigPut } = makeGetAndPutBlock(transactionBlockstore)
     for (const nb of newBlocks) {
       bigPut(nb)
     }
-    bigPut(prollyRootBlock)
+    // bigPut(prollyRootBlock)
   })
   return { cids: events.cids, node: newProllyRootNode }
 }
 
 /**
  * Get the list of events not known by the `since` event
- * @param {import('./block').BlockFetcher} blocks Bucket block storage.
+ * @param {import('../test/block.js').BlockFetcher} blocks Bucket block storage.
  * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
  * @param {import('./clock').EventLink<EventData>} since Event to compare against.
  * @returns {Promise<import('./clock').EventLink<EventData>[]>}
@@ -239,7 +237,7 @@ export async function eventsSince (blocks, head, since) {
 
 /**
  *
- * @param {import('./block').BlockFetcher} blocks Bucket block storage.
+ * @param {import('../test/block.js').BlockFetcher} blocks Bucket block storage.
  * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
  *
  * @returns {Promise<import('./prolly').Entry[]>}
@@ -257,7 +255,7 @@ export async function getAll (blocks, head) {
 }
 
 /**
- * @param {import('./block').BlockFetcher} blocks Bucket block storage.
+ * @param {import('../test/block.js').BlockFetcher} blocks Bucket block storage.
  * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
  * @param {string} key The key of the value to retrieve.
  */
@@ -270,3 +268,15 @@ export async function get (blocks, head, key) {
   const { result, cids } = await prollyRootNode.get(key)
   return { result, cids, clockCIDs }
 }
+
+export async function * vis (blocks, head) {
+  if (!head.length) {
+    return { cids: new CIDCounter(), result: null }
+  }
+  const { node: prollyRootNode, cids } = await root(blocks, head)
+  const lines = []
+  for await (const line of prollyRootNode.vis()) {
+    yield line
+  }
+  return { vis: lines.join('\n'), cids }
+}

package/src/valet.js

@@ -3,17 +3,8 @@ import { CID } from 'multiformats/cid'
 import { openDB } from 'idb'
 import cargoQueue from 'async/cargoQueue.js'
 
-// const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
-// let storageSupported = false
-// try {
-//   storageSupported = window.localStorage && true
-// } catch (e) {}
-// const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
-
 export default class Valet {
-  #cars = new Map() // cars by cid
-  #cidToCar = new Map() // cid to car
-  #db = null
+  idb = null
   #uploadQueue = null
   #alreadyEnqueued = new Set()
 
@@ -23,7 +14,8 @@ export default class Valet {
    */
   uploadFunction = null
 
-  constructor () {
+  constructor (name = 'default') {
+    this.name = name
     this.#uploadQueue = cargoQueue(async (tasks, callback) => {
       console.log(
         'queue worker',
@@ -59,9 +51,8 @@ export default class Valet {
   }
 
   withDB = async (dbWorkFun) => {
-    // if (!storageSupported) return
-    if (!this.#db) {
-      this.#db = await openDB('valet', 2, {
+    if (!this.idb) {
+      this.idb = await openDB(`fp.${this.name}.valet`, 2, {
         upgrade (db, oldVersion, newVersion, transaction) {
           if (oldVersion < 1) {
             db.createObjectStore('cars') // todo use database name
@@ -75,7 +66,7 @@ export default class Valet {
         }
       })
     }
-    return await dbWorkFun(this.#db)
+    return await dbWorkFun(this.idb)
   }
 
   /**
@@ -84,11 +75,6 @@ export default class Valet {
    * @param {*} value
    */
   async parkCar (carCid, value, cids) {
-    // this.#cars.set(carCid, value)
-    // for (const cid of cids) {
-    //   this.#cidToCar.set(cid, carCid)
-    // }
-
     await this.withDB(async (db) => {
       const tx = db.transaction(['cars', 'cidToCar'], 'readwrite')
       await tx.objectStore('cars').put(value, carCid)
@@ -130,45 +116,3 @@ export default class Valet {
     })
   }
 }
-
-// export class MemoryValet {
-//   #cars = new Map() // cars by cid
-//   #cidToCar = new Map() // cid to car
-
-//   /**
-//    *
-//    * @param {string} carCid
-//    * @param {*} value
-//    */
-//   async parkCar (carCid, value, cids) {
-//     this.#cars.set(carCid, value)
-//     for (const cid of cids) {
-//       this.#cidToCar.set(cid, carCid)
-//     }
-//   }
-
-//   async getBlock (dataCID) {
-//     return await this.#valetGet(dataCID)
-//   }
-
-//   /**
-//    * Internal function to load blocks from persistent storage.
-//    * Currently it just searches all the cars for the block, but in the future
-//    * we need to index the block CIDs to the cars, and reference that to find the block.
-//    * This index will also allow us to use accelerator links for the gateway when needed.
-//    * It can itself be a prolly tree...
-//    * @param {string} cid
-//    * @returns {Promise<Uint8Array|undefined>}
-//    */
-//   #valetGet = async (cid) => {
-//     const carCid = this.#cidToCar.get(cid)
-//     if (carCid) {
-//       const carBytes = this.#cars.get(carCid)
-//       const reader = await CarReader.fromBytes(carBytes)
-//       const gotBlock = await reader.get(CID.parse(cid))
-//       if (gotBlock) {
-//         return gotBlock.bytes
-//       }
-//     }
-//   }
-// }

package/{src → test}/block.js

@@ -1,8 +1,8 @@
 import { parse } from 'multiformats/link'
 
 /**
- * @typedef {{ cid: import('./link').AnyLink, bytes: Uint8Array }} AnyBlock
- * @typedef {{ get: (link: import('./link').AnyLink) => Promise<AnyBlock | undefined> }} BlockFetcher
+ * @typedef {{ cid: import('../src/link').AnyLink, bytes: Uint8Array }} AnyBlock
+ * @typedef {{ get: (link: import('../src/link').AnyLink) => Promise<AnyBlock | undefined> }} BlockFetcher
  */
 
 /** @implements {BlockFetcher} */
@@ -11,7 +11,7 @@ export class MemoryBlockstore {
   #blocks = new Map()
 
   /**
-   * @param {import('./link').AnyLink} cid
+   * @param {import('../src/link').AnyLink} cid
    * @returns {Promise<AnyBlock | undefined>}
    */
   async get (cid) {
@@ -21,7 +21,7 @@ export class MemoryBlockstore {
   }
 
   /**
-   * @param {import('./link').AnyLink} cid
+   * @param {import('../src/link').AnyLink} cid
    * @param {Uint8Array} bytes
    */
   async put (cid, bytes) {
@@ -30,7 +30,7 @@ export class MemoryBlockstore {
   }
 
   /**
-   * @param {import('./link').AnyLink} cid
+   * @param {import('../src/link').AnyLink} cid
    * @param {Uint8Array} bytes
    */
   putSync (cid, bytes) {
@@ -53,7 +53,7 @@ export class MultiBlockFetcher {
     this.#fetchers = fetchers
   }
 
-  /** @param {import('./link').AnyLink} link */
+  /** @param {import('../src/link').AnyLink} link */
   async get (link) {
     for (const f of this.#fetchers) {
       const v = await f.get(link)

package/test/clock.test.js

@@ -440,13 +440,8 @@ describe('Clock', () => {
     toSync = await testFindEventsToSync(blocks, sinceHead)
     assert.equal(toSync.length, 0)
 
-    // todo do these since heads make sense?
     sinceHead = [...head0, ...head2]
     toSync = await testFindEventsToSync(blocks, sinceHead)
-    // console.log('need', toSync.map(b => b.value.data))
-    // assert.equal(toSync.length, 2) // 0
-    // assert.equal(toSync[0].cid.toString(), event1.cid.toString())
-    // assert.equal(toSync[1].cid.toString(), event2.cid.toString())
   })
 
   it('add two events with some shared parents', async () => {

package/test/db-index.test.js

@@ -2,13 +2,13 @@ import { describe, it, beforeEach } from 'mocha'
 import assert from 'node:assert'
 import Blockstore from '../src/blockstore.js'
 import Fireproof from '../src/fireproof.js'
-import Index from '../src/db-index.js'
+import DbIndex from '../src/db-index.js'
 console.x = function () {}
 
-describe('Index query', () => {
+describe('DbIndex query', () => {
   let database, index
   beforeEach(async () => {
-    database = new Fireproof(new Blockstore(), []) // todo: these need a cloud name aka w3name, add this after we have cloud storage of blocks
+    database = Fireproof.storage()
     const docs = [
       { _id: 'a1s3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c', name: 'alice', age: 40 },
       { _id: 'b2s3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c', name: 'bob', age: 40 },
@@ -24,7 +24,7 @@ describe('Index query', () => {
       assert(response.id, 'should have id')
       assert.equal(response.id, id)
     }
-    index = new Index(database, function (doc, map) {
+    index = new DbIndex(database, function (doc, map) {
       map(doc.age, doc.name)
     })
   })
@@ -36,8 +36,10 @@ describe('Index query', () => {
     assert.equal(result.rows[0].key, 43)
     assert(result.rows[0].value === 'carol', 'correct value')
   })
-  it.skip('query exact key', async () => {
-    const result = await index.query({ key: 43 })
+  it('query exact key', async () => {
+    let result = await index.query({ range: [41, 44] })
+    assert(result.rows[0].key === 43, 'correct key')
+    result = await index.query({ key: 43 })
     assert(result, 'did return result')
     assert(result.rows)
     assert.equal(result.rows.length, 1, 'one row matched')
@@ -197,12 +199,12 @@ describe('Index query', () => {
   })
 })
 
-describe('Index query with bad index definition', () => {
+describe('DbIndex query with bad index definition', () => {
   let database, index
   beforeEach(async () => {
     database = new Fireproof(new Blockstore(), []) // todo: these need a cloud name aka w3name, add this after we have cloud storage of blocks
     await database.put({ _id: 'a1s3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c', name: 'alice', age: 40 })
-    index = new Index(database, function (doc, map) {
+    index = new DbIndex(database, function (doc, map) {
       map(doc.oops.missingField, doc.name)
     })
   })
@@ -214,5 +216,4 @@ describe('Index query with bad index definition', () => {
       console.error = oldErrFn
     })
   })
-  it.skip('reproduce missing block error from browser so we can turn off always rebuild', async () => {})
 })

package/test/fireproof.test.js

@@ -2,6 +2,7 @@ import { describe, it, beforeEach } from 'mocha'
 import assert from 'node:assert'
 import Blockstore from '../src/blockstore.js'
 import Fireproof from '../src/fireproof.js'
+import * as codec from '@ipld/dag-cbor'
 
 let database, resp0
 
@@ -9,14 +10,18 @@ let database, resp0
 
 describe('Fireproof', () => {
   beforeEach(async () => {
-    database = new Fireproof(new Blockstore(), []) // todo: these need a cloud name aka w3name, add this after we have cloud storage of blocks
+    database = Fireproof.storage('helloName')
     resp0 = await database.put({
       _id: '1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c',
       name: 'alice',
       age: 42
     })
   })
-
+  it('takes an optional name', () => {
+    assert.equal(database.name, 'helloName')
+    const x = database.blocks.valet.idb
+    assert.equal(x.name.toString(), 'fp.helloName.valet')
+  })
   it('put and get document', async () => {
     assert(resp0.id, 'should have id')
     assert.equal(resp0.id, '1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
@@ -27,13 +32,13 @@ describe('Fireproof', () => {
   })
   it('mvcc put and get document with _clock that matches', async () => {
     assert(resp0.clock, 'should have clock')
-    assert.equal(resp0.clock[0].toString(), 'bafyreieth2ckopwivda5mf6vu76xwqvox3q5wsaxgbmxy2dgrd4hfuzmma')
+    assert.equal(resp0.clock[0].toString(), 'bafyreiadhnnxgaeeqdxujfew6zxr4lnjyskkrg26cdjvk7tivy6dt4xmsm')
     const theDoc = await database.get('1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
     theDoc._clock = database.clock
     const put2 = await database.put(theDoc)
     assert.equal(put2.id, '1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
     assert.equal(put2.clock.length, 1)
-    assert.equal(put2.clock[0].toString(), 'bafyreida2c2ckhjfoz5ulmbbfe66ey4svvedrl4tzbvtoxags2qck7lj2i')
+    assert.equal(put2.clock[0].toString(), 'bafyreib2kck2fv73lgahfcd5imarslgxcmachbxxavhtwahx5ppjfts4qe')
   })
   it('get should return an object instance that is not the same as the one in the db', async () => {
     const theDoc = await database.get('1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
@@ -46,7 +51,7 @@ describe('Fireproof', () => {
   it('get with mvcc option', async () => {
     const theDoc = await database.get('1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c', { mvcc: true })
     assert(theDoc._clock, 'should have _clock')
-    assert.equal(theDoc._clock[0].toString(), 'bafyreieth2ckopwivda5mf6vu76xwqvox3q5wsaxgbmxy2dgrd4hfuzmma')
+    assert.equal(theDoc._clock[0].toString(), 'bafyreiadhnnxgaeeqdxujfew6zxr4lnjyskkrg26cdjvk7tivy6dt4xmsm')
   })
   it('get with mvcc option where someone else changed another document first', async () => {
     const theDoc = await database.get('1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c', { mvcc: true })
@@ -237,6 +242,34 @@ describe('Fireproof', () => {
     assert.equal(prevBob.age, 11)
   })
 
+  it('provides docs since tiny', async () => {
+    const result = await database.changesSince()
+    assert.equal(result.rows.length, 1)
+    assert.equal(result.rows[0].key, '1ef3b32a-3c3a-4b5e-9c1c-8c5c0c5c0c5c')
+
+    // console.log('result', result)
+
+    // const result2 = await database.changesSince(result.clock)
+    // console.log('result2', result2)
+    // assert.equal(result2.rows.length, 0)
+
+    const bKey = 'befbef-3c3a-4b5e-9c1c-bbbbbb'
+    const bvalue = {
+      _id: bKey,
+      name: 'bob',
+      age: 44
+    }
+    const response = await database.put(bvalue)
+    assert(response.id, 'should have id')
+    assert.equal(response.id, bKey)
+
+    const res3 = await database.changesSince()
+    assert.equal(res3.rows.length, 2)
+
+    const res4 = await database.changesSince(result.clock)
+    assert.equal(res4.rows.length, 1)
+  })
+
   it('provides docs since', async () => {
     const result = await database.changesSince()
     assert.equal(result.rows.length, 1)
@@ -275,8 +308,6 @@ describe('Fireproof', () => {
 
     const res5 = await database.changesSince(res4.clock)
 
-    // await database.visClock()
-
     assert.equal(res5.rows.length, 1)
 
     const res6 = await database.changesSince(result2.clock)
@@ -308,22 +339,48 @@ describe('Fireproof', () => {
     assert.equal((await database.changesSince()).rows.length, 1)
     let resp, doc, changes
     for (let index = 0; index < 200; index++) {
-      const id = '' + (300 - index).toString()
+      const id = '1' + (301 - index).toString()
+      console.log(`Putting id: ${id}, index: ${index}`)
       resp = await database.put({ index, _id: id }).catch(e => {
-        assert.equal(e.message, 'put failed on  _id: ' + id)
+        assert.fail(`put failed on _id: ${id}, error: ${e.message}`)
       })
-      assert(resp.id)
+      assert(resp.id, `Failed to obtain resp.id for _id: ${id}`)
+
+      console.log(`vis for update id: ${id}, index:`, index)
+      for await (const line of database.vis()) {
+        console.log(line)
+      }
+
       doc = await database.get(resp.id).catch(e => {
-        console.trace('failed', e)
-        assert.equal(e.message, 'get failed on _id: ' + id)
+        console.log('failed', e)
+        assert.fail(`get failed on _id: ${id}, error: ${e.message}`)
+      })
+
+      assert.equal(doc.index, index, `doc.index is not equal to index for _id: ${id}`)
+      changes = await database.changesSince().catch(async e => {
+        assert.fail(`changesSince failed on _id: ${id}, error: ${e.message}`)
       })
-      assert.equal(doc.index, index)
-      changes = await database.changesSince().catch(e => {
-        assert.equal(e.message, 'changesSince failed on  _id: ' + id)
+      changes.rows.forEach(row => {
+        for (const key in row) {
+          const value = row[key]
+          assert(!/^bafy/.test(value), `Unexpected "bafy..." value found at index ${index} in row ${JSON.stringify(row)}`)
+        }
       })
-      assert.equal(changes.rows.length, index + 2)
+      if (index > 3) {
+        const stored = await database.blocks.get('bafyreicumn7tvssch4xslbe4jjq55c6w3jt4yxyjagkr2tengsudato7vi').catch((e) => {
+          console.log(`Error getting block for index ${index}: ${e.message}`)
+        })
+        if (stored) {
+          const doc = codec.decode(await stored.bytes)
+          // console.log('stored', JSON.stringify(dec))
+          assert.equal(doc.closed, false)
+        }
+      }
+      console.log('changes: ', index, changes.rows.length, JSON.stringify(changes.rows))
+      assert.equal(changes.rows.length, index + 2, `failed on ${index}, with ${changes.rows.length} ${id}`)
     }
   }).timeout(20000)
+
   it('concurrent transactions', async () => {
     assert.equal((await database.changesSince()).rows.length, 1)
     const promises = []

package/test/helpers.js

@@ -3,7 +3,7 @@ import crypto from 'node:crypto'
 import * as Link from 'multiformats/link'
 import * as raw from 'multiformats/codecs/raw'
 import { sha256 } from 'multiformats/hashes/sha2'
-import { MemoryBlockstore } from '../src/block.js'
+import { MemoryBlockstore } from './block.js'
 
 // console.x = console.log
 // console.log = function (...args) {

package/test/prolly.test.js

@@ -110,7 +110,7 @@ describe('Prolly', () => {
     assert.equal(bvalue.toString(), data[0][1].toString())
   })
 
-  it.skip('linear put hundreds of values', async () => {
+  it.skip('passing, slow: linear put hundreds of values', async () => {
     const blocks = new Blockstore()
     const alice = new TestPail(blocks, [])
 
@@ -173,7 +173,7 @@ class TestPail {
   /** @param {string} key */
   async get (key) {
     const resp = await get(this.blocks, this.head, key)
-    console.log('prolly GET', key, resp)
+    // console.log('prolly GET', key, resp)
     return resp.result
   }
 

package/test/proofs.test.js

@@ -14,11 +14,11 @@ describe('Proofs', () => {
     doc = await database.get(ok.id, { mvcc: true })
   })
 
-  it.skip('first put result shoud not include proof', async () => {
+  it('first put result shoud not include proof', async () => {
     assert(ok.proof)
     assert(ok.proof.data)
     assert(ok.proof.clock)
-    console.log('ok', ok)
+    // console.log('ok', ok)
     assert.equal(ok.proof.data.length, 0)
     assert.equal(ok.proof.clock.length, 0)
 
@@ -31,7 +31,7 @@ describe('Proofs', () => {
     assert(ok2.proof)
     assert(ok2.proof.data)
     assert(ok2.proof.clock)
-    console.log('ok2', ok2)
+    // console.log('ok2', ok2)
     assert.equal(ok2.proof.data.length, 1)
     assert.equal(ok2.proof.clock.length, 1)
 
@@ -47,7 +47,7 @@ describe('Proofs', () => {
     assert(doc._proof.clock)
     assert.equal(doc._proof.data.length, 1)
     assert.equal(doc._proof.clock.length, 1)
-    assert.equal(doc._proof.data[0], 'bafyreibsbxxd4ueujryihk6xza2ekwhzsh6pzuu5fysft5ilz7cbw6bjju')
-    assert.equal(doc._proof.clock[0].toString(), 'bafyreiactx5vku7zueq27i5zdrgcjnczxvepceo5yszjqb2exufwrwxg44')
+    assert.equal(doc._proof.data[0], 'bafyreieilmvxq6wudu46i2ssmuyrmaszr4onzlqxzlvngrczbn7ppyvloq')
+    assert.equal(doc._proof.clock[0].toString(), 'bafyreict4aip45uwnm4xcsn4oikh73t5n7nzdmc2u36rdbguroun2yaf2y')
   })
 })

package/test/reproduce-fixture-bug.test.js

@@ -58,8 +58,8 @@ const reproduceBug = async (database) => {
   const id = '02pkji8'
   const doc = await database.get(id)
   // (await database.put({ completed: !completed, ...doc }))
-  const ok = await database.put(doc)
+  await database.put(doc)
   await database.todosByList.query({ range: [0, 1] })
 
-  console.log('ok', ok)
+  // console.log('ok', ok)
 }

package/scripts/propernames/gen.sh

@@ -1,3 +0,0 @@
-while read p; do
-  pail put $p $(node ../randomcid.js) --max-shard-size=10000
-done </usr/share/dict/propernames

package/scripts/randomcid.js

@@ -1,12 +0,0 @@
-import crypto from 'node:crypto'
-import { CID } from 'multiformats/cid'
-import * as raw from 'multiformats/codecs/raw'
-import { sha256 } from 'multiformats/hashes/sha2'
-
-async function main () {
-  const bytes = crypto.webcrypto.getRandomValues(new Uint8Array(32))
-  const hash = await sha256.digest(bytes)
-  process.stdout.write(CID.create(1, raw.code, hash).toString())
-}
-
-main()

package/scripts/words/gen.js

@@ -1,55 +0,0 @@
-import fs from 'node:fs'
-import { Readable } from 'node:stream'
-import { CarWriter } from '@ipld/car'
-import { CID } from 'multiformats/cid'
-import * as raw from 'multiformats/codecs/raw'
-import { sha256 } from 'multiformats/hashes/sha2'
-import { ShardBlock, put } from '../../index.js'
-import { MemoryBlockstore } from '../../block.js'
-
-/** @param {string} str */
-async function stringToCID (str) {
-  const hash = await sha256.digest(new TextEncoder().encode(str))
-  return CID.create(1, raw.code, hash)
-}
-
-async function main () {
-  const data = await fs.promises.readFile('/usr/share/dict/words', 'utf8')
-  const words = data.split(/\n/)
-  const cids = await Promise.all(words.map(stringToCID))
-  const blocks = new MemoryBlockstore()
-  const rootblk = await ShardBlock.create()
-  blocks.putSync(rootblk.cid, rootblk.bytes)
-
-  console.time(`put x${words.length}`)
-  /** @type {import('../../shard').ShardLink} */
-  let root = rootblk.cid
-  for (const [i, word] of words.entries()) {
-    const res = await put(blocks, root, word, cids[i])
-    root = res.root
-    for (const b of res.additions) {
-      blocks.putSync(b.cid, b.bytes)
-    }
-    for (const b of res.removals) {
-      blocks.deleteSync(b.cid)
-    }
-    if (i % 1000 === 0) {
-      console.log(`${Math.floor(i / words.length * 100)}%`)
-    }
-  }
-  console.timeEnd(`put x${words.length}`)
-
-  // @ts-expect-error
-  const { writer, out } = CarWriter.create(root)
-  const finishPromise = new Promise(resolve => {
-    Readable.from(out).pipe(fs.createWriteStream('./pail.car')).on('finish', resolve)
-  })
-
-  for (const b of blocks.entries()) {
-    await writer.put(b)
-  }
-  await writer.close()
-  await finishPromise
-}
-
-main()