@fireproof/core 0.6.4 → 0.7.0-alpha.0

package/src/valet.js

@@ -1,107 +1,45 @@
-import { CarReader } from '@ipld/car'
-import { CID } from 'multiformats/cid'
 import { sha256 } from 'multiformats/hashes/sha2'
-import { parse } from 'multiformats/link'
 import * as CBW from '@ipld/car/buffer-writer'
 import * as raw from 'multiformats/codecs/raw'
 import * as Block from 'multiformats/block'
-import * as dagcbor from '@ipld/dag-cbor'
-import { openDB } from 'idb'
-import cargoQueue from 'async/cargoQueue.js'
+import { Loader } from './loader.js'
 // @ts-ignore
-
-// @ts-ignore
-import { bf, simpleCompare as compare } from 'prolly-trees/utils'
+import { bf } from 'prolly-trees/utils'
 // @ts-ignore
 import { nocache as cache } from 'prolly-trees/cache'
-// import { makeGetBlock } from './prolly.js'
 import { encrypt, decrypt } from './crypto.js'
 import { Buffer } from 'buffer'
-// @ts-ignore
-import * as codec from 'encrypted-block'
-
-import { create, load } from 'ipld-hashmap'
 
-import { rawSha1 as sha1sync } from './sha1.js'
 const chunker = bf(30)
 
-const blockOpts = { cache, chunker, codec: dagcbor, hasher: sha256, compare }
-
-const NO_ENCRYPT = typeof process !== 'undefined' && !!process.env?.NO_ENCRYPT
-// ? process.env.NO_ENCRYPT : import.meta && import.meta.env.VITE_NO_ENCRYPT
-
 export class Valet {
   idb = null
   name = null
   uploadQueue = null
   alreadyEnqueued = new Set()
-  keyMaterial = null
-  keyId = 'null'
-  valetRoot = null
-  valetRootCid = null // set by hydrate
-  valetRootCarCid = null // most recent diff
 
-  valetCidBlocks = new VMemoryBlockstore()
   instanceId = Math.random().toString(36).slice(2)
 
-  /**
-   * Function installed by the database to upload car files
-   * @type {null|function(string, Uint8Array):Promise<void>}
-   */
-  uploadFunction = null
-
-  constructor (name = 'default', keyMaterial) {
+  constructor (name = 'default', config = {}) {
     this.name = name
-    this.setKeyMaterial(keyMaterial)
-    this.uploadQueue = cargoQueue(async (tasks, callback) => {
-      // console.log(
-      //   'queue worker',
-      //   tasks.length,
-      //   tasks.reduce((acc, t) => acc + t.value.length, 0)
-      // )
-      if (this.uploadFunction) {
-        // todo we can coalesce these into a single car file
-        // todo remove idb usage here
-        for (const task of tasks) {
-          await this.uploadFunction(task.carCid, task.value)
-          // todo update syncCidMap to say this has been synced
-          // const carMeta = await db.get('cidToCar', task.carCid)
-          // delete carMeta.pending
-          // await db.put('cidToCar', carMeta)
-        }
-      }
-      callback()
-    })
-
-    this.uploadQueue.drain(async () => {
-      // todo read syncCidMap and sync any that are still unsynced
-      //   return await this.withDB(async db => {
-      //     const carKeys = (await db.getAllFromIndex('cidToCar', 'pending')).map(c => c.car)
-      //     for (const carKey of carKeys) {
-      //       await this.uploadFunction(carKey, await db.get('cars', carKey))
-      //       const carMeta = await db.get('cidToCar', carKey)
-      //       delete carMeta.pending
-      //       await db.put('cidToCar', carMeta)
-      //     }
-      //   })
+    // console.log('new Valet', name, config.primary)
+    this.primary = Loader.appropriate(name, config.primary)
+    this.secondary = config.secondary ? Loader.appropriate(name, config.secondary) : null
+    // set up a promise listener that applies all the headers to the clock
+    // when they resolve
+    const readyP = [this.primary.ready]
+    if (this.secondary) readyP.push(this.secondary.ready)
+
+    this.ready = Promise.all(readyP).then((blocksReady) => {
+      // console.log('blocksReady valet', this.name, blocksReady)
+      return blocksReady
     })
   }
 
-  getKeyMaterial () {
-    return this.keyMaterial
-  }
-
-  setKeyMaterial (km) {
-    if (km && !NO_ENCRYPT) {
-      const hex = Uint8Array.from(Buffer.from(km, 'hex'))
-      this.keyMaterial = km
-      const hash = sha1sync(hex)
-      this.keyId = Buffer.from(hash).toString('hex')
-    } else {
-      this.keyMaterial = null
-      this.keyId = 'null'
-    }
-    // console.trace('keyId', this.name, this.keyId)
+  async saveHeader (header) {
+    // each storage needs to add its own carCidMapCarCid to the header
+    if (this.secondary) { this.secondary.saveHeader(header) } // todo: await?
+    return await this.primary.saveHeader(header)
   }
 
   /**
@@ -113,33 +51,13 @@ export class Valet {
    */
   async writeTransaction (innerBlockstore, cids) {
     if (innerBlockstore.lastCid) {
-      if (this.keyMaterial) {
-        // console.log('encrypting car', innerBlockstore.label)
-        // should we pass cids in instead of iterating frin innerBlockstore?
-        const newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, this.keyMaterial)
-        await this.parkCar(newCar.cid.toString(), newCar.bytes, cids)
-      } else {
-        const newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
-        await this.parkCar(newCar.cid.toString(), newCar.bytes, cids)
-      }
+      await parkCar(this.primary, innerBlockstore, cids)
+      if (this.secondary) await parkCar(this.secondary, innerBlockstore, cids)
     } else {
       throw new Error('missing lastCid for car header')
     }
   }
 
-  withDB = async dbWorkFun => {
-    if (!this.idb) {
-      this.idb = await openDB(`fp.${this.keyId}.${this.name}.valet`, 3, {
-        upgrade (db, oldVersion, newVersion, transaction) {
-          if (oldVersion < 1) {
-            db.createObjectStore('cars')
-          }
-        }
-      })
-    }
-    return await dbWorkFun(this.idb)
-  }
-
   /**
    * Iterate over all blocks in the store.
    *
@@ -155,216 +73,49 @@ export class Valet {
     // }
   }
 
-  setRootCarCid (cid) {
-    this.valetRootCarCid = cid
-    this.valetRoot = null
-    this.valetRootCid = null
-  }
-
-  // todo memoize this
-  async getCarCIDForCID (cid) {
-    // make a car reader for this.valetRootCarCid
-    if (!this.valetRootCarCid) return { result: null }
-
-    let indexNode
-    if (this.valetRoot) {
-      indexNode = this.valetRoot
-    } else {
-      const combinedReader = await this.getCombinedReader(this.valetRootCarCid)
-      if (!this.valetRootCid) {
-        const root = combinedReader.root.cid
-        // console.log('roots', this.instanceId, this.name, root, this.valetRootCarCid, this.valetRootCid)
-        this.valetRootCid = root
-      }
-      indexNode = await load(combinedReader, this.valetRootCid, {
-        blockHasher: blockOpts.hasher,
-        blockCodec: blockOpts.codec
-      })
-    }
-
-    const got = await indexNode.get(cid)
-    // console.log('getCarCIDForCID', cid, got)
-    return { result: got }
-  }
-
-  async getCombinedReader (carCid) {
-    let carMapReader
-    if (this.valetRootCarCid) {
-      // todo only need this if we are cold starting
-      carMapReader = await this.getCarReader(this.valetRootCarCid)
-    }
-
-    const theseValetCidBlocks = this.valetCidBlocks
-    // console.log('theseValetCidBlocks', theseValetCidBlocks)
-    const combinedReader = {
-      root: carMapReader?.root,
-      put: async (cid, bytes) => {
-        // console.log('mapPut', cid, bytes.length)
-        return await theseValetCidBlocks.put(cid, bytes)
-      },
-      get: async cid => {
-        // console.log('mapGet', cid)
-        try {
-          const got = await theseValetCidBlocks.get(cid)
-          return got.bytes
-        } catch (e) {
-          // console.log('get from car', cid, carMapReader)
-          if (!carMapReader) throw e
-          const bytes = await carMapReader.get(cid)
-          await theseValetCidBlocks.put(cid, bytes)
-          // console.log('mapGet', cid, bytes.length, bytes.constructor.name)
-          return bytes
-        }
-      }
-    }
-    return combinedReader
-  }
-
-  /**
-   *
-   * @param {string} carCid
-   * @param {*} value
-   */
-  async parkCar (carCid, value, cids) {
-    // console.log('parkCar', this.instanceId, this.name, carCid, cids)
-    const combinedReader = await this.getCombinedReader(carCid)
-    const mapNode = await addCidsToCarIndex(
-      combinedReader,
-      this.valetRoot,
-      this.valetRootCid,
-      Array.from(cids).map(cid => ({ key: cid.toString(), value: carCid.toString() }))
-    )
-
-    this.valetRoot = mapNode
-    this.valetRootCid = mapNode.cid
-    // make a block set with all the cids of the map
-    const saveValetBlocks = new VMemoryBlockstore() //  todo this blockstore should read from the last valetCid car also
-
-    for await (const cidx of mapNode.cids()) {
-      const bytes = await combinedReader.get(cidx)
-      saveValetBlocks.put(cidx, bytes)
-    }
-    let newValetCidCar
-    if (this.keyMaterial) {
-      newValetCidCar = await blocksToEncryptedCarBlock(this.valetRootCid, saveValetBlocks, this.keyMaterial)
-    } else {
-      newValetCidCar = await blocksToCarBlock(this.valetRootCid, saveValetBlocks)
-    }
-    // console.log('newValetCidCar', this.name, Math.floor(newValetCidCar.bytes.length / 1024))
-    await this.writeCars([
-      {
-        cid: carCid,
-        bytes: value,
-        replaces: null
-      },
-      {
-        cid: newValetCidCar.cid,
-        bytes: newValetCidCar.bytes,
-        replaces: null
-        // replaces: this.valetRootCarCid // todo
-      }
-    ])
-
-    this.valetRootCarCid = newValetCidCar.cid // goes to clock
-
-    // console.log('parked car', carCid, value.length, Array.from(cids))
-    // upload to web3.storage if we have credentials
-    if (this.uploadFunction) {
-      if (this.alreadyEnqueued.has(carCid)) {
-        // console.log('already enqueued', carCid)
-        return
-      }
-      // don't await this, it will be done in the queue
-      // console.log('add to queue', carCid, value.length)
-      this.uploadQueue.push({ carCid, value })
-      this.alreadyEnqueued.add(carCid)
-    } else {
-      // console.log('no upload function', carCid, value.length, this.uploadFunction)
-    }
-  }
-
-  async writeCars (cars) {
-    return await this.withDB(async db => {
-      const tx = db.transaction(['cars'], 'readwrite')
-      for (const { cid, bytes, replaces } of cars) {
-        await tx.objectStore('cars').put(bytes, cid.toString())
-        // todo remove old maps
-        if (replaces) {
-          await tx.objectStore('cars').delete(replaces.toString())
-        }
-      }
-      return await tx.done
-    })
-  }
-
   remoteBlockFunction = null
 
-  async getCarReader (carCid) {
-    carCid = carCid.toString()
-    const carBytes = await this.withDB(async db => {
-      const tx = db.transaction(['cars'], 'readonly')
-      // console.log('getCarReader', carCid)
-      return await tx.objectStore('cars').get(carCid)
-    })
-    const reader = await CarReader.fromBytes(carBytes)
-    if (this.keyMaterial) {
-      const roots = await reader.getRoots()
-      const readerGetWithCodec = async cid => {
-        const got = await reader.get(cid)
-        // console.log('got.', cid.toString())
-        let useCodec = codec
-        if (cid.toString().indexOf('bafy') === 0) {
-          // todo cleanup types
-          useCodec = dagcbor
-        }
-        const decoded = await Block.decode({
-          ...got,
-          codec: useCodec,
-          hasher: sha256
-        })
-        // console.log('decoded', decoded.value)
-        return decoded
-      }
-      const { blocks } = await blocksFromEncryptedCarBlock(roots[0], readerGetWithCodec, this.keyMaterial)
-
-      // last block is the root ??? todo
-      const rootBlock = blocks[blocks.length - 1]
-
-      return {
-        root: rootBlock,
-        get: async dataCID => {
-          // console.log('getCarReader dataCID', dataCID)
-          dataCID = dataCID.toString()
-          const block = blocks.find(b => b.cid.toString() === dataCID)
-          // console.log('getCarReader block', block)
-          if (block) {
-            return block.bytes
-          }
-        }
-      }
-    } else {
-      return {
-        root: reader.getRoots()[0],
-        get: async dataCID => {
-          const gotBlock = await reader.get(CID.parse(dataCID))
-          if (gotBlock) {
-            return gotBlock.bytes
+  async getValetBlock (dataCID) {
+    // console.log('getValetBlock primary', dataCID)
+    try {
+      const { block } = await this.primary.getLoaderBlock(dataCID)
+      return block
+    } catch (e) {
+      // console.log('getValetBlock error', e)
+      if (this.secondary) {
+        // console.log('getValetBlock secondary', dataCID)
+        try {
+          const { block, reader } = await this.secondary.getLoaderBlock(dataCID)
+          const cids = new Set()
+          for await (const { cid } of reader.entries()) {
+            // console.log(cid, bytes)
+            cids.add(cid.toString())
           }
+          reader.get = reader.gat // some consumers prefer get
+          // console.log('replicating', reader.root)
+          reader.lastCid = reader.root.cid
+          await parkCar(this.primary, reader, [...cids])
+          return block
+        } catch (e) {
+          // console.log('getValetBlock secondary error', e)
         }
       }
     }
   }
+}
 
-  // todo memoize this
-  async getValetBlock (dataCID) {
-    // console.log('get valet block', dataCID)
-    const { result: carCid } = await this.getCarCIDForCID(dataCID)
-    if (!carCid) {
-      throw new Error('Missing block: ' + dataCID)
-    }
-    const reader = await this.getCarReader(carCid)
-    return await reader.get(dataCID)
+async function parkCar (storage, innerBlockstore, cids) {
+  // console.log('parkCar', this.instanceId, this.name, carCid, cids)
+  let newCar
+  if (storage.keyMaterial) {
+    // console.log('encrypting car', innerBlockstore.label)
+    newCar = await blocksToEncryptedCarBlock(innerBlockstore.lastCid, innerBlockstore, storage.keyMaterial)
+  } else {
+    // todo should we pass cids in instead of iterating innerBlockstore?
+    newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
   }
+  // console.log('new car', newCar.cid.toString())
+  return await storage.saveCar(newCar.cid.toString(), newCar.bytes, cids)
 }
 
 export const blocksToCarBlock = async (rootCids, blocks) => {
@@ -400,10 +151,11 @@ export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blo
   const encryptionKey = Buffer.from(keyMaterial, 'hex')
   const encryptedBlocks = []
   const theCids = []
+  // console.trace('blocksToEncryptedCarBlock', blocks)
   for (const { cid } of blocks.entries()) {
     theCids.push(cid.toString())
   }
-  // console.log('encrypting', theCids.length, 'blocks', theCids.includes(innerBlockStoreClockRootCid.toString()))
+  // console.log('encrypting', theCids.length, 'blocks', theCids.includes(innerBlockStoreClockRootCid.toString()), keyMaterial)
   // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
   let last
   for await (const block of encrypt({
@@ -426,7 +178,7 @@ export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blo
 // { root, get, key, cache, chunker, hasher }
 
 const memoizeDecryptedCarBlocks = new Map()
-const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
+export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
   if (memoizeDecryptedCarBlocks.has(cid.toString())) {
     return memoizeDecryptedCarBlocks.get(cid.toString())
   } else {
@@ -453,57 +205,3 @@ const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
     return blocksPromise
   }
 }
-
-const addCidsToCarIndex = async (blockstore, valetRoot, valetRootCid, bulkOperations) => {
-  let indexNode
-  if (valetRootCid) {
-    if (valetRoot) {
-      indexNode = valetRoot
-    } else {
-      indexNode = await load(blockstore, valetRootCid, { blockHasher: blockOpts.hasher, blockCodec: blockOpts.codec })
-    }
-  } else {
-    indexNode = await create(blockstore, {
-      bitWidth: 4,
-      bucketSize: 2,
-      blockHasher: blockOpts.hasher,
-      blockCodec: blockOpts.codec
-    })
-  }
-  // console.log('adding', bulkOperations.length, 'cids to index')
-  for (const { key, value } of bulkOperations) {
-    // console.log('adding', key, value)
-    await indexNode.set(key, value)
-  }
-  return indexNode
-}
-
-export class VMemoryBlockstore {
-  /** @type {Map<string, Uint8Array>} */
-  blocks = new Map()
-  instanceId = Math.random().toString(36).slice(2)
-
-  async get (cid) {
-    const bytes = this.blocks.get(cid.toString())
-    // console.log('getvm', bytes.constructor.name, this.instanceId, cid, bytes && bytes.length)
-    if (bytes.length === 253) {
-      // console.log('getvm', bytes.())
-    }
-    if (!bytes) throw new Error('block not found ' + cid.toString())
-    return { cid, bytes }
-  }
-
-  /**
-   * @param {import('../src/link').AnyLink} cid
-   * @param {Uint8Array} bytes
-   */
-  async put (cid, bytes) {
-    this.blocks.set(cid.toString(), bytes)
-  }
-
-  * entries () {
-    for (const [str, bytes] of this.blocks) {
-      yield { cid: parse(str), bytes }
-    }
-  }
-}

package/dist/hooks/use-fireproof.js

@@ -1,150 +0,0 @@
-// @ts-ignore
-import { useEffect, useState, useCallback, createContext } from 'react';
-import { Fireproof, Index } from '@fireproof/core';
-/**
-@typedef {Object} FireproofCtxValue
-@property {Function} addSubscriber - A function to add a subscriber with a label and function.
-@property {Fireproof} database - An instance of the Fireproof class.
-@property {Function} useLiveQuery - A hook to return a query result
-@property {Function} useLiveDocument - A hook to return a live document
-@property {boolean} ready - A boolean indicating whether the database is ready.
-@param {string} label - A label for the subscriber.
-@param {Function} fn - A function to be added as a subscriber.
-@returns {void}
-*/
-export const FireproofCtx = createContext({
-    addSubscriber: () => { },
-    database: null,
-    ready: false
-});
-// const inboundSubscriberQueue = new Map()
-let startedSetup = false;
-let database;
-const initializeDatabase = name => {
-    if (database)
-        return;
-    database = Fireproof.storage(name);
-};
-/**
-@function useFireproof
-React hook to initialize a Fireproof database, automatically saving and loading the clock.
-You might need to import { nodePolyfills } from 'vite-plugin-node-polyfills' in your vite.config.ts
-@deprecated - npm install @fireproof/react instead
-@param {string} name - The path to the database file
-@param {function(database): void} [defineDatabaseFn] - Synchronous function that defines the database, run this before any async calls
-@param {function(database): Promise<void>} [setupDatabaseFn] - Asynchronous function that sets up the database, run this to load fixture data etc
-@returns {FireproofCtxValue} { useLiveQuery, useLiveDocument, database, ready }
-*/
-export function useFireproof(name = 'useFireproof', defineDatabaseFn = () => { }, setupDatabaseFn = async () => { }) {
-    const [ready, setReady] = useState(false);
-    initializeDatabase(name);
-    /**
-     * @deprecated - use database.subscribe instead
-     */
-    const addSubscriber = (label, fn) => {
-        // todo test that the label is not needed
-        return database.registerListener(fn);
-        // inboundSubscriberQueue.set(label, fn)
-    };
-    useEffect(() => {
-        const doSetup = async () => {
-            if (ready)
-                return;
-            if (startedSetup)
-                return;
-            startedSetup = true;
-            defineDatabaseFn(database); // define indexes before querying them
-            if (database.clock.length === 0) {
-                await setupDatabaseFn(database);
-            }
-            setReady(true);
-        };
-        doSetup();
-    }, [ready]);
-    function useLiveDocument(initialDoc) {
-        const id = initialDoc._id;
-        const [doc, setDoc] = useState(initialDoc);
-        const saveDoc = async (newDoc) => {
-            await database.put({ _id: id, ...newDoc });
-        };
-        const refreshDoc = useCallback(async () => {
-            // todo add option for mvcc checks
-            const got = await database.get(id).catch(() => initialDoc);
-            setDoc(got);
-        }, [id, initialDoc]);
-        useEffect(() => database.subscribe(change => {
-            if (change.find(c => c.key === id)) {
-                refreshDoc(); // todo use change.value
-            }
-        }), [id, refreshDoc]);
-        useEffect(() => {
-            refreshDoc();
-        }, []);
-        return [doc, saveDoc];
-    }
-    function useLiveQuery(mapFn, query = null, initialRows = []) {
-        const [rows, setRows] = useState({ rows: initialRows, proof: {} });
-        const [index, setIndex] = useState(null);
-        const refreshRows = useCallback(async () => {
-            if (!index)
-                return;
-            const got = await index.query(query || {});
-            setRows(got);
-        }, [index, JSON.stringify(query)]);
-        useEffect(() => {
-            // todo listen to index changes
-            return database.subscribe(() => {
-                refreshRows();
-            });
-        }, [refreshRows]);
-        useEffect(() => {
-            refreshRows();
-        }, [index]);
-        useEffect(() => {
-            const index = new Index(database, null, mapFn); // this should only be created once
-            setIndex(index);
-        }, [mapFn.toString()]);
-        return rows;
-    }
-    return {
-        addSubscriber,
-        useLiveQuery,
-        useLiveDocument,
-        database,
-        ready
-    };
-}
-// const husherMap = new Map()
-// const husher = (id, workFn, ms) => {
-//   if (!husherMap.has(id)) {
-//     const start = Date.now()
-//     husherMap.set(
-//       id,
-//       workFn().finally(() => setTimeout(() => husherMap.delete(id), ms - (Date.now() - start)))
-//     )
-//   }
-//   return husherMap.get(id)
-// }
-// const hushed =
-//   (id, workFn, ms) =>
-//     (...args) =>
-//       husher(id, () => workFn(...args), ms)
-// let storageSupported = false
-// try {
-//   storageSupported = window.localStorage && true
-// } catch (e) {}
-// export function localGet (key) {
-//   if (storageSupported) {
-//     return localStorage && localStorage.getItem(key)
-//   }
-// }
-// function localSet (key, value) {
-//   if (storageSupported) {
-//     return localStorage && localStorage.setItem(key, value)
-//   }
-// }
-// function localRemove(key) {
-//   if (storageSupported) {
-//     return localStorage && localStorage.removeItem(key)
-//   }
-// }

package/dist/src/crypto-poly.js

@@ -1,4 +0,0 @@
-import crypto from 'crypto-browserify';
-export function randomBytes(n) {
-    return crypto.randomBytes(n);
-}

package/dist/src/link.js

@@ -1 +0,0 @@
-export {};