@fireproof/core 0.3.7 → 0.3.9

package/README.md

@@ -18,7 +18,7 @@ Fireproof is alpha software, you should only use it if you are planning to contr
 ## Usage
 
 ```js
-import { Fireproof } from 'fireproof';
+import Fireproof from 'fireproof';
 
 async function main() {
   const database = Fireproof.storage('my-db');

package/hooks/use-fireproof.js

@@ -0,0 +1,135 @@
+/* global localStorage */
+// @ts-ignore
+import { useEffect, useState, createContext } from 'react'
+import { Fireproof, Listener, Hydrator } from '../index'
+
+// export interface FireproofCtxValue {
+//   addSubscriber: (label: String, fn: Function) => void
+//   database: Fireproof
+//   ready: boolean
+//   persist: () => void
+// }
+export const FireproofCtx = createContext({
+  addSubscriber: () => {},
+  database: null,
+  ready: false
+})
+
+const inboundSubscriberQueue = new Map()
+
+let startedSetup = false
+let database
+let listener
+const initializeDatabase = name => {
+  if (database) return
+  database = Fireproof.storage(name)
+  listener = new Listener(database)
+}
+
+/**
+ * @function useFireproof
+ * React hook to initialize a Fireproof database, automatically saving and loading the clock.
+ * You might need to `import { nodePolyfills } from 'vite-plugin-node-polyfills'` in your vite.config.ts
+ * @param [defineDatabaseFn] Synchronous function that defines the database, run this before any async calls
+ * @param [setupDatabaseFn] Asynchronous function that sets up the database, run this to load fixture data etc
+ * @returns {FireproofCtxValue} { addSubscriber, database, ready }
+ */
+export function useFireproof (
+  defineDatabaseFn = () => {},
+  setupDatabaseFn = async () => {},
+  name
+) {
+  const [ready, setReady] = useState(false)
+  initializeDatabase(name || 'useFireproof')
+  const localStorageKey = 'fp.' + database.name
+
+  const addSubscriber = (label, fn) => {
+    inboundSubscriberQueue.set(label, fn)
+  }
+
+  const listenerCallback = async event => {
+    localSet(localStorageKey, JSON.stringify(database))
+    if (event._external) return
+    for (const [, fn] of inboundSubscriberQueue) fn()
+  }
+
+  useEffect(() => {
+    const doSetup = async () => {
+      if (ready) return
+      if (startedSetup) return
+      startedSetup = true
+      defineDatabaseFn(database) // define indexes before querying them
+      console.log('Initializing database', database.name)
+      const fp = localGet(localStorageKey) // todo use db.name
+      if (fp) {
+        try {
+          const serialized = JSON.parse(fp)
+          // console.log('serialized', JSON.stringify(serialized.indexes.map(c => c.clock)))
+          console.log(`Loading previous database clock. (localStorage.removeItem('${localStorageKey}') to reset)`)
+          await Hydrator.fromJSON(serialized, database)
+          const changes = await database.changesSince()
+          if (changes.rows.length < 2) {
+            // console.log('Resetting database')
+            throw new Error('Resetting database')
+          }
+        } catch (e) {
+          console.error(`Error loading previous database clock. ${fp} Resetting.`, e)
+          await Hydrator.zoom(database, [])
+          await setupDatabaseFn(database)
+          localSet(localStorageKey, JSON.stringify(database))
+        }
+      } else {
+        await setupDatabaseFn(database)
+        localSet(localStorageKey, JSON.stringify(database))
+      }
+      setReady(true)
+      listener.on('*', listenerCallback)// hushed('*', listenerCallback, 250))
+    }
+    doSetup()
+  }, [ready])
+
+  return {
+    addSubscriber,
+    database,
+    ready,
+    persist: () => {
+      localSet(localStorageKey, JSON.stringify(database))
+    }
+  }
+}
+
+// const husherMap = new Map()
+// const husher = (id, workFn, ms) => {
+//   if (!husherMap.has(id)) {
+//     const start = Date.now()
+//     husherMap.set(
+//       id,
+//       workFn().finally(() => setTimeout(() => husherMap.delete(id), ms - (Date.now() - start)))
+//     )
+//   }
+//   return husherMap.get(id)
+// }
+// const hushed =
+//   (id, workFn, ms) =>
+//     (...args) =>
+//       husher(id, () => workFn(...args), ms)
+
+let storageSupported = false
+try {
+  storageSupported = window.localStorage && true
+} catch (e) {}
+export function localGet (key) {
+  if (storageSupported) {
+    return localStorage && localStorage.getItem(key)
+  }
+}
+function localSet (key, value) {
+  if (storageSupported) {
+    return localStorage && localStorage.setItem(key, value)
+  }
+}
+// function localRemove(key) {
+//   if (storageSupported) {
+//     return localStorage && localStorage.removeItem(key)
+//   }
+// }

package/index.js

@@ -0,0 +1,6 @@
+import Fireproof from './src/fireproof.js'
+import Index from './src/db-index.js'
+import Listener from './src/listener.js'
+import Hydrator from './src/hydrator.js'
+
+export { Fireproof, Index, Listener, Hydrator }

package/package.json

@@ -1,10 +1,9 @@
 {
   "name": "@fireproof/core",
-  "version": "0.3.7",
+  "version": "0.3.9",
   "description": "Realtime database for IPFS",
-  "main": "src/index.js",
+  "main": "index.js",
   "type": "module",
-  "types": "./dist/src/index.d.ts",
   "scripts": {
     "keygen": "node scripts/keygen.js",
     "test": "standard && npm run test:unencrypted && npm run test:mocha",
@@ -15,8 +14,7 @@
     "prepublishOnly": "cp ../../README.md .",
     "postpublish": "rm README.md",
     "lint": "standard",
-    "lint:fix": "standard --fix",
-    "build": "tsc --build"
+    "lint:fix": "standard --fix"
   },
   "keywords": [
     "database",
@@ -34,6 +32,7 @@
   ],
   "license": "Apache-2.0 OR MIT",
   "dependencies": {
+    "prolly-trees": "1.0.4",
     "@ipld/car": "^5.1.0",
     "@ipld/dag-cbor": "^9.0.0",
     "archy": "^1.0.0",
@@ -44,8 +43,6 @@
     "encrypted-block": "^0.0.3",
     "idb": "^7.1.1",
     "multiformats": "^11.0.1",
-    "node-polyfill-webpack-plugin": "^2.0.1",
-    "prolly-trees": "1.0.4",
     "sade": "^1.8.1"
   },
   "devDependencies": {
@@ -54,10 +51,7 @@
     "flexsearch": "^0.7.31",
     "mocha": "^10.2.0",
     "nanoid": "^4.0.0",
-    "standard": "^17.0.0",
-    "typescript": "^5.0.2",
-    "webpack": "^5.78.0",
-    "webpack-cli": "^5.0.1"
+    "standard": "^17.0.0"
   },
   "mocha": {
     "require": [
@@ -68,7 +62,7 @@
     "ignore": [
       "examples/**/*.tsx",
       "examples/**/dist",
-      "out/**"
+      "out/**", "dist/**"
     ]
   },
   "repository": {
@@ -79,33 +73,6 @@
     "url": "https://github.com/fireproof-storage/fireproof/issues"
   },
   "homepage": "https://fireproof.storage",
-  "exports": {
-    ".": {
-      "types": "./dist/src/index.d.ts",
-      "import": "./src/index.js"
-    },
-    "./fireproof": {
-      "types": "./dist/src/fireproof.d.ts",
-      "import": "./src/fireproof.js"
-    },
-    "./hydrator": {
-      "types": "./dist/src/hydrator.d.ts",
-      "import": "./src/hydrator.js"
-    },
-    "./listener": {
-      "types": "./dist/src/listener.d.ts",
-      "import": "./src/listener.js"
-    },
-    "./db-index": {
-      "types": "./dist/src/db-index.d.ts",
-      "import": "./src/db-index.js"
-    }
-  },
-  "files": [
-    "src",
-    "dist",
-    "README.md"
-  ],
   "workspaces": [
     "examples/todomvc"
   ]

package/scripts/keygen.js

@@ -0,0 +1,3 @@
+import { randomBytes } from 'crypto'
+
+console.log(randomBytes(32).toString('hex'))

package/src/blockstore.js

@@ -1,7 +1,6 @@
-// @ts-nocheck
 import { parse } from 'multiformats/link'
 import { CID } from 'multiformats'
-import { Valet } from './valet.js'
+import Valet from './valet.js'
 
 // const sleep = ms => new Promise(r => setTimeout(r, ms))
 
@@ -28,7 +27,7 @@ const husher = (id, workFn) => {
  * A blockstore that caches writes to a transaction and only persists them when committed.
  * @implements {Blockstore}
  */
-export class TransactionBlockstore {
+export default class TransactionBlockstore {
   /** @type {Map<string, Uint8Array>} */
   committedBlocks = new Map()
 

package/src/clock.js

@@ -1,4 +1,3 @@
-// @ts-nocheck
 import { Block, encode, decode } from 'multiformats/block'
 import { sha256 } from 'multiformats/hashes/sha2'
 import * as cbor from '@ipld/dag-cbor'
@@ -23,7 +22,7 @@ import { CIDCounter } from 'prolly-trees/utils'
  * Advance the clock by adding an event.
  *
  * @template T
- * @param {import('./blockstore').BlockFetcher} blocks Block storage.
+ * @param {import('../test/block').BlockFetcher} blocks Block storage.
  * @param {EventLink<T>[]} head The head of the clock.
  * @param {EventLink<T>} event The event to add.
  * @returns {Promise<EventLink<T>[]>} The new head of the clock.
@@ -90,7 +89,7 @@ export class EventBlock extends Block {
 
 /** @template T */
 export class EventFetcher {
-  /** @param {import('./blockstore').BlockFetcher} blocks */
+  /** @param {import('../test/block').BlockFetcher} blocks */
   constructor (blocks) {
     /** @private */
     this._blocks = blocks
@@ -169,7 +168,7 @@ async function contains (events, a, b) {
 
 /**
  * @template T
- * @param {import('./blockstore').BlockFetcher} blocks Block storage.
+ * @param {import('../test/block').BlockFetcher} blocks Block storage.
  * @param {EventLink<T>[]} head
  * @param {object} [options]
  * @param {(b: EventBlockView<T>) => string} [options.renderNodeLabel]

package/src/crypto.js

@@ -1,4 +1,3 @@
-// @ts-nocheck
 import * as codec from 'encrypted-block'
 import {
   create,

package/src/db-index.js

@@ -1,20 +1,15 @@
-// @ts-nocheck
-// @ts-ignore
 import { create, load } from 'prolly-trees/db-index'
 // import { create, load } from '../../../../prolly-trees/src/db-index.js'
 
 import { sha256 as hasher } from 'multiformats/hashes/sha2'
-// @ts-ignore
 import { nocache as cache } from 'prolly-trees/cache'
-// @ts-ignore
 import { bf, simpleCompare } from 'prolly-trees/utils'
 import { makeGetBlock } from './prolly.js'
 import { cidsToProof } from './fireproof.js'
 
 import * as codec from '@ipld/dag-cbor'
 // import { create as createBlock } from 'multiformats/block'
-import { TransactionBlockstore, doTransaction } from './blockstore.js'
-// @ts-ignore
+import TransactionBlockstore, { doTransaction } from './blockstore.js'
 import charwise from 'charwise'
 
 const ALWAYS_REBUILD = false // todo: make false
@@ -92,7 +87,7 @@ const indexEntriesForChanges = (changes, mapFn) => {
  * @param {Function} mapFn - The map function to apply to each entry in the database.
  *
  */
-export class DbIndex {
+export default class DbIndex {
   constructor (database, mapFn, clock, opts = {}) {
     // console.log('DbIndex constructor', database.constructor.name, typeof mapFn, clock)
     /**
@@ -177,7 +172,7 @@ export class DbIndex {
   /**
    * Query object can have {range}
    * @param {DbQuery} query - the query range to use
-   * @returns {Promise<{proof: {}, rows: Array<{id: string, key: string, value: any}>}>}
+   * @returns {Promise<{rows: Array<{id: string, key: string, value: any}>}>}
    * @memberof DbIndex
    * @instance
    */
@@ -264,8 +259,8 @@ export class DbIndex {
 
 /**
  * Update the DbIndex with the given entries
- * @param {import('./blockstore.js').Blockstore} blocks
- * @param {{root, cid}} inIndex
+ * @param {Blockstore} blocks
+ * @param {Block} inRoot
  * @param {DbIndexEntry[]} indexEntries
  * @private
  */

package/src/fireproof.js

@@ -1,27 +1,23 @@
-// @ts-nocheck
 import { randomBytes } from 'crypto'
 import { visMerkleClock, visMerkleTree, vis, put, get, getAll, eventsSince } from './prolly.js'
-import { TransactionBlockstore, doTransaction } from './blockstore.js'
+import TransactionBlockstore, { doTransaction } from './blockstore.js'
 import charwise from 'charwise'
-// import { CID } from 'multiformats/dist/types/src/cid.js'
 
 // const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
 
-// class Proof {}
-
 /**
  * @class Fireproof
  * @classdesc Fireproof stores data in IndexedDB and provides a Merkle clock.
  *  This is the main class for saving and loading JSON and other documents with the database. You can find additional examples and
  *  usage guides in the repository README.
  *
- * @param {import('./blockstore.js').TransactionBlockstore} blocks - The block storage instance to use documents and indexes
+ * @param {Blockstore} blocks - The block storage instance to use documents and indexes
  * @param {CID[]} clock - The Merkle clock head to use for the Fireproof instance.
  * @param {object} [config] - Optional configuration options for the Fireproof instance.
  * @param {object} [authCtx] - Optional authorization context object to use for any authentication checks.
  *
  */
-export class Fireproof {
+export default class Fireproof {
   listeners = new Set()
 
   /**
@@ -165,7 +161,7 @@ export class Fireproof {
    *
    * @param {string} key - the ID of the document to retrieve
    * @param {Object} [opts] - options
-   * @returns {Promise<{_id: string}>} - the document with the specified ID
+   * @returns {Object<{_id: string, ...doc: Object}>} - the document with the specified ID
    * @memberof Fireproof
    * @instance
    */
@@ -194,9 +190,8 @@ export class Fireproof {
    *
    * @param {Object} doc - the document to be added
    * @param {string} doc._id - the document ID. If not provided, a random ID will be generated.
-   * @param {CID[]} doc._clock - the document ID. If not provided, a random ID will be generated.
-   * @param {Proof} doc._proof - CIDs referenced by the update
-   * @returns {Promise<{ id: string, clock: CID[]  }>} - The result of adding the document to the database
+   * @param {Object} doc.* - the document data to be added
+   * @returns {Object<{ id: string, clock: CID[]  }>} - The result of adding the document to the database
    * @memberof Fireproof
    * @instance
    */
@@ -208,8 +203,8 @@ export class Fireproof {
 
   /**
    * Deletes a document from the database
-   * @param {string | any} docOrId - the document ID
-   * @returns {Promise<{ id: string, clock: CID[] }>} - The result of deleting the document from the database
+   * @param {string} id - the document ID
+   * @returns {Object<{ id: string, clock: CID[] }>} - The result of deleting the document from the database
    * @memberof Fireproof
    * @instance
    */
@@ -231,8 +226,8 @@ export class Fireproof {
   /**
    * Updates the underlying storage with the specified event.
    * @private
-   * @param {{del?: true, key : string, value?: any}} decodedEvent - the event to add
-   * @returns {Promise<{ proof:{}, id: string, clock: CID[] }>} - The result of adding the event to storage
+   * @param {Object<{key : string, value: any}>} event - the event to add
+   * @returns {Object<{ id: string, clock: CID[] }>} - The result of adding the event to storage
    */
   async putToProllyTree (decodedEvent, clock = null) {
     const event = encodeEvent(decodedEvent)

package/src/hydrator.js

@@ -1,10 +1,10 @@
-import { DbIndex } from './db-index.js'
-import { Fireproof } from './fireproof.js'
+import DbIndex from './db-index.js'
+import Fireproof from './fireproof.js'
 import { CID } from 'multiformats'
 
 const parseCID = cid => typeof cid === 'string' ? CID.parse(cid) : cid
 
-export class Hydrator {
+export default class Hydrator {
   static fromJSON (json, database) {
     database.hydrate({ clock: json.clock.map(c => parseCID(c)), name: json.name, key: json.key })
     if (json.indexes) {

package/src/listener.js

@@ -1,4 +1,3 @@
-// @ts-nocheck
 /**
  * A Fireproof database Listener allows you to react to events in the database.
  *
@@ -10,7 +9,7 @@
  */
 // import { ChangeEvent } from './db-index'
 
-export class Listener {
+export default class Listener {
   subcribers = new Map()
   doStopListening = null
 

package/src/prolly.js

@@ -1,4 +1,3 @@
-// @ts-nocheck
 import {
   advance,
   EventFetcher,
@@ -38,9 +37,23 @@ export const makeGetBlock = (blocks) => {
 }
 
 /**
- *
- * @param {*} param0
- * @returns
+ * Creates and saves a new event.
+ * @param {import('./blockstore.js').Blockstore} inBlocks - A persistent blockstore.
+ * @param {MemoryBlockstore} mblocks - A temporary blockstore.
+ * @param {Function} getBlock - A function that gets a block.
+ * @param {Function} bigPut - A function that puts a block.
+ * @param {import('prolly-trees/map').Root} root - The root node.
+ * @param {Object<{ key: string, value: any, del: boolean }>} event - The update event.
+ * @param {CID[]} head - The head of the event chain.
+ * @param {Array<import('multiformats/block').Block>} additions - A array of additions.
+ * @param {Array<mport('multiformats/block').Block>>} removals - An array of removals.
+ * @returns {Promise<{
+ *   root: import('prolly-trees/map').Root,
+ *   additions: Map<string, import('multiformats/block').Block>,
+ *   removals: Array<string>,
+ *   head: CID[],
+ *   event: CID[]
+ * }>}
  */
 async function createAndSaveNewEvent ({
   inBlocks,
@@ -185,7 +198,7 @@ const doProllyBulk = async (inBlocks, head, event) => {
 /**
  * Put a value (a CID) for the given key. If the key exists it's value is overwritten.
  *
- * @param {import('./blockstore.js').BlockFetcher} blocks Bucket block storage.
+ * @param {import('../test/block.js').BlockFetcher} blocks Bucket block storage.
  * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
  * @param {string} key The key of the value to put.
  * @param {CID} value The value to put.
@@ -237,7 +250,7 @@ export async function put (inBlocks, head, event, options) {
 /**
  * Determine the effective prolly root given the current merkle clock head.
  *
- * @param {import('./blockstore.js').BlockFetcher} blocks Bucket block storage.
+ * @param {import('../test/block.js').BlockFetcher} blocks Bucket block storage.
  * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
  */
 export async function root (inBlocks, head) {
@@ -257,10 +270,10 @@ export async function root (inBlocks, head) {
 
 /**
  * Get the list of events not known by the `since` event
- * @param {import('./blockstore.js').BlockFetcher} blocks Bucket block storage.
+ * @param {import('../test/block.js').BlockFetcher} blocks Bucket block storage.
  * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
  * @param {import('./clock').EventLink<EventData>} since Event to compare against.
- * @returns {Promise<{clockCIDs: CIDCounter, result: EventData[]}>}
+ * @returns {Promise<import('./clock').EventLink<EventData>[]>}
  */
 export async function eventsSince (blocks, head, since) {
   if (!head.length) {
@@ -273,10 +286,10 @@ export async function eventsSince (blocks, head, since) {
 
 /**
  *
- * @param {import('./blockstore.js').BlockFetcher} blocks Bucket block storage.
+ * @param {import('../test/block.js').BlockFetcher} blocks Bucket block storage.
  * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
  *
- * @returns {Promise<{clockCIDs: CIDCounter, result: EventData[]}>}
+ * @returns {Promise<import('./prolly').Entry[]>}
  *
  */
 export async function getAll (blocks, head) {
@@ -294,7 +307,7 @@ export async function getAll (blocks, head) {
 }
 
 /**
- * @param {import('./blockstore.js').BlockFetcher} blocks Bucket block storage.
+ * @param {import('../test/block.js').BlockFetcher} blocks Bucket block storage.
  * @param {import('./clock').EventLink<EventData>[]} head Merkle clock head.
  * @param {string} key The key of the value to retrieve.
  */

package/src/sha1.js

@@ -1,4 +1,3 @@
-// @ts-nocheck
 // from https://github.com/duzun/sync-sha1/blob/master/rawSha1.js
 // MIT License Copyright (c) 2020 Dumitru Uzun
 // Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -30,7 +29,7 @@
 *
 * @return  {Uint8Array}  sha1 hash
 */
-export function rawSha1 (b) {
+export default function rawSha1 (b) {
   let i = b.byteLength
   let bs = 0
   let A; let B; let C; let D; let G

package/src/valet.js

@@ -1,4 +1,3 @@
-// @ts-nocheck
 import { CarReader } from '@ipld/car'
 import { CID } from 'multiformats/cid'
 import { sha256 } from 'multiformats/hashes/sha2'
@@ -13,13 +12,13 @@ import { nocache as cache } from 'prolly-trees/cache'
 import { encrypt, decrypt } from './crypto.js'
 import { Buffer } from 'buffer'
 import * as codec from 'encrypted-block'
-import { rawSha1 as sha1sync } from './sha1.js'
+import sha1sync from './sha1.js'
 const chunker = bf(3)
 
 const NO_ENCRYPT =
   typeof process !== 'undefined' ? process.env.NO_ENCRYPT : import.meta && import.meta.env.VITE_NO_ENCRYPT
 
-export class Valet {
+export default class Valet {
   idb = null
   name = null
   uploadQueue = null
@@ -235,8 +234,7 @@ const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, ke
     key: encryptionKey,
     hasher: sha256,
     chunker,
-    cache,
-    // codec: dagcbor, // should be crypto?
+    codec: dagcbor, // should be crypto?
     root: innerBlockStoreClockRootCid
   })) {
     encryptedBlocks.push(block)
@@ -264,8 +262,8 @@ const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
         key: decryptionKey,
         chunker,
         hasher: sha256,
-        cache
-        // codec: dagcbor
+        cache,
+        codec: dagcbor
       })) {
         decryptedBlocks.push(block)
         cids.add(block.cid.toString())

package/test/block.js

@@ -0,0 +1,65 @@
+import { parse } from 'multiformats/link'
+
+/**
+ * @typedef {{ cid: import('../src/link').AnyLink, bytes: Uint8Array }} AnyBlock
+ * @typedef {{ get: (link: import('../src/link').AnyLink) => Promise<AnyBlock | undefined> }} BlockFetcher
+ */
+
+/** @implements {BlockFetcher} */
+export class MemoryBlockstore {
+  /** @type {Map<string, Uint8Array>} */
+  blocks = new Map()
+
+  /**
+   * @param {import('../src/link').AnyLink} cid
+   * @returns {Promise<AnyBlock | undefined>}
+   */
+  async get (cid) {
+    const bytes = this.blocks.get(cid.toString())
+    if (!bytes) return
+    return { cid, bytes }
+  }
+
+  /**
+   * @param {import('../src/link').AnyLink} cid
+   * @param {Uint8Array} bytes
+   */
+  async put (cid, bytes) {
+    // console.log('put', cid)
+    this.blocks.set(cid.toString(), bytes)
+  }
+
+  /**
+   * @param {import('../src/link').AnyLink} cid
+   * @param {Uint8Array} bytes
+   */
+  putSync (cid, bytes) {
+    this.blocks.set(cid.toString(), bytes)
+  }
+
+  * entries () {
+    for (const [str, bytes] of this.blocks) {
+      yield { cid: parse(str), bytes }
+    }
+  }
+}
+
+export class MultiBlockFetcher {
+  /** @type {BlockFetcher[]} */
+  fetchers
+
+  /** @param {BlockFetcher[]} fetchers */
+  constructor (...fetchers) {
+    this.fetchers = fetchers
+  }
+
+  /** @param {import('../src/link').AnyLink} link */
+  async get (link) {
+    for (const f of this.fetchers) {
+      const v = await f.get(link)
+      if (v) {
+        return v
+      }
+    }
+  }
+}