@fireproof/core 0.16.6 → 0.17.0

package/README.md

@@ -1,6 +1,6 @@
 
 
-# <img src="https://fireproof.storage/static/img/flame.svg" alt="Fireproof logo" width="25"> Fireproof
+# <img src="https://fireproof.storage/static/img/flame.svg" alt="Fireproof logo" width="25"> [Fireproof](https://fireproof.storage) – live database for the web
 
 <p align="right">
   <a href="https://github.com/fireproof-storage/fireproof/actions/workflows/test.yml">
@@ -10,7 +10,11 @@
 
 Simplify your application state with a live database. Automatically update your UI based on local or remote changes, and optionally integrate with any cloud for replication and sharing.
 
-Fireproof is an embedded JavaScript document database designed to streamline app development. Data resides locally, with optional encrypted cloud storage and realtime collaboration. Features like live queries, database branches and snapshots, and file attachments make Fireproof ideal for browser-based apps big or small. 
+Fireproof is an embedded JavaScript document database designed to streamline app development. Data resides locally, with optional encrypted cloud storage and real-time sync and collaboration. Features like live queries, database branches and snapshots, and file attachments make Fireproof ideal for browser-based apps big or small. 
+
+Fireproof works in the browser, server, edge function, and any other JavaScript environment, with [connectors for popular backend services like AWS, Netlify, and PartyKit.]()
+
+### Install Anywhere
 
 Get started with just the NPM module:
 
@@ -24,13 +28,19 @@ or target specific builds via:
 npm install @fireproof/core
 ```
 
-or add the database to to any web page via HTML script tag:
+The default build is optimized for browsers, to load the node build add `/node`:
+
+```js
+import { fireproof } from '@fireproof/core/node'
+```
+
+Add the database to any web page via HTML script tag:
 
 ```html
 <script src="https://cdn.jsdelivr.net/npm/@fireproof/core/dist/web/fireproof.iife.js"></script>
 ```
 
-You can go ahead and write features now, and connect to any cloud backend later.
+Go ahead and write features, then [connect to any cloud backend](https://www.npmjs.com/package/@fireproof/connect) later.
  
  ### JavaScript Example
 
@@ -50,7 +60,6 @@ const result = await db.query("age", { range: [40, 52] })
 
 Jump to the docs site [for JavaScript API basics.](https://use-fireproof.com/docs/database-api/basics) You can [find a real-world JavaScript app here.](https://github.com/mlc-ai/web-stable-diffusion/pull/52) Fireproof has been tested in many JavaScript environments. Read more about [bundler support](https://use-fireproof.com/docs/bundling).
 
-
 ### React Example 
 
 Fireproof has React hooks so you can avoid boilerplate and write expressive code. Instead of dealing with React contexts and reducers, simple hooks make your JSON documents feel like `setState()` objects.
@@ -67,7 +76,7 @@ Read the [step-by-step React tutorial](https://use-fireproof.com/docs/react-tuto
 
 ## Why choose Fireproof 
 
-Fireproof has a unique take on distributed data integrity, rooted in immutable data and cryptographically verifiable protocols. This allows you to add live data to your app without complex configuration or installation (it's just an npm module) and if you decide to connect to the cloud you can easily choose storage providers or connect to your own S3 bucket. End-to-end encryption allows you to manage keys separately from data, defining custom security policies, so you can get started writing app features today, and connect to any environment when you are ready.
+Fireproof has a unique take on distributed data integrity, rooted in immutable data and cryptographically verifiable protocols. This allows you to add live data to any app without complex configuration or installation (it's just an npm module) and if you decide to connect to the cloud you can easily choose storage providers or connect to your own S3 bucket. End-to-end encryption allows you to manage keys separately from data, defining custom security policies, so you can get started writing app features today, and connect to any environment when you are ready. This infrastructure independence makes Fireproof great for brownfield and greenfield projects alike.
 
 [Read more about the thinking behind Fireproof on our blog.](https://fireproof.storage/blog/) The community is active on [Discord](https://discord.gg/cCryrNHePH) and [X](https://twitter.com/FireproofStorge), among other places.
 

package/dist/browser/fireproof.cjs

@@ -91,8 +91,9 @@ var import_block = require("multiformats/block");
 var import_link = require("multiformats/link");
 var import_sha2 = require("multiformats/hashes/sha2");
 var codec = __toESM(require("@ipld/dag-cbor"), 1);
-var import_crdt = require("@alanshaw/pail/crdt");
-var import_clock = require("@alanshaw/pail/clock");
+var import_crdt = require("@web3-storage/pail/crdt");
+var import_clock = require("@web3-storage/pail/clock");
+var Batch = __toESM(require("@web3-storage/pail/crdt/batch"), 1);
 var import_encrypted_blockstore = require("@fireproof/encrypted-blockstore");
 
 // src/files.ts
@@ -164,30 +165,43 @@ var UnixFSFileBuilder = class {
 };
 
 // src/crdt-helpers.ts
-async function applyBulkUpdateToCrdt(tblocks, head, updates, options) {
-  let result;
-  for (const update of updates) {
-    const link = await writeDocContent(tblocks, update);
-    result = await (0, import_crdt.put)(tblocks, head, update.key, link, options);
-    const resRoot = result.root.toString();
-    const isReturned = result.additions.some((a) => a.cid.toString() === resRoot);
-    if (!isReturned) {
-      const hasRoot = await tblocks.get(result.root);
-      if (!hasRoot) {
-        throw new Error(
-          `missing root in additions: ${result.additions.length} ${resRoot} keys: ${updates.map((u) => u.key).toString()}`
-        );
-        result.head = head;
-      }
+function time(tag) {
+}
+function timeEnd(tag) {
+}
+async function applyBulkUpdateToCrdt(tblocks, head, updates) {
+  let result = null;
+  if (updates.length > 1) {
+    const batch = await Batch.create(tblocks, head);
+    for (const update of updates) {
+      const link = await writeDocContent(tblocks, update);
+      await batch.put(update.key, link);
     }
-    if (result.event) {
-      for (const { cid, bytes } of [...result.additions, result.event]) {
-        tblocks.putSync(cid, bytes);
+    result = await batch.commit();
+  } else {
+    for (const update of updates) {
+      const link = await writeDocContent(tblocks, update);
+      result = await (0, import_crdt.put)(tblocks, head, update.key, link);
+      const resRoot = result.root.toString();
+      const isReturned = result.additions.some((a) => a.cid.toString() === resRoot);
+      if (!isReturned) {
+        const hasRoot = await tblocks.get(result.root);
+        if (!hasRoot) {
+          throw new Error(
+            `missing root in additions: ${result.additions.length} ${resRoot} keys: ${updates.map((u) => u.key).toString()}`
+          );
+        }
       }
-      head = result.head;
     }
   }
-  return { head };
+  if (!result)
+    throw new Error("Missing result");
+  if (result.event) {
+    for (const { cid, bytes } of [...result.additions, ...result.removals, result.event]) {
+      tblocks.putSync(cid, bytes);
+    }
+  }
+  return { head: result.head };
 }
 async function writeDocContent(blocks, update) {
   let value;
@@ -325,38 +339,34 @@ async function gatherUpdates(blocks, eventsFetcher, head, since, updates = [], k
     const { value: event } = await eventsFetcher.get(link);
     if (!event)
       continue;
-    const { key, value } = event.data;
-    if (keys.has(key)) {
-      if (event.parents) {
-        updates = await gatherUpdates(
-          blocks,
-          eventsFetcher,
-          event.parents,
-          since,
-          updates,
-          keys,
-          didLinks,
-          limit
-        );
-      }
-    } else {
-      keys.add(key);
-      const docValue = await getValueFromLink(blocks, value);
-      updates.push({ key, value: docValue.doc, del: docValue.del, clock: link });
-      limit--;
-      if (event.parents) {
-        updates = await gatherUpdates(
-          blocks,
-          eventsFetcher,
-          event.parents,
-          since,
-          updates,
-          keys,
-          didLinks,
-          limit
-        );
+    const { type } = event.data;
+    let ops = [];
+    if (type === "batch") {
+      ops = event.data.ops;
+    } else if (type === "put") {
+      ops = [event.data];
+    }
+    for (let i = ops.length - 1; i >= 0; i--) {
+      const { key, value } = ops[i];
+      if (!keys.has(key)) {
+        const docValue = await getValueFromLink(blocks, value);
+        updates.push({ key, value: docValue.doc, del: docValue.del, clock: link });
+        limit--;
+        keys.add(key);
       }
     }
+    if (event.parents) {
+      updates = await gatherUpdates(
+        blocks,
+        eventsFetcher,
+        event.parents,
+        since,
+        updates,
+        keys,
+        didLinks,
+        limit
+      );
+    }
   }
   return updates;
 }
@@ -378,25 +388,32 @@ async function doCompact(blockLog, head) {
     return;
   }
   isCompacting = true;
+  time("compact head");
   for (const cid of head) {
     const bl = await blockLog.get(cid);
     if (!bl)
       throw new Error("Missing head block: " + cid.toString());
   }
-  for await (const entry of getAllEntries(blockLog, head)) {
-  }
-  for await (const [, link] of (0, import_crdt.entries)(blockLog, head)) {
-    const bl = await blockLog.get(link);
-    if (!bl)
-      throw new Error("Missing entry block: " + link.toString());
+  timeEnd("compact head");
+  time("compact all entries");
+  for await (const _entry of getAllEntries(blockLog, head)) {
   }
+  timeEnd("compact all entries");
+  time("compact clock vis");
   for await (const _line of (0, import_clock.vis)(blockLog, head)) {
   }
+  timeEnd("compact clock vis");
+  time("compact root");
   const result = await (0, import_crdt.root)(blockLog, head);
+  timeEnd("compact root");
+  time("compact root blocks");
   for (const { cid, bytes } of [...result.additions, ...result.removals]) {
     blockLog.loggedBlocks.putSync(cid, bytes);
   }
+  timeEnd("compact root blocks");
+  time("compact changes");
   await clockChangesSince(blockLog, head, [], {});
+  timeEnd("compact changes");
   isCompacting = false;
 }
 async function getBlock(blocks, cidString) {
@@ -760,8 +777,8 @@ function makeMapFnFromName(name) {
 }
 
 // src/crdt-clock.ts
-var import_clock2 = require("@alanshaw/pail/clock");
-var import_crdt2 = require("@alanshaw/pail/crdt");
+var import_clock2 = require("@web3-storage/pail/clock");
+var import_crdt2 = require("@web3-storage/pail/crdt");
 
 // src/apply-head-queue.ts
 function applyHeadQueue(worker) {
@@ -958,12 +975,12 @@ var CRDT = class {
       }
     });
   }
-  async bulk(updates, options) {
+  async bulk(updates) {
     await this.ready;
     const prevHead = [...this.clock.head];
     const meta = await this.blockstore.transaction(
       async (blocks) => {
-        const { head } = await applyBulkUpdateToCrdt(blocks, this.clock.head, updates, options);
+        const { head } = await applyBulkUpdateToCrdt(blocks, this.clock.head, updates);
         updates = updates.map(({ key, value, del, clock }) => {
           readFiles(this.blockstore, { doc: value });
           return { key, value, del, clock };