@le-space/orbitdb-storage-bridge 0.13.0 → 0.14.1

package/README.md

@@ -329,3 +329,20 @@ and how to run each suite.
 ## License
 
 MIT License
+
+## Trademarks
+
+LoRa® is a trademark of Semtech Corporation. Meshtastic® is a registered
+trademark of Meshtastic LLC. Other names used here — OrbitDB, IPFS, Filecoin,
+Storacha, Pinata, Lighthouse, Aleph — belong to their respective owners and
+appear only to say what this package interoperates with. This project is not
+affiliated with or endorsed by any of them.
+
+The first two appear in this repository only in prose about what the courier
+seam is *for*: `courier-sync` is transport-neutral and this package contains no
+code from either project and depends on neither. That is deliberate rather than
+incidental. The Meshtastic client libraries (`@meshtastic/core`,
+`@meshtastic/transport-web-bluetooth`) are **GPL-3.0-only**, and the courier
+that drives them lives in [funkpost](https://github.com/NiKrause/funkpost), on
+the GPL side of that line. This package is MIT and stays that way; the door
+only opens one way.

package/lib/courier-sync.js

@@ -165,6 +165,61 @@ function isOplogEntry(value) {
   );
 }
 
+/**
+ * What the peer can be assumed to hold, given the heads it named.
+ *
+ * The heads alone are not a stop set. An OrbitDB entry's `refs` are skip-list
+ * back-references that point *past* its parent, so a walk that stops only at
+ * the named hashes follows a ref around them and carries on to the root: a peer
+ * missing one entry was sent the whole log, measured at 12 blocks and 8742 B
+ * where 2 blocks and 1533 B were owed (funkpost's two phones over LoRa, #127).
+ * At half a kilobyte a minute that is the difference between a list that syncs
+ * and one that cannot.
+ *
+ * A head is a claim about everything below it, so the closure below those heads
+ * is what the peer holds. It is walked over blocks *we* hold; where we cannot
+ * follow, that ancestry stays unknown and so stays out of the stop set, which
+ * errs towards sending — the direction that costs bytes rather than
+ * correctness.
+ *
+ * Reading the whole ancestry locally to avoid transmitting it is a good trade
+ * on any carrier: the reads are a blockstore away, the bytes are airtime.
+ */
+async function reachableFrom(db, roots) {
+  const held = new Set();
+  const queue = [...roots];
+  while (queue.length > 0) {
+    const hash = queue.shift();
+    if (held.has(hash)) continue;
+    held.add(hash);
+    // The index first, and only then the blocks. `IPFSBlockStorage.get` on a
+    // miss waits out a network timeout — measured at 20 s against a Helia node
+    // with no peers, which is every phone in the field — and the peer's heads
+    // are precisely where misses live: their newest entry is the one we have
+    // not got. `has` reads the log's index and answers at once.
+    //
+    // The rule is already written down twenty lines below, in the closure
+    // check, and this walk broke it: 0.14.0 built correct deltas and took
+    // twenty seconds to do it, which stalled the exchange past every timeout
+    // around it (funkpost#170). The suite could not see it, because its nodes
+    // are built offline and a miss there fails instantly.
+    if (!(await db.log.has(hash))) continue; // theirs, not ours: stop here
+    const bytes = await db.log.storage.get(hash).catch(() => null);
+    if (!bytes) continue; // not ours to follow; their ancestry ends here for us
+    let value;
+    try {
+      value = dagCbor.decode(bytes);
+    } catch {
+      continue;
+    }
+    if (!isOplogEntry(value)) continue;
+    for (const parent of [...value.next, ...(value.refs || [])]) {
+      if (!held.has(parent)) queue.push(parent);
+    }
+  }
+  return held;
+}
+
 /**
  * Compute the delta a peer with `theirHeads` is missing: entry blocks from our
  * heads down to their heads, the identity blocks those entries reference, and
@@ -184,10 +239,19 @@ function isOplogEntry(value) {
  * @returns {Promise<{heads: Array<string>, blocks: Array<{hash: string, bytes: Uint8Array}>}>}
  */
 export async function createDelta({ db, theirHeads = [] }) {
-  const stop = new Set(theirHeads);
   const heads = await db.log.heads();
   const headHashes = heads.map((entry) => entry.hash);
 
+  // The peer stands exactly where we do: nothing is owed, and the ancestry
+  // need not be read to find that out. This is the steady state between two
+  // quiet peers, so it is worth answering before the walk below.
+  const named = new Set(theirHeads);
+  if (headHashes.length > 0 && headHashes.every((hash) => named.has(hash))) {
+    return { heads: headHashes, blocks: [] };
+  }
+
+  const stop = await reachableFrom(db, theirHeads);
+
   const seen = new Set();
   const identityHashes = new Set();
   const entryBlocks = [];
@@ -267,7 +331,9 @@ export async function createDelta({ db, theirHeads = [] }) {
  * @param {Object} params
  * @param {Object} params.db An open OrbitDB database
  * @param {{heads: Array<string>, blocks: Array<{hash: string, bytes: Uint8Array}>}} params.delta
- * @returns {Promise<{complete: boolean, joined: number, missing: Array<string>, entries: Array<Object>}>}
+ * @returns {Promise<{complete: boolean, joined: number, missing: Array<string>,
+ *   entries: Array<Object>, heads: number, outcome: Object}>} `heads` is how
+ *   many were offered and `outcome` says what became of each — see `noJoins`.
  */
 export async function applyDelta({ db, delta }) {
   return applyDeltaToStores({
@@ -289,7 +355,39 @@ function dbBlockstore(db) {
   };
 }
 
+/**
+ * Why a head did not join.
+ *
+ * The join loop below has exactly five exits, and from outside four of them
+ * look the same: no join, and — since "synced" only fires when something
+ * joined — no event at all. That silence is what left a day of field logs
+ * unreadable. Two phones over LoRa received five complete deltas and joined
+ * nothing all day, and the log could not say whether the courier was working
+ * or broken, because the two findings that matter produce identical silence:
+ *
+ *   held    the entry was already in the log. Another route brought it first;
+ *           the courier delivered something nobody needed, which is wasteful
+ *           but correct.
+ *   absent  the sender named a head and did not send it. A defect in the
+ *           delta it built, and the database does not move.
+ *
+ * Opposite repairs, one symptom. `malformed` and `refused` should not happen
+ * at all; they are counted separately rather than folded into `absent` so
+ * that "should not happen" stays falsifiable in a field log.
+ *
+ * Reported for every delivery through the "applied" event, including the
+ * deliveries that came to nothing — those are the interesting ones.
+ */
+const noJoins = () => ({
+  joined: 0,
+  held: 0,
+  absent: 0,
+  malformed: 0,
+  refused: 0,
+});
+
 async function applyDeltaToStores({ blockstore, log, events, delta }) {
+  const heads = delta.heads || [];
   const inDelta = new Map();
   for (const block of delta.blocks || []) {
     inDelta.set(block.hash, block.bytes);
@@ -312,32 +410,56 @@ async function applyDeltaToStores({ blockstore, log, events, delta }) {
     }
   }
   if (missing.length > 0) {
-    return { complete: false, joined: 0, missing, entries: [] };
+    return {
+      complete: false,
+      joined: 0,
+      missing,
+      entries: [],
+      heads: heads.length,
+      outcome: noJoins(),
+    };
   }
 
-  let joined = 0;
+  const outcome = noJoins();
   const entries = [];
-  for (const hash of delta.heads || []) {
-    if (await log.has(hash)) continue;
+  for (const hash of heads) {
+    if (await log.has(hash)) {
+      outcome.held++;
+      continue;
+    }
     const bytes = inDelta.get(hash);
-    if (!bytes) continue;
+    if (!bytes) {
+      outcome.absent++;
+      continue;
+    }
     const value = dagCbor.decode(bytes);
-    if (!isOplogEntry(value)) continue;
+    if (!isOplogEntry(value)) {
+      outcome.malformed++;
+      continue;
+    }
     const entry = { ...value, hash };
-    const updated = await log.joinEntry(entry);
-    if (updated) {
-      joined++;
-      entries.push(entry);
+    if (!(await log.joinEntry(entry))) {
+      outcome.refused++;
+      continue;
     }
+    outcome.joined++;
+    entries.push(entry);
   }
 
   // Database.applyOperation emits 'update' when the pubsub Sync delivers an
   // entry; a courier delivery is the same event from the application's side.
-  if (events && joined > 0) {
+  if (events && outcome.joined > 0) {
     for (const entry of entries) events.emit("update", entry);
   }
 
-  return { complete: true, joined, missing: [], entries };
+  return {
+    complete: true,
+    joined: outcome.joined,
+    missing: [],
+    entries,
+    heads: heads.length,
+    outcome,
+  };
 }
 
 /**
@@ -373,6 +495,14 @@ async function applyDeltaToStores({ blockstore, log, events, delta }) {
  *   carrier that cannot keep up before the oldest is dropped.
  * @returns {Promise<Object>} sync handle: { start, stop, announce, hello,
  *   presence, forgetPeers, db(), events }
+ *
+ * Events, via `sync.on(name, cb)`:
+ *   "message" { direction, type, bytes }  one message on or off the carrier
+ *   "synced"  { joined, entries }         the database moved
+ *   "applied" { complete, heads, missing, joined, held, absent, malformed,
+ *               refused }                 what a `blocks` delivery came to,
+ *                                         fired even when it came to nothing
+ *   "error"   Error                       a delivery that threw
  */
 export async function createCourierSync({
   db = null,
@@ -411,7 +541,7 @@ export async function createCourierSync({
   const pendingBlocks = new Map(); // hash -> bytes, parked until the database can open
   const peers = new Map(); // sender id (hex) -> when we last heard it
   let lastHeardAt = null; // any traffic for this database, identified or not
-  const listeners = { synced: [], message: [], error: [] };
+  const listeners = { synced: [], applied: [], message: [], error: [] };
   let database = db;
   // Opened on first contact but not handed out: the bootstrap is not in it yet.
   // The protocol works on it all the same, so repair stays incremental.
@@ -723,6 +853,16 @@ export async function createCourierSync({
     } finally {
       applying = false;
     }
+    // What the delivery came to, always — including when it came to nothing.
+    // A delta that moves the database is a "synced"; a delta that moves
+    // nothing is either a courier doing no harm or a courier doing no good,
+    // and this is the only line that tells them apart.
+    emit("applied", {
+      complete: result.complete,
+      heads: result.heads,
+      missing: result.missing.length,
+      ...result.outcome,
+    });
     if (!result.complete) {
       post(
         {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@le-space/orbitdb-storage-bridge",
-  "version": "0.13.0",
+  "version": "0.14.1",
   "description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
   "main": "lib/orbitdb-storacha-bridge.js",
   "svelte": "dist/components/",