@le-space/orbitdb-storage-bridge 0.14.1 → 0.16.0

package/lib/courier-sync.js

@@ -17,15 +17,84 @@
  *     all three. Returns an unsubscribe function.
  *
  * Wire messages (dag-cbor encoded, one per courier payload):
- *   { v, tag, p, t: "announce", heads: [hash] }
+ *   { v, tag, p, t: "announce", heads: [hash], r? }   r: please reconcile
  *   { v, tag, p, t: "want", cids: [hash], have: [hash] }
  *   { v, tag, p, t: "blocks", heads: [hash], blocks: [{ hash, bytes }] }
+ *   { v, tag, p, t: "op", id: bytes8, o: { op, key, value } }   one change
  *   { v, tag, p, t: "hello" }   is anybody keeping this database out there?
  *   { v, tag, p, t: "here" }    the answer
  * `tag` is a short hash of the database address, so couriers can be shared
  * between databases without cross-talk while the address itself stays off
  * the air (the mesh reads everything).
  *
+ * Two ways to carry a change, and the cheap one is not replication.
+ *
+ * `announce`/`want`/`blocks` move the log itself: signed entries, their
+ * ancestry, the identity that vouches for them. That is real OrbitDB
+ * replication — every peer ends up with the same log and the same hashes — and
+ * it is what a cold join needs, because a peer that has never seen the database
+ * has to be given the manifest and the whole history.
+ *
+ * It is also expensive out of all proportion to a todo. The operation inside
+ * that entry — `{op: "PUT", key, value}`, which OrbitDB hands to every
+ * `update` listener already — is the whole of what anyone meant to send.
+ *
+ * Measured over funkpost's Meshtastic courier, framing, ARQ and duty-cycle
+ * pacing included, for one further change to a list both sides already hold:
+ *
+ *   | plane        | bytes on the air |
+ *   | ------------ | ---------------- |
+ *   | `delta`      | 1774 B           |
+ *   | `operations` |   86 B           |
+ *
+ * Twenty times. On a carrier moving half a kilobyte a minute that is three and
+ * a half minutes against ten seconds, for one ticked box.
+ *
+ * So `op` ships the operation and lets the far side perform it locally, as its
+ * own write, with its own identity. What that buys and what it costs:
+ *
+ *   - The two logs **diverge**: each device holds its own entry for the same
+ *     change, with a different hash. For a keyvalue database the materialised
+ *     view still agrees, and once an IP path returns OrbitDB's own sync merges
+ *     both logs and the divergence becomes history rather than disagreement.
+ *   - Two devices editing the **same key** while both are offline can show
+ *     different values until that merge, because each applies the other's
+ *     operation after its own. A CRDT would not have this; an oplog with
+ *     last-write-wins does. For a shared todo list it is an acceptable trade
+ *     and it must be a deliberate one.
+ *   - **The receiver must be allowed to write.** This is the requirement that
+ *     surprises: on the delta plane the far side only *stores* an entry someone
+ *     else signed, so a database whose access controller names one writer
+ *     replicates fine. Here the far side performs the change as its own write,
+ *     so an access controller that excludes it refuses the operation outright —
+ *     *"Key … is not allowed to write to the log"*. `write: ["*"]` makes it
+ *     work, which is what funkpost's mesh-todo already does.
+ *
+ *     That is also where a pairing between two devices belongs, rather than in
+ *     a new protocol of its own: two devices that have met put each other's
+ *     OrbitDB identity in the write set, and "anyone holding the channel key"
+ *     becomes "these two". The access controller is the authorisation
+ *     mechanism this plane needs, and it already exists.
+ *   - Nothing is signed end to end. The receiver vouches for the change with
+ *     its own identity, so trust rests entirely on the carrier — which on a
+ *     Meshtastic channel means everyone holding the channel key. The sender id
+ *     in `p` is not authentication and must never be read as such.
+ *   - Reconciliation is asked for, not inferred: `announce()` sets `r` on the
+ *     wire and the far side answers with a delta. An announce without it is
+ *     noted and left alone, because divergent logs are this plane's ordinary
+ *     state rather than a gap.
+ *   - A lost `op` is simply lost; there is no ancestry to notice the hole.
+ *     Note that `send`'s own comment — that a dropped message is re-derivable
+ *     because a peer still wanting it asks again — holds for `announce`, `want`
+ *     and `blocks` and **not** for `op`: an operation the outbox sheds under
+ *     `maxOutbox` takes its change with it.
+ *     `announce()` is the repair: it runs the delta plane, which is complete by
+ *     construction. Cheap path for the ordinary change, expensive path when
+ *     something has to be made right.
+ *
+ * `liveUpdates: "operations"` opts in. The default stays `"delta"`, so nothing
+ * changes for a consumer that does not ask.
+ *
  * `p` is a four-byte sender id, and it is what makes *presence* possible: a
  * carrier can tell you a radio is in range, which is not the question. The
  * question is whether another program is keeping the same database, and only
@@ -52,6 +121,20 @@ export const COURIER_SYNC_VERSION = 1;
 
 const TAG_LENGTH = 8;
 
+// Eight bytes naming a change, derived from the entry hash so that every
+// retransmission of the same change carries the same id and applies once.
+const CHANGE_ID_LENGTH = 8;
+// How many change ids to remember. A duplicate arriving after this many others
+// is applied twice: one extra entry in the log, not a wrong value.
+const SEEN_OPS = 512;
+// How many operations may wait for a deliberate send before the batch goes as
+// a delta instead. Not a memory limit: the outbox holds `maxOutbox` (32) and
+// sheds the oldest when it overflows, and an operation is the one message here
+// that is *not* re-derivable — so a long batch of them would lose writes
+// quietly. A delta is one message and complete by construction, which is the
+// right shape once a batch is large enough to threaten that.
+const MAX_PENDING_OPS = 16;
+
 // Four bytes of sender id: enough that two peers in one conversation collide
 // with probability ~1 in 4 billion, small enough to ride on every message.
 const PEER_ID_LENGTH = 4;
@@ -185,6 +268,19 @@ function isOplogEntry(value) {
  * Reading the whole ancestry locally to avoid transmitting it is a good trade
  * on any carrier: the reads are a blockstore away, the bytes are airtime.
  */
+/**
+ * Eight bytes naming one change, the same on every retransmission of it.
+ *
+ * Derived from the entry hash rather than from the operation, so that two
+ * genuinely separate writes of the same value — ticking a box off and on and
+ * off again — stay separate changes, while a duplicate delivery of one write
+ * is recognised and applied once.
+ */
+async function changeId(entryHash) {
+  const digest = await sha256.digest(new TextEncoder().encode(entryHash));
+  return digest.digest.slice(0, CHANGE_ID_LENGTH);
+}
+
 async function reachableFrom(db, roots) {
   const held = new Set();
   const queue = [...roots];
@@ -478,6 +574,11 @@ async function applyDeltaToStores({ blockstore, log, events, delta }) {
  * @param {string} [params.address] Database address, required when `db` is not given
  * @param {Object} params.courier The byte courier (see module docs)
  * @param {Object} [params.dbOptions] Extra options for the lazy `orbitdb.open`
+ * @param {"delta"|"operations"} [params.liveUpdates="delta"] How a local write
+ *   travels. `"delta"` announces and lets the peer ask, which moves signed
+ *   entries and keeps both logs identical. `"operations"` ships the operation
+ *   itself — thirteen times smaller, at the cost of divergent logs and no
+ *   end-to-end signature. See the module docs before choosing it.
  * @param {boolean} [params.announceOnLocalUpdate=true] Announce as soon as a
  *   local write lands. Default keeps the eager behaviour. Set false where the
  *   courier is expensive — a duty-cycled radio, say — and the application would
@@ -512,6 +613,7 @@ export async function createCourierSync({
   dbOptions = {},
   rejoinIntervalMs = 15000,
   announceOnLocalUpdate = true,
+  liveUpdates = "delta",
   peerId = randomPeerId(),
   peerTimeoutMs = PEER_TIMEOUT_MS,
   sendTimeoutMs = SEND_TIMEOUT_MS,
@@ -527,6 +629,9 @@ export async function createCourierSync({
   if (!(peerId instanceof Uint8Array) || peerId.length !== PEER_ID_LENGTH) {
     throw new Error(`peerId must be ${PEER_ID_LENGTH} bytes`);
   }
+  if (liveUpdates !== "delta" && liveUpdates !== "operations") {
+    throw new Error('liveUpdates must be "delta" or "operations"');
+  }
   const databaseAddress = address || (db && db.address);
   if (!databaseAddress) {
     throw new Error("Either an open db or a database address is required");
@@ -542,6 +647,13 @@ export async function createCourierSync({
   const peers = new Map(); // sender id (hex) -> when we last heard it
   let lastHeardAt = null; // any traffic for this database, identified or not
   const listeners = { synced: [], applied: [], message: [], error: [] };
+  // Written locally and not yet sent, oldest first. Only fills when the
+  // operation plane is on and the application has opted out of sending on
+  // every write — the batching case, where a button decides.
+  const pendingOps = [];
+  // Change ids already applied, oldest first. Insertion-ordered, so the oldest
+  // key is the first one Map iteration yields.
+  const seenOps = new Map();
   let database = db;
   // Opened on first contact but not handed out: the bootstrap is not in it yet.
   // The protocol works on it all the same, so repair stays incremental.
@@ -726,7 +838,13 @@ export async function createCourierSync({
   const ourHeadHashes = async (target = local()) =>
     target ? (await target.log.heads()).map((entry) => entry.hash) : [];
 
-  const announce = async () => {
+  /**
+   * @param {boolean} [reconcile] Ask the peer to close the gap, not merely to
+   *   note where we stand. Only the operation plane distinguishes the two:
+   *   there, divergent logs are the ordinary state rather than a gap, so a
+   *   reconciliation has to be asked for or it would run after every change.
+   */
+  const announce = async (reconcile = false) => {
     if (!local()) {
       // Nothing local yet — not even the manifest. An announce of empty heads
       // cannot get one from a peer whose log is also empty, so first contact
@@ -735,7 +853,34 @@ export async function createCourierSync({
       await send({ t: "want", cids: [], have: [] });
       return;
     }
-    await send({ t: "announce", heads: await ourHeadHashes() });
+    const message = { t: "announce", heads: await ourHeadHashes() };
+    if (reconcile) message.r = true;
+    await send(message);
+  };
+
+  /**
+   * Everything written since the last deliberate send, as operations.
+   *
+   * Answers how many went, so the caller can tell "I sent your changes" from
+   * "there was nothing to send, so I asked the peer to reconcile" — which is
+   * what an announce means when the queue is empty.
+   *
+   * Past `MAX_PENDING_OPS` the batch goes as a delta instead. Not arbitrary:
+   * the outbox holds `maxOutbox` messages and sheds the oldest, and an
+   * operation is the one message here that cannot be re-derived, so a long
+   * batch of them would lose writes without saying so. One delta is one
+   * message and carries all of it.
+   */
+  const flushOperations = async () => {
+    if (pendingOps.length === 0) return 0;
+    const going = pendingOps.splice(0, pendingOps.length);
+    if (going.length > MAX_PENDING_OPS) {
+      emit("message", { direction: "out", type: "batch-too-long", bytes: 0 });
+      await announce(true);
+      return going.length;
+    }
+    for (const entry of going) await shipOperation(entry);
+    return going.length;
   };
 
   /**
@@ -769,15 +914,26 @@ export async function createCourierSync({
   };
 
   const watchLocalUpdates = () => {
-    // Opted out: the application announces when it decides to, not when a write
-    // happens. Incoming announces are still answered, so a peer asking for
-    // blocks is served — going quiet must not mean going deaf.
-    if (!announceOnLocalUpdate) return;
+    // On the delta plane, opting out means not watching at all: the application
+    // announces when it decides to. On the operation plane it means something
+    // else — the write still has to be *remembered*, or the deliberate send has
+    // nothing to send. So the hook goes on either way there, and only what it
+    // does with the entry changes.
+    //
+    // Incoming announces are still answered in both cases: going quiet must not
+    // mean going deaf.
+    if (!announceOnLocalUpdate && liveUpdates !== "operations") return;
     if (!database || offUpdate) return;
-    const onUpdate = () => {
+    const onUpdate = (entry) => {
       if (applying) return; // courier-applied entries already end in an announce
+      if (liveUpdates === "operations" && !announceOnLocalUpdate) {
+        pendingOps.push(entry);
+        return; // waits for the application to ask
+      }
       queue = queue
-        .then(() => announceSoon())
+        .then(() =>
+          liveUpdates === "operations" ? shipOperation(entry) : announceSoon(),
+        )
         .catch((error) => emit("error", error));
     };
     database.events.on("update", onUpdate);
@@ -796,6 +952,16 @@ export async function createCourierSync({
         );
       return;
     }
+    // On the operation plane both sides write their own entry for the same
+    // change, so the peer's heads being strangers to us is the ordinary state
+    // and not a gap to close. Without this, the acknowledgement that ends a
+    // blocks delivery reads as a reconcile request, and on logs that are
+    // permanently divergent each answer earns another: measured over the memory
+    // courier at nine messages for one reconcile against six with it. Both
+    // terminate — this is chatter, not a loop — and it does not change what an
+    // operation costs, which is one message either way.
+    if (liveUpdates === "operations" && !message.r) return;
+
     const ours = await ourHeadHashes();
     const theirSet = new Set(theirHeads);
     const theyLack = ours.filter((hash) => !theirSet.has(hash));
@@ -887,8 +1053,114 @@ export async function createCourierSync({
       emit("synced", { joined: result.joined, entries: result.entries });
     }
     // Tells the peer where we now stand — their diff turns empty and the
-    // exchange goes quiet; doubles as an end-to-end acknowledgement.
-    await announce();
+    // exchange goes quiet; doubles as an end-to-end acknowledgement. Not a
+    // reconcile request: two peers answering each other's would never stop.
+    await announce(false);
+  };
+
+  /**
+   * Remember a change id. Answers whether it was new.
+   *
+   * Our own changes are remembered as they go out, so a mesh repeating us —
+   * or a peer echoing the operation onward — cannot make us perform our own
+   * write a second time.
+   */
+  const rememberOp = (id) => {
+    const key = hex(id);
+    if (seenOps.has(key)) return false;
+    seenOps.set(key, Date.now());
+    while (seenOps.size > SEEN_OPS) seenOps.delete(seenOps.keys().next().value);
+    return true;
+  };
+
+  /**
+   * Perform someone else's change as our own write.
+   *
+   * Only the operations a keyvalue, documents or events database emits. A type
+   * whose operations are named differently is not carried this way, and saying
+   * so out loud is better than writing something that silently does nothing.
+   */
+  const performOperation = async (target, op) => {
+    const run = () => {
+      if (op.op === "PUT") return target.put(op.key, op.value);
+      if (op.op === "DEL") return target.del(op.key);
+      if (op.op === "ADD") return target.add(op.value);
+      throw new Error(`cannot perform operation ${op.op} over the courier`);
+    };
+    try {
+      return await run();
+    } catch (error) {
+      // The refusal every consumer of this plane meets first, and OrbitDB's own
+      // message does not say why it is happening here rather than on the delta
+      // plane. Name the cause where it is read.
+      if (/not allowed to write/i.test(error?.message || "")) {
+        throw new Error(
+          "the operation plane performs the change as a local write, so this " +
+            "device must be in the database's write set — open it with an " +
+            'access controller that admits it (mesh-todo uses write: ["*"]), ' +
+            'or stay on liveUpdates: "delta"',
+          { cause: error },
+        );
+      }
+      throw error;
+    }
+  };
+
+  /** One local change, on its way as an operation rather than as a log entry. */
+  const shipOperation = async (entry) => {
+    const op = entry && entry.payload;
+    if (!op || typeof op.op !== "string" || !entry.hash) return;
+    const id = await changeId(entry.hash);
+    rememberOp(id); // ours: never perform it on ourselves
+    post({ t: "op", id, o: op });
+  };
+
+  const handleOperation = async (message) => {
+    const id = message.id;
+    const op = message.o;
+    if (!(id instanceof Uint8Array) || id.length !== CHANGE_ID_LENGTH) return;
+    if (!op || typeof op.op !== "string") return;
+
+    const target = local();
+    // An operation carries no manifest, so it cannot bootstrap a database. Ask
+    // for the whole thing instead and let the delta plane answer.
+    if (!target) {
+      post({ t: "want", cids: [], have: [] }, { to: senderOf(message) });
+      return;
+    }
+
+    const report = (applied) =>
+      emit("applied", {
+        complete: true,
+        heads: 1,
+        missing: 0,
+        joined: applied ? 1 : 0,
+        held: applied ? 0 : 1,
+        absent: 0,
+        malformed: 0,
+        refused: 0,
+        via: "operation",
+      });
+
+    // Duplicates are ordinary on a broadcast carrier with retransmission, and
+    // performing one twice would put a second entry in the log for a change
+    // that already happened.
+    if (!rememberOp(id)) return report(false);
+
+    // The write we are about to make fires the database's own "update", which
+    // is what an application listens to — so the far side's list moves without
+    // anything else being wired. The guard is what stops that same event from
+    // sending the change straight back out.
+    applying = true;
+    try {
+      await performOperation(target, op);
+    } finally {
+      applying = false;
+    }
+    // No "synced": that event carries the entries that were joined, and nothing
+    // was joined here — the database performed a write of its own. "applied"
+    // is where a courier delivery reports what it came to.
+    report(true);
   };
 
   const handlePayload = (bytes) => {
@@ -917,6 +1189,7 @@ export async function createCourierSync({
         if (message.t === "announce") return handleAnnounce(message);
         if (message.t === "want") return handleWant(message);
         if (message.t === "blocks") return handleBlocks(message);
+        if (message.t === "op") return handleOperation(message);
       })
       .catch((error) => emit("error", error));
   };
@@ -935,7 +1208,9 @@ export async function createCourierSync({
       started = true;
       unsubscribe = courier.onPayload(handlePayload);
       watchLocalUpdates();
-      await announce();
+      // A start reconciles: whatever happened while this program was not
+      // running is exactly what the delta plane is for. Once per session.
+      await announce(true);
       // A joiner that has not bootstrapped keeps re-asking on its own until
       // the database opens — so a bootstrap the lossy channel dropped heals
       // without the user pressing "join" again. Cleared the moment the
@@ -953,7 +1228,18 @@ export async function createCourierSync({
       }
     },
     /** Re-announce — recovery poke after suspected loss. */
-    announce: () => announce(),
+    /**
+     * What an application's "send" button means.
+     *
+     * With changes waiting, it sends those — that is what the press was about.
+     * With none, it is a request to reconcile, which is what it always was.
+     * Internal announces never flush: the acknowledgement that ends a blocks
+     * delivery must not spend somebody's queued writes on its own.
+     */
+    announce: async () => {
+      if ((await flushOperations()) > 0) return;
+      await announce(true);
+    },
     /** This instance's sender id, as it appears on the wire. */
     peerId: hex(peerId),
     /**

package/lib/extract-blocks.js

@@ -256,3 +256,76 @@ export async function extractDatabaseBlocks(database, options = {}) {
   logger.info(`   📊 Extracted ${blocks.size} total blocks`);
   return { blocks, blockSources, manifestCID };
 }
+
+/**
+ * Several databases in one backup: every block of each, in one Map, and the
+ * metadata that names them all.
+ *
+ * The metadata is the shape `backupDatabaseCAR` writes and `isValidMetadata`
+ * accepts — `databases` was always a list, and here it finally holds more than
+ * one. Each entry also names the database's **heads**: `restoreFromBlocks` can
+ * find them from the blocks alone, but a stated head says which entries the
+ * writer considered current when the backup ran, and costs a few bytes.
+ *
+ * Blocks two databases share — a writer's identity — are kept once: the Map is
+ * keyed by the CID string OrbitDB uses, and the same block has the same name.
+ *
+ * @param {Object[]|Object<string, Object>} databases opened OrbitDB databases,
+ *   as a list or by name
+ * @param {Object} [options]
+ * @param {(progress: { stage: "database", index: number, total: number,
+ *   name: string, address: string, entries: number, blocks: number }) => void}
+ *   [options.onProgress] called after each database, with what it added
+ * @param {number} [options.timestamp] ms since the epoch; injected for tests
+ * @returns {Promise<{ blocks: Map<string, { cid: CID, bytes: Uint8Array }>,
+ *   metadata: Object }>}
+ */
+export async function bundleDatabases(databases, options = {}) {
+  const list = Array.isArray(databases) ? databases : Object.values(databases ?? {});
+  if (list.length === 0) throw new Error("bundleDatabases needs at least one database");
+
+  const blocks = new Map();
+  const described = [];
+  let totalEntries = 0;
+
+  for (const [index, database] of list.entries()) {
+    const before = blocks.size;
+    const { blocks: own, manifestCID } = await extractDatabaseBlocks(database);
+    for (const [name, block] of own) if (!blocks.has(name)) blocks.set(name, block);
+
+    const entries = (await database.log.values()).length;
+    const heads = (await database.log.heads()).map((head) => head.hash);
+    totalEntries += entries;
+    described.push({
+      address: database.address,
+      name: database.name,
+      type: database.type,
+      manifestCID,
+      entryCount: entries,
+      heads,
+    });
+    options.onProgress?.({
+      stage: "database",
+      index,
+      total: list.length,
+      name: database.name,
+      address: database.address,
+      entries,
+      blocks: blocks.size - before,
+    });
+  }
+
+  return {
+    blocks,
+    metadata: {
+      version: "1.0",
+      timestamp: options.timestamp ?? Date.now(),
+      databaseCount: described.length,
+      totalBlocks: blocks.size,
+      totalEntries,
+      // The first database's, so a reader of the one-database shape still finds one.
+      manifestCID: described[0].manifestCID,
+      databases: described,
+    },
+  };
+}