@storacha/clawracha 0.2.2 → 0.3.0

package/dist/mdsync/index.js

@@ -1,22 +1,21 @@
 /**
  * mdsync — CRDT markdown storage on top of UCN Pail.
  *
- * Stores RGA-backed markdown trees at Pail keys. Each key's value is a
- * MarkdownEntry containing:
+ * Each key's value is a single DAG-CBOR block (DeserializedMarkdownEntry)
+ * containing:
  *   - The current RGA tree (full document state)
  *   - An RGA of MarkdownEvents (causal history scoped to this key)
  *   - The last changeset applied (for incremental replay during merge)
  *
  * On read, if the Pail has multiple heads (concurrent writes), we resolve
  * by walking from the common ancestor forward, replaying each branch's
- * changesets and merging event RGAs — analogous to how Pail itself resolves
- * concurrent root updates.
+ * changesets and merging event RGAs.
  */
 import { MemoryBlockstore, withCache } from "@storacha/ucn/block";
 import * as CRDT from "@web3-storage/pail/crdt";
 import { CID } from "multiformats";
 import * as cbor from "@ipld/dag-cbor";
-import { fromMarkdown, encodeTree, encodeRGA, RGA, decodeRGA, decodeTree, computeChangeSet, applyRGAChangeSet, encodeChangeSet, decodeChangeSet, toMarkdown, mergeRGATrees, } from "@storacha/md-merge";
+import { fromMarkdown, RGA, computeChangeSet, applyRGAChangeSet, toMarkdown, mergeRGATrees, serializeTree, deserializeTree, serializeRGA, deserializeRGA, serializeChangeSet, deserializeChangeSet as deserializeChangeSetStruct, stripUndefined, } from "@storacha/md-merge";
 import * as Pail from "@web3-storage/pail";
 import { decode, encode } from "multiformats/block";
 import { sha256 } from "multiformats/hashes/sha2";
@@ -69,40 +68,78 @@ const nohierarchyComparator = (a, b) => a.toString() === b.toString() ? 0 : a.to
 /**
  * Build a comparator from the event RGA's weighted BFS ordering.
  * Events earlier in the BFS (closer to the root of the causal tree)
- * compare as "less than" later events. This is used as the EventComparator
- * for all RGA tree operations — it determines how concurrent inserts are
- * ordered in the document.
+ * compare as "less than" later events.
  */
 const makeComparator = (events) => {
     const ordered = events.toWeightedArray();
     const index = new Map(ordered.map((e, i) => [e.toString(), i]));
     return (a, b) => (index.get(a.toString()) ?? -1) - (index.get(b.toString()) ?? -1);
 };
+// ---- Single-block serialization ----
 /**
- * Create the first MarkdownEntry for a key — bootstraps the RGA tree and
+ * Encode a DeserializedMarkdownEntry as a single DAG-CBOR block.
+ * All data (tree, events, changeset) is inlined — no CID references.
+ */
+export const encodeMarkdownEntry = async (entry) => {
+    const flat = {
+        type: entry.type,
+        root: serializeTree(entry.root),
+        events: serializeRGA(entry.events, serializeMarkdownEvent),
+    };
+    if (entry.type === "update") {
+        flat.changeset = serializeChangeSet(entry.changeset);
+    }
+    return await encode({
+        value: stripUndefined(flat),
+        codec: cbor,
+        hasher: sha256,
+    });
+};
+/**
+ * Decode a single DAG-CBOR block back to a DeserializedMarkdownEntry.
+ */
+export const decodeMarkdownEntry = async (block) => {
+    const decoded = await decode({
+        bytes: block.bytes,
+        codec: cbor,
+        hasher: sha256,
+    });
+    const flat = decoded.value;
+    const eventRGA = deserializeRGA(flat.events, parseMarkdownEvent, deserializeMarkdownEvent, nohierarchyComparator);
+    const comparator = makeComparator(eventRGA);
+    const root = deserializeTree(flat.root, parseMarkdownEvent, comparator);
+    if (flat.type === "initial") {
+        return { type: "initial", root, events: eventRGA };
+    }
+    const changeset = deserializeChangeSetStruct(flat.changeset, parseMarkdownEvent);
+    return { type: "update", root, events: eventRGA, changeset };
+};
+// ---- Core helpers ----
+/**
+ * Create the first entry for a key — bootstraps the RGA tree and
  * event RGA from raw markdown. Used when a key doesn't exist yet.
  */
-const firstPut = async (newMarkdown, parents) => {
+const firstPut = (newMarkdown, parents) => {
     const markdownEvent = new MarkdownEvent(parents);
     const eventRGA = new RGA(nohierarchyComparator);
     eventRGA.insert(undefined, markdownEvent, markdownEvent);
-    const eventBlock = await encodeRGA(eventRGA, serializeMarkdownEvent);
     // Only one event, so comparator is trivial (all nodes have the same event).
     const rgaRoot = fromMarkdown(newMarkdown, markdownEvent, (a, b) => 0);
-    return serializeMarkdownEntry({
+    return {
         type: "initial",
         root: rgaRoot,
         events: eventRGA,
-    });
+    };
 };
 // ---- Public API ----
 /**
  * First put into an empty Pail (v0 = no existing revision).
- * Returns the markdown entry CID and blocks to store. Caller is
- * responsible for creating the Pail revision via Revision.v0Put.
+ * Returns a single block to store. Caller is responsible for
+ * creating the Pail revision via Revision.v0Put.
  */
 export const v0Put = async (newMarkdown) => {
-    return firstPut(newMarkdown, []);
+    const entry = firstPut(newMarkdown, []);
+    return encodeMarkdownEntry(entry);
 };
 /**
  * Put markdown content at a key in an existing Pail.
@@ -112,135 +149,36 @@ export const v0Put = async (newMarkdown) => {
  * an RGA changeset against the resolved tree, applies it, and stores the
  * updated entry.
  *
- * Returns the markdown entry CID and blocks to store. Caller is
- * responsible for creating the Pail revision via Revision.put.
+ * Returns a single block to store, or null if no changes detected.
+ * Caller is responsible for creating the Pail revision via Revision.put.
  */
 export const put = async (blocks, current, key, newMarkdown) => {
     const mdEntry = await resolveValue(blocks, current, key);
     if (!mdEntry) {
         // Key doesn't exist yet — bootstrap with firstPut.
-        return firstPut(newMarkdown, current.revision.map((r) => r.event.cid));
+        const entry = firstPut(newMarkdown, current.revision.map((r) => r.event.cid));
+        return encodeMarkdownEntry(entry);
     }
     const { events: eventRGA, root: rgaRoot } = mdEntry;
     // Create a new event anchored to the current Pail revision heads.
     const mdEvent = new MarkdownEvent(current.revision.map((r) => r.event.cid));
-    // Insert after the last event in weighted order — this places the new event
-    // deeper than all existing events in the causal tree, correctly representing
-    // that it incorporates all known history.
+    // Insert after the last event in weighted order.
     const orderedNodes = eventRGA.toWeightedNodes();
     eventRGA.insert(orderedNodes[orderedNodes.length - 1].id, mdEvent, mdEvent);
     // Diff the current tree against the new markdown and apply.
     const changeset = computeChangeSet(rgaRoot, newMarkdown, mdEvent);
     if (changeset.changes.length === 0) {
-        return null; // No changes to apply, skip writing a new entry.
+        return null; // No changes to apply.
     }
     const comparator = makeComparator(eventRGA);
     const newRoot = applyRGAChangeSet(rgaRoot, changeset, comparator);
-    return serializeMarkdownEntry({
+    return encodeMarkdownEntry({
         type: "update",
         root: newRoot,
         events: eventRGA,
         changeset,
     });
 };
-// ---- Block fetching helpers ----
-/** Decode a MarkdownEntry (CID pointers) from a DAG-CBOR block. */
-const getMarkdownEntry = async (blocks, mdEntryCid) => {
-    const mdEntryBlock = await blocks.get(mdEntryCid);
-    if (!mdEntryBlock) {
-        throw new Error(`Could not find markdown entry block for CID ${mdEntryCid}`);
-    }
-    return (await decode({ bytes: mdEntryBlock.bytes, codec: cbor, hasher: sha256 })).value;
-};
-/** Fetch and decode the event RGA from its block CID. */
-const getEventRGAFromCID = async (blocks, eventsCid) => {
-    const eventBlock = await blocks.get(eventsCid);
-    if (!eventBlock) {
-        throw new Error(`Could not find event block for CID ${eventsCid}`);
-    }
-    const eventRGA = await decodeRGA(eventBlock, parseMarkdownEvent, deserializeMarkdownEvent, nohierarchyComparator);
-    return eventRGA;
-};
-/** Fetch and decode an RGAChangeSet from its block CID. */
-const getRGAChangeSetFromCID = async (blocks, changesetCid) => {
-    const changesetBlock = await blocks.get(changesetCid);
-    if (!changesetBlock) {
-        throw new Error(`Could not find changeset block for CID ${changesetCid}`);
-    }
-    return await decodeChangeSet(changesetBlock, parseMarkdownEvent);
-};
-/** Fetch and decode an RGA tree from its block CID, using the event RGA for ordering. */
-const getRGATreeFromRootCID = async (blocks, rootCid, eventRGA) => {
-    const comparator = makeComparator(eventRGA);
-    const rootBlock = await blocks.get(rootCid);
-    if (!rootBlock) {
-        throw new Error(`Could not find root block for CID ${rootCid}`);
-    }
-    return await decodeTree(rootBlock, parseMarkdownEvent, comparator);
-};
-/**
- * Fully deserialize a MarkdownEntry from its CID — fetches and decodes
- * the entry, event RGA, tree, and (if update) changeset.
- */
-const deserializedMarkdownEntryCID = async (blocks, mdEntryCid) => {
-    const mdEntry = await getMarkdownEntry(blocks, mdEntryCid);
-    const eventRGA = await getEventRGAFromCID(blocks, mdEntry.events);
-    const rgaRoot = await getRGATreeFromRootCID(blocks, mdEntry.root, eventRGA);
-    if (mdEntry.type === "initial") {
-        return {
-            type: "initial",
-            root: rgaRoot,
-            events: eventRGA,
-        };
-    }
-    const changeset = await getRGAChangeSetFromCID(blocks, mdEntry.changeset);
-    return {
-        type: "update",
-        root: rgaRoot,
-        events: eventRGA,
-        changeset,
-    };
-};
-/**
- * Serialize a DeserializedMarkdownEntry to DAG-CBOR blocks.
- * Returns the entry's CID and all blocks that need to be stored
- * (event RGA, tree, changeset if update, and the entry itself).
- */
-const serializeMarkdownEntry = async (entry) => {
-    let blocks = [];
-    const eventBlock = await encodeRGA(entry.events, serializeMarkdownEvent);
-    blocks.push(eventBlock);
-    const rootBlock = await encodeTree(entry.root);
-    blocks.push(rootBlock);
-    const changesetBlock = entry.type === "update"
-        ? await encodeChangeSet(entry.changeset)
-        : undefined;
-    if (changesetBlock) {
-        blocks.push(changesetBlock);
-    }
-    const mdEntry = entry.type === "initial"
-        ? {
-            type: "initial",
-            root: rootBlock.cid,
-            events: eventBlock.cid,
-        }
-        : {
-            type: "update",
-            root: rootBlock.cid,
-            events: eventBlock.cid,
-            changeset: changesetBlock?.cid,
-        };
-    const mdEntryBlock = await encode({
-        value: mdEntry,
-        codec: cbor,
-        hasher: sha256,
-    });
-    blocks.push(mdEntryBlock);
-    return {
-        mdEntryCid: mdEntryBlock.cid,
-        additions: blocks,
-    };
-};
 // ---- Resolution (multi-head merge) ----
 /**
  * Resolve the current markdown value for a key, merging concurrent heads.
@@ -249,16 +187,8 @@ const serializeMarkdownEntry = async (entry) => {
  *
  * If there are multiple heads, finds their common ancestor in the Pail's
  * merkle clock, then replays all events from ancestor → heads in causal order.
- * For each "put" event:
- *   - If "initial": bootstraps the entry (must be the first event for this key)
- *   - If "update": merges the event's causal history into the running event RGA,
- *     then applies the changeset to the running tree using the merged comparator
- * For "del" events: clears the entry (file deleted).
- *
- * This is analogous to how Pail resolves concurrent root updates — walk from
- * common ancestor, replay operations in deterministic order.
  */
-const resolveValue = async (blocks, current, key) => {
+const resolveValue = async (blocks, current, key, decrypt) => {
     const mdEntryBlockCid = await Pail.get(blocks, current.root, key);
     if (!mdEntryBlockCid) {
         return undefined;
@@ -266,9 +196,18 @@ const resolveValue = async (blocks, current, key) => {
     // Cache event blocks in memory so EventFetcher can find them.
     blocks = withCache(blocks, new MemoryBlockstore(current.revision.map((r) => r.event)));
     const events = new EventFetcher(blocks);
+    // Fetch entry bytes: decrypt callback for private spaces, or raw block fetch.
+    const getEntryBytes = async (cid) => {
+        if (decrypt)
+            return decrypt(cid);
+        const block = await blocks.get(cid);
+        if (!block)
+            throw new Error(`Could not find block for CID ${cid}`);
+        return block.bytes;
+    };
     // Fast path: single head, no merge needed.
     if (current.revision.length === 1) {
-        return await deserializedMarkdownEntryCID(blocks, mdEntryBlockCid);
+        return decodeMarkdownEntry({ bytes: await getEntryBytes(mdEntryBlockCid) });
     }
     // Multi-head: find common ancestor and replay events in causal order.
     const ancestor = await CRDT.findCommonAncestor(events, current.revision.map((r) => r.event.cid));
@@ -279,9 +218,10 @@ const resolveValue = async (blocks, current, key) => {
     const aevent = await events.get(ancestor);
     let { root } = aevent.value.data;
     const rootMDEntryCid = await Pail.get(blocks, root, key);
-    let mdEntry = rootMDEntryCid
-        ? await deserializedMarkdownEntryCID(blocks, rootMDEntryCid)
-        : undefined;
+    let mdEntry;
+    if (rootMDEntryCid) {
+        mdEntry = await decodeMarkdownEntry({ bytes: await getEntryBytes(rootMDEntryCid) });
+    }
     // Get all events from ancestor → heads, sorted in deterministic causal order.
     const sorted = await CRDT.findSortedEvents(events, current.revision.map((r) => r.event.cid), ancestor);
     // Filter to only events that touch this key.
@@ -310,33 +250,24 @@ const resolveValue = async (blocks, current, key) => {
             data = { ...data, ...op };
         }
         if (data.type === "put") {
-            // Fetch the MarkdownEntry that was stored with this event.
             const mdEntryCid = await Pail.get(blocks, data.root, key);
             if (!mdEntryCid) {
                 throw new Error(`Could not find markdown entry for CID ${data.root} and key ${key}`);
             }
-            const newMDEntry = await getMarkdownEntry(blocks, mdEntryCid);
+            const newMDEntry = await decodeMarkdownEntry({ bytes: await getEntryBytes(mdEntryCid) });
             if (newMDEntry.type === "initial") {
-                // First write for this key — bootstrap from the stored entry.
-                const newEventRGA = await getEventRGAFromCID(blocks, newMDEntry.events);
-                const newRoot = await getRGATreeFromRootCID(blocks, newMDEntry.root, newEventRGA);
                 if (mdEntry) {
-                    // Concurrent initial — two branches independently created this key.
-                    // Merge event histories and merge the two RGA trees.
-                    mdEntry.events.merge(newEventRGA);
+                    // Concurrent initial — merge event histories and RGA trees.
+                    mdEntry.events.merge(newMDEntry.events);
                     const comparator = makeComparator(mdEntry.events);
                     mdEntry = {
                         type: "initial",
                         events: mdEntry.events,
-                        root: mergeRGATrees(mdEntry.root, newRoot, comparator),
+                        root: mergeRGATrees(mdEntry.root, newMDEntry.root, comparator),
                     };
                 }
                 else {
-                    mdEntry = {
-                        type: "initial",
-                        events: newEventRGA,
-                        root: newRoot,
-                    };
+                    mdEntry = newMDEntry;
                 }
             }
             else {
@@ -344,23 +275,17 @@ const resolveValue = async (blocks, current, key) => {
                 if (!mdEntry) {
                     throw new Error(`Expected existing markdown entry for update event, found none for CID ${mdEntryCid}`);
                 }
-                // Merge the event's causal history into our running history.
-                // RGA merge is commutative — order of merging doesn't matter.
-                const eventRGA = mdEntry.events;
-                eventRGA.merge(await getEventRGAFromCID(blocks, newMDEntry.events));
-                const changeset = await getRGAChangeSetFromCID(blocks, newMDEntry.changeset);
-                // Rebuild comparator from merged event history, then apply changeset.
-                const comparator = makeComparator(eventRGA);
+                mdEntry.events.merge(newMDEntry.events);
+                const comparator = makeComparator(mdEntry.events);
                 mdEntry = {
                     type: "update",
-                    events: eventRGA,
-                    root: applyRGAChangeSet(mdEntry.root, changeset, comparator),
-                    changeset,
+                    events: mdEntry.events,
+                    root: applyRGAChangeSet(mdEntry.root, newMDEntry.changeset, comparator),
+                    changeset: newMDEntry.changeset,
                 };
             }
         }
         else if (data.type === "del") {
-            // Key deleted — clear state. If re-created later, it'll be a new "initial".
             mdEntry = undefined;
         }
     }
@@ -370,8 +295,8 @@ const resolveValue = async (blocks, current, key) => {
  * Get the current markdown string for a key, resolving concurrent heads.
  * Returns undefined if the key doesn't exist.
  */
-export const get = async (blocks, current, key) => {
-    const mdEntry = await resolveValue(blocks, current, key);
+export const get = async (blocks, current, key, decrypt) => {
+    const mdEntry = await resolveValue(blocks, current, key, decrypt);
     if (!mdEntry) {
         return undefined;
     }

package/dist/plugin.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EACV,iBAAiB,EAGlB,MAAM,qBAAqB,CAAC;AAkC7B,MAAM,CAAC,OAAO,UAAU,MAAM,CAAC,GAAG,EAAE,iBAAiB,QAspBpD"}
+{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAKH,OAAO,KAAK,EACV,iBAAiB,EAGlB,MAAM,qBAAqB,CAAC;AA0B7B,MAAM,CAAC,OAAO,UAAU,MAAM,CAAC,GAAG,EAAE,iBAAiB,QAqkBpD"}