yrby 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5928987b16a5d3a3933b95875dff276e4e18364b46ed0e33baa8e48b5b2e847
-  data.tar.gz: 2676689ddee9403765353c4612055fb120ce9e1c6dcdbeb956c482eed313374d
+  metadata.gz: 2dcb5650668e6cc3f1e0ef1349426ba875d5580a945e90a62216e61a24bf9cef
+  data.tar.gz: eead1e7159caeacf517febe05149d0c55491dcceaf6fbbbe2f3c40fdf7620978
 SHA512:
-  metadata.gz: 7e13256e4881804c3330bf69ce86203bd2aea4bda9313db042f7d368221130f6624285cbddea1a9b61b967c3831635ace45b1291496100872ad8a93c97be2e10
-  data.tar.gz: fbf1c8c9ec0f0b23190578d562f75fb14a14ec3d562b54b86437c5abda5bd48a97f2da216927f66fde46b5d52fa12d38b4f39b8073b8c6222d656fd09fd2c24f
+  metadata.gz: 68948615ae023becf04deea8c767cdd075ccd7097bbcecefb52868a7c2257fe4bd39d20bbac83b1f6a47d78a2889d76e590a83b7df0d9f6e1ccea1a8fdfa115d
+  data.tar.gz: 70716f2948661046b71843e66dcec62e5cf3e2b5c1509e9ea76c49db6039909bc02f510a90e3b9f44054b20ea3c2297d10878834b3c751abd1a28c72d0d2cabc

data/CHANGELOG.md

@@ -6,6 +6,34 @@ to follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+## [0.3.0] - 2026-07-01
+
+### Fixed
+
+- **Sync no longer serves un-integrable pending structs.** When a doc holds a
+  *pending* struct (a gappy update whose causally-prior update is missing — e.g.
+  legacy data recorded before the `update_ready?` gate existed), its integrated
+  state vector is empty but `encode_state_as_update` merges the pending bytes back
+  in. Answering a peer's `SyncStep1` with that state handed the peer content it
+  couldn't integrate, so it parked the same pending forever and the empty-SV /
+  non-empty-content mismatch drove endless resync traffic (observed as a browser
+  re-sending frames several times a second). `handle_sync_message` now answers
+  `SyncStep1` with **integrated-only** state, so a server never serves a struct it
+  can't integrate itself. Neutralizes existing poisoned server state on deploy —
+  no migration needed. The server's own pending is untouched and still heals if
+  the missing dependency later arrives (only then does the content become
+  visible in sync). Live delta relay (`Update` frames) is unchanged.
+
+### Added
+
+- `Doc#pending?` — true if the doc holds un-integrable pending structs or a
+  pending delete set (content waiting on a missing causally-prior update).
+- `Doc#compacted_state_update` — like `encode_state_as_update` (full state) but
+  **gap-free**: excludes pending structs/delete set. Use it when persisting or
+  serving state other peers will apply. Non-destructive — the doc keeps its
+  pending (so it can still heal), and `encode_state_as_update` stays lossless for
+  raw-update recovery.
+
 ## [0.2.3] - 2026-07-01
 
 ### Fixed

data/README.md

@@ -142,18 +142,39 @@ doc = Y::Doc.new(12345) # specific client ID (used for CRDT identity)
 
 # Encoding
 doc.encode_state_vector           # => current state vector
-doc.encode_state_as_update        # => full update
+doc.encode_state_as_update        # => full update (lossless: keeps pending)
 doc.encode_state_as_update(sv)    # => update diff against state vector
+doc.compacted_state_update        # => full update, gap-free (excludes pending)
 
 # Applying updates
 doc.apply_update(update_bytes)    # apply raw V1 update
+doc.pending?                      # => true if holding un-integrable pending structs
 
 # Sync protocol
 doc.sync_step1                    # => SyncStep1 message (this doc's state vector)
 doc.handle_sync_message(data)     # => [msg_type, sync_type, response]; answers a
-                                  #    peer's SyncStep1 with a SyncStep2
+                                  #    peer's SyncStep1 with an integrated-only
+                                  #    SyncStep2 (never serves pending structs)
 ```
 
+### Pending structs and gap-free state
+
+If a doc applies an update whose causally-prior update is missing (a "gappy"
+update), yrs parks it as a **pending** struct: the integrated state vector stays
+empty, but the pending block is held as a recovery buffer and heals if the
+missing dependency later arrives. `Doc#pending?` reports this.
+
+Pending structs are *not* document state, so they must not cross the sync
+boundary — a peer that receives one can't integrate it and gets stuck. Two
+guarantees keep serving safe:
+
+- `handle_sync_message` answers `SyncStep1` with **integrated-only** state, so a
+  server never serves a struct it can't integrate itself (this is automatic).
+- `Doc#compacted_state_update` gives you the same gap-free full-state update for
+  when you persist or hand off state yourself. It's non-destructive (the doc
+  keeps its pending), while `encode_state_as_update` stays lossless so you can
+  still preserve the raw pending bytes for recovery.
+
 ### Protocol codec (module functions)
 
 Classifying and unwrapping wire frames is stateless, so it's exposed as

data/ext/yrby/src/lib.rs

@@ -8,7 +8,10 @@ use yrs::{Doc, GetString, ReadTxn, Transact};
 
 mod protocol;
 mod read;
-use protocol::{classify_message, merged_doc_update, update_advances_doc, update_is_ready};
+use protocol::{
+    classify_message, has_pending, integrated_update, merged_doc_update, update_advances_doc,
+    update_is_ready,
+};
 
 /// Wrapper around yrs Doc.
 ///
@@ -188,6 +191,28 @@ impl RbDoc {
         })
     }
 
+    /// True if the doc holds un-integrable pending structs or a pending delete
+    /// set — content that couldn't integrate because a causally-prior update is
+    /// missing. Such content is a recovery buffer, not document state; it heals if
+    /// the missing dependency later arrives. A pure read.
+    fn pending(&self) -> bool {
+        let doc = &self.0;
+        nogvl(move || has_pending(doc))
+    }
+
+    /// Like `encode_state_as_update` (full state), but **gap-free**: it excludes
+    /// any pending (un-integrable) structs and pending delete set. Use this when
+    /// persisting or serving state that other peers will apply — serving pending
+    /// content poisons their sync. Non-destructive: this doc keeps its pending, so
+    /// a genuine gap still heals if its dependency arrives. (`encode_state_as_update`
+    /// stays lossless for raw-update recovery.)
+    fn compacted_state_update(&self) -> Result<RString, Error> {
+        let doc = &self.0;
+        let update = nogvl(move || integrated_update(doc, &yrs::StateVector::default()))
+            .map_err(yrb_error)?;
+        Ok(binary_string(&update))
+    }
+
     /// Encode state as update (optionally diffed against a state vector)
     fn encode_state_as_update(&self, args: &[Value]) -> Result<RString, Error> {
         let sv_bytes: Option<Vec<u8>> = if args.is_empty() {
@@ -263,9 +288,13 @@ impl RbDoc {
                 match msg {
                     Message::Sync(sync_msg) => match sync_msg {
                         SyncMessage::SyncStep1(sv) => {
-                            // Respond with SyncStep2
-                            let txn = doc.transact();
-                            let update = txn.encode_state_as_update_v1(&sv);
+                            // Respond with SyncStep2 carrying only *integrated*
+                            // state. Never hand a peer un-integrable pending
+                            // structs: the peer would park the same pending
+                            // forever and the state-vector/content mismatch drives
+                            // endless resync traffic. (integrated_update is a no-op
+                            // fast path when nothing is pending.)
+                            let update = integrated_update(doc, &sv)?;
                             let response = Message::Sync(SyncMessage::SyncStep2(update));
                             Ok((0, 0, response.encode_v1()))
                         }
@@ -366,6 +395,11 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     doc_class.define_method("read_text", method!(RbDoc::read_text, 1))?;
     doc_class.define_method("read_xml", method!(RbDoc::read_xml, 1))?;
     doc_class.define_method("read_map", method!(RbDoc::read_map, 1))?;
+    doc_class.define_method("pending?", method!(RbDoc::pending, 0))?;
+    doc_class.define_method(
+        "compacted_state_update",
+        method!(RbDoc::compacted_state_update, 0),
+    )?;
     doc_class.define_method("update_ready?", method!(RbDoc::update_ready, 1))?;
     doc_class.define_method("update_advances?", method!(RbDoc::update_advances, 1))?;
     doc_class.define_method("sync_step1", method!(RbDoc::sync_step1, 0))?;

data/ext/yrby/src/protocol.rs

@@ -8,7 +8,7 @@ use yrs::encoding::read::{Cursor, Read};
 use yrs::sync::protocol::MessageReader;
 use yrs::sync::{Message, SyncMessage};
 use yrs::updates::decoder::{Decode, DecoderV1};
-use yrs::{Doc, ReadTxn, Transact};
+use yrs::{Doc, ReadTxn, StateVector, Transact, Update, WriteTxn};
 
 /// Classify a frame: a non-zero code only for exactly one well-formed message
 /// that consumes the whole buffer (the codes are the match arms below).
@@ -143,21 +143,52 @@ pub(crate) fn update_advances_doc(doc: &Doc, update_bytes: &[u8]) -> Result<bool
     }
 }
 
-/// True if the doc holds pending structs or a pending delete set: blocks that
-/// couldn't integrate because a dependency is missing. Test-only: asserts the
-/// causal-chain parking behavior in the unit tests below.
-#[cfg(test)]
-pub(crate) fn doc_has_pending(doc: &Doc) -> bool {
+/// True if the doc holds un-integrable pending structs or a pending delete set:
+/// blocks that couldn't integrate because a causally-prior update is missing. A
+/// pure read; does not mutate.
+pub(crate) fn has_pending(doc: &Doc) -> bool {
     let txn = doc.transact();
     txn.store().pending_update().is_some() || txn.store().pending_ds().is_some()
 }
 
+/// Encode the doc's **integrated** state as a v1 update diffed against `sv`,
+/// excluding any pending (un-integrable) structs and pending delete set.
+///
+/// Pending blocks are a recovery buffer, not document state. Serving them across
+/// the sync boundary hands a peer content it can't integrate, so the peer parks
+/// the same pending forever and the state-vector/content mismatch drives endless
+/// resync traffic. `encode_state_as_update_v1` merges pending back in (see yrs
+/// `merge_pending_v1`), so to get a gap-free encode we rebuild the state into a
+/// throwaway doc and `prune_pending` there before re-encoding.
+///
+/// Non-destructive: the prune happens only on the throwaway copy; `doc` keeps its
+/// pending, so a genuine gap still heals if its missing dependency later arrives.
+pub(crate) fn integrated_update(doc: &Doc, sv: &StateVector) -> Result<Vec<u8>, String> {
+    // Fast path: with nothing pending the direct encode is already gap-free, so
+    // the clean common case keeps the zero-copy behavior.
+    if !has_pending(doc) {
+        return Ok(doc.transact().encode_state_as_update_v1(sv));
+    }
+    let full = doc
+        .transact()
+        .encode_state_as_update_v1(&StateVector::default());
+    let clean = Doc::new();
+    {
+        let mut txn = clean.transact_mut();
+        txn.apply_update(Update::decode_v1(&full).map_err(|e| e.to_string())?)
+            .map_err(|e| e.to_string())?;
+        txn.prune_pending();
+    }
+    let out = clean.transact().encode_state_as_update_v1(sv);
+    Ok(out)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
     use yrs::sync::Awareness;
     use yrs::updates::encoder::Encode;
-    use yrs::Text;
+    use yrs::{GetString, Text};
 
     fn text_update(content: &str) -> Vec<u8> {
         let doc = Doc::new();
@@ -418,24 +449,213 @@ mod tests {
             !update_is_ready(&doc, u3).unwrap(),
             "u3 depends on the missing u2"
         );
-        assert!(
-            !doc_has_pending(&doc),
-            "nothing pending until u3 is applied"
-        );
+        assert!(!has_pending(&doc), "nothing pending until u3 is applied");
 
         // Applying u3 anyway parks it as a pending struct.
         doc.transact_mut()
             .apply_update(yrs::Update::decode_v1(u3).unwrap())
             .unwrap();
-        assert!(
-            doc_has_pending(&doc),
-            "u3 is pending: its parent u2 is missing"
-        );
+        assert!(has_pending(&doc), "u3 is pending: its parent u2 is missing");
 
         // Once u2 arrives (via resync), u3 integrates and pending clears.
         doc.transact_mut()
             .apply_update(yrs::Update::decode_v1(u2).unwrap())
             .unwrap();
-        assert!(!doc_has_pending(&doc), "u2 arrived; u3 integrated");
+        assert!(!has_pending(&doc), "u2 arrived; u3 integrated");
+    }
+
+    // Build a causal gap: `first` inserts "a", `dependent` inserts "b" after it,
+    // so `dependent` alone parks as pending on a doc that lacks `first`.
+    fn gap_pair() -> (Vec<u8>, Vec<u8>) {
+        let src = Doc::new();
+        let txt = src.get_or_insert_text("notepad");
+        txt.insert(&mut src.transact_mut(), 0, "a");
+        let first = src
+            .transact()
+            .encode_state_as_update_v1(&yrs::StateVector::default());
+        let sv = src.transact().state_vector();
+        txt.insert(&mut src.transact_mut(), 1, "b");
+        let dependent = src.transact().encode_state_as_update_v1(&sv);
+        (first, dependent)
+    }
+
+    #[test]
+    fn integrated_update_strips_pending_and_is_non_destructive() {
+        let (_first, dependent) = gap_pair();
+        let doc = Doc::new();
+        doc.transact_mut()
+            .apply_update(yrs::Update::decode_v1(&dependent).unwrap())
+            .unwrap();
+        assert!(has_pending(&doc), "the gappy update parked as pending");
+
+        // encode_state_as_update carries the pending; integrated_update does not.
+        let full = doc
+            .transact()
+            .encode_state_as_update_v1(&yrs::StateVector::default());
+        let gap_free = integrated_update(&doc, &yrs::StateVector::default()).unwrap();
+        assert_ne!(full, gap_free, "integrated_update drops the pending bytes");
+
+        // Applying the gap-free encode to a fresh peer must NOT poison it.
+        let peer = Doc::new();
+        peer.transact_mut()
+            .apply_update(yrs::Update::decode_v1(&gap_free).unwrap())
+            .unwrap();
+        assert!(
+            !has_pending(&peer),
+            "peer got no pending from the gap-free state"
+        );
+
+        // Non-destructive: the source doc keeps its pending (so it can still heal).
+        assert!(
+            has_pending(&doc),
+            "integrated_update did not mutate the source"
+        );
+    }
+
+    #[test]
+    fn integrated_update_fast_path_matches_direct_encode_when_clean() {
+        // No pending -> byte-identical to encode_state_as_update (zero-copy path).
+        let (first, _dependent) = gap_pair();
+        let doc = Doc::new();
+        doc.transact_mut()
+            .apply_update(yrs::Update::decode_v1(&first).unwrap())
+            .unwrap();
+        assert!(!has_pending(&doc));
+        let direct = doc
+            .transact()
+            .encode_state_as_update_v1(&yrs::StateVector::default());
+        let via = integrated_update(&doc, &yrs::StateVector::default()).unwrap();
+        assert_eq!(direct, via);
+    }
+
+    #[test]
+    fn a_healed_gap_serves_its_content() {
+        // After the missing dependency arrives, the (formerly pending) content is
+        // integrated and integrated_update includes it.
+        let (first, dependent) = gap_pair();
+        let doc = Doc::new();
+        doc.transact_mut()
+            .apply_update(yrs::Update::decode_v1(&dependent).unwrap())
+            .unwrap();
+        doc.transact_mut()
+            .apply_update(yrs::Update::decode_v1(&first).unwrap())
+            .unwrap();
+        assert!(!has_pending(&doc), "gap healed once first arrived");
+        let gap_free = integrated_update(&doc, &yrs::StateVector::default()).unwrap();
+        let peer = Doc::new();
+        peer.transact_mut()
+            .apply_update(yrs::Update::decode_v1(&gap_free).unwrap())
+            .unwrap();
+        assert_eq!(
+            peer.get_or_insert_text("notepad")
+                .get_string(&peer.transact()),
+            "ab"
+        );
+    }
+
+    // A gappy insert from its own independent client: inserts two chars and
+    // returns only the second delta, which depends on the (missing) first.
+    fn independent_gappy_insert() -> Vec<u8> {
+        let src = Doc::new();
+        let txt = src.get_or_insert_text("notepad");
+        txt.insert(&mut src.transact_mut(), 0, "x");
+        let sv = src.transact().state_vector();
+        txt.insert(&mut src.transact_mut(), 1, "y");
+        let txn = src.transact();
+        txn.encode_state_as_update_v1(&sv)
+    }
+
+    #[test]
+    fn integrated_update_keeps_content_and_drops_pending_when_mixed() {
+        // The realistic case: a doc with real integrated content AND a pending
+        // struct. Pruning must keep the content and drop only the pending.
+        let (first, _dep) = gap_pair();
+        let doc = Doc::new();
+        doc.transact_mut()
+            .apply_update(yrs::Update::decode_v1(&first).unwrap())
+            .unwrap(); // integrated "a"
+        doc.transact_mut()
+            .apply_update(yrs::Update::decode_v1(&independent_gappy_insert()).unwrap())
+            .unwrap(); // + a pending struct from another client
+        assert!(has_pending(&doc));
+
+        let gap_free = integrated_update(&doc, &yrs::StateVector::default()).unwrap();
+        let peer = Doc::new();
+        peer.transact_mut()
+            .apply_update(yrs::Update::decode_v1(&gap_free).unwrap())
+            .unwrap();
+        assert_eq!(
+            peer.get_or_insert_text("notepad")
+                .get_string(&peer.transact()),
+            "a",
+            "kept the integrated content"
+        );
+        assert!(!has_pending(&peer), "dropped the pending");
+    }
+
+    #[test]
+    fn integrated_update_diffs_against_a_peer_sv_and_excludes_pending() {
+        // The production signature: `handle_sync_message` calls
+        // integrated_update(doc, peer_sv). A peer already holding the integrated
+        // content should get a diff carrying no new content and no pending.
+        let (first, _dep) = gap_pair();
+        let server = Doc::new();
+        server
+            .transact_mut()
+            .apply_update(yrs::Update::decode_v1(&first).unwrap())
+            .unwrap();
+        server
+            .transact_mut()
+            .apply_update(yrs::Update::decode_v1(&independent_gappy_insert()).unwrap())
+            .unwrap();
+
+        let peer = Doc::new();
+        peer.transact_mut()
+            .apply_update(yrs::Update::decode_v1(&first).unwrap())
+            .unwrap();
+        let peer_sv = peer.transact().state_vector();
+
+        let diff = integrated_update(&server, &peer_sv).unwrap();
+        peer.transact_mut()
+            .apply_update(yrs::Update::decode_v1(&diff).unwrap())
+            .unwrap();
+        assert_eq!(
+            peer.get_or_insert_text("notepad")
+                .get_string(&peer.transact()),
+            "a"
+        );
+        assert!(!has_pending(&peer), "the diff carried no pending");
+    }
+
+    #[test]
+    fn integrated_update_strips_a_pending_delete_set() {
+        // A deletion whose target struct is absent parks as a pending *delete
+        // set* -- the delete-side counterpart to a pending struct.
+        let src = Doc::new();
+        let txt = src.get_or_insert_text("notepad");
+        txt.insert(&mut src.transact_mut(), 0, "z");
+        let sv = src.transact().state_vector();
+        txt.remove_range(&mut src.transact_mut(), 0, 1);
+        let deletion = src.transact().encode_state_as_update_v1(&sv); // delete-only
+
+        let doc = Doc::new();
+        doc.transact_mut()
+            .apply_update(yrs::Update::decode_v1(&deletion).unwrap())
+            .unwrap();
+        assert!(
+            has_pending(&doc),
+            "the orphan deletion parked as a pending delete set"
+        );
+
+        let gap_free = integrated_update(&doc, &yrs::StateVector::default()).unwrap();
+        let peer = Doc::new();
+        peer.transact_mut()
+            .apply_update(yrs::Update::decode_v1(&gap_free).unwrap())
+            .unwrap();
+        assert!(!has_pending(&peer), "the pending delete set was not served");
+        assert!(
+            has_pending(&doc),
+            "non-destructive: source keeps its pending"
+        );
     }
 }

data/lib/y/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Y
-  VERSION = "0.2.3"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: yrby
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - JP Camara