yrby 0.6.0 → 0.7.0

data/ext/yrby/Cargo.toml

@@ -14,6 +14,9 @@ magnus = "0.8"
 rb-sys = "0.9"
 yrs = { version = "0.27", features = ["sync"] }
 serde_json = "1.0"
+yrs-html-core = { path = "crates/html-core" }
+yrs-lexical-html = { path = "crates/lexical-html" }
+yrs-prosemirror-html = { path = "crates/prosemirror-html" }
 
 [dev-dependencies]
 rb-sys-env = "0.1"

data/ext/yrby/crates/html-core/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "yrs-html-core"
+description = "Internal core for the yrs-*-html renderer crates; depend on those instead"
+repository = "https://github.com/jpcamara/yrby"
+readme = "README.md"
+version = "0.1.0"
+edition = "2024"
+rust-version = "1.85"
+authors = ["JP Camara <johnpcamara@gmail.com>"]
+license = "MIT"
+publish = false
+
+[dependencies]
+yrs = { version = "0.27", features = ["sync"] }
+serde_json = "1.0"

data/ext/yrby/{src/render_rules.rs → crates/html-core/src/lib.rs}

@@ -45,6 +45,12 @@ pub struct Emitter {
     frames: Vec<Vec<Segment>>,
 }
 
+impl Default for Emitter {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl Emitter {
     pub fn new() -> Self {
         Emitter {

data/ext/yrby/crates/lexical-html/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "yrs-lexical-html"
+description = "Render Lexical-shaped yrs documents to HTML, no browser or Node required"
+repository = "https://github.com/jpcamara/yrby"
+readme = "README.md"
+version = "0.1.0"
+edition = "2024"
+rust-version = "1.85"
+authors = ["JP Camara <johnpcamara@gmail.com>"]
+license = "MIT"
+publish = false
+
+[dependencies]
+yrs = { version = "0.27", features = ["sync"] }
+serde_json = "1.0"
+yrs-html-core = { path = "../html-core" }

data/ext/yrby/{src/lexical_html.rs → crates/lexical-html/src/lib.rs}

@@ -39,15 +39,19 @@
 //! with the unwrapped span. Case-transform format bits are never rendered
 //! (their text-transform style is outside the sanitize whitelist).
 //!
-//! Custom nodes: rules are registered by `__type` (see `render_rules`) and
+//! Custom nodes: rules are registered by `__type` (see `yrs-html-core`) and
 //! consulted before the built-in arms, so they extend the schema or override
 //! a built-in. Declarative rules render here; callback rules emit
 //! `Segment::Deferred` for the caller to fill in after the render.
 
-use crate::render_rules::{
-    resolve_parts, xml_attrs_json, xml_ref_attr, Content, Emitter, NodeRule, Rules, Segment,
-    TypeMap,
-};
+// README examples are living code: compile-checked on every cargo test.
+#[cfg(doctest)]
+#[doc = include_str!("../README.md")]
+mod readme_examples {}
+
+// The full rules surface, re-exported: depend on this crate alone;
+// yrs-html-core is an internal implementation crate.
+pub use yrs_html_core::*;
 use yrs::types::text::YChange;
 use yrs::{
     Any, GetString, Map, Out, ReadTxn, Text, Xml, XmlElementRef, XmlFragment, XmlFragmentRef,
@@ -121,12 +125,12 @@ pub fn render_segments<T: ReadTxn>(
     Some(em.into_segments())
 }
 
-/// Rule-free rendering to a plain string — the fixture-parity surface most
-/// tests pin. With no callback rules, segments always flatten.
-#[cfg(test)]
+/// Rule-free rendering to a plain string — the simplest way to use this
+/// crate standalone, and the fixture-parity surface the tests pin. With no
+/// callback rules, segments always flatten.
 pub fn render<T: ReadTxn>(txn: &T, fragment: &XmlFragmentRef) -> Option<String> {
     render_segments(txn, fragment, &Rules::empty()).map(|segs| {
-        crate::render_rules::flatten(segs)
+        yrs_html_core::flatten(segs)
             .into_html()
             .expect("no callback rules registered")
     })
@@ -1297,7 +1301,7 @@ mod tests {
         let txn = doc.transact();
         let frag = txn.get_xml_fragment("root").unwrap();
         let segs = render_segments(&txn, &frag, &rules).unwrap();
-        let html = crate::render_rules::flatten(segs).into_html().unwrap();
+        let html = yrs_html_core::flatten(segs).into_html().unwrap();
         assert_eq!(
             html,
             "<p><a class=\"app-link\" href=\"https://x.example\">site</a>\

data/ext/yrby/crates/prosemirror-html/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "yrs-prosemirror-html"
+description = "Render ProseMirror-shaped yrs documents to HTML, no browser or Node required"
+repository = "https://github.com/jpcamara/yrby"
+readme = "README.md"
+version = "0.1.0"
+edition = "2024"
+rust-version = "1.85"
+authors = ["JP Camara <johnpcamara@gmail.com>"]
+license = "MIT"
+publish = false
+
+[dependencies]
+yrs = { version = "0.27", features = ["sync"] }
+serde_json = "1.0"
+yrs-html-core = { path = "../html-core" }

data/ext/yrby/{src/prosemirror_html.rs → crates/prosemirror-html/src/lib.rs}

@@ -36,10 +36,14 @@
 //! Declarative rules render here; callback rules emit `Segment::Deferred` for
 //! the caller to fill in after the render.
 
-use crate::render_rules::{
-    any_attr_string, resolve_parts, xml_attrs_json, xml_ref_attr, Content, Emitter, MarkRule,
-    NodeRule, Rules, Segment, TypeMap,
-};
+// README examples are living code: compile-checked on every cargo test.
+#[cfg(doctest)]
+#[doc = include_str!("../README.md")]
+mod readme_examples {}
+
+// The full rules surface, re-exported: depend on this crate alone;
+// yrs-html-core is an internal implementation crate.
+pub use yrs_html_core::*;
 use yrs::types::text::YChange;
 use yrs::types::Attrs;
 use yrs::{
@@ -96,12 +100,12 @@ pub fn render_segments<T: ReadTxn>(
     Some(em.into_segments())
 }
 
-/// Rule-free rendering to a plain string — the fixture-parity surface most
-/// tests pin. With no callback rules, segments always flatten.
-#[cfg(test)]
+/// Rule-free rendering to a plain string — the simplest way to use this
+/// crate standalone, and the fixture-parity surface the tests pin. With no
+/// callback rules, segments always flatten.
 pub fn render<T: ReadTxn>(txn: &T, fragment: &XmlFragmentRef) -> Option<String> {
     render_segments(txn, fragment, &Rules::empty()).map(|segs| {
-        crate::render_rules::flatten(segs)
+        yrs_html_core::flatten(segs)
             .into_html()
             .expect("no callback rules registered")
     })
@@ -1232,7 +1236,7 @@ mod tests {
         let txn = doc.transact();
         let segs = render_segments(&txn, &frag, &rules).unwrap();
         assert_eq!(
-            crate::render_rules::flatten(segs).into_html().unwrap(),
+            yrs_html_core::flatten(segs).into_html().unwrap(),
             "<aside class=\"callout callout--warning\"><div class=\"para\">careful</div></aside>"
         );
     }
@@ -1290,7 +1294,7 @@ mod tests {
         }
         let txn = doc.transact();
         let map = collect_node_types(&txn, &frag).unwrap();
-        let json = crate::render_rules::type_map_json(&map, |ty| {
+        let json = yrs_html_core::type_map_json(&map, |ty| {
             if is_builtin(ty) {
                 Some("builtin")
             } else {

data/ext/yrby/src/lib.rs

@@ -7,16 +7,16 @@ use yrs::updates::decoder::Decode;
 use yrs::updates::encoder::Encode;
 use yrs::{Doc, GetString, ReadTxn, Transact};
 
-mod lexical_html;
-mod prosemirror_html;
 mod protocol;
 mod read;
-mod render_rules;
 use protocol::{
     classify_message, has_pending, integrated_update, merged_doc_update, update_advances_doc,
     update_is_ready,
 };
 use render_rules::{Rules, Segment};
+pub(crate) use yrs_html_core as render_rules;
+use yrs_lexical_html as lexical_html;
+use yrs_prosemirror_html as prosemirror_html;
 
 /// Wrapper around yrs Doc.
 ///
@@ -297,13 +297,13 @@ impl RbDoc {
                 match msg {
                     Message::Sync(sync_msg) => match sync_msg {
                         SyncMessage::SyncStep1(sv) => {
-                            // Respond with SyncStep2 carrying only *integrated*
-                            // state. Never hand a peer un-integrable pending
-                            // structs: the peer would park the same pending
-                            // forever and the state-vector/content mismatch drives
-                            // endless resync traffic. (integrated_update is a no-op
-                            // fast path when nothing is pending.)
-                            let update = integrated_update(doc, &sv)?;
+                            // Respond with SyncStep2 carrying the doc's full
+                            // state, pending included, matching Y.js's
+                            // encodeStateAsUpdate. A peer parks a pending
+                            // struct exactly as this doc does and heals it
+                            // when the missing dependency arrives.
+                            let txn = doc.transact();
+                            let update = txn.encode_state_as_update_v1(&sv);
                             let response = Message::Sync(SyncMessage::SyncStep2(update));
                             Ok((0, 0, response.encode_v1()))
                         }

data/ext/yrby/src/protocol.rs

@@ -135,6 +135,27 @@ pub(crate) fn update_advances_doc(doc: &Doc, update_bytes: &[u8]) -> Result<bool
     let update = yrs::Update::decode_v1(update_bytes).map_err(|e| e.to_string())?;
     let has_deletes = !update.delete_set().is_empty();
 
+    // A merged update can hide an internal gap behind a Skip block, and
+    // blocks past the gap can still integrate (yrs plants a Skip hole in the
+    // store) when their origins don't need the missing range. That moves
+    // neither the doc's public state vector (yrs caps it at the first hole)
+    // nor pending: invisible to every comparison below, so genuinely novel
+    // content would report "doesn't advance": applied, acked, but never
+    // recorded or broadcast. The update's own state_vector() caps at its
+    // first Skip the same way, so any insertion past a non-zero cap reveals
+    // such a gap: conservatively report it as advancing (record it; a
+    // duplicate in the log is harmless, dropped content is not). Standard
+    // Yjs providers never emit these frames (a client's own updates and
+    // diffs are gap-free), so this guards crafted or pathologically merged
+    // input, not a hot path.
+    let capped = update.state_vector();
+    for (client, ranges) in update.insertions(true).iter() {
+        let cap = capped.get(client);
+        if cap > 0 && ranges.iter().any(|range| range.end > cap) {
+            return Ok(true);
+        }
+    }
+
     // Fast path: blocks beyond the doc's state vector are content the doc
     // lacks — the update advances, no probe needed. The common case (a novel
     // edit) exits here; only retries and ambiguous diffs pay for the probe.
@@ -663,6 +684,48 @@ mod tests {
         );
     }
 
+    #[test]
+    fn a_gappy_update_with_novel_post_hole_content_advances() {
+        // A merged update hiding an internal gap behind a Skip, whose
+        // post-hole blocks integrate anyway (their origins don't need the
+        // missing range). Integration plants a Skip hole, so neither the
+        // public state vector nor pending moves, before the hidden-gap
+        // check, the probe comparison reported genuinely novel content as
+        // "doesn't advance": update_ready? accepted it, the doc applied it,
+        // but it was never recorded or broadcast. Unreachable through
+        // standard Yjs providers (a client's own updates and diffs are
+        // gap-free); this is a crafted/hostile frame shape.
+        let a = Doc::new();
+        let t1 = a.get_or_insert_text("one");
+        t1.insert(&mut a.transact_mut(), 0, "first");
+        let u1 = a
+            .transact()
+            .encode_state_as_update_v1(&yrs::StateVector::default());
+        t1.insert(&mut a.transact_mut(), 5, " second"); // the future hole
+        let sv2 = a.transact().state_vector();
+        // Novel content on a DIFFERENT root: no dependency on the hole.
+        let t3 = a.get_or_insert_text("two");
+        t3.insert(&mut a.transact_mut(), 0, "third");
+        let u3 = a.transact().encode_state_as_update_v1(&sv2);
+
+        let server = Doc::new();
+        server
+            .transact_mut()
+            .apply_update(yrs::Update::decode_v1(&u1).unwrap())
+            .unwrap();
+        // The crafted frame: u1 + u3 merged, u2's range an internal Skip.
+        let gappy = yrs::merge_updates_v1([&u1, &u3]).unwrap();
+
+        assert!(
+            update_is_ready(&server, &gappy).unwrap(),
+            "sanity: the sync flow would accept and apply this frame"
+        );
+        assert!(
+            update_advances_doc(&server, &gappy).unwrap(),
+            "novel content hiding behind a Skip must be recorded, not dropped"
+        );
+    }
+
     // Build a causal gap: `first` inserts "a", `dependent` inserts "b" after it,
     // so `dependent` alone parks as pending on a doc that lacks `first`.
     fn gap_pair() -> (Vec<u8>, Vec<u8>) {

data/ext/yrby/src/read.rs

@@ -352,7 +352,7 @@ mod tests {
         // lexxy-realtime's test server and saving GET /content/:room.
         use yrs::updates::decoder::Decode;
         use yrs::Update;
-        let bytes = include_bytes!("fixtures/lexical_linebreak.bin");
+        let bytes = include_bytes!("../crates/lexical-html/src/fixtures/lexical_linebreak.bin");
         let doc = Doc::new();
         doc.transact_mut()
             .apply_update(Update::decode_v1(bytes).unwrap())
@@ -369,7 +369,7 @@ mod tests {
         // upload's caption as its own line — while the divider stays silent.
         use yrs::updates::decoder::Decode;
         use yrs::{Transact, Update};
-        let bytes = include_bytes!("fixtures/lexxy_full.bin");
+        let bytes = include_bytes!("../crates/lexical-html/src/fixtures/lexxy_full.bin");
         let doc = Doc::new();
         doc.transact_mut()
             .apply_update(Update::decode_v1(bytes).unwrap())
@@ -408,7 +408,7 @@ mod tests {
         // must come through (get_string alone dropped all the nested ones).
         use yrs::updates::decoder::Decode;
         use yrs::{Transact, Update};
-        let bytes = include_bytes!("fixtures/lexical_rich.bin");
+        let bytes = include_bytes!("../crates/lexical-html/src/fixtures/lexical_rich.bin");
         let doc = Doc::new();
         {
             let mut txn = doc.transact_mut();