yrby 0.2.3 → 0.3.1

data/ext/yrby/src/read.rs

@@ -62,6 +62,17 @@ fn lexical_type<T: ReadTxn>(txn: &T, t: &XmlTextRef) -> String {
     }
 }
 
+/// The `__type` of an embedded Lexical `Y.Map`. Two kinds appear inside a
+/// block: text-node metadata (`"text"`) and node maps like the LineBreakNode
+/// (`"linebreak"`). Structure confirmed from live-editor bytes (see the
+/// captured-fixture test).
+fn lexical_map_type<T: ReadTxn>(txn: &T, m: &MapRef) -> String {
+    match m.get(txn, "__type") {
+        Some(Out::Any(Any::String(s))) => s.to_string(),
+        _ => String::new(),
+    }
+}
+
 /// Gather the text of an inline Lexical element (its text runs and any nested
 /// inline elements) without introducing block breaks.
 fn inline_lexical_text<T: ReadTxn>(txn: &T, t: &XmlTextRef, buf: &mut String) {
@@ -69,7 +80,12 @@ fn inline_lexical_text<T: ReadTxn>(txn: &T, t: &XmlTextRef, buf: &mut String) {
         match d.insert {
             Out::Any(Any::String(s)) => buf.push_str(&s),
             Out::YXmlText(child) => inline_lexical_text(txn, &child, buf),
-            _ => {} // per-text-node metadata map, decorator embeds: no text
+            Out::YMap(m) => match lexical_map_type(txn, &m).as_str() {
+                "linebreak" => buf.push('\n'),
+                "tab" => buf.push('\t'),
+                _ => {} // per-text-node metadata: no text of its own
+            },
+            _ => {} // decorator embeds: no text
         }
     }
 }
@@ -81,17 +97,30 @@ fn walk_lexical_block<T: ReadTxn>(txn: &T, t: &XmlTextRef, out: &mut Vec<String>
     for d in t.diff(txn, YChange::identity) {
         match d.insert {
             Out::Any(Any::String(s)) => line.push_str(&s),
+            // Node maps: linebreak/tab carry no text, so emit the character
+            // they represent ("foo⏎bar" must not become "foobar"). Metadata
+            // maps ("text") stay silent.
+            Out::YMap(m) => match lexical_map_type(txn, &m).as_str() {
+                "linebreak" => line.push('\n'),
+                "tab" => line.push('\t'),
+                _ => {}
+            },
             Out::YXmlText(child) => {
-                if is_inline_lexical_type(&lexical_type(txn, &child)) {
-                    inline_lexical_text(txn, &child, &mut line);
-                } else {
-                    if !line.is_empty() {
-                        out.push(std::mem::take(&mut line));
+                let ty = lexical_type(txn, &child);
+                match ty.as_str() {
+                    // Defensive only: real Lexical stores these as Y.Map embeds.
+                    "linebreak" => line.push('\n'),
+                    "tab" => line.push('\t'),
+                    _ if is_inline_lexical_type(&ty) => inline_lexical_text(txn, &child, &mut line),
+                    _ => {
+                        if !line.is_empty() {
+                            out.push(std::mem::take(&mut line));
+                        }
+                        walk_lexical_block(txn, &child, out);
                     }
-                    walk_lexical_block(txn, &child, out);
                 }
             }
-            _ => {} // per-text-node metadata map; embeds we don't read for text
+            _ => {} // decorator embeds we don't read for text
         }
     }
     if !line.is_empty() {
@@ -256,6 +285,53 @@ mod tests {
         assert_eq!(map_json(&txn, &map), "{}");
     }
 
+    #[test]
+    fn lexical_soft_line_break_and_tab_emit_their_characters() {
+        // A paragraph "foo⏎bar" (shift-enter): Lexical stores the LineBreakNode
+        // as an embedded Y.Map with __type=linebreak (the same shape as the
+        // per-text-node metadata maps, which must stay silent). It must come
+        // through as '\n', not vanish and glue the words. Same for tab.
+        use yrs::{Text, XmlTextPrelim};
+        let doc = Doc::new();
+        let frag = doc.get_or_insert_xml_fragment("lex");
+        {
+            let mut txn = doc.transact_mut();
+            let block = frag.push_back(&mut txn, XmlTextPrelim::new(""));
+            let meta: MapPrelim = [("__type", yrs::In::from("text"))].into_iter().collect();
+            block.insert_embed(&mut txn, 0, meta); // metadata map: no text
+            block.push(&mut txn, "foo");
+            let br: MapPrelim = [("__type", yrs::In::from("linebreak"))]
+                .into_iter()
+                .collect();
+            block.insert_embed(&mut txn, 4, br);
+            block.push(&mut txn, "bar");
+            let tab: MapPrelim = [("__type", yrs::In::from("tab"))].into_iter().collect();
+            block.insert_embed(&mut txn, 8, tab);
+            block.push(&mut txn, "baz");
+        }
+        let txn = doc.transact();
+        assert_eq!(xml_blocks_text(&txn, &frag), "foo\nbar\tbaz");
+    }
+
+    #[test]
+    fn lexical_real_captured_linebreak_extracts_as_newline() {
+        // Ground truth: bytes captured from a LIVE Lexxy editor (agent-browser
+        // typing "foo", pressing Shift+Enter, typing "barbaz"), served by the
+        // yrby test server's durable store. The hand-built test above models
+        // this structure; this one IS the structure. Regenerate by driving
+        // lexxy-realtime's test server and saving GET /content/:room.
+        use yrs::updates::decoder::Decode;
+        use yrs::Update;
+        let bytes = include_bytes!("fixtures/lexical_linebreak.bin");
+        let doc = Doc::new();
+        doc.transact_mut()
+            .apply_update(Update::decode_v1(bytes).unwrap())
+            .unwrap();
+        let txn = doc.transact();
+        let frag = txn.get_xml_fragment("root").unwrap();
+        assert_eq!(xml_blocks_text(&txn, &frag), "foo\nbarbaz");
+    }
+
     #[test]
     fn lexical_complex_doc_extracts_all_nested_text() {
         // A real Lexxy/Lexical doc with every block type: headings, formatted

data/lib/y/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Y
-  VERSION = "0.2.3"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: yrby
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.1
 platform: ruby
 authors:
 - JP Camara
@@ -77,6 +77,7 @@ extensions:
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
+- Cargo.lock
 - Cargo.toml
 - LICENSE
 - README.md
@@ -86,10 +87,7 @@ files:
 - ext/yrby/src/protocol.rs
 - ext/yrby/src/read.rs
 - lib/y.rb
-- lib/y/decoder.rb
-- lib/y/decoder/version.rb
 - lib/y/version.rb
-- lib/yrby-decoder.rb
 - lib/yrby.rb
 homepage: https://github.com/jpcamara/yrby
 licenses:

data/lib/y/decoder/version.rb

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-module Y
-  module Decoder
-    VERSION = "0.1.0.alpha1"
-  end
-end

data/lib/y/decoder.rb

@@ -1,66 +0,0 @@
-# frozen_string_literal: true
-
-require "y"
-require "y/decoder/version"
-
-module Y
-  # Plain-text reconstruction of a stored Yjs document, in pure Ruby — for search
-  # indexing and previews. The core `yrby` gem moves and stores opaque CRDT
-  # updates without reading them; this reads the text out of the shared type the
-  # editor uses (Lexical's `Y.XmlText`, plain `Y.Text`, or ProseMirror's
-  # `Y.XmlFragment`), in-process, on the native extension core already ships — no
-  # Node, no subprocess, no binary.
-  #
-  #   state = doc.encode_state_as_update        # opaque CRDT bytes from the store
-  #   Y::Decoder.text(state)              # => "hello world"
-  #   Y::Decoder.preview(state, 280)      # => "hello world…"
-  #
-  # Full-fidelity reconstruction (the exact Lexical EditorState / HTML, which
-  # needs @lexical/yjs) is a separate, opt-in concern — see the `yrby-decode`
-  # package's Bun binary. This gem stays pure Ruby on purpose.
-  module Decoder
-    class Error < Y::Error; end
-
-    module_function
-
-    # Plain text of the document. `field` pins the root key (Lexical: the editor
-    # id; ProseMirror: "default"); omit it to use the document's sole root.
-    def text(state, field: nil)
-      field ||= Y::Doc.new.tap { |d| d.apply_update(state) }.root_names.first
-      return "" unless field
-
-      # A plain `Y.Text` root (a simple shared-text editor) reads straight out.
-      # (A yrs root's type is fixed by its first typed access, so each reader
-      # gets a fresh doc to try a different shared type against the same state.)
-      direct = load(state).read_text(field)
-      return normalize(direct) if direct && !direct.strip.empty?
-
-      # Lexical (each block a sibling `Y.XmlText`) and ProseMirror (blocks are
-      # `Y.XmlElement`s) both come back from read_xml as block-per-line markup;
-      # strip any element tags to plain text.
-      markup = load(state).read_xml(field)
-      markup ? normalize(strip_tags(markup)) : ""
-    end
-
-    # A compact, single-line preview for list UIs.
-    def preview(state, limit: 280, field: nil)
-      body = text(state, field: field).gsub(/\s+/, " ").strip
-      body.length > limit ? "#{body[0, limit].rstrip}…" : body
-    end
-
-    def load(state)
-      Y::Doc.new.tap { |doc| doc.apply_update(state) }
-    end
-
-    def strip_tags(markup)
-      markup.gsub(/<[^>]*>/, " ")
-    end
-
-    def normalize(text)
-      text.gsub(/[ \t]+/, " ")     # collapse runs of spaces/tabs
-          .gsub(/ *\n */, "\n")    # trim spaces left around block separators
-          .gsub(/\n{3,}/, "\n\n")  # cap blank-line runs
-          .strip
-    end
-  end
-end

data/lib/yrby-decoder.rb

@@ -1,4 +0,0 @@
-# frozen_string_literal: true
-
-# Entry point matching the gem name, so `Bundler.require` loads it automatically.
-require "y/decoder"