yrby 0.7.1 → 0.8.1

data/ext/yrby/src/read.rs

@@ -1,4 +1,4 @@
-//! Pure content-reading helpers over yrs shared types — no magnus/Ruby, so they
+//! Pure content-reading helpers over yrs shared types, with no magnus/Ruby, so they
 //! can be unit-tested directly in Rust (like `protocol.rs`). The binding layer in
 //! `lib.rs` is a thin wrapper that opens a transaction and calls these.
 
@@ -6,8 +6,8 @@ use std::collections::HashMap;
 use std::sync::Arc;
 use yrs::types::text::YChange;
 use yrs::{
-    Any, Array, GetString, Map, MapRef, Out, ReadTxn, Text, Xml, XmlFragment, XmlFragmentRef,
-    XmlOut, XmlTextRef,
+    Any, Array, ArrayRef, GetString, Map, MapRef, Out, ReadTxn, Text, Xml, XmlFragment,
+    XmlFragmentRef, XmlOut, XmlTextRef,
 };
 
 /// Read an XML-shaped root as text, one top-level block per line.
@@ -18,7 +18,7 @@ use yrs::{
 ///   (`<paragraph>…`). `get_string` already recurses these (tags included; the
 ///   caller strips them), so we keep that path.
 /// - **Lexical** (Lexxy) stores every node as a `Y.XmlText`, and nests child
-///   blocks (list items, table cells, nested lists) as *embedded* `Y.XmlText`s —
+///   blocks (list items, table cells, nested lists) as *embedded* `Y.XmlText`s,
 ///   which `get_string` silently omits, dropping all that content. So for a
 ///   Lexical block we walk its content (`Text::diff`) instead: text runs build a
 ///   line, inline children (links) join it, and nested block children flush the
@@ -34,7 +34,7 @@ pub fn xml_blocks_text<T: ReadTxn>(txn: &T, fragment: &XmlFragmentRef) -> String
                 // them (tags kept, caller strips). A Lexical decorator is an
                 // XmlElement *with* a `__type`: attachments carry readable text
                 // (a mention's plain text, an upload's caption); the rest
-                // (horizontal rule) have none — skip rather than emit their
+                // (horizontal rule) have none: skip rather than emit their
                 // `<UNDEFINED …>` serialization.
                 if e.get_attribute(txn, "__type").is_none() {
                     out.push(e.get_string(txn));
@@ -158,35 +158,70 @@ fn walk_lexical_block<T: ReadTxn>(txn: &T, t: &XmlTextRef, out: &mut Vec<String>
     }
 }
 
-/// Read a `Y.Map` root as a JSON object string (keys sorted for stable output).
+/// Read a `Y.Map` root as a JSON object string (object keys sorted at every
+/// depth for stable, diffable output).
 ///
-/// The complement to `read_text`/`read_xml` for structured state — e.g. a shared
+/// The complement to `read_text`/`read_xml` for structured state, e.g. a shared
 /// "view state" map. Values are converted recursively: primitives pass through;
 /// nested `Y.Map`/`Y.Array` recurse; `Y.Text`/XML values stringify. The caller
 /// parses the JSON (yrs's own `Out::to_json` is crate-private, so we walk the
 /// `Out` variants ourselves here).
 pub fn map_json<T: ReadTxn>(txn: &T, map: &MapRef) -> String {
-    let mut pairs: Vec<(String, Any)> = map
-        .iter(txn)
-        .map(|(k, v)| (k.to_string(), out_to_any(txn, &v)))
-        .collect();
-    pairs.sort_by(|a, b| a.0.cmp(&b.0)); // deterministic key order
-    let mut out = String::from("{");
-    for (i, (k, v)) in pairs.iter().enumerate() {
-        if i > 0 {
-            out.push(',');
-        }
-        // Any::to_json serializes from the start of the buffer (it doesn't
-        // append), so each piece goes into its own String, then concatenated.
-        out.push_str(&any_to_json(&Any::String(Arc::from(k.as_str())))); // JSON-escaped key
-        out.push(':');
-        out.push_str(&any_to_json(v));
+    let mut hm: HashMap<String, Any> = HashMap::new();
+    for (k, v) in map.iter(txn) {
+        hm.insert(k.to_string(), out_to_any(txn, &v));
     }
-    out.push('}');
-    out
+    any_to_json(&Any::Map(Arc::new(hm)))
+}
+
+/// A Y.Array's elements as a JSON array string, recursing through nested
+/// shared types exactly as `map_json` does (a Y.Map root and a Y.Array root are
+/// the two document shapes yrs can read from the top). Element order is the
+/// array's own order; nested object keys are sorted like map_json.
+pub fn array_json<T: ReadTxn>(txn: &T, array: &ArrayRef) -> String {
+    let items: Vec<Any> = array.iter(txn).map(|o| out_to_any(txn, &o)).collect();
+    any_to_json(&Any::Array(items.into()))
 }
 
+/// JSON-serialize an `Any` with object keys sorted at every depth. yrs's own
+/// `Any::to_json` iterates `Any::Map` (a `HashMap`) in arbitrary order, so a
+/// nested map would serialize unpredictably and differ run to run; sorting here
+/// makes the whole tree deterministic. Scalars defer to `Any::to_json`, which
+/// writes from the start of a buffer (it does not append), hence each scalar
+/// gets its own fresh `String`.
 fn any_to_json(a: &Any) -> String {
+    match a {
+        Any::Array(items) => {
+            let mut out = String::from("[");
+            for (i, item) in items.iter().enumerate() {
+                if i > 0 {
+                    out.push(',');
+                }
+                out.push_str(&any_to_json(item));
+            }
+            out.push(']');
+            out
+        }
+        Any::Map(entries) => {
+            let mut pairs: Vec<(&String, &Any)> = entries.iter().collect();
+            pairs.sort_by(|a, b| a.0.cmp(b.0)); // deterministic key order
+            let mut out = String::from("{");
+            for (i, (key, value)) in pairs.into_iter().enumerate() {
+                if i > 0 {
+                    out.push(',');
+                }
+                out.push_str(&scalar_json(&Any::String(Arc::from(key.as_str())))); // JSON-escaped key
+                out.push(':');
+                out.push_str(&any_to_json(value));
+            }
+            out.push('}');
+            out
+        }
+        scalar => scalar_json(scalar),
+    }
+}
+
+fn scalar_json(a: &Any) -> String {
     let mut s = String::new();
     a.to_json(&mut s);
     s
@@ -307,6 +342,27 @@ mod tests {
         assert_eq!(map_json(&txn, &map), r#"{"user":{"name":"Ada"}}"#);
     }
 
+    #[test]
+    fn map_json_sorts_nested_object_keys() {
+        // Nested maps sort their keys too, not just the root: yrs stores map
+        // entries in a HashMap, so without this the inner object serializes in
+        // arbitrary, run-varying order.
+        let doc = Doc::new();
+        let map = doc.get_or_insert_map("state");
+        {
+            let mut txn = doc.transact_mut();
+            let inner = map.insert(&mut txn, "user", MapPrelim::default());
+            inner.insert(&mut txn, "name", "Ada");
+            inner.insert(&mut txn, "active", true);
+            inner.insert(&mut txn, "id", 7_i64);
+        }
+        let txn = doc.transact();
+        assert_eq!(
+            map_json(&txn, &map),
+            r#"{"user":{"active":true,"id":7,"name":"Ada"}}"#
+        );
+    }
+
     #[test]
     fn map_json_empty_is_object() {
         let doc = Doc::new();
@@ -315,6 +371,61 @@ mod tests {
         assert_eq!(map_json(&txn, &map), "{}");
     }
 
+    #[test]
+    fn array_json_serializes_primitives_in_order() {
+        let doc = Doc::new();
+        let array = doc.get_or_insert_array("rows");
+        {
+            let mut txn = doc.transact_mut();
+            array.push_back(&mut txn, "a");
+            array.push_back(&mut txn, 2_i64);
+            array.push_back(&mut txn, true);
+        }
+        let txn = doc.transact();
+        assert_eq!(array_json(&txn, &array), r#"["a",2,true]"#);
+    }
+
+    #[test]
+    fn array_json_recurses_into_nested_maps() {
+        // The kanban/spreadsheet shape: an array of Y.Maps. Each element must
+        // come through as its own object, keys sorted like map_json.
+        let doc = Doc::new();
+        let array = doc.get_or_insert_array("cards");
+        {
+            let mut txn = doc.transact_mut();
+            let card = array.push_back(&mut txn, MapPrelim::default());
+            card.insert(&mut txn, "text", "Ship it");
+            card.insert(&mut txn, "column", "done");
+        }
+        let txn = doc.transact();
+        assert_eq!(
+            array_json(&txn, &array),
+            r#"[{"column":"done","text":"Ship it"}]"#
+        );
+    }
+
+    #[test]
+    fn array_json_recurses_into_nested_arrays() {
+        let doc = Doc::new();
+        let array = doc.get_or_insert_array("grid");
+        {
+            let mut txn = doc.transact_mut();
+            let row = array.push_back(&mut txn, yrs::ArrayPrelim::default());
+            row.push_back(&mut txn, 1_i64);
+            row.push_back(&mut txn, 2_i64);
+        }
+        let txn = doc.transact();
+        assert_eq!(array_json(&txn, &array), r#"[[1,2]]"#);
+    }
+
+    #[test]
+    fn array_json_empty_is_array() {
+        let doc = Doc::new();
+        let array = doc.get_or_insert_array("rows");
+        let txn = doc.transact();
+        assert_eq!(array_json(&txn, &array), "[]");
+    }
+
     #[test]
     fn lexical_soft_line_break_and_tab_emit_their_characters() {
         // A paragraph "foo⏎bar" (shift-enter): Lexical stores the LineBreakNode
@@ -365,8 +476,8 @@ mod tests {
     #[test]
     fn lexxy_full_schema_doc_extracts_attachment_text_too() {
         // The full-schema capture (see lexical_html.rs): attachments must now
-        // contribute readable text — a mention's plain text inline, an
-        // upload's caption as its own line — while the divider stays silent.
+        // contribute readable text (a mention's plain text inline, an
+        // upload's caption as its own line) while the divider stays silent.
         use yrs::updates::decoder::Decode;
         use yrs::{Transact, Update};
         let bytes = include_bytes!("../crates/lexical-html/src/fixtures/lexxy_full.bin");

data/lib/generators/yrby/install/install_generator.rb

@@ -5,15 +5,16 @@ require "generators/yrby/tables/tables_generator"
 
 module Yrby
   module Generators
-    # `bin/rails generate yrby:install` — a DocumentChannel speaking the
-    # y-websocket protocol over the gem's document storage, plus the storage
-    # migration (via yrby:tables). The models ship in the gem; only the
-    # migration lands in the app.
+    # `bin/rails generate yrby:install` runs yrby:tables to create the storage
+    # migration. The models and Y::DocumentChannel ship in the gem, so that is
+    # the only file it generates by default. Pass --channel to also generate
+    # an application channel for custom authorization or room-keyed documents.
     class InstallGenerator < ::Rails::Generators::Base
       source_root File.expand_path("templates", __dir__)
+      class_option :channel, type: :boolean, default: false, desc: "Generate a custom DocumentChannel"
 
       def create_channel
-        template "document_channel.rb", "app/channels/document_channel.rb"
+        template "document_channel.rb", "app/channels/document_channel.rb" if options[:channel]
       end
 
       def create_tables
@@ -25,15 +26,25 @@ module Yrby
 
           Next steps:
 
-            1. Authorize document access: implement `authorized?` in
-               app/channels/document_channel.rb (it denies everyone until you do).
-            2. bin/rails db:migrate
-            3. Install the yrby-client npm package and connect an editor:
+            1. bin/rails db:migrate
+            2. Render a collaborative document on a page where the user may
+               edit the record:
 
-                 import { ActionCableProvider } from "yrby-client"
-                 const provider = new ActionCableProvider(doc, consumer,
-                   "DocumentChannel", { id: documentId })
-                 provider.connect()
+                 <%= collaborative_document_tag @post, :body %>
+
+            3. Install the yrby-client npm package. The tag renders an
+               element that connects automatically. Once it syncs, your code
+               gets the document and can bind it to any Yjs editor:
+
+                 import "yrby-client/element"
+
+                 document.addEventListener("yrby:synced", ({ target, detail }) => {
+                   const editor = bindYourEditor(target, detail.doc, detail.provider)
+                   detail.signal.addEventListener("abort", () => editor.destroy(), { once: true })
+                 })
+
+          Run with --channel to also generate app/channels/document_channel.rb,
+          and implement its authorized? method before you use it.
 
           The README's Editors section links working integrations for
           Tiptap, Lexxy, Rhino Editor, and CodeMirror.

data/lib/generators/yrby/install/templates/document_channel.rb

@@ -16,11 +16,7 @@ class DocumentChannel < ApplicationCable::Channel
   # and yrby-client retries it.
   on_change { |key, update| Y::Document.append(key, update) }
 
-  def subscribed
-    return reject unless authorized?(params[:id])
-
-    sync_subscribed(params[:id])
-  end
+  def subscribed = sync_subscribed(params[:id])
 
   def receive(data) = sync_receive(data, params[:id])
 
@@ -28,8 +24,9 @@ class DocumentChannel < ApplicationCable::Channel
 
   # Everyone is denied until you fill this in. Wire it to your app's auth:
   # identify current_user on the cable connection, then check they may read
-  # and write this document. Don't lean on on_change raising for access
-  # control; that path exists for store failures.
+  # and write this document. This runs once when the client subscribes, and
+  # the subscription stays authorized until it ends. Raising in on_change is
+  # for store failures, so don't use it for access control.
   def authorized?(_document_key)
     false
   end

data/lib/generators/yrby/tables/tables_generator.rb

@@ -10,10 +10,13 @@ module Yrby
     # yrby:install, and by other gems building on the same storage.
     #
     # Template notes (kept here, not in the emitted migration): state is
-    # 4.gigabytes - 1 (longblob on MySQL; a compacted snapshot is the whole
-    # document; a 16 MB cap would break compaction) and payload is
-    # 16.megabytes - 1 (one update can carry a big paste or a client's
-    # accumulated offline edits; the 64 KB default blob is too small).
+    # 1.gigabyte - 1: the largest limit every adapter accepts. MySQL maps
+    # anything over 16 MB to longblob (a compacted snapshot is the whole
+    # document; a 16 MB cap would break compaction); Postgres raises above
+    # 1 GB - 1 and ignores the limit on bytea otherwise; SQLite ignores
+    # limits. Payload is 16.megabytes - 1 (one update can carry a big paste
+    # or a client's accumulated offline edits; the 64 KB default blob is
+    # too small).
     # The partial unique index's WHERE only keeps
     # key-only rows out of the index: uniqueness holds without it, since
     # unique indexes treat NULLs as distinct on every supported database,

data/lib/generators/yrby/tables/templates/create_y_tables.rb

@@ -6,7 +6,7 @@ class CreateYTables < ActiveRecord::Migration<%= migration_version %>
       t.string :key, null: false, index: { unique: true }
       t.references :record, polymorphic: true, null: true, index: false
       t.string :name
-      t.binary :state, limit: 4.gigabytes - 1
+      t.binary :state, limit: 1.gigabyte - 1
       t.timestamps
       t.index %i[record_type record_id name], unique: true,
               where: "record_type IS NOT NULL",

data/lib/y/collaborative/attribute.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Y
+  module Collaborative
+    # The collaborative document for one attribute of one record, such as a
+    # post's body. Ruby code and the shipped channel both use it, so they read
+    # and write the same rows.
+    class Attribute
+      attr_reader :record, :name
+
+      def initialize(record, name)
+        raise ArgumentError, "collaboration requires a persisted record" unless record.persisted?
+
+        @record = record
+        @name = name.to_s.dup.freeze
+      end
+
+      # The document as update bytes, or nil before the first write.
+      def load_state = existing_row&.load_state
+
+      # Records one change and returns once it's saved. The channel waits for
+      # this to return before it acknowledges or broadcasts the change.
+      def append(update) = document_row.append(update)
+
+      # Builds a new Y::Doc from storage on every call.
+      def y_doc
+        Y::Doc.new.tap do |doc|
+          state = load_state
+          doc.apply_update(state) if state
+        end
+      end
+
+      # The key clients sync the document under, such as "post/1/body". If a
+      # key-only channel created the document before it was linked to a
+      # record, this returns that original key.
+      def key = existing_row&.key || Y::Document.key_for(record, name)
+
+      # Finds or creates the Y::Document or Y::EncryptedDocument row. Writes
+      # and maintenance work such as compaction go through it.
+      def document_row = document_class.for(record, name)
+
+      private
+
+      # Returns the row if it exists without creating one, so reading a
+      # document never adds a row. It selects only id and key because the
+      # row's load_state reads the snapshot itself, and loading the snapshot
+      # here too would fetch it twice.
+      def existing_row = document_class.select(:id, :key).find_by(record:, name:)
+
+      def document_class = record.class.collaborative_document_class(name)
+    end
+  end
+end

data/lib/y/collaborative/helper.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Y
+  module Collaborative
+    # The view side of a collaborative document. The engine includes it into
+    # Action View.
+    #
+    #   <%= collaborative_document_tag @post, :body %>
+    #
+    # renders the auto-connecting element:
+    #
+    #   <yrby-document grant="<signed sgid>" name="body"></yrby-document>
+    #
+    # Importing "yrby-client/element" registers the element. It subscribes to
+    # Y::DocumentChannel by itself and exposes the synced Y.Doc through its
+    # `doc` property and `yrby:synced` event, much like the element behind
+    # turbo_stream_from.
+    #
+    # The grant is a signed GlobalID scoped to this record and attribute
+    # (record.collaborative_sgid(name)). Only render the tag on pages where
+    # the user may edit the record, because by default anyone holding the
+    # grant can open the document. To also check the user's current
+    # permissions when they subscribe, use Y::DocumentChannel.authorize_document.
+    #
+    # Extra options pass through to the element and a block becomes its
+    # content, so the element can wrap the mount point an editor binds to:
+    #
+    #   <%= collaborative_document_tag @post, :body, id: "editor" %>
+    #
+    # expires_in: limits how long the grant lasts. refresh: is a URL the
+    # element fetches a new grant from when a subscription is rejected, so it
+    # can resubscribe without a page load. The app's action there re-runs its
+    # own authorization and renders { grant: record.collaborative_sgid(name) }:
+    #
+    #   <%= collaborative_document_tag @post, :body, expires_in: 10.minutes,
+    #                                  refresh: grant_post_path(@post) %>
+    module Helper
+      def collaborative_document_tag(record, name, expires_in: nil, refresh: nil, **, &)
+        grant = record.collaborative_sgid(name, **{ expires_in: expires_in }.compact)
+        attributes = { grant: grant, name: name }
+        attributes[:refresh] = refresh if refresh
+        # The helper's attributes go last because a later key wins in a keyword
+        # splat, so a template can't override the grant or the name.
+        tag.yrby_document(**, **attributes, &)
+      end
+    end
+  end
+end

data/lib/y/collaborative.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+require "global_id"
+
+module Y
+  # The signed token that connects a page to a channel for record-backed
+  # collaborative documents.
+  #
+  # The client never picks its own document. The server signs a token for the
+  # record and attribute, the page renders it, and the channel looks the
+  # record up from it. The token is a signed GlobalID scoped to one attribute.
+  #
+  #   # the view
+  #   tag.div data: { grant: post.collaborative_sgid(:body) }
+  #
+  #   # the channel
+  #   def authorized?(_key)
+  #     record.present? && record.editable_by?(current_user)
+  #   end
+  #
+  #   # Not memoized, because AnyCable gives each command a fresh channel
+  #   # instance and a cached record would go stale. Y::DocumentChannel
+  #   # does the same.
+  #   def record
+  #     Y::Collaborative.locate(params[:grant], :body)
+  #   end
+  #
+  # The engine includes this into ActiveRecord::Base so any channel's
+  # authorized? can use it. lexxy-realtime uses the same token flow.
+  module Collaborative
+    extend ActiveSupport::Concern
+
+    class << self
+      # The signed-GlobalID purpose for one collaborative attribute. A token
+      # signed for one attribute verifies only against that attribute's
+      # purpose, so it can't locate the record through any other attribute.
+      # The purpose leaves out the channel name, so a custom channel and the
+      # shipped one can both resolve the same tokens for an attribute.
+      def sgid_purpose(name) = "yrby/#{name}"
+
+      # Looks up the record for a token from `collaborative_sgid(name)`.
+      # Returns nil for an invalid, tampered, expired, or wrong-attribute
+      # token, and for a record that no longer exists.
+      def locate(sgid, name)
+        GlobalID::Locator.locate_signed(sgid, for: sgid_purpose(name))
+      rescue ActiveRecord::RecordNotFound
+        nil
+      end
+    end
+
+    included do
+      class_attribute :collaborative_document_options,
+                      instance_accessor: false, default: {}.freeze
+    end
+
+    class_methods do
+      # Declares how one attribute's document is stored. The shipped channel
+      # and Ruby reads both follow it, and undeclared attributes use plain
+      # Y::Document.
+      def has_collaborative_document(name, encrypted: false) # rubocop:disable Naming/PredicatePrefix
+        self.collaborative_document_options = collaborative_document_options.merge(
+          name.to_s => { encrypted: encrypted }.freeze
+        ).freeze
+      end
+
+      # Returns the model that stores this attribute's document,
+      # Y::EncryptedDocument for an attribute declared encrypted and
+      # Y::Document for the rest. It resolves the constant on every call so
+      # that declaring an attribute doesn't load the engine's models while the
+      # app's models are still loading.
+      def collaborative_document_class(name)
+        encrypted = collaborative_document_options.dig(name.to_s, :encrypted)
+        encrypted ? Y::EncryptedDocument : Y::Document
+      end
+    end
+
+    # Returns the document for one attribute, stored the way the model
+    # declared. It responds to load_state, append, y_doc, and key.
+    def collaborative_document(name)
+      Attribute.new(self, name)
+    end
+
+    # A signed token that a channel can pass to Y::Collaborative.locate to get
+    # this record back, for this attribute only. Pass expires_in: to limit how
+    # long the grant lasts. Without it GlobalID's own default applies, which is
+    # one month under Rails. We pass the option through only when it's given,
+    # because an explicit nil would mean "never expire".
+    def collaborative_sgid(name, expires_in: nil)
+      options = { for: Y::Collaborative.sgid_purpose(name) }
+      options[:expires_in] = expires_in if expires_in
+      to_sgid(**options).to_s
+    end
+  end
+end
+
+require "y/collaborative/attribute"
+require "y/collaborative/helper"

data/lib/y/lexxy.rb

@@ -2,7 +2,7 @@
 
 module Y
   # The Lexxy renderer: Y::Lexical (core Lexical) plus the Lexxy-specific
-  # schema, applied beneath the app's rules — an app rule for one of these
+  # schema, applied beneath the app's rules: an app rule for one of these
   # types simply replaces it. This is the byte-parity class: the fixture
   # tests hold `Y::Lexxy.new(doc).to_html` identical to a live editor's own
   # serialized value.
@@ -36,14 +36,14 @@ module Y
       %(<th class="lexxy-content__table-cell--header" #{style}>#{node.content}</th>)
     end
 
-    # Attribute order follows Lexxy's export — checked items put aria-checked
+    # Attribute order follows Lexxy's export: checked items put aria-checked
     # before value; items holding a nested list append the
     # lexxy-nested-listitem class after value.
     def self.list_item(node)
       out = +"<li"
       checked = node.attrs["__checked"]
-      out << %( aria-checked="#{checked}") unless checked.nil?
-      out << %( value="#{node.attrs["__value"] || 1}")
+      out << %( aria-checked="#{RenderRules.escape_attr(checked)}") unless checked.nil?
+      out << %( value="#{RenderRules.escape_attr(node.attrs["__value"] || 1)}")
       out << %( class="lexxy-nested-listitem") if node.child_types.include?("list")
       "#{out}>#{node.content}</li>"
     end