braid-text 0.5.4 → 0.5.7

package/client/simpleton-sync.js

@@ -55,6 +55,18 @@
 //
 // content_type: used for Accept and Content-Type headers
 //
+// on_error?: (error) => void
+//     called when an error occurs (e.g., network failure, digest mismatch)
+//
+// on_online?: (is_online) => void
+//     called when the connection status changes
+//
+// on_ack?: () => void
+//     called when all outstanding PUTs have been acknowledged
+//
+// send_digests?: boolean
+//     if truthy, includes a Repr-Digest header with each PUT
+//
 // returns { changed, abort }
 //     call changed() whenever there is a local change,
 //     and the system will call a combination of get_state and
@@ -106,10 +118,9 @@ function simpleton_client(url, {
     get_patches,
     get_state,
     content_type,
-    headers: custom_headers,    // The user can pass in custom headers
-                              // that are forwared into the fetch
+    headers,                  // The user can pass in custom headers
+                              // that are forwarded into fetches
     on_error,
-    on_res,
     on_online,
     on_ack,
     send_digests
@@ -122,38 +133,34 @@ function simpleton_client(url, {
     var max_outstanding_changes = 10 // throttle limit
     var throttled = false
     var throttled_updates = []
-    var ac = new AbortController()
 
-    // ── Subscription (GET) ──────────────────────────────────────────────
-    //
-    // Opens a long-lived GET subscription with retry: () => true, meaning
-    // any disconnection (network error, HTTP error) triggers automatic
-    // reconnection with backoff.
-    //
-    // The parents callback sends client_version on reconnect, so the
-    // server knows where we left off and can send patches from there.
-    //
-    // IMPORTANT: No changed() / flush is called on reconnect. The
-    // subscription simply resumes. Any queued PUTs are retried by
-    // braid_fetch independently.
-    braid_fetch(url, {
-        peer,
-        subscribe: true,
-        heartbeats: 20,
-        signal: ac.signal,
-        retry: () => true,
-        parents: () => client_version.length ? client_version : null,
-        onSubscriptionStatus: status => on_online && on_online(status.online),
-        headers: { ...custom_headers,
-                   "Merge-Type": "simpleton",
-                   ...content_type && {Accept: content_type} },
-    }).then(res => {
-        if (on_res) on_res(res)
-        res.subscribe(async update => {
+    // extend the headers with merge-type and peer
+    headers = {
+        ...headers,
+        "Merge-Type": "simpleton",
+        Peer: peer,
+    }
+
+    // Manages both the GET subscription and PUT requests through a single
+    // channel with automatic reconnection and PUT queuing.
+    var channel = reliable_update_channel(url, {
+        reconnect_from_parents: () => client_version.length ? client_version : null,
+        get_headers: {
+            ...headers,
+            ...content_type && {Accept: content_type}
+        },
+        put_headers: {
+            ...headers,
+            ...content_type && {"Content-Type": content_type}
+        },
+        on_update: async update => {
             // ── Parent check ────────────────────────────────────────
             // Core simpleton invariant: only accept updates whose
-            // parents match our client_version exactly. This ensures
-            // we stay on a single line of time.
+            // parents form a continuous chain. We compare against the
+            // last queued update's version (if throttled) or
+            // client_version. When throttled, matching updates are
+            // queued but not applied — they'll be applied later when
+            // the throttle clears (see changed()).
             update.parents.sort()
             var last_queued = throttled_updates.length
                 ? throttled_updates[throttled_updates.length - 1].version
@@ -161,8 +168,16 @@ function simpleton_client(url, {
             if (versions_eq(last_queued, update.parents))
                 if (throttled) throttled_updates.push(update)
                 else await apply_update(update)
-        }, on_error)
-    }).catch(on_error)
+        },
+        on_status: status => on_online && on_online(status.online),
+        on_error: err => on_error && on_error(err),
+
+        // this api is preliminary and undocumented;
+        // we use it to tell the reliable_update_channel to die,
+        // if there is a digest mismatch on the server,
+        // which will result in a 550 status code
+        no_retry_status_codes: [550]
+    })
 
     async function apply_update(update) {
         // ── Parse and convert patches ───────────────────────────────
@@ -246,7 +261,7 @@ function simpleton_client(url, {
     // ── Public interface ────────────────────────────────────────────────
     return {
       // ── abort() — cancel the subscription ─────────────────────────
-      abort: () => ac.abort(),
+      abort: () => channel.close(),
 
       // ── changed() — call when local edits occur ───────────────────
       // This is the entry point for sending local edits. It:
@@ -326,28 +341,15 @@ function simpleton_client(url, {
                 //   - HTTP 550 (Repr-Digest mismatch / out of sync):
                 //     give up, throw — client must be re-created
                 outstanding_changes++
-                try {
-                    var r = await braid_fetch(url, {
-                        method: "PUT",
-                        peer, version, parents, patches,
-                        retry: (res) => res.status !== 550,
-                        headers: {
-                            ...custom_headers,
-                            "Merge-Type": "simpleton",
-                            ...send_digests && {
-                                "Repr-Digest": await get_digest(client_state) },
-                            ...content_type && {
-                                "Content-Type": content_type }
-                        }
-                    })
-                    if (!r.ok) throw new Error(`bad http status: ${r.status}`)
-                } catch (e) {
-                    // A 550 means Repr-Digest check failed — we're out
-                    // of sync. The client must be torn down and
-                    // re-created from scratch.
-                    on_error(e)
-                    throw e
-                }
+
+                await channel.put({
+                    version, parents, patches,
+                    headers: {
+                        ...send_digests && {
+                            "Repr-Digest": await get_digest(client_state) }
+                    }
+                })
+                
                 throttled = false
                 outstanding_changes--
                 if (on_ack && !outstanding_changes) on_ack()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "braid-text",
-  "version": "0.5.4",
+  "version": "0.5.7",
   "description": "Library for collaborative text over http using braid.",
   "author": "Braid Working Group",
   "repository": "braid-org/braid-text",
@@ -25,7 +25,7 @@
   ],
   "dependencies": {
     "@braid.org/diamond-types-node": "^2.0.1",
-    "braid-http": "^1.3.106"
+    "braid-http": "~1.3.123"
   },
   "peerDependencies": {
     "yjs": "^13.6.0"

package/server-demo.js

@@ -14,14 +14,15 @@ var server = require("http").createServer(async (req, res) => {
     if (req.method === 'OPTIONS') return res.end()
 
     var q = req.url.split('?').slice(-1)[0]
-    if (q === 'editor' || q === 'markdown-editor') {
+    if (q === 'editor' || q === 'markdown-editor' || q === 'yjs-editor' || q === 'demo') {
         res.writeHead(200, { "Content-Type": "text/html", "Cache-Control": "no-cache" })
         require("fs").createReadStream(`./client/${q}.html`).pipe(res)
         return
     }
 
     if (req.url === '/simpleton-sync.js' || req.url === '/web-utils.js'
-        || req.url === '/textarea-highlights.js' || req.url === '/cursor-sync.js') {
+        || req.url === '/textarea-highlights.js' || req.url === '/cursor-sync.js'
+        || req.url === '/yjs-sync.js') {
         res.writeHead(200, { "Content-Type": "text/javascript", "Cache-Control": "no-cache" })
         require("fs").createReadStream("./client" + req.url).pipe(res)
         return

package/server.js

@@ -353,8 +353,7 @@ function create_braid_text() {
 
         var peer = req.headers['peer'],
             merge_type = req.headers['merge-type'] || 'simpleton'
-        // TODO: accept 'yjs' merge_type here for Braid-HTTP yjs clients
-        if (merge_type !== 'simpleton' && merge_type !== 'dt')
+        if (merge_type !== 'simpleton' && merge_type !== 'dt' && merge_type !== 'yjs')
             return my_end(400, `Unknown merge type: ${merge_type}`)
 
         var is_read  = req.method === 'GET' || req.method === 'HEAD',
@@ -744,16 +743,14 @@ function create_braid_text() {
                 if (getting.history === 'since-parents')
                     throw new Error('yjs-text from arbitrary parents not yet implemented')
 
-                var patches = braid_text.from_yjs_binary(
+                var yjs_updates = braid_text.from_yjs_binary(
                     Y.encodeStateAsUpdate(resource.yjs.doc))
 
                 if (!getting.subscribe)
-                    return { version: resource.version, parents: [], patches }
+                    return yjs_updates
 
-                if (patches.length)
-                    options.subscribe({
-                        version: resource.version, parents: [], patches
-                    })
+                for (var u of yjs_updates)
+                    options.subscribe(u)
             }
 
             if (getting.subscribe) {
@@ -911,7 +908,10 @@ function create_braid_text() {
                     ? options.yjs_update : new Uint8Array(options.yjs_update)
             } else if (patches && patches.length && patches[0].unit === 'yjs-text') {
                 yjs_text_patches = patches
-                yjs_binary = braid_text.to_yjs_binary(patches)
+                yjs_binary = braid_text.to_yjs_binary([{
+                    version: options.version?.[0],
+                    patches
+                }])
             }
 
             if (yjs_binary) {
@@ -959,7 +959,8 @@ function create_braid_text() {
                         }
                         for (var b of dt_bytes) resource.dt.doc.mergeBytes(b)
                         resource.version = resource.dt.doc.getRemoteVersion().map(x => x.join("-")).sort()
-                        if (!resource.dt.known_versions[syn_actor]) resource.dt.known_versions[syn_actor] = new RangeSet()
+                        if (!resource.dt.known_versions[syn_actor])
+                            resource.dt.known_versions[syn_actor] = new RangeSet()
                         resource.dt.known_versions[syn_actor].add_range(0, syn_seq - 1)
                         await resource.dt.log.save(resource.dt.doc.getPatchSince(yjs_v_before))
 
@@ -977,8 +978,15 @@ function create_braid_text() {
                             if (!peer || client.peer !== peer)
                                 await client.send_update(
                                     client.accept_encoding_dt
-                                        ? { version: resource.version, parents: version_before_yjs_sync, body: resource.dt.doc.getPatchSince(yjs_v_before), encoding: 'dt' }
-                                    : { version: resource.version, parents: version_before_yjs_sync, patches: xf }
+                                        ? { version: resource.version,
+                                            parents: version_before_yjs_sync,
+                                            body: resource.dt.doc.getPatchSince(yjs_v_before),
+                                            encoding: 'dt'
+                                          }
+                                    : { version: resource.version,
+                                        parents: version_before_yjs_sync,
+                                        patches: xf
+                                      }
                                 )
                         }
                     }
@@ -986,18 +994,15 @@ function create_braid_text() {
 
                 // Broadcast to yjs-text subscribers (skip sender)
                 if (resource.yjs) {
-                    // If we received yjs-text patches, reuse them; otherwise
+                    // If we received yjs-text updates, reuse them; otherwise
                     // derive them from the binary update
-                    var yjs_patches = yjs_text_patches
-                        || braid_text.from_yjs_binary(yjs_binary)
-                    if (yjs_patches.length) {
+                    var yjs_updates = yjs_text_patches
+                        ? [{version: options.version, patches: yjs_text_patches}]
+                        : braid_text.from_yjs_binary(yjs_binary)
+                    for (var yjs_update of yjs_updates) {
                         for (var client of resource.yjs.clients) {
-                            if (!peer || client.peer !== peer) {
-                                await client.send_update({
-                                    version: resource.version,
-                                    patches: yjs_patches
-                                })
-                            }
+                            if (!peer || client.peer !== peer)
+                                await client.send_update(yjs_update)
                         }
                     }
                 }
@@ -1233,15 +1238,12 @@ function create_braid_text() {
                 // and we are mixing them. The update-level .version is the DT frontier.
                 // Each patch's .version is a Yjs item ID (clientID-clock).
                 if (resource.yjs && captured_yjs_update) {
-                    var yjs_patches = braid_text.from_yjs_binary(captured_yjs_update)
-                    for (var client of resource.yjs.clients) {
-                        if (!peer || client.peer !== peer) {
-                            await client.send_update({
-                                version: resource.version,   // DT version space
-                                patches: yjs_patches         // patches[].version: Yjs version space
-                            })
-                        }
-                    }
+                    var yjs_updates = braid_text.from_yjs_binary(captured_yjs_update)
+                    if (braid_text.verbose) console.log('DT→Yjs broadcast:', yjs_updates.length, 'updates to', resource.yjs.clients.size, 'yjs clients')
+                    for (var yjs_update of yjs_updates)
+                        for (var client of resource.yjs.clients)
+                            if (!peer || client.peer !== peer)
+                                await client.send_update(yjs_update)
                 }
 
                 // Persist Yjs delta
@@ -1355,7 +1357,9 @@ function create_braid_text() {
             if (braid_text.db_folder) {
                 await db_folder_init()
                 var pages = new Set()
-                for (let x of await require('fs').promises.readdir(braid_text.db_folder)) if (/\.(dt|yjs)\.\d+$/.test(x)) pages.add(decode_filename(x.replace(/\.(dt|yjs)\.\d+$/, '')))
+                for (let x of await require('fs').promises.readdir(braid_text.db_folder))
+                    if (/\.(dt|yjs)\.\d+$/.test(x))
+                        pages.add(decode_filename(x.replace(/\.(dt|yjs)\.\d+$/, '')))
                 return [...pages.keys()]
             } else return Object.keys(braid_text.cache)
         } catch (e) { return [] }
@@ -3072,48 +3076,60 @@ function create_braid_text() {
     // Convert a Yjs binary update to yjs-text range patches.
     // Decodes the binary without needing a Y.Doc.
     // Returns array of {unit: 'yjs-text', range: '...', content: '...'}
+    // Convert a Yjs binary update to an array of braid updates,
+    // each with a version and patches in yjs-text format.
     braid_text.from_yjs_binary = function(update) {
         require_yjs()
         var decoded = Y.decodeUpdate(
             update instanceof Uint8Array ? update : new Uint8Array(update))
-        var patches = []
+        var updates = []
 
-        // Convert inserted structs to yjs-text patches
+        // Each inserted struct becomes one update with one insert patch.
+        // GC'd structs (deleted items) have content.len but no content.str —
+        // we emit placeholder text since the delete set will remove it anyway.
         for (var struct of decoded.structs) {
-            if (!struct.content?.str) continue  // skip non-text items
+            var text = struct.content?.str
+            if (!text && struct.content?.len) text = '_'.repeat(struct.content.len)
+            if (!text) continue  // skip non-text items (e.g. format, embed)
             var id = struct.id
             var origin = struct.origin
             var rightOrigin = struct.rightOrigin
             var left = origin ? `${origin.client}-${origin.clock}` : ''
             var right = rightOrigin ? `${rightOrigin.client}-${rightOrigin.clock}` : ''
-            patches.push({
-                unit: 'yjs-text',
-                range: `(${left}:${right})`,
-                content: struct.content.str,
-                version: `${id.client}-${id.clock}`
+            updates.push({
+                version: [`${id.client}-${id.clock}`],
+                patches: [{
+                    unit: 'yjs-text',
+                    range: `(${left}:${right})`,
+                    content: text,
+                }]
             })
         }
 
-        // Convert delete set entries to yjs-text patches
+        // Each delete range becomes one update with one delete patch
         for (var [clientID, deleteItems] of decoded.ds.clients) {
             for (var item of deleteItems) {
                 var left = `${clientID}-${item.clock}`
                 var right = `${clientID}-${item.clock + item.len - 1}`
-                patches.push({
-                    unit: 'yjs-text',
-                    range: `[${left}:${right}]`,
-                    content: ''
+                updates.push({
+                    version: [`${clientID}-${item.clock}`],
+                    patches: [{
+                        unit: 'yjs-text',
+                        range: `[${left}:${right}]`,
+                        content: ''
+                    }]
                 })
             }
         }
 
-        return patches
+        return updates
     }
 
     // Convert yjs-text range patches to a Yjs binary update.
+    // Convert braid updates with yjs-text patches to a Yjs binary update.
     // This is the inverse of from_yjs_binary.
-    // Insert patches must have a .version field with the item ID as "client-clock".
-    braid_text.to_yjs_binary = function(patches) {
+    // Accepts an array of updates, each with {version, patches}.
+    braid_text.to_yjs_binary = function(updates) {
         require_yjs()
         var lib0_encoding = require('lib0/encoding')
         var encoder = new Y.UpdateEncoderV1()
@@ -3122,26 +3138,30 @@ function create_braid_text() {
         var inserts_by_client = new Map()
         var deletes_by_client = new Map()
 
-        for (var p of patches) {
-            var parsed = parse_yjs_range(p.range)
-            if (!parsed) throw new Error(`invalid yjs-text range: ${p.range}`)
-
-            if (p.content.length > 0) {
-                // Insert — version is the item ID as "client-clock"
-                if (!p.version) throw new Error('insert patch requires .version = "client-clock"')
-                var v_parts = p.version.match(/^(\d+)-(\d+)$/)
-                if (!v_parts) throw new Error('invalid insert patch version: ' + p.version)
-                var item_id = { client: parseInt(v_parts[1]), clock: parseInt(v_parts[2]) }
-                var list = inserts_by_client.get(item_id.client) || []
-                list.push({ id: item_id, origin: parsed.left, rightOrigin: parsed.right, content: p.content })
-                inserts_by_client.set(item_id.client, list)
-            } else {
-                // Delete
-                if (!parsed.left) throw new Error('delete patch requires left ID')
-                var client = parsed.left.client
-                var list = deletes_by_client.get(client) || []
-                list.push({ clock: parsed.left.clock, len: parsed.right ? parsed.right.clock - parsed.left.clock + 1 : 1 })
-                deletes_by_client.set(client, list)
+        for (var update of updates) {
+            if (!update.patches) continue
+            for (var p of update.patches) {
+                var parsed = parse_yjs_range(p.range)
+                if (!parsed) throw new Error(`invalid yjs-text range: ${p.range}`)
+
+                if (p.content.length > 0) {
+                    // Insert — version on the update is the item ID as ["client-clock"]
+                    var v_str = Array.isArray(update.version) ? update.version[0] : update.version
+                    if (!v_str) throw new Error('insert update requires .version = ["client-clock"]')
+                    var v_parts = v_str.match(/^(\d+)-(\d+)$/)
+                    if (!v_parts) throw new Error('invalid update version: ' + v_str)
+                    var item_id = { client: parseInt(v_parts[1]), clock: parseInt(v_parts[2]) }
+                    var list = inserts_by_client.get(item_id.client) || []
+                    list.push({ id: item_id, origin: parsed.left, rightOrigin: parsed.right, content: p.content })
+                    inserts_by_client.set(item_id.client, list)
+                } else {
+                    // Delete
+                    if (!parsed.left) throw new Error('delete patch requires left ID')
+                    var client = parsed.left.client
+                    var list = deletes_by_client.get(client) || []
+                    list.push({ clock: parsed.left.clock, len: parsed.right ? parsed.right.clock - parsed.left.clock + 1 : 1 })
+                    deletes_by_client.set(client, list)
+                }
             }
         }