maquina_stream 0.1.0

data/lib/maquina_stream/document.rb

@@ -0,0 +1,296 @@
+# frozen_string_literal: true
+
+module MaquinaStream
+  # Splits a rendered message into top-level blocks, and decides which of them
+  # are safe to freeze.
+  #
+  # ```ruby
+  # document = MaquinaStream::Document.new(markdown, config: config, sid: "m8f21")
+  # document.blocks          # every top-level block, in order
+  # document.sealed_blocks   # frozen, never re-sent
+  # document.open_block      # the tail, patched on every frame
+  # ```
+  #
+  # The whole buffer is rendered once and then sliced. Blocks are never rendered
+  # in isolation: a block that mentions [docs] needs the link reference
+  # definition that lives at the bottom of the message, and rendering it alone
+  # silently loses it.
+  class Document
+    # The raw markdown this document was built from.
+    attr_reader :markdown
+
+    # The Configuration it reads — `seal_lag` in particular.
+    attr_reader :config
+
+    # The stream id every block id is prefixed with, or nil for an anonymous
+    # render.
+    attr_reader :sid
+
+    # The render mode passed through to Renderer, one of Renderer::MODES.
+    attr_reader :mode
+
+    # Builds a document over one markdown buffer.
+    #
+    # `sid:` is the record's `maquina_stream_id`. Pass it: block ids are built
+    # from it, and a document rendered without one produces ids
+    # (`ms-b0`, `ms-b1`) that collide the moment two messages share a page.
+    #
+    # Nothing is rendered until #blocks or #html is called.
+    def initialize(markdown, config: MaquinaStream.config, sid: nil, mode: :streaming)
+      @markdown = markdown.to_s
+      @config = config
+      @sid = sid
+      @mode = mode
+    end
+
+    # Every top-level Block, in document order, each already stamped with its
+    # id and digest. Memoized: the whole buffer is rendered once, on the first
+    # call.
+    def blocks
+      @blocks ||= build_blocks
+    end
+
+    # Sealed blocks trail the tail by config.seal_lag. Markdown reinterprets
+    # retroactively - a paragraph becomes a heading when its underline arrives,
+    # a table's delimiter row turns the line above into a header - so a block is
+    # only safe to freeze once enough later blocks exist that nothing can reach
+    # back into it.
+    #
+    # The lag is not the whole story. A link reference definition resolves links
+    # in blocks arbitrarily far above it, so no fixed lag makes a block with an
+    # unresolved reference safe. The pointer stops there instead, and moves on
+    # once the definition arrives.
+    def sealed_blocks
+      blocks.first(seal_pointer)
+    end
+
+    # The blocks behind the seal pointer: still able to change, and therefore
+    # still eligible for a patch on the next frame.
+    def unsealed_blocks
+      blocks.drop(seal_pointer)
+    end
+
+    # How far sealing reached, as an index into #blocks. Sealing is a prefix,
+    # so this is both the count of sealed blocks and the position of the first
+    # unsealed one.
+    def seal_pointer
+      @seal_pointer ||= blocks.count(&:sealed?)
+    end
+
+    # The last block — the tail the model is currently writing into.
+    def open_block
+      blocks.last
+    end
+
+    # The whole document as one rendered HTML string, before it is sliced into
+    # blocks. Memoized.
+    def html
+      @html ||= Renderer.call(markdown, mode: mode, config: config)
+    end
+
+    # `[[id, digest], …]` for the sealed blocks. What Manifest is built from.
+    def manifest_entries
+      sealed_blocks.map(&:to_manifest_entry)
+    end
+
+    private
+      # Blocks are matched to their source range by the index the post-pass
+      # stamped on them, never by position in the rendered output. The sanitizer
+      # drops nodes - an HTML comment, a disallowed element - and a positional
+      # match silently shifts every range after the first drop.
+      def build_blocks
+        rendered = block_nodes(Nokogiri::HTML5.fragment(html))
+        ranges = source_ranges
+        indices = block_indices(rendered)
+        sealed_count = [rendered.length - config.seal_lag, 0].max
+
+        rendered.each_with_index.map do |node, position|
+          source_index = indices[position]
+          range = ranges[source_index]
+
+          # The digest covers the block's content, before the element-level
+          # attributes below are stamped on. Those change when a block seals or
+          # when the caret moves past it, and neither changes what it says.
+          digest = Digest::SHA256.hexdigest("#{source_index}:#{node.inner_html}")[0, 16]
+
+          # The DOM contract, from docs/javascript.md. The id is index-derived
+          # so idiomorph pairs the node instead of recreating it.
+          node["id"] = sid ? "ms-#{sid}-b#{source_index}" : "ms-b#{source_index}"
+          node["data-ms-block"] = ""
+          node["data-ms-block-index"] = source_index.to_s
+          node["data-ms-block-digest"] = digest
+
+          Block.new(
+            index: source_index,
+            markdown: slice(range),
+            html: node.to_html,
+            line_range: range,
+            sid: sid,
+            sealed: false,
+            digest: digest
+          )
+        end.then { |built| apply_seal(built, sealed_count) }
+      end
+
+      # Inline-level elements. Content inside a block, never a block of their
+      # own — and every one of them can be stranded at the top level by the
+      # unwrap below.
+      INLINE_ELEMENTS = Set[
+        "a", "abbr", "b", "br", "cite", "code", "del", "dfn", "em", "i", "img",
+        "input", "ins", "kbd", "mark", "q", "s", "samp", "small", "span",
+        "strong", "sub", "sup", "time", "u", "var", "wbr"
+      ]
+
+      # Splitting must not be able to lose a character.
+      #
+      # The sanitizer unwraps an element it does not know — <thinking>,
+      # <tool_call>, <citation>, any tag a model invents to carry meaning for the
+      # application — and keeps its children. CommonMark has already made that
+      # tag an HTML block, so the text under it comes back as a bare text node at
+      # the TOP level of the fragment, where `select(&:element?)` used to drop it
+      # on the floor. The same happens to an inline element the unwrap strands
+      # there.
+      #
+      # The wrapping belongs here rather than in the sanitizer's final pass. The
+      # sanitizer is a pure allowlist over an arbitrary fragment: it runs again
+      # on the client over fragments that are deliberately inline, and a pass
+      # that invented a <p> around them would change what the caller asked to
+      # sanitize. Document is the object that claims every character which
+      # survives sanitizing lands in exactly one block, so it is the object that
+      # has to make the claim true.
+      #
+      # Consecutive orphans are wrapped together — text plus the <em> beside it
+      # stay one block, as they read — and a run that is only whitespace is
+      # inter-block separation, not content, so it is left where it is.
+      def block_nodes(fragment)
+        fragment.children.to_a
+          .slice_when { |before, after| orphan?(before) != orphan?(after) }
+          .flat_map { |run| orphan?(run.first) ? [wrap_orphans(run)].compact : run.select(&:element?) }
+      end
+
+      def orphan?(node)
+        return true unless node.element?
+
+        INLINE_ELEMENTS.include?(node.name.downcase)
+      end
+
+      def wrap_orphans(run)
+        return nil if run.map(&:text).join.strip.empty?
+
+        wrapper = Nokogiri::XML::Node.new("p", run.first.document)
+        wrapper["data-ms-element"] = "p"
+        run.first.add_previous_sibling(wrapper)
+        run.each { |node| wrapper.add_child(node) }
+        wrapper
+      end
+
+      # The post-pass stamps `data-ms-block-index` on every top-level element it
+      # sees; a block wrapped above never passed under it, and neither did an
+      # element the unwrap promoted from inside one. Those take the first free
+      # index at or after their position, so ids stay unique — the one property
+      # idiomorph needs from them. They are not always in ascending order, and
+      # they do not have to be: block order is the order of #blocks.
+      def block_indices(nodes)
+        used = nodes.filter_map { |node| node["data-ms-block-index"]&.to_i }.to_set
+        cursor = 0
+
+        nodes.each_with_index.map do |node, position|
+          stamped = node["data-ms-block-index"]&.to_i
+          next stamped if stamped
+
+          cursor = [cursor, position].max
+          cursor += 1 while used.include?(cursor)
+          used << cursor
+          cursor
+        end
+      end
+
+      # Sealing is a prefix: the pointer is the position it reaches, so a block
+      # that cannot seal holds every block after it open too.
+      def apply_seal(built, sealed_count)
+        limit = [sealed_count, unresolved_position(built) || sealed_count].min
+        built.each_with_index.map { |block, position| (position < limit) ? block.seal : block }
+      end
+
+      def unresolved_position(built)
+        built.index { |block| unresolved_references?(block.markdown) }
+      end
+
+      REFERENCE_USE = /\[[^\]\n]*\]\[([^\]\n]*)\]/
+      # A definition counts only once its line is complete: "[docs]:" with no
+      # destination yet resolves nothing, and "[docs]: https://exa" resolves to
+      # a truncated host that changes when the rest arrives. Either would seal a
+      # block whose links are still moving.
+      REFERENCE_DEFINITION = /^ {0,3}\[([^\]\n]+)\]:[ \t]*\S+[^\n]*\n/
+
+      def unresolved_references?(source)
+        source.scan(REFERENCE_USE).flatten.any? do |label|
+          !defined_reference_labels.include?(label.strip.downcase)
+        end
+      end
+
+      def defined_reference_labels
+        @defined_reference_labels ||= markdown.scan(REFERENCE_DEFINITION).flatten.map { |l| l.strip.downcase }.to_set
+      end
+
+      # Line ranges come from a parse of the raw buffer, not from the rendered
+      # output: the post-pass rewrites elements and the sanitizer may drop one,
+      # and neither carries a source position afterwards.
+      #
+      # HTML blocks report no sourcepos at all (confirmed against commonmarker
+      # 2.10.0 in test/sourcepos_test.rb), and a link reference definition
+      # renders no element whatsoever - so coverage has gaps, and a block with no
+      # range of its own inherits the lines between its neighbours.
+      def source_ranges
+        parsed = Commonmarker.to_html(
+          prepared,
+          options: Renderer::COMMONMARKER_OPTIONS,
+          plugins: Renderer::COMMONMARKER_PLUGINS
+        )
+
+        nodes = Nokogiri::HTML5.fragment(parsed).children.select(&:element?)
+        explicit = nodes.map { |node| parse_sourcepos(node["data-sourcepos"]) }
+
+        fill_gaps(explicit)
+      end
+
+      def parse_sourcepos(value)
+        return nil unless value
+
+        start_line, end_line = value.split("-").map { |part| part.split(":").first.to_i }
+        (start_line..end_line)
+      end
+
+      def fill_gaps(ranges)
+        ranges.each_with_index.map do |range, index|
+          next range if range
+
+          previous_end = ranges[0...index].compact.last&.last
+          next_start = ranges[(index + 1)..].compact.first&.first
+
+          from = (previous_end || 0) + 1
+          to = (next_start || line_count + 1) - 1
+          (from > to) ? (from..from) : (from..to)
+        end
+      end
+
+      # The same string the renderer parsed: repaired by maquina_remend and
+      # normalised by Renderer::TagBlocks. Ranges and slices have to share one
+      # coordinate system, and the rendered document is in this one.
+      def prepared
+        @prepared ||= Renderer.prepare(markdown)
+      end
+
+      def lines
+        @lines ||= prepared.lines
+      end
+
+      def line_count = lines.length
+
+      def slice(range)
+        return "" unless range
+
+        lines[(range.first - 1)..(range.last - 1)].to_a.join
+      end
+  end
+end

data/lib/maquina_stream/engine.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module MaquinaStream
+  # The Rails engine. Mount it for the repair endpoints:
+  #
+  # ```ruby
+  # mount MaquinaStream::Engine => "/maquina_stream"
+  # ```
+  #
+  # It contributes two routes (see MaquinaStream::ManifestsController and
+  # MaquinaStream::BlocksController), the engine's stylesheets and JavaScript,
+  # and its importmap pins. It contributes no migrations and no models: the
+  # host owns persistence.
+  class Engine < ::Rails::Engine
+    isolate_namespace MaquinaStream
+
+    initializer "maquina_stream.streamable" do
+      ActiveSupport.on_load(:active_record) do
+        # Hosts `include MaquinaStream::Streamable` themselves; requiring it
+        # here only guarantees it is loaded before any host model boots.
+        require "maquina_stream/streamable"
+      end
+    end
+
+    # NoBuild: the engine's JavaScript ships as source and is served by the
+    # asset pipeline, pinned into the host's importmap. There is no package.json
+    # and no npm dependency anywhere in this gem.
+    #
+    # Both initializers are no-ops when the host has neither propshaft/sprockets
+    # nor importmap-rails: the engine's Ruby side does not require them.
+    initializer "maquina_stream.assets" do |app|
+      next unless app.config.respond_to?(:assets)
+
+      app.config.assets.paths << root.join("app/javascript")
+      app.config.assets.paths << root.join("app/assets/stylesheets")
+    end
+
+    initializer "maquina_stream.importmap", before: "importmap" do |app|
+      next unless app.config.respond_to?(:importmap)
+
+      app.config.importmap.paths << root.join("config/importmap.rb")
+      # Cache-bust the host's importmap when engine JavaScript changes in dev.
+      app.config.importmap.cache_sweepers << root.join("app/javascript")
+    end
+  end
+end

data/lib/maquina_stream/errors.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module MaquinaStream
+  # Base class for everything this engine raises. Rescue it to catch all of
+  # them at once.
+  Error = Class.new(StandardError)
+
+  # Raised when a host model does not satisfy the MaquinaStream::Streamable
+  # contract. The message names the method, the column and the class, so the
+  # fix is in the error rather than in a document.
+  ContractError = Class.new(Error)
+
+  # Raised when a required host seam has not been configured — `find_stream`,
+  # in practice. `authorize` denies instead of raising, because denial is the
+  # safe answer.
+  ConfigurationError = Class.new(Error)
+end

data/lib/maquina_stream/export.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module MaquinaStream
+  # A whole message, back out as markdown.
+  #
+  # ```ruby
+  # MaquinaStream::Export.markdown(message) # => String
+  # ```
+  #
+  # The buffer already is markdown, so export is mostly a question of what to do
+  # with the parts that are not clean:
+  #
+  # * **A cancelled stream ends mid-token.** The raw buffer would export a half
+  #   written bold run or an unterminated fence, so the buffer is repaired first
+  #   — the same preprocessor the renderer uses, so an export matches what was
+  #   on screen.
+  # * **Deferred content exports as its source**, which is the Phase 6 fallback,
+  #   decided once and applied everywhere. A diagram exports as its `mermaid`
+  #   fence, verbatim, because that is what the model wrote and what another
+  #   tool can read. There is nothing to substitute: rendering it would mean
+  #   shipping the renderer to the server.
+  #
+  # A status footer is appended for anything that did not finish, because a
+  # cancelled message that exports as though it were complete is a lie in a file
+  # somebody keeps.
+  module Export
+    # The seal statuses that get a footer. `:complete` does not; neither does a
+    # stream that is still open, which exports as far as it has got.
+    INCOMPLETE = %i[cancelled errored timed_out].freeze
+
+    class << self
+      # A whole message as markdown, as a String.
+      #
+      # `annotate: false` suppresses the status footer — for a caller that is
+      # re-ingesting the text rather than handing a human a file, and that will
+      # carry the status some other way.
+      #
+      # The footer comes from the `maquina_stream.export.<status>` locale key,
+      # falling back to `> [status]`.
+      def markdown(record, config: MaquinaStream.config, annotate: true)
+        buffer = MaquinaRemend.call(record.maquina_stream_buffer.to_s)
+        status = status_of(record)
+
+        return buffer unless annotate && INCOMPLETE.include?(status)
+
+        "#{buffer.rstrip}\n\n#{note_for(status, config)}\n"
+      end
+
+      private
+        def status_of(record)
+          return :open if record.maquina_stream_open?
+          return nil unless record.respond_to?(:maquina_stream_status)
+
+          record.maquina_stream_status&.to_sym
+        end
+
+        def note_for(status, config)
+          I18n.t(
+            "maquina_stream.export.#{status}",
+            locale: I18n.locale || config.locale,
+            default: "> [#{status}]"
+          )
+        end
+    end
+  end
+end

data/lib/maquina_stream/frame.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module MaquinaStream
+  # One broadcast: what changed since the last one.
+  #
+  # | Field | What is in it |
+  # |---|---|
+  # | `appends` | blocks the browser has never seen, sent whole |
+  # | `patch` | unsealed blocks whose HTML moved since the last frame |
+  #
+  # Frames come out of Broadcaster. A host reads them — to assert a bandwidth
+  # budget, or to drive a transport of its own — rather than building them.
+  #
+  # `patch` is a list rather than a single open block. The seal lag keeps `seal_lag` blocks unsealed at all times, and any
+  # of them can still change — a paragraph two blocks back becomes a heading
+  # when its underline arrives. Sending only the last one would leave the others
+  # wrong until a repair.
+  class Frame
+    # This frame's sequence number, from the host's row. Monotonic, one per
+    # frame that actually went out. A client that sees a gap knows it missed
+    # something.
+    attr_reader :seq
+
+    # Blocks the browser has never seen, as Block objects. Sent whole.
+    attr_reader :appends
+
+    # Unsealed blocks whose HTML moved since the last frame, as Block objects.
+    # Applied by morph, keyed on id.
+    attr_reader :patch
+
+    # Builds a frame. Broadcaster does this.
+    def initialize(seq:, appends: [], patch: [], final: false)
+      @seq = seq
+      @appends = appends
+      @patch = patch
+      @final = final
+    end
+
+    # The final seal. It is marked on the wire because the repair path triggers
+    # on it: docs/design.md calls this the trigger that makes all intra-stream
+    # drift cosmetic and self-correcting, and a client cannot know a frame is
+    # the last one by looking at it.
+    def final? = @final
+
+    # Whether this frame carries nothing. An empty frame is skipped, unless it
+    # is the final one.
+    def empty?
+      appends.empty? && patch.empty?
+    end
+
+    # Every block in this frame, appends first.
+    def blocks
+      appends + patch
+    end
+
+    # The HTML this frame puts on the wire, in bytes. What the bandwidth
+    # budget is measured in.
+    def bytesize
+      blocks.sum { |block| block.html.to_s.bytesize }
+    end
+
+    # A summary: sequence, finality, and the block ids on each side. Ids
+    # rather than HTML, so it stays readable in a log.
+    def to_h
+      {
+        seq: seq,
+        final: final?,
+        appends: appends.map(&:id),
+        patch: patch.map(&:id)
+      }
+    end
+  end
+end

data/lib/maquina_stream/manifest.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module MaquinaStream
+  # What the browser is told a message currently *is*, in a bounded number of
+  # bytes.
+  #
+  # ```ruby
+  # { seq: 412, cutoff: 38, rollup: "7c1f…",
+  #   blocks: [["ms-m8f21-b38", "a91c…"], ["ms-m8f21-b39", "4fe2…"]] }
+  # ```
+  #
+  # Not HTML. The client diffs this against its own DOM, asks for the blocks
+  # whose digests differ, and morphs only those — so repair costs what has
+  # drifted rather than what the message weighs.
+  #
+  # ## Why it is windowed
+  #
+  # docs/design.md said the manifest is "a few hundred bytes regardless of
+  # message size". Measured, listing every sealed block gave 1.8KB for a 2KB
+  # message and 88KB for a 100KB one — it tracked length almost exactly, because
+  # block count does. A keyframe every four seconds carrying 88KB is the
+  # bandwidth problem the manifest was introduced to prevent.
+  #
+  # So the manifest carries the last `window` sealed blocks in full, plus one
+  # rollup digest covering everything older. A client whose rollup matches knows
+  # its history is intact and only has to consider the window; a client whose
+  # rollup differs asks for the whole thing, which is rare and is what a cold
+  # page load does anyway.
+  #
+  # Payload is then bounded by the window, not by the message.
+  #
+  # ## Wire format
+  #
+  # | Key | Meaning |
+  # |---|---|
+  # | `seq` | the sequence number this manifest describes |
+  # | `cutoff` | how many sealed blocks fall behind the window |
+  # | `rollup` | one digest covering every block behind the cutoff |
+  # | `blocks` | `[[id, digest], …]` for the blocks inside the window |
+  class Manifest
+    # The sequence number this manifest describes.
+    attr_reader :seq
+
+    # Every sealed Block, in order — including the ones behind the window,
+    # which #rollup covers and #windowed_entries omits.
+    attr_reader :blocks
+
+    # How many sealed blocks are listed in full. Everything older is covered by
+    # #rollup. `Float::INFINITY` when the manifest was built with `full: true`.
+    attr_reader :window
+
+    # Builds the manifest for a record, over its current buffer.
+    #
+    # `full: true` drops the window and lists every sealed block — what a
+    # client asks for when its #rollup disagrees with ours and the windowed
+    # diff is therefore not enough. Rare, and no more expensive than the cold
+    # page load it resembles.
+    def self.for(record, config: MaquinaStream.config, window: nil, full: false)
+      document = Document.new(
+        record.maquina_stream_buffer,
+        config: config,
+        sid: record.maquina_stream_id,
+        mode: record.maquina_stream_open? ? :streaming : :static
+      )
+
+      new(
+        seq: record.maquina_stream_sequence,
+        blocks: document.sealed_blocks,
+        window: full ? Float::INFINITY : (window || config.manifest_window)
+      )
+    end
+
+    # Builds a manifest over a list of sealed blocks. `.for` is the usual
+    # entry point.
+    def initialize(seq:, blocks:, window: 50)
+      @seq = seq
+      @blocks = blocks
+      @window = window
+    end
+
+    # Every sealed block, id and digest. The wire format sends a slice of this.
+    def entries
+      blocks.map(&:to_manifest_entry)
+    end
+
+    # How many entries fall behind the window and are covered by #rollup
+    # instead of being listed. Zero for a full manifest.
+    def cutoff
+      return 0 if window.infinite?
+
+      [entries.length - window, 0].max
+    end
+
+    # The entries actually sent: everything from #cutoff onwards.
+    def windowed_entries
+      entries.drop(cutoff)
+    end
+
+    # One digest covering every block older than the window. Order matters: two
+    # clients holding the same blocks in a different order are not in the same
+    # state.
+    def rollup
+      Digest::SHA256.hexdigest(entries.take(cutoff).flatten.join(" "))[0, 16]
+    end
+
+    # The wire format, as a Hash. What the manifest endpoint renders.
+    def to_h
+      {seq: seq, cutoff: cutoff, rollup: rollup, blocks: windowed_entries}
+    end
+
+    # The wire format, as JSON.
+    def to_json(*args)
+      to_h.to_json(*args)
+    end
+
+    # Which of the client's blocks disagree with ours, within the window.
+    #
+    # Blocks the client has and we do not are not reported: the server is the
+    # authority on what exists, and a stale block is removed by the morph rather
+    # than by a separate instruction.
+    def diff(client_digests)
+      client = client_digests.to_h
+
+      windowed_entries.filter_map do |id, digest|
+        id unless client[id] == digest
+      end
+    end
+
+    # True when the client's view of the history behind the window disagrees
+    # with ours, and the windowed diff is therefore not enough.
+    def stale_history?(client_rollup)
+      cutoff.positive? && client_rollup != rollup
+    end
+  end
+end