maquina_stream 0.1.0

data/lib/maquina_stream/renderer/tag_blocks.rb

@@ -0,0 +1,276 @@
+# frozen_string_literal: true
+
+module MaquinaStream
+  class Renderer
+    # Gives a **registered** app-meaning tag its own HTML block, by putting
+    # blank lines around its opening and closing tags before commonmarker ever
+    # sees the buffer.
+    #
+    # ```ruby
+    # MaquinaStream::Renderer::TagBlocks.call(markdown, names: MaquinaStream.tags.keys)
+    # ```
+    #
+    # ## The bug this exists for
+    #
+    # CommonMark ends an HTML block at the first blank line (spec §4.6,
+    # condition 7). A tag a model wrote across several paragraphs therefore has
+    # its closing tag emitted *inside a paragraph*:
+    #
+    # ```
+    # <thinking>\nFirst para.\n\nSecond para.\n</thinking>\n\nAfter the tag.
+    # #  <thinking>
+    # #  First para.
+    # #  <p>Second para.<br />\n</thinking></p>
+    # #  <p>After the tag.</p>
+    # ```
+    #
+    # The HTML5 parser sees a stray end tag, ignores it, and never closes
+    # `<thinking>` — so the rest of the message is parsed *inside* it and the
+    # host's partial is handed content the model wrote after the tag closed.
+    # That is a content leak, not a cosmetic defect: the partial is where the
+    # host says "this is the model's private reasoning".
+    #
+    # With a blank line after the opener and before the closer, each tag is its
+    # own HTML block, the element closes where the model closed it, and the
+    # paragraphs between them are ordinary markdown.
+    #
+    # ## Why it normalises markdown rather than HTML
+    #
+    # Rebalancing end tags in the commonmarker output would mean hand-editing an
+    # HTML string that is model output — the one thing this engine never does.
+    # This pass inserts newlines into the markdown and changes nothing else;
+    # the trust boundary stays exactly where it was, with the sanitizer running
+    # last over parsed HTML.
+    #
+    # ## What it will not touch
+    #
+    # It is a privilege boundary, and it is written to be boring:
+    #
+    # * **Only registered names.** `names:` is what the host passed to
+    #   `MaquinaStream.register_tag`. An unregistered `<script>`, `<iframe>` or
+    #   `<img>` is not matched, not moved and not re-parsed. With an empty
+    #   registry the input is returned byte for byte.
+    # * **Code wins.** Matching runs over MaquinaRemend::Scanner#masked_text,
+    #   which blanks fenced code blocks and balanced inline code spans while
+    #   preserving offsets, so a `<thinking>` inside a fence or a backtick span
+    #   is text and stays text. Indented code is covered by the indent rule
+    #   below rather than by the scanner, which does not track it, and the raw
+    #   HTML regions the scanner does not model either — comments, `<script>`,
+    #   `<pre>` and friends — are masked here by RAW_REGIONS.
+    # * **Complete tags only.** A match is a whole open or close tag on one
+    #   line, with CommonMark's attribute grammar — quoted values may contain
+    #   `>`, and a tag broken across a newline is not a tag. `<thinkingXYZ>`
+    #   does not match `thinking`; `<thinking/>` opens nothing.
+    # * **Block position only.** The opening tag must *begin its line* under
+    #   four columns of indent, which is the shape that starts an HTML block and
+    #   therefore the shape that leaks. Anything deeper may be indented code or
+    #   list content and is left alone; an inline `<citation>…</citation>` in
+    #   the middle of a sentence is left alone because it already works, and is
+    #   what the registry was built for.
+    # * **Pairs only.** An opener with no closer, or a closer with no opener,
+    #   inserts nothing. Nesting is matched innermost-first, as HTML does it.
+    #
+    # Insertion is idempotent: a buffer that already has the blank lines comes
+    # back byte-identical, which is what makes the pass safe to run on every
+    # frame of a stream.
+    class TagBlocks
+      # An HTML block opener may be indented up to three columns; the fourth
+      # makes it indented code.
+      MAX_INDENT = 3
+
+      # An indent that still starts an HTML block, and one that is deep enough
+      # to be indented code or list content instead.
+      BLOCK_INDENT = /\A {0,#{MAX_INDENT}}\z/
+      CODE_INDENT = /\A[ \t]{#{MAX_INDENT + 1},}\z/
+
+      # CommonMark's attribute grammar (spec §6.6), restricted to spaces and
+      # tabs. A tag whose attributes wrap onto a second line is not a complete
+      # tag for HTML-block purposes, so it must not be one for us either.
+      ATTRIBUTE = /[ \t]+[a-zA-Z_:][a-zA-Z0-9_.:-]*(?:[ \t]*=[ \t]*(?:[^ \t"'=<>`]+|'[^']*'|"[^"]*"))?/
+
+      # The raw regions CommonMark reads to a closing marker rather than to a
+      # blank line: HTML block types 1 through 5 — `<script>`, `<pre>`,
+      # `<style>` and `<textarea>`, comments, processing instructions,
+      # declarations and CDATA.
+      #
+      # MaquinaRemend::Scanner masks fences and inline code, which is the
+      # markdown half of "this is text, not markup"; it does not model these,
+      # because no repair it makes has ever needed to. They matter here for one
+      # reason: a blank line inserted inside one of them **ends it early**, and
+      # text the model had buried in a comment or a `<script>` — text the
+      # sanitizer would have dropped whole — comes back out as live markdown.
+      # Masking them means a registered name written inside one is never a tag.
+      #
+      # An unterminated region masks to the end of the buffer, which is the
+      # conservative answer while a message is still streaming.
+      RAW_REGIONS = [
+        /<!--.*?(?:-->|\z)/m,
+        /<!\[CDATA\[.*?(?:\]\]>|\z)/m,
+        /<\?.*?(?:\?>|\z)/m,
+        /<![A-Za-z].*?(?:>|\z)/m,
+        %r{<(script|pre|style|textarea)\b.*?(?:</\1\s*>|\z)}mi
+      ].freeze
+
+      class << self
+        # Normalises `markdown` for the given tag `names` and returns it.
+        # Returns the argument unchanged when nothing is registered, when no
+        # registered tag appears in block position, or when the blank lines are
+        # already there.
+        def call(markdown, names: MaquinaStream.tags.keys)
+          new(names: names).call(markdown)
+        end
+      end
+
+      # The registered tag names this instance will normalise, downcased.
+      attr_reader :names
+
+      def initialize(names:)
+        @names = Array(names).map { |name| name.to_s.downcase }.reject(&:empty?).uniq
+      end
+
+      def call(markdown)
+        return markdown if markdown.nil? || markdown.empty? || names.empty?
+
+        insertions = plan(markdown.to_s)
+        return markdown if insertions.empty?
+
+        splice(markdown.to_s, insertions)
+      end
+
+      private
+        # Longest first, so `<answer>` cannot be matched inside `<answerable>`
+        # by a shorter alternative winning the alternation.
+        def pattern
+          @pattern ||= begin
+            alternation = names.sort_by { |name| -name.length }.map { |name| Regexp.escape(name) }.join("|")
+            /<(\/?)(#{alternation})(?![a-zA-Z0-9_:-])(#{ATTRIBUTE}*)[ \t]*(\/?)>/i
+          end
+        end
+
+        # `[position, text]` pairs, one per newline this pass wants to insert.
+        def plan(text)
+          pairs(text).flat_map { |open, close| insertions_for(text, open, close) }.compact
+        end
+
+        # Matched open/close ranges, innermost first, over the masked buffer so
+        # that code context is honoured. Mirrors the stack in
+        # MaquinaRemend::Handlers::AppTags rather than inventing a second one.
+        def pairs(text)
+          masked = mask_raw_regions(MaquinaRemend::Scanner.new(text).masked_text)
+          stack = []
+
+          masked.to_enum(:scan, pattern).each_with_object([]) do |_, found|
+            match = Regexp.last_match
+            name = match[2].downcase
+            range = match.begin(0)...match.end(0)
+
+            if match[1].empty?
+              # A self-closing `<citation id="1"/>` opens nothing.
+              stack << [name, range] if match[4].empty?
+            elsif (index = stack.rindex { |open_name, _| open_name == name })
+              found << [stack[index][1], range]
+              stack.slice!(index..)
+            end
+          end
+        end
+
+        # Blanks every RAW_REGIONS span, keeping the buffer's length so that
+        # match offsets are still offsets into the original text. Same trick,
+        # and the same masking character, as MaquinaRemend::Scanner.
+        def mask_raw_regions(masked)
+          RAW_REGIONS.reduce(masked) do |text, pattern|
+            text.gsub(pattern) { MaquinaRemend::Scanner::MASK * ::Regexp.last_match(0).length }
+          end
+        end
+
+        # Four questions per pair, each answering with one newline or with
+        # nothing: is there a blank line above the opener, below the opener,
+        # above the closer, below the closer. A pair that is not in block
+        # position, or whose closer sits in what could be indented code, is
+        # skipped entirely.
+        def insertions_for(text, open, close)
+          return [] unless block_positioned?(text, open)
+          return [] unless closer_placeable?(text, close)
+
+          [
+            blank_line_before(text, open),
+            blank_line_after(text, open),
+            blank_line_before(text, close),
+            blank_line_after(text, close)
+          ]
+        end
+
+        # A registered tag that **begins a line** under four columns of indent is
+        # the host's block-level component, and is the only shape this pass
+        # touches. Anything else is either already correct — an inline
+        # `<citation>…</citation>` in the middle of a sentence renders fine and
+        # is left exactly as the model wrote it — or ambiguous, because four
+        # columns could be indented code or list continuation, and ambiguous
+        # means untouched.
+        def block_positioned?(text, range)
+          text[line_start(text, range.first)...range.first].match?(BLOCK_INDENT)
+        end
+
+        # The closer may carry prose in front of it — that is the common case,
+        # `Second para.</thinking>` — but four columns of leading whitespace
+        # could be indented code, and a repair there would corrupt it.
+        def closer_placeable?(text, range)
+          indent = text[line_start(text, range.first)...range.first]
+
+          !indent.match?(CODE_INDENT)
+        end
+
+        # A blank line above the tag. Prose in front of it on the same line is
+        # pushed down instead — `Second para.</thinking>` becomes two blocks —
+        # and a tag that already has a blank line above it is left alone, which
+        # is what makes the pass idempotent.
+        def blank_line_before(text, range)
+          start = line_start(text, range.first)
+          head = text[start...range.first]
+
+          return [range.first, "\n\n"] unless head.match?(/\A[ \t]*\z/)
+          return nil if start.zero? || text[0...start].match?(/(?:\A|\n)[ \t]*\n\z/)
+
+          [start, "\n"]
+        end
+
+        # A blank line below the tag, by the same rules. It matters on the
+        # closer as much as on the opener: without it the text after
+        # `</thinking>` continues the closer's HTML block and is emitted as raw
+        # HTML instead of being rendered as the markdown the model wrote.
+        def blank_line_after(text, range)
+          tail = rest_of_line(text, range.last)
+
+          return [range.last, "\n\n"] unless tail.match?(/\A[ \t]*\z/)
+
+          following = text[(range.last + tail.length + 1)..]
+          return nil if following.nil? || following.match?(/\A[ \t]*(?:\n|\z)/)
+
+          [range.last, "\n"]
+        end
+
+        # `rindex` reads a negative start as an offset from the end of the
+        # string, so offset 0 has to answer for itself rather than search.
+        def line_start(text, offset)
+          return 0 if offset.zero?
+
+          (text.rindex("\n", offset - 1) || -1) + 1
+        end
+
+        def rest_of_line(text, offset)
+          text[offset...(text.index("\n", offset) || text.length)].to_s
+        end
+
+        # Two pairs can ask for the same newline — a closer and the next
+        # opener, say. Each position is written once, with the longest text
+        # asked for there, so the pass stays idempotent.
+        def splice(text, insertions)
+          merged = insertions.group_by(&:first).transform_values { |group| group.map(&:last).max_by(&:length) }
+
+          merged.keys.sort.reverse_each.with_object(+text) do |position, buffer|
+            buffer.insert(position, merged[position])
+          end
+        end
+    end
+  end
+end

data/lib/maquina_stream/renderer/view_context.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module MaquinaStream
+  class Renderer
+    # Renders engine partials without a request.
+    #
+    # The renderer is a pure function, but components are ERB — so it builds its
+    # own view context from the engine's view paths rather than borrowing a
+    # controller's. "No request context" is the constraint; ActionView is not
+    # request context.
+    class ViewContext
+      class MissingActionView < Error; end
+
+      def self.build
+        @build ||= new
+      end
+
+      # Templates are looked up once per process, so a component change in
+      # development needs a reload — the same trade every engine partial makes.
+      def self.reset!
+        @build = nil
+      end
+
+      def render(partial, locals = {})
+        view.render(partial: partial, locals: locals)
+      end
+
+      private
+        def view
+          @view ||= begin
+            require_action_view!
+
+            lookup = ActionView::LookupContext.new(view_paths)
+            view = ActionView::Base.with_empty_template_cache.new(lookup, {}, nil)
+            # Components render through the seam, and the seam is a helper.
+            view.extend(components_helper)
+            view
+          end
+        end
+
+        def view_paths
+          if defined?(ActionController::Base)
+            ActionController::Base.view_paths
+          else
+            [File.expand_path("../../../app/views", __dir__)]
+          end
+        end
+
+        # The helper lives in app/helpers and is normally autoloaded by the
+        # engine. Requiring it directly is what lets the renderer work in a
+        # plain Ruby process with ActionView and no Rails application - which is
+        # the whole claim the pure-function constraint makes.
+        def components_helper
+          unless defined?(MaquinaStream::ComponentsHelper)
+            require File.expand_path("../../../app/helpers/maquina_stream/components_helper", __dir__)
+          end
+
+          MaquinaStream::ComponentsHelper
+        end
+
+        def require_action_view!
+          return if defined?(ActionView::Base)
+
+          raise MissingActionView, <<~MESSAGE
+            MaquinaStream::Renderer needs ActionView to render component partials.
+            It does not need a request, a controller or a running server — but it
+            is an engine, and its components are ERB.
+          MESSAGE
+        end
+    end
+  end
+end

data/lib/maquina_stream/renderer.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "json"
+require "commonmarker"
+require "nokogiri"
+require "maquina_remend"
+
+require_relative "renderer/view_context"
+require_relative "renderer/fence"
+require_relative "renderer/post_pass"
+require_relative "renderer/tag_blocks"
+
+module MaquinaStream
+  # Markdown in, sanitized HTML out.
+  #
+  # ```ruby
+  # MaquinaStream::Renderer.call(markdown, mode: :streaming) # => SafeBuffer
+  # ```
+  #
+  # Pure: no request, no controller, no stubbing. The same function serves the
+  # live stream, a page reload, a replay and an export, and `mode` changes
+  # nothing but whether the reveal attributes are emitted. If this ever starts
+  # needing request context, that is a design error rather than a plumbing one.
+  #
+  # ## The pipeline
+  #
+  # 1. `MaquinaRemend.call` repairs the unterminated markdown a half-written
+  #    buffer always ends in — an open bold run, an unclosed fence.
+  # 2. Renderer::TagBlocks puts blank lines around the opening and closing tags
+  #    of a *registered* tag, so a block-level `<thinking>` is its own HTML
+  #    block rather than a closing tag stranded inside a paragraph.
+  # 3. Commonmarker parses it, with source positions.
+  # 4. Renderer::PostPass rewrites elements: fences, custom tags, registered
+  #    element overrides, text direction, block indices.
+  # 5. Sanitizer runs, unconditionally, last.
+  #
+  # Renderer returns one HTML string for the whole buffer. It does **not**
+  # split it into blocks or stamp ids and digests on them — that is Document,
+  # and without it a page cannot be repaired at all. Host code almost always
+  # wants MaquinaStream.render or Document rather than this.
+  class Renderer
+    # The two render modes. Both produce the same document byte for byte; the
+    # mode is carried so the post-pass knows whether the message is still being
+    # written.
+    MODES = %i[streaming static].freeze
+
+    # unsafe: true lets registered custom tags through the parser. It is not a
+    # relaxation: the sanitizer is the gate, it runs unconditionally, and it runs
+    # last. Model output is assumed hostile at every step before it.
+    COMMONMARKER_OPTIONS = {
+      parse: {sourcepos_chars: true},
+      render: {sourcepos: true, unsafe: true, github_pre_lang: false},
+      extension: {
+        table: true,
+        strikethrough: true,
+        autolink: true,
+        tasklist: true,
+        footnotes: true
+      }
+    }.freeze
+
+    # commonmarker highlights with syntect by default, which would ship a second
+    # highlighter's inline styles into a pipeline that already owns highlighting
+    # (Rouge, at fence close only) and forbids inline colour. Confirmed against
+    # commonmarker 2.10.0 in test/sourcepos_test.rb.
+    COMMONMARKER_PLUGINS = {syntax_highlighter: nil}.freeze
+
+    # The mode this renderer was built with, one of MODES.
+    attr_reader :mode
+
+    # The Configuration this renderer reads.
+    attr_reader :config
+
+    # Renders `markdown` in one call. The usual entry point.
+    #
+    # `mode:` is `:streaming` or `:static`; anything else raises ArgumentError.
+    # `config:` defaults to the global MaquinaStream.config, and is a keyword
+    # rather than a lookup so the renderer stays callable from a plain Ruby
+    # process with no Rails around it.
+    def self.call(markdown, mode: :streaming, config: MaquinaStream.config)
+      new(mode: mode, config: config).call(markdown)
+    end
+
+    # The buffer as commonmarker will see it: repaired, then normalised so a
+    # registered block-level tag stands on its own. Document parses this rather
+    # than the raw buffer, so its source positions are the ones the rendered
+    # document was built from — two parses of two different strings do not line
+    # up, and the block ids are derived from the alignment.
+    #
+    # Byte-identical to `MaquinaRemend.call(markdown)` when no tag is
+    # registered.
+    def self.prepare(markdown)
+      TagBlocks.call(MaquinaRemend.call(markdown))
+    end
+
+    # Builds a reusable renderer. Raises ArgumentError unless `mode:` is one of
+    # MODES.
+    def initialize(mode: :streaming, config: MaquinaStream.config)
+      raise ArgumentError, "mode must be one of #{MODES.join(", ")}" unless MODES.include?(mode)
+
+      @mode = mode
+      @config = config
+    end
+
+    # Renders one markdown string to sanitized HTML.
+    #
+    # Returns an `html_safe` String — a plain String when ActiveSupport is not
+    # loaded — and an empty one for nil or whitespace-only input.
+    def call(markdown)
+      return safe("") if markdown.nil? || markdown.strip.empty?
+
+      prepared = self.class.prepare(markdown)
+      parsed = Commonmarker.to_html(prepared, options: COMMONMARKER_OPTIONS, plugins: COMMONMARKER_PLUGINS)
+      fragment = Nokogiri::HTML5.fragment(parsed)
+
+      PostPass.new(fragment, markdown: prepared, mode: mode, config: config).call
+
+      safe(Sanitizer.call(fragment.to_html, config: config))
+    end
+
+    private
+      # ActiveSupport is present inside the engine, but the renderer is expected
+      # to run in a plain process too, so the wrapper degrades to a String.
+      def safe(html)
+        html.respond_to?(:html_safe) ? html.html_safe : html
+      end
+  end
+end