maquina_remend 0.1.0

data/lib/maquina_remend/handlers/app_tags.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module MaquinaRemend
+  module Handlers
+    # Closes an application tag whose closing tag has not streamed in yet.
+    #
+    # Models emit XML-ish tags that carry application meaning rather than
+    # presentation — `<thinking>`, `<answer>`, `<tool_call>`, `<citation>`,
+    # `<scratchpad>`. MaquinaRemend::Handlers::HtmlTags already drops one that
+    # is still half-typed. This handler covers the other half of the problem: a
+    # *complete* opening tag whose closer is still minutes of tokens away.
+    #
+    # Enabled by `app_tags:`, whose value is the list of tag names to close;
+    # MaquinaRemend::APP_TAGS by default, `false` to switch the handler off.
+    #
+    # ```ruby
+    # MaquinaRemend.call("<thinking>\nI am partway thro")
+    # # => "<thinking>\nI am partway thro</thinking>"
+    # ```
+    #
+    # ## Why an unclosed app tag is a broken document
+    #
+    # CommonMark ends an HTML block at the first blank line (spec §4.6,
+    # condition 7). An unclosed `<thinking>` therefore stops containing the
+    # stream as soon as the model types a blank line — measured with
+    # commonmarker 2.10:
+    #
+    # ```
+    # "<thinking>\nreasoning\n\nmore\n\nDone."
+    # #  <thinking>
+    # #  reasoning
+    # #  <p>more</p>
+    # #  <p>Done.</p>
+    # ```
+    #
+    # The paragraphs escape the tag in the AST, while in the browser's DOM they
+    # are swallowed by it — an unknown element is never implicitly closed, so
+    # every later block nests inside `<thinking>` and the whole answer renders
+    # as reasoning. Closing the tag makes the two agree.
+    #
+    # ## Where the closer goes
+    #
+    # Placement is not cosmetic; the wrong separator produces
+    # `<p></thinking></p>`, which nests the closer inside a paragraph:
+    #
+    # | Innermost open tag | Appended | Renders as |
+    # |---|---|---|
+    # | in the current paragraph | `</thinking>` flush | one balanced HTML block |
+    # | before the last blank line | a blank line, then one closer per line | a closing HTML block per tag |
+    #
+    # Nested tags are closed innermost first. A tag inside a fenced code block
+    # or an inline code span is content, not markup: those spans are masked out
+    # by MaquinaRemend::Scanner before the stack is built, so the fence wins.
+    # A self-closing `<citation id="1"/>` opens nothing and is skipped. While
+    # the tail is inside an *unterminated* fence nothing is repaired at all, this
+    # handler included: the tag stays open until the fence closes, because a
+    # closer appended there would land inside the code block.
+    #
+    # ## Why it runs last
+    #
+    # It appends past the end of every other repair. Running it earlier would
+    # let MaquinaRemend::Handlers::Emphasis close a bold run *after* the closing
+    # tag — `<thinking>\n**bold\n</thinking>**` — putting the emphasis outside
+    # the element it belongs to.
+    class AppTags < Base
+      # Any complete tag: group 1 is the closing slash, group 2 the name,
+      # group 3 the self-closing slash. Attributes are skipped over rather than
+      # parsed; a tag whose `>` has not arrived is not matched at all, because
+      # MaquinaRemend::Handlers::HtmlTags has already dropped it.
+      TAG = %r{<(/?)([A-Za-z][A-Za-z0-9_:-]*)(?:\s[^<>]*?)?(/?)>}
+
+      private
+        def option_key = :app_tags
+
+        def names
+          @names ||= Array(options[:app_tags]).map { |name| name.to_s.downcase }
+        end
+
+        def repair(text, scanner)
+          return text if names.empty?
+
+          stack = unclosed(scanner)
+          return text if stack.empty?
+
+          candidate = text + closers(stack, scanner)
+          stable?(candidate) ? candidate : text
+        end
+
+        # The closers land past the end of every other repair, so the *next*
+        # pass reads them as content: a buffer ending in a lone `*` has nothing
+        # to emphasise until `</answer>` is appended after it, and then the pass
+        # after that emphasises the closing tag. Rather than guess which handler
+        # is at risk, ask them. If replaying the rest of the pipeline over the
+        # closed buffer would change it, the tag is left open for this one frame
+        # and closed on the next, when the tail has moved on — an unstable
+        # repair is worse than a tag that is closed a character late.
+        #
+        # Custom handlers are left out of the probe: they are the host's to keep
+        # idempotent, and one that rewrites unconditionally would otherwise
+        # switch this repair off for good.
+        def stable?(candidate)
+          Pipeline.new(**options.merge(app_tags: false, handlers: [])).call(candidate) == candidate
+        end
+
+        # The tags still open at the end of the buffer, outermost first, each as
+        # a `[name, offset]` pair. Built over the masked text so that a tag
+        # written inside a fence or a code span never opens anything.
+        def unclosed(scanner)
+          stack = []
+
+          scanner.masked_text.to_enum(:scan, TAG).each do
+            match = Regexp.last_match
+            name = match[2].downcase
+            next unless names.include?(name)
+
+            if match[1].empty?
+              stack << [name, match.begin(0)] if match[3].empty?
+            elsif (index = stack.rindex { |open, _| open == name })
+              stack.slice!(index..)
+            end
+          end
+
+          stack
+        end
+
+        # The text to append. Flush against the tail when the innermost open tag
+        # is in the same block as the tail, and otherwise after a blank line,
+        # one closer per line — a line holding two closing tags is not an HTML
+        # block and would be wrapped in a paragraph instead.
+        def closers(stack, scanner)
+          tags = stack.reverse.map { |name, _| "</#{name}>" }
+          return tags.join if stack.last[1] >= scanner.paragraph_offset
+
+          "\n" * [2 - scanner.text[/\n*\z/].length, 0].max + tags.join("\n")
+        end
+    end
+  end
+end

data/lib/maquina_remend/handlers/base.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module MaquinaRemend
+  # One class per repair. Each is listed against the option that switches it on
+  # in MaquinaRemend::DEFAULTS, and each runs in the fixed order given by
+  # MaquinaRemend::Pipeline::BUILT_INS.
+  #
+  # A *custom* handler does not live here and does not inherit from
+  # MaquinaRemend::Handlers::Base — it only has to respond to
+  # `#call(text, context)`. See MaquinaRemend.call for that contract. This
+  # namespace is documented because reading a built-in is the fastest way to
+  # see what a careful repair looks like.
+  module Handlers
+    # A handler sees the buffer and the context, and returns a buffer. It rescans
+    # rather than trusting an earlier handler's view of the text, because an
+    # earlier handler may have appended to the tail it is about to read.
+    #
+    # ## Writing one
+    #
+    # A built-in overrides two private methods:
+    #
+    # * `#option_key` — the MaquinaRemend::DEFAULTS key that enables it. A
+    #   handler covering more than one option overrides `#enabled?` instead, as
+    #   MaquinaRemend::Handlers::Links and MaquinaRemend::Handlers::Math do.
+    # * `#repair(text, scanner)` — the repair itself, returning the new buffer
+    #   or `text` unchanged. It is only reached when the handler is enabled and
+    #   the tail is not inside a fence; #call has already checked both.
+    #
+    # Two private helpers are available to a subclass: `#streaming_tail?`, true
+    # when the buffer's last line has not been terminated by a newline, and
+    # `#escape_matches`, which backslash-escapes a pattern in the paragraph
+    # without ever touching a character inside inline code or math.
+    #
+    # None of this applies to a host's own handler, which is a plain object with
+    # a two-argument `#call`; MaquinaRemend.call has that contract.
+    class Base
+      # The resolved option hash the pipeline was built with.
+      attr_reader :options
+
+      # * `options` — the resolved hash from MaquinaRemend::Pipeline#options,
+      #   not a partial one. Handlers read their own flags out of it.
+      def initialize(options)
+        @options = options
+      end
+
+      # Returns the buffer, repaired or untouched.
+      #
+      # Returns `text` unchanged when this handler's option is off, or when the
+      # tail sits inside a fenced code block — the built-ins never complete
+      # anything inside a fence. Otherwise the work is delegated to the
+      # subclass's `#repair`.
+      #
+      # The scanner is passed in when the pipeline already has a current one:
+      # scanning is the expensive part, and most handlers leave the text alone.
+      # Custom handlers keep the two-argument contract documented on
+      # MaquinaRemend.call.
+      def call(text, context, scanner = nil)
+        return text unless enabled?
+        return text if context.in_code_fence?
+
+        repair(text, scanner || Scanner.new(text))
+      end
+
+      private
+        # Which option switches this handler off. Handlers covering more than one
+        # option override #enabled? instead.
+        def option_key
+          raise NotImplementedError
+        end
+
+        def enabled?
+          options.fetch(option_key, false)
+        end
+
+        def repair(text, _scanner)
+          text
+        end
+
+        # True when the buffer's last line is still arriving, rather than having
+        # been terminated by a newline. Repairs that would rewrite a complete
+        # document belong behind this.
+        def streaming_tail?(text)
+          !text.end_with?("\n")
+        end
+
+        # Backslash-escapes every match of `pattern` in the paragraph.
+        #
+        # Matches are located in the masked paragraph and applied to the raw one,
+        # so a character inside inline code or math is never escaped: the mask
+        # blanks those spans but preserves their length, which keeps the offsets
+        # usable against the original text.
+        def escape_matches(scanner, pattern, skip_line: ->(_line) { false })
+          raw = scanner.paragraph.dup
+          offsets = []
+          line_offset = 0
+
+          scanner.masked_paragraph.lines.each do |line|
+            unless skip_line.call(line)
+              line.to_enum(:scan, pattern).each { offsets << line_offset + Regexp.last_match.begin(0) }
+            end
+            line_offset += line.length
+          end
+
+          offsets.reverse_each { |index| raw.insert(index, "\\") }
+          scanner.prefix + raw
+        end
+    end
+  end
+end

data/lib/maquina_remend/handlers/comparison_operators.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module MaquinaRemend
+  module Handlers
+    # Case 27. A ">" used as a comparison inside a line is escaped, so it cannot
+    # be mistaken for a blockquote marker when the line is re-parsed. A ">" that
+    # genuinely starts a line is left alone: that one really is a blockquote.
+    #
+    # Enabled by `comparison_operators:`, on by default.
+    #
+    # ```ruby
+    # MaquinaRemend.call("- a > b")  # => "- a \\> b"
+    # MaquinaRemend.call("> quoted") # => "> quoted"
+    # ```
+    class ComparisonOperators < Base
+      # A `>` surrounded by whitespace and not already escaped — a comparison,
+      # not a blockquote marker.
+      COMPARISON = /(?<=\s)(?<!\\)>(?=\s)/
+      # A line that genuinely starts with a blockquote marker. Lines matching
+      # this are skipped entirely.
+      BLOCKQUOTE_LINE = /\A {0,3}>/
+
+      private
+        def option_key = :comparison_operators
+
+        def repair(_text, scanner)
+          escape_matches(scanner, COMPARISON, skip_line: ->(line) { line.match?(BLOCKQUOTE_LINE) })
+        end
+    end
+  end
+end

data/lib/maquina_remend/handlers/dangling_escape.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module MaquinaRemend
+  module Handlers
+    # Case 14. A backslash at the very end of the buffer is half a token — the
+    # start of "\frac", "\neq", "\*" — and the rest of it has not arrived.
+    #
+    # It has to go before anything else is appended, because a backslash escapes
+    # whatever lands next: closing block math after a dangling backslash yields
+    # "\$$", an escaped dollar that leaves the math still open, so the next pass
+    # appends another "$$", and the next, and the next. Found by replaying real
+    # streamed model output, not by reading the spec.
+    #
+    # Only at end of buffer. A backslash at the end of a *line* is a hard line
+    # break and is left exactly where it is.
+    #
+    # Enabled by `dangling_escape:`, on by default. Runs first in
+    # MaquinaRemend::Pipeline::BUILT_INS.
+    #
+    # ```ruby
+    # MaquinaRemend.call("$$x = \\frac{-b \\") # => "$$x = \\frac{-b $$"
+    # ```
+    class DanglingEscape < Base
+      # The run of backslashes at the very end of the buffer. The run is
+      # captured whole because its parity decides everything: an even run is a
+      # complete escaped backslash, an odd one has a dangling half-token.
+      TRAILING_BACKSLASHES = /(\\+)\z/
+
+      private
+        def option_key = :dangling_escape
+
+        def repair(text, _scanner)
+          return text unless streaming_tail?(text)
+
+          match = TRAILING_BACKSLASHES.match(text)
+          return text unless match
+          return text if match[1].length.even? # an escaped backslash is complete
+
+          text[0...-1]
+        end
+    end
+  end
+end

data/lib/maquina_remend/handlers/emphasis.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module MaquinaRemend
+  module Handlers
+    # Cases 01-04. Closes emphasis runs the stream has opened but not finished.
+    #
+    # Delimiters are matched with a stack rather than counted, so nesting works:
+    # "**bold *italic" leaves two openers and gets "*" then "**" appended, in
+    # that order. Underscores between word characters are skipped entirely —
+    # they are identifiers, not emphasis (case 24).
+    #
+    # Enabled by `italic:` for one-character runs, `bold:` for two and
+    # `bold_italic:` for three or more; any one of the three switches the
+    # handler on, and each run is then checked against the flag for its own
+    # length.
+    #
+    # ```ruby
+    # MaquinaRemend.call("**bold *italic") # => "**bold *italic***"
+    # MaquinaRemend.call("some_var_name")  # => "some_var_name"
+    # ```
+    class Emphasis < Base
+      # One delimiter run: a `*` or `_` and every repeat of the same character
+      # immediately after it.
+      RUN = /(?<char>[*_])\k<char>*/
+
+      # Run length to the option that governs it. Runs longer than three are
+      # clamped to three, so `****` is governed by `bold_italic:`.
+      LENGTH_OPTIONS = {1 => :italic, 2 => :bold, 3 => :bold_italic}.freeze
+
+      private
+        def enabled?
+          LENGTH_OPTIONS.values.any? { |key| options[key] }
+        end
+
+        def repair(text, scanner)
+          openers = unclosed_runs(scanner.masked_paragraph)
+          return text if openers.empty?
+
+          text + openers.reverse.map { |run| completion_for(run) }.join
+        end
+
+        def completion_for(run)
+          run
+        end
+
+        def unclosed_runs(paragraph)
+          stack = []
+
+          paragraph.to_enum(:scan, RUN).each do
+            match = Regexp.last_match
+            run = match[0]
+            next if skip?(paragraph, match)
+            next unless completable?(run)
+
+            if closes?(stack, paragraph, match) || closing_in_progress?(stack, paragraph, match)
+              stack.pop
+            elsif opens?(paragraph, match)
+              stack.push(run)
+            end
+          end
+
+          stack
+        end
+
+        def completable?(run)
+          key = LENGTH_OPTIONS[[run.length, 3].min]
+          options.fetch(key, false)
+        end
+
+        # Underscores inside a word never open or close emphasis.
+        def skip?(paragraph, match)
+          return false unless match[0].start_with?("_")
+
+          word_char?(preceding(paragraph, match)) && word_char?(following(paragraph, match))
+        end
+
+        def opens?(paragraph, match)
+          char = following(paragraph, match)
+          !char.nil? && !char.match?(/\s/)
+        end
+
+        # A delimiter run sitting at the very end of the buffer is the closer
+        # being typed, not a new opener: "**Best Use **" is a bold run whose
+        # closer arrived with a space in front of it. Appending another "**"
+        # would both mangle the text and break idempotence, since the appended
+        # run would itself dangle on the next pass.
+        def closing_in_progress?(stack, paragraph, match)
+          return false if stack.empty?
+          return false unless paragraph[match.end(0)..].to_s.match?(/\A\s*\z/)
+
+          stack.last[0] == match[0][0]
+        end
+
+        def closes?(stack, paragraph, match)
+          return false if stack.empty?
+
+          char = preceding(paragraph, match)
+          !char.nil? && !char.match?(/\s/) && stack.last[0] == match[0][0]
+        end
+
+        def preceding(paragraph, match)
+          index = match.begin(0) - 1
+          index.negative? ? nil : paragraph[index]
+        end
+
+        def following(paragraph, match)
+          paragraph[match.end(0)]
+        end
+
+        def word_char?(char)
+          !char.nil? && char.match?(/[[:alnum:]]/)
+        end
+    end
+  end
+end

data/lib/maquina_remend/handlers/html_tags.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module MaquinaRemend
+  module Handlers
+    # Case 13. A tag whose ">" has not arrived yet is dropped, along with the
+    # whitespace in front of it. Leaving it in place makes the renderer swallow
+    # everything that streams in afterwards as attribute soup.
+    #
+    # Enabled by `html_tags:`, on by default.
+    #
+    # ```ruby
+    # MaquinaRemend.call("text <div cla") # => "text"
+    # ```
+    class HtmlTags < Base
+      # An HTML tag opened at the tail of the buffer whose `>` has not arrived.
+      TRUNCATED_TAG = %r{<[a-zA-Z/][^<>]*\z}
+
+      private
+        def option_key = :html_tags
+
+        def repair(text, scanner)
+          return text unless streaming_tail?(text)
+
+          paragraph = scanner.masked_paragraph
+          match = TRUNCATED_TAG.match(paragraph)
+          return text unless match
+
+          scanner.prefix + scanner.paragraph[0...match.begin(0)].rstrip
+        end
+    end
+  end
+end

data/lib/maquina_remend/handlers/inline_code.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module MaquinaRemend
+  module Handlers
+    # Case 05. An odd number of backticks in the paragraph means a code span is
+    # still open; close it so the renderer does not treat the rest of the
+    # message as code.
+    #
+    # Enabled by `inline_code:`, on by default. Appends one backtick, so
+    # `` `code `` becomes `` `code` ``.
+    class InlineCode < Base
+      private
+        def option_key = :inline_code
+
+        def repair(text, scanner)
+          scanner.inline_code_open? ? text + "`" : text
+        end
+    end
+  end
+end

data/lib/maquina_remend/handlers/links.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module MaquinaRemend
+  module Handlers
+    # Cases 07-09. A link or image whose label or destination is still arriving.
+    #
+    # `link_mode: :protocol` closes the syntax and points the destination at a
+    # placeholder. A half-streamed destination is never kept: "https://exa" is a
+    # perfectly valid URL to the wrong host, and model output is untrusted.
+    #
+    # `link_mode: :text_only` unwraps the whole thing to its label, which is the
+    # right choice for a host that would rather show no link than a dead one.
+    #
+    # Enabled by `links:` for `[label](url)` and by `images:` for `![alt](url)` —
+    # either flag switches the handler on, and each match is then checked
+    # against the flag for its own kind. Shaped by `link_mode:`; see
+    # MaquinaRemend::DEFAULTS.
+    #
+    # ```ruby
+    # MaquinaRemend.call("[label")                # => "[label]()"
+    # MaquinaRemend.call("[label](https://exa")   # => "[label](#)"
+    # MaquinaRemend.call("![alt", images: false)  # => "![alt"
+    # ```
+    #
+    # Runs before MaquinaRemend::Handlers::Emphasis, for the reason
+    # MaquinaRemend::Pipeline::BUILT_INS gives.
+    class Links < Base
+      # What a half-streamed destination is replaced by under
+      # `link_mode: :protocol`.
+      PLACEHOLDER = "#"
+
+      # A link or image whose label is still being typed: `[label` or `![alt`,
+      # running to the end of the buffer.
+      LABEL_OPEN = /(?<image>!)?\[(?<label>[^\[\]]*)\z/
+      # A link or image whose label closed but whose destination is still
+      # arriving: `[label](https://exa`.
+      DESTINATION_OPEN = /(?<image>!)?\[(?<label>[^\[\]]*)\]\((?<destination>[^()\s]*)\z/
+
+      private
+        def enabled? = options[:links] || options[:images]
+
+        def repair(text, scanner)
+          paragraph = scanner.masked_paragraph
+
+          if (match = DESTINATION_OPEN.match(paragraph))
+            close_destination(text, match)
+          elsif (match = LABEL_OPEN.match(paragraph))
+            close_label(text, match)
+          else
+            text
+          end
+        end
+
+        def close_destination(text, match)
+          return text unless allowed?(match)
+
+          if text_only?
+            text[0...-match[0].length] + match[:label]
+          else
+            text[0...-match[:destination].length] + PLACEHOLDER + ")"
+          end
+        end
+
+        def close_label(text, match)
+          return text unless allowed?(match)
+
+          text_only? ? text[0...-match[0].length] + match[:label] : text + "]()"
+        end
+
+        def allowed?(match)
+          match[:image] ? options[:images] : options[:links]
+        end
+
+        def text_only? = options[:link_mode] == :text_only
+    end
+  end
+end

data/lib/maquina_remend/handlers/math.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module MaquinaRemend
+  module Handlers
+    # Cases 10 and 11. Block math is closed by default; inline math is not,
+    # because "$5 and $10" is currency far more often than it is mathematics
+    # (case 26). Turning inline_math on is a decision about a corpus, and only
+    # the host knows its corpus.
+    #
+    # Enabled by `block_math:` (on) for `$$...$$` and by `inline_math:` (off)
+    # for `$...$`.
+    #
+    # ```ruby
+    # MaquinaRemend.call("$$x = 1")                     # => "$$x = 1$$"
+    # MaquinaRemend.call("costs $5 and $10")            # => "costs $5 and $10"
+    # MaquinaRemend.call("$x = 1", inline_math: true)   # => "$x = 1$"
+    # ```
+    class Math < Base
+      private
+        def enabled? = options[:block_math] || options[:inline_math]
+
+        def repair(text, scanner)
+          text = close_block(text, scanner)
+          close_inline(text, Scanner.new(text))
+        end
+
+        def close_block(text, scanner)
+          return text unless options[:block_math]
+
+          scanner.math_open? ? text + "$$" : text
+        end
+
+        def close_inline(text, scanner)
+          return text unless options[:inline_math]
+
+          paragraph = scanner.masked_paragraph.gsub("$$", "")
+          paragraph.scan(/(?<!\\)\$/).length.odd? ? text + "$" : text
+        end
+    end
+  end
+end

data/lib/maquina_remend/handlers/setext_heading.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module MaquinaRemend
+  module Handlers
+    # Case 12. A setext underline that is still arriving — "Title\n=" — already
+    # parses as a heading, so the repair is not about the block type. It is
+    # about the underline being visibly half-drawn while the rest of it streams
+    # in: pad it to the width of the heading text and it stops growing on screen.
+    #
+    # Only fires on a tail that is still streaming. A buffer that ended with a
+    # newline has a complete underline, and rewriting it would break the
+    # byte-identical passthrough that case 28 asserts.
+    #
+    # Enabled by `setext_headings:`, on by default.
+    #
+    # ```ruby
+    # MaquinaRemend.call("Title\n=") # => "Title\n====="
+    # ```
+    class SetextHeading < Base
+      # A setext underline on its own line: up to three spaces of indent, then
+      # a run of `=` or `-`, then nothing but trailing whitespace.
+      UNDERLINE = /\A( {0,3})(=+|-+)[ \t]*\z/
+
+      private
+        def option_key = :setext_headings
+
+        def repair(text, _scanner)
+          return text unless streaming_tail?(text)
+
+          lines = text.lines
+          return text if lines.length < 2
+
+          underline = lines.last.chomp
+          title = lines[-2].chomp
+          return text unless (match = UNDERLINE.match(underline))
+          return text if title.strip.empty?
+          return text if match[2].length >= title.strip.length
+
+          lines[-1] = match[1] + match[2][0] * title.strip.length
+          lines.join
+        end
+    end
+  end
+end