rouge-carve 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b5dd23d4f12415b4c19ed7333a5bd1555ae21e4f547b4f0a8f52b7829f8ad94
-  data.tar.gz: 1f9a6066c9c55b5457eda908909ff6b865d84fedaebbe18a37fdbe4f9e62a76d
+  metadata.gz: 300be9217acf879b361b092d5bd9f814858ba169272f98f730cdbfa86a2e75bf
+  data.tar.gz: 1b4e02989bd311b9d0423e1021250d5ddac1982420c15016d80600ddb82386b0
 SHA512:
-  metadata.gz: bdc875e2a50141600798ff0efcd8741e751516dbb3de632f55039a62e881f4edbe4561942e8b39228e6603c1e6cef428d5320323b2617213d5f8ceed1053e550
-  data.tar.gz: 8d1983500c85ecef17ac25bac9aad4ebf76b477bf75ee721d1343ef09a82cf424c86c4bf9cb65b948361183bf97cce2bffe90630f20fa519368d5253fed1f13c
+  metadata.gz: 63d049afb99f11ac50d7b64a2829c8fec7bc220b1501b621cbbe2e6a87160e75cc326bc94f3724f3280650e12e9be2ce0a00bb160f1c9012e717fc2d538916b9
+  data.tar.gz: d3dce00b0304ec63f85dc9e258f78e7c516d7a9048e1ac9e76e8745140c2931a8ee3e0de70abc7a7a804203b511707d901f7ecd44c152ad08bdaeaa3e22dd35d

data/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## [Unreleased]
 
+## [0.1.4] - 2026-10-23
+
+- `csv` and `tsv` fence bodies are highlighted: every field has its own token type and the separator is an operator, colored apart from the fields in the default styles (#25, #26, #27, #28).
+- A lone `+` is a continuation marker, tokenized as an operator, only where the engine reads one: at an open container's marker column with a non-list block below, or as the first block after an opener (#29, #30, #32). A prose line ending in `+` is not scoped (#31).
+- A quote-prefixed `+` needs one space after each `>`: `>+`, `>>+` and `>> +` are paragraph text (#33).
+- A quote opened on a list item line (`- > q`) scopes its `>`, and `- > +` is a marker when the next line starts a non-list block at column 0, at the line's column or at the owning quote's column (markup-carve/carve#3421). A stacked `> > +` takes its follower only at column 0 or the first quote's column (#34).
+- A `:` left of the `::` column of the definition list it would continue is paragraph text (markup-carve/carve#3458) (#34).
+
 ## [0.1.3] - 2026-10-14
 
 0.1.2 was tagged without its `RougeCarve::VERSION` bump, so it never reached RubyGems. 0.1.3 is the first published release with its change.

data/README.md

@@ -49,6 +49,7 @@ markup, symbol shortcodes, mentions, tags and the typographic runs.
 as Ruby, because that is what a reader expects. A ` ```=html ` raw block is not:
 its info string names an output format, not a language, so the payload stays
 opaque.
+CSV and TSV fences highlight separators, quoted fields, escaped quotes, and unquoted fields (the header record apart from the data rows).
 
 **Fence widths are respected.** A three-backtick line inside a four-backtick
 fence is content, not a closer. The width is closed over per block rather than

data/lib/rouge/lexers/carve.rb

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*- #
 # frozen_string_literal: true
 
+require_relative 'carve_continuation'
+
 module Rouge
   module Lexers
     # Carve is a lightweight markup language for documents and the web. Its
@@ -48,6 +50,74 @@ module Rouge
     #
     # Spec: https://markup-carve.github.io/carve/
     class Carve < RegexLexer
+      class DelimitedTable < RegexLexer
+        # Unquoted fields are Name::Variable (colored in the default Rouge
+        # styles, unlike a bare Name); the header record's unquoted fields are
+        # Name::Attribute.
+        start { @header = true }
+
+        def field_token
+          @header ? Name::Attribute : Name::Variable
+        end
+
+        state :root do
+          rule /"/ do
+            token Str
+            push :unquoted
+            push :quoted
+          end
+          rule /[,\t]/ do |m|
+            if m[0] == self.class::SEPARATOR
+              token Keyword
+            else
+              token field_token
+              push :unquoted
+            end
+          end
+          rule /[^,\t\n]+/ do
+            token field_token
+            push :unquoted
+          end
+          rule /\n/ do
+            token Text
+            @header = false
+          end
+        end
+
+        state :unquoted do
+          rule /[,\t]/ do |m|
+            if m[0] == self.class::SEPARATOR
+              token Keyword
+              pop!
+            else
+              token field_token
+            end
+          end
+          rule /[^,\t\n]+/ do
+            token field_token
+          end
+          rule /\n/ do
+            token Text
+            @header = false
+            pop!
+          end
+        end
+
+        state :quoted do
+          rule /""/, Str::Escape
+          rule /"/, Str, :pop!
+          rule /[^"]+/, Str
+        end
+      end
+
+      class CsvTable < DelimitedTable
+        SEPARATOR = ','.freeze
+      end
+
+      class TsvTable < DelimitedTable
+        SEPARATOR = "\t".freeze
+      end
+
       title 'Carve'
       desc 'Carve, a lightweight markup language for documents and the web'
 
@@ -110,8 +180,13 @@ module Rouge
       # is a word, not a marker.
       ORDERED = '(?:\\d+|[A-Za-z]|[ivxlcdm]+|[IVXLCDM]+)[.)]'
 
+      # The states read a column-0 `+` with a flush-left block below as an
+      # attachment. Whether a `+` is the marker is settled by CarveContinuation.
+      PLUS_NEXT = /(?=\n[^ \t\n])/.freeze
+
       # A list marker at the start of a line, as the list rules below read it.
       LIST_MARKER = "(?:[-*]|#{ORDERED}|\\.)(?:#{ATTRS})?(?:[ \\t]|$)"
+      OPENER_RUN = "(?:(?:[-*]|#{ORDERED}|\\.)(?:#{ATTRS})? +[ \\t]*|> )+"
 
       # A table row needs its closing pipe: `| not a table` is a paragraph.
       ROW_END = /(?=(?:\\.|[^\\\n])*\|[ \t]*(?:#{ATTRS})?[ \t]*$)/.freeze
@@ -199,6 +274,7 @@ module Rouge
       # enters its state through #open_block, which knows which of the two it
       # is running in.
       def self.block_openers(bol)
+        root_margin = bol == '^' ? '()' : '( )'
         proc do
           # A raw block: the `=FORMAT` info string routes the payload to that
           # output format verbatim. Emitted as one token including the `=`,
@@ -233,14 +309,14 @@ module Rouge
           end
 
           # A caption line attaches to the block above or below it.
-          rule %r/#{bol}(#{MARGIN})(\^)([ \t]+)/ do
+          rule %r/#{bol}(#{MARGIN})(\^)( +)(?=[ \t]*[^ \t\n])/ do
             groups Text, Punctuation, Text
             open_block :caption
           end
 
           # Headings. Carve has no setext form, so a `#` run is the only
           # spelling and a trailing `{...}` is NOT an attribute block here.
-          rule %r/#{bol}(#{MARGIN})(\#{1,6})([ \t]+)/ do
+          rule %r/#{bol}(#{MARGIN})(\#{1,6})( +)(?=[ \t]*[^ \t\n])/ do
             groups Text, Punctuation, Text
             open_block :heading
           end
@@ -257,7 +333,7 @@ module Rouge
           # definition ends with its line.
           rule %r/#{bol}(#{MARGIN})(\[\^)([^\]\n]+)(\]:)( )/ do
             groups Text, Punctuation, Name::Label, Punctuation, Text
-            open_block :defline
+            open_block(:defline, :defline, host: :footnote)
           end
           rule %r/#{bol}(#{MARGIN})(\*\[)([^\]\n]+)(\]:)( )/ do
             groups Text, Punctuation, Name::Entity, Punctuation, Text
@@ -270,47 +346,125 @@ module Rouge
 
           # A definition-list term (`::`) and its definition (`:`). A term
           # may wrap onto lazy lines, and a definition's text is an item's.
-          rule %r/#{bol}(#{MARGIN})(::)([ \t]+)/ do
+          rule %r/#{bol}(#{MARGIN})(::)( +)(?=[ \t]*[^ \t\n])/ do
             groups Text, Punctuation, Text
+            remember_host(:term)
             open_block :dlterm, :heading
           end
-          rule %r/#{bol}(#{MARGIN})(:)(?=[ \t])/ do
-            groups Text, Punctuation
+          rule %r/#{bol}#{root_margin}(:)( +[ \t]*)(\+)()$#{PLUS_NEXT}/ do
+            if definition_term_before?
+              groups Text, Punctuation, Text, Operator, Text
+            else
+              groups Text, Punctuation, Text, Text, Text
+            end
             open_block :listitem, :quoteline
           end
+          # Without a term above, `: b` is paragraph text (a lazy line too).
+          rule %r/#{bol}(#{MARGIN})(:)(?= +[ \t]*[^ \t\n])/ do
+            # the scan from the last definition is short; the term scan runs once
+            if (definition_before? || definition_term_before?) && !separator_below_term?
+              groups Text, Punctuation
+              term = [@term_stream, @term_pos, @term_quote_depth]
+              open_block :listitem, :quoteline
+              # the next `: d` may still belong to the same term
+              @term_stream, @term_pos, @term_quote_depth = term
+              @def_stream = @current_stream
+              @def_pos = @last_host_pos
+              @def_quote_depth = @term_quote_depth
+            else
+              groups Text, Text
+              open_block :paragraph
+            end
+          end
 
           # A blockquote marker must be followed by a space or end the line;
           # `>foo` and `>> x` are paragraphs, and `> > x` nests. What follows
           # a marker may open a block of its own, so the openers are tried
           # again there.
-          rule %r/#{bol}(#{MARGIN})(>)(?=[ \t]|$)/ do
+          rule %r/#{bol}(#{MARGIN})(>)(?= |$)/ do |m|
             groups Text, Punctuation
-            state?(:quotestart) ? goto(:quotestart) : push(:quotestart)
+            record_quote_depth(quote_depth_on_line + 1)
+            remember_host(:quote)
+            rooted = state?(:quotestart) ? m[1] == ' ' : m[1].empty?
+            if rooted
+              state?(:quotestart) ? goto(:quotestart) : push(:quotestart)
+            else
+              state?(:quotestart) ? goto(:quoteline) : push(:quoteline)
+            end
+          end
+
+          # A task can attach its first block on the marker line.
+          rule %r/#{bol}#{root_margin}([-*](?:#{ATTRS})?)( +[ \t]*)(\[[^\]\n]\])( +[ \t]*)(\+)()$#{PLUS_NEXT}/ do
+            groups Text, Punctuation, Text, Name::Constant, Text, Operator, Text
+            open_block :listitem, :quoteline
+          end
+
+          rule %r/#{bol}#{root_margin}(#{OPENER_RUN})(\+)()$#{PLUS_NEXT}/ do |m|
+            token Text, m[1]
+            scanner = StringScanner.new(m[2])
+            until scanner.eos?
+              if (marker = scanner.scan(/(?:#{ORDERED}|\.)(?:#{ATTRS})?/))
+                token Num::Integer, marker
+              elsif (marker = scanner.scan(/[-*](?:#{ATTRS})?|>/))
+                token Punctuation, marker
+              else
+                token Text, scanner.getch
+              end
+            end
+            depth = quote_depth_on_line
+            token(state?(:quotestart) && !quote_level_follows?(depth) ? Text : Operator, m[3])
+            token Text, m[4]
+            m[2].rstrip.end_with?('>') ? open_block(:quoteline) : open_block(:listitem, :quoteline)
+          end
+
+          # A quote opened on a list item's marker line (`- > q`): its `>` is a
+          # marker, and a `+` behind it may be the quote's first block
+          # (markup-carve/carve#3421).
+          rule %r/#{bol}(#{MARGIN})((?:(?:[-*](?:#{ATTRS})?(?: +\[[^\]\n]\])?|#{ORDERED}(?:#{ATTRS})?|\.(?:#{ATTRS})?) +)+)((?:> +)*>)(?:( )(\+)()$|(?= |$))/ do |m|
+            token Text, m[1]
+            scanner = StringScanner.new(m[2] + m[3])
+            until scanner.eos?
+              if (marker = scanner.scan(/(?:#{ORDERED}|\.)(?:#{ATTRS})?/))
+                token Num::Integer, marker
+              elsif (marker = scanner.scan(/\[[^\]\n]\]/))
+                token Name::Constant, marker
+              elsif (marker = scanner.scan(/[-*](?:#{ATTRS})?|>/))
+                token Punctuation, marker
+              else
+                token Text, scanner.scan(/ +/)
+              end
+            end
+            if m[5]
+              token Text, m[4]
+              # CarveContinuation settles whether this `+` is the marker.
+              token Operator, m[5]
+            end
+            open_block(:quoteline)
           end
 
           # Task items before plain bullets, so the state marker is its own
           # token. The state is any single character, not only a space or an
           # x: `[>]` is deferred and `[-]` is dropped. A `+` is not a bullet in
           # Carve; it continues a container.
-          rule %r/#{bol}(#{MARGIN})((?:[-*]|#{ORDERED})(?:#{ATTRS})?)([ \t]+)(\[[^\]\n]\])/ do
+          rule %r/#{bol}(#{MARGIN})((?:[-*]|#{ORDERED})(?:#{ATTRS})?)( +)(\[[^\]\n]\])(?= +[ \t]*[^ \t\n])/ do
             groups Text, Punctuation, Text, Name::Constant
             open_block :listitem, :quoteline
           end
 
           # Bullets. A run of markers on one line opens nested lists at once
           # (`- - A`), and attributes may be glued straight onto the marker.
-          rule %r/#{bol}(#{MARGIN})((?:[-*][ \t]+)*[-*](?:#{ATTRS})?)(?=[ \t]|$)/ do
+          rule %r/#{bol}(#{MARGIN})((?:[-*] +)*[-*](?:#{ATTRS})?)(?= +[ \t]*[^ \t\n])/ do
             groups Text, Punctuation
             open_block :listitem, :quoteline
           end
 
           # Ordered markers: numeric, alphabetic, roman, and the bare `.` that
           # continues the enclosing sequence.
-          rule %r/#{bol}(#{MARGIN})(#{ORDERED}(?:#{ATTRS})?)(?=[ \t]|$)/ do
+          rule %r/#{bol}(#{MARGIN})(#{ORDERED}(?:#{ATTRS})?)(?= +[ \t]*[^ \t\n])/ do
             groups Text, Num::Integer
             open_block :listitem, :quoteline
           end
-          rule %r/#{bol}(#{MARGIN})(\.(?:#{ATTRS})?)(?=[ \t]|$)/ do
+          rule %r/#{bol}(#{MARGIN})(\.(?:#{ATTRS})?)(?= +[ \t]*[^ \t\n])/ do
             groups Text, Num::Integer
             open_block :listitem, :quoteline
           end
@@ -371,6 +525,15 @@ module Rouge
           groups Text, Comment::Preproc, Comment
         end
 
+        rule %r/^(\+)()$#{PLUS_NEXT}/ do |m|
+          if continuation_host_before?
+            groups Operator, Text
+          else
+            token Text, m[0]
+            push :paragraph
+          end
+        end
+
         mixin :blockopeners
 
         # A standalone attribute block, which may span lines.
@@ -400,6 +563,9 @@ module Rouge
 
       state :paraline do
         rule(%r/^(?=[ \t]*(?:\n|\z))/) { pop! }
+        rule(%r/^(?=\+$#{PLUS_NEXT})/) do
+          continuation_host_before? ? pop! : continue_paragraph(:paragraph)
+        end
         rule(%r/^(?=#{MARGIN}(?:#{INTERRUPT}))/) { pop! }
         rule(%r/^(?=#{MARGIN}(`{3,}|~{3,}))/) { |m| fence_closed_below?(m[1]) ? pop! : continue_paragraph(:paragraph) }
         rule(%r/^(?=#{MARGIN}\^[ \t])/) { @lone_image ? pop! : continue_paragraph(:paragraph) }
@@ -422,6 +588,12 @@ module Rouge
 
         state line do
           rule(%r/^(?=[ \t]*(?:\n|\z))/) { pop! }
+          if name == :listitem
+            rule %r/^(\+)()$#{PLUS_NEXT}/ do
+              groups Operator, Text
+              goto name
+            end
+          end
           rule(%r/^(?=#{MARGIN}(?:#{INTERRUPT}|#{LIST_MARKER}|:[ \t]))/) { pop! }
           rule(%r/^(?=#{MARGIN}(`{3,}|~{3,}))/) { |m| fence_closed_below?(m[1]) ? pop! : goto(name) }
           rule(//) { goto name }
@@ -849,8 +1021,50 @@ module Rouge
         end
       end
 
+      # The lexer states cannot carry the open containers' marker columns, so
+      # the `+` scope is settled afterwards from a document-level pass: a `+`
+      # it names is the continuation operator, every other `+` the states
+      # gave Operator is text.
+      def stream_tokens(str, &block)
+        markers = CarveContinuation.markers(str).to_h { |offset| [offset, true] }
+        candidates = CarveContinuation.candidates(str).to_h { |offset| [offset, true] }
+        pos = 0
+        super(str) do |tok, val|
+          if val.include?('+') && candidates.any?
+            retag_plus(tok, val, pos, markers, candidates, &block)
+          else
+            yield tok, val
+          end
+          pos += val.length
+        end
+      end
+
       private
 
+      def retag_plus(tok, val, pos, markers, candidates)
+        run = +''
+        val.each_char.with_index do |ch, i|
+          scope = if ch != '+' || !candidates[pos + i] || opaque?(tok) then tok
+                  elsif markers[pos + i] then Operator
+                  elsif tok == Operator then Text
+                  else tok
+                  end
+          if scope != tok
+            yield tok, run unless run.empty?
+            run = +''
+            yield scope, ch
+          else
+            run << ch
+          end
+        end
+        yield tok, run unless run.empty?
+      end
+
+      # Comment and literal payloads keep their scope.
+      def opaque?(tok)
+        tok <= Comment || tok <= Literal
+      end
+
       # The token a state gives a character that opens nothing.
       PLAIN_TOKENS = {
         heading: Generic::Heading, quoteline: Generic::Emph,
@@ -1083,7 +1297,9 @@ module Rouge
       # opener that is a single token. After a blockquote marker the opener
       # replaces the quote's start state, and a single-token opener hands the
       # rest of the line to quoted text.
-      def open_block(name = nil, quoted = name || :quoteline)
+      def open_block(name = nil, quoted = name || :quoteline, host: nil)
+        kind = host || (name == :listitem ? :list : name == :quoteline ? :quote : nil)
+        remember_host(kind) if kind
         if state?(:quotestart)
           goto quoted
         elsif name
@@ -1091,6 +1307,193 @@ module Rouge
         end
       end
 
+      def remember_host(kind)
+        @host_stream = @current_stream
+        @last_host_pos = @current_stream.pos
+        @last_host_kind = kind
+        if kind == :term
+          @term_stream = @current_stream
+          @term_pos = @last_host_pos
+          @term_quote_depth = state?(:quotestart) ? quote_depth_on_line : 0
+        elsif kind != :quote
+          @term_pos = nil
+        end
+        @host_scan = StringScanner.new(@current_stream.string)
+        @host_scan.pos = @last_host_pos
+        @host_scan.scan(/[^\n]*\n/)
+        @host_scan_valid = true
+        @host_scan_fence = nil
+        @host_scan_previous = nil
+      end
+
+      def quote_depth_on_line
+        line_start = line_start_byte
+        @quote_line_start == line_start ? @quote_depth_current : 0
+      end
+
+      def record_quote_depth(depth)
+        @quote_line_start = line_start_byte
+        @quote_depth_current = depth
+      end
+
+      def line_start_byte
+        text = @current_stream.string
+        pos = @current_stream.pos
+        pos -= 1 while pos.positive? && text.getbyte(pos - 1) != 10
+        pos
+      end
+
+      def next_line_after_current
+        scan = StringScanner.new(@current_stream.string)
+        scan.pos = @current_stream.pos
+        return nil unless scan.scan(/[^\n]*\n/)
+
+        scan.scan(/[^\n]*/)
+      end
+
+      def quote_level_follows?(depth)
+        line = next_line_after_current
+        return false unless line
+        depth.times do
+          return false unless line.start_with?('> ')
+
+          line = line[2..]
+        end
+        !line.empty? && !line.match?(/\A[ \t]/)
+      end
+
+      def continuation_host_before?
+        return false unless @host_stream.equal?(@current_stream) && @last_host_pos
+        return false if @last_host_kind == :term
+        return false if @last_host_kind == :quote && !quote_block_follows?
+
+        return false unless @host_scan_valid
+
+        limit = @current_stream.pos
+        while (line = @host_scan.check(/[^\n]*\n/))
+          break if @host_scan.pos + line.bytesize > limit
+
+          @host_scan.scan(/[^\n]*\n/)
+          line = line.chomp("\n")
+          if @host_scan_fence
+            fence = @host_scan_fence
+            @host_scan_fence = nil if line.match?(/\A#{Regexp.escape(fence)}[#{Regexp.escape(fence[0])}]*[ \t]*\z/)
+          elsif line.match?(/\A[ \t]*\z/)
+            @host_scan_valid = false
+          elsif (opening = line.match(/\A(`{3,}|~{3,})/))
+            @host_scan_valid = false unless @host_scan_previous&.match?(/\A\+[ \t]*\z/)
+            @host_scan_fence = opening[1]
+          elsif line.match?(/\A(?:\#{1,6} +|::{1,}|\^ +|\||(?:[-*_][ \t]*){3,}|\[[^\]]+\]: )/)
+            @host_scan_valid = false unless @host_scan_previous&.match?(/\A\+[ \t]*\z/)
+          end
+          @host_scan_previous = line
+          return false unless @host_scan_valid
+        end
+        true
+      end
+
+      def definition_term_before?
+        return false unless @term_stream.equal?(@current_stream) && @term_pos
+        depth = state?(:quotestart) ? quote_depth_on_line : 0
+        return false unless depth == @term_quote_depth
+
+        between = @current_stream.string.byteslice(@term_pos...@current_stream.pos)
+        lines = between.split("\n", -1)
+        lines.shift
+        lines.pop
+        if depth.positive?
+          prefix = '> ' * depth
+          return false unless lines.all? { |line| line.empty? || line.start_with?(prefix) }
+
+          lines.map! { |line| line.empty? ? line : line[prefix.length..] }
+        end
+        return false if lines.count { |line| line.match?(/\A[ \t]*\z/) } > 1
+
+        lines.none? { |line| line.match?(/\A(?:\#{1,6} +|:{3,}|[-*] +|> |\||`{3,}|~{3,}|(?:[-*_][ \t]*){3,}\z)/) }
+      end
+
+      # A `:` left of the `::` column of every open term above it ends the
+      # definition list and is paragraph text (markup-carve/carve#3458).
+      def separator_below_term?
+        text = @current_stream.string
+        unless @separator_source.equal?(text)
+          @separator_source = text
+          @separator_bytes = text.b
+          @separator_below = {}
+        end
+        start = line_start_byte
+        depth = state?(:quotestart) ? quote_depth_on_line : 0
+        strip = lambda do |line|
+          depth.times { line = line.start_with?('> ') ? line[2..] : line.sub(/\A>\z/, '') }
+          line
+        end
+        indent = ->(line) { line[/\A[ \t]*/].gsub("\t", '    ').length }
+        column = indent.call(strip.call(text.byteslice(start...(@separator_bytes.index("\n", start) || text.bytesize))))
+        term = nil
+        below = nil
+        finish = start - 1
+        while finish.positive?
+          from = (@separator_bytes.rindex("\n", finish - 1) || -1) + 1
+          line = strip.call(text.byteslice(from...finish))
+          finish = from - 1
+          next if line.strip.empty?
+
+          if line.match?(/\A[ \t]*::[ \t]/)
+            term = [term, indent.call(line)].compact.min
+            break if term <= column
+          elsif line.match?(/\A[ \t]*:[ \t]/)
+            # An earlier separator at this column with no term between reads the same.
+            if term.nil? && indent.call(line) == column && @separator_below.key?(from)
+              below = @separator_below[from]
+              break
+            end
+          elsif term && indent.call(line) < term
+            break
+          end
+        end
+        below = !term.nil? && column < term if below.nil?
+        @separator_below[start] = below
+      end
+
+      # After a definition, only a heading, quote, table or div ends the list.
+      def definition_before?
+        return false unless @def_stream.equal?(@current_stream) && @def_pos
+        depth = state?(:quotestart) ? quote_depth_on_line : 0
+        return false unless depth == @def_quote_depth
+
+        lines = @current_stream.string.byteslice(@def_pos...@current_stream.pos).split("\n", -1)
+        lines.shift
+        lines.pop
+        prefix = '> ' * depth
+        blank = false
+        lines.all? do |line|
+          if line.empty? || line == prefix.rstrip
+            blank = true
+            next true
+          end
+          next false unless line.start_with?(prefix)
+
+          rest = line[prefix.length..]
+          # a flush-left block after a blank line, or one that interrupts, ends it
+          next false if blank && rest.match?(/\A[^ \t]/)
+
+          blank = false
+          !rest.match?(/\A(?:\#{1,6} +|:{3,}|> |\||::|`{3,}|~{3,}|(?:[-*_][ \t]*){3,}\z)/)
+        end
+      end
+
+      def quote_block_follows?
+        line = next_line_after_current
+        return false unless line
+        return false if line.empty? || line.match?(/\A[ \t]/)
+
+        while line.start_with?('> ')
+          line = line[2..]
+          return false if line.empty? || line.match?(/\A[ \t]/)
+        end
+        line != '>'
+      end
+
       def continue_paragraph(name)
         @lone_image = false
         goto name
@@ -1199,6 +1602,9 @@ module Rouge
       def fence_lexer(language)
         return nil if language.nil? || language.empty?
 
+        return CsvTable.new if language == 'csv'
+        return TsvTable.new if language == 'tsv'
+
         Lexer.find_fancy(language)
       rescue StandardError
         nil

data/lib/rouge/lexers/carve_continuation.rb

@@ -0,0 +1,357 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    # Decides which `+` characters of a Carve document are continuation
+    # markers (Part 9 §17 L3, CARVE-P9-031). The marker is relative: it sits at
+    # an open container's marker column, so the answer needs the container
+    # stack the line-based lexer states do not carry. This pass walks the
+    # document once and returns the character offsets of the markers.
+    #
+    # It is a reduced block model: list items, quotes, footnotes, definitions
+    # and fences, with lazy paragraph lines. Shapes it cannot place (tables,
+    # divs, attribute blocks) read as text.
+    class CarveContinuation
+      TAB = 4
+      ORDERED = /\A(?:\d+|[A-Za-z]|[ivxlcdm]+|[IVXLCDM]+)[.)]/.freeze
+      BULLET = /\A[-*](?=[ \t]|\z)/.freeze
+      FENCE = /\A(`{3,}|~{3,})/.freeze
+
+      Container = Struct.new(:kind, :col, :content, :quotes, keyword_init: true)
+
+      # A line whose only content after quote prefixes and openers is `+`:
+      # the shapes where the `+` is a marker or a refused marker.
+      CANDIDATE = /
+        \A(?:[ \t]|>(?=[ \t])|(?:[-*]|(?:\d+|[A-Za-z]|[ivxlcdm]+|[IVXLCDM]+)[.)])(?:\{[^}\n]*\})?(?=[ \t])
+        |\[[^\]\n]\](?=[ \t])|::?(?=[ \t])|\[\^[^\]\n]+\]:(?=[ ]))*\+[ \t]*\r?\z
+      /x.freeze
+
+      def self.markers(source)
+        new(source).markers
+      end
+
+      # Offsets of every `+` that stands where a marker could.
+      def self.candidates(source)
+        offset = 0
+        source.split("\n", -1).each_with_object([]) do |line, found|
+          found << offset + line.rindex('+') if line.match?(CANDIDATE)
+          offset += line.length + 1
+        end
+      end
+
+      def initialize(source)
+        @lines = source.split("\n", -1)
+        @markers = []
+      end
+
+      def markers
+        stack = []
+        fence = nil
+        para = nil
+        force = 0
+        offset = 0
+        blank = false
+        @lines.each_with_index do |raw, index|
+          line = raw.chomp("\r")
+          cols = columns(line)
+          matched, pos, quotes = match(stack, line, cols, force)
+          rest = line[pos..] || ''
+
+          if fence && (fence[:forced] || matched >= fence[:depth])
+            fence = nil if rest.match?(/\A[ \t]*#{Regexp.escape(fence[:char])}{#{fence[:width]},}[ \t]*\z/)
+            offset += raw.length + 1
+            force = 0
+            next
+          end
+          fence = nil
+
+          lone = rest.match(/\A([ \t]*)\+[ \t]*\z/)
+          attached = 0
+          if rest.match?(/\A[ \t]*\z/)
+            # a blank line ends every quote it does not prefix
+            stack.slice!(matched..)
+            para = nil
+          elsif lone
+            col = cols[pos + lone[1].length]
+            # A blank line above closes a description or a footnote for a `+`.
+            target = stack.rindex do |c|
+              c.col == col && c.quotes == quotes && c.kind != :term &&
+                !(blank && %i[definition note].include?(c.kind))
+            end
+            # Only an item refuses a list below it (the sibling rule).
+            if target && follower?(index + 1, quotes, col, quote: quotes.zero?,
+                                                         list: stack[target].kind != :item)
+              @markers << offset + pos + lone[1].length
+              stack.slice!((target + 1)..)
+              attached = stack.length
+              para = nil
+            else
+              stack.slice!(matched..) unless para
+              para ||= stack.length
+            end
+          elsif para && (!opener?(rest) || lazy_list?(stack, rest, quotes) || unclosed_fence?(rest, index))
+            # a lazy line keeps every container open
+          else
+            stack.slice!(matched..)
+            para, fence, plus = open_blocks(stack, line, cols, pos, quotes, para, index)
+            fence[:forced] = true if fence && force.positive?
+            if plus
+              @markers << offset + plus
+              attached = stack.length
+            end
+          end
+          force = attached
+          blank = rest.match?(/\A[ \t]*\z/)
+          offset += raw.length + 1
+        end
+        @markers
+      end
+
+      private
+
+      # The visual column of every character position, tabs expanded.
+      def columns(line)
+        cols = []
+        col = 0
+        line.each_char do |ch|
+          cols << col
+          col = ch == "\t" ? (col / TAB + 1) * TAB : col + 1
+        end
+        cols << col
+        cols
+      end
+
+      # How many containers the line continues, where its content starts, and
+      # how many quote prefixes it passed.
+      # The containers up to +force+ are the ones an attached line enters.
+      def match(stack, line, cols, force)
+        pos = 0
+        quotes = 0
+        matched = 0
+        stack.each do |c|
+          blank = line[pos..].to_s.match?(/\A[ \t]*\z/)
+          if c.kind == :quote
+            ws = line[pos..][/\A[ \t]*/]
+            at = pos + ws.length
+            unless line[at] == '>' && (line[at + 1].nil? || line[at + 1].match?(/[ \t]/))
+              break unless matched < force
+
+              matched += 1
+              next
+            end
+
+            pos = at + 1
+            pos += 1 if line[pos]&.match?(/[ \t]/)
+            quotes += 1
+          elsif blank
+            # a blank line keeps an item open
+          else
+            ws = line[pos..][/\A[ \t]*/]
+            if cols[pos + ws.length] >= c.content || matched < force
+              pos = advance(line, cols, pos, c.content)
+            else
+              break
+            end
+          end
+          matched += 1
+        end
+        [matched, pos, quotes]
+      end
+
+      # Move +pos+ over whitespace up to +col+.
+      def advance(line, cols, pos, col)
+        pos += 1 while line[pos]&.match?(/[ \t]/) && cols[pos + 1] <= col
+        pos
+      end
+
+      def opener?(rest)
+        text = rest.lstrip
+        text.match?(BULLET) || text.match?(ORDERED) && text.match?(/\A\S+[ \t]/) ||
+          text.start_with?('>', '|') || text.match?(FENCE) ||
+          text.match?(/\A(?:\#{1,6}[ \t]|:::|::?[ \t]|\[\^[^\]]+\]:[ ])/)
+      end
+
+      # A list marker cannot interrupt a paragraph unless it continues a list
+      # already open in the line's own quote context.
+      def lazy_list?(stack, rest, quotes)
+        text = rest.lstrip
+        return false unless list_marker?(text)
+
+        stack.none? { |c| c.kind == :item && c.quotes == quotes }
+      end
+
+      # Only a closed fence interrupts a paragraph.
+      def unclosed_fence?(rest, index)
+        f = rest.lstrip.match(FENCE)
+        return false unless f
+
+        @last_closer ||= {}
+        last = @last_closer[f[1]] ||= begin
+          closer = /\A[ \t>]*#{Regexp.escape(f[1][0])}{#{f[1].length},}[ \t]*\r?\z/
+          @lines.rindex { |l| l.match?(closer) } || -1
+        end
+        last <= index
+      end
+
+      def list_marker?(text)
+        text.match?(BULLET) || text.match?(/#{ORDERED.source}(?=[ \t]|\z)/)
+      end
+
+      # The block a line at +col+ opens is the one the marker attaches.
+      # Inside a quote, `> +` does not attach a quote right after the prefix.
+      def follower?(index, quotes, col, quote: true, list: false)
+        line = @lines[index]
+        return false if line.nil?
+
+        line = line.chomp("\r")
+        cols = columns(line)
+        pos = 0
+        quotes.times do
+          ws = line[pos..][/\A[ \t]*/]
+          return false unless line[pos + ws.length] == '>'
+
+          pos += ws.length + 1
+          pos += 1 if line[pos]&.match?(/[ \t]/)
+        end
+        ws = line[pos..][/\A[ \t]*/]
+        text = line[(pos + ws.length)..]
+        return false if text.nil? || text.empty?
+        return false unless cols[pos + ws.length] == col
+        return false if !quote && ws.empty? && text.start_with?('>')
+
+        list || !list_marker?(text)
+      end
+
+      # Push the containers a line opens. Returns the paragraph depth, an open
+      # fence and the offset of a first-block marker.
+      def open_blocks(stack, line, cols, pos, quotes, para, index)
+        base = pos
+        start = stack.length
+        first = true
+        last = nil
+        loop do
+          ws = line[pos..][/\A[ \t]*/]
+          at = pos + ws.length
+          text = line[at..]
+          col = cols[at]
+          # `>+`, `>>+` and `>> +` open no quote: each `>` needs a space after it.
+          if text.start_with?('>') && (text[1].nil? || text[1] == ' ')
+            break if first && at != base
+
+            stack << Container.new(kind: :quote, col: col, content: col + 2, quotes: quotes)
+            quotes += 1
+            pos = at + 1
+            pos += 1 if line[pos] == ' '
+            last = :quote
+          elsif (m = text.match(/\A(?:[-*]|#{ORDERED.source})(?:\{[^}\n]*\})?(?= |\z)/))
+            
+            after = at + m[0].length
+            gap = line[after..][/\A[ \t]*/]
+            content = gap.empty? || cols[after + gap.length] - cols[after] > 4 ? cols[after] + 1 : cols[after + gap.length]
+            # attributes glued to the marker take no columns
+            content -= cols[after] - cols[at + m[0].index('{').to_i] if m[0].include?('{')
+            stack << Container.new(kind: :item, col: col, content: content, quotes: quotes)
+            pos = after + gap.length
+            last = m[0].start_with?('-', '*') ? :bullet : :ordered
+          elsif last == :bullet && (m = text.match(/\A\[[^\]\n]\](?= )/))
+            pos = at + m[0].length
+            pos += line[pos..][/\A[ \t]*/].length
+            last = :task
+          elsif (first || last == :quote) && text.match?(/\A\[\^[^\]\n]+\]: /)
+            m = text.match(/\A\[\^[^\]\n]+\]: /)
+            stack << Container.new(kind: :note, col: col, content: col + 2, quotes: quotes)
+            pos = at + m[0].length
+            last = :note
+          elsif (first || last == :quote) && text.match?(/\A:: /)
+            @term_open = stack.length
+            stack << Container.new(kind: :term, col: col, content: col + 2, quotes: quotes)
+            return [nil, nil, nil]
+          elsif (first || last == :quote) && @term_open == stack.length && text.match?(/\A: /)
+            stack << Container.new(kind: :definition, col: col, content: col + 2, quotes: quotes)
+            pos = at + 2
+            last = :definition
+          else
+            break
+          end
+          first = false
+        end
+
+        ws = line[pos..][/\A[ \t]*/]
+        text = line[(pos + ws.length)..] || ''
+        @term_open = last == :definition ? stack.length - 1 : nil
+        if last && text.match?(/\A\+[ \t]*\z/)
+          case last
+          when :quote
+            hosted = (start...stack.length).find { |i| stack[i].kind == :item }
+            spaced = !hosted || !line[base...pos].match?(/\S[ \t]{2,}/)
+            return [nil, nil, pos] if spaced && ws.empty? && first_block_follower?(index + 1, stack, cols[base], hosted, start)
+          when :note
+            return [nil, nil, pos + ws.length] if first_block_follower?(index + 1, stack, cols[base])
+          else
+            return [nil, nil, pos + ws.length]
+          end
+        end
+        return [nil, nil, nil] if text.empty?
+
+        # at document level a fence opens at column 0 only
+        if (f = text.match(FENCE)) && (stack.any? || ws.empty? || pos != base)
+          return [nil, { char: f[1][0], width: f[1].length, depth: stack.length }, nil]
+        end
+
+        [stack.length, nil, nil]
+      end
+
+      # A first-block marker after a quote or a footnote attaches a non-list
+      # block at document column 0 or at the column where the line's openers
+      # began; right after an enclosing quote's prefix counts as column 0 of
+      # that quote. A line that re-enters the opened container is its text.
+      # A quote opened behind a list item on the same line (+hosted+, the
+      # stack index of the first item the line opens) reads the follower at
+      # its raw column: the containers from that item on are not entered, and
+      # the item's and the owning quote's columns are eligible too
+      # (markup-carve/carve#3421, corpus 649 to 656). Its markers are one space
+      # apart: `-  > +` is text.
+      def first_block_follower?(index, stack, base_col, hosted = nil, start = nil)
+        line = @lines[index]
+        return false if line.nil?
+
+        line = line.chomp("\r")
+        cols = columns(line)
+        pos = 0
+        reached = 0
+        after_quote = true
+        (hosted ? stack[0...hosted] : stack).each do |c|
+          ws = line[pos..][/\A[ \t]*/]
+          at = pos + ws.length
+          if c.kind == :quote
+            break unless line[at] == '>' && (line[at + 1].nil? || line[at + 1] == ' ')
+
+            pos = at + 1
+            pos += 1 if line[pos] == ' '
+            after_quote = true
+          else
+            break unless line[at] && cols[at] >= c.content
+
+            pos = advance(line, cols, pos, c.content)
+            after_quote = false
+          end
+          reached += 1
+        end
+        return false if !hosted && reached == stack.length
+
+        ws = line[pos..][/\A[ \t]*/]
+        at = pos + ws.length
+        text = line[at..] || ''
+        return false if text.empty? || list_marker?(text)
+        return false if hosted && text.match?(/\A\+[ \t]*\z/)
+
+        # Stacked quotes opened on one line (`> > +`) take the first one's column.
+        own = hosted || start.nil? ? stack.last.col : stack[start].col
+        return cols[at].zero? || cols[at] == own if reached.zero?
+
+        cols[at] == base_col || (after_quote && ws.empty?) || (hosted && [stack[hosted].col, stack.last.col].include?(cols[at]))
+      end
+    end
+  end
+end

data/lib/rouge-carve/version.rb

@@ -3,5 +3,5 @@
 # Kept apart from the entry file so the gemspec can read it without loading
 # Rouge, which is not installed yet when Bundler evaluates the gemspec.
 module RougeCarve
-  VERSION = '0.1.3'
+  VERSION = '0.1.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rouge-carve
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Mark Scherer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-10-14 00:00:00.000000000 Z
+date: 2026-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rouge
@@ -47,6 +47,7 @@ files:
 - lib/rouge-carve/version.rb
 - lib/rouge/demos/carve
 - lib/rouge/lexers/carve.rb
+- lib/rouge/lexers/carve_continuation.rb
 homepage: https://github.com/markup-carve/rouge-carve
 licenses:
 - MIT