coradoc-adoc 2.0.34 → 2.0.36

data/lib/coradoc/asciidoc/lint.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Coradoc
+  module AsciiDoc
+    # AsciiDoc lint rules (#165). Each rule checks the AsciiDoc model
+    # tree in a model-driven way — no text scanning. Rules self-register
+    # into Coradoc::Lint.registry when this file is required; adding a
+    # rule is adding one file under lint/ plus its require here.
+    #
+    # NOTE: a "list starts deeper than level 1" rule is structurally
+    # unobservable here — the parser canonicalizes depth markers
+    # (#285), so a lone '**' list already reads as level 1 on the
+    # model tree. That concern lives in the parser/serializer, not
+    # in a lint.
+    module Lint
+      require_relative 'lint/heading_levels'
+      require_relative 'lint/document_title'
+      require_relative 'lint/empty_section'
+      require_relative 'lint/duplicate_ids'
+
+      RULES = [HeadingLevels, DocumentTitle, EmptySection,
+               DuplicateIds].freeze
+
+      RULES.each { |rule| Coradoc::Lint.registry.register(rule) }
+    end
+  end
+end

data/lib/coradoc/asciidoc/model/text_element.rb

@@ -40,14 +40,27 @@ module Coradoc
           line_break.to_s.empty?
         end
 
+        # Escape whole runs of '*'/'_' that touch whitespace on either
+        # side. Run-aware two-pass scan — the lookaround + alternation
+        # + quantifier regex is modeled as polynomial-redos by CodeQL.
         def self.escape_keychars(string)
           subs = { '*' => '\*', '_' => '\_' }
-          string
-            .gsub(/((?<=\s)[*_]+)|[*_]+(?=\s)/) do |n|
-            n.chars.map do |char|
-              subs[char]
-            end.join
+          chars = string.chars
+          escaped = Array.new(chars.length, false)
+          i = 0
+          while i < chars.length
+            unless subs.key?(chars[i])
+              i += 1
+              next
+            end
+            j = i
+            j += 1 while j < chars.length && subs.key?(chars[j])
+            touches = (i.positive? && chars[i - 1].match?(/\s/)) ||
+                      (j < chars.length && chars[j].match?(/\s/))
+            (i...j).each { |k| escaped[k] = true } if touches
+            i = j
           end
+          chars.each_with_index.map { |ch, k| escaped[k] ? subs[ch] : ch }.join
         end
 
         # Get text content as a string. Handles the polymorphic shape of

data/lib/coradoc/asciidoc/parser/base.rb

@@ -85,17 +85,16 @@ module Coradoc
           warn e.parse_failure_cause.ascii_tree
         end
 
-        # The Ruby engine is the required surface for this grammar.
-        # parsanol 1.3.44 fixes capture WRITES across the bridge
-        # (parsanol-ruby#80 — verified with a continuation-style repro),
-        # but the native tree still diverges from the Ruby-engine tree
-        # on attribute-list constructs: 69 suite failures natively from
-        # transform rules receiving raw Hashes where models are
-        # expected (e.g. Image::AttributeExtractor.promote_positional).
-        # Drop the override when native/Ruby tree parity holds for this
-        # grammar.
-        def parse(string, **options)
-          super(string, **options, mode: options.fetch(:mode, :ruby))
+        # The compiled PARG artifact (grammar/coradoc-adoc.parg, the
+        # single source of truth) is the parse backend; the Ruby-DSL
+        # grammar defined by this class remains the parity reference
+        # (parg_parity_spec + engine_differential_spec assert the two
+        # produce identical trees). The artifact routes through
+        # parsanol's compiled-program fast lane (parsanol-ruby#162,
+        # 1.3.81) and parses native by default; the historical
+        # ruby-mode artifact divergence is fixed (#185, 1.3.82).
+        def parse(string, **)
+          GrammarBackend.parse(string, **)
         end
 
         def rule_dispatch(rule_name, *, **)
@@ -111,6 +110,24 @@ module Coradoc
         end
       end
 
+      # Lazily-loaded compiled-grammar backend (the shipped artifact).
+      module GrammarBackend
+        class << self
+          def parse(string, **)
+            TreeMaterializer.convert(artifact.parse('document', string, **))
+          end
+
+          private
+
+          def artifact
+            require 'parsanol/parg'
+            @artifact ||= Parsanol::PARG::Artifact.load(
+              File.expand_path('../../../../grammar/coradoc-adoc.artifact.json', __dir__)
+            )
+          end
+        end
+      end
+
       # Wrap every parser rule for Parsanol memoization. Must run after all
       # parser modules are included in Base so that instance_method(rule_name)
       # finds the methods defined by every module.

data/lib/coradoc/asciidoc/parser/citation.rb

@@ -22,6 +22,16 @@ module Coradoc
         def escaped_xref
           str('\\') >> str('<<').as(:text)
         end
+
+        # General AsciiDoc escape: \X (X punctuation, minus < for
+        # escaped_xref and \ itself) yields the literal character X
+        # without firing any inline rule (#281).
+        def escape_char
+          # Explicit class (all ASCII punctuation except < for
+          # escaped_xref and \ itself): the native engine does not
+          # compile && class intersections.
+          str('\\') >> match("[!\"#$%&'()*+,\\-./:;=?@\\[\\]^_`{|}~]").as(:text)
+        end
       end
     end
   end

data/lib/coradoc/asciidoc/parser/document_attributes.rb

@@ -19,7 +19,12 @@ module Coradoc
 
         def document_attribute
           str(':') >> attribute_name.as(:key) >> str(':') >>
-            space? >> (attribute_value | str('')).as(:value) >> (line_ending | eof?)
+            # attribute_value itself already matches empty (its
+            # zero-width branch), so the former `| str('')` fallback was
+            # dead — ordered choice commits to attribute_value's empty
+            # match first — and the VM's shadowed-alternative lint
+            # (parsanol 1.3.55+) correctly rejects it.
+            space? >> attribute_value.as(:value) >> (line_ending | eof?)
         end
       end
     end

data/lib/coradoc/asciidoc/parser/inline.rb

@@ -46,10 +46,14 @@ module Coradoc
         # highlight_unconstrained was previously inconsistent (excluded
         # newlines). All four are now uniform.
 
+        # Constrained markers delimit at boundaries only (#281): the
+        # opening marker must not follow a word char, the closing one
+        # must not precede one — `foo_bar_baz` stays plain text.
         def constrained_mark(marker, reject_paragraph_break: false, content: nil)
-          open_guard = str(marker) >> str(marker).absent?
+          open_guard = Parsanol::Atoms::Lookbehind.regex('[[:word:]]', positive: false) >>
+                       str(marker) >> str(marker).absent?
           content_rule = content || default_constrained_content(marker)
-          close_guard = str(marker) >> str(marker).absent?
+          close_guard = str(marker) >> str(marker).absent? >> match('[[:word:]]').absent?
           sequence = open_guard >> content_rule >> close_guard
           sequence >>= str("\n\n").absent? if reject_paragraph_break
           sequence
@@ -138,7 +142,7 @@ module Coradoc
         end
 
         def link
-          ((str('http').present? | str('https').present? | str('ftp').present?) >>
+          ((str('http') | str('https') | str('ftp')).present? >>
             match('[A-Za-z0-9_.\\-:/&?=+,%#~;]+').repeat(1).as(:path) >>
             (str('[') >> match('[^\\]]').repeat(1).as(:text) >> str(']')).maybe
           ).as(:link) |
@@ -201,19 +205,22 @@ module Coradoc
         end
 
         def inline_chars?
-          match('[\[*#_{<^~`]').present? |
-            typographic_quote.present? |
-            str('http').present? |
-            str('https').present? |
-            str('link:').present? |
-            str('image:').present? |
-            str('+++').present? |
-            str('pass:').present? |
-            term_type.present? |
-            str('footnote').present? |
-            stem_type.present? |
-            str('\\<<').present? |
-            hard_line_break_marker?
+          # Single lookahead over the whole dispatch alternation; the former
+          # per-branch .present? chain put zero-width branches in every
+          # non-last slot (shadowed-alternative lint, parsanol 1.3.55+).
+          (match('[\\\\\[*#_{<^~`]') |
+             typographic_quote |
+             str('http') |
+             str('https') |
+             str('link:') |
+             str('image:') |
+             str('+++') |
+             str('pass:') |
+             term_type |
+             str('footnote') |
+             stem_type |
+             str('\\<<') |
+             hard_line_break_marker?).present?
         end
 
         # AsciiDoc hard line break: a space followed by `+` at end of line,
@@ -221,8 +228,8 @@ module Coradoc
         # the enclosing paragraph/verse. Recognised ahead of `text_unformatted`
         # so the marker isn't swallowed as plain text.
         def hard_line_break_marker?
-          (str(' +') >> str("\n")).present? |
-            (str('\\') >> str("\n")).present?
+          ((str(' +') >> str("\n")) |
+             (str('\\') >> str("\n"))).present?
         end
 
         def hard_line_break
@@ -244,7 +251,7 @@ module Coradoc
           monospace_unconstrained monospace_constrained
           superscript subscript
           attribute_reference
-          escaped_xref cross_reference
+          escaped_xref escape_char cross_reference
           term_inline term_inline2
           footnote stem
           link inline_image

data/lib/coradoc/asciidoc/parser/list.rb

@@ -7,10 +7,20 @@ module Coradoc
       module List
         def list(nesting_level = 1)
           (
-          unordered_list(nesting_level) |
-             ordered_list(nesting_level) |
-             definition_list
-        ).as(:list)
+            unordered_list(nesting_level) |
+               ordered_list(nesting_level) |
+               definition_list
+          ).as(:list)
+        end
+
+        # A list item at ANY depth starts a list construct: a nested
+        # list (`* a` followed by `** b`) must stop the parent item's
+        # continuation text and attach as `nested` instead. Full items
+        # (not bare markers) so `**un**constrained` — no space after
+        # the marker — stays ordinary text.
+        def list_start
+          (1..5).map { |level| unordered_list(level) | ordered_list(level) }
+                .reduce(:|)
         end
 
         def list_continuation
@@ -133,6 +143,13 @@ module Coradoc
         end
 
         def ulist_item(nesting_level = 1)
+          # A bare marker line (`*` then newline) is an empty list
+          # item — without this arm it fell through to a continuation
+          # text line and corrupted the preceding item (#139).
+          empty_item = ulist_marker(nesting_level).as(:marker) >>
+                       str(' [[[').absent? >>
+                       line_ending.as(:lines)
+
           item = ulist_marker(nesting_level).as(:marker) >>
                  str(' [[[').absent? >>
                  match("\n").absent? >> space >>
@@ -145,7 +162,7 @@ module Coradoc
             item >>= (list_marker(nesting_level + 1).present? >>
                    list(nesting_level + 1)).repeat(0).as(:nested)
           end
-          ulist_marker(nesting_level).present? >> item.as(:list_item)
+          (empty_item | item).as(:list_item)
         end
 
         # Continuation lines of a list item's first paragraph. AsciiDoc
@@ -242,7 +259,7 @@ module Coradoc
           monospace_unconstrained monospace_constrained
           superscript subscript
           attribute_reference
-          escaped_xref cross_reference
+          escaped_xref escape_char cross_reference
           term_inline term_inline2
           footnote stem
           link inline_image
@@ -259,8 +276,8 @@ module Coradoc
         end
 
         def list_body_hard_break_marker?
-          (str(' +') >> str("\n")).present? |
-            (str('\\') >> str("\n")).present?
+          ((str(' +') >> str("\n")) |
+            (str('\\') >> str("\n"))).present?
         end
 
         # Recognizes line endings for a list body source line, including

data/lib/coradoc/asciidoc/parser/paragraph.rb

@@ -9,7 +9,7 @@ module Coradoc
             (attribute_list >> newline).absent? >>
             block_delimiter.absent? >>
             (str('|===') >> newline).absent? >>
-            list.absent? >>
+            list_start.absent? >>
             list_prefix.absent? >>
             list_continuation.absent? >>
             element_id.absent? >>
@@ -49,7 +49,9 @@ module Coradoc
               (paragraph_text_line(true).repeat(1, 1) >>
                    (newline.repeat(1).as(:line_break) | eof?)).repeat(0, 1))
             ).as(:lines) >>
-            (newline.repeat(0) | eof?)
+            # the former "| eof?" was dead: newline.repeat(0) already
+            # matches empty (shadowed-alternative lint, parsanol 1.3.55+)
+            newline.repeat(0)
           ).as(:paragraph)
         end
 

data/lib/coradoc/asciidoc/parser/positioned_string.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'parsanol'
+
+module Coradoc
+  module AsciiDoc
+    module Parser
+      # An immutable String carrying its byte offset and source input.
+      #
+      # The native engine hands pooled, mutable Parsanol::Slice objects
+      # back in the parse tree; storing those in document models is
+      # unsafe (the pool resets them) and they fail serializer type
+      # checks. TreeMaterializer converts every Slice into a
+      # PositionedString at the GrammarBackend boundary, so models only
+      # ever see Strings — while SourceLineExtractor can still recover
+      # positions (line_and_column delegates the computation to a
+      # throwaway Slice).
+      class PositionedString < String
+        attr_reader :byte_position, :source_input
+
+        def initialize(content, byte_position: 0, source_input: nil)
+          super(content)
+          @byte_position = byte_position
+          @source_input = source_input
+        end
+
+        def self.from_slice(slice)
+          new(slice.to_s, byte_position: slice.offset, source_input: slice.input)
+        end
+
+        def line_and_column
+          raise ArgumentError, 'Line/column requires input' unless @source_input
+
+          @line_and_column ||= Parsanol::Slice.new(@byte_position, to_s,
+                                                   @source_input).line_and_column
+        end
+      end
+    end
+  end
+end

data/lib/coradoc/asciidoc/parser/rule_dispatcher.rb

@@ -47,7 +47,7 @@ module Coradoc
               end
               cache[key] = rule_name
             end
-            parser_instance.public_send(cache[key])
+            TreeMaterializer.convert(parser_instance.public_send(cache[key]))
           end
 
           private
@@ -59,7 +59,8 @@ module Coradoc
           # Wrapping `__send__` in particular causes infinite recursion.
           # @return [Hash{Symbol => Array<Module>}]
           def collect_rule_names
-            parser_constants = Parser.constants - %i[Base Cache FixFiles RuleDispatcher]
+            parser_constants = Parser.constants - %i[Base Cache FixFiles RuleDispatcher
+                                                     PositionedString TreeMaterializer]
             parser_constants.each_with_object({}) do |const, acc|
               Parser.const_get(const).instance_methods(false).each do |name|
                 acc[name] ||= []

data/lib/coradoc/asciidoc/parser/tree_materializer.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'parsanol'
+
+module Coradoc
+  module AsciiDoc
+    module Parser
+      # Converts a native-engine parse tree: every
+      # Parsanol::Slice becomes an immutable PositionedString. Runs at
+      # the GrammarBackend boundary so the transformer and document
+      # models never see pooled slices, without losing source
+      # positions.
+      module TreeMaterializer
+        module_function
+
+        def convert(node)
+          case node
+          when Parsanol::Slice then PositionedString.from_slice(node)
+          when Hash then node.transform_values { |value| convert(value) }
+          when Array then node.map { |value| convert(value) }
+          else
+            node
+          end
+        end
+      end
+    end
+  end
+end

data/lib/coradoc/asciidoc/parser.rb

@@ -7,6 +7,8 @@ module Coradoc
       autoload :Cache, "#{__dir__}/parser/cache"
       autoload :FrontmatterParser, "#{__dir__}/parser/frontmatter_parser"
       autoload :Inline, "#{__dir__}/parser/inline"
+      autoload :PositionedString, "#{__dir__}/parser/positioned_string"
+      autoload :TreeMaterializer, "#{__dir__}/parser/tree_materializer"
     end
   end
 end

data/lib/coradoc/asciidoc/serializer/serializers/block/source_code.rb

@@ -8,7 +8,9 @@ module Coradoc
           class SourceCode < Core
             def to_adoc(model, _options = {})
               @model = model
-              "\n\n#{gen_anchor}[source,#{model.lang}]\n#{gen_delimiter}\n" <<
+              lang = model.lang.to_s.strip
+              header = lang.empty? ? '[source]' : "[source,#{lang}]"
+              "\n\n#{gen_anchor}#{header}\n#{gen_delimiter}\n" <<
                 gen_lines << "\n#{gen_delimiter}\n\n"
             end
           end

data/lib/coradoc/asciidoc/serializer/serializers/list/item.rb

@@ -87,6 +87,10 @@ module Coradoc
                 "+\n#{elem.to_adoc}"
               end.join
               nest = model.nested.nil? || (model.nested.is_a?(Array) && model.nested.empty?) ? '' : model.nested.to_adoc
+              # The nested list serializer opens with a block separator;
+              # adjacent to the item line it would read as a detached
+              # sibling list on re-parse.
+              nest = nest.sub(/\A\n+/, '')
               out = " #{_anchor}#{out}#{model.line_break}"
               out + attach + nest
             end

data/lib/coradoc/asciidoc/serializer/serializers/table.rb

@@ -5,14 +5,20 @@ module Coradoc
     module Serializer
       module Serializers
         class Table < Base
-          def to_adoc(model, _options = {})
+          def to_adoc(model, options_or_context = {})
             @model = model
+            context = normalize_context(options_or_context)
             _anchor = @model.anchor.nil? ? '' : "#{serialize_child(@model.anchor)}\n"
             _attrs = @model.attrs.to_s.empty? ? '' : "#{serialize_child(@model.attrs)}\n"
             _title = @model.title.nil? || @model.title.empty? ? '' : ".#{serialize_child(@model.title)}\n"
             _content = @model.rows.map { |row| serialize_child(row) }.join
 
-            "\n\n#{_anchor}#{_attrs}#{_title}|===\n#{_content}|===\n"
+            # Blank-line separation after the table so a following
+            # block never glues onto the closer (#151) — same
+            # convention as the paragraph serializer, suppressed for
+            # the document's last element.
+            separator = context.last_element ? '' : "\n"
+            "\n\n#{_anchor}#{_attrs}#{_title}|===\n#{_content}|===\n#{separator}"
           end
         end
       end

data/lib/coradoc/asciidoc/serializer/serializers/text_element.rb

@@ -9,7 +9,9 @@ module Coradoc
           def to_adoc(model, _options = {})
             result = ''
             result += "[[#{model.id}]] " if model.id
-            result += serialize_content(model.content)
+            content = serialize_content(model.content)
+            content = Coradoc::Util::AsciiDoc.escape_text_delimiters(content) if content.is_a?(String)
+            result += content
             result += model.line_break.to_s
             result
           end

data/lib/coradoc/asciidoc/transform/from_core_model/list_building.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Coradoc
+  module AsciiDoc
+    module Transform
+      # List construction for the AsciiDoc view: nested lists flatten
+      # into the item stream with depth markers so AsciiDoc reads them
+      # as nested rather than sibling lists (#68) — a marker-type
+      # change alone signals nesting; same-type nesting goes one level
+      # deeper.
+      module ListBuilding
+        def transform_list(list, depth = 1)
+          items = []
+          Array(list.items).each do |item|
+            nested_blocks = [item.nested_list,
+                             *Array(item.children).grep(CoreModel::ListBlock)].compact
+            text = inline_text(item)
+            attached = attached_blocks(item)
+            if text.empty? && attached.empty? && nested_blocks.any?
+              # A wrapper item whose only content is its nested list
+              # cannot render as a deeper list in AsciiDoc (lists start
+              # at level 1, #68), so splice the nested list at this depth.
+              nested_blocks.each do |child|
+                child_depth = child.marker_type == list.marker_type ? depth : 1
+                items.concat(transform_list(child, child_depth).items)
+              end
+              next
+            end
+            items << Coradoc::AsciiDoc::Model::List::Item.new(
+              content: text,
+              marker: item.marker || (default_marker(list.marker_type) * depth),
+              attached: attached
+            )
+            nested_blocks.each do |child|
+              child_depth = child.marker_type == list.marker_type ? depth + 1 : 1
+              items.concat(transform_list(child, child_depth).items)
+            end
+          end
+
+          case list.marker_type
+          when 'ordered'
+            Coradoc::AsciiDoc::Model::List::Ordered.new(items: items)
+          when 'definition'
+            Coradoc::AsciiDoc::Model::List::Definition.new(items: items)
+          else
+            Coradoc::AsciiDoc::Model::List::Unordered.new(items: items)
+          end
+        end
+
+        # Inline children render on the item line; CoreModel::Block
+        # children become attached blocks (`+` continuations) instead of
+        # being glued into the item text.
+        def inline_text(item)
+          case (rc = item.renderable_content)
+          when String then rc
+          when Array
+            rc.each_with_object(+'') do |child, out|
+              next if child.is_a?(CoreModel::Block) || child.is_a?(CoreModel::ListBlock)
+
+              out << case child
+                     when CoreModel::TextContent then child.text.to_s
+                     when String then child
+                     when CoreModel::Base then child.flat_text.to_s
+                     end
+            end
+          else
+            ''
+          end
+        end
+
+        def attached_blocks(item)
+          Array(item.children)
+            .grep(CoreModel::Block)
+            .flat_map { |block| FromCoreModel.flatten_children([block]) }
+        end
+
+        def default_marker(marker_type)
+          case marker_type
+          when 'ordered' then '.'
+          else '*'
+          end
+        end
+      end
+    end
+  end
+end