polytypo 1.17.1 → 1.17.3

data/lib/polytypo/modes/markdown.rb

@@ -287,12 +287,108 @@ module Polytypo
         parser_source = normalize_lone_cr(mask_frontmatter(source, block))
 
         spans = []
+        state = { stack: [], frontmatter_end: block.nil? ? 0 : block.end, labels: [], url_ends: Set.new }
         ParseError.wrap do
-          doc = Commonmarker.parse(parser_source, options: { parse: { sourcepos_chars: true } })
-          walk(doc, chars, line_starts, spans, { stack: [], frontmatter_end: block.nil? ? 0 : block.end })
+          doc = Commonmarker.parse(
+            parser_source,
+            options: { parse: { sourcepos_chars: true }, extension: { footnotes: true } }
+          )
+          walk(doc, chars, line_starts, spans, state)
+        end
+        trim_url_tails(chars, Spans.normalize_spans(remove_ranges(spans, state[:labels])), state[:url_ends])
+      end
+
+      URL_PREFIXES = ["www.", "http://", "https://"].freeze
+
+      # modes.md 3.7.3 (spec 1.17.2): does a bare web address begin at +i+ and fit before +limit+?
+      # ASCII case-insensitive, and no test on the character before it.
+      def self.starts_url?(chars, i, limit)
+        URL_PREFIXES.any? do |word|
+          next false if i + word.length > limit
+
+          word.each_char.with_index.all? do |ch, k|
+            c = chars[i + k]
+            (c.ord >= 0x41 && c.ord <= 0x5A ? c.downcase : c) == ch
+          end
+        end
+      end
+      private_class_method :starts_url?
+
+      URL_TERMINATORS = [SPACE, TAB, LF, CR, "<"].freeze
+
+      # modes.md 3.7.3 (spec 1.17.2): the tail of a bare web address, found by scanning the source
+      # rather than asking a parser, because parsers disagree about where one may start and end. A
+      # span that begins at the U+0020 or U+0009 ending such an address is flagged +url_tail+, which
+      # guards its first code point (modes.md 3.4, Spans.filter_boundary_edits).
+      def self.trim_url_tails(chars, spans, url_ends)
+        out = []
+        spans.each do |span|
+          from = span.start
+          i = span.start
+          guard = span.url_tail == true ||
+                  (url_ends.include?(span.start) && [SPACE, TAB].include?(chars[span.start]))
+          while i < span.end
+            unless (i == span.start && url_ends.include?(i)) || starts_url?(chars, i, span.end)
+              i += 1
+              next
+            end
+            e = i
+            e += 1 while e < span.end && !URL_TERMINATORS.include?(chars[e])
+            if e == i
+              i += 1
+              next
+            end
+            out << Spans::Span.new(from, i, guard ? true : nil) if i > from
+            guard = e < span.end && [SPACE, TAB].include?(chars[e])
+            from = e
+            i = e
+          end
+          out << Spans::Span.new(from, span.end, guard ? true : nil) if from < span.end
         end
-        spans
+        out
+      end
+      private_class_method :trim_url_tails
+
+      def self.remove_ranges(spans, ranges)
+        return spans if ranges.empty?
+
+        spans.flat_map do |span|
+          ranges.reduce([span]) do |pieces, r|
+            pieces.flat_map do |q|
+              next [q] if r.end <= q.start || r.start >= q.end
+
+              res = []
+              res << Spans::Span.new(q.start, r.start) if q.start < r.start
+              res << Spans::Span.new(r.end, q.end) if r.end < q.end
+              res
+            end
+          end
+        end
+      end
+      private_class_method :remove_ranges
+
+      # modes.md 3.7.3 (spec 1.17.2): the text of a shortcut or collapsed reference is its label.
+      # A node exists only when the parser formed it from a definition, so the kind is read off the
+      # source after the closing bracket of the text: nothing (shortcut), "[]" (collapsed); a
+      # resource "(...)" or a non-empty "[label]" (full) leaves the text processable.
+      def self.reference_label_range(node, chars, line_starts)
+        sp = node.source_position
+        start_offset = offset_of(line_starts, sp[:start_line], sp[:start_column])
+        end_offset = offset_of(line_starts, sp[:end_line], sp[:end_column])
+        last = node.to_a.last
+        return nil if last.nil?
+
+        lsp = last.source_position
+        close = offset_of(line_starts, lsp[:end_line], lsp[:end_column]) + 1
+        return nil unless chars[close] == "]"
+
+        shortcut = end_offset == close
+        collapsed = end_offset == close + 2 && chars[close + 1] == "[" && chars[close + 2] == "]"
+        return nil unless shortcut || collapsed
+
+        Spans::Span.new(start_offset + (node.type == :image ? 2 : 1), close)
       end
+      private_class_method :reference_label_range
 
       # line_starts[i] is the 0-based code-point index at which comrak's 1-based line i begins.
       # comrak counts CommonMark line endings, which are U+000A, U+000D, and U+000D U+000A --
@@ -347,12 +443,20 @@ module Polytypo
         if type == :link || type == :image
           sp = node.source_position
           start_offset = offset_of(line_starts, sp[:start_line], sp[:start_column])
-          unless chars[start_offset] == "["
+          opener = type == :image ? start_offset + 1 : start_offset
+          unless chars[opener] == "[" && (type == :link || chars[start_offset] == "!")
             # No leading "[" at the link's own source position: this is an autolink
             # (<https://…> or a bare GFM autolink literal), not a real [text](url) link --
-            # skipped whole, per modes.md 3.7.3.
+            # skipped whole, per modes.md 3.7.3. The end of a literal that begins with a web
+            # prefix is where the tail scan resumes.
+            if type == :link && chars[start_offset] != "<" &&
+               starts_url?(chars, start_offset, chars.length)
+              state[:url_ends] << (offset_of(line_starts, sp[:end_line], sp[:end_column]) + 1)
+            end
             return
           end
+          label = reference_label_range(node, chars, line_starts)
+          state[:labels] << label unless label.nil?
           # A real link/image: fall through to the generic child walk below, which emits the
           # link text (or image alt text) as processable prose. The destination/title are never
           # represented as child nodes at all, so there is nothing further to exclude.

data/lib/polytypo/modes/runner.rb

@@ -13,12 +13,16 @@ module Polytypo
     module Runner
       # The same sequence as Engine::Pipeline.run_rules, with the two boundary filters of
       # modes.md 3.4 interposed.
-      def self.run_rules_over_spans(cp, plan, locale_data, ctx)
+      def self.run_rules_over_spans(cp, plan, locale_data, ctx, guarded = [])
         current = cp
+        markdown = ctx.mode == "markdown"
         plan.each do |rule_id|
           fn = Engine::Registry.rule(rule_id)
           edits = fn.call(current, locale_data, ctx)
-          filtered = Spans.filter_boundary_edits(current, edits, Spans.span_ranges_of(current))
+          filtered = Spans.filter_boundary_edits(
+            current, edits, Spans.span_ranges_of(current),
+            delimiter_neighbours: markdown, guarded: guarded
+          )
           next if filtered.empty?
 
           current = Engine::Edits.apply_edits(current, filtered, rule_id)
@@ -61,7 +65,8 @@ module Polytypo
         return [] if normalized.empty?
 
         concatenated = Spans.concatenate_spans(source_cp, normalized)
-        transformed = run_rules_over_spans(concatenated, plan, locale_data, ctx)
+        guarded = normalized.map { |s| s.url_tail == true }
+        transformed = run_rules_over_spans(concatenated, plan, locale_data, ctx, guarded)
         pieces = Spans.split_on_marker(transformed, normalized.length)
         normalized.each_with_index.map { |span, i| [span, pieces[i]] }
       end
@@ -91,12 +96,16 @@ module Polytypo
         normalized = Spans.normalize_spans(spans)
         return [] if normalized.empty?
 
+        guarded = normalized.map { |s| s.url_tail == true }
         Engine::Pipeline.run_rules_recording(
           Spans.concatenate_spans(source_cp, normalized),
           plan, locale_data, ctx,
           Spans.origin_of_spans(normalized),
           source_cp.length
-        ) { |current, edits| Spans.filter_boundary_edits(current, edits, Spans.span_ranges_of(current)) }
+        ) do |current, edits|
+          Spans.filter_boundary_edits(current, edits, Spans.span_ranges_of(current),
+                                      delimiter_neighbours: ctx.mode == "markdown", guarded: guarded)
+        end
       end
     end
   end

data/lib/polytypo/modes/spans.rb

@@ -18,7 +18,10 @@ module Polytypo
       # A processable span, identified by its offsets in the original source, addressed as
       # code-point indices (ARCHITECTURE.md section 4.2) -- Ruby String indices are already
       # code-point indices for a UTF-8 String, no separate byte/char distinction to manage here.
-      Span = Struct.new(:start, :end)
+      #
+      # +url_tail+ is true (or nil) on a span that begins at the U+0020 or U+0009 ending a bare web
+      # address, so the filter can guard its first code point (modes.md 3.7.3, spec 1.17.2).
+      Span = Struct.new(:start, :end, :url_tail)
 
       # A span's extent in the concatenated code-point array: s0/s1 of modes.md 3.4. `:first`
       # deliberately shadows Struct#first with the equivalent one-argument-less accessor -- this
@@ -50,7 +53,7 @@ module Polytypo
             )
           end
           if span.start == last.end
-            out[-1] = Span.new(last.start, span.end)
+            out[-1] = Span.new(last.start, span.end, last.url_tail)
           else
             out << span
           end
@@ -106,6 +109,35 @@ module Polytypo
       end
       private_class_method :span_containing
 
+      DELIMITERS = [0x2A, 0x5F, 0x7E].freeze
+
+      def self.markdown_whitespace?(c)
+        return false if c.nil?
+
+        [0x09, 0x0A, 0x0C, 0x0D, 0x20, 0xA0, 0x1680, 0x202F, 0x205F, 0x3000].include?(c) ||
+          (c >= 0x2000 && c <= 0x200A)
+      end
+      private_class_method :markdown_whitespace?
+
+      # modes.md 3.4 delimiter-neighbour rule (markdown only, spec 1.17.2): an edit is discarded
+      # when, beside U+002A, U+005F or U+007E, exactly one of the old and new neighbour is
+      # CommonMark whitespace. A marker belongs to neither class, nor does a position outside the
+      # array. An insertion has q = p - 1.
+      def self.changes_delimiter_flank?(cp, p, q, replacement)
+        r = replacement.length
+        before_edit = p.positive? ? cp[p - 1] : nil
+        if DELIMITERS.include?(before_edit)
+          after = r.positive? ? replacement[0] : cp[q + 1]
+          return true if markdown_whitespace?(cp[p]) != markdown_whitespace?(after)
+        end
+        if q + 1 < cp.length && DELIMITERS.include?(cp[q + 1])
+          after = r.positive? ? replacement[r - 1] : before_edit
+          return true if markdown_whitespace?(q.negative? ? nil : cp[q]) != markdown_whitespace?(after)
+        end
+        false
+      end
+      private_class_method :changes_delimiter_flank?
+
       # modes.md 3.4, two safety nets, both pure functions of (p, q, r, s0, s1):
       #
       #  1. No edit may contain a marker -- one that does is a bug, discarded rather than
@@ -118,7 +150,11 @@ module Polytypo
       #     clause tests the CHARACTER, and only U+0020 needs testing -- it is the one code
       #     point any rule emits whose meaning comes from its position rather than from itself
       #     (modes.md 5 item 2).
-      def self.filter_boundary_edits(cp, edits, ranges)
+      #
+      # +delimiter_neighbours+ (markdown only) enables the delimiter-neighbour rule of modes.md 3.4,
+      # and +guarded+ holds, per span, whether its first code point is the terminator of a bare web
+      # address (modes.md 3.7.3); both spec 1.17.2.
+      def self.filter_boundary_edits(cp, edits, ranges, delimiter_neighbours: false, guarded: [])
         edits.select do |edit|
           next false if (edit.start...edit.end).any? { |i| Engine.marker?(cp[i]) }
 
@@ -126,7 +162,13 @@ module Polytypo
           q = edit.end - 1
           d = edit.end - edit.start
           r = edit.replacement.length
+          next false if delimiter_neighbours && changes_delimiter_flank?(cp, p, q, edit.replacement)
+
           span = span_containing(ranges, p)
+          if span && p == span.first && guarded[ranges.index(span)] == true &&
+             (r.zero? || edit.replacement[0] != SPACE)
+            next false
+          end
           next true unless span && (p == span.first || q == span.last)
           next false if r > d
           next false if r.positive? && p == span.first && edit.replacement[0] == SPACE && cp[p] != SPACE

data/lib/polytypo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Polytypo
-  VERSION = "1.17.1"
+  VERSION = "1.17.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: polytypo
 version: !ruby/object:Gem::Version
-  version: 1.17.1
+  version: 1.17.3
 platform: ruby
 authors:
 - Iurii Rogulia