motion-markdown-it 4.4.0 → 8.4.1

data/lib/motion-markdown-it/rules_core/normalize.rb

@@ -4,11 +4,9 @@ module MarkdownIt
   module RulesCore
     class Normalize
 
-      TABS_SCAN_RE = /[\n\t]/
-      NEWLINES_RE  = /\r[\n\u0085]|[\u2424\u2028\u0085]/
+      NEWLINES_RE  = /\r[\n\u0085]?|[\u2424\u2028\u0085]/
       NULL_RE      = /\u0000/
 
-
       #------------------------------------------------------------------------------
       def self.inline(state)
         # Normalize newlines
@@ -17,26 +15,6 @@ module MarkdownIt
         # Replace NULL characters
         str = str.gsub(NULL_RE, '\uFFFD')
 
-        # Replace tabs with proper number of spaces (1..4)
-        if str.include?("\t")
-          lineStart  = 0
-          lastTabPos = 0
-
-          str = str.gsub(TABS_SCAN_RE) do
-            md      = Regexp.last_match
-            match   = md.to_s
-            offset  = md.begin(0)
-            if str.charCodeAt(offset) == 0x0A
-              lineStart   = offset + 1
-              lastTabPos  = 0
-              next match
-            end
-            result      = '    '.slice_to_end((offset - lineStart - lastTabPos) % 4)
-            lastTabPos  = offset - lineStart + 1
-            result
-          end
-        end
-
         state.src = str
       end
     end

data/lib/motion-markdown-it/rules_core/replacements.rb

@@ -34,19 +34,31 @@ module MarkdownIt
 
       #------------------------------------------------------------------------------
       def self.replace_scoped(inlineTokens)
+        inside_autolink = 0
+
         (inlineTokens.length - 1).downto(0) do |i|
           token = inlineTokens[i]
-          if (token.type == 'text')
+          if token.type == 'text' && inside_autolink == 0
             token.content = token.content.gsub(SCOPED_ABBR_RE) {|match| self.replaceFn(match, $1)}
           end
+
+          if token.type == 'link_open' && token.info == 'auto'
+            inside_autolink -= 1
+          end
+
+          if token.type == 'link_close' && token.info == 'auto'
+            inside_autolink += 1
+          end
         end
       end
 
       #------------------------------------------------------------------------------
       def self.replace_rare(inlineTokens)
+        inside_autolink = 0
+
         (inlineTokens.length - 1).downto(0) do |i|
           token = inlineTokens[i]
-          if (token.type == 'text')
+          if token.type == 'text' && inside_autolink == 0
             if (RARE_RE =~ token.content)
               token.content = token.content.
                           gsub(/\+-/, '±').
@@ -61,6 +73,14 @@ module MarkdownIt
                           gsub(/(^|[^-\s])--([^-\s]|$)/m, "\\1\u2013\\2")
             end
           end
+
+          if token.type == 'link_open' && token.info == 'auto'
+            inside_autolink -= 1
+          end
+
+          if token.type == 'link_close' && token.info == 'auto'
+            inside_autolink += 1
+          end
         end
       end
 

data/lib/motion-markdown-it/rules_core/smartquotes.rb

@@ -4,12 +4,11 @@ module MarkdownIt
   module RulesCore
     class Smartquotes
       extend Common::Utils
-      
+
       QUOTE_TEST_RE = /['"]/
       QUOTE_RE      = /['"]/
       APOSTROPHE    = "\u2019" # ’
 
-
       #------------------------------------------------------------------------------
       def self.replaceAt(str, index, ch)
         return str[0, index] + ch + str.slice_to_end(index + 1)
@@ -34,7 +33,7 @@ module MarkdownIt
           stack = (j < stack.length ? stack.slice(0, j + 1) : stack.fill(nil, stack.length...(j+1)))
 
           next if (token.type != 'text')
-          
+
           text = token.content
           pos  = 0
           max  = text.length
@@ -50,12 +49,42 @@ module MarkdownIt
             pos      = t.begin(0) + 1
             isSingle = (t[0] == "'")
 
-            # treat begin/end of the line as a whitespace
-            lastChar = t.begin(0) - 1 >= 0 ? text.charCodeAt(t.begin(0) - 1) : 0x20
-            nextChar = pos < max ? text.charCodeAt(pos) : 0x20
+            # Find previous character,
+            # default to space if it's the beginning of the line
+            #
+            lastChar = 0x20
+
+            if t.begin(0) - 1 >= 0
+              lastChar = text.charCodeAt(t.begin(0) - 1)
+            else
+              (i - 1).downto(0) do |j|
+                break if tokens[j].type == 'softbreak' || tokens[j].type == 'hardbreak' # lastChar defaults to 0x20
+                next if tokens[j].type != 'text'
+
+                lastChar = tokens[j].content.charCodeAt(tokens[j].content.length - 1)
+                break
+              end
+            end
+
+            # Find next character,
+            # default to space if it's the end of the line
+            #
+            nextChar = 0x20
 
-            isLastPunctChar = isMdAsciiPunct(lastChar) || isPunctChar(lastChar.chr(Encoding::UTF_8))
-            isNextPunctChar = isMdAsciiPunct(nextChar) || isPunctChar(nextChar.chr(Encoding::UTF_8))
+            if pos < max
+              nextChar = text.charCodeAt(pos)
+            else
+              (i + 1).upto(tokens.length - 1) do |j|
+                break if tokens[j].type == 'softbreak' || tokens[j].type == 'hardbreak' # nextChar defaults to 0x20
+                next if tokens[j].type != 'text'
+
+                nextChar = tokens[j].content.charCodeAt(0)
+                break
+              end
+            end
+
+            isLastPunctChar = isMdAsciiPunct(lastChar) || isPunctChar(fromCodePoint(lastChar))
+            isNextPunctChar = isMdAsciiPunct(nextChar) || isPunctChar(fromCodePoint(nextChar))
 
             isLastWhiteSpace = isWhiteSpace(lastChar)
             isNextWhiteSpace = isWhiteSpace(nextChar)
@@ -112,7 +141,7 @@ module MarkdownIt
                     openQuote  = state.md.options[:quotes][0]
                     closeQuote = state.md.options[:quotes][1]
                   end
-                  
+
                   # replace token.content *before* tokens[item.token].content,
                   # because, if they are pointing at the same token, replaceAt
                   # could mess up indices when quote length != 1
@@ -124,7 +153,7 @@ module MarkdownIt
 
                   text = token.content
                   max  = text.length
-                  
+
                   stack = (j < stack.length ? stack.slice(0, j) : stack.fill(nil, stack.length...(j)))  # stack.length = j
                   continue_outer_loop = true    # continue OUTER;
                   break
@@ -133,7 +162,7 @@ module MarkdownIt
               end
             end
             next if continue_outer_loop
-            
+
             if (canOpen)
               stack.push({
                 token: i,
@@ -164,7 +193,7 @@ module MarkdownIt
           blkIdx -= 1
         end
       end
-      
+
     end
   end
 end

data/lib/motion-markdown-it/rules_inline/autolink.rb

@@ -5,8 +5,7 @@ module MarkdownIt
     class Autolink
 
       EMAIL_RE    = /^<([a-zA-Z0-9.!#$\%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)>/
-      AUTOLINK_RE = /^<([a-zA-Z.\-]{1,25}):([^<>\x00-\x20]*)>/
-
+      AUTOLINK_RE = /^<([a-zA-Z][a-zA-Z0-9+.\-]{1,31}):([^<>\x00-\x20]*)>/
 
       #------------------------------------------------------------------------------
       def self.autolink(state, silent)
@@ -21,8 +20,6 @@ module MarkdownIt
         if (AUTOLINK_RE =~ tail)
           linkMatch = tail.match(AUTOLINK_RE)
 
-          return false if !URL_SCHEMAS.include?(linkMatch[1].downcase)
-
           url = linkMatch[0].slice(1...-1)
           fullUrl = state.md.normalizeLink.call(url)
           return false if (!state.md.validateLink.call(fullUrl))
@@ -30,11 +27,15 @@ module MarkdownIt
           if (!silent)
             token         = state.push('link_open', 'a', 1)
             token.attrs   = [ [ 'href', fullUrl ] ]
+            token.markup  = 'autolink'
+            token.info    = 'auto'
 
             token         = state.push('text', '', 0)
             token.content = state.md.normalizeLinkText.call(url)
 
             token         = state.push('link_close', 'a', -1)
+            token.markup  = 'autolink'
+            token.info    = 'auto'
           end
 
           state.pos += linkMatch[0].length

data/lib/motion-markdown-it/rules_inline/balance_pairs.rb

@@ -0,0 +1,48 @@
+# For each opening emphasis-like marker find a matching closing one
+#------------------------------------------------------------------------------
+module MarkdownIt
+  module RulesInline
+    class BalancePairs
+
+      #------------------------------------------------------------------------------
+      def self.link_pairs(state)
+        delimiters = state.delimiters
+        max = state.delimiters.length
+
+        0.upto(max - 1) do |i|
+          lastDelim = delimiters[i]
+
+          next if !lastDelim[:close]
+
+          j = i - lastDelim[:jump] - 1
+
+          while j >= 0
+            currDelim = delimiters[j]
+
+            if currDelim[:open] &&
+               currDelim[:marker] == lastDelim[:marker] &&
+               currDelim[:end] < 0 &&
+               currDelim[:level] == lastDelim[:level]
+
+              # typeofs are for backward compatibility with plugins
+              # not needed:  typeof currDelim.length !== 'undefined' &&
+              #              typeof lastDelim.length !== 'undefined' &&
+              odd_match = (currDelim[:close] || lastDelim[:open]) &&
+                          (currDelim[:length] + lastDelim[:length]) % 3 == 0
+
+              if !odd_match
+                lastDelim[:jump] = i - j
+                lastDelim[:open] = false
+                currDelim[:end]  = i
+                currDelim[:jump] = 0
+                break
+              end
+            end
+
+            j -= currDelim[:jump] + 1
+          end
+        end
+      end
+    end
+  end
+end

data/lib/motion-markdown-it/rules_inline/emphasis.rb

@@ -4,166 +4,121 @@ module MarkdownIt
   module RulesInline
     class Emphasis
       extend MarkdownIt::Common::Utils
-      
-      # parse sequence of emphasis markers,
-      # "start" should point at a valid marker
-      #------------------------------------------------------------------------------
-      def self.scanDelims(state, start)
-        pos            = start
-        left_flanking  = true
-        right_flanking = true
-        max            = state.posMax
-        marker         = state.src.charCodeAt(start)
-
-        # treat beginning of the line as a whitespace
-        lastChar = start > 0 ? state.src.charCodeAt(start - 1) : 0x20
-
-        while (pos < max && state.src.charCodeAt(pos) == marker)
-          pos += 1
-        end
-
-        count = pos - start
-
-        # treat end of the line as a whitespace
-        nextChar = pos < max ? state.src.charCodeAt(pos) : 0x20
-
-        isLastPunctChar = isMdAsciiPunct(lastChar) || isPunctChar(lastChar.chr(Encoding::UTF_8))
-        isNextPunctChar = isMdAsciiPunct(nextChar) || isPunctChar(nextChar.chr(Encoding::UTF_8))
 
-        isLastWhiteSpace = isWhiteSpace(lastChar)
-        isNextWhiteSpace = isWhiteSpace(nextChar)
-
-        if (isNextWhiteSpace)
-          left_flanking = false
-        elsif (isNextPunctChar)
-          if (!(isLastWhiteSpace || isLastPunctChar))
-            left_flanking = false
-          end
+      # Insert each marker as a separate text token, and add it to delimiter list
+      #
+      def self.tokenize(state, silent)
+        start   = state.pos
+        marker  = state.src.charCodeAt(start)
+
+        return false if silent
+
+        return false if (marker != 0x5F && marker != 0x2A) #  _ and *
+
+        scanned = state.scanDelims(state.pos, marker == 0x2A)
+
+        0.upto(scanned[:length] - 1) do |i|
+          token         = state.push('text', '', 0)
+          token.content = fromCodePoint(marker)
+
+          state.delimiters.push({
+            # Char code of the starting marker (number).
+            #
+            marker: marker,
+
+            # Total length of these series of delimiters.
+            #
+            length: scanned[:length],
+
+            # An amount of characters before this one that's equivalent to
+            # current one. In plain English: if this delimiter does not open
+            # an emphasis, neither do previous `jump` characters.
+            #
+            # Used to skip sequences like "*****" in one step, for 1st asterisk
+            # value will be 0, for 2nd it's 1 and so on.
+            #
+            jump:   i,
+
+            # A position of the token this delimiter corresponds to.
+            #
+            token:  state.tokens.length - 1,
+
+            # Token level.
+            #
+            level:  state.level,
+
+            # If this delimiter is matched as a valid opener, `end` will be
+            # equal to its position, otherwise it's `-1`.
+            #
+            end:    -1,
+
+            # Boolean flags that determine if this delimiter could open or close
+            # an emphasis.
+            #
+            open:   scanned[:can_open],
+            close:  scanned[:can_close]
+          })
         end
 
-        if (isLastWhiteSpace)
-          right_flanking = false
-        elsif (isLastPunctChar)
-          if (!(isNextWhiteSpace || isNextPunctChar))
-            right_flanking = false
-          end
-        end
-
-        if (marker == 0x5F) # _
-          # "_" inside a word can neither open nor close an emphasis
-          can_open  = left_flanking  && (!right_flanking || isLastPunctChar)
-          can_close = right_flanking && (!left_flanking  || isNextPunctChar)
-        else
-          can_open  = left_flanking
-          can_close = right_flanking
-        end
+        state.pos += scanned[:length]
 
-        return { can_open: can_open, can_close: can_close, delims: count }
+        return true
       end
 
-      #------------------------------------------------------------------------------
-      def self.emphasis(state, silent)
-        max    = state.posMax
-        start  = state.pos
-        marker = state.src.charCodeAt(start)
-
-        return false if (marker != 0x5F && marker != 0x2A) #  _ *
-        return false if (silent) # don't run any pairs in validation mode
-
-        res = scanDelims(state, start)
-        startCount = res[:delims]
-        if (!res[:can_open])
-          state.pos += startCount
-          # Earlier we checked !silent, but this implementation does not need it
-          state.pending += state.src.slice(start...state.pos)
-          return true
-        end
 
-        state.pos = start + startCount
-        stack = [ startCount ]
-
-        while (state.pos < max)
-          if (state.src.charCodeAt(state.pos) == marker)
-            res = scanDelims(state, state.pos)
-            count = res[:delims]
-            if (res[:can_close])
-              oldCount = stack.pop()
-              newCount = count
-
-              while (oldCount != newCount)
-                if (newCount < oldCount)
-                  stack.push(oldCount - newCount)
-                  break
-                end
-
-                # assert(newCount > oldCount)
-                newCount -= oldCount
-
-                break if (stack.length == 0)
-                state.pos += oldCount
-                oldCount = stack.pop()
-              end
-
-              if (stack.length == 0)
-                startCount = oldCount
-                found      = true
-                break
-              end
-              state.pos += count
-              next
-            end
-
-            stack.push(count) if (res[:can_open])
-            state.pos += count
-            next
+      # Walk through delimiter list and replace text tokens with tags
+      #
+      def self.postProcess(state)
+        delimiters = state.delimiters
+        max = state.delimiters.length
+
+        i = max - 1
+        while i >= 0
+          startDelim = delimiters[i]
+
+          (i -= 1) and next if startDelim[:marker] != 0x5F && startDelim[:marker] != 0x2A #  _ and *
+
+          # Process only opening markers
+          (i -= 1) and next if startDelim[:end] == -1
+
+          endDelim = delimiters[startDelim[:end]]
+
+          # If the previous delimiter has the same marker and is adjacent to this one,
+          # merge those into one strong delimiter.
+          #
+          # `<em><em>whatever</em></em>` -> `<strong>whatever</strong>`
+          #
+          isStrong = i > 0 &&
+                     delimiters[i - 1][:end] == startDelim[:end] + 1 &&
+                     delimiters[i - 1][:token] == startDelim[:token] - 1 &&
+                     delimiters[startDelim[:end] + 1][:token] == endDelim[:token] + 1 &&
+                     delimiters[i - 1][:marker] == startDelim[:marker]
+
+          ch = fromCodePoint(startDelim[:marker])
+
+          token         = state.tokens[startDelim[:token]]
+          token.type    = isStrong ? 'strong_open' : 'em_open'
+          token.tag     = isStrong ? 'strong' : 'em'
+          token.nesting = 1
+          token.markup  = isStrong ? ch + ch : ch
+          token.content = ''
+
+          token         = state.tokens[endDelim[:token]]
+          token.type    = isStrong ? 'strong_close' : 'em_close'
+          token.tag     = isStrong ? 'strong' : 'em'
+          token.nesting = -1
+          token.markup  = isStrong ? ch + ch : ch
+          token.content = ''
+
+          if isStrong
+            state.tokens[delimiters[i - 1][:token]].content = ''
+            state.tokens[delimiters[startDelim[:end] + 1][:token]].content = ''
+            i -= 1
           end
 
-          state.md.inline.skipToken(state)
-        end
-
-        if (!found)
-          # parser failed to find ending tag, so it's not valid emphasis
-          state.pos = start
-          return false
+          i -= 1
         end
-
-        # found!
-        state.posMax = state.pos
-        state.pos    = start + startCount
-
-        # Earlier we checked !silent, but this implementation does not need it
-
-        # we have `startCount` starting and ending markers,
-        # now trying to serialize them into tokens
-        count = startCount
-        while count > 1
-          token        = state.push('strong_open', 'strong', 1)
-          token.markup = marker.chr + marker.chr
-          count -= 2
-        end
-        if (count % 2 == 1)
-          token        = state.push('em_open', 'em', 1)
-          token.markup = marker.chr
-        end
-
-        state.md.inline.tokenize(state)
-
-        if (count % 2 == 1)
-          token        = state.push('em_close', 'em', -1)
-          token.markup = marker.chr
-        end
-        count = startCount
-        while count > 1
-          token        = state.push('strong_close', 'strong', -1)
-          token.markup = marker.chr + marker.chr
-          count -= 2
-        end
-
-        state.pos     = state.posMax + startCount
-        state.posMax  = max
-        return true
       end
-
     end
   end
 end