motion-markdown-it 0.4.0.3.0

data/lib/motion-markdown-it/rules_core/normalize.rb

@@ -0,0 +1,44 @@
+# Normalize input string
+#------------------------------------------------------------------------------
+module MarkdownIt
+  module RulesCore
+    class Normalize
+
+      TABS_SCAN_RE = /[\n\t]/
+      NEWLINES_RE  = /\r[\n\u0085]|[\u2424\u2028\u0085]/
+      NULL_RE      = /\u0000/
+
+
+      #------------------------------------------------------------------------------
+      def self.inline(state)
+        # Normalize newlines
+        str = state.src.gsub(NEWLINES_RE, "\n")
+
+        # Replace NULL characters
+        str = str.gsub(NULL_RE, '\uFFFD')
+
+        # Replace tabs with proper number of spaces (1..4)
+        if str.include?("\t")
+          lineStart  = 0
+          lastTabPos = 0
+
+          str = str.gsub(TABS_SCAN_RE) do
+            md      = Regexp.last_match
+            match   = md.to_s
+            offset  = md.begin(0)
+            if str.charCodeAt(offset) == 0x0A
+              lineStart   = offset + 1
+              lastTabPos  = 0
+              next match
+            end
+            result      = '    '.slice_to_end((offset - lineStart - lastTabPos) % 4)
+            lastTabPos  = offset - lineStart + 1
+            result
+          end
+        end
+
+        state.src = str
+      end
+    end
+  end
+end

data/lib/motion-markdown-it/rules_core/replacements.rb

@@ -0,0 +1,90 @@
+# Simple typographyc replacements
+#
+# '' → ‘’
+# "" → “”. Set '«»' for Russian, '„“' for German, empty to disable
+# (c) (C) → ©
+# (tm) (TM) → ™
+# (r) (R) → ®
+# +- → ±
+# (p) (P) -> §
+# ... → … (also ?.... → ?.., !.... → !..)
+# ???????? → ???, !!!!! → !!!, `,,` → `,`
+# -- → –, --- → —
+#------------------------------------------------------------------------------
+module MarkdownIt
+  module RulesCore
+    class Replacements
+
+      # TODO:
+      # - fractionals 1/2, 1/4, 3/4 -> ½, ¼, ¾
+      # - miltiplication 2 x 4 -> 2 × 4
+
+      RARE_RE = /\+-|\.\.|\?\?\?\?|!!!!|,,|--/
+
+      SCOPED_ABBR_RE = /\((c|tm|r|p)\)/i
+      SCOPED_ABBR = {
+        'c' => '©',
+        'r' => '®',
+        'p' => '§',
+        'tm' => '™'
+      }
+
+      #------------------------------------------------------------------------------
+      def self.replaceFn(match, name)
+        return SCOPED_ABBR[name.downcase]
+      end
+
+      #------------------------------------------------------------------------------
+      def self.replace_scoped(inlineTokens)
+        (inlineTokens.length - 1).downto(0) do |i|
+          token = inlineTokens[i]
+          if (token.type == 'text')
+            token.content = token.content.gsub(SCOPED_ABBR_RE) {|match| self.replaceFn(match, $1)}
+          end
+        end
+      end
+
+      #------------------------------------------------------------------------------
+      def self.replace_rare(inlineTokens)
+        (inlineTokens.length - 1).downto(0) do |i|
+          token = inlineTokens[i]
+          if (token.type == 'text')
+            if (RARE_RE =~ token.content)
+              token.content = token.content.
+                          gsub(/\+-/, '±').
+                          # .., ..., ....... -> …
+                          # but ?..... & !..... -> ?.. & !..
+                          gsub(/\.{2,}/, '…').gsub(/([?!])…/, "\\1..").
+                          gsub(/([?!]){4,}/, '\\1\\1\\1').gsub(/,{2,}/, ',').
+                          # em-dash
+                          gsub(/(^|[^-])---([^-]|$)/m, "\\1\u2014\\2").
+                          # en-dash
+                          gsub(/(^|\s)--(\s|$)/m, "\\1\u2013\\2").
+                          gsub(/(^|[^-\s])--([^-\s]|$)/m, "\\1\u2013\\2")
+            end
+          end
+        end
+      end
+
+
+      #------------------------------------------------------------------------------
+      def self.replace(state)
+        return if (!state.md.options[:typographer])
+
+        (state.tokens.length - 1).downto(0) do |blkIdx|
+          next if (state.tokens[blkIdx].type != 'inline')
+
+          if (SCOPED_ABBR_RE =~ state.tokens[blkIdx].content)
+            replace_scoped(state.tokens[blkIdx].children)
+          end
+
+          if (RARE_RE =~ state.tokens[blkIdx].content)
+            replace_rare(state.tokens[blkIdx].children)
+          end
+
+        end
+      end
+
+    end
+  end
+end

data/lib/motion-markdown-it/rules_core/smartquotes.rb

@@ -0,0 +1,158 @@
+# Convert straight quotation marks to typographic ones
+#------------------------------------------------------------------------------
+module MarkdownIt
+  module RulesCore
+    class Smartquotes
+      extend Common::Utils
+      
+      QUOTE_TEST_RE = /['"]/
+      QUOTE_RE      = /['"]/
+      APOSTROPHE    = "\u2019" # ’
+
+
+      #------------------------------------------------------------------------------
+      def self.replaceAt(str, index, ch)
+        return str[0, index] + ch + str.slice_to_end(index + 1)
+      end
+
+      #------------------------------------------------------------------------------
+      def self.process_inlines(tokens, state)
+        stack = []
+
+        (0...tokens.length).each do |i|
+          token = tokens[i]
+
+          thisLevel = tokens[i].level
+
+          j = stack.length - 1
+          while j >= 0
+            break if (stack[j][:level] <= thisLevel)
+            j -= 1
+          end
+
+          # stack.length = j + 1
+          stack = (j < stack.length ? stack.slice(0, j + 1) : stack.fill(nil, stack.length...(j+1)))
+
+          next if (token.type != 'text')
+          
+          text = token.content
+          pos  = 0
+          max  = text.length
+
+          # OUTER loop
+          while pos < max
+            continue_outer_loop = false
+            t = QUOTE_RE.match(text, pos)
+            break if t.nil?
+
+            canOpen  = true
+            canClose = true
+            pos      = t.begin(0) + 1
+            isSingle = (t[0] == "'")
+
+            # treat begin/end of the line as a whitespace
+            lastChar = t.begin(0) - 1 >= 0 ? text.charCodeAt(t.begin(0) - 1) : 0x20
+            nextChar = pos < max ? text.charCodeAt(pos) : 0x20
+
+            isLastPunctChar = isMdAsciiPunct(lastChar) || isPunctChar(lastChar.chr)
+            isNextPunctChar = isMdAsciiPunct(nextChar) || isPunctChar(nextChar.chr)
+
+            isLastWhiteSpace = isWhiteSpace(lastChar)
+            isNextWhiteSpace = isWhiteSpace(nextChar)
+
+            if (isNextWhiteSpace)
+              canOpen = false
+            elsif (isNextPunctChar)
+              if (!(isLastWhiteSpace || isLastPunctChar))
+                canOpen = false
+              end
+            end
+
+            if (isLastWhiteSpace)
+              canClose = false
+            elsif (isLastPunctChar)
+              if (!(isNextWhiteSpace || isNextPunctChar))
+                canClose = false
+              end
+            end
+
+            if (nextChar == 0x22 && t[0] == '"') # "
+              if (lastChar >= 0x30 && lastChar <= 0x39)   # >= 0  && <= 9
+                # special case: 1"" - count first quote as an inch
+                canClose = canOpen = false
+              end
+            end
+
+            if (canOpen && canClose)
+              # treat this as the middle of the word
+              canOpen  = false
+              canClose = isNextPunctChar
+            end
+
+            if (!canOpen && !canClose)
+              # middle of word
+              if (isSingle)
+                token.content = replaceAt(token.content, t.begin(0), APOSTROPHE)
+              end
+              next
+            end
+
+            if (canClose)
+              # this could be a closing quote, rewind the stack to get a match
+              j = stack.length - 1
+              while j >= 0
+                item = stack[j]
+                break if (stack[j][:level] < thisLevel)
+                if (item[:single] == isSingle && stack[j][:level] == thisLevel)
+                  item = stack[j]
+                  if (isSingle)
+                    tokens[item[:token]].content = replaceAt(tokens[item[:token]].content, item[:pos], state.md.options[:quotes][2])
+                    token.content = replaceAt(token.content, t.begin(0), state.md.options[:quotes][3])
+                  else
+                    tokens[item[:token]].content = replaceAt(tokens[item[:token]].content, item[:pos], state.md.options[:quotes][0])
+                    token.content = replaceAt(token.content, t.begin(0), state.md.options[:quotes][1])
+                  end
+                  # stack.length = j
+                  stack = (j < stack.length ? stack.slice(0, j) : stack.fill(nil, stack.length...(j)))
+                  continue_outer_loop = true    # continue OUTER;
+                  break
+                end
+                j -= 1
+              end
+            end
+            next if continue_outer_loop
+            
+            if (canOpen)
+              stack.push({
+                token: i,
+                pos: t.begin(0),
+                single: isSingle,
+                level: thisLevel
+              })
+            elsif (canClose && isSingle)
+              token.content = replaceAt(token.content, t.begin(0), APOSTROPHE)
+            end
+          end
+        end
+      end
+
+
+      #------------------------------------------------------------------------------
+      def self.smartquotes(state)
+        return if (!state.md.options[:typographer])
+
+        blkIdx = state.tokens.length - 1
+        while blkIdx >= 0
+          if (state.tokens[blkIdx].type != 'inline' || !(QUOTE_TEST_RE =~ state.tokens[blkIdx].content))
+            blkIdx -= 1
+            next
+          end
+
+          process_inlines(state.tokens[blkIdx].children, state)
+          blkIdx -= 1
+        end
+      end
+      
+    end
+  end
+end

data/lib/motion-markdown-it/rules_core/state_core.rb

@@ -0,0 +1,20 @@
+# Core state object
+#------------------------------------------------------------------------------
+module MarkdownIt
+  module RulesCore
+    class StateCore
+
+      attr_accessor   :src, :env, :tokens, :inlineMode, :md
+
+      #------------------------------------------------------------------------------
+      def initialize(src, md, env)
+        @src        = src
+        @env        = env
+        @tokens     = []
+        @inlineMode = false
+        @md         = md    # link to parser instance
+      end
+
+    end
+  end
+end

data/lib/motion-markdown-it/rules_inline/autolink.rb

@@ -0,0 +1,74 @@
+# Process autolinks '<protocol:...>'
+#------------------------------------------------------------------------------
+module MarkdownIt
+  module RulesInline
+    class Autolink
+
+      EMAIL_RE    = /^<([a-zA-Z0-9.!#$\%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)>/
+      AUTOLINK_RE = /^<([a-zA-Z.\-]{1,25}):([^<>\x00-\x20]*)>/
+
+
+      #------------------------------------------------------------------------------
+      def self.autolink(state, silent)
+        pos = state.pos
+
+        return false if (state.src.charCodeAt(pos) != 0x3C)  # <
+
+        tail = state.src.slice_to_end(pos)
+
+        return false if !tail.include?('>')
+
+        if (AUTOLINK_RE =~ tail)
+          linkMatch = tail.match(AUTOLINK_RE)
+
+          return false if !URL_SCHEMAS.include?(linkMatch[1].downcase)
+
+          url = linkMatch[0].slice(1...-1)
+          fullUrl = state.md.normalizeLink.call(url)
+          return false if (!state.md.validateLink.call(fullUrl))
+
+          if (!silent)
+            token         = state.push('link_open', 'a', 1)
+            token.attrs   = [ [ 'href', fullUrl ] ]
+
+            token         = state.push('text', '', 0)
+            token.content = state.md.normalizeLinkText.call(url)
+
+            token         = state.push('link_close', 'a', -1)
+          end
+
+          state.pos += linkMatch[0].length
+          return true
+        end
+
+        if (EMAIL_RE =~ tail)
+          emailMatch = tail.match(EMAIL_RE)
+
+          url = emailMatch[0].slice(1...-1)
+          fullUrl = state.md.normalizeLink.call('mailto:' + url)
+          return false if (!state.md.validateLink.call(fullUrl))
+
+          if (!silent)
+            token         = state.push('link_open', 'a', 1)
+            token.attrs   = [ [ 'href', fullUrl ] ]
+            token.markup  = 'autolink'
+            token.info    = 'auto'
+
+            token         = state.push('text', '', 0)
+            token.content = state.md.normalizeLinkText.call(url)
+
+            token         = state.push('link_close', 'a', -1)
+            token.markup  = 'autolink'
+            token.info    = 'auto'
+          end
+
+          state.pos += emailMatch[0].length
+          return true
+        end
+
+        return false
+      end
+
+    end
+  end
+end

data/lib/motion-markdown-it/rules_inline/backticks.rb

@@ -0,0 +1,51 @@
+# Parse backticks
+#------------------------------------------------------------------------------
+module MarkdownIt
+  module RulesInline
+    class Backticks
+
+      #------------------------------------------------------------------------------
+      def self.backtick(state, silent)
+        pos = state.pos
+        ch = state.src.charCodeAt(pos)
+
+        return false if (ch != 0x60)  #  ` 
+
+        start = pos
+        pos  += 1
+        max  = state.posMax
+
+        while (pos < max && state.src.charCodeAt(pos) == 0x60)  # `
+          pos += 1
+        end
+
+        marker = state.src.slice(start...pos)
+
+        matchStart = matchEnd = pos
+
+        while ((matchStart = state.src.index('`', matchEnd)) != nil)
+          matchEnd = matchStart + 1
+
+          while (matchEnd < max && state.src.charCodeAt(matchEnd) == 0x60) # `
+            matchEnd += 1
+          end
+
+          if (matchEnd - matchStart == marker.length)
+            if (!silent)
+              token         = state.push('code_inline', 'code', 0)
+              token.markup  = marker
+              token.content = state.src.slice(pos...matchStart).gsub(/[ \n]+/, ' ').strip
+            end
+            state.pos = matchEnd
+            return true
+          end
+        end
+
+        state.pending += marker if (!silent)
+        state.pos     += marker.length
+        return true
+      end
+
+    end
+  end
+end

data/lib/motion-markdown-it/rules_inline/emphasis.rb

@@ -0,0 +1,172 @@
+# Process *this* and _that_
+#------------------------------------------------------------------------------
+module MarkdownIt
+  module RulesInline
+    class Emphasis
+      extend MarkdownIt::Common::Utils
+      
+      # parse sequence of emphasis markers,
+      # "start" should point at a valid marker
+      #------------------------------------------------------------------------------
+      def self.scanDelims(state, start)
+        pos       = start
+        can_open  = true
+        can_close = true
+        max       = state.posMax
+        marker    = state.src.charCodeAt(start)
+
+        # treat beginning of the line as a whitespace
+        lastChar = start > 0 ? state.src.charCodeAt(start - 1) : 0x20
+
+        while (pos < max && state.src.charCodeAt(pos) == marker)
+          pos += 1
+        end
+
+        if (pos >= max)
+          can_open = false
+        end
+
+        count = pos - start
+
+        # treat end of the line as a whitespace
+        nextChar = pos < max ? state.src.charCodeAt(pos) : 0x20
+
+        isLastPunctChar = isMdAsciiPunct(lastChar) || isPunctChar(lastChar.chr(Encoding::UTF_8))
+        isNextPunctChar = isMdAsciiPunct(nextChar) || isPunctChar(nextChar.chr(Encoding::UTF_8))
+
+        isLastWhiteSpace = isWhiteSpace(lastChar)
+        isNextWhiteSpace = isWhiteSpace(nextChar)
+
+        if (isNextWhiteSpace)
+          can_open = false
+        elsif (isNextPunctChar)
+          if (!(isLastWhiteSpace || isLastPunctChar))
+            can_open = false
+          end
+        end
+
+        if (isLastWhiteSpace)
+          can_close = false
+        elsif (isLastPunctChar)
+          if (!(isNextWhiteSpace || isNextPunctChar))
+            can_close = false
+          end
+        end
+
+        if (marker == 0x5F) # _
+          if (can_open && can_close)
+            # "_" inside a word can neither open nor close an emphasis
+            can_open  = false
+            can_close = isNextPunctChar
+          end
+        end
+
+        return { can_open: can_open, can_close: can_close, delims: count }
+      end
+
+      #------------------------------------------------------------------------------
+      def self.emphasis(state, silent)
+        max    = state.posMax
+        start  = state.pos
+        marker = state.src.charCodeAt(start)
+
+        return false if (marker != 0x5F && marker != 0x2A) #  _ *
+        return false if (silent) # don't run any pairs in validation mode
+
+        res = scanDelims(state, start)
+        startCount = res[:delims]
+        if (!res[:can_open])
+          state.pos += startCount
+          # Earlier we checked !silent, but this implementation does not need it
+          state.pending += state.src.slice(start...state.pos)
+          return true
+        end
+
+        state.pos = start + startCount
+        stack = [ startCount ]
+
+        while (state.pos < max)
+          if (state.src.charCodeAt(state.pos) == marker)
+            res = scanDelims(state, state.pos)
+            count = res[:delims]
+            if (res[:can_close])
+              oldCount = stack.pop()
+              newCount = count
+
+              while (oldCount != newCount)
+                if (newCount < oldCount)
+                  stack.push(oldCount - newCount)
+                  break
+                end
+
+                # assert(newCount > oldCount)
+                newCount -= oldCount
+
+                break if (stack.length == 0)
+                state.pos += oldCount
+                oldCount = stack.pop()
+              end
+
+              if (stack.length == 0)
+                startCount = oldCount
+                found      = true
+                break
+              end
+              state.pos += count
+              next
+            end
+
+            stack.push(count) if (res[:can_open])
+            state.pos += count
+            next
+          end
+
+          state.md.inline.skipToken(state)
+        end
+
+        if (!found)
+          # parser failed to find ending tag, so it's not valid emphasis
+          state.pos = start
+          return false
+        end
+
+        # found!
+        state.posMax = state.pos
+        state.pos    = start + startCount
+
+        # Earlier we checked !silent, but this implementation does not need it
+
+        # we have `startCount` starting and ending markers,
+        # now trying to serialize them into tokens
+        count = startCount
+        while count > 1
+          token        = state.push('strong_open', 'strong', 1)
+          token.markup = marker.chr + marker.chr
+          count -= 2
+        end
+        if (count % 2 == 1)
+          token        = state.push('em_open', 'em', 1)
+          token.markup = marker.chr
+        end
+
+        state.md.inline.tokenize(state)
+
+        if (count % 2 == 1)
+          token        = state.push('em_close', 'em', -1)
+          token.markup = marker.chr
+        end
+        count = startCount
+        while count > 1
+          token        = state.push('strong_close', 'strong', -1)
+          token.markup = marker.chr + marker.chr
+          count -= 2
+        end
+
+        state.pos     = state.posMax + startCount
+        state.posMax  = max
+        return true
+      end
+
+    end
+  end
+end