rouge 2.0.7 → 2.1.0

data/lib/rouge/lexers/coffeescript.rb

@@ -42,7 +42,7 @@ module Rouge
 
       state :comments_and_whitespace do
         rule /\s+/m, Text
-        rule /###.*?###/m, Comment::Multiline
+        rule /###\s*\n.*?###/m, Comment::Multiline
         rule /#.*$/, Comment::Single
       end
 

data/lib/rouge/lexers/console.rb

@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+  module Lexers
+    class ConsoleLexer < Lexer
+      tag 'console'
+      aliases 'terminal', 'shell_session', 'shell-session'
+      filenames '*.cap'
+      desc 'A generic lexer for shell sessions. Accepts ?lang and ?output lexer options, a ?prompt option, and ?comments to enable # comments.'
+
+      option :lang, 'the shell language to lex (default: shell)'
+      option :output, 'the output language (default: plaintext?token=Generic.Output)'
+      option :prompt, 'comma-separated list of strings that indicate the end of a prompt. (default: $,#,>,;)'
+      option :comments, 'enable hash-comments at the start of a line - otherwise interpreted as a prompt. (default: false, implied by ?prompt not containing `#`)'
+
+      def initialize(*)
+        super
+        @prompt = list_option(:prompt) { nil }
+        @lang = lexer_option(:lang) { 'shell' }
+        @output = lexer_option(:output) { PlainText.new(token: Generic::Output) }
+        @comments = bool_option(:comments) { :guess }
+      end
+
+      def prompt_regex
+        @prompt_regex ||= begin
+          /^#{prompt_prefix_regex}(?:#{end_chars.map(&Regexp.method(:escape)).join('|')})/
+        end
+      end
+
+      def end_chars
+        @end_chars ||= if @prompt.any?
+          @prompt.reject { |c| c.empty? }
+        else
+          %w($ # > ;)
+        end
+      end
+
+      # whether to allow comments. if manually specifying a prompt that isn't
+      # simply "#", we flag this to on
+      def allow_comments?
+        case @comments
+        when :guess
+          @prompt && !@prompt.empty? && !end_chars.include?('#')
+        else
+          @comments
+        end
+      end
+
+      def prompt_prefix_regex
+        if allow_comments?
+          /^[^<#]*?/
+        else
+          /^.*?/
+        end
+      end
+
+      def lang_lexer
+        @lang_lexer ||= case @lang
+        when Lexer
+          @lang
+        when nil
+          Shell.new(options)
+        when Class
+          @lang.new(options)
+        when String
+          Lexer.find(@lang).new(options)
+        end
+      end
+
+      def output_lexer
+        @output_lexer ||= case @output
+        when nil
+          PlainText.new(token: Generic::Output)
+        when Lexer
+          @output
+        when Class
+          @output.new(options)
+        when String
+          Lexer.find(@output).new(options)
+        end
+      end
+
+      def line_regex
+        /(\\.|[^\\])*?(\n|$)/m
+      end
+
+      def comment_regex
+        /\A\s*?#/
+      end
+
+      def stream_tokens(input, &output)
+        input = StringScanner.new(input)
+        lang_lexer.reset!
+        output_lexer.reset!
+
+        process_line(input, &output) while !input.eos?
+      end
+
+      def process_line(input, &output)
+        input.scan(line_regex)
+
+        if input[0] =~ /\A\s*(?:<[.]+>|[.]+)\s*\z/
+          puts "console: matched snip #{input[0].inspect}" if @debug
+          output_lexer.reset!
+          lang_lexer.reset!
+
+          yield Comment, input[0]
+        elsif prompt_regex =~ input[0]
+          puts "console: matched prompt #{input[0].inspect}" if @debug
+          output_lexer.reset!
+
+          yield Generic::Prompt, $&
+
+          # make sure to take care of initial whitespace
+          # before we pass to the lang lexer so it can determine where
+          # the "real" beginning of the line is
+          $' =~ /\A\s*/
+          yield Text, $& unless $&.empty?
+
+          lang_lexer.lex($', continue: true, &output)
+        elsif comment_regex =~ input[0].strip
+          puts "console: matched comment #{input[0].inspect}" if @debug
+          output_lexer.reset!
+          lang_lexer.reset!
+
+          yield Comment, input[0]
+        else
+          puts "console: matched output #{input[0].inspect}" if @debug
+          lang_lexer.reset!
+
+          output_lexer.lex(input[0], continue: true, &output)
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/csharp.rb

@@ -26,7 +26,7 @@ module Rouge
         static switch this throw true try typeof unchecked unsafe
         virtual void volatile while
         add alias async await get global partial remove set value where
-        yield
+        yield nameof
         ascending by descending equals from group in into join let on
         orderby select
       )
@@ -47,19 +47,38 @@ module Rouge
         rule %r(/[*].*?[*]/)m, Comment::Multiline
       end
 
+      state :nest do
+        rule /{/, Punctuation, :nest
+        rule /}/, Punctuation, :pop!
+        mixin :root
+      end
+
+      state :splice_string do
+        rule /\\./, Str
+        rule /{/, Punctuation, :nest
+        rule /"|\n/, Str, :pop!
+        rule /./, Str
+      end
+
+      state :splice_literal do
+        rule /""/, Str
+        rule /{/, Punctuation, :nest
+        rule /"/, Str, :pop!
+        rule /./, Str
+      end
+
       state :root do
         mixin :whitespace
 
         rule /^\s*\[.*?\]/, Name::Attribute
-        # rule /[$]\s*"/, Str, :splice_string
-        # rule /[$]\s*<#/, Str, :splice_recstring
-        # rule /<#/, Str, :recstring
+        rule /[$]\s*"/, Str, :splice_string
+        rule /[$]@\s*"/, Str, :splice_literal
 
         rule /(<\[)\s*(#{id}:)?/, Keyword
         rule /\]>/, Keyword
 
         rule /[~!%^&*()+=|\[\]{}:;,.<>\/?-]/, Punctuation
-        rule /@"(\\.|.)*?"/, Str
+        rule /@"(""|[^"])*"/m, Str
         rule /"(\\.|.)*?["\n]/, Str
         rule /'(\\.|.)'/, Str::Char
         rule /0x[0-9a-f]+[lu]?/i, Num
@@ -69,12 +88,12 @@ module Rouge
           (e[+-][0-9]+)? # exponent
           [fldu]? # type
         )ix, Num
+        rule /\b(?:class|struct|interface)\b/, Keyword, :class
+        rule /\b(?:namespace|using)\b/, Keyword, :namespace
         rule /^#[ \t]*(#{cpp_keywords.join('|')})\b.*?\n/,
           Comment::Preproc
         rule /\b(#{keywords.join('|')})\b/, Keyword
         rule /\b(#{keywords_type.join('|')})\b/, Keyword::Type
-        rule /class|struct/, Keyword, :class
-        rule /namespace|using/, Keyword, :namespace
         rule /#{id}(?=\s*[(])/, Name::Function
         rule id, Name
       end

data/lib/rouge/lexers/digdag.rb

@@ -0,0 +1,72 @@
+require 'set'
+module Rouge
+  module Lexers
+    load_lexer 'yaml.rb'
+
+    class Digdag < YAML
+      title 'digdag'
+      desc 'A simple, open source, multi-cloud workflow engine (https://www.digdag.io/)'
+      tag 'digdag'
+      filenames '*.dig'
+
+      mimetypes 'application/x-digdag'
+
+      def self.analyze_text(text)
+        # disable YAML.analyze_text
+      end
+
+      # http://docs.digdag.io/operators.html
+      # as of digdag v0.9.10
+      KEYWORD_PATTERN = Regexp.union(%w(
+        call
+        require
+        loop
+        for_each
+        if
+        fail
+        echo
+
+        td
+        td_run
+        td_ddl
+        td_load
+        td_for_each
+        td_wait
+        td_wait_table
+        td_partial_delete
+        td_table_export
+
+        pg
+
+        mail
+        http
+        s3_wait
+        redshift
+        redshift_load
+        redshift_unload
+        emr
+
+        gcs_wait
+        bq
+        bq_ddl
+        bq_extract
+        bq_load
+
+        sh
+        py
+        rb
+        embulk
+      ).map { |name| "#{name}>"} + %w(
+        _do
+        _parallel
+      ))
+
+      prepend :block_nodes do
+        rule /(#{KEYWORD_PATTERN})(:)(?=\s|$)/ do |m|
+          groups Keyword::Reserved, Punctuation::Indicator
+          set_indent m[0], :implicit => true
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/docker.rb

@@ -11,7 +11,7 @@ module Rouge
       mimetypes 'text/x-dockerfile-config'
 
       KEYWORDS = %w(
-        FROM MAINTAINER CMD EXPOSE ENV ADD ENTRYPOINT VOLUME WORKDIR
+        FROM MAINTAINER CMD LABEL EXPOSE ENV ADD COPY ENTRYPOINT VOLUME USER WORKDIR ARG STOPSIGNAL HEALTHCHECK SHELL
       ).join('|')
 
       start { @shell = Shell.new(@options) }

data/lib/rouge/lexers/dot.rb

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+  module Lexers
+    class Dot < RegexLexer
+      title "DOT"
+      desc "graph description language"
+
+      tag 'dot'
+      filenames '*.dot'
+      mimetypes 'text/vnd.graphviz'
+
+      start do
+        @html = HTML.new(options)
+      end
+
+      state :comments_and_whitespace do
+        rule /\s+/, Text
+        rule %r(#.*?\n), Comment::Single
+        rule %r(//.*?\n), Comment::Single
+        rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, Comment::Multiline
+      end
+
+      state :html do
+        rule /[^<>]+/ do
+          delegate @html
+        end
+        rule /<.+?>/m do
+          delegate @html
+        end
+        rule />/, Punctuation, :pop!
+      end
+
+      state :ID do
+        rule /([a-zA-Z][a-zA-Z_0-9]*)(\s*)(=)/ do |m|
+          token Name, m[1]
+          token Text, m[2]
+          token Punctuation, m[3]
+        end
+        rule /[a-zA-Z][a-zA-Z_0-9]*/, Name::Variable
+        rule /([0-9]+)?\.[0-9]+/, Num::Float
+        rule /[0-9]+/, Num::Integer
+        rule /"(\\"|[^"])*"/, Str::Double
+        rule /</ do
+          token Punctuation
+          @html.reset!
+          push :html
+        end
+      end
+
+      state :a_list do
+        mixin :comments_and_whitespace
+        mixin :ID
+        rule /[=;,]/, Punctuation
+        rule /\]/, Operator, :pop!
+      end
+
+      state :root do
+        mixin :comments_and_whitespace
+        rule /\b(strict|graph|digraph|subgraph|node|edge)\b/i, Keyword
+        rule /[{};:=]/, Punctuation
+        rule /-[->]/, Operator
+        rule /\[/, Operator, :a_list
+        mixin :ID
+      end
+    end
+  end
+end

data/lib/rouge/lexers/elixir.rb

@@ -15,13 +15,6 @@ module Rouge
 
       mimetypes 'text/x-elixir', 'application/x-elixir'
 
-      BRACES = [
-        ['\{', '\}', 'cb'],
-        ['\[', '\]', 'sb'],
-        ['\(', '\)', 'pa'],
-        ['\<', '\>', 'lt']
-      ]
-
       state :root do
         rule /\s+/m, Text
         rule /#.*$/, Comment::Single
@@ -46,9 +39,9 @@ module Rouge
         rule /@[a-zA-Z_]\w*|&\d/, Name::Variable
         rule %r{\b(0[xX][0-9A-Fa-f]+|\d(_?\d)*(\.(?![^\d\s])
              (_?\d)*)?([eE][-+]?\d(_?\d)*)?|0[bB][01]+)\b}x, Num
-        rule %r{~r\/.*\/}, Str::Regex
 
         mixin :strings
+        mixin :sigil_strings
       end
 
       state :strings do
@@ -58,25 +51,6 @@ module Rouge
         rule /'.*?'/, Str::Single
         rule %r{(?<!\w)\?(\\(x\d{1,2}|\h{1,2}(?!\h)\b|0[0-7]{0,2}(?![0-7])\b[^x0MC])|(\\[MC]-)+\w|[^\s\\])}, Str::Other
 
-        BRACES.each do |_, _, name|
-          mixin :"braces_#{name}"
-        end
-      end
-
-      BRACES.each do |lbrace, rbrace, name|
-        state :"braces_#{name}" do
-          rule /%[a-z]#{lbrace}/, Str::Double, :"braces_#{name}_intp"
-          rule /%[A-Z]#{lbrace}/, Str::Double, :"braces_#{name}_no_intp"
-        end
-
-        state :"braces_#{name}_intp" do
-          rule /#{rbrace}[a-z]*/, Str::Double, :pop!
-          mixin :enddoublestr
-        end
-
-        state :"braces_#{name}_no_intp" do
-          rule /.*#{rbrace}[a-z]*/, Str::Double, :pop!
-        end
       end
 
       state :dqs do
@@ -103,6 +77,57 @@ module Rouge
         mixin :interpoling
         rule /[^#"]+/, Str::Double
       end
+
+      state :sigil_strings do
+        # ~-sigiled strings
+        # ~(abc), ~[abc], ~<abc>, ~|abc|, ~r/abc/, etc
+        # Cribbed and adjusted from Ruby lexer
+        delimiter_map = { '{' => '}', '[' => ']', '(' => ')', '<' => '>' }
+        # Match a-z for custom sigils too
+        sigil_opens = Regexp.union(delimiter_map.keys + %w(| / ' "))
+        rule /~([A-Za-z])?(#{sigil_opens})/ do |m|
+          open = Regexp.escape(m[2])
+          close = Regexp.escape(delimiter_map[m[2]] || m[2])
+          interp = /[SRCW]/ === m[1]
+          toktype = Str::Other
+
+          puts "    open: #{open.inspect}" if @debug
+          puts "    close: #{close.inspect}" if @debug
+
+          # regexes
+          if 'Rr'.include? m[1]
+            toktype = Str::Regex
+            push :regex_flags
+          end
+
+          if 'Ww'.include? m[1]
+            push :list_flags
+          end
+
+          token toktype
+
+          push do
+            rule /#{close}/, toktype, :pop!
+
+            if interp
+              mixin :interpoling
+              rule /#/, toktype
+            else
+              rule /[\\#]/, toktype
+            end
+
+            rule /[^##{open}#{close}\\]+/m, toktype
+          end
+        end
+      end
+
+      state :regex_flags do
+        rule /[fgimrsux]*/, Str::Regex, :pop!
+      end
+
+      state :list_flags do
+        rule /[csa]?/, Str::Other, :pop!
+      end
     end
   end
 end