rouge 3.18.0

data/lib/rouge/lexers/reasonml.rb

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    load_lexer 'ocaml/common.rb'
+
+    class ReasonML < OCamlCommon
+      title "ReasonML"
+      desc 'New syntax on top of OCaml ecosystem (reasonml.github.io)'
+      tag 'reasonml'
+      filenames '*.re', '*.rei'
+      mimetypes 'text/x-reasonml'
+
+      def self.keywords
+        @keywords ||= super + Set.new(%w(
+          switch
+        ))
+      end
+
+      state :root do
+        rule %r/\s+/m, Text
+        rule %r/false|true|[(][)]|\[\]/, Name::Builtin::Pseudo
+        rule %r/#{@@upper_id}(?=\s*[.])/, Name::Namespace, :dotted
+        rule %r/`#{@@id}/, Name::Tag
+        rule @@upper_id, Name::Class
+        rule %r/[\/][*](?![\/])/, Comment, :comment
+        rule @@id do |m|
+          match = m[0]
+          if self.class.keywords.include? match
+            token Keyword
+          elsif self.class.word_operators.include? match
+            token Operator::Word
+          elsif self.class.primitives.include? match
+            token Keyword::Type
+          else
+            token Name
+          end
+        end
+
+        rule %r/[(){}\[\];]+/, Punctuation
+        rule @@operator, Operator
+
+        rule %r/-?\d[\d_]*(.[\d_]*)?(e[+-]?\d[\d_]*)/i, Num::Float
+        rule %r/0x\h[\h_]*/i, Num::Hex
+        rule %r/0o[0-7][0-7_]*/i, Num::Oct
+        rule %r/0b[01][01_]*/i, Num::Bin
+        rule %r/\d[\d_]*/, Num::Integer
+
+        rule %r/'(?:(\\[\\"'ntbr ])|(\\[0-9]{3})|(\\x\h{2}))'/, Str::Char
+        rule %r/'[^'\/]'/, Str::Char
+        rule %r/'/, Keyword
+        rule %r/"/, Str::Double, :string
+        rule %r/[~?]#{@@id}/, Name::Variable
+      end
+
+      state :comment do
+        rule %r|[^/*)]+|, Comment
+        rule %r|[/][*]|, Comment, :push
+        rule %r|[*][/]|, Comment, :pop!
+        rule %r|[*/]|, Comment
+      end
+    end
+  end
+end

data/lib/rouge/lexers/rego.rb

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+    module Lexers
+        class Rego < RegexLexer
+            title "Rego"
+            desc "The Rego open-policy-agent (OPA) policy language (openpolicyagent.org)"
+            tag 'rego'
+            filenames '*.rego'
+
+            state :basic do
+              rule %r/\s+/, Text
+              rule %r/#.*/, Comment::Single
+      
+              rule %r/[\[\](){}|.,;!]/, Punctuation
+      
+              rule %r/"[^"]*"/, Str::Double
+      
+              rule %r/-?\d+\.\d+([eE][+-]?\d+)?/, Num::Float
+              rule %r/-?\d+([eE][+-]?\d+)?/, Num
+
+              rule %r/\\u[0-9a-fA-F]{4}/, Num::Hex
+              rule %r/\\["\/bfnrt]/, Str::Escape
+            end
+      
+            state :atoms do
+              rule %r/(true|false|null)/, Keyword::Constant
+              rule %r/[[:word:]]*/, Str::Symbol
+            end
+      
+            state :operators do
+              rule %r/(=|!=|>=|<=|>|<|\+|-|\*|%|\/|\||&|:=)/, Operator
+              rule %r/(default|not|package|import|as|with|else|some)/, Operator
+              rule %r/[\/:?@^~]+/, Operator
+            end
+      
+            state :root do
+              mixin :basic
+              mixin :operators
+              mixin :atoms
+            end
+        end
+    end
+end

data/lib/rouge/lexers/robot_framework.rb

@@ -0,0 +1,249 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class RobotFramework < RegexLexer
+      tag 'robot_framework'
+      aliases 'robot', 'robot-framework'
+
+      title "Robot Framework"
+      desc 'Robot Framework is a generic open source automation testing framework (robotframework.org)'
+
+      filenames '*.robot'
+      mimetypes 'text/x-robot'
+
+      def initialize(opts = {})
+        super(opts)
+        @col = 0
+        @next = nil
+        @is_template = false
+      end
+
+      def self.settings_with_keywords
+        @settings_with_keywords ||= Set.new [
+          "library", "resource", "setup", "teardown", "template", "suite setup",
+          "suite teardown", "task setup", "task teardown", "task template",
+          "test setup", "test teardown", "test template", "variables"
+        ]
+      end
+      
+      def self.settings_with_args
+        @settings_with_args ||= Set.new [
+          "arguments", "default tags", "documentation", "force tags",
+          "metadata", "return", "tags", "timeout", "task timeout",
+          "test timeout"
+        ]
+      end
+
+      id = %r/(?:\\|[^|$@&% \t\n])+(?: (?:\\.|[^|$@&% \t\n])+)*/
+      bdd = %r/(?:Given|When|Then|And|But) /i
+      sep = %r/ +\| +|[ ]{2,}|\t+/
+
+      start do
+        push :prior_text
+      end
+
+      state :prior_text do
+        rule %r/^[^*].*/, Text
+        rule(//) { pop! }
+      end
+
+      # Mixins
+
+      state :whitespace do
+        rule %r/\s+/, Text::Whitespace
+      end
+
+      state :section_include do
+        mixin :end_section
+        mixin :sep
+        mixin :newline
+      end
+
+      state :end_section do
+        rule(/(?=^(?:\| )?\*)/) { pop! }
+      end
+
+      state :return do
+        rule(//) { pop! }
+      end
+
+      state :sep do
+        rule %r/\| /, Text::Whitespace
+
+        rule sep do
+          token Text::Whitespace
+          @col = @col + 1
+          if @next
+            push @next
+          elsif @is_template
+            push :args
+          elsif @col == 1
+            @next = :keyword
+            push :keyword
+          else
+            push :args
+          end
+          push :cell_start
+        end
+
+        rule %r/\.\.\. */ do
+          token Text::Whitespace
+          @col = @col + 1
+          push :args
+        end
+        
+        rule %r/ ?\|/, Text::Whitespace
+      end
+
+      state :newline do
+        rule %r/\n/ do
+          token Text::Whitespace
+          @col = 0
+          @next = nil
+          push :cell_start
+        end
+      end
+
+      # States
+
+      state :root do
+        mixin :whitespace
+
+        rule %r/^(?:\| )?\*[* ]*([A-Z]+(?: [A-Z]+)?).*/i do |m|
+          token Generic::Heading, m[0]
+          case m[1].chomp("s").downcase
+          when "setting" then push :section_settings
+          when "test case" then push :section_tests
+          when "task" then push :section_tasks
+          when "keyword" then push :section_keywords
+          when "variable" then push :section_variables
+          end
+        end
+      end
+
+      state :section_settings do
+        mixin :section_include
+
+        rule %r/([A-Z]+(?: [A-Z]+)?)(:?)/i do |m|
+          match = m[1].downcase
+          @next = if self.class.settings_with_keywords.include? match
+                    :keyword
+                  elsif self.class.settings_with_args.include? match
+                    :args
+                  end
+          groups Name::Builtin::Pseudo, Punctuation
+        end
+      end
+
+      state :section_tests do
+        mixin :section_include
+        
+        rule %r/[$@&%{}]+/, Name::Label
+        rule %r/( )(?![ |])/, Name::Label
+
+        rule id do
+          @is_template = false
+          token Name::Label
+        end
+      end
+
+      state :section_tasks do
+        mixin :section_tests
+      end
+
+      state :section_keywords do
+        mixin :section_include
+  
+        rule %r/[$@&%]\{/ do
+          token Name::Variable
+          push :var
+        end
+        
+        rule %r/[$@&%{}]+/, Name::Label
+        rule %r/( )(?![ |])/, Name::Label
+        
+        rule id, Name::Label
+      end
+
+      state :section_variables do
+        mixin :section_include
+
+        rule %r/[$@&%]\{/ do
+          token Name::Variable
+          @next = :args
+          push :var
+        end
+      end
+
+      state :cell_start do
+        rule %r/#.*/, Comment
+        mixin :return
+      end
+
+      state :keyword do
+        rule %r/(\[)([A-Z]+(?: [A-Z]+)?)(\])/i do |m|
+          groups Punctuation, Name::Builtin::Pseudo, Punctuation
+          
+          match = m[2].downcase
+          @is_template = true if match == "template"
+          if self.class.settings_with_keywords.include? match
+            @next = :keyword
+          elsif self.class.settings_with_args.include? match
+            @next = :args
+          end
+
+          pop!
+        end
+
+        rule %r/[$@&%]\{/ do
+          token Name::Variable
+          @next = :keyword unless @next.nil?
+          push :var
+        end
+
+        rule %r/FOR/i do
+          token Name::Function
+          @next = :keyword unless @next.nil?
+        end
+
+        rule %r/( )(?![ |])/, Name::Function
+
+        rule bdd, Name::Builtin
+        rule id do
+          token Name::Function
+          @next = nil
+        end
+
+        mixin :return
+      end
+
+      state :args do
+        rule %r/[$@&%]\{/ do
+          token Name::Variable
+          @next = :keyword unless @next.nil?
+          push :var
+        end
+
+        rule %r/[$@&%]+/, Str
+        rule %r/( )(?![ |])/, Str
+        rule id, Str
+        
+        mixin :return
+      end
+
+      state :var do
+        rule %r/(\})( )(=)/ do
+          groups Name::Variable, Text::Whitespace, Punctuation
+          pop!
+        end
+        rule %r/[$@&%]\{/, Name::Variable, :var
+        rule %r/[{\[]/, Name::Variable, :var
+        rule %r/[}\]]/, Name::Variable, :pop!
+        rule %r/[^$@&%{}\[\]]+/, Name::Variable
+        rule %r/\}\[/, Name::Variable
+      end
+    end
+  end
+end

data/lib/rouge/lexers/ruby.rb

@@ -0,0 +1,454 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Ruby < RegexLexer
+      title "Ruby"
+      desc "The Ruby programming language (ruby-lang.org)"
+      tag 'ruby'
+      aliases 'rb'
+      filenames '*.rb', '*.ruby', '*.rbw', '*.rake', '*.gemspec', '*.podspec',
+                'Rakefile', 'Guardfile', 'Gemfile', 'Capfile', 'Podfile',
+                'Vagrantfile', '*.ru', '*.prawn', 'Berksfile', '*.arb',
+                'Dangerfile', 'Fastfile', 'Deliverfile', 'Appfile'
+
+      mimetypes 'text/x-ruby', 'application/x-ruby'
+
+      def self.detect?(text)
+        return true if text.shebang? 'ruby'
+      end
+
+      state :symbols do
+        # symbols
+        rule %r(
+          :  # initial :
+          @{0,2} # optional ivar, for :@foo and :@@foo
+          [a-z_]\w*[!?]? # the symbol
+        )xi, Str::Symbol
+
+        # special symbols
+        rule %r(:(?:\*\*|[-+]@|[/\%&\|^`~]|\[\]=?|<<|>>|<=?>|<=?|===?)),
+          Str::Symbol
+
+        rule %r/:'(\\\\|\\'|[^'])*'/, Str::Symbol
+        rule %r/:"/, Str::Symbol, :simple_sym
+      end
+
+      state :sigil_strings do
+        # %-sigiled strings
+        # %(abc), %[abc], %<abc>, %.abc., %r.abc., etc
+        delimiter_map = { '{' => '}', '[' => ']', '(' => ')', '<' => '>' }
+        rule %r/%([rqswQWxiI])?([^\w\s])/ do |m|
+          open = Regexp.escape(m[2])
+          close = Regexp.escape(delimiter_map[m[2]] || m[2])
+          interp = /[rQWxI]/ === m[1]
+          toktype = Str::Other
+
+          puts "    open: #{open.inspect}" if @debug
+          puts "    close: #{close.inspect}" if @debug
+
+          # regexes
+          if m[1] == 'r'
+            toktype = Str::Regex
+            push :regex_flags
+          end
+
+          token toktype
+
+          push do
+            uniq_chars = "#{open}#{close}".squeeze
+            uniq_chars = '' if open == close && open == "\\#"
+            rule %r/\\[##{uniq_chars}\\]/, Str::Escape
+            # nesting rules only with asymmetric delimiters
+            if open != close
+              rule %r/#{open}/ do
+                token toktype
+                push
+              end
+            end
+            rule %r/#{close}/, toktype, :pop!
+
+            if interp
+              mixin :string_intp_escaped
+              rule %r/#/, toktype
+            else
+              rule %r/[\\#]/, toktype
+            end
+
+            rule %r/[^##{uniq_chars}\\]+/m, toktype
+          end
+        end
+      end
+
+      state :strings do
+        mixin :symbols
+        rule %r/\b[a-z_]\w*?[?!]?:\s+/, Str::Symbol, :expr_start
+        rule %r/'(\\\\|\\'|[^'])*'/, Str::Single
+        rule %r/"/, Str::Double, :simple_string
+        rule %r/(?<!\.)`/, Str::Backtick, :simple_backtick
+      end
+
+      state :regex_flags do
+        rule %r/[mixounse]*/, Str::Regex, :pop!
+      end
+
+      # double-quoted string and symbol
+      [[:string, Str::Double, '"'],
+       [:sym, Str::Symbol, '"'],
+       [:backtick, Str::Backtick, '`']].each do |name, tok, fin|
+        state :"simple_#{name}" do
+          mixin :string_intp_escaped
+          rule %r/[^\\#{fin}#]+/m, tok
+          rule %r/[\\#]/, tok
+          rule %r/#{fin}/, tok, :pop!
+        end
+      end
+
+      keywords = %w(
+        BEGIN END alias begin break case defined\? do else elsif end
+        ensure for if in next redo rescue raise retry return super then
+        undef unless until when while yield
+      )
+
+      keywords_pseudo = %w(
+        loop include extend raise
+        alias_method attr catch throw private module_function
+        public protected true false nil __FILE__ __LINE__
+      )
+
+      builtins_g = %w(
+        attr_reader attr_writer attr_accessor
+
+        __id__ __send__ abort ancestors at_exit autoload binding callcc
+        caller catch chomp chop class_eval class_variables clone
+        const_defined\? const_get const_missing const_set constants
+        display dup eval exec exit extend fail fork format freeze
+        getc gets global_variables gsub hash id included_modules
+        inspect instance_eval instance_method instance_methods
+        instance_variable_get instance_variable_set instance_variables
+        lambda load local_variables loop method method_missing
+        methods module_eval name object_id open p print printf
+        private_class_method private_instance_methods private_methods proc
+        protected_instance_methods protected_methods public_class_method
+        public_instance_methods public_methods putc puts raise rand
+        readline readlines require require_relative scan select self send set_trace_func
+        singleton_methods sleep split sprintf srand sub syscall system
+        taint test throw to_a to_s trace_var trap untaint untrace_var warn
+      )
+
+      builtins_q = %w(
+        autoload block_given const_defined eql equal frozen
+        include instance_of is_a iterator kind_of method_defined
+        nil private_method_defined protected_method_defined
+        public_method_defined respond_to tainted
+      )
+
+      builtins_b = %w(chomp chop exit gsub sub)
+
+      start do
+        push :expr_start
+        @heredoc_queue = []
+      end
+
+      state :whitespace do
+        mixin :inline_whitespace
+        rule %r/\n\s*/m, Text, :expr_start
+        rule %r/#.*$/, Comment::Single
+
+        rule %r(=begin\b.*?\n=end\b)m, Comment::Multiline
+      end
+
+      state :inline_whitespace do
+        rule %r/[ \t\r]+/, Text
+      end
+
+      state :root do
+        mixin :whitespace
+        rule %r/__END__/, Comment::Preproc, :end_part
+
+        rule %r/0_?[0-7]+(?:_[0-7]+)*/, Num::Oct
+        rule %r/0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/, Num::Hex
+        rule %r/0b[01]+(?:_[01]+)*/, Num::Bin
+
+        decimal = %r/[\d]+(?:_\d+)*/
+        exp = %r/e[+-]?\d+/i
+        rule %r/#{decimal}(?:\.#{decimal}#{exp}?|#{exp})/, Num::Float
+        rule decimal, Num::Integer
+
+        # names
+        rule %r/@@[a-z_]\w*/i, Name::Variable::Class
+        rule %r/@[a-z_]\w*/i, Name::Variable::Instance
+        rule %r/\$\w+/, Name::Variable::Global
+        rule %r(\$[!@&`'+~=/\\,;.<>_*\$?:"]), Name::Variable::Global
+        rule %r/\$-[0adFiIlpvw]/, Name::Variable::Global
+        rule %r/::/, Operator
+
+        mixin :strings
+
+        rule %r/(?:#{keywords.join('|')})(?=\W|$)/, Keyword, :expr_start
+        rule %r/(?:#{keywords_pseudo.join('|')})\b/, Keyword::Pseudo, :expr_start
+
+        rule %r(
+          (module)
+          (\s+)
+          ([a-zA-Z_][a-zA-Z0-9_]*(::[a-zA-Z_][a-zA-Z0-9_]*)*)
+        )x do
+          groups Keyword, Text, Name::Namespace
+        end
+
+        rule %r/(def\b)(\s*)/ do
+          groups Keyword, Text
+          push :funcname
+        end
+
+        rule %r/(class\b)(\s*)/ do
+          groups Keyword, Text
+          push :classname
+        end
+
+        rule %r/(?:#{builtins_q.join('|')})[?]/, Name::Builtin, :expr_start
+        rule %r/(?:#{builtins_b.join('|')})!/,  Name::Builtin, :expr_start
+        rule %r/(?<!\.)(?:#{builtins_g.join('|')})\b/,
+          Name::Builtin, :method_call
+
+        mixin :has_heredocs
+
+        # `..` and `...` for ranges must have higher priority than `.`
+        # Otherwise, they will be parsed as :method_call
+        rule %r/\.{2,3}/, Operator, :expr_start
+
+        rule %r/[A-Z][a-zA-Z0-9_]*/, Name::Constant, :method_call
+        rule %r/(\.|::)(\s*)([a-z_]\w*[!?]?|[*%&^`~+-\/\[<>=])/ do
+          groups Punctuation, Text, Name::Function
+          push :method_call
+        end
+
+        rule %r/[a-zA-Z_]\w*[?!]/, Name, :expr_start
+        rule %r/[a-zA-Z_]\w*/, Name, :method_call
+        rule %r/\*\*|<<?|>>?|>=|<=|<=>|=~|={3}|!~|&&?|\|\||\./,
+          Operator, :expr_start
+        rule %r/[-+\/*%=<>&!^|~]=?/, Operator, :expr_start
+        rule(/[?]/) { token Punctuation; push :ternary; push :expr_start }
+        rule %r<[\[({,:\\;/]>, Punctuation, :expr_start
+        rule %r<[\])}]>, Punctuation
+      end
+
+      state :has_heredocs do
+        rule %r/(?<!\w)(<<[-~]?)(["`']?)([a-zA-Z_]\w*)(\2)/ do |m|
+          token Operator, m[1]
+          token Name::Constant, "#{m[2]}#{m[3]}#{m[4]}"
+          @heredoc_queue << [['<<-', '<<~'].include?(m[1]), m[3]]
+          push :heredoc_queue unless state? :heredoc_queue
+        end
+
+        rule %r/(<<[-~]?)(["'])(\2)/ do |m|
+          token Operator, m[1]
+          token Name::Constant, "#{m[2]}#{m[3]}#{m[4]}"
+          @heredoc_queue << [['<<-', '<<~'].include?(m[1]), '']
+          push :heredoc_queue unless state? :heredoc_queue
+        end
+      end
+
+      state :heredoc_queue do
+        rule %r/(?=\n)/ do
+          goto :resolve_heredocs
+        end
+
+        mixin :root
+      end
+
+      state :resolve_heredocs do
+        mixin :string_intp_escaped
+
+        rule %r/\n/, Str::Heredoc, :test_heredoc
+        rule %r/[#\\\n]/, Str::Heredoc
+        rule %r/[^#\\\n]+/, Str::Heredoc
+      end
+
+      state :test_heredoc do
+        rule %r/[^#\\\n]*$/ do |m|
+          tolerant, heredoc_name = @heredoc_queue.first
+          check = tolerant ? m[0].strip : m[0].rstrip
+
+          # check if we found the end of the heredoc
+          puts "    end heredoc check #{check.inspect} = #{heredoc_name.inspect}" if @debug
+          if check == heredoc_name
+            @heredoc_queue.shift
+            # if there's no more, we're done looking.
+            pop! if @heredoc_queue.empty?
+            token Name::Constant
+          else
+            token Str::Heredoc
+          end
+
+          pop!
+        end
+
+        rule(//) { pop! }
+      end
+
+      state :funcname do
+        rule %r/\s+/, Text
+        rule %r/\(/, Punctuation, :defexpr
+        rule %r(
+          (?:([a-zA-Z_]\w*)(\.))?
+          (
+            [a-zA-Z_]\w*[!?]? |
+            \*\*? | [-+]@? | [/%&\|^`~] | \[\]=? |
+            <<? | >>? | <=>? | >= | ===?
+          )
+        )x do |m|
+          puts "matches: #{[m[0], m[1], m[2], m[3]].inspect}" if @debug
+          groups Name::Class, Operator, Name::Function
+          pop!
+        end
+
+        rule(//) { pop! }
+      end
+
+      state :classname do
+        rule %r/\s+/, Text
+        rule %r/\w+(::\w+)+/, Name::Class
+
+        rule %r/\(/ do
+          token Punctuation
+          push :defexpr
+          push :expr_start
+        end
+
+        # class << expr
+        rule %r/<</ do
+          token Operator
+          goto :expr_start
+        end
+
+        rule %r/[A-Z_]\w*/, Name::Class, :pop!
+
+        rule(//) { pop! }
+      end
+
+      state :ternary do
+        rule %r/(:)(\s+)/ do
+          groups Punctuation, Text
+          goto :expr_start
+        end
+
+        rule %r/:(?![^#\n]*?[:\\])/ do
+          token Punctuation
+          goto :expr_start
+        end
+
+        mixin :root
+      end
+
+      state :defexpr do
+        rule %r/(\))(\.|::)?/ do
+          groups Punctuation, Operator
+          pop!
+        end
+        rule %r/\(/ do
+          token Punctuation
+          push :defexpr
+          push :expr_start
+        end
+
+        mixin :root
+      end
+
+      state :in_interp do
+        rule %r/}/, Str::Interpol, :pop!
+        mixin :root
+      end
+
+      state :string_intp do
+        rule %r/[#][{]/, Str::Interpol, :in_interp
+        rule %r/#(@@?|\$)[a-z_]\w*/i, Str::Interpol
+      end
+
+      state :string_intp_escaped do
+        mixin :string_intp
+        rule %r/\\([\\abefnrstv#"']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})/,
+          Str::Escape
+        rule %r/\\./, Str::Escape
+      end
+
+      state :method_call do
+        rule %r(/) do
+          token Operator
+          goto :expr_start
+        end
+
+        rule(/(?=\n)/) { pop! }
+
+        rule(//) { goto :method_call_spaced }
+      end
+
+      state :method_call_spaced do
+        mixin :whitespace
+
+        rule %r([%/]=) do
+          token Operator
+          goto :expr_start
+        end
+
+        rule %r((/)(?=\S|\s*/)) do
+          token Str::Regex
+          goto :slash_regex
+        end
+
+        mixin :sigil_strings
+
+        rule(%r((?=\s*/))) { pop! }
+
+        rule(/\s+/) { token Text; goto :expr_start }
+        rule(//) { pop! }
+      end
+
+      state :expr_start do
+        mixin :inline_whitespace
+
+        rule %r(/) do
+          token Str::Regex
+          goto :slash_regex
+        end
+
+        # char operator.  ?x evaulates to "x", unless there's a digit
+        # beforehand like x>=0?n[x]:""
+        rule %r(
+          [?](\\[MC]-)*     # modifiers
+          (\\([\\abefnrstv\#"']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)
+          (?!\w)
+        )x, Str::Char, :pop!
+
+        # special case for using a single space.  Ruby demands that
+        # these be in a single line, otherwise it would make no sense.
+        rule %r/(\s*)(%[rqswQWxiI]? \S* )/ do
+          groups Text, Str::Other
+          pop!
+        end
+
+        mixin :sigil_strings
+
+        rule(//) { pop! }
+      end
+
+      state :slash_regex do
+        mixin :string_intp
+        rule %r(\\\\), Str::Regex
+        rule %r(\\/), Str::Regex
+        rule %r([\\#]), Str::Regex
+        rule %r([^\\/#]+)m, Str::Regex
+        rule %r(/) do
+          token Str::Regex
+          goto :regex_flags
+        end
+      end
+
+      state :end_part do
+        # eat up the rest of the stream as Comment::Preproc
+        rule %r/.+/m, Comment::Preproc, :pop!
+      end
+    end
+  end
+end