rouge 4.1.3 → 4.7.0

data/lib/rouge/lexers/php.rb

@@ -100,6 +100,11 @@ module Rouge
       end
 
       state :names do
+        rule %r/(?:public|protected|private)\(set\)/i do
+          push :in_visibility
+          token Keyword
+        end
+
         rule %r/#{id_with_ns}(?=\s*\()/ do |m|
           name = m[0].downcase
           if self.class.keywords.include? name
@@ -179,7 +184,7 @@ module Rouge
 
         # numbers
         rule %r/(\d[_\d]*)?\.(\d[_\d]*)?(e[+-]?\d[_\d]*)?/i, Num::Float
-        rule %r/0[0-7][0-7_]*/, Num::Oct
+        rule %r/0o?[0-7][0-7_]*/i, Num::Oct
         rule %r/0b[01][01_]*/i, Num::Bin
         rule %r/0x[a-f0-9][a-f0-9_]*/i, Num::Hex
         rule %r/\d[_\d]*/, Num::Integer
@@ -243,6 +248,13 @@ module Rouge
           groups Keyword::Declaration, Text, Name::Class
         end
 
+        rule %r/(enum)
+                (\s+)
+                (#{id_with_ns})/ix do |m|
+          groups Keyword::Declaration, Text, Name::Class
+          push :in_enum
+        end
+
         mixin :names
 
         rule %r/[;,\(\)\{\}\[\]]/, Punctuation
@@ -280,6 +292,9 @@ module Rouge
       end
 
       state :in_const do
+        rule %r/(\??#{id})(\s+)(#{id})/i do
+          groups Keyword::Type, Text, Name::Constant
+        end
         rule id, Name::Constant
         rule %r/=/, Operator, :in_assign
         mixin :escape
@@ -307,7 +322,8 @@ module Rouge
         rule %r/,/, Punctuation
         rule %r/[.]{3}/, Punctuation
         rule %r/=/, Operator, :in_assign
-        rule %r/\b(?:public|protected|private|readonly)\b/i, Keyword
+        rule %r/\b(?:public|protected|private|readonly)(?:\(set\)|\b)/i, Keyword
+        rule %r/\breadonly\b/i, Keyword
         rule %r/\??#{id}/, Keyword::Type, :in_assign
         mixin :escape
         mixin :whitespace
@@ -369,6 +385,7 @@ module Rouge
       end
 
       state :in_visibility do
+        rule %r/\b(?:public|protected|private)(?:\(set\)|\b)/i, Keyword
         rule %r/\b(?:readonly|static)\b/i, Keyword
         rule %r/(?=(abstract|const|function)\b)/i, Keyword, :pop!
         rule %r/\??#{id}/, Keyword::Type, :pop!
@@ -376,6 +393,29 @@ module Rouge
         mixin :whitespace
         mixin :return
       end
+
+      state :in_enum do
+        rule %r/:/, Punctuation, :in_enum_base_type
+        rule %r/\{/, Punctuation, :in_enum_body
+        mixin :escape
+        mixin :whitespace
+        mixin :return
+      end
+
+      state :in_enum_base_type do
+        rule id, Keyword::Type, :pop!
+        mixin :escape
+        mixin :whitespace
+        mixin :return
+      end
+
+      state :in_enum_body do
+        rule %r/\}/, Punctuation, :pop!
+        rule %r/(case)(\s+)(#{id})/i do
+          groups Keyword, Text, Name::Constant
+        end
+        mixin :php
+      end
     end
   end
 end

data/lib/rouge/lexers/plsql.rb

@@ -472,7 +472,7 @@ module Rouge
         # A double-quoted string refers to a database object in our default SQL
         rule %r/"/, Operator, :double_string
         # preprocessor directive treated as special comment
-        rule %r/(\$(?:IF|THEN|ELSE|ELSIF|ERROR|END|(?:\$\$?\w[\w\d]*)))(\s+)/im do
+        rule %r/(\$(?:IF|THEN|ELSE|ELSIF|ERROR|END|(?:\$\$?[a-z]\w*)))(\s+)/im do
           groups Comment::Preproc, Text
         end
 
@@ -503,7 +503,7 @@ module Rouge
         # Special processing for keywords with multiple contexts
         #
         # this madness is to keep the word "replace" from being treated as a builtin function in this context
-        rule %r/(create)(\s+)(?:(or)(\s+)(replace)(\s+))?(package|function|procedure|type)(?:(\s+)(body))?(\s+)(\w[\w\d\$]*)/im do
+        rule %r/(create)(\s+)(?:(or)(\s+)(replace)(\s+))?(package|function|procedure|type)(?:(\s+)(body))?(\s+)([a-z][\w$]*)/im do
           groups Keyword::Reserved, Text, Keyword::Reserved, Text, Keyword::Reserved, Text, Keyword::Reserved, Text, Keyword::Reserved, Text, Name
         end
         # similar for MERGE keywords
@@ -515,7 +515,7 @@ module Rouge
         # General keyword classification with sepcial attention to names
         # in a chained "dot" notation.
         #
-        rule %r/(\w[\w\d\$]*)(\.(?=\w))?/ do |m|
+        rule %r/([a-zA-Z][\w$]*)(\.(?=\w))?/ do |m|
           if self.class.keywords_type.include? m[1].upcase
             tok = Keyword::Type 
           elsif self.class.keywords_func.include? m[1].upcase
@@ -556,11 +556,11 @@ module Rouge
 
       state :dotnames do
         # if we are followed by a dot and another name, we are an ordinary name
-        rule %r/(\w[\w\d\$]*)(\.(?=\w))/ do
+        rule %r/([a-zA-Z][\w\$]*)(\.(?=\w))/ do
           groups Name, Punctuation
         end
         # this rule WILL be true if something pushed into our state. That is our state contract
-        rule %r/\w[\w\d\$]*/ do |m|
+        rule %r/[a-zA-Z][\w\$]*/ do |m|
           if self.class.keywords_func.include? m[0].upcase
             # The Function lookup allows collection methods like COUNT, FIRST, LAST, etc.. to be 
             # classified correctly. Occasionally misidentifies ordinary names as builtin functions,

data/lib/rouge/lexers/python.rb

@@ -8,8 +8,8 @@ module Rouge
       desc "The Python programming language (python.org)"
       tag 'python'
       aliases 'py'
-      filenames '*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac',
-                '*.bzl', 'BUCK', 'BUILD', 'BUILD.bazel', 'WORKSPACE'
+      filenames '*.py', '*.pyi', '*.pyw', '*.sc', 'SConstruct', 'SConscript',
+                '*.tac', '*.bzl', 'BUCK', 'BUILD', 'BUILD.bazel', 'WORKSPACE'
       mimetypes 'text/x-python', 'application/x-python'
 
       def self.detect?(text)
@@ -20,22 +20,24 @@ module Rouge
         @keywords ||= %w(
           assert break continue del elif else except exec
           finally for global if lambda pass print raise
-          return try while yield as with from import yield
+          return try while yield as with from import
           async await nonlocal
         )
       end
 
       def self.builtins
         @builtins ||= %w(
-          __import__ abs all any apply ascii basestring bin bool buffer
-          bytearray bytes callable chr classmethod cmp coerce compile
-          complex delattr dict dir divmod enumerate eval execfile exit
-          file filter float format frozenset getattr globals hasattr hash hex id
-          input int intern isinstance issubclass iter len list locals
-          long map max memoryview min next object oct open ord pow property range
-          raw_input reduce reload repr reversed round set setattr slice
-          sorted staticmethod str sum super tuple type unichr unicode
-          vars xrange zip
+          __import__ abs aiter all anext any apply ascii
+          basestring bin bool buffer breakpoint bytearray bytes
+          callable chr classmethod cmp coerce compile complex
+          delattr dict dir divmod enumerate eval exec execfile exit
+          file filter float format frozenset getattr globals
+          hasattr hash help hex
+          id input int intern isinstance issubclass iter len list locals long
+          map max memoryview min next object oct open ord pow print property
+          range raw_input reduce reload repr reversed round set setattr slice
+          sorted staticmethod str sum super tuple type unichr unicode vars
+          xrange zip
         )
       end
 
@@ -45,25 +47,26 @@ module Rouge
 
       def self.exceptions
         @exceptions ||= %w(
-          ArithmeticError AssertionError AttributeError
-          BaseException BlockingIOError BrokenPipeError BufferError
-          BytesWarning ChildProcessError ConnectionAbortedError
-          ConnectionError ConnectionRefusedError ConnectionResetError
-          DeprecationWarning EOFError EnvironmentError
-          Exception FileExistsError FileNotFoundError
-          FloatingPointError FutureWarning GeneratorExit IOError
-          ImportError ImportWarning IndentationError IndexError
-          InterruptedError IsADirectoryError KeyError KeyboardInterrupt
-          LookupError MemoryError ModuleNotFoundError NameError
-          NotADirectoryError NotImplemented NotImplementedError OSError
-          OverflowError OverflowWarning PendingDeprecationWarning
-          ProcessLookupError RecursionError ReferenceError ResourceWarning
-          RuntimeError RuntimeWarning StandardError StopAsyncIteration
-          StopIteration SyntaxError SyntaxWarning SystemError SystemExit
-          TabError TimeoutError TypeError UnboundLocalError UnicodeDecodeError
-          UnicodeEncodeError UnicodeError UnicodeTranslateError
-          UnicodeWarning UserWarning ValueError VMSError Warning
-          WindowsError ZeroDivisionError
+          ArithmeticError AssertionError AttributeError BaseException
+          BaseExceptionGroup BlockingIOError BrokenPipeError BufferError
+          BytesWarning ChildProcessError ConnectionAbortedError ConnectionError
+          ConnectionRefusedError ConnectionResetError DeprecationWarning
+          EOFError EnvironmentError EncodingWarning Exception ExceptionGroup
+          FileExistsError FileNotFoundError FloatingPointError FutureWarning
+          GeneratorExit IOError ImportError ImportWarning IndentationError
+          IndexError InterruptedError IsADirectoryError
+          KeyError KeyboardInterrupt LookupError
+          MemoryError ModuleNotFoundError
+          NameError NotADirectoryError NotImplemented NotImplementedError
+          OSError OverflowError OverflowWarning PendingDeprecationWarning
+          PermissionError ProcessLookupError PythonFinalizationError
+          RecursionError ReferenceError ResourceWarning RuntimeError RuntimeWarning
+          StandardError StopAsyncIteration StopIteration SyntaxError SyntaxWarning
+          SystemError SystemExit TabError TimeoutError TypeError
+          UnboundLocalError UnicodeDecodeError UnicodeEncodeError UnicodeError
+          UnicodeTranslateError UnicodeWarning UserWarning ValueError VMSError
+          Warning WindowsError
+          ZeroDivisionError
         )
       end
 
@@ -121,12 +124,14 @@ module Rouge
 
         # TODO: not in python 3
         rule %r/`.*?`/, Str::Backtick
-        rule %r/([rfbu]{0,2})('''|"""|['"])/i do |m|
+        rule %r/([rtfbu]{0,2})('''|"""|['"])/i do |m|
           groups Str::Affix, Str::Heredoc
           current_string.register type: m[1].downcase, delim: m[2]
           push :generic_string
         end
 
+        mixin :soft_keywords
+
         # using negative lookbehind so we don't match property names
         rule %r/(?<!\.)#{identifier}/ do |m|
           if self.class.keywords.include? m[0]
@@ -166,6 +171,28 @@ module Rouge
         rule identifier, Name::Class, :pop!
       end
 
+      state :soft_keywords do
+        rule %r/
+          (^[ \t]*)
+          (match|case)\b
+          (?![ \t]*
+            (?:[:,;=^&|@~)\]}] |
+              (?:#{Python.keywords.join('|')})\b))
+        /x do |m|
+          token Text::Whitespace, m[1]
+          token Keyword, m[2]
+          push :soft_keywords_inner
+        end
+      end
+
+      state :soft_keywords_inner do
+        rule %r((\s+)([^\n_]*)(_\b)) do |m|
+          groups Text::Whitespace, Text, Keyword
+        end
+
+        rule(//) { pop! }
+      end
+
       state :raise do
         rule %r/from\b/, Keyword
         rule %r/raise\b/, Keyword

data/lib/rouge/lexers/robot_framework.rb

@@ -10,7 +10,7 @@ module Rouge
       title "Robot Framework"
       desc 'Robot Framework is a generic open source automation testing framework (robotframework.org)'
 
-      filenames '*.robot'
+      filenames '*.robot', '*.resource'
       mimetypes 'text/x-robot'
 
       def initialize(opts = {})

data/lib/rouge/lexers/ruby.rb

@@ -11,7 +11,7 @@ module Rouge
       filenames '*.rb', '*.ruby', '*.rbw', '*.rake', '*.gemspec', '*.podspec',
                 'Rakefile', 'Guardfile', 'Gemfile', 'Capfile', 'Podfile',
                 'Vagrantfile', '*.ru', '*.prawn', 'Berksfile', '*.arb',
-                'Dangerfile', 'Fastfile', 'Deliverfile', 'Appfile'
+                'Dangerfile', 'Fastfile', 'Deliverfile', 'Appfile', '*.thor', 'Thorfile'
 
       mimetypes 'text/x-ruby', 'application/x-ruby'
 
@@ -24,7 +24,7 @@ module Rouge
         rule %r(
           :  # initial :
           @{0,2} # optional ivar, for :@foo and :@@foo
-          [a-z_]\w*[!?]? # the symbol
+          [\p{Ll}_]\p{Word}*[!?]? # the symbol
         )xi, Str::Symbol
 
         # special symbols
@@ -39,7 +39,7 @@ module Rouge
         # %-sigiled strings
         # %(abc), %[abc], %<abc>, %.abc., %r.abc., etc
         delimiter_map = { '{' => '}', '[' => ']', '(' => ')', '<' => '>' }
-        rule %r/%([rqswQWxiI])?([^\w\s])/ do |m|
+        rule %r/%([rqswQWxiI])?([^\p{Word}\s])/ do |m|
           open = Regexp.escape(m[2])
           close = Regexp.escape(delimiter_map[m[2]] || m[2])
           interp = /[rQWxI]/ === m[1] || !m[1]
@@ -57,7 +57,7 @@ module Rouge
           token toktype
 
           push do
-            uniq_chars = "#{open}#{close}".squeeze
+            uniq_chars = [open, close].uniq.join
             uniq_chars = '' if open == close && open == "\\#"
             rule %r/\\[##{uniq_chars}\\]/, Str::Escape
             # nesting rules only with asymmetric delimiters
@@ -83,7 +83,7 @@ module Rouge
 
       state :strings do
         mixin :symbols
-        rule %r/\b[a-z_]\w*?[?!]?:\s+/, Str::Symbol, :expr_start
+        rule %r/\b[\p{Ll}_]\p{Word}*?[?!]?:\s+/, Str::Symbol, :expr_start
         rule %r/'(\\\\|\\'|[^'])*'/, Str::Single
         rule %r/"/, Str::Double, :simple_string
         rule %r/(?<!\.)`/, Str::Backtick, :simple_backtick
@@ -177,9 +177,9 @@ module Rouge
         rule decimal, Num::Integer
 
         # names
-        rule %r/@@[a-z_]\w*/i, Name::Variable::Class
-        rule %r/@[a-z_]\w*/i, Name::Variable::Instance
-        rule %r/\$\w+/, Name::Variable::Global
+        rule %r/@@[\p{Ll}_]\p{Word}*/i, Name::Variable::Class
+        rule %r/@[\p{Ll}_]\p{Word}*/i, Name::Variable::Instance
+        rule %r/\$\p{Word}+/, Name::Variable::Global
         rule %r(\$[!@&`'+~=/\\,;.<>_*\$?:"]), Name::Variable::Global
         rule %r/\$-[0adFiIlpvw]/, Name::Variable::Global
         rule %r/::/, Operator
@@ -193,7 +193,7 @@ module Rouge
         rule %r(
           (module)
           (\s+)
-          ([a-zA-Z_][a-zA-Z0-9_]*(::[a-zA-Z_][a-zA-Z0-9_]*)*)
+          ([\p{L}_][\p{L}0-9_]*(::[\p{L}_][\p{L}0-9_]*)*)
         )x do
           groups Keyword, Text, Name::Namespace
         end
@@ -219,14 +219,14 @@ module Rouge
         # Otherwise, they will be parsed as :method_call
         rule %r/\.{2,3}/, Operator, :expr_start
 
-        rule %r/[A-Z][a-zA-Z0-9_]*/, Name::Constant, :method_call
-        rule %r/(\.|::)(\s*)([a-z_]\w*[!?]?|[*%&^`~+-\/\[<>=])/ do
+        rule %r/[\p{Lu}][\p{L}0-9_]*/, Name::Constant, :method_call
+        rule %r/(\.|::)(\s*)([\p{Ll}_]\p{Word}*[!?]?|[*%&^`~+-\/\[<>=])/ do
           groups Punctuation, Text, Name::Function
           push :method_call
         end
 
-        rule %r/[a-zA-Z_]\w*[?!]/, Name, :expr_start
-        rule %r/[a-zA-Z_]\w*/, Name, :method_call
+        rule %r/[\p{L}_]\p{Word}*[?!]/, Name, :expr_start
+        rule %r/[\p{L}_]\p{Word}*/, Name, :method_call
         rule %r/\*\*|<<?|>>?|>=|<=|<=>|=~|={3}|!~|&&?|\|\||\./,
           Operator, :expr_start
         rule %r/[-+\/*%=<>&!^|~]=?/, Operator, :expr_start
@@ -236,7 +236,7 @@ module Rouge
       end
 
       state :has_heredocs do
-        rule %r/(?<!\w)(<<[-~]?)(["`']?)([a-zA-Z_]\w*)(\2)/ do |m|
+        rule %r/(?<!\p{Word})(<<[-~]?)(["`']?)([\p{L}_]\p{Word}*)(\2)/ do |m|
           token Operator, m[1]
           token Name::Constant, "#{m[2]}#{m[3]}#{m[4]}"
           @heredoc_queue << [['<<-', '<<~'].include?(m[1]), m[3]]
@@ -293,11 +293,11 @@ module Rouge
         rule %r/\s+/, Text
         rule %r/\(/, Punctuation, :defexpr
         rule %r(
-          (?:([a-zA-Z_]\w*)(\.))?
+          (?:([\p{L}_]\p{Word}*)(\.))?
           (
-            [a-zA-Z_]\w*[!?]? |
+            [\p{L}_]\p{Word}*[!?]? |
             \*\*? | [-+]@? | [/%&\|^`~] | \[\]=? |
-            <<? | >>? | <=>? | >= | ===?
+            <=>? | <<? | >>? | >= | ===?
           )
         )x do |m|
           puts "matches: #{[m[0], m[1], m[2], m[3]].inspect}" if @debug
@@ -310,7 +310,7 @@ module Rouge
 
       state :classname do
         rule %r/\s+/, Text
-        rule %r/\w+(::\w+)+/, Name::Class
+        rule %r/\p{Word}+(::\p{Word}+)+/, Name::Class
 
         rule %r/\(/ do
           token Punctuation
@@ -324,7 +324,7 @@ module Rouge
           goto :expr_start
         end
 
-        rule %r/[A-Z_]\w*/, Name::Class, :pop!
+        rule %r/[\p{Lu}_]\p{Word}*/, Name::Class, :pop!
 
         rule(//) { pop! }
       end
@@ -364,7 +364,7 @@ module Rouge
 
       state :string_intp do
         rule %r/[#][{]/, Str::Interpol, :in_interp
-        rule %r/#(@@?|\$)[a-z_]\w*/i, Str::Interpol
+        rule %r/#(@@?|\$)[\p{Ll}_]\p{Word}*/i, Str::Interpol
       end
 
       state :string_intp_escaped do
@@ -419,7 +419,7 @@ module Rouge
         rule %r(
           [?](\\[MC]-)*     # modifiers
           (\\([\\abefnrstv\#"']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)
-          (?!\w)
+          (?!\p{Word})
         )x, Str::Char, :pop!
 
         # special case for using a single space.  Ruby demands that

data/lib/rouge/lexers/sass/common.rb

@@ -155,7 +155,7 @@ module Rouge
       state :attr_common do
         mixin :has_interp
         rule id do |m|
-          if CSS.attributes.include? m[0]
+          if CSS.properties.include? m[0]
             token Name::Label
           else
             token Name::Attribute

data/lib/rouge/lexers/scala.rb

@@ -13,7 +13,8 @@ module Rouge
       mimetypes 'text/x-scala', 'application/x-scala'
 
       # As documented in the ENBF section of the scala specification
-      # http://www.scala-lang.org/docu/files/ScalaReference.pdf
+      # https://scala-lang.org/files/archive/spec/2.13/13-syntax-summary.html
+      # https://en.wikipedia.org/wiki/Unicode_character_property#General_Category
       whitespace = /\p{Space}/
       letter = /[\p{L}$_]/
       upper = /[\p{Lu}$_]/
@@ -24,8 +25,10 @@ module Rouge
       # negative lookahead to filter out other classes
       op = %r(
         (?!#{whitespace}|#{letter}|#{digits}|#{parens}|#{delims})
-        [\u0020-\u007F\p{Sm}\p{So}]
+        [-!#%&*/:?@\\^\p{Sm}\p{So}]
       )x
+      # manually removed +<=>|~ from regexp because they're in property Sm
+      # pp CHRS:(0x00..0x7f).map(&:chr).grep(/\p{Sm}/)
 
       idrest = %r(#{letter}(?:#{letter}|#{digits})*(?:(?<=_)#{op}+)?)x
 

data/lib/rouge/lexers/sql.rb

@@ -115,7 +115,7 @@ module Rouge
         rule %r/"/, Name::Variable, :double_string
         rule %r/`/, Name::Variable, :backtick
 
-        rule %r/\w[\w\d]*/ do |m|
+        rule %r/\w+/ do |m|
           if self.class.keywords_type.include? m[0].upcase
             token Name::Builtin
           elsif self.class.keywords.include? m[0].upcase

data/lib/rouge/lexers/svelte.rb

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    load_lexer 'html.rb'
+
+    class Svelte < HTML
+      desc 'Svelte single-file components (https://svelte.dev/)'
+      tag 'svelte'
+      filenames '*.svelte'
+      mimetypes 'text/x-svelte', 'application/x-svelte'
+
+      def initialize(*)
+        super
+        # todo add support for typescript script blocks
+        @js = Javascript.new(options)
+      end
+
+      # Shorthand syntax for passing attributes - ex, `{src}` instead of `src={src}`
+      prepend :tag do
+        rule %r/(\{)\s*([a-zA-Z0-9_]+)\s*(})/m do
+          groups Str::Interpol, Name::Variable, Str::Interpol
+          pop!
+        end
+      end
+
+      prepend :attr do
+        # Duplicate template_start mixin here with a pop!
+        # Because otherwise we'll never exit the attr state
+        rule %r/\{/ do
+          token Str::Interpol
+          pop!
+          push :template
+        end
+      end
+
+      # handle templates within attribute single/double quotes
+      prepend :dq do
+        mixin :template_start
+      end
+
+      prepend :sq do
+        mixin :template_start
+      end
+
+      prepend :root do
+        # detect curly braces within HTML text (outside of tags/attributes)
+        rule %r/([^<&{]*)(\{)(\s*)/ do
+          groups Text, Str::Interpol, Text
+          push :template
+        end
+      end
+
+      state :template_start do
+        # open template
+        rule %r/\s*\{\s*/, Str::Interpol, :template
+      end
+
+      state :template do
+        # template end
+        rule %r/}/, Str::Interpol, :pop!
+
+        # Allow JS lexer to handle matched curly braces within template
+        rule(/(?<=^|[^\\])\{+.*?(?<=^|[^\\])\}+/) do
+          delegate @js
+        end
+
+        # keywords
+        rule %r/@(debug|html)\b/, Keyword
+        rule %r/(#await)(.*)(then|catch)(\s+)(\w+)/ do |m|
+          token Keyword, m[1]
+          delegate @js, m[2]
+          token Keyword, m[3]
+          token Text, m[4]
+          delegate @js, m[5]
+        end
+        rule %r/([#\/])(await|each|if|key)\b/, Keyword
+        rule %r/(:else)(\s+)(if)?\b/ do
+          groups Keyword, Text, Keyword
+        end
+        rule %r/:?(catch|then)\b/, Keyword
+
+        # allow JS parser to handle anything that's not a curly brace
+        rule %r/[^{}]+/ do
+          delegate @js
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/syzlang.rb

@@ -27,7 +27,6 @@ module Rouge
 
       comment = /#.*$/
       inline_spaces = /[ \t]+/
-      eol_spaces = /[\n\r]+/
       spaces = /\s+/
 
       state :inline_break do

data/lib/rouge/lexers/tcl.rb

@@ -57,9 +57,9 @@ module Rouge
 
       def self.gen_command_state(name='')
         state(:"command#{name}") do
-          mixin :word
+          rule %r/#/, Comment, :comment
 
-          rule %r/##{NOT_CHARS[END_LINE]}+/, Comment::Single
+          mixin :word
 
           rule %r/(?=#{CHARS[END_WORD]})/ do
             push :"params#{name}"
@@ -163,6 +163,11 @@ module Rouge
         rule %r/\\./m, Str::Escape
       end
 
+      state :comment do
+        rule %r/.*[^\\]\n/, Comment, :pop!
+        rule %r/.*\\\n/, Comment
+      end
+
       state :string do
         rule %r/"/, Str::Double, :pop!
         mixin :interp

data/lib/rouge/lexers/terraform.rb

@@ -11,7 +11,7 @@ module Rouge
 
       tag 'terraform'
       aliases 'tf'
-      filenames '*.tf'
+      filenames '*.tf', '*.tfvars', '*.tofu'
 
       def self.keywords
         @keywords ||= Set.new %w(
@@ -39,8 +39,9 @@ module Rouge
 
       state :strings do
         rule %r/\\./, Str::Escape
+        rule %r/(\$[\$]+|%[%]+)(\{)/, Str
         rule %r/\$\{/ do
-          token Keyword
+          token Punctuation
           push :interpolation
         end
       end
@@ -66,7 +67,7 @@ module Rouge
 
       state :interpolation do
         rule %r/\}/ do
-          token Keyword
+          token Punctuation
           pop!
         end