rouge 4.0.0 → 4.1.0

data/lib/rouge/lexers/praat.rb

@@ -15,109 +15,138 @@ module Rouge
         return true if text.shebang? 'praat'
       end
 
-      keywords = %w(
-        if then else elsif elif endif fi for from to endfor endproc while
-        endwhile repeat until select plus minus demo assert stopwatch
-        nocheck nowarn noprogress editor endeditor clearinfo
-      )
-
-      functions_string = %w(
-        backslashTrigraphsToUnicode chooseDirectory chooseReadFile
-        chooseWriteFile date demoKey do environment extractLine extractWord
-        fixed info left mid percent readFile replace replace_regex right
-        selected string unicodeToBackslashTrigraphs
-      )
-
-      functions_numeric = %w(
-        abs appendFile appendFileLine appendInfo appendInfoLine arccos arccosh
-        arcsin arcsinh arctan arctan2 arctanh barkToHertz beginPause
-        beginSendPraat besselI besselK beta beta2 binomialP binomialQ boolean
-        ceiling chiSquareP chiSquareQ choice comment cos cosh createDirectory
-        deleteFile demoClicked demoClickedIn demoCommandKeyPressed
-        demoExtraControlKeyPressed demoInput demoKeyPressed
-        demoOptionKeyPressed demoShiftKeyPressed demoShow demoWaitForInput
-        demoWindowTitle demoX demoY differenceLimensToPhon do editor endPause
-        endSendPraat endsWith erb erbToHertz erf erfc exitScript exp
-        extractNumber fileReadable fisherP fisherQ floor gaussP gaussQ hash
-        hertzToBark hertzToErb hertzToMel hertzToSemitones imax imin
-        incompleteBeta incompleteGammaP index index_regex integer invBinomialP
-        invBinomialQ invChiSquareQ invFisherQ invGaussQ invSigmoid invStudentQ
-        length ln lnBeta lnGamma log10 log2 max melToHertz min minusObject
-        natural number numberOfColumns numberOfRows numberOfSelected
-        objectsAreIdentical option optionMenu pauseScript
-        phonToDifferenceLimens plusObject positive randomBinomial randomGauss
-        randomInteger randomPoisson randomUniform real readFile removeObject
-        rindex rindex_regex round runScript runSystem runSystem_nocheck
-        selectObject selected semitonesToHertz sentence sentencetext sigmoid
-        sin sinc sincpi sinh soundPressureToPhon sqrt startsWith studentP
-        studentQ tan tanh text variableExists word writeFile writeFileLine
-        writeInfo writeInfoLine
-      )
-
-      functions_array = %w(
-        linear randomGauss randomInteger randomUniform zero
-      )
-
-      functions_matrix = %w(
-        linear mul mul_fast mul_metal mul_nt mul_tn mul_tt outer peaks
-        randomGamma randomGauss randomInteger randomUniform softmaxPerRow
-        solve transpose zero
-      )
-
-      functions_string_vector = %w(
-        empty fileNames folderNames readLinesFromFile splitByWhitespace
-      )
-
-      objects = %w(
-        Activation AffineTransform AmplitudeTier Art Artword Autosegment
-        BarkFilter BarkSpectrogram CCA Categories Cepstrogram Cepstrum
-        Cepstrumc ChebyshevSeries ClassificationTable Cochleagram Collection
-        ComplexSpectrogram Configuration Confusion ContingencyTable Corpus
-        Correlation Covariance CrossCorrelationTable CrossCorrelationTableList
-        CrossCorrelationTables DTW DataModeler Diagonalizer Discriminant
-        Dissimilarity Distance Distributions DurationTier EEG ERP ERPTier
-        EditCostsTable EditDistanceTable Eigen Excitation Excitations
-        ExperimentMFC FFNet FeatureWeights FileInMemory FilesInMemory Formant
-        FormantFilter FormantGrid FormantModeler FormantPoint FormantTier
-        GaussianMixture HMM HMM_Observation HMM_ObservationSequence HMM_State
-        HMM_StateSequence HMMObservation HMMObservationSequence HMMState
-        HMMStateSequence Harmonicity ISpline Index Intensity IntensityTier
-        IntervalTier KNN KlattGrid KlattTable LFCC LPC Label LegendreSeries
-        LinearRegression LogisticRegression LongSound Ltas MFCC MSpline ManPages
-        Manipulation Matrix MelFilter MelSpectrogram MixingMatrix Movie Network
-        OTGrammar OTHistory OTMulti PCA PairDistribution ParamCurve Pattern
-        Permutation Photo Pitch PitchModeler PitchTier PointProcess Polygon
-        Polynomial PowerCepstrogram PowerCepstrum Procrustes RealPoint RealTier
-        ResultsMFC Roots SPINET SSCP SVD Salience ScalarProduct Similarity
-        SimpleString SortedSetOfString Sound Speaker Spectrogram Spectrum
-        SpectrumTier SpeechSynthesizer SpellingChecker Strings StringsIndex
-        Table TableOfReal TextGrid TextInterval TextPoint TextTier Tier
-        Transition VocalTract VocalTractTier Weight WordList
-      )
-
-      variables_numeric = %w(
-        all average e left macintosh mono pi praatVersion right stereo
-        undefined unix windows
-      )
-
-      variables_string = %w(
-        praatVersion tab shellDirectory homeDirectory
-        preferencesDirectory newline temporaryDirectory
-        defaultDirectory
-      )
-
-      object_attributes = %w(
-        ncol nrow xmin ymin xmax ymax nx ny dx dy
-      )
+      def self.keywords
+        @keywords ||= %w(
+          if then else elsif elif endif fi for from to endfor endproc while
+          endwhile repeat until select plus minus demo assert stopwatch
+          nocheck nowarn noprogress editor endeditor clearinfo
+        )
+      end
+
+      def self.functions_string
+        @functions_string ||= %w(
+          backslashTrigraphsToUnicode$ chooseDirectory$ chooseReadFile$
+          chooseWriteFile$ date$ demoKey$ do$ environment$ extractLine$ extractWord$
+          fixed$ info$ left$ mid$ percent$ readFile$ replace$ replace_regex$ right$
+          selected$ string$ unicodeToBackslashTrigraphs$
+        )
+      end
+
+      def self.functions_numeric
+        @functions_numeric ||= %w(
+          abs appendFile appendFileLine appendInfo appendInfoLine arccos arccosh
+          arcsin arcsinh arctan arctan2 arctanh barkToHertz beginPause
+          beginSendPraat besselI besselK beta beta2 binomialP binomialQ boolean
+          ceiling chiSquareP chiSquareQ choice comment cos cosh createDirectory
+          deleteFile demoClicked demoClickedIn demoCommandKeyPressed
+          demoExtraControlKeyPressed demoInput demoKeyPressed
+          demoOptionKeyPressed demoShiftKeyPressed demoShow demoWaitForInput
+          demoWindowTitle demoX demoY differenceLimensToPhon do editor endPause
+          endSendPraat endsWith erb erbToHertz erf erfc exitScript exp
+          extractNumber fileReadable fisherP fisherQ floor gaussP gaussQ hash
+          hertzToBark hertzToErb hertzToMel hertzToSemitones imax imin
+          incompleteBeta incompleteGammaP index index_regex integer invBinomialP
+          invBinomialQ invChiSquareQ invFisherQ invGaussQ invSigmoid invStudentQ
+          length ln lnBeta lnGamma log10 log2 max melToHertz min minusObject
+          natural number numberOfColumns numberOfRows numberOfSelected
+          objectsAreIdentical option optionMenu pauseScript
+          phonToDifferenceLimens plusObject positive randomBinomial randomGauss
+          randomInteger randomPoisson randomUniform real readFile removeObject
+          rindex rindex_regex round runScript runSystem runSystem_nocheck
+          selectObject selected semitonesToHertz sentence sentencetext sigmoid
+          sin sinc sincpi sinh soundPressureToPhon sqrt startsWith studentP
+          studentQ tan tanh text variableExists word writeFile writeFileLine
+          writeInfo writeInfoLine
+        )
+      end
+
+      def self.functions_array
+        @functions_array ||= %w(
+          linear# randomGauss# randomInteger# randomUniform# zero#
+        )
+      end
+
+      def self.functions_matrix
+        @functions_matrix ||= %w(
+          linear## mul## mul_fast## mul_metal## mul_nt## mul_tn## mul_tt## outer## peaks##
+          randomGamma## randomGauss## randomInteger## randomUniform## softmaxPerRow##
+          solve## transpose## zero##
+        )
+      end
+
+      def self.functions_string_vector
+        @functions_string_vector ||= %w(
+          empty$# fileNames$# folderNames$# readLinesFromFile$# splitByWhitespace$#
+        )
+      end
+
+      def self.functions_builtin
+        @functions_builtin ||=
+          self.functions_string |
+          self.functions_numeric |
+          self.functions_array |
+          self.functions_matrix |
+          self.functions_string_vector
+      end
+
+      def self.objects
+        @objects ||= %w(
+          Activation AffineTransform AmplitudeTier Art Artword Autosegment
+          BarkFilter BarkSpectrogram CCA Categories Cepstrogram Cepstrum
+          Cepstrumc ChebyshevSeries ClassificationTable Cochleagram Collection
+          ComplexSpectrogram Configuration Confusion ContingencyTable Corpus
+          Correlation Covariance CrossCorrelationTable CrossCorrelationTableList
+          CrossCorrelationTables DTW DataModeler Diagonalizer Discriminant
+          Dissimilarity Distance Distributions DurationTier EEG ERP ERPTier
+          EditCostsTable EditDistanceTable Eigen Excitation Excitations
+          ExperimentMFC FFNet FeatureWeights FileInMemory FilesInMemory Formant
+          FormantFilter FormantGrid FormantModeler FormantPoint FormantTier
+          GaussianMixture HMM HMM_Observation HMM_ObservationSequence HMM_State
+          HMM_StateSequence HMMObservation HMMObservationSequence HMMState
+          HMMStateSequence Harmonicity ISpline Index Intensity IntensityTier
+          IntervalTier KNN KlattGrid KlattTable LFCC LPC Label LegendreSeries
+          LinearRegression LogisticRegression LongSound Ltas MFCC MSpline ManPages
+          Manipulation Matrix MelFilter MelSpectrogram MixingMatrix Movie Network
+          OTGrammar OTHistory OTMulti PCA PairDistribution ParamCurve Pattern
+          Permutation Photo Pitch PitchModeler PitchTier PointProcess Polygon
+          Polynomial PowerCepstrogram PowerCepstrum Procrustes RealPoint RealTier
+          ResultsMFC Roots SPINET SSCP SVD Salience ScalarProduct Similarity
+          SimpleString SortedSetOfString Sound Speaker Spectrogram Spectrum
+          SpectrumTier SpeechSynthesizer SpellingChecker Strings StringsIndex
+          Table TableOfReal TextGrid TextInterval TextPoint TextTier Tier
+          Transition VocalTract VocalTractTier Weight WordList
+        )
+      end
+
+      def self.variables_numeric
+        @variables_numeric ||= %w(
+          all average e left macintosh mono pi praatVersion right stereo
+          undefined unix windows
+        )
+      end
+
+      def self.variables_string
+        @variables_string ||= %w(
+          praatVersion$ tab$ shellDirectory$ homeDirectory$
+          preferencesDirectory$ newline$ temporaryDirectory$
+          defaultDirectory$
+        )
+      end
+
+      def self.object_attributes
+        @object_attributes ||= %w(
+          ncol nrow xmin ymin xmax ymax nx ny dx dy
+        )
+      end
 
       state :root do
         rule %r/(\s+)(#.*?$)/ do
           groups Text, Comment::Single
         end
 
-        rule %r/^#.*?$/,         Comment::Single
-        rule %r/;[^\n]*/,        Comment::Single
-        rule %r/\s+/,            Text
+        rule %r/^#.*?$/, Comment::Single
+        rule %r/;[^\n]*/, Comment::Single
+        rule %r/\s+/, Text
 
         rule %r/(\bprocedure)(\s+)/ do
           groups Keyword, Text
@@ -129,12 +158,11 @@ module Rouge
           push :procedure_call
         end
 
-        rule %r/@/,              Name::Function, :procedure_call
+        rule %r/@/, Name::Function, :procedure_call
 
         mixin :function_call
 
         rule %r/\b(?:select all)\b/, Keyword
-        rule %r/\b(?:#{keywords.join('|')})\b/, Keyword
 
         rule %r/(\bform\b)(\s+)([^\n]+)/ do
           groups Keyword, Text, Literal::String
@@ -156,10 +184,27 @@ module Rouge
 
         rule %r/"/, Literal::String, :string
 
-        rule %r/\b(?:#{objects.join('|')})(?=\s+\S+\n)/, Name::Class, :string_unquoted
+        rule %r/\b([A-Z][a-zA-Z0-9]+)(?=\s+\S+\n)/ do |m|
+          match = m[0]
+          if self.class.objects.include?(match)
+            token Name::Class
+            push :string_unquoted
+          else
+            token Keyword
+          end
+        end
 
         rule %r/\b(?=[A-Z])/, Text, :command
         rule %r/(\.{3}|[)(,\$])/, Punctuation
+
+        rule %r/[a_z]+/ do |m|
+          match = m[0]
+          if self.class.keywords.include?(match)
+            token Keyword
+          else
+            token Text
+          end
+        end
       end
 
       state :command do
@@ -178,7 +223,7 @@ module Rouge
           push :comma_list
         end
 
-        rule %r/[\s]/,    Text, :pop!
+        rule %r/[\s]/, Text, :pop!
       end
 
       state :procedure_call do
@@ -186,7 +231,7 @@ module Rouge
 
         rule %r/(:|\s*\()/, Punctuation, :pop!
 
-        rule %r/'/,            Name::Function
+        rule %r/'/, Name::Function
         rule %r/[^:\('\s]+/, Name::Function
 
         rule %r/(?=\s+)/ do
@@ -205,11 +250,17 @@ module Rouge
       end
 
       state :function_call do
-        rule %r/\b(#{functions_string.join('|')})\$(?=\s*[:(])/, Name::Function, :function
-        rule %r/\b(#{functions_array.join('|')})#(?=\s*[:(])/,   Name::Function, :function
-        rule %r/\b(#{functions_matrix.join('|')})##(?=\s*[:(])/,   Name::Function, :function
-        rule %r/\b(#{functions_string_vector.join('|')})\$#(?=\s*[:(])/,   Name::Function, :function
-        rule %r/\b(#{functions_numeric.join('|')})(?=\s*[:(])/,  Name::Function, :function
+        rule %r/\b([a-z][a-zA-Z0-9_.]+)(\$#|##|\$|#)?(?=\s*[:(])/ do |m|
+          match = m[0]
+          if self.class.functions_builtin.include?(match)
+            token Name::Function
+            push :function
+          elsif self.class.keywords.include?(match)
+            token Keyword
+          else
+            token Operator::Word
+          end
+        end
       end
 
       state :function do
@@ -230,7 +281,7 @@ module Rouge
         rule %r/\s*[\]\})\n]/, Text, :pop!
 
         rule %r/\s+/, Text
-        rule %r/"/,   Literal::String, :string
+        rule %r/"/, Literal::String, :string
         rule %r/\b(if|then|else|fi|endif)\b/, Keyword
 
         mixin :function_call
@@ -263,15 +314,28 @@ module Rouge
         mixin :operator
         mixin :number
 
-        rule %r/\b(?:#{variables_string.join('|')})\$/,  Name::Builtin
-        rule %r/\b(?:#{variables_numeric.join('|')})(?!\$)\b/, Name::Builtin
+        rule %r/\b([A-Z][a-zA-Z0-9]+)_/ do |m|
+          match = m[1]
+          if (['Object'] | self.class.objects).include?(match)
+            token Name::Builtin
+            push :object_reference
+          else
+            token Name::Variable
+          end
+        end
 
-        rule %r/\b(Object|#{objects.join('|')})_/ do
-          token Name::Builtin
-          push :object_reference
+        rule %r/\.?[a-z][a-zA-Z0-9_.]*(\$#|##|\$|#)?/ do |m|
+          match = m[0]
+          if self.class.variables_string.include?(match) ||
+             self.class.variables_numeric.include?(match)
+            token Name::Builtin
+          elsif self.class.keywords.include?(match)
+            token Keyword
+          else
+            token Name::Variable
+          end
         end
 
-        rule %r/\.?[a-z][a-zA-Z0-9_.]*(\$#|##|\$|#)?/, Text
         rule %r/[\[\]]/, Text, :comma_list
         mixin :string_interpolated
       end
@@ -284,7 +348,13 @@ module Rouge
         mixin :string_interpolated
         rule %r/([a-z][a-zA-Z0-9_]*|\d+)/, Name::Builtin
 
-        rule %r/\.(#{object_attributes.join('|')})\b/, Name::Builtin, :pop!
+        rule %r/\.([a-z]+)\b/ do |m|
+          match = m[1]
+          if self.class.object_attributes.include?(match)
+            token Name::Builtin
+            pop!
+          end
+        end
 
         rule %r/\$/, Name::Builtin
         rule %r/\[/, Text, :pop!
@@ -292,7 +362,7 @@ module Rouge
 
       state :operator do
         # This rule incorrectly matches === or +++++, which are not operators
-        rule %r/([+\/*<>=!-]=?|[&*|][&*|]?|\^|<>)/,       Operator
+        rule %r/([+\/*<>=!-]=?|[&*|][&*|]?|\^|<>)/, Operator
         rule %r/(?<![\w.])(and|or|not|div|mod)(?![\w.])/, Operator::Word
       end
 
@@ -302,23 +372,23 @@ module Rouge
 
       state :string_unquoted do
         rule %r/\n\s*\.{3}/, Punctuation
-        rule %r/\n/,         Text, :pop!
-        rule %r/\s/,         Text
+        rule %r/\n/, Text, :pop!
+        rule %r/\s/, Text
 
         mixin :string_interpolated
 
-        rule %r/'/,          Literal::String
-        rule %r/[^'\n]+/,    Literal::String
+        rule %r/'/, Literal::String
+        rule %r/[^'\n]+/, Literal::String
       end
 
       state :string do
         rule %r/\n\s*\.{3}/, Punctuation
-        rule %r/"/,          Literal::String,           :pop!
+        rule %r/"/, Literal::String, :pop!
 
         mixin :string_interpolated
 
-        rule %r/'/,          Literal::String
-        rule %r/[^'"\n]+/,   Literal::String
+        rule %r/'/, Literal::String
+        rule %r/[^'"\n]+/, Literal::String
       end
 
       state :old_form do
@@ -363,7 +433,6 @@ module Rouge
 
         rule %r/\bendform\b/, Keyword, :pop!
       end
-
     end
   end
 end

data/lib/rouge/lexers/python.rb

@@ -40,7 +40,7 @@ module Rouge
       end
 
       def self.builtins_pseudo
-        @builtins_pseudo ||= %w(self None Ellipsis NotImplemented False True)
+        @builtins_pseudo ||= %w(None Ellipsis NotImplemented False True)
       end
 
       def self.exceptions
@@ -86,6 +86,8 @@ module Rouge
         rule %r/\\\n/, Text
         rule %r/\\/, Text
 
+        rule %r/@#{dotted_identifier}/i, Name::Decorator
+
         rule %r/(in|is|and|or|not)\b/, Operator::Word
         rule %r/(<<|>>|\/\/|\*\*)=?/, Operator
         rule %r/[-~+\/*%=<>&^|@]=?|!=/, Operator
@@ -93,13 +95,13 @@ module Rouge
         rule %r/(from)((?:\\\s|\s)+)(#{dotted_identifier})((?:\\\s|\s)+)(import)/ do
           groups Keyword::Namespace,
                  Text,
-                 Name::Namespace,
+                 Name,
                  Text,
                  Keyword::Namespace
         end
 
         rule %r/(import)(\s+)(#{dotted_identifier})/ do
-          groups Keyword::Namespace, Text, Name::Namespace
+          groups Keyword::Namespace, Text, Name
         end
 
         rule %r/(def)((?:\s|\\\s)+)/ do
@@ -112,6 +114,9 @@ module Rouge
           push :classname
         end
 
+        rule %r/([a-z_]\w*)[ \t]*(?=(\(.*\)))/m, Name::Function
+        rule %r/([A-Z_]\w*)[ \t]*(?=(\(.*\)))/m, Name::Class
+
         # TODO: not in python 3
         rule %r/`.*?`/, Str::Backtick
         rule %r/([rfbu]{0,2})('''|"""|['"])/i do |m|
@@ -120,8 +125,6 @@ module Rouge
           push :generic_string
         end
 
-        rule %r/@#{dotted_identifier}/i, Name::Decorator
-
         # using negative lookbehind so we don't match property names
         rule %r/(?<!\.)#{identifier}/ do |m|
           if self.class.keywords.include? m[0]

data/lib/rouge/lexers/systemd.rb

@@ -27,7 +27,7 @@ module Rouge
         rule %r/(.*?)(\\\n)/ do
           groups Text, Str::Escape
         end
-        rule %r/(.*)'/, Text, :pop!
+        rule %r/(.*)'?/, Text, :pop!
       end
     end
   end

data/lib/rouge/lexers/vala.rb

@@ -5,7 +5,7 @@ module Rouge
   module Lexers
     class Vala < RegexLexer
       tag 'vala'
-      filenames '*.vala'
+      filenames '*.vala', '*.vapi'
       mimetypes 'text/x-vala'
 
       title "Vala"

data/lib/rouge/lexers/yaml.rb

@@ -18,8 +18,6 @@ module Rouge
         return true if text =~ /\A\s*%YAML/m
       end
 
-      SPECIAL_VALUES = Regexp.union(%w(true false null))
-
       # NB: Tabs are forbidden in YAML, which is why you see things
       # like /[ ]+/.
 
@@ -175,7 +173,7 @@ module Rouge
 
       state :block_nodes do
         # implicit key
-        rule %r/([^#,:?\[\]{}"'\n]+)(:)(?=\s|$)/ do |m|
+        rule %r/([^#,?\[\]{}"'\n]+)(:)(?=\s|$)/ do |m|
           groups Name::Attribute, Punctuation::Indicator
           set_indent m[0], :implicit => true
         end
@@ -340,7 +338,7 @@ module Rouge
         end
 
         rule %r/[ ]+/, Str
-        rule SPECIAL_VALUES, Name::Constant
+        rule %r((true|false|null)\b), Keyword::Constant
         rule %r/\d+(?:\.\d+)?(?=(\r?\n)| +#)/, Literal::Number, :pop!
 
         # regular non-whitespace characters