rouge 3.26.1 → 3.29.0

data/lib/rouge/lexers/rust.rb

@@ -22,10 +22,13 @@ module Rouge
 
       def self.keywords
         @keywords ||= %w(
-          as assert async await break const continue copy do drop else enum extern
-          fail false fn for if impl let log loop match mod move mut priv pub pure
-          ref return self static struct true trait type unsafe use where
-          while box
+          as async await break const continue crate dyn else enum extern false
+          fn for if impl in let log loop match mod move mut pub ref return self
+          Self static struct super trait true type unsafe use where while
+          abstract become box do final macro
+          override priv typeof unsized virtual
+          yield try
+          union
         )
       end
 
@@ -40,6 +43,9 @@ module Rouge
           Right Send Shl Shr size_t Some ssize_t str Sub Success time_t
           u16 u32 u64 u8 usize uint uintptr_t
           Box Vec String Gc Rc Arc
+          u128 i128 Result Sync Pin Unpin Sized Drop drop Fn FnMut FnOnce
+          Clone PartialEq PartialOrd AsMut AsRef From Into Default
+          DoubleEndedIterator ExactSizeIterator Extend IntoIterator Iterator
         )
       end
 
@@ -54,12 +60,12 @@ module Rouge
 
       delim_map = { '[' => ']', '(' => ')', '{' => '}' }
 
-      id = /[a-z_]\w*/i
+      id = /[\p{XID_Start}_]\p{XID_Continue}*/
       hex = /[0-9a-f]/i
       escapes = %r(
-        \\ ([nrt'"\\0] | x#{hex}{2} | u#{hex}{4} | U#{hex}{8})
+        \\ ([nrt'"\\0] | x#{hex}{2} | u\{(#{hex}_*){1,6}\})
       )x
-      size = /8|16|32|64/
+      size = /8|16|32|64|128|size/
 
       # Although not officially part of Rust, the rustdoc tool allows code in
       # comments to begin with `#`. Code like this will be evaluated but not
@@ -82,8 +88,62 @@ module Rouge
 
       state :whitespace do
         rule %r/\s+/, Text
-        rule %r(//[^\n]*), Comment
-        rule %r(/[*].*?[*]/)m, Comment::Multiline
+        mixin :comments
+      end
+
+      state :comments do
+        # Only 3 slashes are doc comments, `////` and beyond become normal
+        # comments again (for some reason), so match this before the
+        # doc line comments rather than figure out a
+        rule %r(////+[^\n]*), Comment::Single
+        # doc line comments — either inner (`//!`), or outer (`///`).
+        rule %r(//[/!][^\n]*), Comment::Doc
+        # otherwise, `//` is just a plain line comme
+        rule %r(//[^\n]*), Comment::Single
+        # /**/ and /***/ are self-closing block comments, not doc. Because this
+        # is self-closing, it doesn't enter the states for nested comments
+        rule %r(/\*\*\*?/), Comment::Multiline
+        # 3+ stars and it's a normal non-doc block comment.
+        rule %r(/\*\*\*+), Comment::Multiline, :nested_plain_block
+        # `/*!` and `/**` begin doc comments. These nest and can have internal
+        # block/doc comments, but they're still part of the documentation
+        # inside.
+        rule %r(/[*][*!]), Comment::Doc, :nested_doc_block
+        # any other /* is a plain multiline comment
+        rule %r(/[*]), Comment::Multiline, :nested_plain_block
+      end
+
+      # Multiline/block comments fully nest. This is true for ones that are
+      # marked as documentation too. The behavior here is:
+      #
+      # - Anything inside a block doc comment is still included in the
+      #   documentation, even if it's a nested non-doc block comment. For
+      #   example: `/** /* still docs */ */`
+      # - Anything inside of a block non-doc comment is still just a normal
+      #   comment, even if it's a nested block documentation comment. For
+      #   example: `/* /** not docs */ */`
+      #
+      # This basically means: if (on the outermost level) the comment starts as
+      # one kind of block comment (either doc/non-doc), then everything inside
+      # of it, including nested block comments of the opposite type, needs to
+      # stay that type.
+      #
+      # Also note that single line comments do nothing anywhere inside of block
+      # comments, thankfully.
+      #
+      # We just define this as two states, because this seems easier than
+      # tracking it with instance vars.
+      [
+        [:nested_plain_block, Comment::Multiline],
+        [:nested_doc_block, Comment::Doc]
+      ].each do |state_name, comment_token|
+        state state_name do
+          rule %r(\*/), comment_token, :pop!
+          rule %r(/\*), comment_token, state_name
+          # We only want to eat at most one `[*/]` at a time,
+          # but we can skip past non-`[*/]` in bulk.
+          rule %r([^*/]+|[*/]), comment_token
+        end
       end
 
       state :root do
@@ -110,6 +170,7 @@ module Rouge
         rule %r/\bmacro_rules!/, Name::Decorator, :macro_rules
         rule %r/#{id}!/, Name::Decorator, :macro
 
+        rule %r/'static\b/, Keyword
         rule %r/'#{id}/, Name::Variable
         rule %r/#{id}/ do |m|
           name = m[0]
@@ -154,33 +215,35 @@ module Rouge
 
       state :has_literals do
         # constants
-        rule %r/\b(?:true|false|nil)\b/, Keyword::Constant
-        # characters
+        rule %r/\b(?:true|false)\b/, Keyword::Constant
+
+        # characters/bytes
         rule %r(
-          ' (?: #{escapes} | [^\\] ) '
+          b?' (?: #{escapes} | [^\\] ) '
         )x, Str::Char
 
-        rule %r/"/, Str, :string
-        rule %r/r(#*)".*?"\1/m, Str
+        rule %r/b?"/, Str, :string
+        rule %r/b?r(#*)".*?"\1/m, Str
 
         # numbers
-        dot = /[.][0-9_]+/
-        exp = /e[-+]?[0-9_]+/
+        dot = /[.][0-9][0-9_]*/
+        exp = /[eE][-+]?[0-9_]+/
         flt = /f32|f64/
 
         rule %r(
-          [0-9_]+
+          [0-9][0-9_]*
           (#{dot}  #{exp}? #{flt}?
           |#{dot}? #{exp}  #{flt}?
           |#{dot}? #{exp}? #{flt}
+          |[.](?![._\p{XID_Start}])
           )
         )x, Num::Float
 
         rule %r(
           ( 0b[10_]+
           | 0x[0-9a-fA-F_]+
-          | 0o[0-7]+
-          | [0-9_]+
+          | 0o[0-7_]+
+          | [0-9][0-9_]*
           ) (u#{size}?|i#{size})?
         )x, Num::Integer
 

data/lib/rouge/lexers/sparql.rb

@@ -41,16 +41,17 @@ module Rouge
         rule %r('''), Str::Single, :string_single_literal
         rule %r('), Str::Single, :string_single
 
-        rule %r([$?]\w+), Name::Variable
-        rule %r((\w*:)(\w+)?) do |m|
+        rule %r([$?][[:word:]]+), Name::Variable
+        rule %r(([[:word:]-]*)(:)([[:word:]-]+)?) do |m|
           token Name::Namespace, m[1]
-          token Str::Symbol, m[2]
+          token Operator, m[2]
+          token Str::Symbol, m[3]
         end
         rule %r(<[^>]*>), Name::Namespace
         rule %r(true|false)i, Keyword::Constant
         rule %r/a\b/, Keyword
 
-        rule %r([A-Z]\w+\b)i do |m|
+        rule %r([A-Z][[:word:]]+\b)i do |m|
           if self.class.builtins.include? m[0].upcase
             token Name::Builtin
           elsif self.class.keywords.include? m[0].upcase

data/lib/rouge/lexers/sqf/keywords.rb

@@ -9,4 +9,4 @@ module Rouge
       end
     end
   end
-end
+end

data/lib/rouge/lexers/sql.rb

@@ -38,8 +38,8 @@ module Rouge
           DYNAMIC DYNAMIC_FUNCTION DYNAMIC_FUNCTION_CODE EACH ELSE
           ENCODING ENCRYPTED END END-EXEC EQUALS ESCAPE EVERY EXCEPT
           ESCEPTION EXCLUDING EXCLUSIVE EXEC EXECUTE EXISTING EXISTS
-          EXPLAIN EXTERNAL EXTRACT FALSE FETCH FINAL FIRST FOR FORCE
-          FOREIGN FORTRAN FORWARD FOUND FREE FREEZE FROM FULL FUNCTION
+          EXPLAIN EXTERNAL EXTRACT FALSE FETCH FINAL FIRST FOLLOWING FOR
+          FORCE FOREIGN FORTRAN FORWARD FOUND FREE FREEZE FROM FULL FUNCTION
           G GENERAL GENERATED GET GLOBAL GO GOTO GRANT GRANTED GROUP
           GROUPING HANDLER HAVING HIERARCHY HOLD HOST IDENTITY IGNORE
           ILIKE IMMEDIATE IMMUTABLE IMPLEMENTATION IMPLICIT IN INCLUDING
@@ -58,10 +58,10 @@ module Rouge
           ORDINALITY OUT OUTER OUTPUT OVERLAPS OVERLAY OVERRIDING
           OWNER PAD PARAMETER PARAMETERS PARAMETER_MODE PARAMATER_NAME
           PARAMATER_ORDINAL_POSITION PARAMETER_SPECIFIC_CATALOG
-          PARAMETER_SPECIFIC_NAME PARAMATER_SPECIFIC_SCHEMA PARTIAL PASCAL
-          PENDANT PLACING PLI POSITION POSTFIX PREFIX PREORDER
+          PARAMETER_SPECIFIC_NAME PARAMATER_SPECIFIC_SCHEMA PARTIAL PARTITION
+          PASCAL PENDANT PLACING PLI POSITION POSTFIX PREFIX PRECEDING PREORDER
           PREPARE PRESERVE PRIMARY PRIOR PRIVILEGES PROCEDURAL PROCEDURE
-          PUBLIC READ READS RECHECK RECURSIVE REF REFERENCES REFERENCING
+          PUBLIC RANGE READ READS RECHECK RECURSIVE REF REFERENCES REFERENCING
           REINDEX RELATIVE RENAME REPEATABLE REPLACE RESET RESTART
           RESTRICT RESULT RETURN RETURNED_LENGTH RETURNED_OCTET_LENGTH
           RETURNED_SQLSTATE RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
@@ -84,7 +84,7 @@ module Rouge
           USAGE USER USER_DEFINED_TYPE_CATALOG USER_DEFINED_TYPE_NAME
           USER_DEFINED_TYPE_SCHEMA USING VACUUM VALID VALIDATOR VALUES
           VARIABLE VERBOSE VERSION VIEW VOLATILE WHEN WHENEVER WHERE
-          WITH WITHOUT WORK WRITE ZONE
+          WINDOW WITH WITHOUT WORK WRITE ZONE
         )
       end
 
@@ -126,7 +126,7 @@ module Rouge
         end
 
         rule %r([+*/<>=~!@#%&|?^-]), Operator
-        rule %r/[;:()\[\],.]/, Punctuation
+        rule %r/[;:()\[\]\{\},.]/, Punctuation
       end
 
       state :multiline_comments do

data/lib/rouge/lexers/stan.rb

@@ -0,0 +1,451 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Stan < RegexLexer
+      title "Stan"
+      desc 'Stan Modeling Language (mc-stan.org)'
+      tag 'stan'
+      filenames '*.stan', '*.stanfunctions'
+
+      # optional comment or whitespace
+      WS = %r((?:\s|//.*?\n|/[*].*?[*]/)+)
+      ID = /[a-zA-Z_][a-zA-Z0-9_]*/
+      RT = /(?:(?:[a-z_]\s*(?:\[[0-9, ]\])?)\s+)*/
+      OP = Regexp.new([
+        # Assigment operators
+        "=",
+
+        # Comparison operators
+        "<", "<=", ">", ">=", "==", "!=",
+
+        # Boolean operators
+        "!", "&&", "\\|\\|",
+
+        # Real-valued arithmetic operators
+        "\\+", "-", "\\*", "/", "\\^",
+
+        # Transposition operator
+        "'",
+
+        # Elementwise functions
+        "\\.\\+", "\\.-", "\\.\\*", "\\./", "\\.\\^",
+
+        # Matrix division operators
+        "\\\\",
+
+        # Compound assigment operators
+        "\\+=", "-=", "\\*=", "/=", "\\.\\*=", "\\./=",
+
+        # Sampling
+        "~",
+
+        # Conditional operator
+        "\\?", ":"
+      ].join("|"))
+
+      def self.keywords
+        @keywords ||= Set.new %w(
+          if else while for break continue print reject return
+        )
+      end
+
+      def self.types
+        @types ||= Set.new %w(
+          int real vector ordered positive_ordered simplex unit_vector
+          row_vector matrix cholesky_factor_corr cholesky_factor_cov corr_matrix
+          cov_matrix data void complex array
+        )
+      end
+
+      def self.reserved
+        @reserved ||= Set.new [
+          # Reserved words from Stan language
+          "for", "in", "while", "repeat", "until", "if", "then", "else", "true",
+          "false", "target", "functions", "model", "data", "parameters",
+          "quantities", "transformed", "generated",
+
+          # Reserved names from Stan implementation
+          "var", "fvar", "STAN_MAJOR", "STAN_MINOR", "STAN_PATCH",
+          "STAN_MATH_MAJOR", "STAN_MATH_MINOR", "STAN_MATH_PATCH",
+
+          # Reserved names from C++
+          "alignas", "alignof", "and", "and_eq", "asm", "auto", "bitand",
+          "bitor", "bool", "break", "case", "catch", "char", "char16_t",
+          "char32_t", "class", "compl", "const", "constexpr", "const_cast",
+          "continue", "decltype", "default", "delete", "do", "double",
+          "dynamic_cast", "else", "enum", "explicit", "export", "extern",
+          "false", "float", "for", "friend", "goto", "if", "inline", "int",
+          "long", "mutable", "namespace", "new", "noexcept", "not", "not_eq",
+          "nullptr", "operator", "or", "or_eq", "private", "protected",
+          "public", "register", "reinterpret_cast", "return", "short", "signed",
+          "sizeof", "static", "static_assert", "static_cast", "struct",
+          "switch", "template", "this", "thread_local", "throw", "true", "try",
+          "typedef", "typeid", "typename", "union", "unsigned", "using",
+          "virtual", "void", "volatile", "wchar_t", "while", "xor", "xor_eq"
+        ]
+      end
+
+      def self.builtin_functions
+        @builtin_functions ||= Set.new [
+          # Integer-Valued Basic Functions
+
+          ## Absolute functions
+          "abs", "int_step",
+
+          ## Bound functions
+          "min", "max",
+
+          ## Size functions
+          "size",
+
+          # Real-Valued Basic Functions
+
+          ## Log probability function
+          "target", "get_lp",
+
+          ## Logical functions
+          "step", "is_inf", "is_nan",
+
+          ## Step-like functions
+          "fabs", "fdim", "fmin", "fmax", "fmod", "floor", "ceil", "round",
+          "trunc",
+
+          ## Power and logarithm functions
+          "sqrt", "cbrt", "square", "exp", "exp2", "log", "log2", "log10",
+          "pow", "inv", "inv_sqrt", "inv_square",
+
+          ## Trigonometric functions
+          "hypot", "cos", "sin", "tan", "acos", "asin", "atan", "atan2",
+
+          ## Hyperbolic trigonometric functions
+          "cosh", "sinh", "tanh", "acosh", "asinh", "atanh",
+
+          ## Link functions
+          "logit", "inv_logit", "inv_cloglog",
+
+          ## Probability-related functions
+          "erf", "erfc", "Phi", "inv_Phi", "Phi_approx", "binary_log_loss",
+          "owens_t",
+
+          ## Combinatorial functions
+          "beta", "inc_beta", "lbeta", "tgamma", "lgamma", "digamma",
+          "trigamma", "lmgamma", "gamma_p", "gamma_q",
+          "binomial_coefficient_log", "choose", "bessel_first_kind",
+          "bessel_second_kind", "modified_bessel_first_kind",
+          "log_modified_bessel_first_kind", "modified_bessel_second_kind",
+          "falling_factorial", "lchoose", "log_falling_factorial",
+          "rising_factorial", "log_rising_factorial",
+
+          ## Composed functions
+          "expm1", "fma", "multiply_log", "ldexp", "lmultiply", "log1p",
+          "log1m", "log1p_exp", "log1m_exp", "log_diff_exp", "log_mix",
+          "log_sum_exp", "log_inv_logit", "log_inv_logit_diff",
+          "log1m_inv_logit",
+
+          ## Special functions
+          "lambert_w0", "lambert_wm1",
+
+          # Complex-Valued Basic Functions
+
+          ## Complex constructors and accessors
+          "to_complex", "get_real", "get_imag",
+
+          ## Complex special functions
+          "arg", "norm", "conj", "proj", "polar",
+
+          # Array Operations
+
+          ## Reductions
+          "sum", "prod", "log_sum_exp", "mean", "variance", "sd", "distance",
+          "squared_distance", "quantile",
+
+          ## Array size and dimension function
+          "dims", "num_elements",
+
+          ## Array broadcasting
+          "rep_array",
+
+          ## Array concatenation
+          "append_array",
+
+          ## Sorting functions
+          "sort_asc", "sort_desc", "sort_indices_asc", "sort_indices_desc",
+          "rank",
+
+          ## Reversing functions
+          "reverse",
+
+          # Matrix Operations
+
+          ## Integer-valued matrix size functions
+          "num_elements", "rows", "cols",
+
+          ## Dot products and specialized products
+          "dot_product", "columns_dot_product", "rows_dot_product", "dot_self",
+          "columns_dot_self", "rows_dot_self", "tcrossprod", "crossprod",
+          "quad_form", "quad_form_diag", "quad_form_sym", "trace_quad_form",
+          "trace_gen_quad_form", "multiply_lower_tri_self_transpose",
+          "diag_pre_multiply", "diag_post_multiply",
+
+          ## Broadcast functions
+          "rep_vector", "rep_row_vector", "rep_matrix",
+          "symmetrize_from_lower_tri",
+
+          ## Diagonal matrix functions
+          "add_diag", "diagonal", "diag_matrix", "identity_matrix",
+
+          ## Container construction functions
+          "linspaced_array", "linspaced_int_array", "linspaced_vector",
+          "linspaced_row_vector", "one_hot_int_array", "one_hot_array",
+          "one_hot_vector", "one_hot_row_vector", "ones_int_array",
+          "ones_array", "ones_vector", "ones_row_vector", "zeros_int_array",
+          "zeros_array", "zeros_vector", "zeros_row_vector", "uniform_simplex",
+
+          ## Slicing and blocking functions
+          "col", "row", "block", "sub_col", "sub_row", "head", "tail",
+          "segment",
+
+          ## Matrix concatenation
+          "append_col", "append_row",
+
+          ## Special matrix functions
+          "softmax", "log_softmax", "cumulative_sum",
+
+          ## Covariance functions
+          "cov_exp_quad",
+
+          ## Linear algebra functions and solvers
+          "mdivide_left_tri_low", "mdivide_right_tri_low", "mdivide_left_spd",
+          "mdivide_right_spd", "matrix_exp", "matrix_exp_multiply",
+          "scale_matrix_exp_multiply", "matrix_power", "trace", "determinant",
+          "log_determinant", "inverse", "inverse_spd", "chol2inv",
+          "generalized_inverse", "eigenvalues_sym", "eigenvectors_sym",
+          "qr_thin_Q", "qr_thin_R", "qr_Q", "qr_R", "cholseky_decompose",
+          "singular_values", "svd_U", "svd_V",
+
+          # Sparse Matrix Operations
+
+          ## Conversion functions
+          "csr_extract_w", "csr_extract_v", "csr_extract_u",
+          "csr_to_dense_matrix",
+
+          ## Sparse matrix arithmetic
+          "csr_matrix_times_vector",
+
+          # Mixed Operations
+          "to_matrix", "to_vector", "to_row_vector", "to_array_2d",
+          "to_array_1d",
+
+          # Higher-Order Functions
+
+          ## Algebraic equation solver
+          "algebra_solver", "algebra_solver_newton",
+
+          ## Ordinary differential equation
+          "ode_rk45", "ode_rk45_tol", "ode_ckrk", "ode_ckrk_tol", "ode_adams",
+          "ode_adams_tol", "ode_bdf", "ode_bdf_tol", "ode_adjoint_tol_ctl",
+
+          ## 1D integrator
+          "integrate_1d",
+
+          ## Reduce-sum function
+          "reduce_sum", "reduce_sum_static",
+
+          ## Map-rect function
+          "map_rect",
+
+          # Deprecated Functions
+          "integrate_ode_rk45", "integrate_ode", "integrate_ode_adams",
+          "integrate_ode_bdf",
+
+          # Hidden Markov Models
+          "hmm_marginal", "hmm_latent_rng", "hmm_hidden_state_prob"
+        ]
+      end
+
+      def self.distributions
+        @distributions ||= Set.new(
+          [
+            # Discrete Distributions
+
+            ## Binary Distributions
+            "bernoulli", "bernoulli_logit", "bernoulli_logit_glm",
+
+            ## Bounded Discrete Distributions
+            "binomial", "binomial_logit", "beta_binomial", "hypergeometric",
+            "categorical", "categorical_logit_glm", "discrete_range",
+            "ordered_logistic", "ordered_logistic_glm", "ordered_probit",
+
+            ## Unbounded Discrete Distributions
+            "neg_binomial", "neg_binomial_2", "neg_binomial_2_log",
+            "neg_binomial_2_log_glm", "poisson", "poisson_log",
+            "poisson_log_glm",
+
+            ## Multivariate Discrete Distributions
+            "multinomial", "multinomial_logit",
+
+            # Continuous Distributions
+
+            ## Unbounded Continuous Distributions
+            "normal", "std_normal", "normal_id_glm", "exp_mod_normal",
+            "skew_normal", "student_t", "cauchy", "double_exponential",
+            "logistic", "gumbel", "skew_double_exponential",
+
+            ## Positive Continuous Distributions
+            "lognormal", "chi_square", "inv_chi_square",
+            "scaled_inv_chi_square", "exponential", "gamma", "inv_gamma",
+            "weibull", "frechet", "rayleigh",
+
+            ## Positive Lower-Bounded Distributions
+            "pareto", "pareto_type_2", "wiener",
+
+            ## Continuous Distributions on [0, 1]
+            "beta", "beta_proportion",
+
+            ## Circular Distributions
+            "von_mises",
+
+            ## Bounded Continuous Distributions
+            "uniform",
+
+            ## Distributions over Unbounded Vectors
+            "multi_normal", "multi_normal_prec", "multi_normal_cholesky",
+            "multi_gp", "multi_gp_cholesky", "multi_student_t",
+            "gaussian_dlm_obs",
+
+            ## Simplex Distributions
+            "dirichlet",
+
+            ## Correlation Matrix Distributions
+            "lkj_corr", "lkj_corr_cholesky",
+
+            ## Covariance Matrix Distributions
+            "wishart", "inv_wishart"
+          ].product([
+            "", "_lpmf", "_lupmf", "_lpdf", "_lcdf", "_lccdf", "_rng", "_log",
+            "_cdf_log", "_ccdf_log"
+          ]).map {|s| "#{s[0]}#{s[1]}"}
+        )
+      end
+
+      def self.constants
+        @constants ||= Set.new [
+          # Mathematical constants
+          "pi", "e", "sqrt2", "log2", "log10",
+
+          # Special values
+          "not_a_number", "positive_infinity", "negative_infinity",
+          "machine_precision"
+        ]
+      end
+
+      state :root do
+        mixin :whitespace
+        rule %r/#include/, Comment::Preproc, :include
+        rule %r/#.*$/, Generic::Deleted
+        rule %r(
+          functions
+          |(?:transformed\s+)?data
+          |(?:transformed\s+)?parameters
+          |model
+          |generated\s+quantities
+        )x, Name::Namespace
+        rule %r(\{), Punctuation, :bracket_scope
+        mixin :scope
+      end
+
+      state :include do
+        rule %r((\s+)(\S+)(\s*)) do |m|
+          token Text, m[1]                          
+          token Comment::PreprocFile, m[2]           
+          token Text, m[3]
+          pop!
+        end
+      end
+
+      state :whitespace do
+        rule %r(\n+)m, Text
+        rule %r(//(\\.|.)*?$), Comment::Single
+        mixin :inline_whitespace
+      end
+
+      state :inline_whitespace do
+        rule %r([ \t\r]+), Text
+        rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, Comment::Multiline
+      end
+
+      state :statements do
+        mixin :whitespace
+        rule %r/#include/, Comment::Preproc, :include
+        rule %r/#.*$/, Generic::Deleted
+        rule %r("), Str, :string
+        rule %r(
+          (
+            ((\d+[.]\d*|[.]?\d+)e[+-]?\d+|\d*[.]\d+|\d+) 
+            (#{WS})[+-](#{WS})
+            ((\d+[.]\d*|[.]?\d+)e[+-]?\d+|\d*[.]\d+|\d+)i
+          )
+          |((\d+[.]\d*|[.]?\d+)e[+-]?\d+|\d*[.]\d+|\d+)i
+          |((\d+[.]\d*|[.]?\d+)e[+-]?\d+|\d*[.]\d+) 
+        )mx, Num::Float
+        rule %r/\d+/, Num::Integer
+        rule %r(\*/), Error
+        rule OP, Operator
+        rule %r([\[\],.;]), Punctuation
+        rule %r([|](?![|])), Punctuation
+        rule %r(T\b), Keyword::Reserved
+        rule %r((lower|upper)\b), Name::Attribute
+        rule ID do |m|
+          name = m[0]
+
+          if self.class.keywords.include? name
+            token Keyword
+          elsif self.class.types.include? name
+            token Keyword::Type
+          elsif self.class.reserved.include? name
+            token Keyword::Reserved
+          else
+            token Name::Variable
+          end
+        end
+      end
+
+      state :scope do
+        mixin :whitespace
+        rule %r(
+          (#{RT})         # Return type
+          (#{ID})         # Function name
+          (?=\([^;]*?\))  # Signature or arguments
+        )mx do |m|
+          recurse m[1]
+
+          name = m[2]
+          if self.class.builtin_functions.include? name
+            token Name::Builtin, name
+          elsif self.class.distributions.include? name
+            token Name::Builtin, name
+          elsif self.class.constants.include? name
+            token Keyword::Constant
+          else
+            token Name::Function, name
+          end
+        end
+        rule %r(\{), Punctuation, :bracket_scope
+        rule %r(\(), Punctuation, :parens_scope
+        mixin :statements
+      end
+
+      state :bracket_scope do
+        mixin :scope
+        rule %r(\}), Punctuation, :pop!
+      end
+
+      state :parens_scope do
+        mixin :scope
+        rule %r(\)), Punctuation, :pop!
+      end
+    end
+  end
+end