rouge 1.1.0 → 1.2.0

data/lib/rouge/lexers/php/builtins.rb

@@ -40,12 +40,14 @@ module Rouge
           b["Exif"] = Set.new %w(exif_imagetype exif_imagetype exif_read_data exif_tagname exif_thumbnail read_exif_data exif_imagetype)
           b["Expect"] = Set.new %w(expect_expectl expect_expectl expect_popen expect_expectl)
           b["FAM"] = Set.new %w(fam_cancel_monitor fam_cancel_monitor fam_close fam_monitor_collection fam_monitor_directory fam_monitor_file fam_next_event fam_open fam_pending fam_resume_monitor fam_suspend_monitor fam_cancel_monitor)
+          b["Fann"] = Set.new %w(fann_cascadetrain_on_data fann_cascadetrain_on_data fann_cascadetrain_on_file fann_clear_scaling_params fann_copy fann_create_from_file fann_create_shortcut_array fann_create_shortcut fann_create_sparse_array fann_create_sparse fann_create_standard_array fann_create_standard fann_create_train_from_callback fann_create_train fann_descale_input fann_descale_output fann_descale_train fann_destroy_train fann_destroy fann_duplicate_train_data fann_get_activation_function fann_get_activation_steepness fann_get_bias_array fann_get_bit_fail_limit fann_get_bit_fail fann_get_cascade_activation_functions_count fann_get_cascade_activation_functions fann_get_cascade_activation_steepnesses_count fann_get_cascade_activation_steepnesses fann_get_cascade_candidate_change_fraction fann_get_cascade_candidate_limit fann_get_cascade_candidate_stagnation_epochs fann_get_cascade_max_cand_epochs fann_get_cascade_max_out_epochs fann_get_cascade_min_cand_epochs fann_get_cascade_min_out_epochs fann_get_cascade_num_candidate_groups fann_get_cascade_num_candidates fann_get_cascade_output_change_fraction fann_get_cascade_output_stagnation_epochs fann_get_cascade_weight_multiplier fann_get_connection_array fann_get_connection_rate fann_get_errno fann_get_errstr fann_get_layer_array fann_get_learning_momentum fann_get_learning_rate fann_get_MSE fann_get_network_type fann_get_num_input fann_get_num_layers fann_get_num_output fann_get_quickprop_decay fann_get_quickprop_mu fann_get_rprop_decrease_factor fann_get_rprop_delta_max fann_get_rprop_delta_min fann_get_rprop_delta_zero fann_get_rprop_increase_factor fann_get_sarprop_step_error_shift fann_get_sarprop_step_error_threshold_factor fann_get_sarprop_temperature fann_get_sarprop_weight_decay_shift fann_get_total_connections fann_get_total_neurons fann_get_train_error_function fann_get_train_stop_function fann_get_training_algorithm fann_init_weights fann_length_train_data fann_merge_train_data fann_num_input_train_data fann_num_output_train_data fann_print_error fann_randomize_weights fann_read_train_from_file fann_reset_errno fann_reset_errstr fann_reset_MSE fann_run fann_save_train fann_save fann_scale_input_train_data fann_scale_input fann_scale_output_train_data fann_scale_output fann_scale_train_data fann_scale_train fann_set_activation_function_hidden fann_set_activation_function_layer fann_set_activation_function_output fann_set_activation_function fann_set_activation_steepness_hidden fann_set_activation_steepness_layer fann_set_activation_steepness_output fann_set_activation_steepness fann_set_bit_fail_limit fann_set_callback fann_set_cascade_activation_functions fann_set_cascade_activation_steepnesses fann_set_cascade_candidate_change_fraction fann_set_cascade_candidate_limit fann_set_cascade_candidate_stagnation_epochs fann_set_cascade_max_cand_epochs fann_set_cascade_max_out_epochs fann_set_cascade_min_cand_epochs fann_set_cascade_min_out_epochs fann_set_cascade_num_candidate_groups fann_set_cascade_output_change_fraction fann_set_cascade_output_stagnation_epochs fann_set_cascade_weight_multiplier fann_set_error_log fann_set_input_scaling_params fann_set_learning_momentum fann_set_learning_rate fann_set_output_scaling_params fann_set_quickprop_decay fann_set_quickprop_mu fann_set_rprop_decrease_factor fann_set_rprop_delta_max fann_set_rprop_delta_min fann_set_rprop_delta_zero fann_set_rprop_increase_factor fann_set_sarprop_step_error_shift fann_set_sarprop_step_error_threshold_factor fann_set_sarprop_temperature fann_set_sarprop_weight_decay_shift fann_set_scaling_params fann_set_train_error_function fann_set_train_stop_function fann_set_training_algorithm fann_set_weight_array fann_set_weight fann_shuffle_train_data fann_subset_train_data fann_test_data fann_test fann_train_epoch fann_train_on_data fann_train_on_file fann_train fann_cascadetrain_on_data)
           b["FrontBase"] = Set.new %w(fbsql_affected_rows fbsql_affected_rows fbsql_autocommit fbsql_blob_size fbsql_change_user fbsql_clob_size fbsql_close fbsql_commit fbsql_connect fbsql_create_blob fbsql_create_clob fbsql_create_db fbsql_data_seek fbsql_database_password fbsql_database fbsql_db_query fbsql_db_status fbsql_drop_db fbsql_errno fbsql_error fbsql_fetch_array fbsql_fetch_assoc fbsql_fetch_field fbsql_fetch_lengths fbsql_fetch_object fbsql_fetch_row fbsql_field_flags fbsql_field_len fbsql_field_name fbsql_field_seek fbsql_field_table fbsql_field_type fbsql_free_result fbsql_get_autostart_info fbsql_hostname fbsql_insert_id fbsql_list_dbs fbsql_list_fields fbsql_list_tables fbsql_next_result fbsql_num_fields fbsql_num_rows fbsql_password fbsql_pconnect fbsql_query fbsql_read_blob fbsql_read_clob fbsql_result fbsql_rollback fbsql_rows_fetched fbsql_select_db fbsql_set_characterset fbsql_set_lob_mode fbsql_set_password fbsql_set_transaction fbsql_start_db fbsql_stop_db fbsql_table_name fbsql_tablename fbsql_username fbsql_warnings fbsql_affected_rows)
           b["FDF"] = Set.new %w(fdf_add_doc_javascript fdf_add_doc_javascript fdf_add_template fdf_close fdf_create fdf_enum_values fdf_errno fdf_error fdf_get_ap fdf_get_attachment fdf_get_encoding fdf_get_file fdf_get_flags fdf_get_opt fdf_get_status fdf_get_value fdf_get_version fdf_header fdf_next_field_name fdf_open_string fdf_open fdf_remove_item fdf_save_string fdf_save fdf_set_ap fdf_set_encoding fdf_set_file fdf_set_flags fdf_set_javascript_action fdf_set_on_import_javascript fdf_set_opt fdf_set_status fdf_set_submit_form_action fdf_set_target_frame fdf_set_value fdf_set_version fdf_add_doc_javascript)
           b["Fileinfo"] = Set.new %w(finfo_buffer finfo_buffer finfo_close finfo_file finfo_open finfo_set_flags mime_content_type finfo_buffer)
           b["filePro"] = Set.new %w(filepro_fieldcount filepro_fieldcount filepro_fieldname filepro_fieldtype filepro_fieldwidth filepro_retrieve filepro_rowcount filepro filepro_fieldcount)
           b["Filesystem"] = Set.new %w(basename basename chgrp chmod chown clearstatcache copy delete dirname disk_free_space disk_total_space diskfreespace fclose feof fflush fgetc fgetcsv fgets fgetss file_exists file_get_contents file_put_contents file fileatime filectime filegroup fileinode filemtime fileowner fileperms filesize filetype flock fnmatch fopen fpassthru fputcsv fputs fread fscanf fseek fstat ftell ftruncate fwrite glob is_dir is_executable is_file is_link is_readable is_uploaded_file is_writable is_writeable lchgrp lchown link linkinfo lstat mkdir move_uploaded_file parse_ini_file parse_ini_string pathinfo pclose popen readfile readlink realpath_cache_get realpath_cache_size realpath rename rewind rmdir set_file_buffer stat symlink tempnam tmpfile touch umask unlink basename)
           b["Filter"] = Set.new %w(filter_has_var filter_has_var filter_id filter_input_array filter_input filter_list filter_var_array filter_var filter_has_var)
+          b["FPM"] = Set.new %w(fastcgi_finish_request fastcgi_finish_request fastcgi_finish_request)
           b["FriBiDi"] = Set.new %w(fribidi_log2vis fribidi_log2vis fribidi_log2vis)
           b["FTP"] = Set.new %w(ftp_alloc ftp_alloc ftp_cdup ftp_chdir ftp_chmod ftp_close ftp_connect ftp_delete ftp_exec ftp_fget ftp_fput ftp_get_option ftp_get ftp_login ftp_mdtm ftp_mkdir ftp_nb_continue ftp_nb_fget ftp_nb_fput ftp_nb_get ftp_nb_put ftp_nlist ftp_pasv ftp_put ftp_pwd ftp_quit ftp_raw ftp_rawlist ftp_rename ftp_rmdir ftp_set_option ftp_site ftp_size ftp_ssl_connect ftp_systype ftp_alloc)
           b["Function handling"] = Set.new %w(call_user_func_array call_user_func_array call_user_func create_function forward_static_call_array forward_static_call func_get_arg func_get_args func_num_args function_exists get_defined_functions register_shutdown_function register_tick_function unregister_tick_function call_user_func_array)
@@ -113,8 +115,8 @@ module Rouge
           b["NSAPI"] = Set.new %w(nsapi_request_headers nsapi_request_headers nsapi_response_headers nsapi_virtual nsapi_request_headers)
           b["OAuth"] = Set.new %w(oauth_get_sbs oauth_get_sbs oauth_urlencode oauth_get_sbs)
           b["Object Aggregation"] = Set.new %w(aggregate_info aggregate_info aggregate_methods_by_list aggregate_methods_by_regexp aggregate_methods aggregate_properties_by_list aggregate_properties_by_regexp aggregate_properties aggregate aggregation_info deaggregate aggregate_info)
-          b["OCI8"] = Set.new %w(oci_bind_array_by_name oci_bind_by_name oci_cancel oci_client_version oci_close oci_commit oci_connect oci_define_by_name oci_error oci_execute oci_fetch_all oci_fetch_array oci_fetch_assoc oci_fetch_object oci_fetch_row oci_fetch oci_field_is_null oci_field_name oci_field_precision oci_field_scale oci_field_size oci_field_type_raw oci_field_type oci_free_descriptor oci_free_statement oci_internal_debug oci_lob_copy oci_lob_is_equal oci_new_collection oci_new_connect oci_new_cursor oci_new_descriptor oci_num_fields oci_num_rows oci_parse oci_password_change oci_pconnect oci_result oci_rollback oci_server_version oci_set_action oci_set_client_identifier oci_set_client_info oci_set_edition oci_set_module_name oci_set_prefetch oci_statement_type)
-          b["OPcache"] = Set.new %w(opcache_invalidate opcache_invalidate opcache_reset opcache_invalidate)
+          b["OCI8"] = Set.new %w(oci_bind_array_by_name oci_bind_array_by_name oci_bind_by_name oci_cancel oci_client_version oci_close oci_commit oci_connect oci_define_by_name oci_error oci_execute oci_fetch_all oci_fetch_array oci_fetch_assoc oci_fetch_object oci_fetch_row oci_fetch oci_field_is_null oci_field_name oci_field_precision oci_field_scale oci_field_size oci_field_type_raw oci_field_type oci_free_descriptor oci_free_statement oci_get_implicit_resultset oci_internal_debug oci_lob_copy oci_lob_is_equal oci_new_collection oci_new_connect oci_new_cursor oci_new_descriptor oci_num_fields oci_num_rows oci_parse oci_password_change oci_pconnect oci_result oci_rollback oci_server_version oci_set_action oci_set_client_identifier oci_set_client_info oci_set_edition oci_set_module_name oci_set_prefetch oci_statement_type oci_bind_array_by_name)
+          b["OPcache"] = Set.new %w(opcache_compile_file opcache_compile_file opcache_invalidate opcache_reset opcache_compile_file)
           b["OpenAL"] = Set.new %w(openal_buffer_create openal_buffer_create openal_buffer_data openal_buffer_destroy openal_buffer_get openal_buffer_loadwav openal_context_create openal_context_current openal_context_destroy openal_context_process openal_context_suspend openal_device_close openal_device_open openal_listener_get openal_listener_set openal_source_create openal_source_destroy openal_source_get openal_source_pause openal_source_play openal_source_rewind openal_source_set openal_source_stop openal_stream openal_buffer_create)
           b["OpenSSL"] = Set.new %w(openssl_cipher_iv_length openssl_cipher_iv_length openssl_csr_export_to_file openssl_csr_export openssl_csr_get_public_key openssl_csr_get_subject openssl_csr_new openssl_csr_sign openssl_decrypt openssl_dh_compute_key openssl_digest openssl_encrypt openssl_error_string openssl_free_key openssl_get_cipher_methods openssl_get_md_methods openssl_get_privatekey openssl_get_publickey openssl_open openssl_pbkdf2 openssl_pkcs12_export_to_file openssl_pkcs12_export openssl_pkcs12_read openssl_pkcs7_decrypt openssl_pkcs7_encrypt openssl_pkcs7_sign openssl_pkcs7_verify openssl_pkey_export_to_file openssl_pkey_export openssl_pkey_free openssl_pkey_get_details openssl_pkey_get_private openssl_pkey_get_public openssl_pkey_new openssl_private_decrypt openssl_private_encrypt openssl_public_decrypt openssl_public_encrypt openssl_random_pseudo_bytes openssl_seal openssl_sign openssl_verify openssl_x509_check_private_key openssl_x509_checkpurpose openssl_x509_export_to_file openssl_x509_export openssl_x509_free openssl_x509_parse openssl_x509_read openssl_cipher_iv_length)
           b["Output Control"] = Set.new %w(flush flush ob_clean ob_end_clean ob_end_flush ob_flush ob_get_clean ob_get_contents ob_get_flush ob_get_length ob_get_level ob_get_status ob_gzhandler ob_implicit_flush ob_list_handlers ob_start output_add_rewrite_var output_reset_rewrite_vars flush)

data/lib/rouge/lexers/puppet.rb

@@ -67,8 +67,7 @@ module Rouge
         rule %r([<>!+*/-]), Operator
 
         rule /(class|include)(\s*)(#{qualname})/ do
-          group Keyword; group Text
-          group Name::Class
+          groups Keyword, Text, Name::Class
         end
 
         rule /node\b/, Keyword, :regex_allowed

data/lib/rouge/lexers/python.rb

@@ -52,9 +52,7 @@ module Rouge
       state :root do
         rule /\n+/m, Text
         rule /^(:)(\s*)([ru]{,2}""".*?""")/mi do
-          group Punctuation
-          group Text
-          group Str::Doc
+          groups Punctuation, Text, Str::Doc
         end
 
         rule /[^\S\n]+/, Text
@@ -69,26 +67,22 @@ module Rouge
         rule /(?:#{keywords.join('|')})\b/, Keyword
 
         rule /(def)((?:\s|\\\s)+)/ do
-          group Keyword # def
-          group Text # whitespae
+          groups Keyword, Text
           push :funcname
         end
 
         rule /(class)((?:\s|\\\s)+)/ do
-          group Keyword
-          group Text
+          groups Keyword, Text
           push :classname
         end
 
         rule /(from)((?:\s|\\\s)+)/ do
-          group Keyword::Namespace
-          group Text
+          groups Keyword::Namespace, Text
           push :fromimport
         end
 
         rule /(import)((?:\s|\\\s)+)/ do
-          group Keyword::Namespace
-          group Text
+          groups Keyword::Namespace, Text
           push :import
         end
 

data/lib/rouge/lexers/racket.rb

@@ -510,9 +510,7 @@ module Rouge
         rule /(?:'|#|`|,@|,|\.)/, Operator
 
         rule /(['#])(\s*)(\()/m do
-          group Str::Symbol
-          group Text
-          group Punctuation
+          groups Str::Symbol, Text, Punctuation
         end
 
         # () [] {} are all permitted as like pairs

data/lib/rouge/lexers/ruby.rb

@@ -42,8 +42,8 @@ module Rouge
           interp = /[rQWxI]/ === m[1]
           toktype = Str::Other
 
-          debug { "    open: #{open.inspect}" }
-          debug { "    close: #{close.inspect}" }
+          puts "    open: #{open.inspect}" if @debug
+          puts "    close: #{close.inspect}" if @debug
 
           # regexes
           if m[1] == 'r'
@@ -241,17 +241,17 @@ module Rouge
           tolerant, heredoc_name = @heredoc_queue.first
           check = tolerant ? m[2].strip : m[2].rstrip
 
-          group Str::Heredoc
-
           # check if we found the end of the heredoc
-          if check == heredoc_name
-            group Name::Constant
+          line_tok = if check == heredoc_name
             @heredoc_queue.shift
             # if there's no more, we're done looking.
             pop! if @heredoc_queue.empty?
+            Name::Constant
           else
-            group Str::Heredoc
+            Str::Heredoc
           end
+
+          groups(Str::Heredoc, line_tok)
         end
 
         rule /[#\\\n]/, Str::Heredoc
@@ -269,7 +269,7 @@ module Rouge
             <<? | >>? | <=>? | >= | ===?
           )
         )x do |m|
-          debug { "matches: #{[m[0], m[1], m[2], m[3]].inspect}" }
+          puts "matches: #{[m[0], m[1], m[2], m[3]].inspect}" if @debug
           groups Name::Class, Operator, Name::Function
           pop!
         end

data/lib/rouge/lexers/rust.rb

@@ -87,7 +87,7 @@ module Rouge
         rule /([.]\s*)?#{id}(?=\s*[(])/m, Name::Function
         rule /[.]\s*#{id}/, Name::Property
         rule /(#{id})(::)/m do
-          group Name::Namespace; group Punctuation
+          groups Name::Namespace, Punctuation
         end
 
         # macros
@@ -109,13 +109,13 @@ module Rouge
 
         rule /[\[{(]/ do |m|
           @macro_delims[delim_map[m[0]]] += 1
-          debug { "    macro_delims: #{@macro_delims.inspect}" }
+          puts "    macro_delims: #{@macro_delims.inspect}" if @debug
           token Punctuation
         end
 
         rule /[\]})]/ do |m|
           @macro_delims[m[0]] -= 1
-          debug { "    macro_delims: #{@macro_delims.inspect}" }
+          puts "    macro_delims: #{@macro_delims.inspect}" if @debug
           pop! if macro_closed?
           token Punctuation
         end

data/lib/rouge/lexers/sass/common.rb

@@ -8,14 +8,14 @@ module Rouge
         rule /@for\b/, Keyword, :for
         rule /@(debug|warn|if|while)/, Keyword, :value
         rule /(@mixin)(\s+)(#{id})/ do
-          group Keyword; group Text; group Name::Function
+          groups Keyword, Text, Name::Function
           push :value
         end
 
         rule /@extend\b/, Keyword, :selector
 
         rule /(@include)(\s+)(#{id})/ do
-          group Keyword; group Text; group Name::Decorator
+          groups Keyword, Text, Name::Decorator
           push :value
         end
 
@@ -23,7 +23,7 @@ module Rouge
 
         # $variable: assignment
         rule /([$]#{id})([ \t]*)(:)/ do
-          group Name::Variable; group Text; group Punctuation
+          groups Name::Variable, Text, Punctuation
           push :value
         end
       end
@@ -157,7 +157,7 @@ module Rouge
         mixin :attr_common
 
         rule /([ \t]*)(:)/ do
-          group Text; group Punctuation
+          groups Text, Punctuation
           push :value
         end
       end

data/lib/rouge/lexers/scala.rb

@@ -0,0 +1,141 @@
+# -*- coding: utf-8 #
+
+module Rouge
+  module Lexers
+    class Scala < RegexLexer
+      desc "The Scala programming language (scala-lang.org)"
+      tag 'scala'
+      aliases 'scala'
+      filenames '*.scala', '*.sbt'
+
+      mimetypes 'text/x-scala', 'application/x-scala'
+
+      # As documented in the ENBF section of the scala specification
+      # http://www.scala-lang.org/docu/files/ScalaReference.pdf
+      whitespace = /\p{Space}/
+      letter = /[\p{L}$_]/
+      upper = /[\p{Lu}$_]/
+      digits = /[0-9]/
+      parens = /[(){}\[\]]/
+      delims = %r([‘’".;,])
+
+      # negative lookahead to filter out other classes
+      op = %r(
+        (?!#{whitespace}|#{letter}|#{digits}|#{parens}|#{delims})
+        [\u0020-\u007F\p{Sm}\p{So}]
+      )x
+
+      idrest = %r(#{letter}(?:#{letter}|#{digits})*(?:(?<=_)#{op}+)?)x
+
+      keywords = %w(
+        abstract case catch def do else extends final finally for forSome
+        if implicit lazy match new override private protected requires return
+        sealed super this throw try val var while with yield
+      )
+
+      state :root do
+        rule /(class|trait|object)(\s+)/ do
+          groups Keyword, Text
+          push :class
+        end
+        rule /'#{idrest}[^']/, Str::Symbol
+        rule /[^\S\n]+/, Text
+
+        rule %r(//.*?\n), Comment::Single
+        rule %r(/\*), Comment::Multiline, :comment
+
+        rule /@#{idrest}/, Name::Decorator
+        rule %r(
+          (#{keywords.join("|")})\b|
+          (<[%:-]|=>|>:|[#=@_\u21D2\u2190])(\b|(?=\s)|$)
+        )x, Keyword
+        rule /:(?!#{op})/, Keyword, :type
+        rule /#{upper}#{idrest}\b/, Name::Class
+        rule /(true|false|null)\b/, Keyword::Constant
+        rule /(import|package)(\s+)/ do
+          groups Keyword, Text
+          push :import
+        end
+
+        rule /(type)(\s+)/ do
+          groups Keyword, Text
+          push :type
+        end
+
+        rule /""".*?"""(?!")/m, Str
+        rule /"(\\\\|\\"|[^"])*"/, Str
+        rule /'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'/, Str::Char
+
+        rule idrest, Name
+        rule /`[^`]+`/, Name
+
+        rule /\[/, Operator, :typeparam
+        rule /[\(\)\{\};,.#]/, Operator
+        rule /#{op}+/, Operator
+
+        rule /([0-9][0-9]*\.[0-9]*|\.[0-9]+)([eE][+-]?[0-9]+)?[fFdD]?/, Num::Float
+        rule /([0-9][0-9]*[fFdD])/, Num::Float
+        rule /0x[0-9a-fA-F]+/, Num::Hex
+        rule /[0-9]+L?/, Num::Integer
+        rule /\n/, Text
+      end
+
+      state :class do
+        rule /(#{idrest}|#{op}+|`[^`]+`)(\s*)(\[)/ do
+          groups Name::Class, Text, Operator
+          push :typeparam
+        end
+
+        rule /\s+/, Text
+        rule /{/, Operator, :pop!
+        rule /\(/, Operator, :pop!
+        rule %r(//.*?\n), Comment::Single, :pop!
+        rule %r(#{idrest}|#{op}+|`[^`]+`), Name::Class, :pop!
+      end
+
+      state :type do
+        rule /\s+/, Text
+        rule /<[%:]|>:|[#_\u21D2]|forSome|type/, Keyword
+        rule /([,\);}]|=>|=)(\s*)/ do
+          groups Operator, Text
+          pop!
+        end
+        rule /[\(\{]/, Operator, :type
+
+        typechunk = /(?:#{idrest}|#{op}+\`[^`]+`)/
+        rule /(#{typechunk}(?:\.#{typechunk})*)(\s*)(\[)/ do
+          groups Keyword::Type, Text, Operator
+          pop!
+          push :typeparam
+        end
+
+        rule /(#{typechunk}(?:\.#{typechunk})*)(\s*)$/ do
+          groups Keyword::Type, Text
+          pop!
+        end
+
+        rule %r(//.*?\n), Comment::Single, :pop!
+        rule /\.|#{idrest}|#{op}+|`[^`]+`/, Keyword::Type
+      end
+
+      state :typeparam do
+        rule /[\s,]+/, Text
+        rule /<[%:]|=>|>:|[#_\u21D2]|forSome|type/, Keyword
+        rule /([\]\)\}])/, Operator, :pop!
+        rule /[\(\[\{]/, Operator, :typeparam
+        rule /\.|#{idrest}|#{op}+|`[^`]+`/, Keyword::Type
+      end
+
+      state :comment do
+        rule %r([^/\*]+), Comment::Multiline
+        rule %r(/\*), Comment::Multiline, :comment
+        rule %r(\*/), Comment::Multiline, :pop!
+        rule %r([*/]), Comment::Multiline
+      end
+
+      state :import do
+        rule %r((#{idrest}|\.)+), Name::Namespace, :pop!
+      end
+    end
+  end
+end

data/lib/rouge/lexers/scheme.rb

@@ -79,9 +79,7 @@ module Rouge
         rule /(?:'|#|`|,@|,|\.)/, Operator
 
         rule /(['#])(\s*)(\()/m do
-          group Str::Symbol
-          group Text
-          group Punctuation
+          groups Str::Symbol, Text, Punctuation
         end
 
         rule /\(/, Punctuation, :command

data/lib/rouge/lexers/sed.rb

@@ -95,14 +95,14 @@ module Rouge
 
         # commands that take a text segment as an argument
         rule /([aic])(\s*)/ do
-          group Keyword; group Text; goto :text
+          groups Keyword, Text; goto :text
         end
 
         rule /[pd]/, Keyword
 
         # commands that take a number argument
         rule /([qQl])(\s+)(\d+)/i do
-          group Keyword; group Text; group Num
+          groups Keyword, Text, Num
           pop!
         end
 
@@ -111,13 +111,13 @@ module Rouge
 
         # commands that take a filename argument
         rule /([rRwW])(\s+)(\S+)/ do
-          group Keyword; group Text; group Name
+          groups Keyword, Text, Name
           pop!
         end
 
         # commands that take a label argument
         rule /([:btT])(\s+)(\S+)/ do
-          group Keyword; group Text; group Name::Label
+          groups Keyword, Text, Name::Label
           pop!
         end
       end

data/lib/rouge/lexers/shell.rb

@@ -39,8 +39,7 @@ module Rouge
         rule /^\S*[\$%>#] +/, Generic::Prompt
 
         rule /(\b\w+)(=)/ do |m|
-          group Name::Variable
-          group Operator
+          groups Name::Variable, Operator
         end
 
         rule /[\[\]{}()=]/, Operator

data/lib/rouge/lexers/smalltalk.rb

@@ -12,14 +12,14 @@ module Rouge
 
       state :root do
         rule /(<)(\w+:)(.*?)(>)/ do
-          group Punctuation; group Keyword; group Text; group Punctuation
+          groups Punctuation, Keyword, Text, Punctuation
         end
 
         # mixin :squeak_fileout
         mixin :whitespaces
         mixin :method_definition
         rule /([|])([\w\s]*)([|])/ do
-          group Punctuation; group Name::Variable; group Punctuation
+          groups Punctuation, Name::Variable, Punctuation
         end
         mixin :objects
         rule /\^|:=|_/, Operator
@@ -30,23 +30,22 @@ module Rouge
 
       state :method_definition do
         rule /([a-z]\w*:)(\s*)(\w+)/i do
-          group Name::Function; group Text; group Name::Variable
+          groups Name::Function, Text, Name::Variable
         end
 
         rule /^(\s*)(\b[a-z]\w*\b)(\s*)$/i do
-          group Text; group Name::Function; group Text
+          groups Text, Name::Function, Text
         end
 
         rule %r(^(\s*)(#{ops}+)(\s*)(\w+)(\s*)$) do
-          group Text; group Name::Function;
-          group Text; group Name::Variable; group Text;
+          groups Text, Name::Function, Text, Name::Variable, Text
         end
       end
 
       state :block_variables do
         mixin :whitespaces
         rule /(:)(\s*)(\w+)/ do
-          group Operator; group Text; group Name::Variable
+          groups Operator, Text, Name::Variable
         end
 
         rule /[|]/, Punctuation, :pop!

data/lib/rouge/lexers/sml.rb

@@ -18,7 +18,11 @@ module Rouge
         )
       end
 
-      id = /[a-z][\w']*/i
+      def self.symbolic_reserved
+        @symbolic_reserved ||= Set.new %w(: | = => -> # :>)
+      end
+
+      id = /[\w']+/i
       symbol = %r([!%&$#/:<=>?@\\~`^|*+-]+)
 
       def self.analyze_text(text)
@@ -75,10 +79,11 @@ module Rouge
       state :core do
         rule /[()\[\]{},;_]|[.][.][.]/, Punctuation
         rule /#"/, Str::Char, :char
+        rule /"/, Str::Double, :string
         rule /~?0x[0-9a-fA-F]+/, Num::Hex
         rule /0wx[0-9a-fA-F]+/, Num::Hex
         rule /0w\d+/, Num::Integer
-        rule /~?\d+[.]\d+[eE]~?\d+/, Num::Float
+        rule /~?\d+([.]\d+)?[eE]~?\d+/, Num::Float
         rule /~?\d+[.]\d+/, Num::Float
         rule /~?\d+/, Num::Integer
 
@@ -98,10 +103,28 @@ module Rouge
           push :dotted
         end
 
-        state :dotted do
-          rule /(#{id})([.])/ do |m|
-            groups(token_for_id_with_dot(m[1]), Punctuation)
-          end
+        rule id do |m|
+          token token_for_id(m[0])
+        end
+
+        rule symbol do |m|
+          token token_for_id(m[0])
+        end
+      end
+
+      state :dotted do
+        rule /(#{id})([.])/ do |m|
+          groups(token_for_id_with_dot(m[1]), Punctuation)
+        end
+
+        rule id do |m|
+          token token_for_id(m[0])
+          pop!
+        end
+
+        rule symbol do |m|
+          token token_for_id(m[0])
+          pop!
         end
       end
 
@@ -109,6 +132,213 @@ module Rouge
         rule /#!.*?\n/, Comment::Preproc
         rule(//) { push :main }
       end
+
+      state :main do
+        mixin :whitespace
+
+        rule /\b(val|and)\b(?!')/, Keyword::Reserved, :vname
+        rule /\b(fun)\b(?!')/ do
+          token Keyword::Reserved
+          goto :main_fun
+          push :fname
+        end
+
+        mixin :delimiters
+        mixin :core
+      end
+
+      state :main_fun do
+        mixin :whitespace
+        rule /\b(fun|and)\b(?!')/, Keyword::Reserved, :fname
+        rule /\bval\b(?!')/ do
+          token Keyword::Reserved
+          goto :main
+          push :vname
+        end
+
+        rule /[|]/, Punctuation, :fname
+        rule /\b(case|handle)\b(?!')/ do
+          token Keyword::Reserved
+          goto :main
+        end
+
+        mixin :delimiters
+        mixin :core
+      end
+
+      state :has_escapes do
+        rule /\\[\\"abtnvfr]/, Str::Escape
+        rule /\\\^[\x40-\x5e]/, Str::Escape
+        rule /\\[0-9]{3}/, Str::Escape
+        rule /\\u\h{4}/, Str::Escape
+        rule /\\\s+\\/, Str::Interpol
+      end
+
+      state :string do
+        rule /[^"\\]+/, Str::Double
+        rule /"/, Str::Double, :pop!
+        mixin :has_escapes
+      end
+
+      state :char do
+        rule /[^"\\]+/, Str::Char
+        rule /"/, Str::Char, :pop!
+        mixin :has_escapes
+      end
+
+      state :breakout do
+        rule /(?=\w+\b(#{SML.keywords.to_a.join('|')})\b(?!'))/ do
+          pop!
+        end
+      end
+
+      state :sname do
+        mixin :whitespace
+        mixin :breakout
+        rule id, Name::Namespace
+        rule(//) { pop! }
+      end
+
+      state :has_annotations do
+        rule /'[\w']*/, Name::Decorator
+        rule /[(]/, Punctuation, :tyvarseq
+      end
+
+      state :fname do
+        mixin :whitespace
+        mixin :has_annotations
+
+        rule id, Name::Function, :pop!
+        rule symbol, Name::Function, :pop!
+      end
+
+      state :vname do
+        mixin :whitespace
+        mixin :has_annotations
+
+        rule /(#{id})(\s*)(=(?!#{symbol}))/m do
+          groups Name::Variable, Text, Punctuation
+          pop!
+        end
+
+        rule /(#{symbol})(\s*)(=(?!#{symbol}))/m do
+          groups Name::Variable, Text, Punctuation
+        end
+
+        rule id, Name::Variable, :pop!
+        rule symbol, Name::Variable, :pop!
+
+        rule(//) { pop! }
+      end
+
+      state :tname do
+        mixin :whitespace
+        mixin :breakout
+        mixin :has_annotations
+
+        rule /'[\w']*/, Name::Decorator
+        rule /[(]/, Punctuation, :tyvarseq
+
+        rule %r(=(?!#{symbol})) do
+          token Punctuation
+          goto :typbind
+        end
+
+        rule id, Keyword::Type
+        rule symbol, Keyword::Type
+      end
+
+      state :typbind do
+        mixin :whitespace
+
+        rule /\b(and)\b(?!')/ do
+          token Keyword::Reserved
+          goto :tname
+        end
+
+        mixin :breakout
+        mixin :core
+      end
+
+      state :dname do
+        mixin :whitespace
+        mixin :breakout
+        mixin :has_annotations
+
+        rule /(=)(\s*)(datatype)\b/ do
+          groups Punctuation, Text, Keyword::Reserved
+          pop!
+        end
+
+        rule %r(=(?!#{symbol})) do
+          token Punctuation
+          goto :datbind
+          push :datcon
+        end
+
+        rule id, Keyword::Type
+        rule symbol, Keyword::Type
+      end
+
+      state :datbind do
+        mixin :whitespace
+        rule /\b(and)\b(?!')/ do
+          token Keyword::Reserved; goto :dname
+        end
+        rule /\b(withtype)\b(?!')/ do
+          token Keyword::Reserved; goto :tname
+        end
+        rule /\bof\b(?!')/, Keyword::Reserved
+        rule /([|])(\s*)(#{id})/ do
+          groups(Punctuation, Text, Name::Class)
+        end
+
+        rule /([|])(\s+)(#{symbol})/ do
+          groups(Punctuation, Text, Name::Class)
+        end
+
+        mixin :breakout
+        mixin :core
+      end
+
+      state :ename do
+        mixin :whitespace
+        rule /(exception|and)(\s+)(#{id})/ do
+          groups Keyword::Reserved, Text, Name::Class
+        end
+
+        rule /(exception|and)(\s*)(#{symbol})/ do
+          groups Keyword::Reserved, Text, Name::Class
+        end
+
+        rule /\b(of)\b(?!')/, Keyword::Reserved
+        mixin :breakout
+        mixin :core
+      end
+
+      state :datcon do
+        mixin :whitespace
+        rule id, Name::Class, :pop!
+        rule symbol, Name::Class, :pop!
+      end
+
+      state :tyvarseq do
+        mixin :whitespace
+        rule /'[\w']*/, Name::Decorator
+        rule id, Name
+        rule /,/, Punctuation
+        rule /[)]/, Punctuation, :pop!
+        rule symbol, Name
+      end
+
+      state :comment do
+        rule /[^(*)]+/, Comment::Multiline
+        rule /[(][*]/ do
+          token Comment::Multiline; push
+        end
+        rule /[*][)]/, Comment::Multiline, :pop!
+        rule /[(*)]/, Comment::Multiline
+      end
     end
   end
 end