rouge 0.2.8 → 0.2.9

data/Gemfile

@@ -3,7 +3,6 @@ source 'http://rubygems.org'
 gemspec
 
 gem 'minitest'
-gem 'sexp_processor', '~> 3.0'
 gem 'wrong', '~> 0.6.2'
 
 gem 'rake'

data/lib/rouge.rb

@@ -1,16 +1,27 @@
 # stdlib
 require 'pathname'
 
+# The containing module for Rouge
 module Rouge
   class << self
-    def highlight(text, lexer, formatter)
+    # Highlight some text with a given lexer and formatter.
+    #
+    # @example
+    #   Rouge.highlight('@foo = 1', 'ruby', 'html')
+    #   Rouge.highlight('var foo = 1;', 'js', 'terminal256')
+    #
+    #   # streaming - chunks become available as they are lexed
+    #   Rouge.highlight(large_string, 'ruby', 'html') do |chunk|
+    #     $stdout.print chunk
+    #   end
+    def highlight(text, lexer, formatter, &b)
       lexer = Lexer.find(lexer) unless lexer.respond_to? :lex
       raise "unknown lexer #{lexer}" unless lexer
 
       formatter = Formatter.find(formatter) unless formatter.respond_to? :format
       raise "unknown formatter #{formatter}" unless formatter
 
-      formatter.format(lexer.lex(text))
+      formatter.format(lexer.lex(text), &b)
     end
   end
 end
@@ -38,6 +49,9 @@ load load_dir.join('rouge/lexers/sql.rb')
 load load_dir.join('rouge/lexers/make.rb')
 load load_dir.join('rouge/lexers/shell.rb')
 load load_dir.join('rouge/lexers/viml.rb')
+load load_dir.join('rouge/lexers/nginx.rb')
+load load_dir.join('rouge/lexers/conf.rb')
+load load_dir.join('rouge/lexers/sed.rb')
 
 load load_dir.join('rouge/lexers/javascript.rb')
 load load_dir.join('rouge/lexers/css.rb')
@@ -60,6 +74,7 @@ load load_dir.join('rouge/lexers/perl.rb')
 load load_dir.join('rouge/lexers/factor.rb')
 load load_dir.join('rouge/lexers/clojure.rb')
 load load_dir.join('rouge/lexers/groovy.rb')
+load load_dir.join('rouge/lexers/io.rb')
 
 load load_dir.join('rouge/lexers/haskell.rb')
 load load_dir.join('rouge/lexers/scheme.rb')

data/lib/rouge/cli.rb

@@ -60,7 +60,7 @@ module Rouge
       formatter = formatter_class.new(normalize_hash_keys(options[:formatter_opts]))
       lexer = lexer_class.new(normalize_hash_keys(options[:lexer_opts]))
 
-      puts Rouge.highlight(source, lexer, formatter)
+      formatter.format(lexer.lex(source), &method(:print))
     end
 
     desc 'style THEME', 'render THEME as css'
@@ -69,7 +69,7 @@ module Rouge
       theme = Theme.find(theme_name)
       raise "unknown theme: #{theme_name}" unless theme
 
-      puts theme.new(options).render
+      theme.new(options).render(&method(:puts))
     end
 
     desc 'list', 'list the available lexers, formatters, and styles'

data/lib/rouge/demos/conf

@@ -0,0 +1,4 @@
+# A generic configuration file
+option1 "val1"
+option2 23
+option3 'val3'

data/lib/rouge/demos/io

@@ -0,0 +1,11 @@
+bottle := method(i,
+  if(i==0, return "no more bottles of beer")
+  if(i==1, return "1 bottle of beer")
+  return i asString .. " bottles of beer"
+)
+
+for(i, 99, 1, -1,
+  write(bottle(i), " on the wall, ", bottle(i), ",\n")
+  write("take one down, pass it around,\n")
+  write(bottle(i - 1), " on the wall.\n\n")
+)

data/lib/rouge/demos/sed

@@ -0,0 +1,4 @@
+/begin/,/end/ {
+  /begin/n # skip over the line that has "begin" on it
+  s/old/new/
+}

data/lib/rouge/formatter.rb

@@ -19,13 +19,18 @@ module Rouge
     end
 
     # Format a token stream.  Delegates to {#format}.
-    def self.format(tokens, opts={})
-      new(opts).format(tokens)
+    def self.format(tokens, opts={}, &b)
+      new(opts).format(tokens, &b)
     end
 
     # Format a token stream.
-    def format(tokens)
-      enum_for(:stream, tokens).to_a.join
+    def format(tokens, &b)
+      return stream(tokens, &b) if block_given?
+
+      out = ''
+      stream(tokens) { |piece| out << piece }
+
+      out
     end
 
     # @deprecated Use {#format} instead.

data/lib/rouge/formatters/html.rb

@@ -73,7 +73,7 @@ module Rouge
         when ''
           yield val
         when nil
-          raise "unknown token: #{tok.inspect}"
+          raise "unknown token: #{tok.inspect} for #{val.inspect}"
         else
           yield '<span class='
           yield tok.shortname.inspect

data/lib/rouge/lexer.rb

@@ -265,13 +265,9 @@ module Rouge
       # be changed once it has begun lexing.  This method will redefine
       # itself on the first call to a noop if "debug" is not set.
       if option(:debug)
-        class << self
-          def debug; puts yield; end
-        end
+        def self.debug; puts yield; end
       else
-        class << self
-          def debug; end
-        end
+        def self.debug; end
       end
 
       debug(&b)
@@ -299,7 +295,7 @@ module Rouge
       # consolidate consecutive tokens of the same type
       last_token = nil
       last_val = nil
-      stream_tokens(StringScanner.new(string)) do |tok, val|
+      stream_tokens(string) do |tok, val|
         next if val.empty?
 
         if tok == last_token

data/lib/rouge/lexers/conf.rb

@@ -0,0 +1,21 @@
+module Rouge
+  module Lexers
+    class Conf < RegexLexer
+      tag 'conf'
+      aliases 'config', 'configuration'
+
+      desc 'A generic lexer for configuration files'
+      filenames '*.conf', '*.config'
+
+      # short and sweet
+      state :root do
+        rule /#.*?\n/, 'Comment'
+        rule /".*?"/, 'Literal.String.Double'
+        rule /'.*?'/, 'Literal.String.Single'
+        rule /[a-z]\w*/i, 'Name'
+        rule /\d+/, 'Literal.Number'
+        rule /[^\d\w#"']+/, 'Text'
+      end
+    end
+  end
+end

data/lib/rouge/lexers/io.rb

@@ -0,0 +1,65 @@
+module Rouge
+  module Lexers
+    class IO < RegexLexer
+      tag 'io'
+      desc 'The IO programming language (http://iolanguage.com)'
+      mimetypes 'text/x-iosrc'
+      filenames '*.io'
+
+      def self.analyze_text(text)
+        return 1 if text.shebang? 'io'
+      end
+
+      def self.constants
+        @constants ||= Set.new %w(nil false true)
+      end
+
+      def self.builtins
+        @builtins ||= Set.new %w(
+          args call clone do doFile doString else elseif for if list
+          method return super then
+        )
+      end
+
+      state :root do
+        rule /\s+/m, 'Text'
+        rule %r(//.*?\n), 'Comment.Single'
+        rule %r(#.*?\n), 'Comment.Single'
+        rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, 'Comment.Multiline'
+        rule %r(/[+]), 'Comment.Multiline', :nested_comment
+
+        rule /"(\\\\|\\"|[^"])*"/, 'Literal.String'
+
+        rule %r(:?:=), 'Keyword'
+        rule /[()]/, 'Punctuation'
+
+        rule %r([-=;,*+><!/|^.%&\[\]{}]), 'Operator'
+
+        rule /[A-Z]\w*/, 'Name.Class'
+
+        rule /[a-z_]\w*/ do |m|
+          name = m[0]
+
+          if self.class.constants.include? name
+            token 'Keyword.Constant'
+          elsif self.class.builtins.include? name
+            token 'Name.Builtin'
+          else
+            token 'Name'
+          end
+        end
+
+        rule %r((\d+[.]?\d*|\d*[.]\d+)(e[+-]?[0-9]+)?)i, 'Literal.Number.Float'
+        rule /\d+/, 'Literal.Number.Integer'
+
+        rule /@@?/, 'Keyword'
+      end
+
+      state :nested_comment do
+        rule %r([^/+]+)m, 'Comment.Multiline'
+        rule %r(/[+]), 'Comment.Multiline', :nested_comment
+        rule %r([+]/), 'Comment.Multiline', :pop!
+      end
+    end
+  end
+end

data/lib/rouge/lexers/nginx.rb

@@ -3,6 +3,7 @@ module Rouge
     class Nginx < RegexLexer
       tag 'nginx'
       mimetypes 'text/x-nginx-conf'
+      filenames 'nginx.conf'
 
       id = /[^\s$;{}()#]+/
 

data/lib/rouge/lexers/perl.rb

@@ -55,7 +55,7 @@ module Rouge
         rule %r({(\\\\|\\}|[^}])*}[egimosx]*), re_tok, :pop!
         rule %r(<(\\\\|\\>|[^>])*>[egimosx]*), re_tok, :pop!
         rule %r(\[(\\\\|\\\]|[^\]])*\][egimosx]*), re_tok, :pop!
-        rule %r(\((\\\\|\\\)|[^\)])*\)[egimosx]*), re_tok, :pop!
+        rule %r[\((\\\\|\\\)|[^\)])*\)[egimosx]*], re_tok, :pop!
         rule %r(@(\\\\|\\\@|[^\@])*@[egimosx]*), re_tok, :pop!
         rule %r(%(\\\\|\\\%|[^\%])*%[egimosx]*), re_tok, :pop!
         rule %r(\$(\\\\|\\\$|[^\$])*\$[egimosx]*), re_tok, :pop!
@@ -87,7 +87,7 @@ module Rouge
         rule %r(s{(\\\\|\\}|[^}])*}\s*), re_tok, :balanced_regex
         rule %r(s<(\\\\|\\>|[^>])*>\s*), re_tok, :balanced_regex
         rule %r(s\[(\\\\|\\\]|[^\]])*\]\s*), re_tok, :balanced_regex
-        rule %r(s\((\\\\|\\\)|[^\)])*\)\s*), re_tok, :balanced_regex
+        rule %r[s\((\\\\|\\\)|[^\)])*\)\s*], re_tok, :balanced_regex
 
         rule %r(m?/(\\\\|\\/|[^/\n])*/[gcimosx]*), re_tok
         rule %r(m(?=[/!\\{<\[\(@%\$])), re_tok, :balanced_regex

data/lib/rouge/lexers/ruby.rb

@@ -135,7 +135,10 @@ module Rouge
       end
 
       state :root do
+        rule /\n\s*/m, 'Text', :expr_start
+        rule /\s+/, 'Text' # NB: NOT /m
         rule /#.*$/, 'Comment.Single'
+
         rule %r(=begin\b.*?end\b)m, 'Comment.Multiline'
         rule /(?:#{keywords.join('|')})\b/, 'Keyword', :expr_start
         rule /(?:#{keywords_pseudo.join('|')})\b/, 'Keyword.Pseudo', :expr_start
@@ -194,8 +197,6 @@ module Rouge
         rule /[-+\/*%=<>&!^|~]=?/, 'Operator', :expr_start
         rule %r<[({,?:\\;/]>, 'Punctuation', :expr_start
         rule %r<[)}]>, 'Punctuation'
-        rule /\n\s*/m, 'Text', :expr_start
-        rule /\s+/, 'Text' # NB: NOT /m
       end
 
       state :has_heredocs do

data/lib/rouge/lexers/sed.rb

@@ -1,21 +1,166 @@
 module Rouge
   module Lexers
-    class Sed < Lexer
-      # TODO: add rules for this
-      class Regex < TextLexer
-        default_option :token, 'Literal.String.Regex'
+    class Sed < RegexLexer
+      desc 'sed, the ultimate stream editor'
+
+      tag 'sed'
+      filenames '*.sed'
+      mimetypes 'text/x-sed'
+
+      def self.analyze_text(text)
+        return 1 if text.shebang? 'sed'
       end
 
-      state :root do
+      class Regex < RegexLexer
+        state :root do
+          rule /\\./, 'Literal.String.Escape'
+          rule /\[/, 'Punctuation', :brackets
+          rule /[$^.*]/, 'Operator'
+          rule /[()]/, 'Punctuation'
+          rule /./, 'Literal.String.Regex'
+        end
+
+        state :brackets do
+          rule /\^?/ do
+            token 'Punctuation'
+            pop!; push :brackets_int
+          end
+        end
+
+        state :brackets_int do
+          # ranges
+          rule /.-./, 'Name.Variable'
+          rule /\]/, 'Punctuation', :pop!
+          rule /./, 'Literal.String.Regex'
+        end
+      end
+
+      class Replacement < RegexLexer
+        state :root do
+          rule /\\./m, 'Literal.String.Escape'
+          rule /&/, 'Operator'
+          rule /[^\\&]+/m, 'Text'
+        end
+      end
+
+      def regex
+        @regex ||= Regex.new(options)
+      end
+
+      def replacement
+        @replacement ||= Replacement.new(options)
+      end
+
+      start { regex.reset!; replacement.reset! }
+
+      state :whitespace do
         rule /\s+/m, 'Text'
-        rule /#.*?\n/, 'Comment'
+        rule(/#.*?\n/) { token 'Comment'; reset_stack }
+        rule(/\n/) { token 'Text'; reset_stack }
+        rule(/;/) { token 'Punctuation'; reset_stack }
+      end
+
+      state :root do
+        mixin :addr_range
+      end
+
+      edot = /\\.|./m
+
+      state :command do
+        mixin :whitespace
+
+        # subst and transliteration
+        rule /(s)(.)(#{edot}*?)(\2)(#{edot}*?)(\2)/m do |m|
+          token 'Keyword', m[1]
+          token 'Punctuation', m[2]
+          delegate regex, m[3]
+          token 'Punctuation', m[4]
+          delegate replacement, m[5]
+          token 'Punctuation', m[6]
+
+
+          pop!; push :flags
+        end
+
+        rule /(y)(.)(#{edot}*?)(\2)(#{edot}*?)(\2)/m do |m|
+          token 'Keyword', m[1]
+          token 'Punctuation', m[2]
+          delegate replacement, m[3]
+          token 'Punctuation', m[4]
+          delegate replacement, m[5]
+          token 'Punctuation', m[6]
+
+          pop!
+        end
+
+        # commands that take a text segment as an argument
+        rule /([aic])(\s*)/ do
+          group 'Keyword'; group 'Text'; pop!; push :text
+        end
+
+        rule /[pd]/, 'Keyword'
 
-        rule /s(.)(\\.|.*?)(\1)/ do
-          token re_tok
-          push :flags
-          push :subst
-          push :regex
+        # commands that take a number argument
+        rule /([qQl])(\s+)(\d+)/i do
+          group 'Keyword'; group 'Text'; group 'Literal.Number'
+          pop!
         end
+
+        # no-argument commands
+        rule /[={}dDgGhHlnpPqx]/, 'Keyword', :pop!
+
+        # commands that take a filename argument
+        rule /([rRwW])(\s+)(\S+)/ do
+          group 'Keyword'; group 'Text'; group 'Name'
+          pop!
+        end
+
+        # commands that take a label argument
+        rule /([:btT])(\s+)(\S+)/ do
+          group 'Keyword'; group 'Text'; group 'Name.Label'
+          pop!
+        end
+      end
+
+      state :addr_range do
+        mixin :whitespace
+
+        ### address ranges ###
+        addr_tok = 'Keyword.Namespace'
+        rule /\d+/, addr_tok
+        rule /[$,~+!]/, addr_tok
+
+        rule %r((/)(\\.|.)*?(/)) do |m|
+          token addr_tok, m[1]; delegate regex, m[2]; token addr_tok, m[3]
+        end
+
+        # alternate regex rage delimiters
+        rule %r((\\)(.)(\\.|.)*?(\2)) do |m|
+          token addr_tok, m[1] + m[2]
+          delegate regex, m[3]
+          token addr_tok, m[4]
+        end
+
+        rule(//) { push :command }
+      end
+
+      state :text do
+        rule /[^\\\n]+/, 'Literal.String'
+        rule /\\\n/, 'Literal.String.Escape'
+        rule /\\/, 'Literal.String'
+        rule /\n/, 'Text', :pop!
+      end
+
+      state :flags do
+        rule /[gp]+/, 'Keyword', :pop!
+
+        # writing to a file with the subst command.
+        # who'da thunk...?
+        rule /([wW])(\s+)(\S+)/ do
+          token 'Keyword'; token 'Text'; token 'Name'
+        end
+
+        rule(//) { pop! }
       end
     end
   end

data/lib/rouge/lexers/shell.rb

@@ -72,6 +72,7 @@ module Rouge
       end
 
       state :data do
+        rule /\s+/, 'Text'
         rule /\\./, 'Literal.String.Escape'
         rule /\$?"/, 'Literal.String.Double', :double_quotes
 
@@ -85,9 +86,8 @@ module Rouge
         rule /\*/, 'Keyword'
 
         rule /;/, 'Text'
-        rule /\s+/, 'Text'
         rule /[^=\*\s{}()$"\'`\\<]+/, 'Text'
-        rule /\d+(?= |\Z)/, 'Number'
+        rule /\d+(?= |\Z)/, 'Literal.Number'
         rule /</, 'Text'
         mixin :interp
       end
@@ -108,7 +108,7 @@ module Rouge
       state :math do
         rule /\)\)/, 'Keyword', :pop!
         rule %r([-+*/%^|&]|\*\*|\|\|), 'Operator'
-        rule /\d+/, 'Number'
+        rule /\d+/, 'Literal.Number'
         mixin :root
       end
 

data/lib/rouge/lexers/text.rb

@@ -13,8 +13,8 @@ module Rouge
         @token ||= Token[option :token]
       end
 
-      def stream_tokens(stream, &b)
-        yield self.token, stream.string
+      def stream_tokens(string, &b)
+        yield self.token, string
       end
     end
   end

data/lib/rouge/regex_lexer.rb

@@ -183,7 +183,9 @@ module Rouge
     #    is consumed with an `'Error'` token, and we continue at (1.)
     #
     # @see #step #step (where (2.) is implemented)
-    def stream_tokens(stream, &b)
+    def stream_tokens(str, &b)
+      stream = StringScanner.new(str)
+
       until stream.eos?
         debug { "lexer: #{self.class.tag}" }
         debug { "stack: #{stack.map(&:name).inspect}" }
@@ -354,7 +356,7 @@ module Rouge
   private
     def with_output_stream(output_stream, &b)
       old_output_stream = @output_stream
-      @output_stream = Yielder.new do |tok, val|
+      @output_stream = Enumerator::Yielder.new do |tok, val|
         debug { "    yielding #{tok.to_s.inspect}, #{val.inspect}" }
         output_stream.call(Token[tok], val)
       end

data/lib/rouge/util.rb

@@ -1,14 +1,4 @@
 module Rouge
-  class Yielder
-    def initialize(&pr)
-      @proc = pr
-    end
-
-    def <<(*a)
-      @proc && @proc.call(*a)
-    end
-  end
-
   class InheritableHash < Hash
     def initialize(parent=nil)
       @parent = parent

data/lib/rouge/version.rb

@@ -1,5 +1,5 @@
 module Rouge
   def self.version
-    "0.2.8"
+    "0.2.9"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rouge
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.2.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-30 00:00:00.000000000 Z
+date: 2012-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -42,6 +42,7 @@ files:
 - lib/rouge/lexers/sql.rb
 - lib/rouge/lexers/common_lisp.rb
 - lib/rouge/lexers/erb.rb
+- lib/rouge/lexers/conf.rb
 - lib/rouge/lexers/scheme.rb
 - lib/rouge/lexers/clojure.rb
 - lib/rouge/lexers/xml.rb
@@ -53,6 +54,7 @@ files:
 - lib/rouge/lexers/perl.rb
 - lib/rouge/lexers/javascript.rb
 - lib/rouge/lexers/diff.rb
+- lib/rouge/lexers/io.rb
 - lib/rouge/lexers/viml.rb
 - lib/rouge/lexers/objective_c.rb
 - lib/rouge/lexers/smalltalk.rb
@@ -109,6 +111,7 @@ files:
 - lib/rouge/demos/scheme
 - lib/rouge/demos/diff
 - lib/rouge/demos/markdown
+- lib/rouge/demos/io
 - lib/rouge/demos/javascript
 - lib/rouge/demos/json
 - lib/rouge/demos/css
@@ -116,10 +119,12 @@ files:
 - lib/rouge/demos/php
 - lib/rouge/demos/shell
 - lib/rouge/demos/make
+- lib/rouge/demos/sed
 - lib/rouge/demos/java
 - lib/rouge/demos/yaml
 - lib/rouge/demos/sql
 - lib/rouge/demos/groovy
+- lib/rouge/demos/conf
 - lib/rouge/demos/nginx
 - lib/rouge/demos/tcl
 - lib/rouge/demos/c