rouge 0.2.3 → 0.2.4

data/lib/rouge.rb

@@ -7,6 +7,9 @@ module Rouge
       lexer = Lexer.find(lexer) unless lexer.respond_to? :lex
       raise "unknown lexer #{lexer}" unless lexer
 
+      formatter = Formatter.find(formatter) unless formatter.respond_to? :format
+      raise "unknown formatter #{formatter}" unless formatter
+
       formatter.format(lexer.lex(text))
     end
   end

data/lib/rouge/formatter.rb

@@ -1,6 +1,7 @@
 module Rouge
   # A Formatter takes a token stream and formats it for human viewing.
   class Formatter
+    # @private
     REGISTRY = {}
 
     # Specify or get the unique tag for this formatter.  This is used
@@ -17,11 +18,17 @@ module Rouge
       REGISTRY[tag]
     end
 
+    # Format a token stream.  Delegates to {#format}.
+    def self.format(tokens, opts={})
+      new(opts).format(tokens)
+    end
+
     # Format a token stream.
     def format(tokens)
       enum_for(:stream, tokens).to_a.join
     end
 
+    # @deprecated Use {#format} instead.
     def render(tokens)
       warn 'Formatter#render is deprecated, use #format instead.'
       format(tokens)

data/lib/rouge/lexer.rb

@@ -68,6 +68,21 @@ module Rouge
         end
       end
 
+      # Specify or get the path name containing a small demo for
+      # this lexer (can be overriden by {demo}).
+      def demo_file(arg=:absent)
+        return @demo_file = Pathname.new(arg) unless arg == :absent
+
+        @demo_file = Pathname.new(__FILE__).dirname.join('demos', tag)
+      end
+
+      # Specify or get a small demo string for this lexer
+      def demo(arg=:absent)
+        return @demo = arg unless arg == :absent
+
+        @demo = File.read(demo_file)
+      end
+
       # @return a list of all lexers.
       def all
         registry.values.uniq

data/lib/rouge/lexers/javascript.rb

@@ -46,46 +46,85 @@ module Rouge
         rule /[^\n]+/, 'Error', :pop!
       end
 
-      keywords = %w(
-        for in while do break return continue switch case default if else
-        throw try catch finally new delete typeof instanceof void this
-      ).join('|')
-
-      declarations = %w(var let with function).join('|')
-
-      reserved = %w(
-        abstract boolean byte char class const debugger double enum export
-        extends final float goto implements import int interface long
-        native package private protected public short static super
-        synchronized throws transient volatile
-      ).join('|')
-
-      constants = %w(true false null NaN Infinity undefined).join('|')
-
-      builtins = %w(
-        Array Boolean Date Error Function Math netscape
-        Number Object Packages RegExp String sun decodeURI
-        decodeURIComponent encodeURI encodeURIComponent
-        Error eval isFinite isNaN parseFloat parseInt document this
-        window
-      ).join('|')
+      def self.keywords
+        @keywords ||= Set.new %w(
+          for in while do break return continue switch case default
+          if else throw try catch finally new delete typeof instanceof
+          void this
+        )
+      end
+
+      def self.declarations
+        @declarations ||= Set.new %w(var let with function)
+      end
+
+      def self.reserved
+        @reserved ||= Set.new %w(
+          abstract boolean byte char class const debugger double enum
+          export extends final float goto implements import int interface
+          long native package private protected public short static
+          super synchronized throws transient volatile
+        )
+      end
+
+      def self.constants
+        @constants ||= Set.new %w(true false null NaN Infinity undefined)
+      end
+
+      def self.builtins
+        @builtins ||= %w(
+          Array Boolean Date Error Function Math netscape
+          Number Object Packages RegExp String sun decodeURI
+          decodeURIComponent encodeURI encodeURIComponent
+          Error eval isFinite isNaN parseFloat parseInt document this
+          window
+        )
+      end
+
+      id = /[$a-zA-Z_][a-zA-Z0-9_]*/
 
       state :root do
         rule /\A\s*#!.*?\n/m, 'Comment.Preproc'
         rule %r((?<=\n)(?=\s|/|<!--)), 'Text', :slash_starts_regex
         mixin :comments_and_whitespace
         rule %r(\+\+ | -- | ~ | && | \|\| | \\(?=\n) | << | >>>? | ===
-               | !== | \? | : )x,
+               | !== )x,
           'Operator', :slash_starts_regex
         rule %r([-<>+*%&|\^/!=]=?), 'Operator', :slash_starts_regex
-        rule /[{(\[;,]/, 'Punctuation', :slash_starts_regex
-        rule /[})\].]/, 'Punctuation'
-        rule /(?:#{keywords})\b/, 'Keyword', :slash_starts_regex
-        rule /(?:#{declarations})\b/, 'Keyword.Declaration', :slash_starts_regex
-        rule /(?:#{reserved})\b/, 'Keyword.Reserved'
-        rule /(?:#{constants})\b/, 'Keyword.Constant'
-        rule /(?:#{builtins})\b/, 'Name.Builtin'
-        rule /[$a-zA-Z_][a-zA-Z0-9_]*/, 'Name.Other'
+        rule /[(\[;,]/, 'Punctuation', :slash_starts_regex
+        rule /[)\].]/, 'Punctuation'
+
+        rule /[?]/ do
+          token 'Punctuation'
+          push :ternary
+          push :slash_starts_regex
+        end
+
+        rule /[{](?=\s*(#{id}|"[^\n]*?")\s*:)/, 'Punctuation', :object
+
+        rule /[{]/ do
+          token 'Punctuation'
+          push :block
+          push :slash_starts_regex
+        end
+
+        rule id do |m|
+          if self.class.keywords.include? m[0]
+            token 'Keyword'
+            push :slash_starts_regex
+          elsif self.class.declarations.include? m[0]
+            token 'Keyword.Declaration'
+            push :slash_starts_regex
+          elsif self.class.reserved.include? m[0]
+            token 'Keyword.Reserved'
+          elsif self.class.constants.include? m[0]
+            token 'Keyword.Constant'
+          elsif self.class.builtins.include? m[0]
+            token 'Name.Builtin'
+          else
+            token 'Name.Other'
+          end
+        end
 
         rule /[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?/, 'Literal.Number.Float'
         rule /0x[0-9a-fA-F]+/, 'Literal.Number.Hex'
@@ -93,6 +132,33 @@ module Rouge
         rule /"(\\\\|\\"|[^"])*"/, 'Literal.String.Double'
         rule /'(\\\\|\\'|[^'])*'/, 'Literal.String.Single'
       end
+
+      # braced parts that aren't object literals
+      state :block do
+        rule /(#{id})(\s*)(:)/ do
+          group 'Name.Label'; group 'Text'
+          group 'Punctuation'
+        end
+
+        rule /[}]/, 'Punctuation', :pop!
+        mixin :root
+      end
+
+      # object literals
+      state :object do
+        rule /[}]/, 'Punctuation', :pop!
+        rule /(#{id})(\s*)(:)/ do
+          group 'Name.Attribute'; group 'Text'
+          group 'Punctuation'
+        end
+        mixin :root
+      end
+
+      # ternary expressions, where <id>: is not a label!
+      state :ternary do
+        rule /:/, 'Punctuation', :pop!
+        mixin :root
+      end
     end
 
     class JSON < RegexLexer

data/lib/rouge/lexers/sass.rb

@@ -37,7 +37,7 @@ module Rouge
 
         rule /:/, 'Name.Attribute', :old_style_attr
 
-        rule(/(?=.+?:(^a-z|$))/) { push :attribute }
+        rule(/(?=.+?:([^a-z]|$))/) { push :attribute }
 
         rule(//) { push :selector }
       end
@@ -76,6 +76,8 @@ module Rouge
             push :value
           end
 
+          rule /@extend\b/, 'Keyword', :selector
+
           rule /(@include)(\s+)(#{id})/ do
             group 'Keyword'; group 'Text'; group 'Name.Decorator'
             push :value
@@ -96,6 +98,7 @@ module Rouge
           rule /[$]#{id}/, 'Name.Variable'
           rule /url[(]/, 'Literal.String.Other', :string_url
           rule /#{id}(?=\s*[(])/, 'Name.Function'
+          rule /%#{id}/, 'Name.Decorator'
 
           # named literals
           rule /(true|false)\b/, 'Name.Pseudo'
@@ -149,6 +152,7 @@ module Rouge
           rule /:/, 'Name.Decorator', :pseudo_class
           rule /[.]/, 'Name.Class', :class
           rule /#/, 'Name.Namespace', :id
+          rule /%/, 'Name.Variable', :placeholder
           rule id, 'Name.Tag'
           rule /&/, 'Keyword'
           rule /[~^*!&\[\]()<>\|+=@:;,.\/?-]/, 'Operator'
@@ -192,6 +196,11 @@ module Rouge
           mixin :selector_piece
         end
 
+        state :placeholder do
+          rule id, 'Name.Variable'
+          mixin :selector_piece
+        end
+
         state :for do
           rule /(from|to|through)/, 'Operator.Word'
           mixin :value

data/lib/rouge/lexers/scheme.rb

@@ -7,69 +7,76 @@ module Rouge
       filenames '*.scm', '*.ss', '*.rkt'
       mimetypes 'text/x-scheme', 'application/x-scheme'
 
-      keywords = %w(
-        lambda define if else cond and or case let let* letrec begin
-        do delay set! => quote quasiquote unquote unquote-splicing
-        define-syntax let-syntax letrec-syntax syntax-rules
-      )
+      def self.keywords
+        @keywords ||= Set.new %w(
+          lambda define if else cond and or case let let* letrec begin
+          do delay set! => quote quasiquote unquote unquote-splicing
+          define-syntax let-syntax letrec-syntax syntax-rules
+        )
+      end
 
-      builtins = %w(
-        * + - / < <= = > >= abs acos angle append apply asin
-        assoc assq assv atan boolean? caaaar caaadr caaar caadar
-        caaddr caadr caar cadaar cadadr cadar caddar cadddr caddr
-        cadr call-with-current-continuation call-with-input-file
-        call-with-output-file call-with-values call/cc car cdaaar cdaadr
-        cdaar cdadar cdaddr cdadr cdar cddaar cddadr cddar cdddar cddddr
-        cdddr cddr cdr ceiling char->integer char-alphabetic? char-ci<=?
-        char-ci<? char-ci=? char-ci>=? char-ci>? char-downcase
-        char-lower-case? char-numeric? char-ready? char-upcase
-        char-upper-case? char-whitespace? char<=? char<? char=? char>=?
-        char>? char? close-input-port close-output-port complex? cons
-        cos current-input-port current-output-port denominator
-        display dynamic-wind eof-object? eq?  equal? eqv? eval
-        even? exact->inexact exact? exp expt floor for-each force gcd
-        imag-part inexact->exact inexact? input-port? integer->char
-        integer? interaction-environment lcm length list list->string
-        list->vector list-ref list-tail list?  load log magnitude
-        make-polar make-rectangular make-string make-vector map
-        max member memq memv min modulo negative? newline not
-        null-environment null? number->string number? numerator odd?
-        open-input-file open-output-file output-port? pair?  peek-char
-        port? positive? procedure? quotient rational? rationalize
-        read read-char real-part real?  remainder reverse round
-        scheme-report-environment set-car! set-cdr! sin sqrt string
-        string->list string->number string->symbol string-append
-        string-ci<=?  string-ci<? string-ci=? string-ci>=? string-ci>?
-        string-copy string-fill! string-length string-ref
-        string-set! string<=? string<? string=? string>=?
-        string>? string? substring symbol->string symbol?
-        tan transcript-off transcript-on truncate values vector
-        vector->list vector-fill! vector-length vector-ref
-        vector-set! vector? with-input-from-file with-output-to-file
-        write write-char zero?
-      )
+      def self.builtins
+        @builtins ||= Set.new %w(
+          * + - / < <= = > >= abs acos angle append apply asin
+          assoc assq assv atan boolean? caaaar caaadr caaar caadar
+          caaddr caadr caar cadaar cadadr cadar caddar cadddr caddr
+          cadr call-with-current-continuation call-with-input-file
+          call-with-output-file call-with-values call/cc car cdaaar cdaadr
+          cdaar cdadar cdaddr cdadr cdar cddaar cddadr cddar cdddar cddddr
+          cdddr cddr cdr ceiling char->integer char-alphabetic? char-ci<=?
+          char-ci<? char-ci=? char-ci>=? char-ci>? char-downcase
+          char-lower-case? char-numeric? char-ready? char-upcase
+          char-upper-case? char-whitespace? char<=? char<? char=? char>=?
+          char>? char? close-input-port close-output-port complex? cons
+          cos current-input-port current-output-port denominator
+          display dynamic-wind eof-object? eq?  equal? eqv? eval
+          even? exact->inexact exact? exp expt floor for-each force gcd
+          imag-part inexact->exact inexact? input-port? integer->char
+          integer? interaction-environment lcm length list list->string
+          list->vector list-ref list-tail list?  load log magnitude
+          make-polar make-rectangular make-string make-vector map
+          max member memq memv min modulo negative? newline not
+          null-environment null? number->string number? numerator odd?
+          open-input-file open-output-file output-port? pair?  peek-char
+          port? positive? procedure? quotient rational? rationalize
+          read read-char real-part real?  remainder reverse round
+          scheme-report-environment set-car! set-cdr! sin sqrt string
+          string->list string->number string->symbol string-append
+          string-ci<=?  string-ci<? string-ci=? string-ci>=? string-ci>?
+          string-copy string-fill! string-length string-ref
+          string-set! string<=? string<? string=? string>=?
+          string>? string? substring symbol->string symbol?
+          tan transcript-off transcript-on truncate values vector
+          vector->list vector-fill! vector-length vector-ref
+          vector-set! vector? with-input-from-file with-output-to-file
+          write write-char zero?
+        )
+      end
 
       id = /[a-z0-9!$\%&*+,\/:<=>?@^_~|-]+/i
 
-      escape = Regexp.method(:escape)
-
       state :root do
         # comments
         rule /;.*$/, 'Comment.Single'
         rule /\s+/m, 'Text'
-        rule /-?\d+\.\d+/, 'Number.Float'
+        rule /-?\d+\.\d+/, 'Literal.Number.Float'
+        rule /-?\d+/, 'Literal.Number.Integer'
+
+        # Racket infinitites
+        rule /[+-]inf[.][f0]/, 'Literal.Number'
+
+        rule /#b[01]+/, 'Literal.Number.Binary'
+        rule /#o[0-7]+/, 'Literal.Number.Oct'
+        rule /#d[0-9]+/, 'Literal.Number.Integer'
+        rule /#x[0-9a-f]+/i, 'Literal.Number.Hex'
+        rule /#[ei][\d.]+/, 'Literal.Number.Other'
 
-        # support for uncommon kinds of numbers -
-        # have to figure out what the characters mean
-        # rule /(#e|#i|#b|#o|#d|#x)[\d.]+/, 'Number'
         rule /"(\\\\|\\"|[^"])*"/, 'Literal.String'
         rule /'#{id}/i, 'Literal.String.Symbol'
         rule /#\\([()\/'"._!\$%& ?=+-]{1}|[a-z0-9]+)/i,
           'Literal.String.Char'
         rule /#t|#f/, 'Name.Constant'
         rule /(?:'|#|`|,@|,|\.)/, 'Operator'
-        rule /(?:#{keywords.map(&escape).join('|')})(?=[^\w-])/,
-          'Keyword'
 
         rule /(['#])(\s*)(\()/m do
           group 'Literal.String.Symbol'
@@ -84,8 +91,18 @@ module Rouge
       end
 
       state :command do
-        rule /(?:#{builtins.map(&escape).join('|')})(?=[^\w-])/, 'Name.Builtin', :pop!
-        rule id, 'Name.Function', :pop!
+        rule id, 'Name.Function' do |m|
+          if self.class.keywords.include? m[0]
+            token 'Keyword'
+          elsif self.class.builtins.include? m[0]
+            token 'Name.Builtin'
+          else
+            token 'Name.Function'
+          end
+
+          pop!
+        end
+
         rule(//) { pop! }
       end
 

data/lib/rouge/version.rb

@@ -1,5 +1,5 @@
 module Rouge
   def self.version
-    "0.2.3"
+    "0.2.4"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rouge
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-16 00:00:00.000000000 Z
+date: 2012-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor