rouge 0.1.2 → 0.2.0

data/lib/rouge/lexers/common_lisp.rb

@@ -207,31 +207,26 @@ module Rouge
         structure-object symbol synonym-stream t two-way-stream vector
       )
 
-      def stream_tokens(*a, &b)
-        super(*a) do |tok, val|
-          case tok
-          when Token['Name.Variable']
-            if BUILTIN_FUNCTIONS.include? val
-              yield Token['Name.Builtin'], val
-            elsif SPECIAL_FORMS.include? val
-              yield Token['Keyword'], val
-            elsif MACROS.include? val
-              yield Token['Name.Builtin'], val
-            elsif LAMBDA_LIST_KEYWORDS.include? val
-              yield Token['Keyword'], val
-            elsif DECLARATIONS.include? val
-              yield Token['Keyword'], val
-            elsif BUILTIN_TYPES.include? val
-              yield Token['Keyword.Type'], val
-            elsif BUILTIN_CLASSES.include? val
-              yield Token['Name.Class'], val
-            else
-              yield tok, val
-            end
-          else
-            yield tok, val
-          end
+      postprocess 'Name.Variable' do |tok, val|
+        tok = if BUILTIN_FUNCTIONS.include? val
+          'Name.Builtin'
+        elsif SPECIAL_FORMS.include? val
+          'Keyword'
+        elsif MACROS.include? val
+          'Name.Builtin'
+        elsif LAMBDA_LIST_KEYWORDS.include? val
+          'Keyword'
+        elsif DECLARATIONS.include? val
+          'Keyword'
+        elsif BUILTIN_TYPES.include? val
+          'Keyword.Type'
+        elsif BUILTIN_CLASSES.include? val
+          'Name.Class'
+        else
+          'Name.Variable'
         end
+
+        token tok, val
       end
 
       nonmacro = /\\.|[a-zA-Z0-9!$%&*+-\/<=>?@\[\]^_{}~]/

data/lib/rouge/lexers/erb.rb

@@ -1,6 +1,6 @@
 module Rouge
   module Lexers
-    class ERB < RegexLexer
+    class ERB < TemplateLexer
       tag 'erb'
       aliases 'eruby', 'rhtml'
 
@@ -11,19 +11,13 @@ module Rouge
       end
 
       def initialize(opts={})
-        @parent = opts.delete(:parent) || 'html'
-        if @parent.is_a? String
-          lexer_class = Lexer.find(@parent)
-          @parent = lexer_class.new(opts)
-        end
-
         @ruby_lexer = Ruby.new(opts)
 
         super(opts)
       end
 
       start do
-        @parent.reset!
+        parent.reset!
         @ruby_lexer.reset!
       end
 
@@ -36,7 +30,7 @@ module Rouge
         rule open, 'Comment.Preproc', :ruby
 
         rule /.+?(?=#{open})|.+/m do
-          delegate @parent
+          delegate parent
         end
       end
 

data/lib/rouge/lexers/factor.rb

@@ -0,0 +1,300 @@
+module Rouge
+  module Lexers
+    class Factor < RegexLexer
+      tag 'factor'
+      filenames '*.factor'
+      mimetypes 'text/x-factor'
+
+      def self.analyze_text(text)
+        return 1 if text.shebang? 'factor'
+      end
+
+      def self.builtins
+        @builtins ||= {}.tap do |builtins|
+          builtins[:kernel] = Set.new %w(
+            or 2bi 2tri while wrapper nip 4dip wrapper? bi*
+            callstack>array both? hashcode die dupd callstack
+            callstack? 3dup tri@ pick curry build ?execute 3bi prepose
+            >boolean if clone eq? tri* ? = swapd 2over 2keep 3keep clear
+            2dup when not tuple? dup 2bi* 2tri* call tri-curry object bi@
+            do unless* if* loop bi-curry* drop when* assert= retainstack
+            assert? -rot execute 2bi@ 2tri@ boa with either? 3drop bi
+            curry?  datastack until 3dip over 3curry tri-curry* tri-curry@
+            swap and 2nip throw bi-curry (clone) hashcode* compose 2dip if
+            3tri unless compose? tuple keep 2curry equal? assert tri 2drop
+            most <wrapper> boolean? identity-hashcode identity-tuple?
+            null new dip bi-curry@ rot xor identity-tuple boolean
+          )
+
+          builtins[:assocs] = Set.new %w(
+            ?at assoc? assoc-clone-like assoc= delete-at* assoc-partition
+            extract-keys new-assoc value? assoc-size map>assoc push-at
+            assoc-like key? assoc-intersect assoc-refine update
+            assoc-union assoc-combine at* assoc-empty? at+ set-at
+            assoc-all? assoc-subset?  assoc-hashcode change-at assoc-each
+            assoc-diff zip values value-at rename-at inc-at enum? at cache
+            assoc>map <enum> assoc assoc-map enum value-at* assoc-map-as
+            >alist assoc-filter-as clear-assoc assoc-stack maybe-set-at
+            substitute assoc-filter 2cache delete-at assoc-find keys
+            assoc-any? unzip
+          )
+
+          builtins[:combinators] = Set.new %w(
+            case execute-effect no-cond no-case? 3cleave>quot 2cleave
+            cond>quot wrong-values? no-cond? cleave>quot no-case case>quot
+            3cleave wrong-values to-fixed-point alist>quot case-find
+            cond cleave call-effect 2cleave>quot recursive-hashcode
+            linear-case-quot spread spread>quot
+          )
+
+          builtins[:math] = Set.new %w(
+            number= if-zero next-power-of-2 each-integer ?1+
+            fp-special? imaginary-part unless-zero float>bits number?
+            fp-infinity? bignum? fp-snan? denominator fp-bitwise= *
+            + power-of-2? - u>= / >= bitand log2-expects-positive <
+            log2 > integer? number bits>double 2/ zero? (find-integer)
+            bits>float float? shift ratio? even? ratio fp-sign bitnot
+            >fixnum complex? /i /f byte-array>bignum when-zero sgn >bignum
+            next-float u< u> mod recip rational find-last-integer >float
+            (all-integers?) 2^ times integer fixnum? neg fixnum sq bignum
+            (each-integer) bit? fp-qnan? find-integer complex <fp-nan>
+            real double>bits bitor rem fp-nan-payload all-integers?
+            real-part log2-expects-positive? prev-float align unordered?
+            float fp-nan? abs bitxor u<= odd? <= /mod rational? >integer
+            real? numerator
+          )
+
+          builtins[:sequences] = Set.new %w(
+            member-eq? append assert-sequence= find-last-from
+            trim-head-slice clone-like 3sequence assert-sequence? map-as
+            last-index-from reversed index-from cut* pad-tail
+            remove-eq! concat-as but-last snip trim-tail nths
+            nth 2selector sequence slice?  <slice> partition
+            remove-nth tail-slice empty? tail* if-empty
+            find-from virtual-sequence? member? set-length
+            drop-prefix unclip unclip-last-slice iota map-sum
+            bounds-error? sequence-hashcode-step selector-for
+            accumulate-as map start midpoint@ (accumulate) rest-slice
+            prepend fourth sift accumulate! new-sequence follow map! like
+            first4 1sequence reverse slice unless-empty padding virtual@
+            repetition? set-last index 4sequence max-length set-second
+            immutable-sequence first2 first3 replicate-as reduce-index
+            unclip-slice supremum suffix! insert-nth trim-tail-slice
+            tail 3append short count suffix concat flip filter sum
+            immutable? reverse! 2sequence map-integers delete-all start*
+            indices snip-slice check-slice sequence?  head map-find
+            filter! append-as reduce sequence= halves collapse-slice
+            interleave 2map filter-as binary-reduce slice-error? product
+            bounds-check? bounds-check harvest immutable virtual-exemplar
+            find produce remove pad-head last replicate set-fourth
+            remove-eq shorten reversed?  map-find-last 3map-as
+            2unclip-slice shorter? 3map find-last head-slice pop* 2map-as
+            tail-slice* but-last-slice 2map-reduce iota? collector-for
+            accumulate each selector append! new-resizable cut-slice
+            each-index head-slice* 2reverse-each sequence-hashcode
+            pop set-nth ?nth <flat-slice> second join when-empty
+            collector immutable-sequence? <reversed> all? 3append-as
+            virtual-sequence subseq? remove-nth! push-either new-like
+            length last-index push-if 2all? lengthen assert-sequence
+            copy map-reduce move third first 3each tail? set-first prefix
+            bounds-error any? <repetition> trim-slice exchange surround
+            2reduce cut change-nth min-length set-third produce-as
+            push-all head? delete-slice rest sum-lengths 2each head*
+            infimum remove! glue slice-error subseq trim replace-slice
+            push repetition map-index trim-head unclip-last mismatch
+          )
+
+          builtins[:namespaces] = Set.new %w(
+            global +@ change set-namestack change-global init-namespaces
+            on off set-global namespace set with-scope bind with-variable
+            inc dec counter initialize namestack get get-global make-assoc
+          )
+
+          builtins[:arrays] = Set.new %w(
+            <array> 2array 3array pair >array 1array 4array pair?
+            array resize-array array?
+          )
+
+          builtins[:io] = Set.new %w(
+            +character+ bad-seek-type? readln each-morsel
+            stream-seek read print with-output-stream contents
+            write1 stream-write1 stream-copy stream-element-type
+            with-input-stream stream-print stream-read stream-contents
+            stream-tell tell-output bl seek-output bad-seek-type nl
+            stream-nl write flush stream-lines +byte+ stream-flush
+            read1 seek-absolute? stream-read1 lines stream-readln
+            stream-read-until each-line seek-end with-output-stream*
+            seek-absolute with-streams seek-input seek-relative?
+            input-stream stream-write read-partial seek-end?
+            seek-relative error-stream read-until with-input-stream*
+            with-streams* tell-input each-block output-stream
+            stream-read-partial each-stream-block each-stream-line
+          )
+
+          builtins[:strings] = Set.new %w(
+            resize-string >string <string> 1string string string?
+          )
+
+          builtins[:vectors] = Set.new %w(
+            with-return restarts return-continuation with-datastack
+            recover rethrow-restarts <restart> ifcc set-catchstack
+            >continuation< cleanup ignore-errors restart?
+            compute-restarts attempt-all-error error-thread
+            continue <continuation> attempt-all-error? condition?
+            <condition> throw-restarts error catchstack continue-with
+            thread-error-hook continuation rethrow callcc1
+            error-continuation callcc0 attempt-all condition
+            continuation? restart return
+          )
+
+          builtins[:continuations] = Set.new %w(
+            with-return restarts return-continuation with-datastack
+            recover rethrow-restarts <restart> ifcc set-catchstack
+            >continuation< cleanup ignore-errors restart?
+            compute-restarts attempt-all-error error-thread
+            continue <continuation> attempt-all-error? condition?
+            <condition> throw-restarts error catchstack continue-with
+            thread-error-hook continuation rethrow callcc1
+            error-continuation callcc0 attempt-all condition
+            continuation? restart return
+          )
+        end
+      end
+
+      state :root do
+        rule /\s+/m, 'Text'
+
+        rule /(:|::|MACRO:|MEMO:|GENERIC:|HELP:)(\s+)(\S+)/m do
+          group 'Keyword'; group 'Text'
+          group 'Name.Function'
+        end
+
+        rule /(M:|HOOK:|GENERIC#)(\s+)(\S+)(\s+)(\S+)/m do
+          group 'Keyword'; group 'Text'
+          group 'Name.Class'; group 'Text'
+          group 'Name.Function'
+        end
+
+        rule /\((?=\s)/, 'Name.Function', :stack_effect
+        rule /;(?=\s)/, 'Keyword'
+
+        rule /(USING:)((?:\s|\\\s)+)/m do
+          group 'Keyword.Namespace'; group 'Text'
+          push :import
+        end
+
+        rule /(IN:|USE:|UNUSE:|QUALIFIED:|QUALIFIED-WITH:)(\s+)(\S+)/m do
+          group 'Keyword.Namespace'; group 'Text'; group 'Name.Namespace'
+        end
+
+        rule /(FROM:|EXCLUDE:)(\s+)(\S+)(\s+)(=>)/m do
+          group 'Keyword.Namespace'; group 'Text'
+          group 'Name.Namespace'; group 'Text'
+          group 'Punctuation'
+        end
+
+        rule /(?:ALIAS|DEFER|FORGET|POSTPONE):/, 'Keyword.Namespace'
+
+        rule /(TUPLE:)(\s+)(\S+)(\s+)(<)(\s+)(\S+)/m do
+          group 'Keyword'; group 'Text'
+          group 'Name.Class'; group 'Text'
+          group 'Punctuation'; group 'Text'
+          group 'Name.Class'
+          push :slots
+        end
+
+        rule /(TUPLE:)(\s+)(\S+)/m do
+          group 'Keyword'; group 'Text'; group 'Name.Class'
+          push :slots
+        end
+
+        rule /(UNION:|INTERSECTION:)(\s+)(\S+)/m do
+          group 'Keyword'; group 'Text'; group 'Name.Class'
+        end
+
+        rule /(PREDICATE:)(\s+)(\S+)(\s+)(<)(\s+)(\S+)/m do
+          group 'Keyword'; group 'Text'
+          group 'Name.Class'; group 'Text'
+          group 'Punctuation'; group 'Text'
+          group 'Name.Class'
+        end
+
+        rule /(C:)(\s+)(\S+)(\s+)(\S+)/m do
+          group 'Keyword'; group 'Text'
+          group 'Name.Function'; group 'Text'
+          group 'Name.Class'
+        end
+
+        rule %r(
+          (INSTANCE|SLOT|MIXIN|SINGLETONS?|CONSTANT|SYMBOLS?|ERROR|SYNTAX
+           |ALIEN|TYPEDEF|FUNCTION|STRUCT):
+        )x, 'Keyword'
+
+        rule /(?:<PRIVATE|PRIVATE>)/, 'Keyword.Namespace'
+
+        rule /(MAIN:)(\s+)(\S+)/ do
+          group 'Keyword.Namespace'; group 'Text'; group 'Name.Function'
+        end
+
+        # strings
+        rule /"""\s+.*?\s+"""/, 'Literal.String'
+        rule /"(\\.|[^\\])*?"/, 'Literal.String'
+        rule /(CHAR:)(\s+)(\\[\\abfnrstv]*|\S)(?=\s)/, 'Literal.String.Char'
+
+        # comments
+        rule /!\s+.*$/, 'Comment'
+        rule /#!\s+.*$/, 'Comment'
+
+        # booleans
+        rule /[tf](?=\s)/, 'Name.Constant'
+
+        # numbers
+        rule /-?\d+\.\d+(?=\s)/, 'Literal.Number.Float'
+        rule /-?\d+(?=\s)/, 'Literal.Number.Integer'
+
+        rule /HEX:\s+[a-fA-F\d]+(?=\s)/m, 'Literal.Number.Hex'
+        rule /BIN:\s+[01]+(?=\s)/, 'Literal.Number.Bin'
+        rule /OCT:\s+[0-7]+(?=\s)/, 'Literal.Number.Oct'
+
+        rule %r([-+/*=<>^](?=\s)), 'Operator'
+
+        rule /(?:deprecated|final|foldable|flushable|inline|recursive)(?=\s)/,
+          'Keyword'
+
+        # words, to be postprocessed for builtins and things
+        rule /\S+/, 'Postprocess.Word'
+      end
+
+      state :stack_effect do
+        rule /\s+/, 'Text'
+        rule /\(/, 'Name.Function', :stack_effect
+        rule /\)/, 'Name.Function', :pop!
+
+        rule /--/, 'Name.Function'
+        rule /\S+/, 'Name.Variable'
+      end
+
+      state :slots do
+        rule /\s+/, 'Text'
+        rule /;(?=\s)/, 'Keyword', :pop!
+        rule /\S+/, 'Name.Variable'
+      end
+
+      state :import do
+        rule /;(?=\s)/, 'Keyword', :pop!
+        rule /\s+/, 'Text'
+        rule /\S+/, 'Name.Namespace'
+      end
+
+      postprocess 'Postprocess.Word' do |tok, val|
+        tok = if self.class.builtins.values.any? { |b| b.include? val }
+          'Name.Builtin'
+        else
+          'Name'
+        end
+
+        token tok, val
+      end
+    end
+  end
+end

data/lib/rouge/lexers/haml.rb

@@ -0,0 +1,238 @@
+module Rouge
+  module Lexers
+    class Haml < RegexLexer
+      tag 'haml'
+      aliases 'HAML'
+
+      filenames '*.haml'
+      mimetypes 'text/x-haml'
+
+      def self.analyze_text(text)
+        return 0.1 if text.start_with? '!!!'
+      end
+
+      # option :filters is a hash of filter name to lexer of how
+      # various filters should be highlighted.  By default, :javascript
+      # and :stylesheet are supported.
+      def initialize(opts={})
+        (opts.delete(:filters) || {}).each do |name, lexer|
+          unless lexer.respond_to? :lex
+            lexer = Lexer.find(lexer) or raise "unknown lexer: #{lexer}"
+            lexer = lexer.new(options)
+          end
+
+          self.filters[name.to_s] = lexer
+        end
+
+        super(opts)
+      end
+
+      def ruby
+        @ruby ||= Ruby.new(options)
+      end
+
+      def html
+        @html ||= HTML.new(options)
+      end
+
+      def filters
+        @filters ||= {
+          'javascript' => Javascript.new(options),
+          'css' => CSS.new(options),
+          'ruby' => ruby,
+          'erb' => ERB.new(options),
+          # TODO
+          # 'sass' => Sass.new(options),
+          # 'textile' => Textile.new(options),
+          # 'markdown' => Markdown.new(options),
+          # 'maruku' => Maruku.new(options),
+        }
+      end
+
+      start { ruby.reset!; html.reset! }
+
+      # push a state for the next indented block
+      def starts_block(block_state)
+        @block_state = block_state
+        @block_indentation = @last_indentation || ''
+        debug { "    haml: starts_block #{block_state.inspect}" }
+        debug { "    haml: block_indentation: #{@block_indentation.inspect}" }
+      end
+
+      def indentation(indent_str)
+        debug { "    haml: indentation #{indent_str.inspect}" }
+        debug { "    haml: block_indentation: #{@block_indentation.inspect}" }
+        @last_indentation = indent_str
+
+        # if it's an indent and we know where to go next,
+        # push that state.  otherwise, push content and
+        # clear the block state.
+        if (@block_state &&
+            indent_str.start_with?(@block_indentation) &&
+            indent_str != @block_indentation
+        )
+          push @block_state
+        else
+          @block_state = @block_indentation = nil
+          push :content
+        end
+      end
+
+      identifier = /[\w:-]+/
+      ruby_var = /[a-z]\w*/
+
+      # Haml can include " |\n" anywhere,
+      # which is ignored and used to wrap long lines.
+      # To accomodate this, use this custom faux dot instead.
+      dot = /[ ]\|\n(?=.*[ ]\|)|./
+
+      # In certain places, a comma at the end of the line
+      # allows line wrapping as well.
+      comma_dot = /,\s*\n|#{dot}/
+
+      state :root do
+        rule /\s*\n/, 'Text'
+        rule(/\s*/) { |m| token 'Text'; indentation(m[0]) }
+      end
+
+      state :content do
+        mixin :css
+        rule(/%#{identifier}/) { token 'Name.Tag'; pop!; push :tag }
+        rule /!!!#{dot}*\n/, 'Name.Namespace', :pop!
+        rule %r(
+          (/) (\[#{dot}*?\]) (#{dot}*\n)
+        )x do
+          group 'Comment'; group 'Comment.Special'; group 'Comment'
+          pop!
+        end
+
+        rule %r(/#{dot}*\n) do
+          token 'Comment'
+          pop!
+          starts_block :html_comment_block
+        end
+
+        rule /-##{dot}*\n/ do
+          token 'Comment'
+          pop!
+          starts_block :haml_comment_block
+        end
+
+        rule /-/ do
+          token 'Punctuation'
+          reset_stack
+          push :ruby_line
+        end
+
+        # filters
+        rule /:(#{dot}*)\n/ do |m|
+          token 'Name.Decorator'
+          pop!
+          starts_block :filter_block
+
+          filter_name = m[1].strip
+
+          @filter_lexer = self.filters[filter_name]
+          @filter_lexer.reset! unless @filter_lexer.nil?
+
+          debug { "    haml: filter #{filter_name.inspect} #{@filter_lexer.inspect}" }
+        end
+
+        mixin :eval_or_plain
+      end
+
+      state :css do
+        rule(/\.#{identifier}/) { token 'Name.Class'; pop!; push :tag }
+        rule(/##{identifier}/) { token 'Name.Function'; pop!; push :tag }
+      end
+
+      state :tag do
+        mixin :css
+        rule(/\{#{comma_dot}*?\}/) { delegate ruby }
+        rule(/\[#{dot}*?\]/) { delegate ruby }
+        rule /\(/, 'Punctuation', :html_attributes
+        rule /\s*\n/, 'Text', :pop!
+
+        # whitespace chompers
+        rule /[<>]{1,2}(?=[ \t=])/, 'Punctuation'
+
+        mixin :eval_or_plain
+      end
+
+      state :plain do
+        rule(/([^#\n]|#[^{\n]|(\\\\)*\\#\{)+/) { delegate html }
+        mixin :interpolation
+        rule(/\n/) { token 'Text'; reset_stack }
+      end
+
+      state :eval_or_plain do
+        rule /[&!]?==/, 'Punctuation', :plain
+        rule /[&!]?[=!]/ do
+          token 'Punctuation'
+          reset_stack
+          push :ruby_line
+        end
+
+        rule(//) { push :plain }
+      end
+
+      state :ruby_line do
+        rule /\n/, 'Text', :pop!
+        rule(/,[ \t]*\n/) { delegate ruby }
+        rule /[ ]\|[ \t]*\n/, 'Literal.String.Escape'
+        rule(/.*?(?=(,$| \|)?[ \t]*$)/) { delegate ruby }
+      end
+
+      state :html_attributes do
+        rule /\s+/, 'Text'
+        rule /#{identifier}\s*=/, 'Name.Attribute', :html_attribute_value
+        rule identifier, 'Name.Attribute'
+        rule /\)/, 'Text', :pop!
+      end
+
+      state :html_attribute_value do
+        rule /\s+/, 'Text'
+        rule ruby_var, 'Name.Variable', :pop!
+        rule /@#{ruby_var}/, 'Name.Variable.Instance', :pop!
+        rule /\$#{ruby_var}/, 'Name.Variable.Global', :pop!
+        rule /'(\\\\|\\'|[^'\n])*'/, 'Literal.String', :pop!
+        rule /"(\\\\|\\"|[^"\n])*"/, 'Literal.String', :pop!
+      end
+
+      state :html_comment_block do
+        rule /#{dot}+/, 'Comment'
+        mixin :indented_block
+      end
+
+      state :haml_comment_block do
+        rule /#{dot}+/, 'Comment.Preproc'
+        mixin :indented_block
+      end
+
+      state :filter_block do
+        rule /([^#\n]|#[^{\n]|(\\\\)*\\#\{)+/ do
+          if @filter_lexer
+            delegate @filter_lexer
+          else
+            token 'Name.Decorator'
+          end
+        end
+
+        mixin :interpolation
+        mixin :indented_block
+      end
+
+      state :interpolation do
+        rule /(#\{)(#{dot}*?)(\})/ do |m|
+          token 'Literal.String.Interpol', m[1]
+          delegate ruby, m[2]
+          token 'Literal.String.Interpol', m[3]
+        end
+      end
+
+      state :indented_block do
+        rule(/\n/) { token 'Text'; reset_stack }
+      end
+    end
+  end
+end