coderay-beta 0.9.1

data/lib/coderay/scanner.rb

@@ -0,0 +1,284 @@
+module CodeRay
+
+  require 'coderay/helpers/plugin'
+
+  # = Scanners
+  #
+  # This module holds the Scanner class and its subclasses.
+  # For example, the Ruby scanner is named CodeRay::Scanners::Ruby
+  # can be found in coderay/scanners/ruby.
+  #
+  # Scanner also provides methods and constants for the register
+  # mechanism and the [] method that returns the Scanner class
+  # belonging to the given lang.
+  #
+  # See PluginHost.
+  module Scanners
+    extend PluginHost
+    plugin_path File.dirname(__FILE__), 'scanners'
+
+    require 'strscan'
+
+    # = Scanner
+    #
+    # The base class for all Scanners.
+    #
+    # It is a subclass of Ruby's great +StringScanner+, which
+    # makes it easy to access the scanning methods inside.
+    #
+    # It is also +Enumerable+, so you can use it like an Array of
+    # Tokens:
+    #
+    #   require 'coderay'
+    #   
+    #   c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;"
+    #   
+    #   for text, kind in c_scanner
+    #     puts text if kind == :operator
+    #   end
+    #   
+    #   # prints: (*==)++;
+    #
+    # OK, this is a very simple example :)
+    # You can also use +map+, +any?+, +find+ and even +sort_by+,
+    # if you want.
+    class Scanner < StringScanner
+      
+      extend Plugin
+      plugin_host Scanners
+
+      # Raised if a Scanner fails while scanning
+      ScanError = Class.new(Exception)
+
+      require 'coderay/helpers/word_list'
+
+      # The default options for all scanner classes.
+      #
+      # Define @default_options for subclasses.
+      DEFAULT_OPTIONS = { :stream => false }
+      
+      KINDS_NOT_LOC = [:comment, :doctype]
+
+      class << self
+
+        # Returns if the Scanner can be used in streaming mode.
+        def streamable?
+          is_a? Streamable
+        end
+
+        def normify code
+          code = code.to_s
+          if code.respond_to? :force_encoding
+            begin
+              code.force_encoding 'utf-8'
+              code[/\z/]  # raises an ArgumentError when code contains a non-UTF-8 char
+            rescue ArgumentError
+              code.force_encoding 'binary'
+            end
+          end
+          code.to_unix
+        end
+        
+        def file_extension extension = nil
+          if extension
+            @file_extension = extension.to_s
+          else
+            @file_extension ||= plugin_id.to_s
+          end
+        end
+
+      end
+
+=begin
+## Excluded for speed reasons; protected seems to make methods slow.
+
+  # Save the StringScanner methods from being called.
+  # This would not be useful for highlighting.
+  strscan_public_methods =
+    StringScanner.instance_methods -
+    StringScanner.ancestors[1].instance_methods
+  protected(*strscan_public_methods)
+=end
+
+      # Create a new Scanner.
+      #
+      # * +code+ is the input String and is handled by the superclass
+      #   StringScanner.
+      # * +options+ is a Hash with Symbols as keys.
+      #   It is merged with the default options of the class (you can
+      #   overwrite default options here.)
+      # * +block+ is the callback for streamed highlighting.
+      #
+      # If you set :stream to +true+ in the options, the Scanner uses a
+      # TokenStream with the +block+ as callback to handle the tokens.
+      #
+      # Else, a Tokens object is used.
+      def initialize code='', options = {}, &block
+        raise "I am only the basic Scanner class. I can't scan "\
+          "anything. :( Use my subclasses." if self.class == Scanner
+        
+        @options = self.class::DEFAULT_OPTIONS.merge options
+
+        super Scanner.normify(code)
+
+        @tokens = options[:tokens]
+        if @options[:stream]
+          warn "warning in CodeRay::Scanner.new: :stream is set, "\
+            "but no block was given" unless block_given?
+          raise NotStreamableError, self unless kind_of? Streamable
+          @tokens ||= TokenStream.new(&block)
+        else
+          warn "warning in CodeRay::Scanner.new: Block given, "\
+            "but :stream is #{@options[:stream]}" if block_given?
+          @tokens ||= Tokens.new
+        end
+        @tokens.scanner = self
+
+        setup
+      end
+
+      def reset
+        super
+        reset_instance
+      end
+
+      def string= code
+        code = Scanner.normify(code)
+        super code
+        reset_instance
+      end
+
+      # More mnemonic accessor name for the input string.
+      alias code string
+      alias code= string=
+
+      # Scans the code and returns all tokens in a Tokens object.
+      def tokenize new_string=nil, options = {}
+        options = @options.merge(options)
+        self.string = new_string if new_string
+        @cached_tokens =
+          if @options[:stream]  # :stream must have been set already
+            reset unless new_string
+            scan_tokens @tokens, options
+            @tokens
+          else
+            scan_tokens @tokens, options
+          end
+      end
+
+      def tokens
+        @cached_tokens ||= tokenize
+      end
+      
+      # Whether the scanner is in streaming mode.
+      def streaming?
+        !!@options[:stream]
+      end
+
+      # Traverses the tokens.
+      def each &block
+        raise ArgumentError,
+          'Cannot traverse TokenStream.' if @options[:stream]
+        tokens.each(&block)
+      end
+      include Enumerable
+
+      # The current line position of the scanner.
+      #
+      # Beware, this is implemented inefficiently. It should be used
+      # for debugging only.
+      def line
+        string[0..pos].count("\n") + 1
+      end
+      
+      def column pos = self.pos
+        return 0 if pos <= 0
+        string = string()
+        if string.respond_to?(:bytesize) && (defined?(@bin_string) || string.bytesize != string.size)
+          @bin_string ||= string.dup.force_encoding(:binary)
+          string = @bin_string
+        end
+        pos - (string.rindex(?\n, pos) || 0)
+      end
+      
+      def marshal_dump
+        @options
+      end
+      
+      def marshal_load options
+        @options = options
+      end
+
+    protected
+
+      # Can be implemented by subclasses to do some initialization
+      # that has to be done once per instance.
+      #
+      # Use reset for initialization that has to be done once per
+      # scan.
+      def setup
+      end
+
+      # This is the central method, and commonly the only one a
+      # subclass implements.
+      #
+      # Subclasses must implement this method; it must return +tokens+
+      # and must only use Tokens#<< for storing scanned tokens!
+      def scan_tokens tokens, options
+        raise NotImplementedError,
+          "#{self.class}#scan_tokens not implemented."
+      end
+
+      def reset_instance
+        @tokens.clear unless @options[:keep_tokens]
+        @cached_tokens = nil
+        @bin_string = nil if defined? @bin_string
+      end
+
+      # Scanner error with additional status information
+      def raise_inspect msg, tokens, state = 'No state given!', ambit = 30
+        raise ScanError, <<-EOE % [
+
+
+***ERROR in %s: %s (after %d tokens)
+
+tokens:
+%s
+
+current line: %d  column: %d  pos: %d
+matched: %p  state: %p
+bol? = %p,  eos? = %p
+
+surrounding code:
+%p  ~~  %p
+
+
+***ERROR***
+
+        EOE
+          File.basename(caller[0]),
+          msg,
+          tokens.size,
+          tokens.last(10).map { |t| t.inspect }.join("\n"),
+          line, column, pos,
+          matched, state, bol?, eos?,
+          string[pos - ambit, ambit],
+          string[pos, ambit],
+        ]
+      end
+
+    end
+
+  end
+end
+
+class String
+  # I love this hack. It seems to silence all dos/unix/mac newline problems.
+  def to_unix
+    if index ?\r
+      gsub(/\r\n?/, "\n")
+    else
+      self
+    end
+  end
+end

data/lib/coderay/scanners/_map.rb

@@ -0,0 +1,23 @@
+module CodeRay
+module Scanners
+
+  map \
+    :h => :c,
+    :cplusplus => :cpp,
+    :'c++' => :cpp,
+    :ecma => :java_script,
+    :ecmascript => :java_script,
+    :ecma_script => :java_script,
+    :irb => :ruby,
+    :javascript => :java_script,
+    :js => :java_script,
+    :nitro => :nitro_xhtml,
+    :pascal => :delphi,
+    :plain => :plaintext,
+    :xhtml => :html,
+    :yml => :yaml
+
+  default :plain
+
+end
+end

data/lib/coderay/scanners/c.rb

@@ -0,0 +1,203 @@
+module CodeRay
+module Scanners
+
+  class C < Scanner
+
+    include Streamable
+    
+    register_for :c
+    file_extension 'c'
+
+    RESERVED_WORDS = [
+      'asm', 'break', 'case', 'continue', 'default', 'do',
+      'else', 'enum', 'for', 'goto', 'if', 'return',
+      'sizeof', 'struct', 'switch', 'typedef', 'union', 'while',
+      'restrict',  # added in C99
+    ]
+
+    PREDEFINED_TYPES = [
+      'int', 'long', 'short', 'char',
+      'signed', 'unsigned', 'float', 'double',
+      'bool', 'complex',  # added in C99
+    ]
+
+    PREDEFINED_CONSTANTS = [
+      'EOF', 'NULL',
+      'true', 'false',  # added in C99
+    ]
+    DIRECTIVES = [
+      'auto', 'extern', 'register', 'static', 'void',
+      'const', 'volatile',  # added in C89
+      'inline',  # added in C99
+    ]
+
+    IDENT_KIND = WordList.new(:ident).
+      add(RESERVED_WORDS, :reserved).
+      add(PREDEFINED_TYPES, :pre_type).
+      add(DIRECTIVES, :directive).
+      add(PREDEFINED_CONSTANTS, :pre_constant)
+
+    ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+
+    def scan_tokens tokens, options
+
+      state = :initial
+      label_expected = true
+      case_expected = false
+      label_expected_before_preproc_line = nil
+      in_preproc_line = false
+
+      until eos?
+
+        kind = nil
+        match = nil
+        
+        case state
+
+        when :initial
+
+          if match = scan(/ \s+ | \\\n /x)
+            if in_preproc_line && match != "\\\n" && match.index(?\n)
+              in_preproc_line = false
+              label_expected = label_expected_before_preproc_line
+            end
+            tokens << [match, :space]
+            next
+
+          elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+            kind = :comment
+
+          elsif match = scan(/ \# \s* if \s* 0 /x)
+            match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
+            kind = :comment
+
+          elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
+            label_expected = match =~ /[;\{\}]/
+            if case_expected
+              label_expected = true if match == ':'
+              case_expected = false
+            end
+            kind = :operator
+
+          elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+            kind = IDENT_KIND[match]
+            if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/)
+              kind = :label
+              match << matched
+            else
+              label_expected = false
+              if kind == :reserved
+                case match
+                when 'case', 'default'
+                  case_expected = true
+                end
+              end
+            end
+
+          elsif scan(/\$/)
+            kind = :ident
+          
+          elsif match = scan(/L?"/)
+            tokens << [:open, :string]
+            if match[0] == ?L
+              tokens << ['L', :modifier]
+              match = '"'
+            end
+            state = :string
+            kind = :delimiter
+
+          elsif scan(/#[ \t]*(\w*)/)
+            kind = :preprocessor
+            in_preproc_line = true
+            label_expected_before_preproc_line = label_expected
+            state = :include_expected if self[1] == 'include'
+
+          elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
+            label_expected = false
+            kind = :char
+
+          elsif scan(/0[xX][0-9A-Fa-f]+/)
+            label_expected = false
+            kind = :hex
+
+          elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
+            label_expected = false
+            kind = :oct
+
+          elsif scan(/(?:\d+)(?![.eEfF])L?L?/)
+            label_expected = false
+            kind = :integer
+
+          elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
+            label_expected = false
+            kind = :float
+
+          else
+            getch
+            kind = :error
+
+          end
+
+        when :string
+          if scan(/[^\\\n"]+/)
+            kind = :content
+          elsif scan(/"/)
+            tokens << ['"', :delimiter]
+            tokens << [:close, :string]
+            state = :initial
+            label_expected = false
+            next
+          elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            kind = :char
+          elsif scan(/ \\ | $ /x)
+            tokens << [:close, :string]
+            kind = :error
+            state = :initial
+            label_expected = false
+          else
+            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+          end
+
+        when :include_expected
+          if scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
+            kind = :include
+            state = :initial
+
+          elsif match = scan(/\s+/)
+            kind = :space
+            state = :initial if match.index ?\n
+
+          else
+            state = :initial
+            next
+
+          end
+
+        else
+          raise_inspect 'Unknown state', tokens
+
+        end
+
+        match ||= matched
+        if $DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens
+        end
+        raise_inspect 'Empty token', tokens unless match
+
+        tokens << [match, kind]
+
+      end
+
+      if state == :string
+        tokens << [:close, :string]
+      end
+
+      tokens
+    end
+
+  end
+
+end
+end