purple_shoes 0.0.126 → 0.5.149

data/VERSION

@@ -1 +1 @@
-0.0.126
+0.5.149

data/lib/ext/highlighter/common.rb

@@ -0,0 +1,197 @@
+require 'strscan'
+
+module HH::Syntax
+
+  # A single token extracted by a tokenizer. It is simply the lexeme
+  # itself, decorated with a 'group' attribute to identify the type of the
+  # lexeme.
+  class Token < String
+
+    # the type of the lexeme that was extracted.
+    attr_reader :group
+
+    # the instruction associated with this token (:none, :region_open, or
+    # :region_close)
+    attr_reader :instruction
+
+    # Create a new Token representing the given text, and belonging to the
+    # given group.
+    def initialize( text, group, instruction = :none )
+      super text
+      @group = group
+      @instruction = instruction
+    end
+
+  end
+
+  # The base class of all tokenizers. It sets up the scanner and manages the
+  # looping until all tokens have been extracted. It also provides convenience
+  # methods to make sure adjacent tokens of identical groups are returned as
+  # a single token.
+  class Tokenizer
+
+    # The current group being processed by the tokenizer
+    attr_reader :group
+
+    # The current chunk of text being accumulated
+    attr_reader :chunk
+
+    # Start tokenizing. This sets up the state in preparation for tokenization,
+    # such as creating a new scanner for the text and saving the callback block.
+    # The block will be invoked for each token extracted.
+    def start( text, &block )
+      @chunk = ""
+      @group = :normal
+      @callback = block
+      @text = StringScanner.new( text )
+      setup
+    end
+
+    # Subclasses may override this method to provide implementation-specific
+    # setup logic.
+    def setup
+    end
+
+    # Finish tokenizing. This flushes the buffer, yielding any remaining text
+    # to the client.
+    def finish
+      start_group nil
+      teardown
+    end
+
+    # Subclasses may override this method to provide implementation-specific
+    # teardown logic.
+    def teardown
+    end
+
+    # Subclasses must implement this method, which is called for each iteration
+    # of the tokenization process. This method may extract multiple tokens.
+    def step
+      raise NotImplementedError, "subclasses must implement #step"
+    end
+
+    # Begins tokenizing the given text, calling #step until the text has been
+    # exhausted.
+    def tokenize( text, &block )
+      start text, &block
+      step until @text.eos?
+      finish
+    end
+
+    # Specify a set of tokenizer-specific options. Each tokenizer may (or may
+    # not) publish any options, but if a tokenizer does those options may be
+    # used to specify optional behavior.
+    def set( opts={} )
+      ( @options ||= Hash.new ).update opts
+    end
+
+    # Get the value of the specified option.
+    def option(opt)
+      @options ? @options[opt] : nil
+    end
+
+  private
+
+    EOL = /(?=\r\n?|\n|$)/
+
+    # A convenience for delegating method calls to the scanner.
+    def self.delegate( sym )
+      define_method( sym ) { |*a| @text.__send__( sym, *a ) }
+    end
+
+    delegate :bol?
+    delegate :eos?
+    delegate :scan
+    delegate :scan_until
+    delegate :check
+    delegate :check_until
+    delegate :getch
+    delegate :matched
+    delegate :pre_match
+    delegate :peek
+    delegate :pos
+
+    # Access the n-th subgroup from the most recent match.
+    def subgroup(n)
+      @text[n]
+    end
+
+    # Append the given data to the currently active chunk.
+    def append( data )
+      @chunk << data
+    end
+
+    # Request that a new group be started. If the current group is the same
+    # as the group being requested, a new group will not be created. If a new
+    # group is created and the current chunk is not empty, the chunk's
+    # contents will be yielded to the client as a token, and then cleared.
+    #
+    # After the new group is started, if +data+ is non-nil it will be appended
+    # to the chunk.
+    def start_group( gr, data=nil )
+      flush_chunk if gr != @group
+      @group = gr
+      @chunk << data if data
+    end
+
+    def start_region( gr, data=nil )
+      flush_chunk
+      @group = gr
+      @callback.call( Token.new( data||"", @group, :region_open ) )
+    end
+
+    def end_region( gr, data=nil )
+      flush_chunk
+      @group = gr
+      @callback.call( Token.new( data||"", @group, :region_close ) )
+    end
+
+    def flush_chunk
+      @callback.call( Token.new( @chunk, @group ) ) unless @chunk.empty?
+      @chunk = ""
+    end
+
+    def subtokenize( syntax, text )
+      tokenizer = Syntax.load( syntax )
+      tokenizer.set @options if @options
+      flush_chunk
+      tokenizer.tokenize( text, &@callback )
+    end
+  end
+
+
+  # A default tokenizer for handling syntaxes that are not explicitly handled
+  # elsewhere. It simply yields the given text as a single token.
+  class Default
+
+    # Yield the given text as a single token.
+    def tokenize( text )
+      yield Token.new( text, :normal )
+    end
+
+  end
+
+  # A hash for registering syntax implementations.
+  SYNTAX = Hash.new( Default )
+
+  # Load the implementation of the requested syntax. If the syntax cannot be
+  # found, or if it cannot be loaded for whatever reason, the Default syntax
+  # handler will be returned.
+  def load( syntax )
+    begin
+      require_relative "lang/#{syntax}"
+    rescue LoadError
+    end
+    SYNTAX[ syntax ].new
+  end
+  module_function :load
+
+  # Return an array of the names of supported syntaxes.
+  def all
+    lang_dir = File.join(File.dirname(__FILE__), "syntax", "lang")
+    Dir["#{lang_dir}/*.rb"].map { |path| File.basename(path, ".rb") }
+  end
+  module_function :all
+
+
+end

data/lib/ext/highlighter/lang/ruby.rb

@@ -0,0 +1,316 @@
+
+module HH::Syntax
+
+  # A tokenizer for the Ruby language. It recognizes all common syntax
+  # (and some less common syntax) but because it is not a true lexer, it
+  # will make mistakes on some ambiguous cases.
+  class Ruby < Tokenizer
+
+    # The list of all identifiers recognized as keywords.
+    KEYWORDS =
+      %w{if then elsif else end begin do rescue ensure while for
+         class module def yield raise until unless and or not when
+         case super undef break next redo retry in return alias
+         defined?}
+
+    # Perform ruby-specific setup
+    def setup
+      @selector = false
+      @allow_operator = false
+      @heredocs = []
+    end
+
+    # Step through a single iteration of the tokenization process.
+    def step
+      case
+        when bol? && check( /=begin/ )
+          start_group( :comment, scan_until( /^=end#{EOL}/ ) )
+        when bol? && check( /__END__#{EOL}/ )
+          start_group( :comment, scan_until( /\Z/ ) )
+      else
+        case
+          when check( /def\s+/ )
+            start_group :keyword, scan( /def\s+/ )
+            start_group :method,  scan_until( /(?=[;(\s]|#{EOL})/ )
+          when check( /class\s+/ )
+            start_group :keyword, scan( /class\s+/ )
+            start_group :class,  scan_until( /(?=[;\s<]|#{EOL})/ )
+          when check( /module\s+/ )
+            start_group :keyword, scan( /module\s+/ )
+            start_group :module,  scan_until( /(?=[;\s]|#{EOL})/ )
+          when check( /::/ )
+            start_group :punct, scan(/::/)
+          when check( /:"/ )
+            start_group :symbol, scan(/:/)
+            scan_delimited_region :symbol, :symbol, "", true
+            @allow_operator = true
+          when check( /:'/ )
+            start_group :symbol, scan(/:/)
+            scan_delimited_region :symbol, :symbol, "", false
+            @allow_operator = true
+          when scan( /:[_a-zA-Z@$][$@\w]*[=!?]?/ )
+            start_group :symbol, matched
+            @allow_operator = true
+          when scan( /\?(\\[^\n\r]|[^\\\n\r\s])/ )
+            start_group :char, matched
+            @allow_operator = true
+          when check( /(__FILE__|__LINE__|true|false|nil|self)[?!]?/ )
+            if @selector || matched[-1] == ?? || matched[-1] == ?!
+              start_group :ident,
+                scan(/(__FILE__|__LINE__|true|false|nil|self)[?!]?/)
+            else
+              start_group :constant,
+                scan(/(__FILE__|__LINE__|true|false|nil|self)/)
+            end
+            @selector = false
+            @allow_operator = true
+          when scan(/0([bB][01]+|[oO][0-7]+|[dD][0-9]+|[xX][0-9a-fA-F]+)/)
+            start_group :number, matched
+            @allow_operator = true
+          else
+            case peek(2)
+              when "%r"
+                scan_delimited_region :punct, :regex, scan( /../ ), true
+                @allow_operator = true
+              when "%w", "%q"
+                scan_delimited_region :punct, :string, scan( /../ ), false
+                @allow_operator = true
+              when "%s"
+                scan_delimited_region :punct, :symbol, scan( /../ ), false
+                @allow_operator = true
+              when "%W", "%Q", "%x"
+                scan_delimited_region :punct, :string, scan( /../ ), true
+                @allow_operator = true
+              when /%[^\sa-zA-Z0-9]/
+                scan_delimited_region :punct, :string, scan( /./ ), true
+                @allow_operator = true
+              when "<<"
+                saw_word = ( chunk[-1,1] =~ /[\w!?]/ )
+                start_group :punct, scan( /<</ )
+                if saw_word
+                  @allow_operator = false
+                  return
+                end
+
+                float_right = scan( /-/ )
+                append "-" if float_right
+                if ( type = scan( /['"]/ ) )
+                  append type
+                  delim = scan_until( /(?=#{type})/ )
+                  if delim.nil?
+                    append scan_until( /\Z/ )
+                    return
+                  end
+                else
+                  delim = scan( /\w+/ ) or return
+                end
+                start_group :constant, delim
+                start_group :punct, scan( /#{type}/ ) if type
+                @heredocs << [ float_right, type, delim ]
+                @allow_operator = true
+              else
+                case peek(1)
+                  when /[\n\r]/
+                    unless @heredocs.empty?
+                      scan_heredoc(*@heredocs.shift)
+                    else
+                      start_group :normal, scan( /\s+/ )
+                    end
+                    @allow_operator = false
+                  when /\s/
+                    start_group :normal, scan( /\s+/ )
+                  when "#"
+                    start_group :comment, scan( /#[^\n\r]*/ )
+                  when /[A-Z]/
+                    start_group @selector ? :ident : :constant, scan( /\w+/ )
+                    @allow_operator = true
+                  when /[a-z_]/
+                    word = scan( /\w+[?!]?/ )
+                    if !@selector && KEYWORDS.include?( word )
+                      start_group :keyword, word
+                      @allow_operator = false
+                    elsif
+                      start_group :ident, word
+                      @allow_operator = true
+                    end
+                    @selector = false
+                  when /\d/
+                    start_group :number,
+                      scan( /[\d_]+(\.[\d_]+)?([eE][\d_]+)?/ )
+                    @allow_operator = true
+                  when '"'
+                    scan_delimited_region :punct, :string, "", true
+                    @allow_operator = true
+                  when '/'
+                    if @allow_operator
+                      start_group :punct, scan(%r{/})
+                      @allow_operator = false
+                    else
+                      scan_delimited_region :punct, :regex, "", true
+                      @allow_operator = true
+                    end
+                  when "'"
+                    scan_delimited_region :punct, :string, "", false
+                    @allow_operator = true
+                  when "."
+                    dots = scan( /\.{1,3}/ )
+                    start_group :punct, dots
+                    @selector = ( dots.length == 1 )
+                  when /[@]/
+                    start_group :attribute, scan( /@{1,2}\w*/ )
+                    @allow_operator = true
+                  when /[$]/
+                    start_group :global, scan(/\$/)
+                    start_group :global, scan( /\w+|./ ) if check(/./)
+                    @allow_operator = true
+                  when /[-!?*\/+=<>(\[\{}:;,&|%]/
+                    start_group :punct, scan(/./)
+                    @allow_operator = false
+                  when /[)\]]/
+                    start_group :punct, scan(/./)
+                    @allow_operator = true
+                  else
+                    # all else just falls through this, to prevent
+                    # infinite loops...
+                    append getch
+                end
+            end
+        end
+      end
+    end
+
+    private
+
+      # Scan a delimited region of text. This handles the simple cases (strings
+      # delimited with quotes) as well as the more complex cases of %-strings
+      # and here-documents.
+      #
+      # * +delim_group+ is the group to use to classify the delimiters of the
+      #   region
+      # * +inner_group+ is the group to use to classify the contents of the
+      #   region
+      # * +starter+ is the text to use as the starting delimiter
+      # * +exprs+ is a boolean flag indicating whether the region is an
+      #   interpolated string or not
+      # * +delim+ is the text to use as the delimiter of the region. If +nil+,
+      #   the next character will be treated as the delimiter.
+      # * +heredoc+ is either +false+, meaning the region is not a heredoc, or
+      #   <tt>:flush</tt> (meaning the delimiter must be flushed left), or
+      #   <tt>:float</tt> (meaning the delimiter doens't have to be flush left).
+      def scan_delimited_region( delim_group, inner_group, starter, exprs,
+        delim=nil, heredoc=false )
+      # begin
+        if !delim
+          start_group delim_group, starter
+          delim = scan( /./ )
+          append delim
+
+          delim = case delim
+            when '{' then '}'
+            when '(' then ')'
+            when '[' then ']'
+            when '<' then '>'
+            else delim
+          end
+        end
+
+        start_region inner_group
+
+        items = "\\\\|"
+        if heredoc
+          items << "(^"
+          items << '\s*' if heredoc == :float
+          items << "#{Regexp.escape(delim)}\s*?)#{EOL}"
+        else
+          items << "#{Regexp.escape(delim)}"
+        end
+        items << "|#(\\$|@@?|\\{)" if exprs
+        items = Regexp.new( items )
+
+        loop do
+          p = pos
+          match = scan_until( items )
+          if match.nil?
+            start_group inner_group, scan_until( /\Z/ )
+            break
+          else
+            text = pre_match[p..-1]
+            start_group inner_group, text if text.length > 0
+            case matched.strip
+              when "\\"
+                unless exprs
+                  case peek(1)
+                    when "'"
+                      scan(/./)
+                      start_group :escape, "\\'"
+                    when "\\"
+                      scan(/./)
+                      start_group :escape, "\\\\"
+                    else
+                      start_group inner_group, "\\"
+                  end
+                else
+                  start_group :escape, "\\"
+                  c = getch
+                  append c
+                  case c
+                    when 'x'
+                      append scan( /[a-fA-F0-9]{1,2}/ )
+                    when /[0-7]/
+                      append scan( /[0-7]{0,2}/ )
+                  end
+                end
+              when delim
+                end_region inner_group
+                start_group delim_group, matched
+                break
+              when /^#/
+                do_highlight = (option(:expressions) == :highlight)
+                start_region :expr if do_highlight
+                start_group :expr, matched
+                case matched[1]
+                  when ?{
+                    depth = 1
+                    content = ""
+                    while depth > 0
+                      p = pos
+                      c = scan_until( /[\{}]/ )
+                      if c.nil?
+                        content << scan_until( /\Z/ )
+                        break
+                      else
+                        depth += ( matched == "{" ? 1 : -1 )
+                        content << pre_match[p..-1]
+                        content << matched if depth > 0
+                      end
+                    end
+                    if do_highlight
+                      subtokenize "ruby", content
+                      start_group :expr, "}"
+                    else
+                      append content + "}"
+                    end
+                  when ?$, ?@
+                    append scan( /\w+/ )
+                end
+                end_region :expr if do_highlight
+              else raise "unexpected match on #{matched}"
+            end
+          end
+        end
+      end
+
+      # Scan a heredoc beginning at the current position.
+      #
+      # * +float+ indicates whether the delimiter may be floated to the right
+      # * +type+ is +nil+, a single quote, or a double quote
+      # * +delim+ is the delimiter to look for
+      def scan_heredoc(float, type, delim)
+        scan_delimited_region( :constant, :string, "", type != "'",
+          delim, float ? :float : :flush )
+      end
+  end
+
+  SYNTAX["ruby"] = Ruby
+
+end