eden 0.1.1

data/lib/eden/tokenizers/delimited_literal_tokenizer.rb

@@ -0,0 +1,38 @@
+module Eden
+  module DelimitedLiteralTokenizer
+    def tokenize_delimited_literal
+      advance # Pass the %
+
+      if( /[^A-Za-z0-9]/.match( cchar ) )
+        def_char = 'Q'
+        @state = :double_q_string
+      elsif( /[qQswWrx]/.match( cchar) )
+        def_char = cchar
+        @state = infer_delimited_literal_type  
+        advance
+      else
+        raise "Invalid delimiter character"
+      end
+
+      case def_char
+      when 'r', 'Q', 'W', 'x'
+        token = tokenize_expanded_string( cchar )
+      when 's', 'q', 'w'
+        token = tokenize_non_expanded_string( cchar )
+      end
+
+      return token
+    end
+
+    def infer_delimited_literal_type
+      case cchar
+        when 's' then :symbol
+        when 'w', 'W' then :array_literal
+        when 'q' then :single_q_string
+        when 'Q' then :double_q_string
+        when 'r' then :regex
+        when 'x' then :backquote_string
+      end
+    end
+  end
+end

data/lib/eden/tokenizers/number_tokenizer.rb

@@ -0,0 +1,68 @@
+module Eden
+  module NumberTokenizer
+
+    # Tokenize a non-decimal integer literal - e.g. Ox4F2E, 0b1101
+    def tokenize_integer_literal
+      @expr_state = :end
+      if peek_ahead_for(/[_oObBxX]/)
+        advance(2) # Pass 0x / 0b / 0O
+      else
+        advance # Pass 0 for Octal digits
+      end
+      pattern = {:bin_literal => /[01]/,
+                 :oct_literal => /[0-7]/,
+                 :hex_literal => /[0-9a-fA-F]/}[@state]
+      advance until( pattern.match( cchar ).nil? )
+      capture_token( @state )
+    end
+
+    def tokenize_decimal_literal
+      @expr_state = :end
+      # Capture the sign
+      advance if cchar == '+' || cchar == '-'
+
+      # Handle a lone zero
+      if cchar == '0' && !peek_ahead_for(/[dD]/)
+        advance
+        return capture_token( :dec_literal )
+      end
+
+      # Handle 0d1234 digits
+      advance(2) if cchar == '0' && peek_ahead_for(/[dD]/)
+
+      until( /[0-9.eE]/.match( cchar ).nil? )
+        case cchar
+        when '.'
+          return tokenize_float_literal
+        when 'e', 'E'
+          return tokenize_exponent_literal
+        when '0'..'9'
+          advance
+        else
+        end
+      end
+      capture_token( :dec_literal )
+    end
+
+    # Tokenize a literal with an exponent - e.g. 3.4E+22
+    def tokenize_exponent_literal
+      advance # Pass the e/E
+      advance if cchar == '+' or cchar == '-'
+      advance until( /[0-9]/.match( cchar ).nil? )
+      capture_token( :exp_literal )
+    end
+
+    # Tokenize a float literal - e.g. 2.0, 2.1101
+    def tokenize_float_literal
+      advance # Pass the .
+
+      until( /[0-9eE]/.match( cchar ).nil? )
+        if cchar == 'e' || cchar == 'E'
+          return tokenize_exponent_literal
+        end
+        advance
+      end
+      capture_token( :float_literal )
+    end
+  end
+end

data/lib/eden/tokenizers/operator_tokenizer.rb

@@ -0,0 +1,211 @@
+module Eden
+  module OperatorTokenizer
+    def tokenize_equals_operators
+      advance
+      default_expr_state_transition!
+      case cchar
+      when '>'
+        advance and capture_token(:hash_rocket)
+      when "~"
+        advance and capture_token(:matches)
+      when '='
+        advance
+        if cchar == '='
+          advance and capture_token(:identity_equality)
+        else
+          capture_token(:equality)
+        end
+      else
+        capture_token(:equals)
+      end
+    end
+
+    def tokenize_bang_operators
+      @expr_state = :beg
+      advance
+      case cchar
+      when '='
+        advance and capture_token(:not_equals)
+      when '~'
+        advance and capture_token(:not_matches)
+      else
+        capture_token(:logical_not)
+      end
+    end
+
+    def tokenize_plus_operators
+      advance
+      case cchar
+      when '='
+        @expr_state = :beg
+        advance and capture_token(:plus_equals)
+      when '@'
+        @expr_state = :arg if [:fname, :dot].include?(@expr_state)
+        advance and capture_token(:plus_at)
+      else
+        @expr_state = :beg
+        capture_token(:plus)
+      end
+    end
+
+    def tokenize_minus_operators
+      advance
+      case cchar
+      when '='
+        @expr_state = :beg
+        advance and capture_token(:minus_equals)
+      when '@'
+        @expr_state = :arg if [:fname, :dot].include?(@expr_state)
+        advance and capture_token(:minus_at)
+      else
+        @expr_state = :beg
+        capture_token(:minus)
+      end
+    end
+
+    def tokenize_multiply_operators
+      advance
+      case cchar
+      when '*'
+        @expr_state = :beg
+        advance
+        if cchar == '='
+          advance and capture_token(:exponent_equals)
+        else
+          capture_token(:exponent)
+        end
+      when '='
+        @expr_state = :beg
+        advance and capture_token(:multiply_equals)
+      else
+        # TODO: Add logic to disamiguate between splat and multiply
+        default_expr_state_transition!
+        capture_token(:multiply)
+      end
+    end
+
+    # Tokenizes operators using the \ character, after tokenize_potential_regex
+    # has decided that it's not the start of a regex.
+    def tokenize_divide_operators
+      advance
+      if cchar == '='
+        @expr_state = :beg
+        advance and capture_token(:divide_equals)
+      else
+        default_expr_state_transition!
+        capture_token(:divide)
+      end
+    end
+
+    def tokenize_lt_operators
+      advance 
+      default_expr_state_transition!
+      case cchar
+      when '<'
+        advance
+
+        if ![:end, :dot, :end_arg, :class].include?(@expr_state) && cchar != ' '
+          token = tokenize_heredoc_delimiter
+          return token if token
+        end
+
+        if cchar == '='
+          @expr_state = :beg
+          advance and capture_token(:left_shift_equals)
+        else
+          capture_token(:left_shift)
+        end
+      when '='
+        advance
+        if cchar == '>'
+          advance and capture_token(:sort_operator)
+        else
+          capture_token(:lte)
+        end
+      else
+        capture_token(:lt)
+      end
+    end
+
+    def tokenize_gt_operators
+      advance
+      default_expr_state_transition!
+      case cchar
+      when '>'
+        advance
+        if cchar == '='
+          advance and capture_token(:right_shift_equals)
+        else
+          capture_token(:right_shift)
+        end
+      when '='
+        advance and capture_token(:gte)
+      else
+        capture_token(:gt)
+      end
+    end
+
+    def tokenize_pipe_operators
+      advance
+      case cchar
+      when '|'
+        advance
+        @expr_state = :beg
+        if cchar == '='
+          advance and capture_token(:logical_or_equals)
+        else
+          capture_token(:logical_or)
+        end
+      when '='
+        @expr_state = :beg
+        advance and capture_token(:bitwise_or_equals)
+      else
+        default_expr_state_transition!
+        capture_token(:bitwise_or)
+      end
+    end
+
+    def tokenize_ampersand_operators
+      advance
+      case cchar
+      when '&'
+        advance
+        @expr_state = :beg
+        if cchar == '='
+          advance and capture_token(:logical_and_equals)
+        else
+          capture_token(:logical_and)
+        end
+      when '='
+        @expr_state = :beg
+        advance and capture_token(:bitwise_and_equals)
+      else
+        default_expr_state_transition!
+        capture_token(:bitwise_and)
+      end
+    end
+
+    def tokenize_caret_operators
+      advance
+      if cchar == "="
+        @expr_state = :beg
+        advance and capture_token(:caret_equals)
+      else
+        default_expr_state_transition!
+        capture_token(:caret)
+      end
+    end
+
+    def tokenize_modulo_operators
+      advance
+      if cchar == "="
+        @expr_state = :beg
+        advance and capture_token(:modulo_equals)
+      else
+        default_expr_state_transition!
+        capture_token(:modulo)
+      end
+    end
+
+  end
+end

data/lib/eden/tokenizers/regex_tokenizer.rb

@@ -0,0 +1,37 @@
+module Eden
+  module RegexTokenizer
+    def tokenize_potential_regex
+
+      if @expr_state == :beg || @expr_state == :mid || @expr_state == :class
+        return tokenize_regex
+      end
+
+      if peek_ahead_for(/=/)
+        return tokenize_divide_operators
+      end
+
+      if (@expr_state == :arg || @expr == :cmd_arg) && @line.last_token_is_space?
+        return tokenize_regex
+      end
+
+      return tokenize_divide_operators
+    end
+
+    def tokenize_regex
+      advance # Consume the leading /
+      while true
+        if cchar == '/'
+          advance
+          # Capture the regex option
+          advance if cchar =~ /i|o|m|x|n|e|u|s/
+          return capture_token(:regex)
+        end
+        if cchar == nil #end of file
+          raise "Unclosed Regex found"
+        end
+        advance if cchar == '\\'
+        advance
+      end
+    end
+  end
+end

data/lib/eden/tokenizers/string_tokenizer.rb

@@ -0,0 +1,149 @@
+module Eden
+  module StringTokenizer
+    def tokenize_single_quote_string
+      tokenize_non_expanded_string("'")
+    end
+
+    # If a block is given, it gets run after the final delimiter is detected. The
+    # primary purpose for this is to allow the capture of regex modifiers
+    def tokenize_non_expanded_string( start_delimiter )
+      delimiter_depth = 0
+      matched_delimiter = is_matched_delimiter?( start_delimiter )
+      end_delimiter = find_matching_delimiter( start_delimiter )
+
+      advance # Pass the opening delimiter
+
+      until((cchar == end_delimiter && delimiter_depth == 0) || @i >= @length)
+
+        if matched_delimiter
+          delimiter_depth += 1 if cchar == start_delimiter
+          delimiter_depth -= 1 if cchar == end_delimiter
+        end
+
+        if cchar == '\\'
+          advance(2) # Pass the escaped character
+        else
+          advance
+        end
+      end
+      advance # Pass the closing quote
+
+      if @state == :regex
+        advance if ['i', 'm'].include?( cchar )
+      end
+
+      @expr_state = :end
+      capture_token( @state )
+    end
+
+    def tokenize_backquote_string
+      tokenize_expanded_string( '`' )
+    end
+
+    def tokenize_double_quote_string( in_string_already = false )
+      tokenize_expanded_string('"', in_string_already)
+    end
+
+    def tokenize_expanded_string( start_delimiter, in_string_already = false )
+      saved_state = @state
+      tokens = []
+      end_delimiter = find_matching_delimiter( start_delimiter )
+      advance unless in_string_already # Pass the opening backquote
+      until( cchar == end_delimiter || @i >= @length )
+        if cchar == '\\'
+          advance(2) # Pass the escaped character
+        elsif cchar == '#'
+          advance # include the # character in the string
+          case cchar
+          when '{'
+            @interpolating.push( @state )
+            @delimiters.push( start_delimiter )
+            tokens << Token.new( @state, thunk )
+            reset_thunk!
+            @state = :lcurly
+            tokens << tokenize_single_character
+            return tokens
+          when '@'
+            tokens << capture_token( @state )
+            if peek_ahead_for('@')
+              tokens << tokenize_classvar
+            else
+              tokens << tokenize_instancevar
+            end
+            @state = saved_state
+          when '$'
+            tokens << capture_token( @state )
+            tokens << tokenize_globalvar
+            @state = saved_state
+          end
+        else
+          advance
+        end
+      end
+      advance # Pass the closing delimiter
+      if @state == :regex
+        advance if ['i', 'm'].include?( cchar )
+      end
+      @expr_state = :end
+      tokens << capture_token( @state )
+      return tokens
+    end
+  
+    # Called from tokenize_lt_operators when it identifies that 
+    # << is a heredoc delimiter. Expects that '<<' will already
+    # be included in the current thunk.
+    def tokenize_heredoc_delimiter
+      offset = 2
+      if cchar == '-'
+        advance
+        offset = 3
+      end
+
+      if cchar =~ /[A-Za-z_]/
+        advance
+        advance until /[A-Za-z0-9_]/.match( cchar ).nil?
+      elsif /['"`]/.match(cchar)
+        advance_through_quoted_delimiter(cchar)
+      else
+        return nil
+      end
+      @heredoc_delimiter = thunk[offset..-1]
+      capture_token( :heredoc_delimiter )
+    end
+
+    def tokenize_heredoc_body
+      if @heredoc_delimiter
+        advance until (@sf.source[@i, @heredoc_delimiter.length ] == @heredoc_delimiter && 
+          @sf.source[@i, @heredoc_delimiter.length+1 ] == @heredoc_delimiter + "\n" ||
+          @sf.source[@i, @heredoc_delimiter.length+2 ] == @heredoc_delimiter + "\r\n") ||
+          @i >= @length
+      end
+      @heredoc_delimiter.length.times { advance }
+      @heredoc_delimiter = nil
+      capture_token( :heredoc_body )
+    end
+
+    private
+    # Returns the matching delimiter for the 4 "paired" delimiters
+    def find_matching_delimiter( start_delimiter )
+      case start_delimiter
+      when '{' then '}'
+      when '(' then ')'
+      when '[' then ']'
+      when '<' then '>'
+      else
+        start_delimiter
+      end
+    end
+
+    def is_matched_delimiter?( cchar )
+      !! /[{\(\[<]/.match(cchar)
+    end
+
+    def advance_through_quoted_delimiter( delimiter )
+      advance
+      advance until cchar == delimiter
+      advance
+    end
+  end
+end