farleyknight-ionize 0.1.0

data/lib/ionize/tokenizer.rb

@@ -0,0 +1,544 @@
+require 'rubygems'
+require 'dhaka'
+
+module Ionize
+  module Php
+    class Tokenizer < Dhaka::Tokenizer
+      digits     = ('0'..'9').to_a
+      letters    = ('A'..'Z').to_a + ('a'..'z').to_a
+
+      whitespace = [' ']
+      dollar     = ['$']
+      underscore = ['_']
+      percent    = ['%']
+      slash      = ['/']
+      backslash  = ["\\"]
+      star       = ['*']
+      newline    = ["\n"]
+      tab        = ["\t"]
+      squotes    = ["'"]
+      dquotes    = ['"']
+      dot        = ['.']
+      equals     = ['=']
+      plus       = ['+']
+      minus      = ['-']
+      at         = ['@']
+      pipe       = ['|']
+      greater    = [">"]
+      less_than  = ["<"]
+      caret      = ["^"]
+      ampersand  = ["&"]    
+
+      Symbols = ['[', ']', '(', ')', '{', '}', ':', ';', ',', '!', '-']
+
+      #
+      # Why did I do it this way? Might be easier to read as 
+      # just straight symbols
+      #
+      AllCharacters  = digits + Symbols + whitespace + star +
+        underscore + squotes + dquotes + letters + newline + dot + 
+        tab + equals + dollar + plus + minus + at +
+        pipe + greater + less_than + percent + ampersand +
+        ["^", "?", '#', "\r", "/", "\\"]
+
+      Keywords = %w(as and break case class default do echo 
+                    extends else exit for foreach function global 
+                    if include include_once list new or print 
+                    protected private require require_once return 
+                    static switch var while)
+
+      Whitespace     = [" ", "\t", "\r", "\n"] 
+
+      Identifiers    = letters + underscore + digits
+      NonIdentifiers = Whitespace + ['"', "'", ">", "<", "="] + Symbols + 
+        ["^", "+", "-", "|", ".", "/", "&", "\n", "\\", "$"]
+      
+      # TODO: Adding these methods might be a sight that I should 
+      # switch to the Lexeme approach
+      
+      # Just have to figure out Dhaka's own regex engine
+      
+      def peek
+        @input[@curr_char_index+1] and @input[@curr_char_index+1].chr 
+      end
+
+      def double_peek
+        @input[@curr_char_index+2] and @input[@curr_char_index+2].chr 
+      end
+      
+      def reverse_peek
+        @input[@curr_char_index-1] and @input[@curr_char_index-1].chr 
+      end
+
+      def double_reverse_peek
+        @input[@curr_char_index-2] and @input[@curr_char_index-2].chr 
+      end
+
+      def create_keyword keyword
+        create_token keyword, keyword
+        keyword.length.times { advance }    
+      end
+
+      def create_cast cast
+        create_token 'cast', cast
+        cast.length.times { advance }    
+      end
+      
+      def exactly_matches? keyword
+        @input.slice(@curr_char_index, keyword.length) == keyword
+      end
+      
+      for_state :idle_state do
+        for_default do
+          create_token "html_string", "" and switch_to :html
+        end
+      end
+
+      for_state :html do
+        for_default do
+          curr_token.value << curr_char and advance
+        end
+        
+        for_characters "<" do
+          if exactly_matches? "<?php"
+            create_token "php_start", "<?php" and 5.times { advance }
+            switch_to :php
+          else
+            curr_token.value << curr_char and advance
+          end
+        end
+      end
+
+      for_state :php do
+        for_characters Symbols do
+          create_token curr_char, curr_char and advance
+        end
+
+        for_characters [":"] do
+          if peek == ":"
+            create_token "::", "::" and 2.times { advance }
+          else
+            create_token curr_char, curr_char and advance
+          end
+        end
+
+        for_characters [";"] do
+          create_token curr_char, curr_char and advance
+          # Double semicolons ";;" sometimes show up
+          # Who came up with such an idiom, I dunno
+          advance if curr_char == ";"          
+        end
+
+        for_characters ["/"] do
+          if peek == '/'
+            2.times { advance }
+            switch_to :comment
+          elsif peek == '*'
+            2.times { advance }
+            switch_to :multiline_comment
+          else
+            create_token 'op', '/'
+            advance
+          end
+        end
+
+        for_characters ['#'] do
+          if peek == '!'
+            switch_to :unix_interpreter_directive and 2.times { advance }
+          else
+            switch_to :comment
+          end
+        end
+        
+        for_characters ['?'] do
+          if peek == ">"
+            create_token "php_end", "?>" and 2.times { advance }
+            switch_to :idle_state
+          else
+            create_token '?', '?' and advance
+          end
+        end
+
+        for_characters ['@'] do
+          create_token '@', '@' and advance
+        end
+
+        for_characters ["!"] do
+          if peek == "="
+            if double_peek == "="
+              create_token "op", "!==" and 3.times { advance }
+            else
+              create_token "!=", "!=" and 2.times { advance }
+            end
+          else
+            create_token "!", "!" and advance
+          end
+        end
+
+        for_character ["|"] do
+          if peek == "|" 
+            create_token "||", "||" and 2.times { advance }
+          else
+            create_token "op", "|" and advance
+          end
+        end
+        
+        for_characters ["+"] do
+          if peek == "+"
+            create_token "++", "++" and 2.times { advance }
+          elsif peek == "="
+            create_token "op", "+=" and 2.times { advance }
+          else
+            create_token "+", "+" and advance
+          end
+        end
+
+        for_characters ["-"] do
+          if peek == "-"
+            create_token "--", "--" and 2.times { advance }
+          elsif peek == ">"
+            create_token "->", "->" and 2.times { advance }
+          elsif peek == "="
+            create_token "op", "-=" and 2.times { advance }
+          else
+            create_token "-", "-" and advance
+          end
+        end
+
+        for_characters ['%', '*'] do
+          create_token('op', curr_char) and advance
+        end
+
+        for_characters ['<'] do
+          if exactly_matches? "<?php" or exactly_matches? "<?PHP"
+            create_token "php_start", "<?php" and 5.times { advance }
+          elsif peek == ">"
+            create_token "<>", "<>" and 2.times { advance }
+          elsif exactly_matches? "<<<"
+            3.times { advance } and  switch_to :heredoc
+          elsif peek == '='
+            create_token "op", "<=" and 2.times { advance }
+          else
+            create_token "<", "<" and advance
+          end
+        end
+
+        for_characters [">"] do
+          if peek == '='
+            create_token "op", ">=" and 2.times { advance }
+          else
+            create_token ">", ">" and advance
+          end
+        end
+
+        for_character ['.'] do
+          if peek == "="
+            create_token('op', '.=') and 2.times { advance }
+          else
+            create_token('.', '.') and advance
+          end
+        end
+
+        for_character ['='] do
+          if peek == '='
+            if double_peek == '='
+              create_token('op', '===') and 3.times { advance }
+            else
+              create_token('op', '==') and 2.times { advance }
+            end
+          elsif peek == '>'
+            create_token('=>', '=>') and 2.times { advance }
+          else
+            create_token('=', '=') and advance
+          end
+        end
+
+        for_character ['&'] do
+          if peek == '&'
+            create_token('&&', '&&') and 2.times { advance }
+          else
+            create_token('&', '&') and advance
+          end
+        end
+
+        for_character ['^'] do
+          create_token('op', '^') and advance
+        end
+
+        for_characters digits do
+          create_token 'num', ''
+          switch_to :number
+        end
+        
+        for_character ['$'] do
+          if peek == '$'
+            create_token('variable', '$$') and 2.times { advance }
+          else
+            create_token('variable', '$') and advance
+          end
+          switch_to :word
+        end
+        
+        for_character letters + underscore do      
+          create_token 'word', ''
+          switch_to :word
+        end
+
+        for_character ['"'] do
+          create_token('double_quoted_string', '') and advance
+          switch_to :double_string
+        end
+
+        for_character ["'"] do
+          create_token('single_quoted_string', "") and advance
+          switch_to :single_string
+        end
+        
+        for_characters [' ', "\t", "\r"] do
+          advance
+        end
+
+        # Dhaka seems to to be introducing literal newlines?
+        for_characters ["\\"] do
+          if peek == "n"
+            2.times { advance }
+          end
+        end
+
+        for_characters ["\n"] do 
+          advance
+        end
+      end
+
+      for_state :number do
+        for_characters digits do
+          curr_token.value << curr_char and advance
+        end
+
+        for_characters Symbols + whitespace do
+          switch_to :php
+        end
+      end
+
+      casts = %w{ object int integer bool boolean float double real }  
+      
+      for_state :unix_interpreter_directive do
+        for_characters letters + Symbols + whitespace + ["/"] do
+          advance
+        end
+        
+        for_characters ["\n"] do
+          switch_to :php
+        end
+      end
+
+      for_state :word do
+        for_characters Identifiers do
+          curr_token.value << curr_char 
+          # TODO: Get rid of this or turn it into a case/when
+          if curr_token.value == "else" 
+            # This is the special case where we want to parse:
+            #   elseif
+            # as 
+            #   else if
+            # As this is easier on our grammar
+            curr_token.symbol_name = "else"
+            switch_to :php and advance
+          elsif curr_token.value =~ /FALSE/i
+            # We want to parse special constant 'false' as
+            # separate from other words
+            # 
+            curr_token.symbol_name = "false"
+            switch_to :php and advance
+          elsif curr_token.value =~ /TRUE/i
+            # We want to parse special constant 'true' as
+            # separate from other words
+            # 
+            curr_token.symbol_name = "true"
+            switch_to :php and advance
+          else
+            # Otherwise we just advance to the next thing
+            advance
+          end
+        end
+
+        for_characters NonIdentifiers do 
+          if match = Keywords.detect {|kw| curr_token.value.downcase == kw }
+            curr_token.symbol_name = match
+          elsif match = casts.detect {|cast| curr_token.value == cast }
+            curr_token.symbol_name = "cast"
+          elsif curr_token.value == "and"
+            curr_token.symbol_name = "op"
+          end
+          switch_to :php 
+        end
+      end
+
+      for_state :array_lookup_within_double_string do
+        for_default do
+          curr_token.value << curr_char and advance
+        end
+
+        for_characters ["]"] do
+          curr_token.value << curr_char and advance
+          create_token "double_quoted_string_node", ""
+          switch_to :double_string 
+        end
+      end
+
+      for_state :braced_variable_within_double_string do
+        for_default do
+          curr_token.value << curr_char and advance
+        end
+
+        for_characters ["}"] do        
+          create_token "double_quoted_string_node", "" and advance
+          switch_to :double_string
+        end
+      end
+
+      for_state :variable_within_double_string do
+        for_characters Identifiers + ["$"] do
+          curr_token.value << curr_char and advance
+        end
+
+        for_characters ["["] do
+          curr_token.symbol_name = "array_lookup_node"
+          curr_token.value << curr_char and advance
+          switch_to :array_lookup_within_double_string
+        end
+
+        for_characters NonIdentifiers - ["["] do        
+          create_token "double_quoted_string_node", curr_char and advance
+          switch_to :double_string
+        end
+        
+        for_characters ['"'] do 
+          if reverse_peek == "\\"
+            if double_reverse_peek == "\\"
+              advance and switch_to :php
+            else
+              curr_token.value << curr_char and advance
+            end
+          else
+            advance and switch_to :php
+          end
+        end
+      end
+
+      for_state :double_string do
+        for_default do
+          curr_token.value << curr_char and advance      
+        end
+
+        for_characters ["{"] do
+          if peek == "$"
+            if curr_token.symbol_name == "double_quoted_string"
+              curr_token.symbol_name = "open_double_quoted_string"
+            end
+            create_token "variable", "" and advance
+            switch_to :braced_variable_within_double_string
+          else
+            curr_token.value << curr_char and advance
+          end
+        end
+
+        for_characters ["$"] do
+          if curr_token.symbol_name == "double_quoted_string"
+            curr_token.symbol_name = "open_double_quoted_string"
+          end
+          create_token "variable", curr_char and advance
+          switch_to :variable_within_double_string
+        end       
+
+        for_character ['"'] do
+          if reverse_peek == "\\"
+            if double_reverse_peek == "\\"
+              curr_token.value << curr_char and advance
+              switch_to :php
+            else
+              curr_token.value.gsub!("\\", "")
+              curr_token.value << curr_char and advance
+            end
+          else
+            switch_to :php and advance
+          end
+        end
+      end
+
+      for_state :single_string do
+        for_characters AllCharacters - ["'"] do
+          curr_token.value << curr_char and advance      
+        end
+
+        for_character ["'"] do
+          if reverse_peek == "\\"
+            curr_token.value << curr_char and advance      
+          else
+            switch_to :php and advance
+          end
+        end
+      end
+
+      for_state :comment do
+        for_characters AllCharacters - newline do
+          advance
+        end
+
+        for_characters ["\n", "\r"] do
+          advance
+          switch_to :php
+        end
+      end
+
+      for_state :multiline_comment do
+        for_characters AllCharacters - star do
+          advance
+        end
+
+        for_characters ['*'] do
+          if peek == '/'
+            2.times { advance }
+            switch_to :php
+          else
+            advance
+          end      
+        end
+      end
+
+      # Wow.. this is one crazy hack. I'll have to re-write this
+      # at a later date
+      for_state :heredoc do
+        for_characters AllCharacters do
+          @heredoc_identifier_complete ||= false
+          @heredoc_identifier ||= ""
+
+          if @heredoc_identifier_complete
+            # We have the identifier. Append text to the current token
+            if exactly_matches? @heredoc_identifier
+              @heredoc_identifier.length.times { advance }
+              @heredoc_identifier_complete, @heredoc_identifier = true, ""
+              # We've found the of the heredoc. Go back to idle
+              switch_to :php
+            else
+              # Keep appending..
+              curr_token.value << curr_char and advance
+            end
+          else            
+            # We haven't parsed the heredoc identifier yet. Parse it
+            if curr_char == "\n" or curr_char == "\r"
+              # We've reached the end of the heredoc identifier 
+              # Mark it and create a token for the string
+              @heredoc_identifier_complete = true
+              create_token "double_quoted_string", "" and advance
+            else
+              # Just another character in the identifier
+              @heredoc_identifier << curr_char and advance           
+            end
+          end
+        end
+      end
+
+    end
+  end
+end