farleyknight-ionize 0.1.0

data/README.rdoc

@@ -0,0 +1,59 @@
+= Ionize
+
+== DESCRIPTION:
+
+Tranforms Php to Ruby.
+
+Ionize parses Php, turns the parse result into an s-expression, then translates
+the s-expression into one that can be consumed by Ruby2Ruby. When everything is
+said and done, it will be able to translate an entire Php installation into Ruby.
+
+== FEATURES/PROBLEMS:
+
+Check out http://ionize.farleyknight.com for some examples of what
+works and what doesn't.
+
+== SYNOPSIS:
+
+TODO: Add an example of using a command line tool to convert
+an existing Php installation into a Ruby one
+
+== REQUIREMENTS:
+
+Dhaka, ParseTree, Ruby2Ruby
+
+The Rspec gem is required to run the specs..
+
+, .. anything else?
+
+== INSTALL:
+
+When I register this project with Rubygems, the command will be:
+  sudo gem install ionize
+
+But, at the moment, we're still in development..
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2008 Farley Knight
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,5 @@
+# require 'config/requirements'
+# require 'config/hoe' # setup Hoe + all gem configuration
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }
+

data/bin/ionize

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+# This is stupid..
+require '/home/robinhoode/dev/ruby/ionize/lib/ionize'
+
+module Ionize
+  module Php
+    class Convert
+      def run
+        if ARGV.first =~ /\.php$/
+#          convert_file(ARGV.first)
+        elsif File.stat(ARGV.first).directory?
+          puts "#{ARGV.first} was a directory. Converting.."
+          convert_directory(ARGV.first)
+        else
+          # Should actually print the usage here..
+          raise "First argument was neither php file nor directory: #{ARGV.first.inspect}"
+        end
+      end
+
+      def convert_file(php_file, ruby_file = nil)
+        ruby_file ||= php_file.gsub(".php", ".rb")        
+
+        ruby_dir = File.dirname(ruby_file)
+        FileUtils.mkdir_p(ruby_dir) unless File.exists?(ruby_dir)
+
+        File.open(ruby_file, "w+") do |f|
+          f.write(Ionize::Php.to_ruby(File.read(php_file)))
+        end
+        puts "Converted file #{php_file} and stored it as #{ruby_file}"
+      end
+
+      def convert_directory(php_directory, ruby_directory = nil)
+        ruby_directory ||= "#{php_directory}-ruby"
+        FileUtils.mkdir ruby_directory unless File.exists?(ruby_directory) and File.stat(ruby_directory).directory?
+        Dir["#{php_directory}/**/*"].select {|f| f =~ /\.php$/ }.each do |php_file|
+          ruby_file = php_file.gsub(php_directory, ruby_directory).gsub(".php", ".rb")
+          puts "Found file #{php_file}"
+          convert_file(php_file, ruby_file)
+        end
+      end
+    end
+  end
+end
+
+
+Ionize::Php::Convert.new.run

data/lib/ionize.rb

@@ -0,0 +1,75 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'rubygems'
+require 'dhaka'
+require 'parse_tree'
+require 'ruby2ruby'
+
+require 'ionize/translate'
+require 'ionize/environment'
+
+require 'ionize/parser'
+require 'ionize/compiled_parser'
+
+module Ionize
+  module Php
+    CharacterLimit = 50000
+
+    # Returns a Dhaka AST of a string
+    def self.to_ast(string)
+      # TODO: Error handling could be better..
+      # Maybe another time
+
+      if string.length > CharacterLimit
+        raise "Your input is #{string.length}. Ionize::Php::Parser currently cannot handle inputs larger than #{CharacterLimit}, due to Dhaka's memory usage"
+      end
+      
+      tokens = Php.to_tokens(string)
+      result = Php::CompiledParser.parse(tokens)
+      
+      if result.is_a? Dhaka::ParseErrorResult
+        error = %Q{ 
+Could not parse token.
+Token => #{result.unexpected_token.inspect}
+
+}
+        error << string.slice(result.unexpected_token.input_position, 40) rescue nil
+        raise error
+      elsif result.is_a? Dhaka::TokenizerErrorResult
+        raise result
+      end
+
+      result
+    end
+
+    # Tokens from Dhaka's tokenizer
+    def self.to_tokens(string)
+      require 'ionize/tokenizer'
+      tokenizer = Php::Tokenizer.new(string)
+      tokenizer.switch_to :php unless string =~ /<\?php/i
+      tokens = tokenizer.run
+
+      if tokens.is_a? Dhaka::TokenizerErrorResult
+        raise "Could not tokenize at index #{tokens.unexpected_char_index}\n" + string.slice(tokens.unexpected_char_index, 40).inspect
+      end
+
+      tokens
+    end
+    
+    # An S-expression of the string
+    def self.to_sexp(string)
+      Php::Translate.translate(Php.to_ast(string))
+    end
+
+    # A stripped down version of what Dhaka hands us
+    def self.to_raw_sexp(string)
+      Php::Translate.flatten(Php.to_ast(string))
+    end
+
+    # Abracadbra!
+    def self.to_ruby(string)
+      Php::Translate::Php2Ruby.new.process(Php.to_sexp(string))
+    end
+  end
+end

data/lib/ionize/environment.rb

@@ -0,0 +1,56 @@
+
+load File.dirname(__FILE__) + '/environment/php_array.rb'
+load File.dirname(__FILE__) + '/environment/application.rb'
+
+module Ionize
+  module Php
+    Config = {
+      "magic_quotes_gpc"     => true,
+      "magic_quotes_runtime" => true
+    }
+    module Environment
+      def self.included(target)
+        Kernel.module_eval do
+          def array(*args)
+            PhpArray.from_mixed(*args)
+          end
+          
+          def get_magic_quotes_gpc
+            Php::Config["magic_quotes_gpc"] ? 1 : 0
+          end
+          
+          def get_magic_quotes_runtime
+            Php::Config["magic_quotes_runtime"] ? 1 : 0
+          end
+
+          def gethostbyaddr(string)
+            address = string.split(".").map {|n| n.to_i }.pack("CCCC")
+            Socket.gethostbyaddr(address).first
+          end
+        end
+        
+        String.class_eval do
+          def preg_match(regex)
+            matches = match(regex)
+            return [!matches.nil?, matches.to_a]
+          end
+          
+          def preg_match_all(regex)
+            copy = self.dup
+            results = []
+
+            match = copy.match(regex)
+            while match
+              results << match.to_a
+              copy.gsub!(match.to_a.first, "")
+              match = copy.match(regex)
+            end
+
+            [results.length - 1, results.transpose]
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/ionize/environment/application.rb

@@ -0,0 +1,58 @@
+
+require 'rack'
+
+module Ionize
+  module Php
+    module Environment
+      class ScriptExit < Exception; end
+      
+      class Application
+        def self.run(code)
+          Rack::Handler::Mongrel.run(Application.new(code), :Port => 3000)
+        end
+
+        def initialize(code)
+          @output, @header, @code = "", {}, 200
+          @php_script = Php.to_ruby(code)
+        end
+
+        def GET
+          @get ||= @request.GET
+        end
+
+        def POST
+          @post ||= @request.POST
+        end
+
+        def SERVER
+          @server ||= @requeset.SERVER
+        end
+
+        def exit
+          raise ScriptExit
+        end
+
+        def print(text)
+          @output << text
+        end 
+
+        def header(string)
+          if m = string.match(/Location: (.*)/i)
+            @header["Location"] = m[1]
+            @code = 302
+          end
+        end
+
+        def call(env)
+          @request = Rack::Request.new(env)          
+          begin
+            instance_eval(@php_script)         
+          rescue ScriptExit
+            # Catch an exit() or die() and show that to the user
+          end          
+          [@code, @header, @output]
+        end
+      end
+    end
+  end
+end

data/lib/ionize/environment/php_array.rb

@@ -0,0 +1,95 @@
+
+module Ionize
+  module Php
+    module Environment
+      class PhpArray < Hash
+        def initialize(*args, &block)
+          @index = 0
+          super(*args, &block)
+        end
+
+        def join(*args)
+          values.join(*args)
+        end
+
+        def hash?
+          keys.any? {|k| k.class != Fixnum } 
+        end
+
+        def array?
+          keys.all? {|k| k.class == Fixnum }
+        end
+
+        def combine(second)
+          array = self.class.new
+          second.values.each_with_index do |val, key| 
+            array.put(self[key], val)
+          end
+          array
+        end
+
+        def intersect(second)
+          array = self.class.new
+          second.values.each do |value|
+            if self.values.include? value
+              key = self.to_a.select {|k,v| v == value }.first.first
+              array.put(key, value)
+            end
+          end
+          array
+        end
+
+        def add(value)
+          put(@index, value)
+        end
+
+        def put(key, value)
+          if key.is_a? Fixnum and key >= @index
+            @index = key+1
+          end
+          self[key] = value
+        end
+
+        def to_hash
+          Hash[*(self.to_a).flatten]
+        end
+
+        def merge(other)
+          if self.array? and other.array?
+            self.class.from_array(*(self.values + other.values))
+          else
+            self.class.from_hash(super(other))
+          end
+        end
+
+        def self.from_mixed(*args)
+          array = self.new
+          args.each do |arg|
+            if arg.is_a? Hash
+              arg.each {|key, value| array.put(key, value)}
+            else
+              array.add(arg)
+            end
+          end
+          array
+        end
+
+        def self.from_hash(hash)
+          array = self.new
+          hash.each {|key, value| array.put(key, value)}
+          array
+        end
+        
+        def self.from_array(*args)
+          array = self.new
+          args.each {|arg| array.add(arg)}
+          array
+        end
+
+        def inspect
+          "array(" + super.gsub("{", "").gsub("}", "") + ")"
+        end
+      end
+    end
+  end
+end

data/lib/ionize/parser.rb

@@ -0,0 +1,272 @@
+
+require 'rubygems'
+require 'dhaka'
+
+class Dhaka::Grammar
+  def for_rule(hash)
+    i = 0
+    name = hash.keys.first
+    for_symbol(name) do
+      hash.values.first.split("|").each do |s|
+        send "#{name}_#{i += 1}", *(s.split)
+      end
+    end
+  end
+end
+
+module Ionize
+  module Php
+    class Grammar < Dhaka::Grammar
+      precedences do
+        left     %w{ expr }
+        left     %w{ tertiary }
+        left     %w{ + - }
+        left     %w{ term_plus_expr term_minus_expr }
+        left     %w{ preop op }
+        left     %w{ fun_call fun_call_wo_parens }
+      end
+
+      for_symbol(Dhaka::START_SYMBOL_NAME) do
+        html_file_with_php_with_end_and_html_string  %w{ html_string php_start statements php_end html_string }
+        html_file_with_php_with_end       %w{ html_string php_start statements php_end }
+        html_file_with_php_without_end    %w{ html_string php_start statements }
+        php_file                          %w{ statements }
+      end
+
+      for_symbol 'statements' do
+        multiple_statements               %w{ statement statements }
+        single_statement                  %w{ statement }
+      end
+      
+      for_symbol 'statement' do
+        matched                           %w{ matched }
+        unmatched                         %w{ unmatched }
+        if_statement_b                    " if ( expr ) { statements } ".split
+        if_else_statement_b               " if ( expr ) { statements } else { statements } ".split
+        if_else_statement_b1              " if ( expr ) { statements } else statement ".split
+      end
+
+      for_symbol 'matched' do
+        if_else_statement0                " if ( expr ) ; else matched ".split
+        if_else_statement1                " if ( expr ) matched else matched ".split
+        if_else_statement3                " if ( expr ) matched else { statements } ".split
+        simple                            %w{ simple_statement }
+      end
+
+      for_symbol 'unmatched' do
+        if_statement0                     " if ( expr ) matched ".split
+        if_statement1                     " if ( expr ) unmatched ".split
+        if_else_statement2                " if ( expr ) matched else unmatched ".split
+      end
+
+      for_symbol 'block' do
+        empty_braces                      " { } ".split
+        braced_statements                 " { statements } ".split      
+      end
+      
+      for_symbol 'simple_statement' do
+        line_statement                    %w{ expr ; }
+        class_statement                   " class word { statements } ".split
+        class_extends_statement           " class word extends word { statements } ".split
+        
+        while_statement                   " while ( expr ) block ".split
+        do_while_statement                " do block while ( expr ) ; ".split
+        for_statement                     " for ( expr ; expr ; expr ) block ".split
+        foreach_statement                 " foreach ( expr as expr ) block ".split
+        switch_case_statement             " switch ( expr ) { case_statements } ".split
+        fun_define                        " function word ( fun_args ) { statements } ".split
+        protected_fun_define              " protected function word ( fun_args ) { statements } ".split
+        private_fun_define                " private function word ( fun_args ) { statements } ".split
+        
+        static_declaration                %w{ static variables ; }
+        global_declaration                %w{ global variables ; }
+        class_protected_var_statement     %w{ protected variable ; }
+        class_protected_w_default_stmt    %w{ protected variable = expr ; }
+        class_var_statement               %w{ var variable ; }
+        class_var_with_default_statement  %w{ var variable = expr ; }
+        return_one                        %w{ return expr ; }
+        print_statement                   %w{ print expr ; }      
+        echo_statement                    %w{ echo expr ; }
+        return_zero                       %w{ return ; }
+        break_statement                   %w{ break ; }
+        exit_call_statement               %w{ exit ( ) ; }
+        exit_statement                    %w{ exit ; }
+      end
+
+      for_symbol 'elseif_statements' do
+        one_elseif                        %w{ elseif_statement }
+        multi_else_ifs                    %w{ elseif_statements elseif_statement }
+      end
+
+      for_symbol 'elseif_statement' do
+        else_if                           " else if ( expr ) block ".split
+      end
+
+      for_symbol 'case_statements' do
+        default_only_case                 %w{ default : statements }
+        one_case                          %w{ case expr : opt_statements }
+        default_case                      %w{ case_statements default : statements }
+        multi_case                        %w{ case_statements case expr : opt_statements }
+      end
+
+      for_symbol 'opt_statements' do
+        opt_multiple_statements           %w{ statement statements }
+        opt_single_statement              %w{ statement }
+        opt_no_statements                 %w{ }
+      end
+
+      for_symbol 'expr' do
+        # TODO: This one could get cleaned up
+        tertiary                          %w{ term ? expr : expr }
+        term_postop                       %w{ term postop }
+        post_increment_var                %w{ term ++ }
+        post_decrement_var                %w{ term -- }
+        reference_term                    %w{ & term }
+
+        term_or_expr                      %w{ term || expr }
+        term_or_expr_with_or              %w{ term or expr }
+        term_and_expr                     %w{ term && expr }
+        term_boolean_and_expr             %w{ term & expr }
+        term_and_expr_with_and            %w{ term and expr }
+        term_not_equal_expr               %w{ term <> expr }
+        term_less_than_expr               %w{ term < expr }
+        term_greater_than_expr            %w{ term > expr }
+
+        require_statement                 %w{ require expr }
+        require_once_statement            %w{ require_once expr }
+        include_statement                 %w{ include expr }
+        include_once_statement            %w{ include_once expr }
+
+        list_assign                       %w{ list ( fun_args ) = expr }
+
+#        oo_variable_variable_assign       %w{ term -> variable = expr }
+#        oo_array_assign                   %w{ term -> word [ term ] = expr }
+
+#        term_array_append                 %w{ term [ ] = expr }
+#        term_array_assign                 %w{ term [ expr ] = expr }
+
+        # Unfortunately, I'm going to have to stuff everything
+        # inside this one statement.. so that others that I 
+        # cannot predict can be parsed        
+        term_assign                       %w{ term = expr }
+
+        term_concat_expr                  %w{ expr . term } 
+        term_plus_expr                    %w{ expr + expr }
+        term_minus_expr                   %w{ expr - expr }
+        term_not_equals_expr              %w{ expr != expr }
+        term_hashs_to_expr                %w{ expr => expr }
+
+        term_op_expr                      %w{ term op expr }
+        term                              %w{ term }
+      end
+
+      for_symbol 'term' do
+        # TODO: This one could get cleaned up
+        number                            %w{ number }
+        false_value                       %w{ false }
+        true_value                        %w{ true }
+        a_string                          %w{ a_string }
+        a_variable                        %w{ variable }
+        class_method_call                 %w{ word :: word ( fun_call_args ) }
+
+        array_append                      %w{ term [ ] }
+        array_lookup                      %w{ term [ expr ] } 
+        oo_variable                       %w{ term -> word }
+        oo_call                           %w{ term -> word ( fun_call_args ) }
+        oo_variable_variable              %w{ term -> variable }
+                
+        fun_call_trailing_comma           %w{ word ( fun_call_args , ) }
+        regular_fun_call                  %w{ word ( fun_call_args ) }
+        var_call                          %w{ variable ( fun_call_args ) }
+        user_constant                     %w{ word }
+        new_object                        %w{ new word }
+        new_object_with_args              %w{ new word ( fun_call_args ) }
+        new_variable_class_object         %w{ new variable }
+        braced_lookup                     %w{ variable { expr } }
+
+        pre_increment_var                 %w{ ++ term }
+        pre_decrement_var                 %w{ -- term }
+        negative_var                      %w{ - term }
+        not_var                           %w{ ! term }
+
+        term_error_guard                  %w{ @ expr }
+        
+        preop_term                        %w{ preop term }      
+        paren_cast                        %w{ ( cast ) term }
+        paren_expr                        %w{ ( expr ) }
+      end
+
+      for_symbol 'a_string' do
+        composite_double_string           %w{ open_double_quoted_string composite_string_nodes }
+        double_string                     %w{ double_quoted_string }
+        single_string                     %w{ single_quoted_string }      
+      end
+
+      for_symbol 'composite_string_node' do
+        variable_within_string            %w{ variable }
+        array_lookup_within_string        %w{ array_lookup_node }
+        double_string_within_string       %w{ double_quoted_string_node }
+        single_string_within_string       %w{ single_quoted_string_node }
+      end
+
+      for_symbol 'composite_string_nodes' do
+        multiple_nodes                    %w{ composite_string_nodes composite_string_node }
+        single_node                       %w{ composite_string_node }
+      end
+
+      for_symbol 'variables' do
+        one_variable                      %w{ variable }
+        multiple_variables                %w{ variables , variable }
+      end
+
+      for_symbol 'number' do
+        positive_number                   %w{ num }
+        signed_number                     %w{ op num }
+      end
+
+      for_symbol 'bracketed_expr' do
+        one_bracket                       %w{ single_bracket }
+        multi_brackets                    %w{ bracketed_expr single_bracket }
+      end
+
+      for_symbol 'oo_fun_call' do
+        oo_fun_call_trailing_comma        %w{ word ( fun_call_args , ) }
+        oo_regular_fun_call               %w{ word ( fun_call_args ) }      
+      end
+
+      for_symbol 'fun_args' do
+        no_args                           %w{ }
+        single_arg                        %w{ expr }
+        multiple_args                     %w{ fun_args , expr }
+      end    
+
+      for_symbol 'fun_args_within_call' do
+        no_args_within_call               %w{ }
+        single_arg_within_call            %w{ expr }
+        multiple_args_within_call         %w{ fun_args_within_call , expr }
+      end
+
+      for_symbol 'fun_call_args' do
+        fun_args_within_call              %w{ fun_args_within_call }
+        fun_args_with_trailing_comma      %w{ fun_args_within_call , }
+      end    
+    end
+  end
+end
+
+
+
+my_filename       = "lib/ionize/parser.rb"
+compiled_filename = "lib/ionize/compiled_parser.rb"
+
+if File.stat(my_filename).ctime > File.stat(compiled_filename).ctime
+  puts "Grammar newer than CompiledParser. Recompiling.." 
+  null = Object.new.instance_eval do
+    def method_missing(*args, &blk); end
+    self
+  end
+  File.open("lib/ionize/compiled_parser.rb", "w+") do |f|     
+    f.write(Dhaka::Parser.new(Ionize::Php::Grammar, null).compile_to_ruby_source_as("Ionize::Php::CompiledParser")) 
+  end
+end
+