oracle_to_mysql 1.1.0

data/lib/oracle_to_mysql/command.rb

@@ -0,0 +1,45 @@
+class OracleToMysql::Command
+  attr_accessor :client_class
+  def info(msg)
+    self.output("[info] #{msg}")
+  end
+  def error(msg)
+    self.output("[ERROR] #{msg}")
+    raise OracleToMysql::CommandError.new("#{self.client_class.to_s}#otm_execute died on command=#{self.class.to_s}")
+  end
+  def warn(msg)
+    self.output("[WARN] #{msg}")
+  end
+
+  def started(msg='')
+    self.output("[started t=#{self.client_class.otm_time_elapsed_since_otm_timestamp}]#{msg}")
+  end
+  
+  def finished(msg='')
+    self.output("[finished t=#{self.client_class.otm_time_elapsed_since_otm_timestamp}]#{msg}")
+  end
+
+  # USE THE ones above if possible, it makes the output more uniform and easier to read/parse
+  # Stuff funnels to here, which propogates up to the client class
+  def output(msg)
+    self.client_class.otm_output("[#{self.class.to_s}]#{msg}")
+  end
+     
+  # Client classes call this method     
+  def execute
+    self.started
+    self.execute_internal
+    self.finished
+  end
+  
+  # Commands should override this if they use temp files that need to be cleaned up
+  # The cleanup_temp_files_and_tables command reflects on all commands to find and delete things
+  # referenced in here
+  def temp_file_symbols
+    []
+  end
+  
+  def execute_internal
+    raise "CHILDREN MUST OVERRIDE THIS"
+  end
+end  

data/lib/oracle_to_mysql/command/delete_temp_files.rb

@@ -0,0 +1,16 @@
+module OracleToMysql
+  class Command
+    class DeleteTempFiles < OracleToMysql::Command              
+      def execute_internal
+        self.client_class.otm_all_temp_files.each do |temp_file|
+          begin
+            self.info("Deleting temp file, #{temp_file}")
+            File.unlink(temp_file)
+          rescue Errno::ENOENT => e
+            self.warn("Could not remove temp file #{temp_file}")
+          end                      
+        end
+      end
+    end
+  end
+end

data/lib/oracle_to_mysql/command/fork_and_execute_sqlplus_command.rb

@@ -0,0 +1,61 @@
+module OracleToMysql
+  class Command
+    class ForkAndExecuteSqlplusCommand < OracleToMysql::Command              
+      def command_line_invocation
+        username = self.client_class.otm_source_config_hash['username']
+        password = self.client_class.otm_source_config_hash['password']
+        tns = OracleToMysql.tns_string_from_config(self.client_class.otm_source_config_hash)   #self.client_class.otm_source_config_hash['tns']
+        "sqlplus -S '#{username}/#{password}@#{tns}' @#{self.client_class.otm_get_file_name_for(:oracle_commands)}"      
+      end
+
+      # overridden from parent to help CleanupTempFilesAndTables do it's job
+      # 
+      def temp_file_symbols
+        [:oracle_output]
+      end
+      
+      def execute_internal
+        stderr_str = ""
+        stdout_str = ""     
+        the_command = self.command_line_invocation
+        self.started("sqlplus child being spawned")
+        unless
+          Open4::popen4(the_command) { |pid, stdin, stdout, stderr|
+            stderr_str = stderr.read
+            stdout_str = stdout.read
+            true
+          }
+          self.error("Could not execute #{the_command}")
+          # raise "[#{self.to_s}][create_#{deriv_type.to_s}_with_convert][MBF##{mbf.id.to_s}] Couldn't execute #{convert_cmd}"
+        end
+        self.finished("sqlplus process terminated")
+
+        if stderr_str.length > 0
+          self.error("sqlplus had stderr output: #{stderr_str}")
+        end
+        if stdout_str.length > 0
+          self.warn("sqlplus had stdout output: #{stdout_str}")
+        end
+
+        if File.exists?(self.client_class.otm_get_file_name_for(:oracle_output))
+          spooled_file_size = File.size(self.client_class.otm_get_file_name_for(:oracle_output))
+          sqlplus_file_size = File.size(self.client_class.otm_get_file_name_for(:oracle_commands))
+
+          # This sqlplus file will have the oracle message
+          if spooled_file_size > sqlplus_file_size
+            self.info("sqlplus spooled #{spooled_file_size} bytes of output to #{self.client_class.otm_get_file_name_for(:oracle_output)}")
+          else
+            source_output_contents = IO.read(self.client_class.otm_get_file_name_for(:oracle_output))          
+            self.warn("tiny data output: smaller than the sqlplus commands file, #{spooled_file_size} bytes, it might contains errors rather than data, I will check for Oracle error strings in #{self.client_class.otm_get_file_name_for(:oracle_output)} and die if so")
+            if source_output_contents.match(/^ERROR at line/) && source_output_contents.match(/^ORA-/)
+              self.error("sqlplus error: #{self.client_class.otm_get_file_name_for(:oracle_output)} contains both \"ERROR at line\" and \"ORA-\" in it, check it out, contents=#{source_output_contents}")            
+            end          
+          end
+        else
+          self.error("#{self.client_class.otm_get_file_name_for(:oracle_output)} does not exist")
+        end
+        self.info("sqlplus returned successfully")      
+      end
+    end
+  end
+end

data/lib/oracle_to_mysql/command/write_and_execute_mysql_commands_to_bash_file.rb

@@ -0,0 +1,136 @@
+module OracleToMysql
+  class Command
+    class WriteAndExecuteMysqlCommandsToBashFile < OracleToMysql::Command
+
+      def tables_to_retain
+        self.client_class.otm_retain_options[:n]
+      end
+      
+      # Done this way so the child, _in_replace_mode, can override this
+      #
+      def mysql_command_order
+        [:execute_otm_target_sql,
+         :execute_temp_file_modded_otm_target_sql,
+         :load_data_infile,        
+         :drop_expired_retained_tables,
+         :perform_atomic_rename_of_tables
+        ]
+      end
+      
+########## MYSQL queryies/commands BEGIN
+
+      def execute_otm_target_sql
+        self.client_class.otm_target_sql
+      end
+
+      # The client class should not have to know about otm_temp_target_table
+      # this overrides it just for this instance so the otm_target_sql
+      # gets the temp table name interpolated into it instead of the actual
+      # this is done this way to support the non-atomic rename strategy as well (which does not create a temp table)
+      #
+      def execute_temp_file_modded_otm_target_sql      
+        the_modded_client_class_inst = self.client_class.clone
+        the_modded_client_class_inst.instance_eval(<<-EOS, __FILE__,__LINE__)
+          def otm_target_table
+            "#{self.client_class.otm_temp_target_table}"
+          end
+        EOS
+        the_modded_client_class_inst.otm_target_sql
+      end
+      
+      def load_data_infile
+        "-- Rip through the oracle output data and insert into mysql
+         load data local infile '#{self.client_class.otm_get_file_name_for(:oracle_output)}' 
+         into table #{self.client_class.otm_temp_target_table}
+         fields terminated by '\\t'
+         lines terminated by '\\n'
+        "
+      end      
+      
+      def create_actual_target_table_if_it_doesnt_exist        
+        # If this is the first run, the destination table won't exist yet.  If that's the 
+        # case, create an empty table so the atomic rename works
+        "create table if not exists #{self.client_class.otm_target_table} 
+         select * from #{self.client_class.otm_temp_target_table} where 1=0"
+      end
+      
+      def drop_expired_retained_tables
+        raise "TODO: not implemented yet for retention :n != 1" if tables_to_retain != 1
+         "drop table if exists #{self.client_class.otm_retained_target_table(tables_to_retain)}"
+      end
+      
+      def perform_atomic_rename_of_tables
+        raise "TODO: not implemented yet for retention :n != 1" if tables_to_retain != 1
+        "RENAME table 
+          #{self.client_class.otm_target_table} TO #{self.client_class.otm_retained_target_table(tables_to_retain)}, 
+          #{self.client_class.otm_temp_target_table} TO #{self.client_class.otm_target_table}"
+        # rename table #{self.otm_retained_target_table}
+        # 
+        # 
+        # #{self.client_class.otm_target_table} to #{self.client_class.otm_target_table}_old, #{self.client_class.otm_temp_target_table} to #{self.client_class.otm_target_table};
+        # 
+      end
+      
+      
+########## MYSQL queryies/commands END
+
+      # -- Reflect the table retain options
+      # #{drop_expired_retained_tables_sql};
+
+      def the_mysql_commands
+        the_queries = ''
+        self.mysql_command_order.each do |mysql_command|
+          the_queries << "-- #{mysql_command.to_s}\n" # to make debugging easier
+          the_queries << self.send(mysql_command)
+          the_queries << ";\n"
+        end
+        the_queries    
+      end
+
+      def command_line_invocation
+        h = self.client_class.otm_target_config_hash
+        "mysql -u'#{h['username']}' -h'#{h['host']}' -p'#{h['password']}' #{h['database']} < '#{self.client_class.otm_get_file_name_for(:mysql_commands)}'"
+      end
+      
+      # overridden from parent to help CleanupTempFilesAndTables do it's job
+      # 
+      def temp_file_symbols
+        [:mysql_commands]
+      end
+      
+      def execute_internal
+        bytes_written = 0
+        the_mysql_commands_file = self.client_class.otm_get_file_name_for(:mysql_commands)
+        File.open(the_mysql_commands_file,'w') do |f|
+          bytes_written = f.write(self.the_mysql_commands)
+        end
+        if bytes_written > 0
+          self.info("#{bytes_written} bytes written to #{the_mysql_commands_file}")
+        else
+          self.error("Could not write to #{the_mysql_commands_file}")
+        end
+
+        stderr_str = ""
+        stdout_str = ""     
+        the_command = self.command_line_invocation
+        self.started("mysql child being spawned")
+        unless
+         Open4::popen4(the_command) { |pid, stdin, stdout, stderr|
+           stderr_str = stderr.read
+           stdout_str = stdout.read
+           true
+         }
+         self.error("Could not execute #{the_command}")
+        end
+        self.finished("mysql child terminated")
+
+        if stderr_str.length > 0
+          self.error("mysql child process had stderr output: #{stderr_str}")
+        end
+        if stdout_str.length > 0
+          self.warn("mysql child process: #{stdout_str}")
+        end      
+      end
+    end      
+  end
+end

data/lib/oracle_to_mysql/command/write_and_execute_mysql_commands_to_bash_file_in_replace_mode.rb

@@ -0,0 +1,51 @@
+module OracleToMysql
+  class Command
+    class WriteAndExecuteMysqlCommandsToBashFileInReplaceMode < OracleToMysql::Command::WriteAndExecuteMysqlCommandsToBashFile
+      
+      # OVERRIDDEN 
+      def mysql_command_order        
+        [:execute_otm_target_sql,
+         :drop_expired_retained_tables,   # defined in parent
+         :retention_policy_create_and_populate_tables,
+         :load_data_infile                # overridden in this class
+        ]
+      end
+      
+### MYSQL COMMANDS BEGIN      
+            
+      def retention_policy_create_and_populate_tables
+        raise "TODO: not implemented yet for retention :n != 1" if self.tables_to_retain != 1
+        the_modded_client_class_inst = self.client_class.clone
+        the_modded_client_class_inst.instance_eval(<<-EOS, __FILE__,__LINE__)
+          def otm_target_table
+            "#{self.client_class.otm_retained_target_table(tables_to_retain)}"
+          end
+        EOS
+        
+        return_this = ''
+        return_this << the_modded_client_class_inst.otm_target_sql
+        return_this << ";\n"
+        return_this << "-- Insert existing rows into the retention table\n"
+        return_this << "INSERT INTO #{the_modded_client_class_inst.otm_target_table} SELECT * FROM #{self.client_class.otm_target_table}\n"
+        return_this
+      end
+            
+      # Mostly the same as parent class except:
+      #   * the replace key word which will replace existing rows on the target when there are mysql duplicate key errors
+      #     possible todo: support "ignore" instead of "replace" (i have no reason to do this)
+      #   * it copied into the otm_target_table directly rather than the temp table
+      #  
+      def load_data_infile
+        "-- Rip through the oracle output data and insert into mysql
+         load data local infile '#{self.client_class.otm_get_file_name_for(:oracle_output)}' 
+         replace
+         into table #{self.client_class.otm_target_table}         
+         fields terminated by '\\t'
+         lines terminated by '\\n'
+        "
+      end
+
+### MYSQL COMMANDS END      
+    end
+  end
+end

data/lib/oracle_to_mysql/command/write_sqlplus_commands_to_file.rb

@@ -0,0 +1,64 @@
+module OracleToMysql
+  class Command
+    class WriterSqlplusCommandsToFile < OracleToMysql::Command
+      class MaxLineExceeded < Exception; end
+      # Oracle and sqlplus is so finicky, this captures some of the errors I've come across,
+      # it is not exhuastive
+      def validate_otm_source_sql
+        the_source_sql = self.client_class.otm_source_sql
+
+        # last character minus white space must be a ";"
+        raise "otm_source_sql not terminated with a ';'" if the_source_sql.strip[-1..-1] != ';'
+  
+        # Detect and prevent sqlplus from SP2-0027: Input is too long (> 2499 characters) - line ignored
+        the_source_sql.each_line {|x| raise MaxLineExceeded.new("Ruby detection and prevention of Oracle error SP2-0027: Input is too long (> 2499 characters) - line ignored") if x.length > 2499}
+        
+        # other sqlplus CAVEATS (that could be programmed, TODO-ish)
+        # If you're "IN" statements has more than 1000 elements, Oracle will barf out a ORA-01795 error
+        # 
+      end
+      
+      
+      def string_to_write_commands_file_name
+        self.validate_otm_source_sql
+        the_file_to_spool_output_to = self.client_class.otm_get_file_name_for(:oracle_output)
+        return "
+            WHENEVER SQLERROR EXIT 2;
+            spool on;
+            set heading off
+            set echo off
+            set verify off
+            set termout off
+            SET NEWPAGE 0;
+            SET SPACE 0;
+            SET PAGESIZE 0;
+            SET FEEDBACK OFF;
+            SET TRIMSPOOL ON;
+            SET TAB OFF;
+            set linesize 2000;
+            spool #{the_file_to_spool_output_to}
+            #{self.client_class.otm_source_sql}
+            spool off;
+            exit;          
+            "      
+      end
+      # overridden from parent to help CleanupTempFilesAndTables do it's job
+      # 
+      def temp_file_symbols
+        [:oracle_commands]
+      end
+      
+      def execute_internal
+        bytes_written = 0      
+        File.open(self.client_class.otm_get_file_name_for(:oracle_commands),'w') do |f|        
+          bytes_written = f.write(self.string_to_write_commands_file_name)
+        end
+        if bytes_written > 0
+          self.info("#{bytes_written} bytes written to #{self.client_class.otm_get_file_name_for(:oracle_commands)}")
+        else
+          self.error("could not write to #{self.client_class.otm_get_file_name_for(:oracle_commands)}, bytes_written=#{bytes_written}")
+        end
+      end
+    end         
+  end
+end

data/lib/oracle_to_mysql/must_override_instance_methods.rb

@@ -0,0 +1,16 @@
+module OracleToMysql
+  module MustOverrideInstanceMethods    
+    
+    
+    #### MUST BE OVERRIDEN    
+    def otm_source_sql
+      raise "YOU MUST OVERRIDE THIS"
+    end
+    def otm_target_sql
+      raise "YOU MUST OVERRIDE THIS"
+    end
+    def otm_target_table
+      raise "YOU MUST OVERRIDE THIS"
+    end  
+  end
+end

data/lib/oracle_to_mysql/optional_override_instance_methods.rb

@@ -0,0 +1,65 @@
+module OracleToMysql
+  module OptionalOverrideInstanceMethods    
+    
+    # YOU WILL OFTEN WANT TO OVERRIDE THIS, 
+    def otm_strategy
+      if @otm_strategy.nil?
+        @otm_strategy = self.class.otm_default_strategy
+      end
+      @otm_strategy      
+    end
+              
+    # TO CHANGE the retain options, override this method
+    #    
+    def otm_retain_options
+      if @otm_retain_options.nil?
+        @otm_retain_options = self.class.otm_default_retain_options
+      end
+      @otm_retain_options
+    end
+    
+    def otm_source_config_hash
+      if self.otm_config_hash.has_key?('oracle_source')
+        return otm_config_hash['oracle_source']
+      else
+        raise "Could not find oracle_source key in config file #{self.otm_config_file}, you should override this method"
+      end
+    end
+    def otm_target_config_hash
+      if self.otm_config_hash.has_key?('mysql_target')
+        return otm_config_hash['mysql_target']
+      else
+        raise "Could not find mysql_target key in config file #{self.otm_config_file}, you should override this method"
+      end      
+    end
+    
+    # Override this if you have zany rules about where your config file is, there is a handy otm_config_hash method in the ApiInstanceMethods
+    # you can use to get stuff outta here
+    #
+    def otm_config_file
+      non_rails_path = File.join(Dir.pwd, 'oracle_to_mysql.yml')
+      if defined?(RAILS_ROOT)
+        rails_path = File.join(RAILS_ROOT,'config','database.yml')
+        if rails_path
+          @otm_db_config_yml_file_name = rails_path
+        else
+          raise "Weird, RAILS_ROOT detected but no databases.yml, something is amiss!"
+        end
+      elsif File.exists?(non_rails_path)
+        @otm_db_config_yml_file_name = non_rails_path
+      else
+        raise "ERROR: No otm config file found"
+      end
+    end  
+    
+    # This is the table name that is created first on the server for the :atomic_rename strategy
+    #
+    def otm_temp_target_table
+      "#{self.otm_target_table}_#{self.otm_timestamp.to_i}_temp"
+    end
+    
+    def tmp_directory
+      "/tmp"  
+    end                                       
+  end
+end