oracle_to_mysql 1.1.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.specification

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification 
+name: oracle_to_mysql
+version: !ruby/object:Gem::Version 
+  hash: 23
+  prerelease: false
+  segments: 
+  - 1
+  - 0
+  - 0
+  version: 1.0.0
+platform: ruby
+authors: 
+- Joe Goggins
+- Chris Dinger
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-10-29 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: thoughtbot-shoulda
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+description: Wraps the sqlplus binary and mysql binary does not currently require OCI8 or MySQL gems (might someday tho)
+email: joe.goggins@umn.edu
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/oracle_to_mysql.rb
+- lib/oracle_to_mysql/api_instance_methods.rb
+- lib/oracle_to_mysql/command.rb
+- lib/oracle_to_mysql/command/delete_temp_files.rb
+- lib/oracle_to_mysql/command/fork_and_execute_sqlplus_command.rb
+- lib/oracle_to_mysql/command/write_and_execute_mysql_commands_to_bash_file.rb
+- lib/oracle_to_mysql/command/write_and_execute_mysql_commands_to_bash_file_in_replace_mode.rb
+- lib/oracle_to_mysql/command/write_sqlplus_commands_to_file.rb
+- lib/oracle_to_mysql/must_override_instance_methods.rb
+- lib/oracle_to_mysql/optional_override_instance_methods.rb
+- lib/oracle_to_mysql/private_instance_methods.rb
+- lib/oracle_to_mysql/protected_class_methods.rb
+- test/demo/ps_term_tbl.rb
+- test/demo/ps_term_tbl_accumulative.rb
+- test/demo/test_oracle_to_mysql_against_ps_term_tbl.rb
+- test/helper.rb
+- test/oracle_to_mysql.example.yml
+- test/test_against_ps_term_tbl_accumulative.rb
+- test/test_oracle_to_mysql.rb
+has_rdoc: true
+homepage: http://github.com/joegoggins/oracle_to_mysql
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: A gem for mirroring data from an oracle db to a mysql db
+test_files: 
+- test/demo/ps_term_tbl.rb
+- test/demo/ps_term_tbl_accumulative.rb
+- test/demo/test_oracle_to_mysql_against_ps_term_tbl.rb
+- test/helper.rb
+- test/test_against_ps_term_tbl_accumulative.rb
+- test/test_oracle_to_mysql.rb

data/README.rdoc

@@ -0,0 +1,118 @@
+= OracleToMysql
+A wrapper for the `sqlplus` and `mysql` binary that pulls
+data out of an Oracle database and inserts it into MySQL.
+
+By omitting any Ruby/Mysql & Ruby/Oracle libraries things are kept 
+  fast, minimal, and debuggable.
+
+FYI, 1.0.0 is misleading, it's not some rock solid piece of uber-code.
+It's a weasel of a gem we just started using to replace our existing mirroring infrastructure
+in October 2010.
+
+== Use Case
+As a Ruby developer 
+with access to huge Oracle data warehouses
+I need a way to work with this data in my apps
+without using Oracle. ;)
+
+== System Dependencies
+popen4 (gem)
+sqlplus, and mysql binaries are in $PATH
+
+== Usage
+See the "test" cases in /test, there aren't really tests
+Just some code that uses the test/demo/* models to demonstrate 
+the two main mirror strategies associated with this tool (see below)
+
+For example:
+	class TheMagicMirrorer
+	  include OracleToMysql
+
+	  def otm_source_sql
+	    "select 
+	       col || CHR(9) ||
+	       col || CHR(9) ||
+	       ...
+	     from table
+	     where ... Oracle statement"
+	  end
+
+	  def otm_target_table
+	    "a_mysql_table_name"
+	  end
+	
+	  def otm_target_sql
+	    "create table if not exists #{self.otm_target_table} (... mysql statement   "
+	  end
+	end
+  
+  x = TheMagicMirrorer.new
+  x.otm_execute
+
+Will mirror the contents of otm_source_sql into the table created by otm_target_sql.
+The "|| CHR(9) ||" crap is Oracle sql code that tab deliminates the column content in
+the spooled sqlplus SQL data output.  The Mysql "load data infile" command eats this output.
+
+If you are using with Rails, it expects database.yml to have oracle_source and mysql_target entries to get the
+db connect info, to override the names, see OptionalOverrideInstanceMethods#otm_config_file and #otm_source_config_hash or #otm_target_config_hash
+
+Also, will need to do:
+  gem install POpen4
+
+=== Mirror Strategies
+  :atomic_rename (Default)
+    "load data infile" the spooled oracle tab deliminted data into a temp table first
+    then atomically rename 
+      current_target_table -> old_target_table AND
+      new_temp_table -> new_target_table
+    
+  :accumulative
+    "load data infile" directly into target_table replacing any existing
+    rows in target when source data triggers "ON DUPLICATE KEY"
+
+=== Target Table Retention
+For both strategies, if the target_table already exists the
+default is to keep the existing table around and suffix it with "_old".
+This is called a :n => 1 retention policy.  
+
+The ability to retain n previous tables in the works, handy for data mining, stay tuned.
+
+== Gem Development & Testing
+The "tests" aka demo's assume you have a ps_term_tbl in your Oracle db.
+If you're running PeopleSoft at a University you'll probably have this...
+Otherwise, the tests won't run, it's just meant as an example that works in our world.
+
+You'll need the thoughtbot-shoulda gem if you want to develop/hack on this gem or run the tests
+
+To run tests:
+cd test
+  ruby test_oracle_to_mysql_against_ps_term_tbl.rb
+OR
+  irb -r test_oracle_to_mysql_against_ps_term_tbl.rb
+  # And monkey with the run time...all files in test/demo are loaded, you can tinker with them
+  
+This assumes you have a connection file in the test dir:
+  oracle_to_mysql.yml
+copy and populate from oracle_to_mysql.example.yml
+
+== Note on Patches/Pull Requests
+* Fork the project.
+* Add files to test/demo/* that demonstrate how you are using the tool.
+* Bugfixes = fork, fix, commit, pull request.
+* New Features = let us know what yer thinkin, we might already be working on it
+
+== A few things we'd like to work on soon
+* retention policy of 0: don't keep yesterdays data (aka don't create *_old table in mysql)
+* retention policy > 1: keep N mysql table copies around
+* usage of a Logger object instead of stdout
+* Better configuration of what happens when the otm_execute fails, not sure...some options might include
+  * email someone a backtrace of the exception
+  * log the exception backtrace to a table in the db
+  * either cleanup + delete temp files or keep around (now it just leaves the temp files around)
+
+== Known Issues
+* Since source data is written to disk a tab delimintated file, if the source oracle data contains a \t character in might mess things up (none of our data has tabs so we haven't had problems)
+* Add validations/checks for stuff in validate_otm_source_sql in write_sqlplus_commands_to_file.rb if you encounter goofy sqlplus errors
+  Things that are not easily programmatically detectable ought just have an inline note i suppose
+
+

data/Rakefile

@@ -0,0 +1,55 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "oracle_to_mysql"
+    gem.summary = %Q{A gem for mirroring data from an oracle db to a mysql db}
+    gem.description = %Q{Wraps the sqlplus binary and mysql binary does not currently require OCI8 or MySQL gems (might someday tho)}
+    gem.email = "joe.goggins@umn.edu"
+    gem.homepage = "http://github.com/joegoggins/oracle_to_mysql"
+    gem.authors = ["Joe Goggins","Chris Dinger"]
+    
+    gem.add_dependency "POpen4", ">= 0"
+    gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "oracle_to_mysql #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+1.1.0

data/lib/oracle_to_mysql.rb

@@ -0,0 +1,80 @@
+# A super robust process spawning lib, install via
+#   gem install --remote --include-dependencies POpen4
+#   See http://rubygems.org/gems/POpen4
+#   and http://popen4.rubyforge.org/
+#
+require 'popen4'
+require 'oracle_to_mysql/protected_class_methods'
+require 'oracle_to_mysql/private_instance_methods'
+require 'oracle_to_mysql/must_override_instance_methods'
+require 'oracle_to_mysql/optional_override_instance_methods'
+require 'oracle_to_mysql/api_instance_methods'
+
+# These commands are used internally to actuall do the work
+require 'oracle_to_mysql/command'
+require 'oracle_to_mysql/command/write_sqlplus_commands_to_file'
+require 'oracle_to_mysql/command/fork_and_execute_sqlplus_command.rb'
+require 'oracle_to_mysql/command/write_and_execute_mysql_commands_to_bash_file.rb'
+require 'oracle_to_mysql/command/write_and_execute_mysql_commands_to_bash_file_in_replace_mode.rb'
+require 'oracle_to_mysql/command/delete_temp_files.rb'
+module OracleToMysql
+  class CommandError < Exception; end
+  # used to join the table and timestamp for old retained tables if :n > 1, or if :n = 1 its simply the suffix of the table name
+  OTM_RETAIN_KEY = '_old' 
+  OTM_VALID_STRATEGIES = [:accumulative, :atomic_rename]
+
+
+  # Stuff mixed into a client class calling include OracleToMysql
+  def self.included(caller)
+    caller.class_eval do
+      private   
+      include PrivateInstanceMethods          
+      
+      protected
+      extend  ProtectedClassMethods
+
+      # All customization will come by overriding methods in these classes
+      include OptionalOverrideInstanceMethods   # You might need to override one or two of these
+      include MustOverrideInstanceMethods       # THIS IS WHAT A CLIENT MUST OVERRIDE for .otm_execute to work
+      
+      public
+      include ApiInstanceMethods                # You shouldn't need to override these, probably better to override at a lower level
+    end
+  end
+  
+  # If you add to this, don't forget to add the :my_command_name to otm_execute_command_names (and the class definition file)
+  # and the require statement at the top of this class
+  #
+  def self.command_name_to_class_hash
+    return {
+      :write_sqlplus_commands_to_file => OracleToMysql::Command::WriterSqlplusCommandsToFile,
+      :fork_and_execute_sqlplus_commands_file => OracleToMysql::Command::ForkAndExecuteSqlplusCommand,
+      :write_and_execute_mysql_commands_to_bash_file => OracleToMysql::Command::WriteAndExecuteMysqlCommandsToBashFile,
+      :write_and_execute_mysql_commands_to_bash_file_in_replace_mode => OracleToMysql::Command::WriteAndExecuteMysqlCommandsToBashFileInReplaceMode,
+      # TODO pack-keys and optimize table
+      :delete_temp_files => OracleToMysql::Command::DeleteTempFiles
+    }      
+  end
+  
+  ####
+  # This is a very handy method for debugging a single Command bound to a particular client class
+  #
+  # x=OracleToMysql.get_and_bind_command(:write_sqlplus_commands_to_file, PsTermTbl.new)
+  # x.execute
+  #
+  # or can access any other helper methods on x
+  #
+  ###
+  def self.get_and_bind_command(command_name,client_class)
+    raise "invalid command name #{command_name}" unless self.command_name_to_class_hash.has_key?(command_name)
+    command = self.command_name_to_class_hash[command_name].new
+    command.client_class = client_class
+    return command
+  end
+  
+  # once again, sqlplus the jackass needs a "TNS string" to connect to it's stankin ass
+  # this interpolates crap from a config hash, like host and port and database into it
+  #
+  def self.tns_string_from_config(config_hash)      "(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=#{config_hash['host']})(PORT=#{config_hash['port']})))(CONNECT_DATA=(SERVICE_NAME=#{config_hash['database']})))"
+  end  
+end

data/lib/oracle_to_mysql/api_instance_methods.rb

@@ -0,0 +1,94 @@
+module OracleToMysql
+  module ApiInstanceMethods
+    def otm_execute
+      self.otm_output("[started at #{self.otm_timestamp}]")
+      self.otm_started("#otm_execute (in #{self.otm_strategy.to_s} mode, retain_n = #{self.otm_retain_options[:n]}")
+      self.otm_execute_command_names.each do |command_name|
+        command = OracleToMysql.get_and_bind_command(command_name, self)
+        begin
+          command.execute
+        rescue OracleToMysql::CommandError => e
+          puts "TODO: SHOULD EMAIL SOMEONE OR SOMETHING, THE BUILD FAILED"
+          raise e           
+        rescue Exception => e
+          puts "TODO: CUSTOM EXCEPTION HANDLING FOR A PARTICULAR COMMAND MIGHT HAPPEN HERE..."
+          raise e
+        end        
+      end
+      self.otm_finished("#otm_execute")
+      final_time_elapsed = self.otm_time_elapsed_since_otm_timestamp
+      finished_at = self.otm_timestamp + final_time_elapsed
+      
+      self.otm_output("[finished at #{finished_at}]")
+      self.otm_output("[completed in #{final_time_elapsed} seconds]")
+      self.otm_reset_timestamp
+      return true
+    end
+    
+    # This is used extensively by commands to read and write the the specific files
+    # that are shared accross commands, it is a bit protected-ish though (aka prolly don't want to override this)
+    #
+    def otm_get_file_name_for(sym)
+      f = self.generate_tmp_file_name(sym)
+      case sym
+      when :mysql_commands    
+        "#{f}.sql"
+      when :oracle_commands
+        "#{f}.sql"   # sqlplus is such a jackass, the file name must end in .sql for it to work, fyi
+      when :oracle_output
+        "#{f}.txt"
+      else
+        raise "Invalid file_name #{sym}"
+      end
+    end
+    
+    # For interaction with the configuration file that contains the mysql or oracle stuff
+    # override otm_config_file if you have yer config somewhere crazy
+    #
+    def otm_config_hash
+      if @otm_confile_file_hash.nil?
+        @otm_confile_file_hash = YAML.load(File.read(self.otm_config_file))
+      end
+      @otm_confile_file_hash      
+    end
+    
+    # all tables and file name are versioned against the seconds of this Time obj
+    #
+    def otm_timestamp
+      if @otm_timestamp.nil?
+        @otm_timestamp = Time.now
+      end
+      @otm_timestamp
+    end
+    
+    # You need to do this to invoke .otm_execute twice on the same inst, or it will clobber
+    def otm_reset_timestamp
+      @otm_timestamp = nil # reset it so all temp files have a new identifier, if otm_execute'd is executed again
+    end
+    
+    # Returns all temp files that all commands in the current strategy can create
+    #
+    def otm_all_temp_files
+      return_this = []
+      self.otm_execute_command_names.each do |command_name|
+        command = OracleToMysql.get_and_bind_command(command_name, self)
+        return_this += command.temp_file_symbols.map {|sym| self.otm_get_file_name_for(sym)}
+      end
+      return_this.uniq
+    end
+        
+    # returns an array of all target tables by reflecting the table retention options
+    #
+    def otm_all_target_tables
+      return_this = [self.otm_target_table]
+      if self.otm_retain_options[:n] == 0
+        return_this
+      else
+        (1..self.otm_retain_options[:n]).to_enum.each do |x|
+          return_this << self.otm_retained_target_table(x)
+        end
+        return_this
+      end
+    end
+  end
+end