oracle_to_mysql 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,5 @@
1
+ README.rdoc
2
+ lib/**/*.rb
3
+ bin/*
4
+ features/**/*.feature
5
+ LICENSE
@@ -0,0 +1,111 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: oracle_to_mysql
3
+ version: !ruby/object:Gem::Version
4
+ hash: 23
5
+ prerelease: false
6
+ segments:
7
+ - 1
8
+ - 0
9
+ - 0
10
+ version: 1.0.0
11
+ platform: ruby
12
+ authors:
13
+ - Joe Goggins
14
+ - Chris Dinger
15
+ autorequire:
16
+ bindir: bin
17
+ cert_chain: []
18
+
19
+ date: 2010-10-29 00:00:00 -05:00
20
+ default_executable:
21
+ dependencies:
22
+ - !ruby/object:Gem::Dependency
23
+ name: thoughtbot-shoulda
24
+ prerelease: false
25
+ requirement: &id001 !ruby/object:Gem::Requirement
26
+ none: false
27
+ requirements:
28
+ - - ">="
29
+ - !ruby/object:Gem::Version
30
+ hash: 3
31
+ segments:
32
+ - 0
33
+ version: "0"
34
+ type: :development
35
+ version_requirements: *id001
36
+ description: Wraps the sqlplus binary and mysql binary does not currently require OCI8 or MySQL gems (might someday tho)
37
+ email: joe.goggins@umn.edu
38
+ executables: []
39
+
40
+ extensions: []
41
+
42
+ extra_rdoc_files:
43
+ - LICENSE
44
+ - README.rdoc
45
+ files:
46
+ - .document
47
+ - .gitignore
48
+ - LICENSE
49
+ - README.rdoc
50
+ - Rakefile
51
+ - VERSION
52
+ - lib/oracle_to_mysql.rb
53
+ - lib/oracle_to_mysql/api_instance_methods.rb
54
+ - lib/oracle_to_mysql/command.rb
55
+ - lib/oracle_to_mysql/command/delete_temp_files.rb
56
+ - lib/oracle_to_mysql/command/fork_and_execute_sqlplus_command.rb
57
+ - lib/oracle_to_mysql/command/write_and_execute_mysql_commands_to_bash_file.rb
58
+ - lib/oracle_to_mysql/command/write_and_execute_mysql_commands_to_bash_file_in_replace_mode.rb
59
+ - lib/oracle_to_mysql/command/write_sqlplus_commands_to_file.rb
60
+ - lib/oracle_to_mysql/must_override_instance_methods.rb
61
+ - lib/oracle_to_mysql/optional_override_instance_methods.rb
62
+ - lib/oracle_to_mysql/private_instance_methods.rb
63
+ - lib/oracle_to_mysql/protected_class_methods.rb
64
+ - test/demo/ps_term_tbl.rb
65
+ - test/demo/ps_term_tbl_accumulative.rb
66
+ - test/demo/test_oracle_to_mysql_against_ps_term_tbl.rb
67
+ - test/helper.rb
68
+ - test/oracle_to_mysql.example.yml
69
+ - test/test_against_ps_term_tbl_accumulative.rb
70
+ - test/test_oracle_to_mysql.rb
71
+ has_rdoc: true
72
+ homepage: http://github.com/joegoggins/oracle_to_mysql
73
+ licenses: []
74
+
75
+ post_install_message:
76
+ rdoc_options:
77
+ - --charset=UTF-8
78
+ require_paths:
79
+ - lib
80
+ required_ruby_version: !ruby/object:Gem::Requirement
81
+ none: false
82
+ requirements:
83
+ - - ">="
84
+ - !ruby/object:Gem::Version
85
+ hash: 3
86
+ segments:
87
+ - 0
88
+ version: "0"
89
+ required_rubygems_version: !ruby/object:Gem::Requirement
90
+ none: false
91
+ requirements:
92
+ - - ">="
93
+ - !ruby/object:Gem::Version
94
+ hash: 3
95
+ segments:
96
+ - 0
97
+ version: "0"
98
+ requirements: []
99
+
100
+ rubyforge_project:
101
+ rubygems_version: 1.3.7
102
+ signing_key:
103
+ specification_version: 3
104
+ summary: A gem for mirroring data from an oracle db to a mysql db
105
+ test_files:
106
+ - test/demo/ps_term_tbl.rb
107
+ - test/demo/ps_term_tbl_accumulative.rb
108
+ - test/demo/test_oracle_to_mysql_against_ps_term_tbl.rb
109
+ - test/helper.rb
110
+ - test/test_against_ps_term_tbl_accumulative.rb
111
+ - test/test_oracle_to_mysql.rb
@@ -0,0 +1,118 @@
1
+ = OracleToMysql
2
+ A wrapper for the `sqlplus` and `mysql` binary that pulls
3
+ data out of an Oracle database and inserts it into MySQL.
4
+
5
+ By omitting any Ruby/Mysql & Ruby/Oracle libraries things are kept
6
+ fast, minimal, and debuggable.
7
+
8
+ FYI, 1.0.0 is misleading, it's not some rock solid piece of uber-code.
9
+ It's a weasel of a gem we just started using to replace our existing mirroring infrastructure
10
+ in October 2010.
11
+
12
+ == Use Case
13
+ As a Ruby developer
14
+ with access to huge Oracle data warehouses
15
+ I need a way to work with this data in my apps
16
+ without using Oracle. ;)
17
+
18
+ == System Dependencies
19
+ popen4 (gem)
20
+ sqlplus, and mysql binaries are in $PATH
21
+
22
+ == Usage
23
+ See the "test" cases in /test, there aren't really tests
24
+ Just some code that uses the test/demo/* models to demonstrate
25
+ the two main mirror strategies associated with this tool (see below)
26
+
27
+ For example:
28
+ class TheMagicMirrorer
29
+ include OracleToMysql
30
+
31
+ def otm_source_sql
32
+ "select
33
+ col || CHR(9) ||
34
+ col || CHR(9) ||
35
+ ...
36
+ from table
37
+ where ... Oracle statement"
38
+ end
39
+
40
+ def otm_target_table
41
+ "a_mysql_table_name"
42
+ end
43
+
44
+ def otm_target_sql
45
+ "create table if not exists #{self.otm_target_table} (... mysql statement "
46
+ end
47
+ end
48
+
49
+ x = TheMagicMirrorer.new
50
+ x.otm_execute
51
+
52
+ Will mirror the contents of otm_source_sql into the table created by otm_target_sql.
53
+ The "|| CHR(9) ||" crap is Oracle sql code that tab deliminates the column content in
54
+ the spooled sqlplus SQL data output. The Mysql "load data infile" command eats this output.
55
+
56
+ If you are using with Rails, it expects database.yml to have oracle_source and mysql_target entries to get the
57
+ db connect info, to override the names, see OptionalOverrideInstanceMethods#otm_config_file and #otm_source_config_hash or #otm_target_config_hash
58
+
59
+ Also, will need to do:
60
+ gem install POpen4
61
+
62
+ === Mirror Strategies
63
+ :atomic_rename (Default)
64
+ "load data infile" the spooled oracle tab deliminted data into a temp table first
65
+ then atomically rename
66
+ current_target_table -> old_target_table AND
67
+ new_temp_table -> new_target_table
68
+
69
+ :accumulative
70
+ "load data infile" directly into target_table replacing any existing
71
+ rows in target when source data triggers "ON DUPLICATE KEY"
72
+
73
+ === Target Table Retention
74
+ For both strategies, if the target_table already exists the
75
+ default is to keep the existing table around and suffix it with "_old".
76
+ This is called a :n => 1 retention policy.
77
+
78
+ The ability to retain n previous tables in the works, handy for data mining, stay tuned.
79
+
80
+ == Gem Development & Testing
81
+ The "tests" aka demo's assume you have a ps_term_tbl in your Oracle db.
82
+ If you're running PeopleSoft at a University you'll probably have this...
83
+ Otherwise, the tests won't run, it's just meant as an example that works in our world.
84
+
85
+ You'll need the thoughtbot-shoulda gem if you want to develop/hack on this gem or run the tests
86
+
87
+ To run tests:
88
+ cd test
89
+ ruby test_oracle_to_mysql_against_ps_term_tbl.rb
90
+ OR
91
+ irb -r test_oracle_to_mysql_against_ps_term_tbl.rb
92
+ # And monkey with the run time...all files in test/demo are loaded, you can tinker with them
93
+
94
+ This assumes you have a connection file in the test dir:
95
+ oracle_to_mysql.yml
96
+ copy and populate from oracle_to_mysql.example.yml
97
+
98
+ == Note on Patches/Pull Requests
99
+ * Fork the project.
100
+ * Add files to test/demo/* that demonstrate how you are using the tool.
101
+ * Bugfixes = fork, fix, commit, pull request.
102
+ * New Features = let us know what yer thinkin, we might already be working on it
103
+
104
+ == A few things we'd like to work on soon
105
+ * retention policy of 0: don't keep yesterdays data (aka don't create *_old table in mysql)
106
+ * retention policy > 1: keep N mysql table copies around
107
+ * usage of a Logger object instead of stdout
108
+ * Better configuration of what happens when the otm_execute fails, not sure...some options might include
109
+ * email someone a backtrace of the exception
110
+ * log the exception backtrace to a table in the db
111
+ * either cleanup + delete temp files or keep around (now it just leaves the temp files around)
112
+
113
+ == Known Issues
114
+ * Since source data is written to disk a tab delimintated file, if the source oracle data contains a \t character in might mess things up (none of our data has tabs so we haven't had problems)
115
+ * Add validations/checks for stuff in validate_otm_source_sql in write_sqlplus_commands_to_file.rb if you encounter goofy sqlplus errors
116
+ Things that are not easily programmatically detectable ought just have an inline note i suppose
117
+
118
+
@@ -0,0 +1,55 @@
1
+ require 'rubygems'
2
+ require 'rake'
3
+
4
+ begin
5
+ require 'jeweler'
6
+ Jeweler::Tasks.new do |gem|
7
+ gem.name = "oracle_to_mysql"
8
+ gem.summary = %Q{A gem for mirroring data from an oracle db to a mysql db}
9
+ gem.description = %Q{Wraps the sqlplus binary and mysql binary does not currently require OCI8 or MySQL gems (might someday tho)}
10
+ gem.email = "joe.goggins@umn.edu"
11
+ gem.homepage = "http://github.com/joegoggins/oracle_to_mysql"
12
+ gem.authors = ["Joe Goggins","Chris Dinger"]
13
+
14
+ gem.add_dependency "POpen4", ">= 0"
15
+ gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
16
+ # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
17
+ end
18
+ Jeweler::GemcutterTasks.new
19
+ rescue LoadError
20
+ puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
21
+ end
22
+
23
+ require 'rake/testtask'
24
+ Rake::TestTask.new(:test) do |test|
25
+ test.libs << 'lib' << 'test'
26
+ test.pattern = 'test/**/test_*.rb'
27
+ test.verbose = true
28
+ end
29
+
30
+ begin
31
+ require 'rcov/rcovtask'
32
+ Rcov::RcovTask.new do |test|
33
+ test.libs << 'test'
34
+ test.pattern = 'test/**/test_*.rb'
35
+ test.verbose = true
36
+ end
37
+ rescue LoadError
38
+ task :rcov do
39
+ abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
40
+ end
41
+ end
42
+
43
+ task :test => :check_dependencies
44
+
45
+ task :default => :test
46
+
47
+ require 'rake/rdoctask'
48
+ Rake::RDocTask.new do |rdoc|
49
+ version = File.exist?('VERSION') ? File.read('VERSION') : ""
50
+
51
+ rdoc.rdoc_dir = 'rdoc'
52
+ rdoc.title = "oracle_to_mysql #{version}"
53
+ rdoc.rdoc_files.include('README*')
54
+ rdoc.rdoc_files.include('lib/**/*.rb')
55
+ end
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 1.1.0
@@ -0,0 +1,80 @@
1
+ # A super robust process spawning lib, install via
2
+ # gem install --remote --include-dependencies POpen4
3
+ # See http://rubygems.org/gems/POpen4
4
+ # and http://popen4.rubyforge.org/
5
+ #
6
+ require 'popen4'
7
+ require 'oracle_to_mysql/protected_class_methods'
8
+ require 'oracle_to_mysql/private_instance_methods'
9
+ require 'oracle_to_mysql/must_override_instance_methods'
10
+ require 'oracle_to_mysql/optional_override_instance_methods'
11
+ require 'oracle_to_mysql/api_instance_methods'
12
+
13
+ # These commands are used internally to actuall do the work
14
+ require 'oracle_to_mysql/command'
15
+ require 'oracle_to_mysql/command/write_sqlplus_commands_to_file'
16
+ require 'oracle_to_mysql/command/fork_and_execute_sqlplus_command.rb'
17
+ require 'oracle_to_mysql/command/write_and_execute_mysql_commands_to_bash_file.rb'
18
+ require 'oracle_to_mysql/command/write_and_execute_mysql_commands_to_bash_file_in_replace_mode.rb'
19
+ require 'oracle_to_mysql/command/delete_temp_files.rb'
20
+ module OracleToMysql
21
+ class CommandError < Exception; end
22
+ # used to join the table and timestamp for old retained tables if :n > 1, or if :n = 1 its simply the suffix of the table name
23
+ OTM_RETAIN_KEY = '_old'
24
+ OTM_VALID_STRATEGIES = [:accumulative, :atomic_rename]
25
+
26
+
27
+ # Stuff mixed into a client class calling include OracleToMysql
28
+ def self.included(caller)
29
+ caller.class_eval do
30
+ private
31
+ include PrivateInstanceMethods
32
+
33
+ protected
34
+ extend ProtectedClassMethods
35
+
36
+ # All customization will come by overriding methods in these classes
37
+ include OptionalOverrideInstanceMethods # You might need to override one or two of these
38
+ include MustOverrideInstanceMethods # THIS IS WHAT A CLIENT MUST OVERRIDE for .otm_execute to work
39
+
40
+ public
41
+ include ApiInstanceMethods # You shouldn't need to override these, probably better to override at a lower level
42
+ end
43
+ end
44
+
45
+ # If you add to this, don't forget to add the :my_command_name to otm_execute_command_names (and the class definition file)
46
+ # and the require statement at the top of this class
47
+ #
48
+ def self.command_name_to_class_hash
49
+ return {
50
+ :write_sqlplus_commands_to_file => OracleToMysql::Command::WriterSqlplusCommandsToFile,
51
+ :fork_and_execute_sqlplus_commands_file => OracleToMysql::Command::ForkAndExecuteSqlplusCommand,
52
+ :write_and_execute_mysql_commands_to_bash_file => OracleToMysql::Command::WriteAndExecuteMysqlCommandsToBashFile,
53
+ :write_and_execute_mysql_commands_to_bash_file_in_replace_mode => OracleToMysql::Command::WriteAndExecuteMysqlCommandsToBashFileInReplaceMode,
54
+ # TODO pack-keys and optimize table
55
+ :delete_temp_files => OracleToMysql::Command::DeleteTempFiles
56
+ }
57
+ end
58
+
59
+ ####
60
+ # This is a very handy method for debugging a single Command bound to a particular client class
61
+ #
62
+ # x=OracleToMysql.get_and_bind_command(:write_sqlplus_commands_to_file, PsTermTbl.new)
63
+ # x.execute
64
+ #
65
+ # or can access any other helper methods on x
66
+ #
67
+ ###
68
+ def self.get_and_bind_command(command_name,client_class)
69
+ raise "invalid command name #{command_name}" unless self.command_name_to_class_hash.has_key?(command_name)
70
+ command = self.command_name_to_class_hash[command_name].new
71
+ command.client_class = client_class
72
+ return command
73
+ end
74
+
75
+ # once again, sqlplus the jackass needs a "TNS string" to connect to it's stankin ass
76
+ # this interpolates crap from a config hash, like host and port and database into it
77
+ #
78
+ def self.tns_string_from_config(config_hash) "(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=#{config_hash['host']})(PORT=#{config_hash['port']})))(CONNECT_DATA=(SERVICE_NAME=#{config_hash['database']})))"
79
+ end
80
+ end
@@ -0,0 +1,94 @@
1
+ module OracleToMysql
2
+ module ApiInstanceMethods
3
+ def otm_execute
4
+ self.otm_output("[started at #{self.otm_timestamp}]")
5
+ self.otm_started("#otm_execute (in #{self.otm_strategy.to_s} mode, retain_n = #{self.otm_retain_options[:n]}")
6
+ self.otm_execute_command_names.each do |command_name|
7
+ command = OracleToMysql.get_and_bind_command(command_name, self)
8
+ begin
9
+ command.execute
10
+ rescue OracleToMysql::CommandError => e
11
+ puts "TODO: SHOULD EMAIL SOMEONE OR SOMETHING, THE BUILD FAILED"
12
+ raise e
13
+ rescue Exception => e
14
+ puts "TODO: CUSTOM EXCEPTION HANDLING FOR A PARTICULAR COMMAND MIGHT HAPPEN HERE..."
15
+ raise e
16
+ end
17
+ end
18
+ self.otm_finished("#otm_execute")
19
+ final_time_elapsed = self.otm_time_elapsed_since_otm_timestamp
20
+ finished_at = self.otm_timestamp + final_time_elapsed
21
+
22
+ self.otm_output("[finished at #{finished_at}]")
23
+ self.otm_output("[completed in #{final_time_elapsed} seconds]")
24
+ self.otm_reset_timestamp
25
+ return true
26
+ end
27
+
28
+ # This is used extensively by commands to read and write the the specific files
29
+ # that are shared accross commands, it is a bit protected-ish though (aka prolly don't want to override this)
30
+ #
31
+ def otm_get_file_name_for(sym)
32
+ f = self.generate_tmp_file_name(sym)
33
+ case sym
34
+ when :mysql_commands
35
+ "#{f}.sql"
36
+ when :oracle_commands
37
+ "#{f}.sql" # sqlplus is such a jackass, the file name must end in .sql for it to work, fyi
38
+ when :oracle_output
39
+ "#{f}.txt"
40
+ else
41
+ raise "Invalid file_name #{sym}"
42
+ end
43
+ end
44
+
45
+ # For interaction with the configuration file that contains the mysql or oracle stuff
46
+ # override otm_config_file if you have yer config somewhere crazy
47
+ #
48
+ def otm_config_hash
49
+ if @otm_confile_file_hash.nil?
50
+ @otm_confile_file_hash = YAML.load(File.read(self.otm_config_file))
51
+ end
52
+ @otm_confile_file_hash
53
+ end
54
+
55
+ # all tables and file name are versioned against the seconds of this Time obj
56
+ #
57
+ def otm_timestamp
58
+ if @otm_timestamp.nil?
59
+ @otm_timestamp = Time.now
60
+ end
61
+ @otm_timestamp
62
+ end
63
+
64
+ # You need to do this to invoke .otm_execute twice on the same inst, or it will clobber
65
+ def otm_reset_timestamp
66
+ @otm_timestamp = nil # reset it so all temp files have a new identifier, if otm_execute'd is executed again
67
+ end
68
+
69
+ # Returns all temp files that all commands in the current strategy can create
70
+ #
71
+ def otm_all_temp_files
72
+ return_this = []
73
+ self.otm_execute_command_names.each do |command_name|
74
+ command = OracleToMysql.get_and_bind_command(command_name, self)
75
+ return_this += command.temp_file_symbols.map {|sym| self.otm_get_file_name_for(sym)}
76
+ end
77
+ return_this.uniq
78
+ end
79
+
80
+ # returns an array of all target tables by reflecting the table retention options
81
+ #
82
+ def otm_all_target_tables
83
+ return_this = [self.otm_target_table]
84
+ if self.otm_retain_options[:n] == 0
85
+ return_this
86
+ else
87
+ (1..self.otm_retain_options[:n]).to_enum.each do |x|
88
+ return_this << self.otm_retained_target_table(x)
89
+ end
90
+ return_this
91
+ end
92
+ end
93
+ end
94
+ end