oracle_to_mysql 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,5 @@
1
+ README.rdoc
2
+ lib/**/*.rb
3
+ bin/*
4
+ features/**/*.feature
5
+ LICENSE
@@ -0,0 +1,111 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: oracle_to_mysql
3
+ version: !ruby/object:Gem::Version
4
+ hash: 23
5
+ prerelease: false
6
+ segments:
7
+ - 1
8
+ - 0
9
+ - 0
10
+ version: 1.0.0
11
+ platform: ruby
12
+ authors:
13
+ - Joe Goggins
14
+ - Chris Dinger
15
+ autorequire:
16
+ bindir: bin
17
+ cert_chain: []
18
+
19
+ date: 2010-10-29 00:00:00 -05:00
20
+ default_executable:
21
+ dependencies:
22
+ - !ruby/object:Gem::Dependency
23
+ name: thoughtbot-shoulda
24
+ prerelease: false
25
+ requirement: &id001 !ruby/object:Gem::Requirement
26
+ none: false
27
+ requirements:
28
+ - - ">="
29
+ - !ruby/object:Gem::Version
30
+ hash: 3
31
+ segments:
32
+ - 0
33
+ version: "0"
34
+ type: :development
35
+ version_requirements: *id001
36
+ description: Wraps the sqlplus binary and mysql binary does not currently require OCI8 or MySQL gems (might someday tho)
37
+ email: joe.goggins@umn.edu
38
+ executables: []
39
+
40
+ extensions: []
41
+
42
+ extra_rdoc_files:
43
+ - LICENSE
44
+ - README.rdoc
45
+ files:
46
+ - .document
47
+ - .gitignore
48
+ - LICENSE
49
+ - README.rdoc
50
+ - Rakefile
51
+ - VERSION
52
+ - lib/oracle_to_mysql.rb
53
+ - lib/oracle_to_mysql/api_instance_methods.rb
54
+ - lib/oracle_to_mysql/command.rb
55
+ - lib/oracle_to_mysql/command/delete_temp_files.rb
56
+ - lib/oracle_to_mysql/command/fork_and_execute_sqlplus_command.rb
57
+ - lib/oracle_to_mysql/command/write_and_execute_mysql_commands_to_bash_file.rb
58
+ - lib/oracle_to_mysql/command/write_and_execute_mysql_commands_to_bash_file_in_replace_mode.rb
59
+ - lib/oracle_to_mysql/command/write_sqlplus_commands_to_file.rb
60
+ - lib/oracle_to_mysql/must_override_instance_methods.rb
61
+ - lib/oracle_to_mysql/optional_override_instance_methods.rb
62
+ - lib/oracle_to_mysql/private_instance_methods.rb
63
+ - lib/oracle_to_mysql/protected_class_methods.rb
64
+ - test/demo/ps_term_tbl.rb
65
+ - test/demo/ps_term_tbl_accumulative.rb
66
+ - test/demo/test_oracle_to_mysql_against_ps_term_tbl.rb
67
+ - test/helper.rb
68
+ - test/oracle_to_mysql.example.yml
69
+ - test/test_against_ps_term_tbl_accumulative.rb
70
+ - test/test_oracle_to_mysql.rb
71
+ has_rdoc: true
72
+ homepage: http://github.com/joegoggins/oracle_to_mysql
73
+ licenses: []
74
+
75
+ post_install_message:
76
+ rdoc_options:
77
+ - --charset=UTF-8
78
+ require_paths:
79
+ - lib
80
+ required_ruby_version: !ruby/object:Gem::Requirement
81
+ none: false
82
+ requirements:
83
+ - - ">="
84
+ - !ruby/object:Gem::Version
85
+ hash: 3
86
+ segments:
87
+ - 0
88
+ version: "0"
89
+ required_rubygems_version: !ruby/object:Gem::Requirement
90
+ none: false
91
+ requirements:
92
+ - - ">="
93
+ - !ruby/object:Gem::Version
94
+ hash: 3
95
+ segments:
96
+ - 0
97
+ version: "0"
98
+ requirements: []
99
+
100
+ rubyforge_project:
101
+ rubygems_version: 1.3.7
102
+ signing_key:
103
+ specification_version: 3
104
+ summary: A gem for mirroring data from an oracle db to a mysql db
105
+ test_files:
106
+ - test/demo/ps_term_tbl.rb
107
+ - test/demo/ps_term_tbl_accumulative.rb
108
+ - test/demo/test_oracle_to_mysql_against_ps_term_tbl.rb
109
+ - test/helper.rb
110
+ - test/test_against_ps_term_tbl_accumulative.rb
111
+ - test/test_oracle_to_mysql.rb
@@ -0,0 +1,118 @@
1
+ = OracleToMysql
2
+ A wrapper for the `sqlplus` and `mysql` binary that pulls
3
+ data out of an Oracle database and inserts it into MySQL.
4
+
5
+ By omitting any Ruby/Mysql & Ruby/Oracle libraries things are kept
6
+ fast, minimal, and debuggable.
7
+
8
+ FYI, 1.0.0 is misleading, it's not some rock solid piece of uber-code.
9
+ It's a weasel of a gem we just started using to replace our existing mirroring infrastructure
10
+ in October 2010.
11
+
12
+ == Use Case
13
+ As a Ruby developer
14
+ with access to huge Oracle data warehouses
15
+ I need a way to work with this data in my apps
16
+ without using Oracle. ;)
17
+
18
+ == System Dependencies
19
+ popen4 (gem)
20
+ sqlplus, and mysql binaries are in $PATH
21
+
22
+ == Usage
23
+ See the "test" cases in /test, there aren't really tests
24
+ Just some code that uses the test/demo/* models to demonstrate
25
+ the two main mirror strategies associated with this tool (see below)
26
+
27
+ For example:
28
+ class TheMagicMirrorer
29
+ include OracleToMysql
30
+
31
+ def otm_source_sql
32
+ "select
33
+ col || CHR(9) ||
34
+ col || CHR(9) ||
35
+ ...
36
+ from table
37
+ where ... Oracle statement"
38
+ end
39
+
40
+ def otm_target_table
41
+ "a_mysql_table_name"
42
+ end
43
+
44
+ def otm_target_sql
45
+ "create table if not exists #{self.otm_target_table} (... mysql statement "
46
+ end
47
+ end
48
+
49
+ x = TheMagicMirrorer.new
50
+ x.otm_execute
51
+
52
+ Will mirror the contents of otm_source_sql into the table created by otm_target_sql.
53
+ The "|| CHR(9) ||" crap is Oracle sql code that tab deliminates the column content in
54
+ the spooled sqlplus SQL data output. The Mysql "load data infile" command eats this output.
55
+
56
+ If you are using with Rails, it expects database.yml to have oracle_source and mysql_target entries to get the
57
+ db connect info, to override the names, see OptionalOverrideInstanceMethods#otm_config_file and #otm_source_config_hash or #otm_target_config_hash
58
+
59
+ Also, will need to do:
60
+ gem install POpen4
61
+
62
+ === Mirror Strategies
63
+ :atomic_rename (Default)
64
+ "load data infile" the spooled oracle tab deliminted data into a temp table first
65
+ then atomically rename
66
+ current_target_table -> old_target_table AND
67
+ new_temp_table -> new_target_table
68
+
69
+ :accumulative
70
+ "load data infile" directly into target_table replacing any existing
71
+ rows in target when source data triggers "ON DUPLICATE KEY"
72
+
73
+ === Target Table Retention
74
+ For both strategies, if the target_table already exists the
75
+ default is to keep the existing table around and suffix it with "_old".
76
+ This is called a :n => 1 retention policy.
77
+
78
+ The ability to retain n previous tables in the works, handy for data mining, stay tuned.
79
+
80
+ == Gem Development & Testing
81
+ The "tests" aka demo's assume you have a ps_term_tbl in your Oracle db.
82
+ If you're running PeopleSoft at a University you'll probably have this...
83
+ Otherwise, the tests won't run, it's just meant as an example that works in our world.
84
+
85
+ You'll need the thoughtbot-shoulda gem if you want to develop/hack on this gem or run the tests
86
+
87
+ To run tests:
88
+ cd test
89
+ ruby test_oracle_to_mysql_against_ps_term_tbl.rb
90
+ OR
91
+ irb -r test_oracle_to_mysql_against_ps_term_tbl.rb
92
+ # And monkey with the run time...all files in test/demo are loaded, you can tinker with them
93
+
94
+ This assumes you have a connection file in the test dir:
95
+ oracle_to_mysql.yml
96
+ copy and populate from oracle_to_mysql.example.yml
97
+
98
+ == Note on Patches/Pull Requests
99
+ * Fork the project.
100
+ * Add files to test/demo/* that demonstrate how you are using the tool.
101
+ * Bugfixes = fork, fix, commit, pull request.
102
+ * New Features = let us know what yer thinkin, we might already be working on it
103
+
104
+ == A few things we'd like to work on soon
105
+ * retention policy of 0: don't keep yesterdays data (aka don't create *_old table in mysql)
106
+ * retention policy > 1: keep N mysql table copies around
107
+ * usage of a Logger object instead of stdout
108
+ * Better configuration of what happens when the otm_execute fails, not sure...some options might include
109
+ * email someone a backtrace of the exception
110
+ * log the exception backtrace to a table in the db
111
+ * either cleanup + delete temp files or keep around (now it just leaves the temp files around)
112
+
113
+ == Known Issues
114
+ * Since source data is written to disk a tab delimintated file, if the source oracle data contains a \t character in might mess things up (none of our data has tabs so we haven't had problems)
115
+ * Add validations/checks for stuff in validate_otm_source_sql in write_sqlplus_commands_to_file.rb if you encounter goofy sqlplus errors
116
+ Things that are not easily programmatically detectable ought just have an inline note i suppose
117
+
118
+
@@ -0,0 +1,55 @@
1
+ require 'rubygems'
2
+ require 'rake'
3
+
4
+ begin
5
+ require 'jeweler'
6
+ Jeweler::Tasks.new do |gem|
7
+ gem.name = "oracle_to_mysql"
8
+ gem.summary = %Q{A gem for mirroring data from an oracle db to a mysql db}
9
+ gem.description = %Q{Wraps the sqlplus binary and mysql binary does not currently require OCI8 or MySQL gems (might someday tho)}
10
+ gem.email = "joe.goggins@umn.edu"
11
+ gem.homepage = "http://github.com/joegoggins/oracle_to_mysql"
12
+ gem.authors = ["Joe Goggins","Chris Dinger"]
13
+
14
+ gem.add_dependency "POpen4", ">= 0"
15
+ gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
16
+ # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
17
+ end
18
+ Jeweler::GemcutterTasks.new
19
+ rescue LoadError
20
+ puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
21
+ end
22
+
23
+ require 'rake/testtask'
24
+ Rake::TestTask.new(:test) do |test|
25
+ test.libs << 'lib' << 'test'
26
+ test.pattern = 'test/**/test_*.rb'
27
+ test.verbose = true
28
+ end
29
+
30
+ begin
31
+ require 'rcov/rcovtask'
32
+ Rcov::RcovTask.new do |test|
33
+ test.libs << 'test'
34
+ test.pattern = 'test/**/test_*.rb'
35
+ test.verbose = true
36
+ end
37
+ rescue LoadError
38
+ task :rcov do
39
+ abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
40
+ end
41
+ end
42
+
43
+ task :test => :check_dependencies
44
+
45
+ task :default => :test
46
+
47
+ require 'rake/rdoctask'
48
+ Rake::RDocTask.new do |rdoc|
49
+ version = File.exist?('VERSION') ? File.read('VERSION') : ""
50
+
51
+ rdoc.rdoc_dir = 'rdoc'
52
+ rdoc.title = "oracle_to_mysql #{version}"
53
+ rdoc.rdoc_files.include('README*')
54
+ rdoc.rdoc_files.include('lib/**/*.rb')
55
+ end
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 1.1.0
@@ -0,0 +1,80 @@
1
+ # A super robust process spawning lib, install via
2
+ # gem install --remote --include-dependencies POpen4
3
+ # See http://rubygems.org/gems/POpen4
4
+ # and http://popen4.rubyforge.org/
5
+ #
6
+ require 'popen4'
7
+ require 'oracle_to_mysql/protected_class_methods'
8
+ require 'oracle_to_mysql/private_instance_methods'
9
+ require 'oracle_to_mysql/must_override_instance_methods'
10
+ require 'oracle_to_mysql/optional_override_instance_methods'
11
+ require 'oracle_to_mysql/api_instance_methods'
12
+
13
+ # These commands are used internally to actuall do the work
14
+ require 'oracle_to_mysql/command'
15
+ require 'oracle_to_mysql/command/write_sqlplus_commands_to_file'
16
+ require 'oracle_to_mysql/command/fork_and_execute_sqlplus_command.rb'
17
+ require 'oracle_to_mysql/command/write_and_execute_mysql_commands_to_bash_file.rb'
18
+ require 'oracle_to_mysql/command/write_and_execute_mysql_commands_to_bash_file_in_replace_mode.rb'
19
+ require 'oracle_to_mysql/command/delete_temp_files.rb'
20
+ module OracleToMysql
21
+ class CommandError < Exception; end
22
+ # used to join the table and timestamp for old retained tables if :n > 1, or if :n = 1 its simply the suffix of the table name
23
+ OTM_RETAIN_KEY = '_old'
24
+ OTM_VALID_STRATEGIES = [:accumulative, :atomic_rename]
25
+
26
+
27
+ # Stuff mixed into a client class calling include OracleToMysql
28
+ def self.included(caller)
29
+ caller.class_eval do
30
+ private
31
+ include PrivateInstanceMethods
32
+
33
+ protected
34
+ extend ProtectedClassMethods
35
+
36
+ # All customization will come by overriding methods in these classes
37
+ include OptionalOverrideInstanceMethods # You might need to override one or two of these
38
+ include MustOverrideInstanceMethods # THIS IS WHAT A CLIENT MUST OVERRIDE for .otm_execute to work
39
+
40
+ public
41
+ include ApiInstanceMethods # You shouldn't need to override these, probably better to override at a lower level
42
+ end
43
+ end
44
+
45
+ # If you add to this, don't forget to add the :my_command_name to otm_execute_command_names (and the class definition file)
46
+ # and the require statement at the top of this class
47
+ #
48
+ def self.command_name_to_class_hash
49
+ return {
50
+ :write_sqlplus_commands_to_file => OracleToMysql::Command::WriterSqlplusCommandsToFile,
51
+ :fork_and_execute_sqlplus_commands_file => OracleToMysql::Command::ForkAndExecuteSqlplusCommand,
52
+ :write_and_execute_mysql_commands_to_bash_file => OracleToMysql::Command::WriteAndExecuteMysqlCommandsToBashFile,
53
+ :write_and_execute_mysql_commands_to_bash_file_in_replace_mode => OracleToMysql::Command::WriteAndExecuteMysqlCommandsToBashFileInReplaceMode,
54
+ # TODO pack-keys and optimize table
55
+ :delete_temp_files => OracleToMysql::Command::DeleteTempFiles
56
+ }
57
+ end
58
+
59
+ ####
60
+ # This is a very handy method for debugging a single Command bound to a particular client class
61
+ #
62
+ # x=OracleToMysql.get_and_bind_command(:write_sqlplus_commands_to_file, PsTermTbl.new)
63
+ # x.execute
64
+ #
65
+ # or can access any other helper methods on x
66
+ #
67
+ ###
68
+ def self.get_and_bind_command(command_name,client_class)
69
+ raise "invalid command name #{command_name}" unless self.command_name_to_class_hash.has_key?(command_name)
70
+ command = self.command_name_to_class_hash[command_name].new
71
+ command.client_class = client_class
72
+ return command
73
+ end
74
+
75
+ # once again, sqlplus the jackass needs a "TNS string" to connect to it's stankin ass
76
+ # this interpolates crap from a config hash, like host and port and database into it
77
+ #
78
+ def self.tns_string_from_config(config_hash) "(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=#{config_hash['host']})(PORT=#{config_hash['port']})))(CONNECT_DATA=(SERVICE_NAME=#{config_hash['database']})))"
79
+ end
80
+ end
@@ -0,0 +1,94 @@
1
+ module OracleToMysql
2
+ module ApiInstanceMethods
3
+ def otm_execute
4
+ self.otm_output("[started at #{self.otm_timestamp}]")
5
+ self.otm_started("#otm_execute (in #{self.otm_strategy.to_s} mode, retain_n = #{self.otm_retain_options[:n]}")
6
+ self.otm_execute_command_names.each do |command_name|
7
+ command = OracleToMysql.get_and_bind_command(command_name, self)
8
+ begin
9
+ command.execute
10
+ rescue OracleToMysql::CommandError => e
11
+ puts "TODO: SHOULD EMAIL SOMEONE OR SOMETHING, THE BUILD FAILED"
12
+ raise e
13
+ rescue Exception => e
14
+ puts "TODO: CUSTOM EXCEPTION HANDLING FOR A PARTICULAR COMMAND MIGHT HAPPEN HERE..."
15
+ raise e
16
+ end
17
+ end
18
+ self.otm_finished("#otm_execute")
19
+ final_time_elapsed = self.otm_time_elapsed_since_otm_timestamp
20
+ finished_at = self.otm_timestamp + final_time_elapsed
21
+
22
+ self.otm_output("[finished at #{finished_at}]")
23
+ self.otm_output("[completed in #{final_time_elapsed} seconds]")
24
+ self.otm_reset_timestamp
25
+ return true
26
+ end
27
+
28
+ # This is used extensively by commands to read and write the the specific files
29
+ # that are shared accross commands, it is a bit protected-ish though (aka prolly don't want to override this)
30
+ #
31
+ def otm_get_file_name_for(sym)
32
+ f = self.generate_tmp_file_name(sym)
33
+ case sym
34
+ when :mysql_commands
35
+ "#{f}.sql"
36
+ when :oracle_commands
37
+ "#{f}.sql" # sqlplus is such a jackass, the file name must end in .sql for it to work, fyi
38
+ when :oracle_output
39
+ "#{f}.txt"
40
+ else
41
+ raise "Invalid file_name #{sym}"
42
+ end
43
+ end
44
+
45
+ # For interaction with the configuration file that contains the mysql or oracle stuff
46
+ # override otm_config_file if you have yer config somewhere crazy
47
+ #
48
+ def otm_config_hash
49
+ if @otm_confile_file_hash.nil?
50
+ @otm_confile_file_hash = YAML.load(File.read(self.otm_config_file))
51
+ end
52
+ @otm_confile_file_hash
53
+ end
54
+
55
+ # all tables and file name are versioned against the seconds of this Time obj
56
+ #
57
+ def otm_timestamp
58
+ if @otm_timestamp.nil?
59
+ @otm_timestamp = Time.now
60
+ end
61
+ @otm_timestamp
62
+ end
63
+
64
+ # You need to do this to invoke .otm_execute twice on the same inst, or it will clobber
65
+ def otm_reset_timestamp
66
+ @otm_timestamp = nil # reset it so all temp files have a new identifier, if otm_execute'd is executed again
67
+ end
68
+
69
+ # Returns all temp files that all commands in the current strategy can create
70
+ #
71
+ def otm_all_temp_files
72
+ return_this = []
73
+ self.otm_execute_command_names.each do |command_name|
74
+ command = OracleToMysql.get_and_bind_command(command_name, self)
75
+ return_this += command.temp_file_symbols.map {|sym| self.otm_get_file_name_for(sym)}
76
+ end
77
+ return_this.uniq
78
+ end
79
+
80
+ # returns an array of all target tables by reflecting the table retention options
81
+ #
82
+ def otm_all_target_tables
83
+ return_this = [self.otm_target_table]
84
+ if self.otm_retain_options[:n] == 0
85
+ return_this
86
+ else
87
+ (1..self.otm_retain_options[:n]).to_enum.each do |x|
88
+ return_this << self.otm_retained_target_table(x)
89
+ end
90
+ return_this
91
+ end
92
+ end
93
+ end
94
+ end