table_syncer 0.2.0 → 0.3.0

data/History.txt

@@ -1,3 +1,9 @@
+==0.3.0 2008-10-15
+
+* Now uses the more conservative [and slower] query_with_result = true, instead of the faster and potentially scary query_with_result = false -- see http://betterlogic.com/roger/?p=473
+* Now has a working --extra_sql=SQL
+* Recommended upgrade.
+
 == 0.2.0 2009-09-39
 
 * Now prompts you if you use --commit [so you don't do it on accident]

data/lib/table_syncer/version.rb

@@ -1,7 +1,7 @@
 module TableSyncer
   module VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 2
+    MINOR = 3
     TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')

data/lib/table_syncer.rb

@@ -13,36 +13,50 @@
 # If it doesn't, then run the command output [it will print out the ssh tunnel command appropriate] in a different terminal window
 
 # 2008 Roger Pack Public Domain
-# No warranty of any type :)
+# No warranty expressed or implied of any type :)
 
   require 'rubygems'
   begin
-	require 'mysqlplus'
+	require 'mysqlplus' # it's barely faster
   rescue LoadError
 	require "mysql"
   end
   require 'optparse'
  
   # define some databases and how you connect with them, if foreign
-  example_db_1 = {:host => '127.0.0.1', :user => 'root', :password => '', :db => 'wilkboar_ties'}
-  example_db_2 = {:host => '127.0.0.1', :user => 'root', :password => '', :db => 'new_database'}
+  local_db = {:host => '127.0.0.1', :user => 'root', :password => '', :db => 'local_leadgen_dev'}
+  ties_db = {:user => 'db_user_name_ex_root', :password => 'mysql_password_for_that_user', :db => 'wilkboar_ties', :ssh_host => 'my_host.com', :ssh_user => 'ssh_login'} # ssh example -- doesn't take passwords here but attempts to create an ssh tunnel for you to that host.
 
-  all_database_names = ['example_db_1', 'example_db_2'] # just used so that the --help command is more useful and can print out the database connection names available
+  # I suppose you wouldn't need an ssh_user if you had established the tunnel youself  [ex: if you had already created a tunnel on port 9000 to the remote Mysql port]
+  ties_db_no_host_even = {:user => 'db_user_name_ex_root', :password => 'mysql_password_for_that_user', :db => 'wilkboar_ties', :tunnel_local_port_to_use => 9000} 
+  # if you have multiple remote hosts then you'll want to assign a different port for each host
 
-  # setup defaults -- it will use these databases by default unless you specify otherwise on the command line:
-  db_from_name = 'example_db_1'
-  db_to_name = 'example_db_2'
+  ties_super_advanced = {:user => 'mysql_user', :password => 'mysql_pass_for_that_user', :ssh_host => 'helpme.com', :ssh_port => 888, :ssh_user => 'ssh_login_name', :ssh_local_to_host => '127.0.0.1 -- change if you want it to connect to a different host "from the remote"', :ssh_local_to_port => '4000 instead of the normal 3306'}
+  # ssh_port is the ssh port [instead of 22] for the remote host
+ 
+  all_database_names = ['local_db', 'ties_db', 'ties_db_no_host_even', 'ties_super_advanced'] # only used --help command is more useful and can print out the database connection names available -- not necessary
+
+
+  
+
+  # setup defaults -- it will use these databases by default unless you specify otherwise on the command line [or use nil for none]
+  db_from_name = 'local_db' # replace with yours
+  db_to_name = 'local_db'
   actually_run_queries = false # default to just previewing -- use --commit to actually run it
   verbose = true 
+  default_tables_to_sync = nil # replace with default tables to sync, like ['users']
 
 
-  # now parse incoming options 
   my_options = {}
   auto_create_ssh_tunnels = true
-  tables_to_sync = nil
+  tables_to_sync = nil # leave as nil--this will be propagated by yours, above, or those passed from the command line
   # make sure use normal style options, or nothing :)
   my_options[:skip_the_warning_prompt_for_commit] = false
 
+  # note that ctrl-c doesn't work too well in the middle of this script, but ctrl-z + kill PID does
+
+  # now parse incoming options 
+
   do_structure_sync_only = false
 
   OptionParser.new do |opts|
@@ -52,10 +66,14 @@
       db_from_name = from_name
     end
 
-    opts.on("-t", "--to=TO", "to database name #{all_database_names.inspect} \n\t\t\t\t\t\tedit databases at #{__FILE__}") do |to_name|
+    opts.on("-t", "--to=TO", "to database name #{all_database_names.inspect} \n\t\t\t\t\t\tedit databases descriptions at #{__FILE__}") do |to_name|
 	db_to_name = to_name
     end
 
+    opts.on('--verbose') do
+	my_options[:verbose] = true
+    end
+
     opts.on('-y', 'skip the warning prompt for commit') do
 	my_options[:skip_the_warning_prompt_for_commit] = true
     end
@@ -68,12 +86,10 @@
     end
 
     opts.on("", "--commit", " tell us whether or not to actually run queries -- the default is to not run queries") do
-      print "DOING IT COMMITTING LIVE QUERIES\n"
       actually_run_queries = true
     end
 
     opts.on("-z", "--extra_sql=STRING", "specify an sql sql string to run after the script ends [in the 'to' database]") do |sql|
-      print "extra sql", sql
       my_options[:extra_sql] = sql
     end
 
@@ -81,7 +97,7 @@
 	    verbose = false
     end
     
-    opts.on('--perform_structure_sync_only', "Do a structure morph for the designated tables -- \nnote that it doesnt yet do indices and you need to still pass --commit for it to do the structure change.  Use it with --tables=STRUCTURE_SYNC_THE_WHOLE_THING to drop any existing tables not found in the origin database and structure sync over any existing tables") do
+    opts.on('--perform_structure_sync_only', "Do a structure morph for the designated tables -- Use it with --tables=STRUCTURE_SYNC_THE_WHOLE_THING to drop any existing tables not found in the origin database and structure sync over any existing tables -- see the docs http://code.google.com/p/ruby-roger-useful-functions/wiki/TableSyncer") do
       do_structure_sync_only = true
     end
 
@@ -93,11 +109,14 @@
   eval("db_from_info = #{db_from_name}")
   print "to db: #{db_to_name}\n"
   eval("db_to_info = #{db_to_name}")
-
-  # some error checking
-  raise if defined?(production) and db_to_name == 'production' and actually_run_queries # I never wanted to commit to one db
   raise 'missing a database selected?' unless db_to_info and db_from_info
 
+  # skip the necessity for -y if a db is set to :expendable
+  my_options[:skip_the_warning_prompt_for_commit] = true if db_to_info[:expendable]
+
+  # raise if they attempt to commit to a db which has :read_only => true set in its options
+  raise 'attempting to commit to a read only db ' + db_to_name if db_to_info[:read_only] and actually_run_queries
+
   # custom parse table names they have within the parameters
   unless tables_to_sync
 	extra_table_args = ARGV.select{|arg| arg[0..0] != '-'}
@@ -106,7 +125,9 @@
 	   for arg in extra_table_args
 		tables_to_sync  += Array(arg.split(','))
 	   end	
-        else
+        elsif default_tables_to_sync
+		tables_to_sync = default_tables_to_sync	
+	else
 	   print 'no tables specified! run with --help', "\n"
 	   exit
 	end
@@ -150,19 +171,19 @@ class Hash
 	end		
 end
 
-  def sync_structure(db_to, db_from, table, actually_run_queries)
+  def sync_structure(db_to, db_from, table, actually_run_queries, my_options)
   print "structure syncing #{table}\n"
-  good_structure = db_from.query("desc #{table}").use_result
+  good_structure = db_from.query("desc #{table}")
   all_from_columns = {}
   good_structure.each_hash{|h| all_from_columns[h['Field']] = h }
   good_structure.free
   # we basically cheat and just fakely recreate mismatched columns by "modifying them" to match the creation script given by 'show create table x' for that column
-  good_creation_query = db_from.query("show create table #{table}").use_result
+  good_creation_query = db_from.query("show create table #{table}")
   create_whole_table_script = good_creation_query.fetch_hash['Create Table']
   good_creation_script = create_whole_table_script.split("\n")
   good_creation_query.free
 
-  questionable_to_structure = db_to.query("desc #{table}").use_result rescue nil
+  questionable_to_structure = db_to.query("desc #{table}") rescue nil
   unless questionable_to_structure 
       if actually_run_queries
 	db_to.query(create_whole_table_script)
@@ -170,7 +191,7 @@ end
 	print "would have created new table #{table} thus: #{create_whole_table_script}\n"
 	db_to = db_from # fake it that they match so we don't raise any errors for the duration of this method call
       end
-      questionable_to_structure = db_to.query("desc #{table}").use_result
+      questionable_to_structure = db_to.query("desc #{table}")
   end
 
   all_to_columns = {}
@@ -222,25 +243,36 @@ end
 		db_to.query("ALTER TABLE #{table} DROP #{column_name}") if actually_run_queries
    end
 
-   indices = db_from.query("show index from #{table};").use_result
+   indices = db_from.query("show index from #{table};")
    all_indices = []
    indices.each_hash{|h| h.delete('Cardinality'); all_indices << h } # Cardinality doesn't make a difference...AFAIK
    indices.free
 
-   existing_indices = db_to.query("show index from #{table}").use_result
+   existing_indices = db_to.query("show index from #{table}")
    all_existing_indices = []
    existing_indices.each_hash{|h| h.delete('Cardinality'); all_existing_indices << h }
    existing_indices.free
+   different = []
+   all_different = all_indices - all_existing_indices #.each{|hash| different << hash['Column_name'] unless existing_indices.include?(hash) }
    apparently_lacking = all_indices.map{|index| index['Column_name']} - all_existing_indices.map{|index| index['Column_name']}
 
+   
+
    for index in apparently_lacking
 	    # ltodo if it looks nice and generic then go ahead and add it
    end
 
+
+
    if all_indices != all_existing_indices # this is right
-     print "\n\nWARNING #{table}: you are missing some indexes now or there is some type of discrepancy [these aren't handled yet]-- you may want to add them a la\nCREATE INDEX some_name_usually_column_name_here ON #{table} (column_name_here)\n for apparently the following columns: #{apparently_lacking.inspect}\n\n
-     you might get away with dropping the old table and letting it be recreated -- that might also add the right indices\n\n
-     #{all_indices.inspect}\n != \n#{all_existing_indices.inspect}"
+     print "\n\nWARNING #{table}: you are missing some indexes now or there is some type of discrepancy [indices aren't handled yet]-- you may want to add them a la\nCREATE INDEX some_name_usually_column_name_here ON #{table} (column_name_here)\n for apparently at least the following missing indices: #{apparently_lacking.inspect} 
+
+            you have apparently mismatched indices for: #{all_different.map{|h| h['Column_name']}.inspect}\n\n
+             --you might get away with dropping the old table and letting it be recreated -- that might add the right indices -- run with --verbose to see more info"
+
+     if my_options[:verbose]
+	print "the 'good' one is #{all_indices.inspect}, yours is #{all_existing_indices.inspect}"
+     end
    end 
 end
 
@@ -262,7 +294,7 @@ end
    # ltodo add in the local_to_stuff here
    
    if actually_run_queries and !my_options[:skip_the_warning_prompt_for_commit] 
-	print "Continue (yes/no)?"
+	print "Continue (y/n)?"
 	input = gets
 	if !['y', 'yes'].include? input.downcase.strip
 		print "aborting -- you gave me #{input}"
@@ -286,19 +318,25 @@ end
      puts "This may mean a tunnel is not working" if e.error.include?('127.0.0.1')
      # note that, if you do add ssh -> ssh, you may still only need one connection!
      if db_from_info[:ssh_host] or db_to_info[:ssh_host]
-	ssh_port = db_from_info[:ssh_port] || db_to_info[:ssh_port]
-	ssh_local_to_port = db_from_info[:ssh_local_to_port] || db_to_info[:ssh_local_to_port] || 3306
-	ssh_user = db_from_info[:ssh_user] || db_to_info[:ssh_user]
-	ssh_local_to_host = db_from_info[:ssh_local_to_host] || db_to_info[:ssh_local_to_host] || 'localhost'
-	ssh_host = db_from_info[:ssh_host] || db_to_info[:ssh_host]
-	command = "ssh -N #{ssh_port ? '-p ' + ssh_port.to_s : nil} -L 4000:#{ssh_local_to_host}:#{ssh_local_to_port} #{ssh_user}@#{ssh_host} \n" # NOTE DOES NOT YET ALLOW FOR TWO SSH DB's
+        
+        raise 'two ssh hosts not yet a working feature--request it' if db_from_info[:ssh_host] and db_to_info[:ssh_host] and (db_from_info[:ssh_host] != db_to_info[:ssh_host])
+        ssh_requiring_connection = db_to_info[:ssh_host] ? db_to_info : db_from_info
+	ssh_port = ssh_requiring_connection[:ssh_port]
+	ssh_local_to_port = ssh_requiring_connection[:ssh_local_to_port] || 3306
+	ssh_user = ssh_requiring_connection[:ssh_user]
+	ssh_local_to_host = ssh_requiring_connection[:ssh_local_to_host] || 'localhost'
+	ssh_host = ssh_requiring_connection[:ssh_host]
+
+        local_port_to_use = ssh_requiring_connection[:tunnel_local_port_to_use] || 4000
+        
+	command = "ssh -N #{ssh_port ? '-p ' + ssh_port.to_s : nil} -L #{local_port_to_use}:#{ssh_local_to_host}:#{ssh_local_to_port} #{ssh_user}@#{ssh_host} \n" # note that ssh_local_to_port is 'local on the foreign server'
         if auto_create_ssh_tunnels and !retried
             print "trying  to auto create ssh tunnel via: #{command}\n"
 	    Thread.new { system(command) }
 	    retried = true # this doesn't actually work :P
 	    retry
         else
-	  print "unable to connect to server--try running\n#{command}in another window or try again!"
+	  print "unable to connect to server--try running\n\t#{command}in another window or try again!\n"
         end
      end
      exit
@@ -308,19 +346,15 @@ end
 
   summary_information = '' # so we can print it all (again), at the end
 
-  db_to.query_with_result = false
-  db_from.query_with_result = false # allow us to read them from the wire when they are coming in.  Save a little CPU time :)
-
-      
   # we need to delete any extra tables if they're there in one and not the other
   if do_structure_sync_only and tables_to_sync == ['STRUCTURE_SYNC_THE_WHOLE_THING'] or tables_to_sync == ['ALL_TABLES']
-    tables_from = db_from.query("show tables").use_result
+    tables_from = db_from.query("show tables")
     tables_from_array = []
     tables_from.each_hash {|h| h.each{|k, v| tables_from_array << v}}
     tables_from.free
     tables_to_sync = tables_from_array
     if tables_to_sync == ['STRUCTURE_SYNC_THE_WHOLE_THING'] # then we want to drop some tables if they exist
-      tables_to = db_to.query("show tables").use_result
+      tables_to = db_to.query("show tables")
       tables_to_array = []
       tables_to.each_hash {|h| h.each{|k, v| tables_to_array << v}}
       tables_to.free
@@ -336,13 +370,13 @@ end
   for table in tables_to_sync  do
    print "start #{commit_style} table #{table}" + "**" * 10 + "\n"
    if do_structure_sync_only
-     sync_structure(db_to, db_from, table, actually_run_queries)
+     sync_structure(db_to, db_from, table, actually_run_queries, my_options)
      next
    end
   
    all_to_keys_not_yet_processed = {}
    select_all_to = db_to.query("SELECT * FROM #{table}") # could easily be 'select id', as well note this assumes distinct id's! Otherwise we'd need hashes, one at a time, etc. etc.
-   select_all_to = select_all_to.use_result
+   select_all_to = select_all_to
    select_all_to.each_hash { |to_element|
 	if all_to_keys_not_yet_processed[to_element['id']] # duplicated id's are a fringe case and not yet handled! TODO use hashes or somefin' bet-uh
 		raise "\n\n\n\nERROR detected a duplicated id (or the lack of id at all) in #{table} -- aborting [consider clearing [DELETE FROM #{table} in the 'to' database and trying again, if in a pinch]!\n\n\n\n" 
@@ -351,7 +385,7 @@ end
    }
 
    res = db_from.query("SELECT * from #{table}")
-   res = res.use_result
+   res = res
    count_updated = 0
    count_created = 0
    
@@ -398,7 +432,7 @@ end
      end
      double_check_all_query = "select * from #{table} where id IN (#{ids.join(',')})" # this allows us to make sure we don't delete any doubled ones (which would be a weird situation and too odd to handle), and also so we can have a nice verbose  'we are deleting this row' message
      double_check_result = db_to.query(double_check_all_query)
-     double_check_result = double_check_result.use_result
+     double_check_result = double_check_result
 
 
      victims = {}
@@ -425,9 +459,16 @@ end
   end
   if my_options[:extra_sql] and actually_run_queries
     print "doing sql #{my_options[:extra_sql]}\n"
-    result = db_to.query my_options[:extra_sql]
-    require 'pp'
-    pp "got sql result", result 
+    result = db_to.query( my_options[:extra_sql])
+    # result will only be a result set if it was a select query
+    
+    if result
+      require 'pp'
+      while row = result.fetch_row do
+        pp row
+      end
+      result.free
+    end
   end
   db_from.close if db_from
   db_to.close if db_to
@@ -506,3 +547,5 @@ end
 # ltodo: alias for table names :) czs, ps, etc :)
 
 # we really need to be able to handle many to many: just use a hash + counter based system instead of an id based system
+# ltodo: use mysql table checksum :)
+# ltodo: lacks using port besides 3306 locally

data/website/index.html

@@ -33,7 +33,7 @@
     <h1>table_syncer</h1>
     <div id="version" class="clickable" onclick='document.location = "http://rubyforge.org/projects/table_syncer"; return false'>
       <p>Get Version</p>
-      <a href="http://rubyforge.org/projects/table_syncer" class="numbers">0.2.0</a>
+      <a href="http://rubyforge.org/projects/table_syncer" class="numbers">0.3.0</a>
     </div>
     <h1>&amp;#x2192; &#8216;table_syncer&#8217;</h1>
 <h2>What</h2>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: table_syncer
 version: !ruby/object:Gem::Version 
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors: 
 - Roger Pack
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-09-30 00:00:00 -06:00
+date: 2008-11-22 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -20,7 +20,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.7.0
+        version: 1.8.0
     version: 
 description: tool to synchronize data across databases
 email: 
@@ -33,6 +33,7 @@ extra_rdoc_files:
 - History.txt
 - Manifest.txt
 - PostInstall.txt
+- README.rdoc
 - website/index.txt
 files: 
 - History.txt
@@ -72,7 +73,7 @@ post_install_message: |+
   
 rdoc_options: 
 - --main
-- README.txt
+- README.rdoc
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -90,10 +91,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: table-syncer
-rubygems_version: 1.3.0
+rubygems_version: 1.3.1
 signing_key: 
 specification_version: 2
 summary: tool to synchronize data across databases
 test_files: 
-- test/test_helper.rb
 - test/test_table_syncer.rb
+- test/test_helper.rb