table_syncer 0.2.0

data/History.txt

@@ -0,0 +1,9 @@
+== 0.2.0 2009-09-39
+
+* Now prompts you if you use --commit [so you don't do it on accident]
+* Minor code cleanups :)
+
+== 0.1.0 not a real release
+
+* Now does some structure syncing
+

data/Manifest.txt

@@ -0,0 +1,25 @@
+History.txt
+Manifest.txt
+PostInstall.txt
+README.rdoc
+Rakefile
+bin/table_syncer
+config/hoe.rb
+config/requirements.rb
+lib/table_syncer.rb
+lib/table_syncer/version.rb
+script/console
+script/destroy
+script/generate
+script/txt2html
+setup.rb
+tasks/deployment.rake
+tasks/environment.rake
+tasks/website.rake
+test/test_helper.rb
+test/test_table_syncer.rb
+website/index.html
+website/index.txt
+website/javascripts/rounded_corners_lite.inc.js
+website/stylesheets/screen.css
+website/template.html.erb

data/PostInstall.txt

@@ -0,0 +1,7 @@
+
+For more information on table_syncer, see http://table_syncer.rubyforge.org
+
+NOTE: Change this information in PostInstall.txt 
+You can also delete it if you don't want it.
+
+

data/README.rdoc

@@ -0,0 +1,48 @@
+= table_syncer
+
+* FIX (url)
+
+== DESCRIPTION:
+
+FIX (describe your package)
+
+== FEATURES/PROBLEMS:
+
+* FIX (list of features or problems)
+
+== SYNOPSIS:
+
+  FIX (code sample of usage)
+
+== REQUIREMENTS:
+
+* FIX (list of requirements)
+
+== INSTALL:
+
+* FIX (sudo gem install, anything else)
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2008 FIXME full name
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,4 @@
+require 'config/requirements'
+require 'config/hoe' # setup Hoe + all gem configuration
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }

data/bin/table_syncer

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+#
+#  Created on 2008-9-23.
+#  Copyright (c) 2008. All rights reserved.
+
+begin
+  require 'rubygems'
+  require 'table_syncer' # runs everything
+rescue LoadError
+  # no rubygems to load, so we fail silently
+end

data/config/hoe.rb

@@ -0,0 +1,73 @@
+require 'table_syncer/version'
+
+AUTHOR = 'Roger Pack'  # can also be an array of Authors
+EMAIL = "rogerpack2005@gmail.com"
+DESCRIPTION = "tool to synchronize data across databases"
+GEM_NAME = 'table_syncer' # what ppl will type to install your gem
+RUBYFORGE_PROJECT = 'table-syncer' # The unix name for your project
+HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
+DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+EXTRA_DEPENDENCIES = [
+#  ['activesupport', '>= 1.3.1']
+]    # An array of rubygem dependencies [name, version]
+
+@config_file = "~/.rubyforge/user-config.yml"
+@config = nil
+RUBYFORGE_USERNAME = "rogerdpack"
+def rubyforge_username
+  unless @config
+    begin
+      @config = YAML.load(File.read(File.expand_path(@config_file)))
+    rescue
+      puts <<-EOS
+ERROR: No rubyforge config file found: #{@config_file}
+Run 'rubyforge setup' to prepare your env for access to Rubyforge
+ - See http://newgem.rubyforge.org/rubyforge.html for more details
+      EOS
+      exit
+    end
+  end
+  RUBYFORGE_USERNAME.replace @config["username"]
+end
+
+
+REV = nil
+# UNCOMMENT IF REQUIRED:
+# REV = YAML.load(`svn info`)['Revision']
+VERS = TableSyncer::VERSION::STRING + (REV ? ".#{REV}" : "")
+RDOC_OPTS = ['--quiet', '--title', 'table_syncer documentation',
+    "--opname", "index.html",
+    "--line-numbers",
+    "--main", "README",
+    "--inline-source"]
+
+class Hoe
+  def extra_deps
+    @extra_deps.reject! { |x| Array(x).first == 'hoe' }
+    @extra_deps
+  end
+end
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.new(GEM_NAME, VERS) do |p|
+  p.developer(AUTHOR, EMAIL)
+  p.description = DESCRIPTION
+  p.summary = DESCRIPTION
+  p.url = HOMEPATH
+  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  p.test_globs = ["test/**/test_*.rb"]
+  p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']  #An array of file patterns to delete on clean.
+
+  # == Optional
+  p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
+  #p.extra_deps = EXTRA_DEPENDENCIES
+
+    #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
+  end
+
+CHANGES = $hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
+PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
+$hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
+$hoe.rsync_args = '-av --delete --ignore-errors'
+$hoe.spec.post_install_message = File.open(File.dirname(__FILE__) + "/../PostInstall.txt").read rescue ""

data/config/requirements.rb

@@ -0,0 +1,15 @@
+require 'fileutils'
+include FileUtils
+
+require 'rubygems'
+%w[rake hoe newgem rubigen].each do |req_gem|
+  begin
+    require req_gem
+  rescue LoadError
+    puts "This Rakefile requires the '#{req_gem}' RubyGem."
+    puts "Installation: gem install #{req_gem} -y"
+    exit
+  end
+end
+
+$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))

data/lib/table_syncer.rb

@@ -0,0 +1,508 @@
+#!/usr/bin/env ruby
+# the amazing mysql table syncer! 
+# this program syncs between two database tables to make a recipient match a donor table
+# to use: define some database connections at the top of this file
+# then run --help to see options
+# See also http://code.google.com/p/ruby-roger-useful-functions/wiki/MysqlTableSyncer
+# Note that it automatically creates would_has_run.sql which has the contents of what if would have executed, had you passed --commit
+#
+# Enjoy.
+# Note that with ssh hosts if it fails to connect, it will try and auto-connect to that host.
+# the "auto established" tunnel will then be running in the background.
+# Re-run the script and it will hopefully work the second time.
+# If it doesn't, then run the command output [it will print out the ssh tunnel command appropriate] in a different terminal window
+
+# 2008 Roger Pack Public Domain
+# No warranty of any type :)
+
+  require 'rubygems'
+  begin
+	require 'mysqlplus'
+  rescue LoadError
+	require "mysql"
+  end
+  require 'optparse'
+ 
+  # define some databases and how you connect with them, if foreign
+  example_db_1 = {:host => '127.0.0.1', :user => 'root', :password => '', :db => 'wilkboar_ties'}
+  example_db_2 = {:host => '127.0.0.1', :user => 'root', :password => '', :db => 'new_database'}
+
+  all_database_names = ['example_db_1', 'example_db_2'] # just used so that the --help command is more useful and can print out the database connection names available
+
+  # setup defaults -- it will use these databases by default unless you specify otherwise on the command line:
+  db_from_name = 'example_db_1'
+  db_to_name = 'example_db_2'
+  actually_run_queries = false # default to just previewing -- use --commit to actually run it
+  verbose = true 
+
+
+  # now parse incoming options 
+  my_options = {}
+  auto_create_ssh_tunnels = true
+  tables_to_sync = nil
+  # make sure use normal style options, or nothing :)
+  my_options[:skip_the_warning_prompt_for_commit] = false
+
+  do_structure_sync_only = false
+
+  OptionParser.new do |opts|
+    opts.banner = "Usage: #{$0} [options] table names"
+
+    opts.on("-f", "--from=FROM", "from database name #{all_database_names.inspect}") do |from_name|
+      db_from_name = from_name
+    end
+
+    opts.on("-t", "--to=TO", "to database name #{all_database_names.inspect} \n\t\t\t\t\t\tedit databases at #{__FILE__}") do |to_name|
+	db_to_name = to_name
+    end
+
+    opts.on('-y', 'skip the warning prompt for commit') do
+	my_options[:skip_the_warning_prompt_for_commit] = true
+    end
+
+    opts.on("-s", "--tables=", "tables list comma separated, ex --tables=table_one,table_two -- or if you'd just like to do all tables, then do --tables=ALL_TABLES", Array) do |tables|
+      tables_to_sync = []
+      for entry in tables do
+      	tables_to_sync << entry
+      end
+    end
+
+    opts.on("", "--commit", " tell us whether or not to actually run queries -- the default is to not run queries") do
+      print "DOING IT COMMITTING LIVE QUERIES\n"
+      actually_run_queries = true
+    end
+
+    opts.on("-z", "--extra_sql=STRING", "specify an sql sql string to run after the script ends [in the 'to' database]") do |sql|
+      print "extra sql", sql
+      my_options[:extra_sql] = sql
+    end
+
+    opts.on('-q', '--quiet', 'Non-verbose -- dont output as much junk, just the summaries') do |quiet_true|
+	    verbose = false
+    end
+    
+    opts.on('--perform_structure_sync_only', "Do a structure morph for the designated tables -- \nnote that it doesnt yet do indices and you need to still pass --commit for it to do the structure change.  Use it with --tables=STRUCTURE_SYNC_THE_WHOLE_THING to drop any existing tables not found in the origin database and structure sync over any existing tables") do
+      do_structure_sync_only = true
+    end
+
+  end.parse!
+
+  # grab the right db's
+  db_from_info = db_to_info = nil
+  print "from db: #{db_from_name}\n"
+  eval("db_from_info = #{db_from_name}")
+  print "to db: #{db_to_name}\n"
+  eval("db_to_info = #{db_to_name}")
+
+  # some error checking
+  raise if defined?(production) and db_to_name == 'production' and actually_run_queries # I never wanted to commit to one db
+  raise 'missing a database selected?' unless db_to_info and db_from_info
+
+  # custom parse table names they have within the parameters
+  unless tables_to_sync
+	extra_table_args = ARGV.select{|arg| arg[0..0] != '-'}
+        if extra_table_args.length > 0
+           tables_to_sync = []		
+	   for arg in extra_table_args
+		tables_to_sync  += Array(arg.split(','))
+	   end	
+        else
+	   print 'no tables specified! run with --help', "\n"
+	   exit
+	end
+  else
+	if ARGV.find{|arg| arg[0..0] != '-'}
+		print "warning--ignoring some apparently extra parameters at the end, since you passed in tables via a command line arg"
+	end
+  end
+  ARGV.clear
+  raise 'must specify tables or ALL for structure_sync to work--we\'re conservative and disallow it otherwise' if tables_to_sync.empty? and do_structure_sync_only
+
+  example_out_file = File.open 'would_have_run.sql' , 'w' unless actually_run_queries
+
+
+class Hash
+	def to_sql_update_query(table_name, nonmatching_keys) # ltodo take some 'params' :)
+		raise unless self['id']
+		query = "update #{table_name} set"
+ 		comma = ''
+		self.each_key do |key|
+			query << "#{comma} #{key} = #{self[key] ? "'" + self[key].gsub("'", "\\\\'") + "'": 'NULL'}" if nonmatching_keys.include? key
+			comma = ',' if nonmatching_keys.include? key
+		end
+		query << " where id = #{self['id']}"
+	end
+
+	def to_sql_create_query(table_name)
+		query = "insert into #{table_name} ("
+		comma = ''
+		self.each_key { |key_name|
+			query += "#{comma}#{key_name} "
+			comma = ','
+		}
+		query += ") values ( "
+		comma = ''
+		self.each_key { |key_name|
+			query += "#{comma} #{self[key_name] ? "'" + self[key_name].gsub("'", "\\\\'") + "'" : 'NULL'}" # assume it will leave the others are null, I guess
+			comma = ','
+		}
+	 	query += ");"
+	end		
+end
+
+  def sync_structure(db_to, db_from, table, actually_run_queries)
+  print "structure syncing #{table}\n"
+  good_structure = db_from.query("desc #{table}").use_result
+  all_from_columns = {}
+  good_structure.each_hash{|h| all_from_columns[h['Field']] = h }
+  good_structure.free
+  # we basically cheat and just fakely recreate mismatched columns by "modifying them" to match the creation script given by 'show create table x' for that column
+  good_creation_query = db_from.query("show create table #{table}").use_result
+  create_whole_table_script = good_creation_query.fetch_hash['Create Table']
+  good_creation_script = create_whole_table_script.split("\n")
+  good_creation_query.free
+
+  questionable_to_structure = db_to.query("desc #{table}").use_result rescue nil
+  unless questionable_to_structure 
+      if actually_run_queries
+	db_to.query(create_whole_table_script)
+      else
+	print "would have created new table #{table} thus: #{create_whole_table_script}\n"
+	db_to = db_from # fake it that they match so we don't raise any errors for the duration of this method call
+      end
+      questionable_to_structure = db_to.query("desc #{table}").use_result
+  end
+
+  all_to_columns = {}
+  questionable_to_structure.each_hash{|h| all_to_columns[h['Field']] = h }
+  questionable_to_structure.free
+
+  for column_name, specs in all_from_columns do
+
+  	matching_creation_line = good_creation_script.find{|line| line =~ /^\s*`#{column_name}`/} # starts with column name--kind of fallible, but hey, we're working with english single words here
+
+  	matching_to_column_specs = all_to_columns[column_name]
+  	matching_creation_line = matching_creation_line[0..-2] if matching_creation_line[-1..-1] == ','
+  	unless matching_to_column_specs # get it from the script
+  		# create it
+  		if specs['Extra'] != ''
+  			raise "uh oh currently we don't sync id's they're assumed to exist already! Try deleting the old column #{column_name} or table #{table} entirely"
+  		end
+
+  		running = "ALTER TABLE #{table} ADD COLUMN #{matching_creation_line}"
+  		print "running #{running}-- for #{column_name}\n"
+  		db_to.query running if actually_run_queries
+          else
+  		# we don't want key differences to make a difference--those are handle after
+  		to_specs_non_keyed = matching_to_column_specs.dup
+                  specs_non_keyed = specs.dup
+  		to_specs_non_keyed.delete('Key')
+  		specs_non_keyed.delete('Key')
+  		if specs_non_keyed != to_specs_non_keyed
+  			line = "ALTER TABLE #{table} CHANGE #{column_name} #{matching_creation_line}"
+
+			# for some reason the create table script doesn't include defaults if they're NULL or ''
+			unless line =~ /default/i
+				if specs_non_keyed['Default'] == nil
+				   line += " DEFAULT NULL"
+				else
+				   line += " DEFAULT '#{ specs_non_keyed['Default'] }'"
+			 	end
+			end
+  			print "modifying #{column_name} -- #{line} \n"
+			print "#{specs_non_keyed.inspect} != the to guy: #{to_specs_non_keyed.inspect}"
+  			db_to.query line if actually_run_queries
+  	  end	
+  		all_to_columns.delete(column_name)
+  	end
+   end
+
+   for column_name, description in all_to_columns # left overs
+		print "REMOVING COLUMN #{column_name}"
+		db_to.query("ALTER TABLE #{table} DROP #{column_name}") if actually_run_queries
+   end
+
+   indices = db_from.query("show index from #{table};").use_result
+   all_indices = []
+   indices.each_hash{|h| h.delete('Cardinality'); all_indices << h } # Cardinality doesn't make a difference...AFAIK
+   indices.free
+
+   existing_indices = db_to.query("show index from #{table}").use_result
+   all_existing_indices = []
+   existing_indices.each_hash{|h| h.delete('Cardinality'); all_existing_indices << h }
+   existing_indices.free
+   apparently_lacking = all_indices.map{|index| index['Column_name']} - all_existing_indices.map{|index| index['Column_name']}
+
+   for index in apparently_lacking
+	    # ltodo if it looks nice and generic then go ahead and add it
+   end
+
+   if all_indices != all_existing_indices # this is right
+     print "\n\nWARNING #{table}: you are missing some indexes now or there is some type of discrepancy [these aren't handled yet]-- you may want to add them a la\nCREATE INDEX some_name_usually_column_name_here ON #{table} (column_name_here)\n for apparently the following columns: #{apparently_lacking.inspect}\n\n
+     you might get away with dropping the old table and letting it be recreated -- that might also add the right indices\n\n
+     #{all_indices.inspect}\n != \n#{all_existing_indices.inspect}"
+   end 
+end
+
+   if db_from_info[:ssh_host]
+		db_from_info[:host] = db_from_info[:ssh_local_to_host] || '127.0.0.1'
+		db_from_info[:port] = 4000
+   end
+
+   if db_to_info[:ssh_host]
+		db_to_info[:host] = db_from_info[:ssh_local_to_host] || '127.0.0.1'
+		db_to_info[:port] = 4000
+   end
+   commit_style = actually_run_queries ? '---COMMITTING----' : 'previewing (no changes made)'
+
+   print "#{db_from_info[:db]} => #{db_to_info[:db]}\n\n"
+   print "#{commit_style} run\n\n"
+   print "#{db_from_info[:ssh_host] || db_from_info[:host]}:#{db_from_info[:db]} #{tables_to_sync.inspect}\n"
+   print "\t=> #{db_to_info[:ssh_host] || db_to_info[:host]}:#{db_to_info[:db]} #{tables_to_sync.inspect}\n"
+   # ltodo add in the local_to_stuff here
+   
+   if actually_run_queries and !my_options[:skip_the_warning_prompt_for_commit] 
+	print "Continue (yes/no)?"
+	input = gets
+	if !['y', 'yes'].include? input.downcase.strip
+		print "aborting -- you gave me #{input}"
+		exit
+	end
+   end
+   
+   
+   start_time = Time.now 
+   retried = false
+   begin
+     # connect to the MySQL servers
+     print 'connecting to to DB...', db_to_info[:db]; STDOUT.flush
+     db_to = Mysql.real_connect(db_to_info[:host], db_to_info[:user], db_to_info[:password], db_to_info[:db], db_to_info[:port], nil, Mysql::CLIENT_COMPRESS)
+     print 'connected', "\n", 'now connecting to from DB ', db_from_info[:db]; STDOUT.flush
+     db_from = Mysql.real_connect(db_from_info[:host], db_from_info[:user], db_from_info[:password], db_from_info[:db], db_from_info[:port], nil, Mysql::CLIENT_COMPRESS)
+     print "connected\n"
+   rescue Mysql::Error => e
+     puts "Error code: #{e.errno}"
+     puts "Error message: #{e.error}"
+     puts "This may mean a tunnel is not working" if e.error.include?('127.0.0.1')
+     # note that, if you do add ssh -> ssh, you may still only need one connection!
+     if db_from_info[:ssh_host] or db_to_info[:ssh_host]
+	ssh_port = db_from_info[:ssh_port] || db_to_info[:ssh_port]
+	ssh_local_to_port = db_from_info[:ssh_local_to_port] || db_to_info[:ssh_local_to_port] || 3306
+	ssh_user = db_from_info[:ssh_user] || db_to_info[:ssh_user]
+	ssh_local_to_host = db_from_info[:ssh_local_to_host] || db_to_info[:ssh_local_to_host] || 'localhost'
+	ssh_host = db_from_info[:ssh_host] || db_to_info[:ssh_host]
+	command = "ssh -N #{ssh_port ? '-p ' + ssh_port.to_s : nil} -L 4000:#{ssh_local_to_host}:#{ssh_local_to_port} #{ssh_user}@#{ssh_host} \n" # NOTE DOES NOT YET ALLOW FOR TWO SSH DB's
+        if auto_create_ssh_tunnels and !retried
+            print "trying  to auto create ssh tunnel via: #{command}\n"
+	    Thread.new { system(command) }
+	    retried = true # this doesn't actually work :P
+	    retry
+        else
+	  print "unable to connect to server--try running\n#{command}in another window or try again!"
+        end
+     end
+     exit
+   ensure
+     # ltodo: disconnect from server here [?] -- also do we free, and disconnect, at all, during this? :)
+   end
+
+  summary_information = '' # so we can print it all (again), at the end
+
+  db_to.query_with_result = false
+  db_from.query_with_result = false # allow us to read them from the wire when they are coming in.  Save a little CPU time :)
+
+      
+  # we need to delete any extra tables if they're there in one and not the other
+  if do_structure_sync_only and tables_to_sync == ['STRUCTURE_SYNC_THE_WHOLE_THING'] or tables_to_sync == ['ALL_TABLES']
+    tables_from = db_from.query("show tables").use_result
+    tables_from_array = []
+    tables_from.each_hash {|h| h.each{|k, v| tables_from_array << v}}
+    tables_from.free
+    tables_to_sync = tables_from_array
+    if tables_to_sync == ['STRUCTURE_SYNC_THE_WHOLE_THING'] # then we want to drop some tables if they exist
+      tables_to = db_to.query("show tables").use_result
+      tables_to_array = []
+      tables_to.each_hash {|h| h.each{|k, v| tables_to_array << v}}
+      tables_to.free
+      nukables = tables_to_array - tables_from_array
+      for table in nukables do
+        query = "DROP TABLE #{table}"
+        print "dropping table -- #{query}\n"
+        db_to.query(query) if actually_run_queries
+      end
+    end
+  end
+
+  for table in tables_to_sync  do
+   print "start #{commit_style} table #{table}" + "**" * 10 + "\n"
+   if do_structure_sync_only
+     sync_structure(db_to, db_from, table, actually_run_queries)
+     next
+   end
+  
+   all_to_keys_not_yet_processed = {}
+   select_all_to = db_to.query("SELECT * FROM #{table}") # could easily be 'select id', as well note this assumes distinct id's! Otherwise we'd need hashes, one at a time, etc. etc.
+   select_all_to = select_all_to.use_result
+   select_all_to.each_hash { |to_element|
+	if all_to_keys_not_yet_processed[to_element['id']] # duplicated id's are a fringe case and not yet handled! TODO use hashes or somefin' bet-uh
+		raise "\n\n\n\nERROR detected a duplicated id (or the lack of id at all) in #{table} -- aborting [consider clearing [DELETE FROM #{table} in the 'to' database and trying again, if in a pinch]!\n\n\n\n" 
+	end
+	all_to_keys_not_yet_processed[to_element['id']] = to_element
+   }
+
+   res = db_from.query("SELECT * from #{table}")
+   res = res.use_result
+   count_updated = 0
+   count_created = 0
+   
+   res.each_hash do |from_element|
+	existing = all_to_keys_not_yet_processed[from_element['id']]
+	# now there are a few cases--we can find a matching id->rest locally, or an id->nonmatching (update) or non_id (insert)
+	# the problem is that we need to keep track of which id's we never used, and delete them from the offending table, afterward
+	if existing # we have a match--test if it is truly matching
+		to_element = existing# ltodo rename
+		all_nonmatching_keys = []
+		for key in from_element.keys do
+			if from_element[key] != to_element[key]
+				all_nonmatching_keys << key
+				print  " #{key}\t\t\t[", from_element[key].inspect, "]!!!!======to:[", to_element[key].inspect||'',  ']', "\n" if verbose
+			else
+				# equal, ok
+			end
+
+		end
+		if all_nonmatching_keys.length > 0
+			count_updated += 1
+			query = from_element.to_sql_update_query(table, all_nonmatching_keys)
+			print "update query on #{to_element['name']}: #{query}\n" if verbose
+			db_to.query query if actually_run_queries
+			example_out_file.write query + ";\n" unless actually_run_queries
+		end
+	else
+		count_created += 1
+		create_query = from_element.to_sql_create_query(table)
+		print "insert query on #{from_element['name']}: #{create_query}\n" if verbose
+		db_to.query create_query if actually_run_queries
+		example_out_file.write create_query + ";\n" unless actually_run_queries
+        end
+	all_to_keys_not_yet_processed.delete(from_element['id'])
+   end
+   print "\n" if (count_updated>0 or count_created>0) if verbose
+
+
+   count_deleted = all_to_keys_not_yet_processed.length
+   if count_deleted > 0
+     ids = []
+     for id in all_to_keys_not_yet_processed.keys do
+       ids << id 
+     end
+     double_check_all_query = "select * from #{table} where id IN (#{ids.join(',')})" # this allows us to make sure we don't delete any doubled ones (which would be a weird situation and too odd to handle), and also so we can have a nice verbose  'we are deleting this row' message
+     double_check_result = db_to.query(double_check_all_query)
+     double_check_result = double_check_result.use_result
+
+
+     victims = {}
+
+     double_check_result.each_hash {|victim|
+        raise 'duplicate' if victims[victim['id']]
+        victims[victim['id']] = victim['name']
+     }
+     raise 'weird deleted--got back strange number of rows -- refusing to delete' unless double_check_result.num_rows == count_deleted
+     double_check_result.free
+     for id in all_to_keys_not_yet_processed.keys do
+       query = "delete from #{table} where id = #{id}"
+       print "DELETE query, for #{victims[id]} is #{query}\n" if verbose
+       db_to.query query if actually_run_queries
+       example_out_file.write query + ";\n" unless actually_run_queries
+     end
+   end
+
+   res.free
+   print "done #{commit_style} "
+   summary =  "#{table} -- updated #{count_updated}, created #{count_created}, deleted #{count_deleted}\n"
+   print summary
+   summary_information << summary
+  end
+  if my_options[:extra_sql] and actually_run_queries
+    print "doing sql #{my_options[:extra_sql]}\n"
+    result = db_to.query my_options[:extra_sql]
+    require 'pp'
+    pp "got sql result", result 
+  end
+  db_from.close if db_from
+  db_to.close if db_to
+  print "\n--summary--\n", summary_information
+  print "total transfer time #{Time.now - start_time}\n"
+
+# Todo list:
+# inlined todo notes
+# ltodo: some SSH examples in here
+# ltodo: optionally 'only add from one to the other' -- only add new entries past the current maxes for the table
+# ltodo: one BIG transaction, so that ctrl+c will work. [?]
+# ltodo: read_only directive
+# todo by default output a 'backup' log somewhere, too! oh baby! 10 of them! what the heck! :)
+# just needs docs and rock and roll [publicize] :)
+# note this lacks 'transaction's thus far
+# and its SSH is hackish, requiring the user to start a tunnel in another window
+# ltodo: tell people how to install ssh for windows users [?]
+# TODO sql escape things better (does it a little already) [use mysql escape function itself or whatever it is]
+# TODO could use a 'mass insert' and mass delete to speed things up (several giant strings batch mode) 
+# could do: something along the lines of rsync for database tables--calculate some checksums, foreign host, etc., anything to save on bandwidth lol. It is, however, true that most changes come from the "latter end" of tables so...probably has tons of potential savings
+# ltodo handle ssh => ssh [separate hosts] -- note: may want to specify a 'local port' per ssh host :)
+# could do: download from both people at the same time, per table, or what not muhaha
+# could do: some more free's in here [double check to make sure I free everwhere]
+# TODO when it does something require keyboard input unless they specify --force or something
+# ltodo handle ginormous tables :) that wouldn't ever ever fit in memory :)
+# could do: have this cool 'difference since then' setting thingers...like...ok you sync'ed that since then, you've changed this much we know, and that one has changed that much, we know...so the sum diff is...
+# whoa!
+# ltodo: you can chain together a few updates, too, a la "update x where y; update z where q; update a where b;" or the 'super table mass updater' that uses a temp table lol
+# :)
+# need to use my_options for the syncing stuff, clean up code, too
+# todo: this errs
+#>> gc.description.pretty_inspect=> ""GCU\\\\’s Bachelor of Science in Entrepreneurial Studies program is built on the principles of personal integrity, values, and innovation. Emphasizing the philosophy of being an \\"Entrepreneurial School by Entrepreneurs,\\" the program provides students with the skills to think analytically, ask the right questions, solve problems, and function as an entrepreneur in both small and large companies. Students are prepared to be self-motivated, self-managed, and self-disciplined entrepreneurs with the skill-set to manage their own careers either by starting their own business venture or working within a start-up, entrepreneurial business environment. Interaction with successful entrepreneurs, business consulting opportunities, and individual venture capital projects are highlighted in the program."\n"
+#>> gc.description
+#=> "GCU\\’s Bachelor of Science in Entrepreneurial Studies program is built on the principles of personal integrity, values, and innovation. Emphasizing the philosophy of being an "Entrepreneurial School by Entrepreneurs," the program provides students with the skills to think analytically, ask the right questions, solve problems, and function as an entrepreneur in both small and large companies. Students are prepared to be self-motivated, self-managed, and self-disciplined entrepreneurs with the skill-set to manage their own careers either by starting their own business venture or working within a start-up, entrepreneurial business environment. Interaction with successful entrepreneurs, business consulting opportunities, and individual venture capital projects are highlighted in the program."
+
+
+
+
+=begin
+multiple insertions ex:
+   dbh.query("INSERT INTO animal (name, category)
+                VALUES
+                  ('snake', 'reptile'),
+                  ('frog', 'amphibian'),
+                  ('tuna', 'fish'),
+                  ('racoon', 'mammal')
+              ")
+
+class MultipleDeleter # not thread safe
+  @@batch_max = 1000;
+  def initialize connection, table_name
+   @connection = connection
+   @table_name = table_name
+   ids = []
+  end
+
+  def add_id_t_delete id
+   ids << id
+   if ids.length == @@batch_max
+     send
+   end
+  end
+
+  def send
+     @connection.query "delete from #{@table_name} where id IN (#{ids.join(',')});"
+     ids = []
+  end
+
+  def flush
+    send
+  end
+end
+=end
+
+
+# ltodo: alias for table names :) czs, ps, etc :)
+
+# we really need to be able to handle many to many: just use a hash + counter based system instead of an id based system