benhutton-mysql2psql 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f28d90b5a9100a93f9739bdaeba79548d7d18720
+  data.tar.gz: 0dce5eeec6734e5f70d7b1e84715c20ed28eaac2
+SHA512:
+  metadata.gz: 4bae782d1fb37f864de841670b9fd905737cef6f2df276322156276bee521e72991b783c0c85ba9adebe2091f5f58fc6e7595312a129aba6ebf7727e8f991466
+  data.tar.gz: b38082d08dcdc7af7f740ec792cb33f0364d95be5ebc5872c973f0ac096805c7659dcae364700790a4a79b8bb0b1728c496c1fbd086a67108828bb2de669d404

data/CHANGELOG

@@ -0,0 +1,19 @@
+0.2.2 Feb 13, 2012
+===
+
+* merge contributions from: James Coleman, Aaron Peckham, James Tippett
+* breaking change: note that "supress*" configuration parameters now use correct spelling. Existing config files you may have will need housekeeping.
+* new configuration options: suppress_sequence_update; suppress_indexes; use_timezones
+* adds a Gemfile to help setup development environment
+
+0.1.1 May 3, 2011
+===
+
+* updated for pg driver ~> 0.11.0
+* add a check for mysql cmd, e.g. mysql or mysql5
+* remove deprecated feature per gem -v 1.7.2
+
+0.1.0 September 18, 2010
+===
+
+* Initial packaging as a gem

data/Gemfile

@@ -0,0 +1,8 @@
+source :rubygems
+
+group :development do
+  gem "bundler", "~> 1.0.21"
+  gem "jeweler", "~> 1.5.2"
+end
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,34 @@
+PATH
+  remote: .
+  specs:
+    mysql2psql (0.2.0)
+      mysql (= 2.8.1)
+      mysql2psql
+      pg (~> 0.11.0)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    git (1.2.5)
+    jeweler (1.5.2)
+      bundler (~> 1.0.0)
+      git (>= 1.2.5)
+      rake
+    json (1.6.5)
+    mysql (2.8.1)
+    pg (0.11.0)
+    rake (0.9.2.2)
+    rdoc (3.12)
+      json (~> 1.4)
+    test-unit (2.4.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.21)
+  jeweler (~> 1.5.2)
+  mysql2psql!
+  rake (~> 0.9.2.2)
+  rdoc (~> 3.12)
+  test-unit (>= 2.1.1)

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009-2010 name <email> 
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,148 @@
+= MYSQL2PSQL - MySQL to PostgreSQL Migration Tool
+
+Tool for converting mysql database to postgresql.
+It can create postgresql dump from mysql database or directly load data from mysql to
+postgresql (at about 100 000 records per minute).
+
+It can translate now most data types and indexes, but if you experience some problems, feel free
+to contact me, I'll help you.
+
+== Installation
+Mysql2psql is packaged as a gem. Install as usual (use sudo if required).
+
+  $ gem install mysql2psql
+
+After installation, the "mysql2psql" command will be available.
+When run, it will generate a default configuration file in the current directory if a config
+file is not already found. The default configuration filename is mysql2psql.yml.
+
+  $ mysql2psql
+
+You can use an alternate config file by passing the filename/path as a parameter:
+
+  $ mysql2psql ../another/world.yml
+
+== Running Migrations
+See the configuration file for documentation about settings. Key choices include:
+* whether to dump to a file, or migrate direct to PostgreSQL
+* migrate only the schema definition, only the data, or both schema and data
+
+
+After editing the configuration file and launching tool, you will see something like..
+
+  Creating table friendships...
+  Created table friendships
+  Loading friendships...
+  620000 of 638779 rows loaded. [ETA: 2010/01/21 21:32 (00h:00m:01s)]
+  638779 rows loaded in 1min 3s
+  Indexing table friendships...
+  Indexed table friendships
+  Table creation 0 min, loading 1 min, indexing 0 min, total 1 min
+
+
+== Database driver dependencies
+Mysql2psql uses the 'mysql' and 'pg' gems for database connectivity.
+Mysql2psql gem installation should automatically install these too.
+
+If you encounter any issues with db connectivity, verify that the 'mysql' and 'pg' gems are installed and working correctly first.
+The 'mysql' gem in particular may need a hint on where to find the mysql headers and libraries:
+
+  $ [sudo] gem install mysql -- --with-mysql-dir=/usr/local/mysql
+
+NB: With Ruby 1.8.x, the gem install will usually complain about "No definition for ..." errors. These are non-fatal and just affect the documentation install. This doesn't happen with Ruby 1.9.2.
+  
+== Contributing to Mysql2psql
+
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so no-one unintentionally breaks it in a future version.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so the gem maintainers can cherry-pick around it.
+* Send a pull request
+  
+== Getting the source
+Mysql2psql was first produced by Max Lapshin <max@maxidoors.ru> who maintains the master
+repository at https://github.com/maxlapshin/mysql2postgres
+
+The gem bundling and refactoring for testing was done by Paul Gallagher <gallagher.paul@gmail.com> and
+the repository is at https://github.com/tardate/mysql2postgres
+
+Gem packaging is done with Jeweler. Note that on windows this means you must run under mingw or other shell that supports git to do the gem bundling (but you can gem install the built gem in the normal Windows shell).
+
+== Setting up for development
+
+The project includes a Gemfile so with bundler you can install gem dependencies with:
+
+  gem install bundler # if not already installed
+  bundle install
+
+Note comments 'Database driver dependencies' regarding issues that you may encounter with the mysql gem.
+  
+== Running tests
+If you fork/clone the project, you will want to run the test suite (and add to it if you are developing new features or fixing bugs).
+
+A suite of tests are provided and are run using rake (rake -T for more info)
+  rake test:units
+  rake test:integration
+  rake test
+
+Rake without parameters (or "rake test") will run both the unit and integration tests.
+
+Unit tests are small standalone tests of the mysql2psql codebase that have no external dependencies (like a database)
+
+Integration tests require suitable mysql and postgres databases to be setup in advance.
+Running the integration tests *will* rewrite data and schema for the "mysql2psql_test" databases in mysql and postgres. Don't use this database for anything else!
+
+Setting up mysql on localhost:3306
+- create a database called "mysql2psql_test"
+- setup a user "mysql2psql" with no password
+
+e.g.
+
+  mysqladmin -uroot -p create mysql2psql_test
+  mysql -uroot -p -e "grant all on mysql2psql_test.* to 'mysql2psql'@'localhost';"
+  # verify connection:
+  mysql -umysql2psql -e "select database(),'yes' as connected" mysql2psql_test
+
+
+Setting up PostgreSQL on localhost:5432
+- create a database called "mysql2psql_test"
+- setup a role (user) "mysql2psql" with no password
+
+e.g.
+
+  psql postgres -c "create role mysql2psql with login"
+  psql postgres -c "create database mysql2psql_test with owner mysql2psql encoding='UTF8'"
+  # verify connection:
+  psql mysql2psql_test -U mysql2psql -c "\c"
+
+
+== Notes, Limitations, Outstanding Issues..
+
+* more test coverage, as always...
+
+
+== Contributors
+Project founded by Max Lapshin <max@maxidoors.ru>
+
+Contributors (roughly git log order):
+- Anton Ageev <anton@ageev.name>
+- Samuel Tribehou <cracoucax@gmail.com>
+- Marco Nenciarini <marco.nenciarini@devise.it>
+- James Nobis <jnobis@jnobis.controldocs.com>
+- quel <github@quelrod.net>
+- Holger Amann <keeney@fehu.org>
+- Maxim Dobriakov <closer.main@gmail.com>
+- Michael Kimsal <mgkimsal@gmail.com>
+- Jacob Coby <jcoby@portallabs.com>
+- Neszt Tibor <neszt@tvnetwork.hu>
+- Miroslav Kratochvil <exa.exa@gmail.com>
+- {Paul Gallagher}[https://github.com/tardate] <gallagher.paul@gmail.com>
+- {James Coleman}[https://github.com/jcoleman] <jtc331@gmail.com>
+- {Aaron Peckham}[https://github.com/apeckham]
+- {James Tippett}[https://github.com/jtippett]
+- {Tim Morgan}[https://github.com/seven1m]
+- {dakhota}[https://github.com/dakhota]
+- {Matthew Soldo}[https://github.com/mattsoldo]

data/Rakefile

@@ -0,0 +1,95 @@
+require 'rubygems'
+require 'rake'
+
+$LOAD_PATH.unshift('lib')
+require 'mysql2psql/version'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "mysql2psql"
+    gem.version = Mysql2psql::Version::STRING
+    gem.summary = %Q{Tool for converting mysql database to postgresql}
+    gem.description = %Q{It can create postgresql dump from mysql database or directly load data from mysql to
+    postgresql (at about 100 000 records per minute). Translates most data types and indexes.}
+    gem.email = "gallagher.paul@gmail.com"
+    gem.homepage = "https://github.com/tardate/mysql2postgres"
+    gem.authors = [
+      "Max Lapshin <max@maxidoors.ru>",
+      "Anton Ageev <anton@ageev.name>",
+      "Samuel Tribehou <cracoucax@gmail.com>",
+      "Marco Nenciarini <marco.nenciarini@devise.it>",
+      "James Nobis <jnobis@jnobis.controldocs.com>",
+      "quel <github@quelrod.net>",
+      "Holger Amann <keeney@fehu.org>",
+      "Maxim Dobriakov <closer.main@gmail.com>",
+      "Michael Kimsal <mgkimsal@gmail.com>",
+      "Jacob Coby <jcoby@portallabs.com>",
+      "Neszt Tibor <neszt@tvnetwork.hu>",
+      "Miroslav Kratochvil <exa.exa@gmail.com>",
+      "Paul Gallagher <gallagher.paul@gmail.com>",
+      "James Coleman <jtc331@gmail.com>",
+      "Aaron Peckham",
+      "James Tippett",
+      "Tim Morgan",
+      "dakhota",
+      "Matthew Soldo"
+    ]
+    gem.add_dependency "mysql", "= 2.8.1"
+    gem.add_dependency "pg", "~> 0.11.0"
+    gem.add_development_dependency "test-unit", ">= 2.1.1"
+    gem.add_development_dependency "rake", "~> 0.9.2.2"
+    gem.add_development_dependency "rdoc", "~> 3.12"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'rake/testtask'
+namespace :test do
+  Rake::TestTask.new(:units) do |test|
+    test.libs << 'lib' << 'test/lib'
+    test.pattern = 'test/units/*test.rb'
+    test.verbose = true
+  end
+
+  Rake::TestTask.new(:integration) do |test|
+    test.libs << 'lib' << 'test/lib'
+    test.pattern = 'test/integration/*test.rb'
+    test.verbose = true
+  end
+end
+
+desc "Run all tests"
+task :test do
+  Rake::Task['test:units'].invoke
+  Rake::Task['test:integration'].invoke
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'lib'
+    test.libs << 'test/lib'
+    test.pattern = 'test/**/*test.rb'
+    test.verbose = true
+    test.rcov_opts << "--exclude gems/*"
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :default => :test
+
+require 'rdoc/task'
+RDoc::Task.new do |rdoc|
+  version = Mysql2psql::Version::STRING
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "mysql2psql #{version}"
+  rdoc.rdoc_files.include("README*", "lib/**/*.rb")
+end

data/bin/mysql2psql

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'rubygems'
+require 'mysql2psql'
+
+exit Mysql2psql.new(ARGV).convert

data/lib/mysql2psql/config.rb

@@ -0,0 +1,142 @@
+require 'mysql2psql/config_base'
+
+class Mysql2psql
+
+  class Config < ConfigBase
+
+    def initialize(configfilepath, generate_default_if_not_found = true)
+      unless File.exists?(configfilepath)
+        reset_configfile(configfilepath) if generate_default_if_not_found
+        if File.exists?(configfilepath)
+          raise Mysql2psql::ConfigurationFileInitialized.new("\n
+No configuration file found.
+A new file has been initialized at: #{configfilepath}
+Please review the configuration and retry..\n\n\n")
+        else
+          raise Mysql2psql::ConfigurationFileNotFound.new("cannot load config file #{configfilepath}")
+        end
+      end
+      super(configfilepath)
+    end
+
+    def reset_configfile(filepath)
+      file = File.new(filepath,'w')
+      self.class.template.each_line do | line|
+        file.puts line
+      end
+      file.close
+    end
+
+    # Returns template file text given +options+ hash which may include:
+    # :to_filename - default: nil
+    # :include_tables - default: []
+    # :exclude_tables - default: []
+    # :suppress_data - default: false
+    # :suppress_ddl - default: false
+    # :suppress_sequence_update - default: false
+    # :suppress_indexes - default: false
+    # :force_truncate - default: false
+    # :use_timezones - default: false
+    def self.template(options={})
+      configtext = <<EOS
+mysql:
+ hostname: localhost
+ port: 3306
+ socket:
+ username: mysql2psql
+ password:
+ database: mysql2psql_test
+
+destination:
+ # if file is given, output goes to file, else postgres
+ file: #{ options[:to_filename] || ''}
+ postgres:
+  hostname: localhost
+  port: 5432
+  username: mysql2psql
+  password:
+  database: mysql2psql_test
+
+# if tables is given, only the listed tables will be converted.  leave empty to convert all tables.
+#tables:
+#- table1
+#- table2
+EOS
+      include_tables = options[:include_tables] || []
+      if include_tables.length>0
+        configtext += "\ntables:\n"
+        include_tables.each do |t|
+          configtext += "- #{t}\n"
+        end
+      end
+      configtext += <<EOS
+# if exclude_tables is given, exclude the listed tables from the conversion.
+#exclude_tables:
+#- table3
+#- table4
+
+EOS
+      exclude_tables = options[:exclude_tables] || []
+      if exclude_tables.length>0
+        configtext += "\nexclude_tables:\n"
+        exclude_tables.each do |t|
+          configtext += "- #{t}\n"
+        end
+      end
+
+      if !options[:suppress_data].nil?
+        configtext += <<EOS
+
+# if suppress_data is true, only the schema definition will be exported/migrated, and not the data
+suppress_data: #{options[:suppress_data]}
+EOS
+      end
+
+      if !options[:suppress_ddl].nil?
+        configtext += <<EOS
+
+# if suppress_ddl is true, only the data will be exported/imported, and not the schema
+suppress_ddl: #{options[:suppress_ddl]}
+EOS
+      end
+
+      if !options[:suppress_sequence_update].nil?
+        configtext += <<EOS
+
+# if suppress_sequence_update is true, the sequences for serial (auto-incrementing) columns
+# will not be update to the current maximum value of that column in the database
+# if suppress_ddl is not set to true, then this option is implied to be false as well (unless overridden here)
+suppress_sequence_update: #{options[:suppress_sequence_update]}
+EOS
+      end
+
+      if !options[:force_truncate].nil?
+        configtext += <<EOS
+
+# if force_truncate is true, forces a table truncate before table loading
+force_truncate: #{options[:force_truncate]}
+EOS
+      end
+
+      if !options[:use_timezones].nil?
+        configtext += <<EOS
+        
+# if use_timezones is true, timestamp/time columns will be created in postgres as "with time zone"
+# rather than "without time zone"
+use_timezones: #{options[:use_timezones]}
+EOS
+      end
+
+      if !options[:suppress_indexes].nil?
+        configtext += <<EOS
+
+# if suppress_indexes is true, indexes will not be exported/migrated
+suppress_indexes: #{options[:suppress_indexes]}
+EOS
+      end
+      configtext
+    end
+
+  end
+
+end

data/lib/mysql2psql/config_base.rb

@@ -0,0 +1,39 @@
+require 'yaml'
+require 'mysql2psql/errors'
+
+class Mysql2psql
+  
+  class ConfigBase
+    attr_reader :config, :filepath
+
+    def initialize(configfilepath)
+      @filepath=configfilepath
+      @config = YAML::load(File.read(filepath))
+    end
+    def [](key)
+      self.send( key )
+    end
+    def method_missing(name, *args)
+      token=name.to_s
+      default = args.length>0 ? args[0] : ''
+      must_be_defined = default == :none
+      case token
+      when /mysql/i
+        key=token.sub( /^mysql/, '' )
+        value=config["mysql"][key]
+      when /pg/i
+        key=token.sub( /^pg/, '' )
+        value=config["destination"]["postgres"][key]
+      when /dest/i
+        key=token.sub( /^dest/, '' )
+        value=config["destination"][key]
+      when /only_tables/i
+        value=config["tables"] 
+      else
+        value=config[token]
+      end
+      value.nil? ? ( must_be_defined ? (raise Mysql2psql::UninitializedValueError.new("no value and no default for #{name}")) : default ) : value
+    end
+  end
+  
+end

data/lib/mysql2psql/converter.rb

@@ -0,0 +1,62 @@
+class Mysql2psql
+
+  class Converter
+    attr_reader :reader, :writer, :options
+    attr_reader :exclude_tables, :only_tables, :suppress_data, :suppress_ddl, :supress_sequence_update, :force_truncate, :suppress_indexes
+
+    def initialize(reader, writer, options)
+      @reader = reader
+      @writer = writer
+      @options = options
+      @exclude_tables = options.exclude_tables([])
+      @only_tables = options.only_tables(nil)
+      @suppress_data = options.suppress_data(false)
+      @suppress_ddl = options.suppress_ddl(false)
+      @supress_sequence_update = options.supress_sequence_update(false)
+      @suppress_indexes = options.suppress_indexes(false)
+      @force_truncate = options.force_truncate(false)
+      @use_timezones = options.use_timezones(false)
+    end
+
+    def convert
+      _time1 = Time.now
+
+      tables = reader.tables.
+        reject {|table| @exclude_tables.include?(table.name)}.
+        select {|table| @only_tables ? @only_tables.include?(table.name) : true}
+
+      tables.each do |table|
+        writer.write_sequence_update(table, options)
+      end if !(@suppress_sequence_update && @suppress_ddl)
+
+      tables.each do |table|
+        writer.write_table(table, {:use_timezones => @use_timezones})
+      end unless @suppress_ddl
+ 
+      _time2 = Time.now
+      tables.each do |table|
+        writer.truncate(table) if force_truncate && !suppress_ddl
+        writer.write_contents(table, reader)
+      end unless @suppress_data
+
+      _time3 = Time.now
+      tables.each do |table|
+        writer.write_indexes(table) unless @suppress_indexes
+      end unless @suppress_ddl
+      tables.each do |table|
+        writer.write_constraints(table)
+      end unless @suppress_ddl
+
+
+      writer.close
+      _time4 = Time.now
+      puts "Table creation #{((_time2 - _time1) / 60).round} min, loading #{((_time3 - _time2) / 60).round} min, indexing #{((_time4 - _time3) / 60).round} min, total #{((_time4 - _time1) / 60).round} min"
+      return 0
+    rescue => e
+      $stderr.puts "Mysql2psql: conversion failed: #{e.to_s}"
+      $stderr.puts e.backtrace
+      return -1
+    end
+  end
+
+end

data/lib/mysql2psql/errors.rb

@@ -0,0 +1,16 @@
+
+class Mysql2psql
+  
+  class GeneralError < StandardError
+	end
+
+  class ConfigurationError < StandardError
+	end
+  class UninitializedValueError < ConfigurationError
+	end
+  class ConfigurationFileNotFound < ConfigurationError
+	end
+  class ConfigurationFileInitialized < ConfigurationError
+	end	
+	
+end