sqlite2mysql 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e4abd7634da6d7d4924ac5440ca84a74328e64e0
-  data.tar.gz: 673c82ad3762793877b8d8b9671439b80e51bd47
+  metadata.gz: 4459055881f5d71cbb6f70ba64ac6370f414b6b9
+  data.tar.gz: c7b60bb262fa636a7d53efd98a3d54bf9bd13f88
 SHA512:
-  metadata.gz: 0a7bf80a0be02f3032275bffbf3510328f9618f6d88b08b294ba9a0277d1c3128ba6bd34ae056386ba487845d2f93cd20dfdb3574eb5c57d3cb37c0dc86971b3
-  data.tar.gz: b36b4cd35cdd4cb184fa16bb238ee4b221769d51e76d8969f8e95407711a43b0ca9573556002699b4e023a47de35dc0da34df238493c14fbed3a70ab45c37a71
+  metadata.gz: 4a0b2dbf1af68fe5c2e52029e54f9c71d1133d25808401a83764fddde1b6d2490781affc9a56592b349b1e0471432c9524e540b49f305d18a601bb8e1e548616
+  data.tar.gz: 120080c429c5d492778e2a8d7b587e5f5762c90dcf578c273845c2856155aff7239502c90293567ef0ac66de3740c897d37047e7ad3859f5061794c2f99b6a10

data/exe/sqlite2mysql

@@ -1,78 +1,5 @@
 #!/usr/bin/env ruby
 
-require 'mysql2'
-require 'sqlite3'
+require 'sqlite2mysql'
 
-puts 'Usage: sqlite2mysql sqlite_file.db [mysql_db_name]' if ARGV.size < 1
-
-DATABASE = ARGV.first
-SQL_DB_NAME = ARGV[1] || DATABASE.gsub(/[^0-9a-z]/i, '')
-
-puts 'Collecting Sqlit3 Info' # ===============================================
-
-db = SQLite3::Database.new DATABASE
-
-schema = {}
-
-tables = db.execute 'SELECT name FROM sqlite_master WHERE type="table"'
-
-tables.flatten.each do |t|
-  columns = db.execute("pragma table_info(#{t})")
-
-  formatted_columns = []
-  columns.each do |col|
-    formatted_columns << { name:    col[1],
-                           type:    col[2],
-                           notnull: col[3],
-                           default: col[4] }
-  end
-
-  schema[t] = formatted_columns
-end
-
-puts "Creating MySQL DB: #{SQL_DB_NAME}" # ====================================
-
-RESERVED_WORDS = %w(key int)
-
-def create_table_query(table, columns)
-  query = "CREATE TABLE #{table} ("
-  cols = []
-  columns.each do |col|
-    col[:name] += '_1' if RESERVED_WORDS.include?(col[:name])
-    if col[:type] == ''
-      col[:type] = 'varchar(255)'
-    elsif col[:type].start_with?('float')
-      col[:type] = 'float'
-    end
-    cols << "#{col[:name]} #{col[:type]} #{'NOT NULL' if col[:notnull]}"
-  end
-  query + "#{cols.join(', ')})"
-end
-
-client = Mysql2::Client.new(host: 'localhost', username: 'root')
-
-client.query("DROP DATABASE IF EXISTS #{SQL_DB_NAME}")
-client.query("CREATE DATABASE #{SQL_DB_NAME}")
-client.query("USE #{SQL_DB_NAME}")
-
-schema.keys.each do |table|
-  puts "Creating table: #{table}"
-  client.query(create_table_query(table, schema[table]))
-end
-
-print 'Grab a ☕' # ============================================================
-
-schema.keys.each do |table|
-  puts "\nInserting data: #{table}"
-  data = db.execute("select * from #{table}")
-  data.each_slice(1000) do |slice|
-    slice.each do |row|
-      cleaned_row = row.map do |val|
-        val.is_a?(String) ? client.escape(val) : val
-      end
-      client.query("INSERT INTO #{table} VALUES (\"#{cleaned_row.join('", "')}\")")
-    end
-    print '.'
-  end
-end
-puts ''
+Sqlite2Mysql.run(ARGV)

data/lib/sqlite2mysql.rb

@@ -1,3 +1,39 @@
 require 'sqlite2mysql/version'
+require 'sqlite2mysql/services/arguments'
+require 'sqlite2mysql/services/bound_finder'
+require 'sqlite2mysql/services/mysql'
+require 'sqlite2mysql/services/sqlite'
+require 'sqlite2mysql/services/type_inferrer'
 
-puts 'WARNING: Including sqlite2mysql does nothing, run it from the terminal.'
+class Sqlite2Mysql
+  class << self
+    def run(args)
+      arguments = Arguments.new(args)
+
+      puts 'Collecting Sqlite3 Info'
+
+      db = SqliteClient.new(arguments.sqlite_db, infer_column_types: arguments.infer_types)
+
+      schema = db.build_schema
+
+      puts "Creating MySQL DB: #{arguments.mysql_db}"
+
+      mysql = MysqlClient.new(
+        host:     arguments.mysql_host,
+        username: arguments.username,
+        password: arguments.password,
+        port:     arguments.mysql_port)
+      mysql.recreate(arguments.mysql_db)
+      mysql.build_from_schema(schema)
+
+      print 'Grab a ☕'
+
+      schema.keys.each do |table|
+        puts "\nInserting data: #{table}"
+        data = db.get_data(table)
+        mysql.insert_table(table, data)
+      end
+      puts ''
+    end
+  end
+end

data/lib/sqlite2mysql/services/arguments.rb

@@ -0,0 +1,71 @@
+class Arguments
+  attr_accessor :mysql_host, :username, :password, :mysql_port, :infer_types,
+                :mysql_db, :sqlite_db
+
+  def initialize(args)
+    help(args) if args.size == 0
+
+    set_defaults
+    unmodified_args = args.dup
+    unmodified_args.each do |arg|
+      if arg.start_with?('--')
+        send(arg[2..-1], args)
+        args.delete(arg)
+      end
+    end
+
+    @sqlite_db = args.first
+    @mysql_db = args[1] || @sqlite_db.gsub(/[^0-9a-z]/i, '')
+  end
+
+  def infer(_)
+    @infer_types = true
+  end
+
+  def user(args)
+    @username = get_value_for_flag('--user', args)
+  end
+
+  def pass(args)
+    @password = get_value_for_flag('--pass', args)
+  end
+
+  def port(args)
+    @mysql_port = get_value_for_flag('--port', args)
+  end
+
+  def host(args)
+    @mysql_host = get_value_for_flag('--host', args)
+  end
+
+  def help(_)
+    puts <<-HELP
+Usage:
+  sqlite2mysql sqlite.db [mysql_name]
+
+Options:
+  --help    Show this message
+  --infer   Infer types for columns
+  --user    MySQL username (default: root)
+  --host    MySQL host     (default: localhost)
+  --pass    MySQL password
+  --port    MySQL port
+HELP
+    exit 0
+  end
+
+  private
+
+  def get_value_for_flag(flag, args)
+    index = args.index(flag)
+    args.delete_at(index + 1)
+  end
+
+  def set_defaults
+    @username = 'root'
+    @password = nil
+    @host = 'localhost'
+    @port = nil
+    @infer = false
+  end
+end

data/lib/sqlite2mysql/services/bound_finder.rb

@@ -0,0 +1,23 @@
+class BoundFinder
+  def initialize(client, table, column)
+    @client = client
+    @table = table
+    @column = column
+  end
+
+  def max
+    @client.select("MAX(#{@column})", @table)
+  end
+
+  def min
+    @client.select("MIN(#{@column})", @table)
+  end
+
+  def max_length
+    @client.select("MAX(LENGTH(#{@column}))", @table)
+  end
+
+  def min_length
+    @client.select("MIN(LENGTH(#{@column}))", @table)
+  end
+end

data/lib/sqlite2mysql/services/mysql.rb

@@ -0,0 +1,64 @@
+require 'mysql2'
+
+class MysqlClient
+  def initialize(*args)
+    @client = Mysql2::Client.new(*args)
+  end
+
+  def recreate(name)
+    @client.query("DROP DATABASE IF EXISTS #{name}")
+    @client.query("CREATE DATABASE #{name}")
+    @client.query("USE #{name}")
+  end
+
+  def build_from_schema(schema)
+    schema.keys.each do |table|
+      puts "Creating table: #{table}"
+      create_table(table, schema[table])
+    end
+  end
+
+  def create_table(table, fields)
+    puts create_table_query(table, fields)
+    @client.query(create_table_query(table, fields))
+  end
+
+  def insert_table(table, data)
+    data.each_slice(1000) do |slice|
+      @client.query(chunk_sql(table, slice))
+      print '.'
+    end
+  end
+
+  private
+
+  def chunk_sql(table, chunk)
+    values = []
+    chunk.each do |row|
+      values << "#{row_sql(row)}"
+    end
+    "INSERT INTO #{table} VALUES #{values.join(', ')}"
+  end
+
+  def row_sql(row)
+    values = row.map do |val|
+      if val.is_a?(String)
+        (val.empty? || val.nil? || val == '') ? nil : @client.escape(val)
+      else
+         val
+      end
+    end
+    "(\"#{values.join('", "')}\")"
+  end
+
+  def create_table_query(table, fields)
+    reserved_words = %w(key int)
+    query = "CREATE TABLE #{table} ("
+    cols = []
+    fields.each do |col|
+      col[:name] += '_1' if reserved_words.include?(col[:name])
+      cols << "#{col[:name]} #{col[:type]} #{'NOT NULL' if col[:notnull]}"
+    end
+    query + "#{cols.join(', ')})"
+  end
+end

data/lib/sqlite2mysql/services/sqlite.rb

@@ -0,0 +1,58 @@
+require 'sqlite3'
+require 'time'
+
+class SqliteClient
+  def initialize(filename, infer_column_types: false)
+    @db = SQLite3::Database.new(filename)
+    @infer = infer_column_types
+  end
+
+  def build_schema
+    schema = {}
+    tables = @db.execute 'SELECT name FROM sqlite_master WHERE type="table"'
+
+    tables.flatten.each do |t|
+      schema[t] = column_formatter(t)
+    end
+
+    schema
+  end
+
+  def get_data(table)
+    @db.execute("select * from #{table}")
+  end
+
+  def column_formatter(table)
+    columns = @db.execute("pragma table_info(#{table})")
+
+    formatted_columns = []
+    columns.each do |col|
+      formatted_columns << { name:    col[1],
+                             type:    type_getter(col[2], table, col[1]),
+                             notnull: col[3],
+                             default: col[4] }
+    end
+    formatted_columns
+  end
+
+  def type_getter(type, table, column)
+    if @infer
+      samples = @db.execute("SELECT #{column} FROM #{table} WHERE #{column} IS NOT NULL AND #{column} != '' ORDER BY RANDOM() LIMIT 100").flatten
+      type = TypeInferrer.new(samples, BoundFinder.new(self, table, column)).make_inference
+      puts "Inferring type of #{column} as #{type}"
+      return type
+    else
+      if type == '' || type.nil?
+        return 'varchar(255)'
+      elsif type.start_with?('float')
+        return 'float'
+      else
+        return type
+      end
+    end
+  end
+
+  def select(column, table)
+    @db.execute("SELECT #{column} FROM #{table}").flatten.first
+  end
+end

data/lib/sqlite2mysql/services/type_inferrer.rb

@@ -0,0 +1,85 @@
+class TypeInferrer
+  def initialize(samples, bound_finder)
+    @samples = samples
+    @bound_finder = bound_finder
+  end
+
+  def make_inference
+    possibilities = weigh_possibilities
+
+    case possibilities.max_by(&:last).first
+    when :int
+      return get_integer_type
+    when :float
+      return 'FLOAT'
+    when :date
+      return 'DATE'
+    when :datetime
+      return 'DATETIME'
+    when :string
+      return get_varchar_type
+    end
+  end
+
+  def weigh_possibilities
+    {
+      float: 0,
+      int: 0,
+      date: 0,
+      datetime: 0,
+      string: 0
+    }.tap do |possibilities|
+      @samples.each do |sample|
+        if date_or_time?(sample)
+          if sample.is_a?(Date) || Date.parse(sample).to_time == Time.parse(sample)
+            possibilities[:date] += 1
+          else
+            possibilities[:datetime] += 1
+          end
+        elsif sample.is_a?(Float) || sample.to_i > 0 && sample.to_f != sample.to_i.to_f
+          possibilities[:float] += 1
+        elsif sample.is_a?(Integer) || sample.to_i > 0 || sample == '0'
+          possibilities[:int] += 1
+        else
+          possibilities[:string] += 1
+        end
+      end
+    end
+  end
+
+  def get_integer_type
+    max = @bound_finder.max.to_i
+    min = @bound_finder.min.to_i
+    if min > -128 && max < 127
+      'TINYINT'
+    elsif min > -32768 && max < 32767
+      'SMALLINT'
+    elsif min > -8388608 && max < 8388607
+      'MEDIUMINT'
+    elsif min > -2147483648 && max < 2147483647
+      'INT'
+    else
+      'BIGINT'
+    end
+  end
+
+  def get_varchar_type
+    max_length = @bound_finder.max_length
+    max_length = 1 if max_length == 0 || max_length.nil?
+
+    return 'TEXT' if max_length > 255
+
+    "VARCHAR(#{max_length})"
+  end
+
+  private
+
+  def date_or_time?(sample)
+    sample.is_a?(Date) ||
+      sample.is_a?(Time) ||
+      sample.is_a?(String) &&
+      (%r((\d{1,2}[-\/]\d{1,2}[-\/]\d{4})|(\d{4}[-\/]\d{1,2}[-\/]\d{1,2})).match(sample) &&
+        Date.parse(sample) rescue false &&
+        Time.parse(sample) rescue false)
+  end
+end

data/lib/sqlite2mysql/version.rb

@@ -1,3 +1,3 @@
 module Sqlite2mysql
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/readme.md

@@ -0,0 +1,29 @@
+# sqlite2mysql
+
+### Installation
+
+    gem install sqlite2mysql
+
+Don't include it in your projects, that's not what it's for. This is a command line tool.
+
+### Usage
+
+Run like this:
+
+    sqlite2mysql test.db
+
+This will create a database called testdb in mysql with the exact schema and data as was in `test.db`. You can optionally name it something else in mysql like this:
+
+    sqlite2mysql test.db my_awesome_database
+
+Isn't that handy?
+
+This assumes you can log in as root to your localhost mysql database without a password.
+
+### Contributing
+
+Do.
+
+### License
+
+MIT.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqlite2mysql
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Alexander Standke
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-09-04 00:00:00.000000000 Z
+date: 2015-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,6 +100,11 @@ files:
 - bin/setup
 - exe/sqlite2mysql
 - lib/sqlite2mysql.rb
+- lib/sqlite2mysql/services/arguments.rb
+- lib/sqlite2mysql/services/bound_finder.rb
+- lib/sqlite2mysql/services/mysql.rb
+- lib/sqlite2mysql/services/sqlite.rb
+- lib/sqlite2mysql/services/type_inferrer.rb
 - lib/sqlite2mysql/version.rb
 - readme.md
 - sqlite2mysql.gemspec
@@ -128,3 +133,4 @@ signing_key:
 specification_version: 4
 summary: Simple tool to convert sqlite3 to mysql
 test_files: []
+has_rdoc: