dohmysql 0.1.8 → 0.1.9

data/bin/makedb

@@ -5,6 +5,7 @@ require 'doh/mysql/database_creator'
 
 opts = Doh::Options.new(
 {'drop_first' => [false, "-z", "--drop_first", "if true, will drop the database or tables before creating"] \
+,'skip_migrate' => [false, "-k", "--skip_migrate", "if true, will skip running migrates after creating"] \
 ,'database' => [Doh.config[:default_database], "-d", "--database <database>", "name of the source database -- defaults to Doh.config[:default_database], currently '#{Doh.config[:default_database]}'"] \
 ,'all' => [false, "-a", "--all", "create all databases"] \
 ,'target' => [nil, "-g", "--target <database>", "name of the target database -- defaults to same as source"] \
@@ -20,9 +21,9 @@ db_creator = DohDb::DatabaseCreator.new
 if opts.tables
   db_creator.create_tables(opts.database, opts.drop_first, opts.tables.split(/,/))
 elsif opts.all
-  db_creator.create_all_databases(opts.drop_first)
+  db_creator.create_all_databases(opts.drop_first, !opts.skip_migrate)
 elsif opts.target
-  db_creator.create_database_copy(opts.target, opts.database, opts.drop_first)
+  db_creator.create_database_copy(opts.target, opts.database, opts.drop_first, !opts.skip_migrate)
 else
-  db_creator.create_database(opts.database, opts.drop_first)
+  db_creator.create_database(opts.database, opts.drop_first, !opts.skip_migrate)
 end

data/bin/migrate

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+require 'doh/app/cli'
+require 'doh/mysql/migrate/runner'
+require 'doh/mysql/migrate/analyzer'
+
+COMMAND_INFO = [
+['make', '<name>', 'creates the files for a new migration'],
+['check', '<name>', 'checks that all migrations are valid'],
+['commit', '<name>', 'UNIMPLEMENTED: executes the check command, then git add,commit'],
+['apply', '<name>', 'applies one or more migrations'],
+['revert', '<name>', 'reverts one or more migrations'],
+['verify' ,'<name>', 'verifies that the migration has been integrated into the base files'],
+['merge', '<name>', 'UNIMPLEMENTED: executes the verify command, then git rm,commit'],
+].freeze
+COMMANDS = COMMAND_INFO.collect {|name, args, desc| name}.freeze
+NOTIFY_COMMANDS = %w(apply revert)
+
+cmd_detail = COMMAND_INFO.collect do |name, args, desc|
+  args = ' ' + args unless args.empty?
+  name_rpad = ' ' * (33 - (name.size + args.size))
+  "    #{name}#{args}#{name_rpad}#{desc}"
+end.join("\n")
+
+opts = Doh::Options.new(
+{'database' => [Doh.config[:default_database], "-d", "--database <database>", "name of the database to migrate -- defaults to config[:default_database], currently '#{Doh.config[:default_database]}'"] \
+}, true, "Commands:\n\n#{cmd_detail}")
+
+cmd_name = opts.varargs.shift
+cmd_args = opts.varargs
+
+unless COMMANDS.include?(cmd_name)
+  warn "unrecognized command #{cmd_name}"
+  exit 1
+end
+
+if opts.database.to_s.empty?
+  warn "You must specify a database, either here with -d or in config"
+  exit 1
+end
+
+if ['make', 'apply', 'revert'].include?(cmd_name)
+  success, msg = DohDb::MigrateRunner.new(opts.database).send(cmd_name, *cmd_args)
+elsif ['check', 'verify'].include?(cmd_name)
+  success, msg = DohDb::MigrateAnalyzer.new(opts.database).send(cmd_name, *cmd_args)
+end
+
+if success
+  puts "succcess: " + msg
+  if NOTIFY_COMMANDS.include?(cmd_name)
+    dohlog.notify("success: #{msg}")
+  end
+else
+  warn "***************** FAILURE **********************"
+  warn msg
+  if NOTIFY_COMMANDS.include?(cmd_name)
+    dohlog.notify("FAILURE: #{msg}")
+  end
+  exit 1
+end

data/lib/doh/mysql/database_creator.rb

@@ -2,27 +2,30 @@ require 'doh/core_ext/dir'
 require 'doh/mysql/handle'
 require 'doh/mysql/load_sql'
 require 'yaml'
+require 'doh/mysql/file_util'
 
 module DohDb
 
 class DatabaseCreator
+  MIGRATE_TABLE_DEF = "CREATE TABLE migrate (migrated_at DATETIME NOT NULL, name CHAR(50) NOT NULL)"
+
   def initialize(sqlfiles_directory = nil, connector = nil)
-    @sqlfiles_directory = sqlfiles_directory || File.join(Doh.root, 'data/mysql')
+    @sqlfiles_directory = sqlfiles_directory || DohDb.sql_files_path
     @connector = connector || DohDb.connector_instance
     @include_scripts = true
   end
 
-  def create_database(dbname, drop_first = false)
-    create_one_database(get_handle(''), dbname, dbname, drop_first)
+  def create_database(dbname, drop_first, include_migrates)
+    create_one_database(get_handle(''), dbname, dbname, drop_first, include_migrates)
   end
 
-  def create_database_copy(dest_db, source_db, drop_first = false)
-    create_one_database(get_handle(''), dest_db, source_db, drop_first)
+  def create_database_copy(dest_db, source_db, drop_first, include_migrates)
+    create_one_database(get_handle(''), dest_db, source_db, drop_first, include_migrates)
   end
 
-  def create_all_databases(drop_first = false)
+  def create_all_databases(drop_first, include_migrates)
     dbh = get_handle('')
-    Dir.directories(@sqlfiles_directory).each {|elem| create_one_database(dbh, elem, elem, drop_first)}
+    Dir.directories(@sqlfiles_directory).each {|elem| create_one_database(dbh, elem, elem, drop_first, include_migrates)}
   end
 
   def create_tables(database, drop_first, *table_and_view_names)
@@ -48,23 +51,21 @@ private
     File.join(@sqlfiles_directory, database, subdir, name) + '.sql'
   end
 
-  def find_files(source_db, subdir, ext = '.sql')
-    path = File.join(@sqlfiles_directory, source_db, subdir)
-    return [] unless File.exist?(path)
-    Dir.entries(path).find_all {|entry| entry.end_with?(ext)}.sort.collect {|elem| File.join(path, elem)}
+  def find_files(glob)
+    Dir.glob(File.join(@sqlfiles_directory, glob)).sort
   end
 
   def view_files(source_db)
     path = File.join(@sqlfiles_directory, source_db, 'views')
     return [] unless File.exist?(path)
     ordered_filenames = YAML.load_file(File.join(path, 'order.yml')).collect {|uqfn| File.join(path, uqfn) + '.sql'}
-    ordered_filenames + (find_files(source_db, 'views') - ordered_filenames)
+    ordered_filenames + (find_files("#{source_db}/views/*.sql") - ordered_filenames)
   end
 
   def create_base_table(dbh, database, table_name, drop_first)
     dbh.query("DROP TABLE IF EXISTS #{table_name}") if drop_first
     files = [sql_filename(database, 'tables', table_name)]
-    inserts_file = sql_filename(database, 'insert_sql', table_name)
+    inserts_file = sql_filename(database, 'inserts', table_name)
     files.push(inserts_file) if File.exist?(inserts_file)
     DohDb.load_sql(dbh.config, files)
   end
@@ -74,19 +75,34 @@ private
     DohDb.load_sql(dbh.config, [sql_filename(database, 'views', view_name)])
   end
 
-  def create_one_database(dbh, dest_db, source_db, drop_first)
+  def create_one_database(dbh, dest_db, source_db, drop_first, include_migrates)
     dohlog.info("creating database " + dest_db + " from source files at " + File.join(@sqlfiles_directory, source_db))
     dbh.query("DROP DATABASE IF EXISTS " + dest_db) if drop_first
 
     dbh.query("CREATE DATABASE " + dest_db)
     dbh.query("USE " + dest_db)
+    dbh.query(MIGRATE_TABLE_DEF)
 
     @connector.config[:database] = dest_db
 
-    files = find_files(source_db, 'tables') + find_files(source_db, 'insert_sql') + view_files(source_db)
+    files = find_files("#{source_db}/tables/*.sql") + find_files("#{source_db}/inserts/*.sql") + view_files(source_db)
     DohDb.load_sql(@connector.config, files)
-    return unless @include_scripts
-    find_files(source_db, 'insert_scripts', '.rb').each do |filename|
+    run_scripts(source_db) if @include_scripts
+    apply_migrates(dbh, source_db) if include_migrates
+  end
+
+  def apply_migrates(dbh, source_db)
+    apply_files = find_files("#{source_db}/migrate/*_apply.sql")
+    DohDb.load_sql(@connector.config, apply_files)
+    migrate_names = apply_files.collect {|path| File.basename(path).partition('_apply').first}
+    # NOTE: could package these up into one insert, but it is very small, and will have very few migrates, so not a big deal
+    migrate_names.each do |name|
+      dbh.query("INSERT INTO migrate SET migrated_at = NOW(), name = #{name.to_sql}")
+    end
+  end
+
+  def run_scripts(source_db)
+    find_files("#{source_db}/scripts/*.rb").each do |filename|
       dohlog.info("loading file: #{filename}")
       load(filename)
     end

data/lib/doh/mysql/file_util.rb

@@ -0,0 +1,11 @@
+module DohDb
+
+def self.sql_files_path(database = nil)
+  if database
+    File.join(Doh.root, "data/mysql/#{database}")
+  else
+    File.join(Doh.root, 'data/mysql')
+  end
+end
+
+end

data/lib/doh/mysql/load_sql.rb

@@ -8,7 +8,7 @@ end
 
 # NOTE: this doesn't work on jruby, but keeping it around in case it's determined to be significantly faster than the more portable version in place now
 # in which case, we can use the popen version on the rubys that it works with and the other one as needed
-def self.load_sql_using_popen(dbconfig, filenames)
+def self.load_sql_using_one_popen(dbconfig, filenames)
   dohlog.debug("loading sql file: " + filenames.first) if filenames.size == 1
 
   mysqlcmd = 'mysql' + mysql_arg(dbconfig[:host], 'h') + mysql_arg(dbconfig[:username], 'u') + mysql_arg(dbconfig[:password], 'p') + ' ' + dbconfig[:database]
@@ -19,7 +19,8 @@ def self.load_sql_using_popen(dbconfig, filenames)
   io.close
 end
 
-def self.load_sql(dbconfig, filenames)
+# this works on JRuby, but is slower than popen
+def self.load_sql_using_each_backtick(dbconfig, filenames)
   dohlog.debug("loading sql file: " + filenames.first) if filenames.size == 1
 
   basecmd = 'mysql' + mysql_arg(dbconfig[:host], 'h') + mysql_arg(dbconfig[:username], 'u') + mysql_arg(dbconfig[:password], 'p') + ' ' + dbconfig[:database] + ' < '
@@ -28,5 +29,25 @@ def self.load_sql(dbconfig, filenames)
   end
 end
 
+# pass through so we can change implemenation easily
+def self.load_sql(dbconfig, filenames)
+  DohDb.load_sql_using_each_backtick(dbconfig, filenames)
+end
+
+# trying Open3 now for better error handling; unknown as of yet if this works under JRuby
+def self.load_sql_using_each_open3(dbconfig, filenames)
+  dohlog.debug("loading sql file: " + filenames.first) if filenames.size == 1
+
+  basecmd = 'mysql' + mysql_arg(dbconfig[:host], 'h') + mysql_arg(dbconfig[:username], 'u') + mysql_arg(dbconfig[:password], 'p') + ' ' + dbconfig[:database] + ' < '
+  filenames.each do |elem|
+    mysqlcmd = "#{basecmd} #{elem}"
+    stdin, stdout, stderr = Open3.popen3(mysqlcmd)
+    stdoutstr = stdout.read
+    stdout.close
+    stderrstr = stderr.read
+    raise "#{stderrstr}" if !stderrstr.empty?
+  end
+end
+
 end
 

data/lib/doh/mysql/migrate/analyzer.rb

@@ -0,0 +1,118 @@
+require 'doh/mysql/migrate/runner'
+require 'doh/mysql/database_creator'
+require 'open3'
+
+module DohDb
+
+class MigrateAnalyzer
+  CHECK_DATABASE = 'dohmysql_migrate_check'
+  FILE_PREFIX = '/tmp/dohmysql_migrate_check_'
+
+  def initialize(database)
+    @database = database
+    @directory = File.join(DohDb.sql_files_path(@database), 'migrate')
+    @sql_original = FILE_PREFIX + "#{@database}_original.sql"
+    @sql_final = FILE_PREFIX + "#{@database}_final.sql"
+  end
+
+  def check(migrate_name)
+    DohDb::DatabaseCreator.new.create_database_copy(CHECK_DATABASE, @database, true)
+    dump_sql(@sql_original)
+
+    runner = DohDb::MigrateRunner.new(@database)
+
+    success, msg = runner.apply(migrate_name)
+    return [success, msg] if !success
+
+    success, msg = runner.revert(migrate_name)
+    return [success, msg] if !success
+
+    dump_sql(@sql_final)
+
+    diffstr = diff(@sql_original, @sql_final)
+    return [false, diffstr] unless diffstr.empty?
+
+    File.delete(@sql_original)
+    File.delete(@sql_final)
+
+    [true, "migration #{migrate_name} checked successfully"]
+  end
+
+  def verify(migrate_name)
+    DohDb::DatabaseCreator.new.create_database_copy(CHECK_DATABASE, @database, true, false)
+    dump_sql(@sql_original)
+
+    runner = DohDb::MigrateRunner.new(@database)
+
+    success, msg = runner.apply(migrate_name)
+    return [success, msg] if !success
+
+    dump_sql(@sql_final)
+
+    diffstr = diff(@sql_original, @sql_final)
+    return [false, diffstr] unless diffstr.empty?
+
+    File.delete(@sql_original)
+    File.delete(@sql_final)
+
+    [true, "migration #{migrate_name} verified successfully"]
+  end
+
+private
+  def execute_cmd(cmd)
+    dohlog.debug("executing: #{cmd}")
+    stdin, stdout, stderr = Open3.popen3(cmd)
+    stdoutstr = stdout.read
+    stdout.close
+    stderrstr = stderr.read
+    raise "stderr: #{stderrstr}" if !stderrstr.empty?
+    stdoutstr
+  end
+
+  def dump_sql(fname)
+    sql = execute_cmd("mysqldump -uroot -d #{CHECK_DATABASE}")
+    sql = remove_unwanted_comparisons(sql)
+    sql = sql.split(/[\r|\n]+/)
+    output = []
+    current_keys = []
+    #ignore order of keys
+    sql.each do |line|
+      if line =~ /(.*KEY[^,]*),?/
+        current_keys.push($1)
+      else
+        output.concat(current_keys.sort)
+        output.push(line)
+        current_keys = []
+      end
+    end
+
+    File.open(fname, 'wb') do |file|
+      output.each do |line|
+        file.write(line + "\n")
+      end
+    end
+  end
+
+  def remove_unwanted_comparisons(sql)
+    regexp = /--.-- Temporary table structure for view[^;]*;[^;]*;[^;]*;/m
+    regexp2 = /--.-- Table structure for table `[^;]*_deleteme`[^;]*;[^;]*;[^;]*;[^;]*;[^;]*;/m
+    sql.gsub(regexp, '').gsub(regexp2, '')
+  end
+
+  def remove_safe_diff_lines(result)
+    result.dup.delete_if do |elem|
+      elem =~ /\d+c\d+/ || elem =~ /-- Dump completed on.*/ || elem == '---' || elem =~ /.*ENGINE.*AUTO_INCREMENT=.*/ || elem =~ /.*ENGINE.*DEFAULT CHARSET=.*/
+    end
+  end
+
+  def diff(file1, file2)
+    result = remove_safe_diff_lines(execute_cmd("diff -b #{file1} #{file2}").split(/[\r|\n]+/))
+    if result.empty?
+      ''
+    else
+      "#{file1} and #{file2} differ\n" + result.join("\n")
+    end
+  end
+end
+
+end

data/lib/doh/mysql/migrate/runner.rb

@@ -0,0 +1,63 @@
+require 'doh/mysql/load_sql'
+
+module DohDb
+
+class MigrateRunner
+  def initialize(database)
+    @database = database
+    @table = @database + '.migrate'
+    @directory = File.join(DohDb.sql_files_path(@database), 'migrate')
+    @config = DohDb::connector_instance.config
+  end
+
+  def make(migrate_name)
+    apply_fname = apply_filename(migrate_name)
+    if File.exist?(apply_fname)
+      return [false, "it appears that migration #{migrate_name} already exists"]
+    end
+    `touch #{apply_fname}`
+    `touch #{revert_filename(migrate_name)}`
+    [true, "files for migration #{migrate_name} created in #{@directory}"]
+  end
+
+  def apply(migrate_name)
+    if migrate_exist?(migrate_name)
+      return [false, "migration #{migrate_name} has already been applied"]
+    end
+    load_sql(apply_filename(migrate_name))
+    Doh.db.query("INSERT INTO #@table SET migrated_at = NOW(), name = #{migrate_name.to_sql}")
+    [true, "migration #{migrate_name} applied successfully"]
+  rescue Exception => excpt
+    [false, excpt.message]
+  end
+
+  def revert(migrate_name)
+    unless migrate_exist?(migrate_name)
+      return [false, "migration #{migrate_name} can't be reverted until it has been applied"]
+    end
+    load_sql(revert_filename(migrate_name))
+    Doh.db.query("DELETE FROM #@table WHERE name = #{migrate_name.to_sql}")
+    [true, "migration #{migrate_name} reverted successfully"]
+  rescue Exception => excpt
+    [false, excpt.message]
+  end
+
+private
+  def apply_filename(migrate_name)
+    File.join(@directory, "#{migrate_name}_apply.sql")
+  end
+
+  def revert_filename(migrate_name)
+    File.join(@directory, "#{migrate_name}_revert.sql")
+  end
+
+  def load_sql(filename)
+    DohDb.load_sql_using_each_open3(@config, [filename])
+  end
+
+  def migrate_exist?(migrate_name)
+    Doh.db.select_field("SELECT COUNT(*) FROM #@table WHERE name = #{migrate_name.to_sql}") > 0
+  end
+end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dohmysql
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
   prerelease: 
 platform: ruby
 authors:
@@ -10,11 +10,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-02 00:00:00.000000000Z
+date: 2012-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dohroot
-  requirement: &70310724663340 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -22,21 +22,31 @@ dependencies:
         version: 0.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70310724663340
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 - !ruby/object:Gem::Dependency
   name: dohutil
-  requirement: &70310724662120 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.2.1
+        version: 0.2.3
   type: :runtime
   prerelease: false
-  version_requirements: *70310724662120
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.2.3
 - !ruby/object:Gem::Dependency
   name: mysql2
-  requirement: &70310724660480 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,10 +54,15 @@ dependencies:
         version: 0.3.11
   type: :runtime
   prerelease: false
-  version_requirements: *70310724660480
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.3.11
 - !ruby/object:Gem::Dependency
   name: sqlstmt
-  requirement: &70310724658640 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -55,10 +70,15 @@ dependencies:
         version: 0.1.1
   type: :runtime
   prerelease: false
-  version_requirements: *70310724658640
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.1
 - !ruby/object:Gem::Dependency
   name: dohtest
-  requirement: &70310724657600 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -66,17 +86,24 @@ dependencies:
         version: 0.1.8
   type: :development
   prerelease: false
-  version_requirements: *70310724657600
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.8
 description: wrapper classes around low level mysql gem to provide a better interface
 email:
 - devinfo@atpsoft.com
 executables:
 - makedb
+- migrate
 extensions: []
 extra_rdoc_files:
 - MIT-LICENSE
 files:
 - bin/makedb
+- bin/migrate
 - lib/doh/mysql/abstract_row.rb
 - lib/doh/mysql/cache_connector.rb
 - lib/doh/mysql/connector_instance.rb
@@ -85,10 +112,13 @@ files:
 - lib/doh/mysql/database_creator.rb
 - lib/doh/mysql/db_date.rb
 - lib/doh/mysql/error.rb
+- lib/doh/mysql/file_util.rb
 - lib/doh/mysql/handle.rb
 - lib/doh/mysql/hash_row.rb
 - lib/doh/mysql/load_sql.rb
 - lib/doh/mysql/metadata_util.rb
+- lib/doh/mysql/migrate/analyzer.rb
+- lib/doh/mysql/migrate/runner.rb
 - lib/doh/mysql/raw_row_builder.rb
 - lib/doh/mysql/readonly_row.rb
 - lib/doh/mysql/smart_row.rb
@@ -128,7 +158,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.15
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: friendly mysql client interface
@@ -143,3 +173,4 @@ test_files:
 - test/to_sql.dt.rb
 - test/typecasting.dt.rb
 - test/writable_row.dt.rb
+has_rdoc: