dohruby 0.2.1 → 0.3

data/lib/doh/mysql/handle.rb

@@ -1,18 +1,22 @@
+require 'doh/core/array'
 require 'doh/logger/interface'
 require 'doh/mysql/error'
-require 'doh/mysql/raw_row_builder'
+require 'doh/mysql/typed_row_builder'
+require 'doh/mysql/writable_row'
+require 'doh/mysql/hash_row'
+require 'doh/mysql/smart_row'
 
 module DohDb
 
 class Handle
   def initialize(mysqlh, row_builder = nil)
     @mysqlh = mysqlh
-    @row_builder = row_builder || RawRowBuilder
+    @row_builder = row_builder || TypedRowBuilder.new
   end
 
   def close
     unless closed?
-      Doh::log.info("closing raw mysql handle: #@mysqlh")
+      dohlog.info("closing raw mysql handle: #@mysqlh")
       @mysqlh.close
       @mysqlh = nil
     end
@@ -25,68 +29,190 @@ class Handle
   def query(statement)
     generic_query(statement)
     retval = @mysqlh.affected_rows
-    Doh::log.info("affected #{retval} rows")
+    dohlog.info("affected #{retval} rows")
     retval
   end
 
   def update(statement)
     generic_query(statement)
     retval = @mysqlh.affected_rows
-    Doh::log.info("updated #{retval} rows")
+    dohlog.info("updated #{retval} rows")
     retval
   end
 
+  def update_row(statement)
+    retval = update(statement)
+    raise UnexpectedQueryResult, "updated #{retval} rows; expected 1" unless retval == 1
+    retval
+  end
+
+  def update_hash(hash, table, primary_key_value, primary_key_name)
+    items = hash.keys.collect {|key| key + ' = ' + hash[key].to_sql}
+    query("UPDATE #{table} SET #{items.join(', ')} WHERE #{primary_key_name} = #{primary_key_value}")
+  end
+
   def insert(statement)
     generic_query(statement)
     retval = @mysqlh.insert_id
-    Doh::log.info("insert_id was #{retval}")
+    dohlog.info("insert_id was #{retval}")
     retval
   end
 
+  def insert_hash(hash, table, ignore = false)
+    if ignore then keyword = 'INSERT IGNORE' else keyword = 'INSERT' end
+    insert_hash_helper(hash, table, keyword)
+  end
+
+  def replace_hash(hash, table)
+    insert_hash_helper(hash, table, 'REPLACE')
+  end
+
+  # The most generic form of select.
+  # It calls to_s on the statement object to facilitate the use of sql builder objects.
   def select(statement, row_builder = nil)
     result_set = generic_query(statement)
-    Doh::log.info("selected #{result_set.num_rows} rows")
-    rows = (row_builder || @row_builder.new).build_rows(result_set)
+    dohlog.info("selected #{result_set.num_rows} rows")
+    rows = get_row_builder(row_builder).build_rows(result_set)
     result_set.free
     rows
   end
 
-  def update_row(statement)
-    retval = update(statement)
-    raise QueryResultError.new("updated #{retval} rows; expected 1") unless retval == 1
-    retval
-  end
-
+  # Simple convenience wrapper around the generic select call.
+  # Throws an exception unless the result set is a single row.
+  # Returns the row selected.
   def select_row(statement, row_builder = nil)
     rows = select(statement, row_builder)
-    raise QueryResultError.new("selected #{rows.size} rows; expected 1") unless rows.size == 1
+    raise UnexpectedQueryResult, "selected #{rows.size} rows; expected 1" unless rows.size == 1
     rows[0]
   end
 
+  # Simple convenience wrapper around the generic select call.
+  # Throws an exception unless the result set is empty or a single row.
+  # Returns nil if the result set is empty, or the row selected.
   def select_optional_row(statement, row_builder = nil)
     rows = select(statement, row_builder)
-    raise QueryResultError.new("selected #{rows.size} rows; expected 0 or 1") if rows.size > 1
+    raise UnexpectedQueryResult, "selected #{rows.size} rows; expected 0 or 1" if rows.size > 1
     if rows.empty? then nil else rows[0] end
   end
 
+  # Simple convenience wrapper around select_row.
+  # Returns the first (and typically, the only) field from the selected row.
   def select_field(statement, row_builder = nil)
     select_row(statement, row_builder).at(0)
   end
 
+  # Simple convenience wrapper around select_optional_row.
+  # Returns the first (and typically, the only) field from the selected row, if any, or nil.
   def select_optional_field(statement, row_builder = nil)
     row = select_optional_row(statement, row_builder)
     row && row.at(0)
   end
 
+  # Rows in the result set must have 2 or more fields.
+  # If there are 2 fields, returns a hash where each key is the first field in the result set, and the value is the second field.
+  # If there are more than 2 fields, returns a hash where each key is the first field in the result set,
+  # and the value is the row itself, as a Hash, and without the field used as a key.
+  def select_transpose(statement, row_builder = nil)
+    rows = select(statement, row_builder)
+    return {} if rows.empty?
+    field_count = rows.first.size
+    if field_count < 2
+      raise UnexpectedQueryResult, "must select at least 2 fields in order to transpose"
+    elsif field_count == 2
+      rows.build_hash { |row| [row.at(0), row.at(1)] }
+    else
+      key_field = rows.first.keys.first
+      rows.build_hash do |row|
+        value = row.to_h
+        value.delete(key_field)
+        [row.at(0), value]
+      end
+    end
+  end
+
+  # Returns an array of arrays, where the individual arrays contain just the values from each database row -- they lack field names.
+  def select_values(statement, row_builder = nil)
+    select(statement, row_builder).collect { |row| row.values }
+  end
+
+  # Returns an array of the first (and typically, the only) field of every row in the result set.
+  def select_list(statement, row_builder = nil)
+    select(statement, row_builder).collect { |row| row.at(0) }
+  end
+
+  def multi_select(statement_infos, dflt_row_builder = nil)
+    statements = []
+    row_builders = []
+    statement_infos.each do |info|
+      if info.is_a?(Array)
+        statements.push(info.first)
+        row_builders.push(info.last)
+      else
+        statements.push(info)
+        row_builders.push(nil)
+      end
+    end
+    sqlstr = statements.join(';')
+
+    dohlog.info(sqlstr)
+    @mysqlh.query(sqlstr)
+    retval = []
+    while true
+      custom_builder = row_builders.shift
+      if @mysqlh.field_count > 0
+        result_set = @mysqlh.store_result
+        dohlog.info("selected #{result_set.num_rows} rows")
+        rows = get_row_builder(custom_builder || dflt_row_builder).build_rows(result_set)
+        result_set.free
+        retval.push(rows)
+      end
+      break unless @mysqlh.next_result
+    end
+    retval
+  rescue Exception => excpt
+    dohlog.error("caught exception during query: #{sqlstr}", excpt)
+    raise
+  end
+
 private
   def generic_query(statement)
     sqlstr = statement.to_s
-    Doh::log.info(sqlstr)
+    dohlog.info(sqlstr)
     @mysqlh.query(sqlstr)
+    @mysqlh.store_result if @mysqlh.field_count > 0
   rescue Exception => excpt
-    Doh::log.error("caught exception during query: #{sqlstr}", excpt)
+    dohlog.error("caught exception during query: #{sqlstr}", excpt)
     raise
   end
+
+  def insert_hash_helper(hash, table, keyword)
+    names = []
+    values = []
+    hash.each_pair do |key, value|
+      names.push(key)
+      values.push(value.to_sql)
+    end
+
+    insert("#{keyword} INTO #{table} (#{names.join(',')}) VALUES (#{values.join(',')})")
+  end
+
+  def get_row_builder(row_builder = nil)
+    if row_builder.nil?
+      @row_builder
+    elsif row_builder == :read
+      TypedRowBuilder.new(ReadOnlyRow)
+    elsif row_builder == :hash
+      TypedRowBuilder.new(HashRow)
+    elsif row_builder == :write
+      TypedRowBuilder.new(WritableRow)
+    elsif row_builder == :smart
+      TypedRowBuilder.new(SmartRow)
+    elsif row_builder.respond_to?('build_rows')
+      row_builder
+    else
+      TypedRowBuilder.new(row_builder)
+    end
+  end
 end
 
 end

data/lib/doh/mysql/hash_row.rb

@@ -0,0 +1,13 @@
+module DohDb
+
+class HashRow
+  def self.new(keys, values)
+    hash = {}
+    keys.each_with_index do |key, index|
+      hash[key] = values.at(index)
+    end
+    hash
+  end
+end
+
+end

data/lib/doh/mysql/load_sql.rb

@@ -10,6 +10,7 @@ end
 def self.load_sql(filenames, host, username, password, database)
   mysqlcmd = 'mysql' + mysql_arg(host, 'h') + mysql_arg(username, 'u') + mysql_arg(password, 'p') + ' ' + database
   io = IO::popen(mysqlcmd, 'r+')
+  dohlog.debug("loading sql file: " + filenames.first) if filenames.size == 1
   filenames.each do |elem|
     open(elem) {|file| io << file.read}
   end

data/lib/doh/mysql/metadata_util.rb

@@ -1,32 +1,73 @@
 require 'doh/mysql'
+require 'doh/mysql/to_sql'
 
-module Doh
+module DohDb
 
-@@cached_table_info = {}
+@@cached_column_info = {}
+def self.column_info(table, database = nil)
+  database ||= DohDb::connector_instance.database
+  lookup_str = database + '.' + table
+  return @@cached_column_info[lookup_str] if @@cached_column_info[lookup_str]
+  stmt = "SELECT column_name, is_nullable, data_type, character_maximum_length, numeric_scale, column_type FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=#{database.to_sql} AND TABLE_NAME=#{table.to_sql}"
+  @@cached_column_info[lookup_str] = DohDb::select_transpose(stmt)
+end
+
+def self.field_character_size(table, field, database = nil)
+  column_info(table, database).fetch(field, {}).fetch('character_maximum_length')
+end
 
-def self.chop_strings(table, row, ignore_non_db_fields = true, log = false)
-  result = ignore_non_db_fields ? {} : row.dup
-  self.column_info(table).each do |field, size, data_type|
-    Doh::log.debug("processing field: #{field.inspect}, size: #{size.inspect}, data_type: #{data_type.inspect}") if log
-    result[field] = row[field]
-    if row.keys.include?(field) && ['char', 'varchar'].include?(data_type.downcase) && row[field] && row[field].size > size.to_i
-      Doh::log.info("chopping field: #{row[field].inspect} to #{size} chars.") if log
-      result[field] = result[field].to_s[0, size.to_i]
+def self.chop_character_fields!(table, row)
+  column_info(table).each do |field, attribs|
+    maxlen = attribs['character_maximum_length']
+    if maxlen && row[field].to_s.size > maxlen
+      row[field] = row[field].to_s[0, maxlen]
     end
   end
-  result
+  row
+end
+
+def self.chop_character_fields(table, row)
+  chop_character_fields!(table, row.dup)
+end
+
+def self.field_exist?(table, field, database = nil)
+  column_info(table, database).key?(field)
+end
+
+@@primary_keys = {}
+def self.find_primary_key(table, database = nil)
+  if table.index('.')
+    database = table.before('.')
+    table = table.after('.')
+  else
+    database ||= DohDb::connector_instance.database
+  end
+
+  dbhash = @@primary_keys[database]
+  if dbhash.nil?
+    dbhash = DohDb::select_transpose("SELECT table_name, column_name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=#{database.to_sql} AND ORDINAL_POSITION=1")
+    raise "no information found for database #{database}" if dbhash.empty?
+    @@primary_keys[database] = dbhash
+  end
+
+  retval = dbhash[table]
+  raise "attempting to find_primary_key for table that doesn't exist: #{database}.#{table}" if retval.nil?
+  retval
+end
+
+def self.table_exist?(table, database = nil)
+  !find_primary_key(table, database).nil?
 end
 
-def self.column_info(table)
-  return @@cached_table_info[table] if @@cached_table_info[table]
-  @@cached_table_info[table] = query_column_info(table)
+def self.field_list(table, database = nil)
+  column_info(table, database).keys
 end
 
-def self.query_column_info(table)
-  column_info = DohDb::select("SELECT column_name, character_maximum_length, data_type FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='#{table}'")
-  column_info = column_info.collect {|column| [column['column_name'], column['character_maximum_length'], column['data_type']]}
-  Doh::log.debug("got column info for table: #{table}, info: #{column_info.inspect}")
-  column_info
+@@tables_by_database = {}
+def self.all_tables(database = nil)
+  database ||= DohDb::connector_instance.database
+  @@tables_by_database[database] ||
+    @@tables_by_database[database] ||= DohDb::select_list("SELECT table_name FROM information_schema.tables WHERE table_schema = '#{database}'")
 end
 
 end

data/lib/doh/mysql/migrate.rb

@@ -0,0 +1,122 @@
+require 'doh/mysql/load_sql'
+require 'doh/mysql/version'
+
+module DohDb
+
+class DatabaseMigrator
+  def initialize(dest_db = nil, source_db = nil, directory = nil)
+    @dest_db = dest_db || DohApp::config['primary_database']
+    @source_db = source_db || @dest_db
+    @table = @dest_db + '.version'
+    @directory = directory || File.join(DohApp::home, 'database', @source_db, 'migrate')
+  end
+
+  # Migrates to the latest version.
+  # Returns nil if true if no migrations took place, or [original_version, new_version].
+  def update
+    get_original_version
+    make_latest_version
+    migrate_result
+  end
+
+  # Rolls back one version.
+  # Returns nil if true if no migrations took place, or [original_version, new_version].
+  def rollback
+    get_original_version
+    make_earlier_version(@original_version - 1)
+    migrate_result
+  end
+
+  # Migrates to a specific version.
+  # Returns nil if true if no migrations took place, or [original_version, new_version].
+  def make_version(new_version)
+    get_original_version
+    if new_version > @original_version
+      make_later_version(new_version)
+    else
+      make_earlier_version(new_version)
+    end
+    migrate_result
+  end
+
+  def up_filename(version)
+    filename_prefix(version) + '_up.sql'
+  end
+
+  def down_filename(version)
+    filename_prefix(version) + '_down.sql'
+  end
+
+  def get_unlocked_migration
+    latest_version = DohDb::latest_database_version(@dest_db)
+    previous_version, previous_svn_revision = DohDb::locked_version_info(@dest_db)
+    raise 'you need to lock the database before adding a migration (migrate.rb -l)' if !previous_svn_revision
+    if latest_version == previous_version
+      upfile = up_filename(latest_version + 1)
+      downfile = down_filename(latest_version + 1)
+      `touch #{upfile}`
+      `touch #{downfile}`
+      `svn add #{upfile}`
+      `svn add #{downfile}`
+    else
+      upfile = up_filename(latest_version)
+      downfile = down_filename(latest_version)
+    end
+    [true, "up:#{upfile} down:#{downfile}"]
+  end
+
+private
+  def get_original_version
+    @original_version =  DohDb::select_field("SELECT version FROM #@table").to_i
+  end
+
+  def migrate_result
+    if @new_version.nil? || @new_version == @original_version
+      nil
+    else
+      [@original_version, @new_version]
+    end
+  end
+
+  def filename_prefix(version)
+    @directory + '/' + ('000' + version.to_s).lastn(3)
+  end
+
+  def load_sql_file(filename)
+    DohDb::load_sql_connector([filename], DohDb::connector_instance, @dest_db)
+  end
+
+  def set_version(number)
+    DohDb::query("UPDATE #@table SET version = #{number}")
+    @new_version = number
+  end
+
+  def make_latest_version
+    next_version = @original_version + 1
+    while File.exist?(filename = up_filename(next_version))
+      load_sql_file(filename)
+      set_version(next_version)
+      next_version += 1
+    end
+  end
+
+  def make_later_version(new_version)
+    next_version = @original_version + 1
+    while next_version <= new_version
+      load_sql_file(up_filename(next_version))
+      set_version(next_version)
+      next_version += 1
+    end
+  end
+
+  def make_earlier_version(new_version)
+    next_version = @original_version
+    while next_version > new_version
+      load_sql_file(down_filename(next_version))
+      next_version -= 1
+      set_version(next_version)
+    end
+  end
+end
+
+end

data/lib/doh/mysql/migrate_check.rb

@@ -0,0 +1,139 @@
+require 'doh/mysql/migrate'
+require 'doh/mysql/database_creator'
+require 'doh/mysql/version'
+require 'open3'
+
+module DohDb
+
+class MigrateChecker
+  def initialize(database = nil)
+    @database = database || DohApp::config['primary_database']
+    @data_directory = File.join(DohApp::home, 'database', @database)
+    @check_database = 'migration_check_database'
+    @sql_previous = "/tmp/check_migration_#{@database}_previous.sql"
+    @sql_previous_up = "/tmp/check_migration_#{@database}_previous_up.sql"
+    @sql_latest = "/tmp/check_migration_#{@database}_latest.sql"
+    @sql_latest_down = "/tmp/check_migration_#{@database}_latest_down.sql"
+  end
+
+  def run
+    svnout = `svn st #@data_directory`
+    unless svnout.strip.empty?
+      return [false, "svn status shows local changes in #@data_directory -- this needs to be resolved before migration can be checked"]
+    end
+
+    @latest_version = DohDb::latest_database_version(@database)
+    if @latest_version == 0
+      return [true, "no migrations to check, database version is still at 0"]
+    end
+
+    @previous_version, @previous_svn_revision = DohDb::locked_version_info(@database)
+    if @latest_version == @previous_version
+      return [true, "no unlocked migration exists, nothing to check"]
+    end
+
+    dbmigrator = DohDb::DatabaseMigrator.new(@check_database, @database)
+    if File.exist?(dbmigrator.up_filename(@latest_version + 1))
+      puts "multiple unlocked versions exist, should merge into version: #{@latest_version}"
+      exit 1
+    end
+    dohlog.debug("comparing database revisions #@previous_version and #@latest_version")
+
+    dohlog.debug("svn updating to revision #@previous_svn_revision, building database, dumping as #{@sql_previous}")
+    execute_cmd("svn up -r#@previous_svn_revision #@data_directory")
+
+    DohDb::DatabaseCreator.new.exclude_scripts.create_database_copy(@check_database, @database, true)
+    dump_sql(@sql_previous)
+
+    dohlog.debug("svn updating to current revision, migrating to latest database version, dumping as #{@sql_previous_up}")
+    execute_cmd("svn up #@data_directory")
+    dbmigrator.update
+    dump_sql(@sql_previous_up)
+
+    dohlog.debug("re-creating database from current database files, dumping as #{@sql_latest}")
+    begin
+      DohDb::DatabaseCreator.new.exclude_scripts.create_database_copy(@check_database, @database, true)
+    rescue Errno::ENOENT => e
+      puts 'got exception during database creation -- possible view in order.yml that doesn\'t exist?'
+      raise e
+    end
+    dump_sql(@sql_latest)
+
+    dohlog.debug("migrating to previous version, dumping as #{@sql_latest_down}")
+    DohDb::DatabaseMigrator.new(@check_database, @database).make_version(@previous_version)
+
+    dump_sql(@sql_latest_down)
+
+    diffstr = diff(@sql_previous, @sql_latest_down)
+    return [false, diffstr] unless diffstr.empty?
+
+    diffstr = diff(@sql_previous_up, @sql_latest)
+    return [false, diffstr] unless diffstr.empty?
+
+    File.delete(@sql_previous)
+    File.delete(@sql_previous_up)
+    File.delete(@sql_latest)
+    File.delete(@sql_latest_down)
+
+    [true, 'unlocked migration compared successfully']
+  end
+
+private
+  def execute_cmd(cmd)
+    dohlog.debug("executing: #{cmd}")
+    stdin, stdout, stderr = Open3.popen3(cmd)
+    stdoutstr = stdout.read
+    stdout.close
+    stderrstr = stderr.read
+    raise "stderr: #{stderrstr}" if !stderrstr.empty?
+    stdoutstr
+  end
+
+
+  def dump_sql(fname)
+    sql = execute_cmd("mysqldump -uroot -d #{@check_database}")
+    sql = remove_unwanted_comparisons(sql)
+    sql = sql.split(/[\r|\n]+/)
+    output = []
+    current_keys = []
+    #ignore order of keys
+    sql.each do |line|
+      if line =~ /(.*KEY[^,]*),?/
+        current_keys.push($1)
+      else
+        output.concat(current_keys.sort)
+        output.push(line)
+        current_keys = []
+      end
+    end
+
+    File.open(fname, 'wb') do |file|
+      output.each do |line|
+        file.write(line + "\n")
+      end
+    end
+  end
+
+  def remove_unwanted_comparisons(sql)
+    regexp = /--.-- Temporary table structure for view[^;]*;[^;]*;[^;]*;/m
+    regexp2 = /--.-- Table structure for table `[^;]*_deleteme`[^;]*;[^;]*;[^;]*;[^;]*;[^;]*;/m
+    sql.gsub(regexp, '').gsub(regexp2, '')
+  end
+
+  def remove_safe_diff_lines(result)
+    result.dup.delete_if do |elem|
+      elem =~ /\d+c\d+/ || elem =~ /-- Dump completed on.*/ || elem == '---' || elem =~ /.*ENGINE.*AUTO_INCREMENT=.*/ || elem =~ /.*ENGINE.*DEFAULT CHARSET=.*/
+    end
+  end
+
+  def diff(file1, file2)
+    result = remove_safe_diff_lines(execute_cmd("diff -b #{file1} #{file2}").split(/[\r|\n]+/))
+    if result.empty?
+      ''
+    else
+      "#{file1} and #{file2} differ\n" + result.join("\n")
+    end
+  end
+end
+
+end

data/lib/doh/mysql/parse.rb

@@ -15,11 +15,13 @@ end
 
 def self.parse_datetime(str)
   raise ArgumentError.new("unexpected value: " + str) unless str.size == 19
+  return nil if str == '0000-00-00 00:00:00'
   DateTime.parse(str)
 end
 
 def self.parse_date(str)
   raise ArgumentError.new("unexpected value: " + str) unless str.size == 10
+  return nil if str == '0000-00-00'
   Date.new(str[0..3].to_i, str[5..6].to_i, str[8..9].to_i)
 end
 

data/lib/doh/mysql/readonly_row.rb

@@ -1,61 +1,24 @@
-module DohDb
-
-class ReadOnlyRow
-  attr_reader :keys, :values
-
-  def initialize(keys, values)
-    @keys = keys
-    @values = values
-  end
-
-  def at(index)
-    @values.at(index)
-  end
-
-  def get(key)
-    index = @keys.index(key)
-    if index
-      @values.at(index)
-    else
-      nil
-    end
-  end
-  alias [] get
-
-  def key?(key)
-    !@keys.index(key).nil?
-  end
-
-  def to_a
-    retval = []
-    @keys.size.times {|index| retval.push([@keys[index], @values[index]])}
-    retval
-  end
+require 'doh/mysql/abstract_row'
 
-  def to_h
-    retval = {}
-    @keys.each_with_index {|key, index| retval[key] = @values.at(index)}
-    retval
-  end
-
-  def inspect
-    to_h.inspect
-  end
+module DohDb
 
-  def each_pair
-    @keys.size.times do |index|
-      yield(@keys.at(index), @values.at(index))
-    end
+class ReadOnlyRow < AbstractRow
+  # can accept 2 arguments: keys array, values array
+  # or 1 argument: a hash
+  def initialize(*args)
+    keys, values = parse_initialize_args(*args)
+    @keys = keys.freeze
+    @values = values.freeze
+    freeze
   end
 
-  undef id
   def method_missing(sym, *ignore)
     key = sym.to_s
     index = @keys.index(key)
     if index
       @values.at(index)
     else
-      raise RuntimeError.new("unknown method or field name: " + key)
+      raise RuntimeError.new("unknown field: " + key)
     end
   end
 end