dbgeni 0.10.0

data/lib/dbgeni/migration_cli.rb

@@ -0,0 +1,204 @@
+module DBGeni
+  class MigrationCLI
+
+    def initialize(base_installer, config, logger)
+      @base       = base_installer
+      @config     = config
+      @logger     = logger
+      set_plugin_hooks
+    end
+
+    def migrations
+      @migration_list ||= DBGeni::MigrationList.new(@config.migration_directory) unless @migration_list
+      @migration_list.migrations
+    end
+
+    def outstanding_migrations
+      ensure_initialized
+      migrations
+      @migration_list.outstanding(@config, connection)
+    end
+
+    def applied_migrations
+      ensure_initialized
+      migrations
+      @migration_list.applied(@config, connection)
+    end
+
+    def applied_and_broken_migrations
+      ensure_initialized
+      migrations
+      @migration_list.applied_and_broken(@config, connection)
+    end
+
+    def list_of_migrations(list)
+      ensure_initialized
+      migrations
+      @migration_list.list(list, @config, connection)
+    end
+
+    # Applying
+
+    def apply_all_migrations(force=nil)
+      ensure_initialized
+      migrations = outstanding_migrations
+      if migrations.length == 0
+        raise DBGeni::NoOutstandingMigrations
+      end
+      apply_migration_list(migrations, force)
+    end
+
+    def apply_next_migration(force=nil)
+      ensure_initialized
+      migrations = outstanding_migrations
+      if migrations.length == 0
+        raise DBGeni::NoOutstandingMigrations
+      end
+      apply_migration_list([migrations.first], force)
+    end
+
+    def apply_until_migration(migration_name, force=nil)
+      ensure_initialized
+      milestone = find_migration(migration_name)
+      outstanding = outstanding_migrations
+      index = outstanding.index milestone
+      unless index
+        # milestone migration doesn't exist or is already applied.
+        raise MigrationNotOutstanding, milestone.to_s
+      end
+      apply_migration_list(outstanding[0..index], force)
+    end
+
+    def apply_list_of_migrations(migration_list, force=nil)
+      ensure_initialized
+      migration_files = list_of_migrations(migration_list)
+      apply_migration_list(migration_files, force)
+    end
+
+
+    def apply_migration(migration, force=nil)
+      ensure_initialized
+      begin
+        run_plugin(@before_up_run_plugin, migration)
+        migration.apply!(@config, connection, force)
+        @logger.info "Applied #{migration.to_s}"
+        run_plugin(@after_up_run_plugin, migration)
+      rescue DBGeni::MigratorCouldNotConnect
+        @logger.error "Failed to connect to the database CLI"
+        raise DBGeni::MigrationApplyFailed, migration.to_s
+      rescue DBGeni::MigrationApplyFailed
+        @logger.error "Failed #{migration.to_s}. Errors in #{migration.logfile}\n\n#{migration.error_messages}\n\n"
+        raise DBGeni::MigrationApplyFailed, migration.to_s
+      end
+    end
+
+    # rolling back
+
+    def rollback_all_migrations(force=nil)
+      ensure_initialized
+      migrations = applied_and_broken_migrations.reverse
+      if migrations.length == 0
+        raise DBGeni::NoAppliedMigrations
+      end
+      apply_migration_list(migrations, force, false)
+    end
+
+    def rollback_last_migration(force=nil)
+      ensure_initialized
+      migrations = applied_and_broken_migrations
+      if migrations.length == 0
+        raise DBGeni::NoAppliedMigrations
+      end
+      # the most recent one is at the end of the array!!
+      apply_migration_list([migrations.last], force, false)
+    end
+
+    def rollback_until_migration(migration_name, force=nil)
+      ensure_initialized
+      milestone = find_migration(migration_name)
+      applied = applied_and_broken_migrations.reverse
+      index = applied.index milestone
+      unless index
+        # milestone migration doesn't exist or is already applied.
+        raise DBGeni::MigrationNotApplied, milestone.to_s
+      end
+      # Note the triple ... in the range to exclude the end element
+      # This is because we don't want to rollback the final migration as its upto but not including
+      apply_migration_list(applied[0...index], force, false)
+    end
+
+    def rollback_list_of_migrations(migration_list, force=nil)
+      ensure_initialized
+      migration_files = list_of_migrations(migration_list).reverse
+      apply_migration_list(migration_files, force, false)
+    end
+
+    def rollback_migration(migration, force=nil)
+      ensure_initialized
+      begin
+        run_plugin(@before_down_run_plugin, migration)
+        migration.rollback!(@config, connection, force)
+        @logger.info  "Rolledback #{migration.to_s}"
+        run_plugin(@after_down_run_plugin, migration)
+      rescue DBGeni::MigratorCouldNotConnect
+        @logger.error "Failed to connect to the database CLI"
+        raise DBGeni::MigrationApplyFailed, migration.to_s
+      rescue DBGeni::MigrationApplyFailed
+        @logger.error "Failed #{migration.to_s}. Errors in #{migration.logfile}\n\n#{migration.error_messages}\n\n"
+        raise DBGeni::MigrationApplyFailed, migration.to_s
+      end
+    end
+
+    private
+
+    def apply_migration_list(migration_list, force, up=true)
+      # TODO - conflicted about how to handle exceptions.
+      # If before_running_migrations throws an exception no
+      # migrations will be run.
+      # If a migration throws an exception, then after_running_migrations
+      # will not be run.
+      params = {
+        :operation => up == true ? 'apply' : 'remove'
+      }
+      run_plugin(@before_running_plugin, migration_list, params)
+      migration_list.each do |m|
+        if up
+          apply_migration(m, force)
+        else
+          rollback_migration(m, force)
+        end
+      end
+      run_plugin(@after_running_plugin, migration_list, params)
+    end
+
+    def run_plugin(hook, object, params={})
+      @base.run_plugin(hook, object, params)
+    end
+
+    private
+
+    def find_migration(migration_name)
+      m = Migration.initialize_from_internal_name(@config.migration_directory, migration_name)
+    end
+
+    def set_plugin_hooks
+      @before_up_run_plugin   = :before_migration_up
+      @after_up_run_plugin    = :after_migration_up
+      @before_down_run_plugin = :before_migration_down
+      @after_down_run_plugin  = :after_migration_down
+      @before_running_plugin  = :before_running_migrations
+      @after_running_plugin   = :after_running_migrations
+    end
+
+
+    def ensure_initialized
+      @base.ensure_initialized
+    end
+
+    def connection
+      @base.connection
+    end
+
+
+  end
+end

data/lib/dbgeni/migration_list.rb

@@ -0,0 +1,91 @@
+module DBGeni
+
+  class MigrationList
+
+    attr_reader :migrations
+    attr_reader :migration_directory
+
+    # This is a bit of a hack. DML migrations reuse all of migrations, infact they were forced in
+    # later as an afterthought, so each migration will initially have a type of 'Migration'. This
+    # changes it to DML.
+    def self.new_dml_migrations(migration_directory)
+      obj = self.new(migration_directory)
+      obj.migrations.each do |m|
+        m.migration_type = 'DML'
+      end
+      obj
+    end
+
+    def initialize(migration_directory)
+      @migration_directory = migration_directory
+      file_list
+    end
+
+    # Returns an array of MIGRATIONS that have been applied
+    # ordered by oldest first
+    def applied(config, connection)
+      migrations_with_status(config, connection, DBGeni::Migration::COMPLETED)
+    end
+
+    def outstanding(config, connection)
+      migrations_with_status(config, connection,
+                             DBGeni::Migration::NEW,
+                             DBGeni::Migration::ROLLEDBACK,
+                             DBGeni::Migration::FAILED,
+                             DBGeni::Migration::PENDING)
+    end
+
+    def applied_and_broken(config, connection)
+      migrations_with_status(config, connection,
+                             DBGeni::Migration::COMPLETED,
+                             DBGeni::Migration::FAILED,
+                             DBGeni::Migration::PENDING)
+    end
+
+    def list(list_of_migrations, config, connection)
+      valid_migrations = []
+      list_of_migrations.each do |m|
+        mig_obj = Migration.initialize_from_internal_name(@migration_directory, m)
+        if i = @migrations.index(mig_obj)
+          valid_migrations.push @migrations[i]
+        else
+          raise DBGeni::MigrationFileNotExist, m
+        end
+      end
+      valid_migrations.sort {|x,y|
+        x.migration_file <=> y.migration_file
+      }
+    end
+
+
+    def migrations_with_status(config, connection, *args)
+      migrations = @migrations.select{|m|
+        args.include? m.status(config, connection)
+      }.sort {|x,y|
+        x.migration_file <=> y.migration_file
+      }
+    end
+
+    private
+
+    def file_list
+      begin
+        # Migrations usually come in pairs, so need to find just the 'up'
+        # ones here, otherwise there will be too many!
+        # The migration filename format is YYYYMMDDHHMM_<up / down >_title.sql
+        files = Dir.entries(@migration_directory).grep(/^\d{12}_up_.+\.sql$/).sort
+      rescue Exception => e
+        puts "Migrations directory: #{@migrations_directory}"
+        raise DBGeni::MigrationDirectoryNotExist, "Migrations directory: #{@migrations_directory}"
+      end
+      @migrations = Array.new
+      files.each do |f|
+        @migrations.push Migration.new(@migration_directory, f)
+      end
+    end
+
+  end
+
+end
+
+

data/lib/dbgeni/migrators/migrator.rb

@@ -0,0 +1,40 @@
+module DBGeni
+
+  module Migrator
+
+    def self.initialize(config, connection)
+      required_class = setup(config.db_type)
+      begin
+        required_method = required_class.method("new")
+      rescue NameError
+        raise DBGeni::InvalidMigratorForDBType, config.db_type
+      end
+      required_method.call(config, connection)
+    end
+
+#    def self.logfile(filename)
+#      name = File.basename(filename)
+#      "#{Time.now.strftime('%Y%m%d%H%M%S')}_#{name}"
+#    end
+
+    private
+
+    def self.setup(db_type)
+      begin
+        require "dbgeni/migrators/#{db_type}"
+      rescue
+        raise DBGeni::NoMigratorForDBType, db_type
+      end
+
+      required_class = nil
+      if Migrator.const_defined?(db_type.capitalize)
+        required_class = Migrator.const_get(db_type.capitalize)
+      else
+        raise DBGeni::NoMigratorForDBType, db_type
+      end
+      required_class
+    end
+
+  end
+
+end

data/lib/dbgeni/migrators/migrator_interface.rb

@@ -0,0 +1,51 @@
+module DBGeni
+  module Migrator
+
+    class MigratorInterface
+
+      attr_reader :logfile
+
+      def apply(migration, force=nil)
+        run_in_client(migration.runnable_migration, force)
+      end
+
+      def rollback(migration, force=nil)
+        run_in_client(migration.runnable_rollback, force)
+      end
+
+      def migration_errors
+      end
+
+      def verify(migration)
+        raise DBGeni::NotImplemented
+      end
+
+      def compile(code, force=false)
+        run_in_client(code.runnable_code, force, true)
+      end
+
+      def remove(code, force=false)
+        raise DBGeni::NotImplemented
+      end
+
+      def code_errors
+        raise DBGeni::NotImplemented
+      end
+
+      private
+
+      def run_in_client(file, force, is_proc=false)
+      end
+
+      def initialize(config, connection)
+        @config = config
+        # this is not actually used to run in the sql script
+        @connection = connection
+        @logfile    = nil
+        @log_dir    = DBGeni::Logger.instance("#{@config.base_directory}/log").detailed_log_dir
+        ensure_executable_exists
+      end
+    end
+
+  end
+end

data/lib/dbgeni/migrators/mysql.rb

@@ -0,0 +1,82 @@
+module DBGeni
+  module Migrator
+
+    class Mysql < DBGeni::Migrator::MigratorInterface
+
+      def initialize(config, connection)
+        super(config, connection)
+      end
+
+      # Defined in super ...
+      # def apply(migration, force=nil)
+      # end
+
+      # def rollback(migration, force=nil)
+      # end
+
+      def migration_errors
+        error = ''
+        # MYSQL prints the errors at the start of the log file
+        begin
+          fh = File.open(@logfile, 'r')
+          error = fh.readline
+          unless error =~ /^ERROR/
+            error = ''
+          end
+        ensure
+          fh.close if fh
+        end
+        error
+      end
+
+      def remove(code, force=false)
+        begin
+          @connection.execute(drop_command(code))
+        rescue Exception => e
+          unless e.to_s =~ /(procedure|function|trigger).+does not exist/i
+            raise DBGeni::CodeRemoveFailed, e.to_s
+          end
+        end
+      end
+
+      def code_errors
+        # In mysql the code errors ar just the same as migration errors
+        errors = migration_errors
+        if errors == ''
+          errors = nil
+        end
+        errors
+      end
+
+      private
+
+      def run_in_client(file, force, is_proc=false)
+        @logfile = "#{@log_dir}/#{File.basename(file)}"
+
+        z = @config.env
+        response = system("mysql -u#{z.username} -p#{z.password} -h#{z.hostname} -P#{z.port} -D#{z.database} -vvv #{force ? '--force' : ''} <#{file} >#{logfile} 2>&1")
+        unless response
+          raise DBGeni::MigrationContainsErrors
+        end
+      end
+
+      def ensure_executable_exists
+        unless Kernel.executable_exists?('mysql')
+          raise DBGeni::DBCLINotOnPath
+        end
+      end
+
+      def drop_command(code)
+        case code.type
+        when DBGeni::Code::TRIGGER
+          "drop trigger #{code.name.downcase}"
+        when DBGeni::Code::FUNCTION
+          "drop function #{code.name.downcase}"
+        when DBGeni::Code::PROCEDURE
+          "drop procedure #{code.name.downcase}"
+        end
+      end
+
+    end
+  end
+end

data/lib/dbgeni/migrators/oracle.rb

@@ -0,0 +1,211 @@
+module DBGeni
+  module Migrator
+
+    class Oracle < DBGeni::Migrator::MigratorInterface
+
+      def initialize(config, connection)
+        super(config, connection)
+      end
+
+      # def apply(migration, force=nil)
+      # end
+
+      # def rollback(migration, force=nil)
+      # end
+
+      def migration_errors
+        has_errors = false
+        buffer = []
+
+        begin
+          File.open(@logfile, 'r').each_line do |l|
+            buffer.push l
+            if buffer.length > 10
+              buffer.shift
+            end
+            if !has_errors && l =~ /^ERROR at line/
+              has_errors = true
+              next
+            end
+            # After we find the ERROR at line, the next line contains the error
+            # message, so we just want to consume it and then exit.
+            # The line we care about will be in the buffer, so just break and join
+            # the buffer.
+            if has_errors
+              break
+            end
+          end
+        rescue Errno::ENOENT
+          # assume this means the log was never written as, generally because
+          # sqlplus didn't connect to Oracle. In this case do nothing
+        end
+        buffer.join("")
+      end
+
+     # def verify(migration)
+     # end
+
+      def compile(code, force=false)
+        run_in_client(code.runnable_code, force, true)
+      end
+
+      def remove(code, force=false)
+        begin
+          @connection.execute(drop_command(code))
+        rescue Exception => e
+          unless e.to_s =~ /(object|trigger) .+ does not exist/
+            raise DBGeni::CodeRemoveFailed, e.to_s
+          end
+        end
+      end
+
+      def code_errors
+        # The error part of the file file either looks like:
+
+        # SQL> show err
+        # No errors.
+        # SQL> spool off
+
+        # or
+
+        # SQL> show err
+        # Errors for PACKAGE BODY PKG1:
+
+        # LINE/COL ERROR
+        # -------- -----------------------------------------------------------------
+        # 5/1      PLS-00103: Encountered the symbol "END" when expecting one of the
+        # following:
+        # Error messages here
+        # SQL> spool off
+
+        # In the first case, return nil, but in the second case get everything after show err
+
+        error_msg = ''
+        start_search = false
+        File.open(@logfile, 'r').each_line do |l|
+          if !start_search && l =~ /^SQL> show err/
+            start_search = true
+            next
+          end
+          if start_search
+            if l =~ /^No errors\./
+              error_msg = nil
+              break
+            elsif l =~ /^SQL> spool off/
+              break
+            else
+              error_msg << l
+            end
+          end
+        end
+        error_msg
+      end
+
+      private
+
+      def run_in_client(file, force, is_proc=false)
+        null_device = '/dev/null'
+        if Kernel.is_windows?
+          null_device = 'NUL:'
+        end
+
+        @logfile = "#{@log_dir}/#{File.basename(file)}"
+
+#        sql_parameters = parameters
+#        unless parameters
+#          sql_parameters = checkSQLPlusParameters(file)
+#        end
+
+        add_terminator = true
+        if is_proc
+          add_terminator = !file_contains_terminator?(file)
+        end
+
+        IO.popen("sqlplus -L #{@config.env.username}/#{@config.env.password}@#{@config.env.database} > #{null_device}", "w") do |p|
+          p.puts "set TERM on"
+          p.puts "set ECHO on"
+          #            if sql_parameters == ''
+          p.puts "set define off"
+          #            end
+          unless force
+            p.puts "whenever sqlerror exit 200"
+          end
+          #            p.puts "START #{File.basename(file)} #{sql_parameters}"
+          p.puts "spool #{@logfile}"
+          # Switch the current schema but only if the override is in the config file
+          if @config.env.install_schema
+            p.puts "alter session set current_schema=#{@config.env.install_schema};"
+          end
+          p.puts "START #{file}"
+          if is_proc
+            p.puts "/" if add_terminator
+            p.puts "show err"
+          end
+          p.puts "spool off"
+          p.puts "exit"
+        end
+          # When the pipe block ends, ruby sets $? with the exit status.  A
+          # good exit status is 0 (zero) anything else means it went wrong
+          # If $? is anything but zero, raise an exception.
+        if $?.exitstatus != 0
+          if $?.exitstatus != 200
+            raise DBGeni::MigratorCouldNotConnect
+          end
+          # Code compile errors never get here as they don't make sqlplus abort.
+          # But if the user does not have privs to create the proc / trigger etc,
+          # the code will abort to here via a insufficient privs error. Or if the code
+          # file doesn't contain 'create or replace .... ' and just garbage it can get here.
+          raise DBGeni::MigrationContainsErrors
+        end
+      end
+
+      def ensure_executable_exists
+        unless Kernel.executable_exists?('sqlplus')
+          raise DBGeni::DBCLINotOnPath
+        end
+      end
+
+      def drop_command(code)
+        case code.type
+        when DBGeni::Code::PACKAGE_SPEC
+          "drop package #{code.name}"
+        when DBGeni::Code::PACKAGE_BODY
+          "drop package body #{code.name}"
+        when DBGeni::Code::TRIGGER
+          "drop trigger #{code.name}"
+        when DBGeni::Code::FUNCTION
+          "drop function #{code.name}"
+        when DBGeni::Code::PROCEDURE
+          "drop procedure #{code.name}"
+        when DBGeni::Code::TYPE
+          "drop type #{code.name}"
+        when DBGeni::Code::UNKNOWN
+          derive_drop_command_from_db(code.name)
+        end
+      end
+
+      def file_contains_terminator?(filename)
+        has_slash = false
+        File.open(filename, 'r').each_line do |l|
+          if l =~ /^\s*\/\s*$/
+            has_slash = true
+            break
+          end
+        end
+        has_slash
+      end
+
+      def derive_drop_command_from_db(object_name)
+        # This is tricky because of packages - will give two objects with the same name :-/
+        raise CannotRemoveUnknownObject
+
+        # owner = @config.env.install_schema ? @config.env.install_schema : @config.env.username
+        # results = @connection.execute ("select object_type
+        #                                 from all_objects
+        #                                 where owner = :b1 and object_name = b2", owner.upcase, object_name.upcase)
+        # unless results
+      end
+
+    end
+  end
+end