dbgeni 0.10.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 662e5bb250525f1053d901edbae3d01ac733878f
+  data.tar.gz: d58cfb81adf70cd4674b47fa0259992fbc15d168
+SHA512:
+  metadata.gz: 3e1a1d3e269c4e96e38abd113c056b70bab9732de9533d2ccf13499a7de5882de56a36c320e442029bf1f27b438f3f810e8aad53b1113dd3d7193189dff49c64
+  data.tar.gz: 242e3e2f823249aed271d674b19eebe23af1553939be7af93cee81af64ef1a88e6bc082cd41b980ea3e747bf52d04b3ea7908e9542d6ef0f3ebf597334a5d7e6

data/Rakefile

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'rake'
+require 'rake/gempackagetask'
+
+
+spec = Gem::Specification.new do |s| 
+  s.name = "dbgeni"
+  s.version = "0.1.0"
+  s.author = "Stephen O'Donnell"
+  s.email = "stephen@betteratoracle.com"
+  s.homepage = "http://somewebsite.com"
+  s.platform = Gem::Platform::RUBY
+  s.summary = "A generic database installer"
+  s.files = FileList['lib/**/*.rb', 'bin/*', '[A-Z]*', 'test/**/*'].reject{ |fn| fn.include? "temp" }
+  s.require_path = "lib"
+  s.bindir       = "bin"
+  s.executables << 'dbgeni'	
+  s.description  = "Generic installer to manage database migrations for various databases"
+#  s.autorequire = "name"
+#  s.test_files = FileList["{test}/**/*test.rb"].to_a
+  s.has_rdoc = false
+#  s.extra_rdoc_files = ["README"]
+#  s.add_dependency("dependency", ">= 0.x.x")
+end
+ 
+Rake::GemPackageTask.new(spec) do |pkg| 
+  pkg.need_tar = true 
+end 
+

data/bin/dbgeni

@@ -0,0 +1,2 @@
+
+require 'dbgeni/cli'

data/lib/dbgeni/base.rb

@@ -0,0 +1,146 @@
+require 'dbgeni/logger'
+require 'dbgeni/blank_slate'
+require 'dbgeni/config'
+require 'dbgeni/environment'
+require 'dbgeni/file_converter'
+require 'dbgeni/base_code'
+require 'dbgeni/migration_list'
+require 'dbgeni/migration'
+require 'dbgeni/code_list'
+require 'dbgeni/code'
+require 'dbgeni/plugin'
+require 'dbgeni/exceptions/exception'
+require 'dbgeni/initializers/initializer'
+require 'dbgeni/migrators/migrator'
+require 'dbgeni/migrators/migrator_interface'
+require 'dbgeni/connectors/connector'
+require 'dbgeni/migration_cli'
+require 'dbgeni/dml_cli'
+
+require 'fileutils'
+
+module DBGeni
+
+  class Base
+    attr_reader :config
+   # attr_reader :migrations
+
+    # This pulls in all the code related methods - listing, applying, removing
+    # TODO - turn this into a class like with the migrations and DML
+    include DBGeni::BaseModules::Code
+
+    def self.installer_for_environment(config_file, environment_name=nil)
+      installer = self.new(config_file)
+      # If environment is nil, then it assumes there is only a single environment
+      # defined. So pass the nil value to select_environment - if there is more than
+      # one environment then select_environment will error out after making a call
+      # to get_environment.
+      installer.select_environment(environment_name)
+      installer
+    end
+
+    def initialize(config_file)
+      load_config(config_file)
+      initialize_logger
+    end
+
+    def select_environment(environment_name)
+      current_environment = selected_environment_name
+      if current_environment != nil && current_environment != environment_name
+        # disconnect from database as the connection may well have changed!
+        disconnect
+      end
+      @config.set_env(environment_name)
+    end
+
+    def selected_environment_name
+      begin
+        @config.env.__environment_name
+      rescue DBGeni::ConfigAmbiguousEnvironment
+        nil
+      end
+    end
+
+
+    def run_plugin(hook, object, params={})
+      pdir = @config.plugin_directory
+      if pdir && pdir != ''
+        unless @plugin_manager
+          @plugin_manager = DBGeni::Plugin.new
+          @plugin_manager.load_plugins(pdir)
+        end
+        @plugin_manager.run_plugins(hook,
+                                    {
+                                      :logger      => @logger,
+                                      :object      => object,
+                                      :environment => @config.env,
+                                      :connection  => connection
+                                    }.merge!(params)
+                                    )
+      end
+    end
+
+
+    def connect
+      raise DBGeni::NoEnvironmentSelected unless selected_environment_name
+      return @connection if @connection
+
+      @connection = DBGeni::Connector.initialize(@config)
+    end
+
+    def disconnect
+      if @connection
+        @connection.disconnect
+      end
+      @connection = nil
+    end
+
+    def connection
+      @connection ||= connect
+    end
+
+    def initialize_database
+      DBGeni::Initializer.initialize(connection, @config)
+    end
+
+    def database_initialized?
+      DBGeni::Initializer.initialized?(connection, @config)
+    end
+
+
+    def ensure_initialized
+      raise DBGeni::DatabaseNotInitialized unless database_initialized?
+    end
+
+    private
+
+    def initialize_logger
+      @logger = DBGeni::Logger.instance("#{@config.base_directory}/log")
+    end
+
+    def load_config(config)
+      @config = Config.load_from_file(config)
+    end
+
+    def delegate_to_migration_cli(meth, *args)
+      @migration_cli ||= MigrationCLI.new(self, @config, @logger)
+      @migration_cli.send(meth, *args)
+    end
+
+    def delegate_to_dml_cli(meth, *args)
+      @dml_cli ||= DmlCLI.new(self, @config, @logger)
+      @dml_cli.send(meth, *args)
+    end
+
+    def method_missing(meth, *args, &blk)
+      if meth =~ /migration/
+        delegate_to_migration_cli(meth.intern, *args)
+      elsif meth =~ /dml/
+        delegate_to_dml_cli(meth.to_s.gsub(/dml/,'migration').intern, *args)
+      else
+        super
+      end
+    end
+
+  end
+end

data/lib/dbgeni/base_code.rb

@@ -0,0 +1,143 @@
+module DBGeni
+  module BaseModules
+    module Code
+
+      # This module isn't much good on its own, but it is here purely to
+      # break up the code in the Base class. This module should be included
+      # into base for code operations to work correctly
+
+      def code
+        @code_list ||= DBGeni::CodeList.new(@config.code_dir)
+        @code_list.code
+      end
+
+      def current_code
+        ensure_initialized
+        code
+        @code_list.current(@config, connection)
+      end
+
+      def outstanding_code
+        ensure_initialized
+        code
+        @code_list.outstanding(@config, connection)
+      end
+
+      def list_of_code(list)
+        ensure_initialized
+        code
+        @code_list.list(list, @config, connection)
+      end
+
+      # Applying
+
+      def apply_all_code(force=nil)
+        ensure_initialized
+        code_files = code
+        if code_files.length == 0
+          raise DBGeni::NoOutstandingCode
+        end
+        apply_code_list(code_files, true)
+      end
+
+      def apply_outstanding_code(force=nil)
+        ensure_initialized
+        code_files = outstanding_code
+        if code_files.length == 0
+          raise DBGeni::NoOutstandingCode
+        end
+        apply_code_list(code_files, force)
+      end
+
+      def apply_list_of_code(object_names, force=nil)
+        ensure_initialized
+        code_files = list_of_code(object_names)
+        apply_code_list(code_files, force)
+      end
+
+      def apply_code(code_obj, force=nil)
+        ensure_initialized
+        begin
+          run_plugin(:before_code_apply, code_obj)
+          code_obj.apply!(@config, connection, force)
+          if code_obj.error_messages
+            # Oracle can apply procs that still have errors. This is expected. Other databases
+            # have errors raised for invalid procs, except when force is on, so this logic is
+            # for when they are being forced through.
+            if @config.db_type == 'oracle'
+              @logger.info "Applied #{code_obj.to_s} (with errors)\n\n#{code_obj.error_messages}\nFull errors in #{code_obj.logfile}\n\n"
+            else
+              @logger.error "Failed to apply #{code_obj.filename}. Errors in #{code_obj.logfile}\n\n#{code_obj.error_messages}\n\n"
+            end
+          else
+            @logger.info "Applied #{code_obj.to_s}"
+          end
+          run_plugin(:after_code_apply, code_obj)
+        rescue DBGeni::MigratorCouldNotConnect
+          @logger.error "Failed to connect to the database CLI"
+          raise DBGeni::CodeApplyFailed
+        rescue DBGeni::CodeApplyFailed => e
+          # The only real way code can get here is if the user had insufficient privs
+          # to create the proc, or there was other bad stuff in the proc file.
+          # In this case, dbgeni should stop - but also treat the error like a migration error
+          # as the error message will be in the logfile in the format standard SQL errors are.
+          @logger.error "Failed to apply #{code_obj.filename}. Errors in #{code_obj.logfile}\n\n#{code_obj.error_messages}\n\n"
+          raise DBGeni::CodeApplyFailed, e.to_s
+        end
+      end
+
+      def remove_all_code(force=nil)
+        ensure_initialized
+        code_files = code
+        if code_files.length == 0
+          raise DBGeni::NoCodeFilesExist
+        end
+        apply_code_list(code_files, force, false)
+      end
+
+      def remove_list_of_code(object_names, force=nil)
+        ensure_initialized
+        code_files = list_of_code(object_names)
+        apply_code_list(code_files, force, false)
+      end
+
+      def remove_code(code_obj, force=nil)
+        ensure_initialized
+        begin
+          run_plugin(:before_code_remove, code_obj)
+          code_obj.remove!(@config, connection, force)
+          @logger.info "Removed #{code_obj.to_s}"
+          run_plugin(:after_code_remove, code_obj)
+        rescue DBGeni::MigratorCouldNotConnect
+          @logger.error "Failed to connect to the database CLI"
+          raise DBGeni::CodeRemoveFailed
+        rescue DBGeni::CodeRemoveFailed => e
+          # Not sure if the code can even get here. Many if timeout waiting for lock on object?
+          # In this case, dbgeni should stop - but also treat the error like a migration error
+
+          # TODO - not sure this is even correct - dropping code doesn't create a logfile ...
+          @logger.error "Failed to remove #{code_obj.filename}. Errors in #{code_obj.logfile}"
+          raise DBGeni::CodeRemoveFailed
+        end
+      end
+
+      private
+
+      def apply_code_list(code_list, force, up=true)
+        params = {
+          :operation => up == true ? 'apply' : 'remove'
+        }
+        run_plugin(:before_modifying_code, code_list, params)
+        code_list.each do |c|
+          if up
+            apply_code(c, force)
+          else
+            remove_code(c, force)
+          end
+        end
+        run_plugin(:after_modifying_code, code_list, params)
+      end
+
+    end
+  end
+end

data/lib/dbgeni/blank_slate.rb

@@ -0,0 +1,24 @@
+module DBGeni
+
+  class ::BlankSlate
+    KEEP_METHODS = [
+                    :respond_to?,
+                    :__id__,
+                    :__send__,
+                    :instance_eval,
+                    :==,
+                    :equal?,
+                    :initialize,
+                    :method_missing,
+                    :instance_variable_set,
+                    :send,
+                    :alias_method
+                    ]
+    suppress_warnings {
+      (instance_methods - KEEP_METHODS).each do |m|
+        undef_method(m)
+      end
+    }
+  end
+
+end

data/lib/dbgeni/cli.rb

@@ -0,0 +1,96 @@
+# hack to get the CLI working witout being properly installed as a gem
+$:.unshift File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "lib"))
+
+$config_file      = './.dbgeni'
+$environment_name = nil
+$force            = false
+$username         = nil
+$password         = nil
+
+if index = ARGV.index('--config-file') or index = ARGV.index('-c')
+  unless ARGV[index+1]
+    puts "error: --config-file switch present, but no file specified"
+    exit
+  end
+  $config_file = ARGV[index+1]
+  # remove all references to config file from the argument list
+  ARGV.delete_at(index)
+  ARGV.delete_at(index)
+end
+
+if index = ARGV.index('--environment-name') or index = ARGV.index('-e')
+  unless ARGV[index+1]
+    puts "error: --environment-name switch present, but no environment specified"
+    exit
+  end
+  $environment_name = ARGV[index+1]
+  # remove all references to config file from the argument list
+  ARGV.delete_at(index)
+  ARGV.delete_at(index)
+end
+
+if index = ARGV.index('--password') or index = ARGV.index('-p')
+  unless ARGV[index+1]
+    puts "error: --password switch present, but no password specified"
+    exit
+  end
+  $password = ARGV[index+1]
+  ARGV.delete_at(index)
+  ARGV.delete_at(index)
+end
+
+if index = ARGV.index('--username') or index = ARGV.index('-u')
+  unless ARGV[index+1]
+    puts "error: --username switch present, but no user specified"
+    exit
+  end
+  $username = ARGV[index+1]
+  ARGV.delete_at(index)
+  ARGV.delete_at(index)
+end
+
+if index = ARGV.index('--force') or index = ARGV.index('-f')
+  $force = true
+  ARGV.delete_at(index)
+end
+
+$build_installer = Proc.new {
+  installer = DBGeni::Base.installer_for_environment($config_file, $environment_name)
+  if $password or $username
+    env = installer.config.env
+    env.__enable_loading
+    if $password
+      env.password $password
+    end
+    if $username
+      env.username $username
+    end
+    env.__completed_loading
+  end
+  installer
+}
+
+require 'dbgeni'
+
+begin
+  require 'dbgeni/commands/commands'
+rescue DBGeni::ConfigSyntaxError => e
+  puts "There is an error in the config file: #{e.to_s}"
+  exit(1)
+rescue DBGeni::ConfigFileNotExist => e
+  puts "The config file #{$config_file} does not exist"
+  exit(1)
+rescue DBGeni::ConfigFileNotSpecified => e
+  puts "No config file was specified"
+  exit(1)
+rescue DBGeni::ConfigAmbiguousEnvironment => e
+  puts "No environment specified and config file defines more than one environment"
+  exit(1)
+rescue DBGeni::EnvironmentNotExist => e
+  puts "The environment #{$environment_name} does not exist"
+  exit(1)
+rescue DBGeni::DBConnectionError => e
+  puts "Failed to establish databse connection: #{e.to_s}"
+  exit(1)
+end
+

data/lib/dbgeni/code.rb

@@ -0,0 +1,235 @@
+module DBGeni
+  class Code
+
+    attr_reader :directory, :filename, :type, :name, :logfile, :error_messages
+    PACKAGE_SPEC = 'PACKAGE SPEC'
+    PACKAGE_BODY = 'PACKAGE BODY'
+    PROCEDURE    = 'PROCEDURE'
+    FUNCTION     = 'FUNCTION'
+    TRIGGER      = 'TRIGGER'
+    TYPE         = 'TYPE'
+
+    UNKNOWN      = 'UNKNOWN'
+
+    APPLIED = 'Applied'
+
+    EXT_MAP = {
+               'pks' => PACKAGE_SPEC,
+               'pkb' => PACKAGE_BODY,
+               'prc' => PROCEDURE,
+               'fnc' => FUNCTION,
+               'trg' => TRIGGER,
+               'typ' => TYPE,
+               'sql' => UNKNOWN
+    }
+
+    def initialize(directory, filename)
+      @directory = directory
+      @filename  = filename
+      @runnable_code = nil
+      set_type
+      set_name
+    end
+
+    def ==(other)
+      if @directory == other.directory && @type == other.type && @name == other.name
+        true
+      else
+        false
+      end
+    end
+
+    def sort_field
+      # Normally alphabetical sorting is enough, but this means pkg specs sort after
+      # bodies. So need to do something about that. The simplest *hack* is to replace
+      # pks with pka to get it to sort before pkb, just for sorting purposes
+      sortable = @filename.gsub(/\.pks$/, '.pka')
+    end
+
+    def db_hash(config, connection)
+      results = connection.execute("select sequence_or_hash
+                                    from #{config.db_table}
+                                    where migration_name = ?
+                                    and   migration_type = ?", @name, @type)
+      results.length == 1 ? results[0][0] : nil
+    end
+#" THis line is just to fix the bad syntax highlighting in emacs!
+
+    def hash
+      # TODO what if file is empty?
+      @hash ||= Digest::SHA1.file(File.join(@directory, @filename)).hexdigest
+    end
+
+    # if the DB hash equals the file hash then it is current
+    def current?(config, connection)
+      hash == nil_to_s(db_hash(config, connection))
+    end
+
+    # if a hash is found in the DB then it is applied.
+    def applied?(config, connection)
+      db_hash(config, connection) ? true : false
+    end
+
+    # If there is no db_hash then its outstanding.
+    def outstanding?(config, connection)
+      db_hash(config, connection) ? false : true
+    end
+
+    def set_applied(config, connection)
+      env(config, connection)
+      insert_or_set_state(APPLIED)
+    end
+
+    def set_removed(config, connection)
+      env(config, connection)
+      remove_db_record
+    end
+
+    def apply!(config, connection, force=false)
+      env(config, connection)
+      ensure_file_exists
+      if current?(config, connection) and force != true
+        raise DBGeni::CodeModuleCurrent, self.to_s
+      end
+      migrator = DBGeni::Migrator.initialize(config, connection)
+      convert_code(config)
+      begin
+        migrator.compile(self)
+        set_applied(config,connection)
+      rescue DBGeni::MigratorCouldNotConnect
+        @error_messages = ""
+        raise DBGeni::MigratorCouldNotConnect
+      rescue DBGeni::MigrationContainsErrors
+        # MYSQL (and sybase is like mysql) and Oracle procedures are handled different.
+        # In Oracle if the code fails to
+        # compile, it can be because missing objects are not there, but in mysql the proc
+        # will compile fine if objects are missing - the only reason it seems to not compile
+        # is if the syntax is bad.
+        # Also, an error loading mysql proc will result in the proc not being on the DB at all.
+        # Can argue either way if DBGeni should stop on code errors. As many oracle compile errors
+        # could be caused by objects that have not been created yet, best for Oracle to continue,
+        # but for mysql I think it is best to stop.
+        if migrator.class.to_s =~ /Oracle/
+          @error_messages = migrator.migration_errors
+        elsif migrator.class.to_s =~ /(Mysql|Sybase)/
+          @error_messages = migrator.code_errors
+        end
+        unless force
+          raise DBGeni::CodeApplyFailed
+        end
+      ensure
+        @logfile = migrator.logfile
+        # Only set this if it has not been set in the exception handler
+        @error_messages ||= migrator.code_errors
+      end
+    end
+
+    def remove!(config, connection, force=false)
+      env(config, connection)
+      ensure_file_exists
+      migrator = DBGeni::Migrator.initialize(config, connection)
+      begin
+        migrator.remove(self)
+        remove_db_record
+      rescue Exception => e
+        raise DBGeni::CodeRemoveFailed, e.to_s
+      end
+    end
+
+    def to_s
+      "#{@type.ljust(12)} #{@name}"
+    end
+
+    def convert_code(config)
+      @runnable_code = DBGeni::FileConverter.convert(@directory, @filename, config)
+    end
+
+    def runnable_code
+      if @runnable_code
+        @runnable_code
+      else
+        File.join(@directory, @filename)
+      end
+    end
+
+    private
+
+    def insert_or_set_state(state)
+      # no hash in the DB means there is no db record...
+      results = db_hash(@config, @connection)
+      if results == nil then
+        add_db_record(state)
+      else
+        update_db_record(state)
+      end
+    end
+
+
+    def add_db_record(state)
+      results = @connection.execute("insert into #{@config.db_table}
+                                    (
+                                       sequence_or_hash,
+                                       migration_name,
+                                       migration_type,
+                                       migration_state,
+                                       start_dtm
+                                    )
+                                    values
+                                    (
+                                       ?,
+                                       ?,
+                                       ?,
+                                       ?,
+                                       #{@connection.date_placeholder('sdtm')}
+                                    )", hash, @name, @type, state, @connection.date_as_string(Time.now))
+    end
+
+
+    def update_db_record(state)
+      results = @connection.execute("update #{@config.db_table}
+                                    set sequence_or_hash  = ?,
+                                        completed_dtm     = #{@connection.date_placeholder('sdtm')},
+                                        migration_state   = ?
+                                    where migration_type  = ?
+                                    and   migration_name  = ?", hash, @connection.date_as_string(Time.now), state, @type, @name)
+    end
+
+
+    def remove_db_record
+      results = @connection.execute("delete from #{@config.db_table}
+                                    where migration_type  = ?
+                                    and   migration_name  = ?", @type, @name)
+    end
+
+    def set_type
+      @filename =~ /\.(.+)$/
+      if EXT_MAP.has_key?($1)
+        @type = EXT_MAP[$1]
+      else
+        raise DBGeni::UnknownCodeType, $1
+      end
+    end
+
+    def set_name
+      # If a filename starts with <digit>+_ then that part should be
+      # stripped away to get the real name. It is purely for ordering.
+      @filename =~ /^(?:\d+_{1}){0,1}(.+)\.[a-z]{3}$/  #/^[\d+_]*(.+)\.[a-z]{3}$/
+      @name = $1.upcase
+    end
+
+    def env(config, connection)
+      @config     = config
+      @connection = connection
+    end
+
+    def ensure_file_exists
+      unless File.exists? File.join(@directory, @filename)
+        raise DBGeni::CodeFileNotExist, File.join(@directory, @filename)
+      end
+    end
+
+    def nil_to_s(obj)
+      obj ? obj : ''
+    end
+  end
+end