database_loader 0.1.0

data/.gitignore

@@ -0,0 +1,3 @@
+pkg/*
+*.gem
+.bundle

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in database_loader.gemspec
+gemspec

data/README.md

@@ -0,0 +1,93 @@
+Database Loader
+===============
+
+Load SQL views, materialized views, grants, etc. into database.
+
+For tables, use Rails migrations.
+
+Currently it works only with **oracle_enahanced** adapter.
+
+## Installation
+
+Include the gem in your Gemfile:
+
+    gem "database_loader"
+
+In Rails 2.x you also need to add this to you Rakefile
+
+    require "database_loader/tasks"
+
+## Structure
+
+    rails_app/
+      db/
+        migrate/
+        sql/
+          local/
+            views/
+              report_v.sql
+          ext/
+            packages/
+              sync_api.sql
+            views/
+              users_v.sql
+            materialized_views/
+            indexes/
+            functions/
+            scripts/
+            grants/
+
+## Load generated SQL into database
+
+    rake db:sql:load:local
+    rake db:sql:load:ext
+
+## Preview generated SQL
+
+    rake db:sql:dump:local
+    rake db:sql:dump:ext
+
+## Packaging a new release
+
+In case you cannot directly access your external database,
+you can generate a package containing all your SQL files and send it to DBA.
+
+    rake db:sql:package:local NAME="v1"
+    rake db:sql:package:ext NAME="v1"
+
+## Configuration
+
+config/initializers/database_loader.rb
+
+    # Types that contain SQL files.
+    DatabaseLoader.types = [ :views, :materialized_views, :indexes, :packages, :functions, :scripts, :grants ]
+
+    # Use :erb, :erubis or :liquid to render SQL files.
+    DatabaseLoader.template_engine = :erb
+
+    # Path to store generated tar packages.
+    DatabaseLoader.package_path = "/tmp"
+
+    # Path to SH template that will be used to generate deployement script (see examples/template.sh)
+    DatabaseLoader.template_path = Rails.root.join("db", "sql", "template.sh")
+
+## Setting up database connections
+
+Convention is to create additional configurations for
+each environment and append the external schema name.
+
+config/database.yml
+
+    development:
+      username: local
+      password: ...
+
+    development_ext:
+      username: external
+      password: ...
+
+
+## TODO
+
+* SQL dependency handling.
+

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/database_loader.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "database_loader/version"
+
+Gem::Specification.new do |s|
+  s.name        = "database_loader"
+  s.version     = DatabaseLoader::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Edgars Beigarts"]
+  s.email       = ["1@wb4.lv"]
+  s.homepage    = "http://github.com/ebeigarts/database_loader"
+  s.description = %q{Load SQL views, materialized views, grants, etc. into database}
+  s.summary     = s.description
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/examples/template.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+echo "usage: $0 username password"
+
+MYDIR=`pwd`
+
+for sql_dir in views materialized_views packages indexes scripts do
+do
+  if test -r ${MYDIR}/${sql_dir}
+  then
+    for sql_file in ${MYDIR}/${sql_dir}/*
+    do
+      if test -r ${sql_file}
+      then
+        echo create ${sql_dir} ${sql_file}
+        sqlplus -s $1/$1 @${sql_file}
+      fi
+    done
+  fi
+done

data/lib/database_loader/railtie.rb

@@ -0,0 +1,8 @@
+module DatabaseLoader
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      DatabaseLoader.set_schemas
+      require "database_loader/tasks"
+    end
+  end
+end

data/lib/database_loader/ruby_file.rb

@@ -0,0 +1,23 @@
+module DatabaseLoader
+  class RubyFile < SqlFile
+    def render
+      "-- Cannot dump ruby file"
+    end
+    alias_method :to_s, :render
+    
+    def excerpt
+      "#{read.first(60)} ..."
+    end
+    
+    def statements
+      [self]
+    end
+    
+    def execute
+      content = read
+      ActiveRecord::Schema.define do
+        instance_eval(content)
+      end
+    end
+  end
+end

data/lib/database_loader/sql_file.rb

@@ -0,0 +1,32 @@
+module DatabaseLoader
+  class SqlFile
+    attr_accessor :path, :username
+
+    def initialize(path)
+      self.path = path
+    end
+
+    def type
+      path.split(/[\/\\]/)[-2].to_sym
+    end
+
+    def name
+      File.basename(path)
+    end
+
+    def read
+      File.read(path)
+    end
+
+    def render
+      Template.new(read).render("username" => username)
+    end
+    alias_method :to_s, :render
+
+    def statements
+      render.split(/\r?\n\/\r?\n/).reject(&:blank?).map do |str|
+        SqlStatement.new(str)
+      end
+    end
+  end
+end

data/lib/database_loader/sql_statement.rb

@@ -0,0 +1,26 @@
+module DatabaseLoader
+  class SqlStatement
+    attr_accessor :contents
+
+    def initialize(contents)
+      self.contents = contents
+    end
+
+    def excerpt
+      text = contents.gsub(/\-\-[^\r\n]*/, "").squish
+      if text.size > 60
+        "#{text.first(60)} ..."
+      else
+        text
+      end
+    end
+
+    def execute
+      ActiveRecord::Base.connection.execute(contents)
+    end
+
+    def to_s
+      contents
+    end
+  end
+end

data/lib/database_loader/tasks.rb

@@ -0,0 +1,136 @@
+require "database_loader"
+
+namespace :db do
+  namespace :sql do
+    namespace :load do
+      DatabaseLoader.schemas.each do |schema|
+        desc "Load SQL statements in #{schema}"
+        task schema do
+          DatabaseLoader.types.each do |type|
+            Rake::Task["db:sql:load:#{schema}:#{type}"].invoke
+          end
+        end
+        namespace schema do
+          DatabaseLoader.types.each do |type|
+            desc "Load SQL #{type.to_s.singularize} statements in #{schema}"
+            task type => :environment do
+              DatabaseLoader.connect_as(schema) do |connection_config|
+                DatabaseLoader.enable_dbms_output
+                DatabaseLoader.files(schema, type, ENV['NAME']).each do |file|
+                  puts "* #{file.type}/#{file.name} ".ljust(80, "-")
+                  file.username = connection_config[:username]
+                  file.statements.each do |statement|
+                    puts "  #{statement.excerpt}"
+                    begin
+                      statement.execute
+                      DatabaseLoader.dbms_output do |line|
+                        puts "! DBMS_OUTPUT: #{line}"
+                      end
+                    rescue => e
+                      $stderr.puts e.message
+                    end
+                  end
+                end
+                DatabaseLoader.invalid_objects.each do |h|
+                  puts "! #{h['name']} #{h['type']} #{h['status']}"
+                end
+                DatabaseLoader.errors.each do |h|
+                  puts "! #{h['name']} #{h['type']} #{h['attribute']} at line #{h['line']}: #{h['text']}"
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+
+    namespace :dump do
+      DatabaseLoader.schemas.each do |schema|
+        desc "Dump SQL statements in #{schema}"
+        task schema do
+          DatabaseLoader.types.each do |type|
+            Rake::Task["db:sql:dump:#{schema}:#{type}"].invoke
+          end
+        end
+        namespace schema do
+          DatabaseLoader.types.each do |type|
+            desc "Dump SQL #{type.to_s.singularize} statements in #{schema}"
+            task type => :environment do
+              connection_config = ActiveRecord::Base.connection.raw_connection.instance_variable_get("@config").symbolize_keys
+              DatabaseLoader.files(schema, type, ENV['NAME']).each do |file|
+                file.username = connection_config[:username]
+                puts "-- #{file.type}/#{file.name} ".ljust(80, "-")
+                puts file.to_s
+                puts
+              end
+            end
+          end
+        end
+      end
+    end
+
+    namespace :package do
+      DatabaseLoader.schemas.each do |schema|
+        desc "Package SQL files for remote installation in #{schema}. rake db:sql:package:#{schema} NAME=CM0000072322 COMMIT=master."
+        task schema => :environment do
+          unless ENV['NAME'].present?
+            STDERR.puts "You have to specify NAME environment"
+          end
+          if ENV['COMMIT']
+            files = `git diff --diff-filter=ACM --name-only #{ENV['COMMIT']} ./db/sql/#{schema}`.strip.split(/\n/).sort
+          else
+            files = `find ./db/sql/#{schema} -type f`.strip.split("\n").sort
+          end
+          connection_config = ActiveRecord::Base.connection.raw_connection.instance_variable_get("@config").symbolize_keys
+          pkg_name = ENV['NAME']
+          pkg_dir = File.join(DatabaseLoader.package_path, connection_config[:username].to_s.downcase)
+          # Package SQL files
+          mkdir_p(File.join(pkg_dir, pkg_name))
+          Dir[File.join(pkg_dir, pkg_name) + "/*/"].each do |dir|
+            rm_rf(dir)
+          end
+          DatabaseLoader.types.each do |type|
+            DatabaseLoader.files(schema, type, "*").each do |file|
+              next unless files.any? { |f| f.include?(file.name) }
+              file.username = connection_config[:username]
+              puts "-- #{file.type}/#{file.name} ".ljust(80, "-")
+              sql = file.to_s
+              sql += "\nEXIT\n" # tell sqlplus to exit
+              # puts sql
+              FileUtils.mkdir_p(File.join(pkg_dir, pkg_name, file.type.to_s))
+              File.open(File.join(pkg_dir, pkg_name, file.type.to_s, file.name), "wb") { |f| f.write sql }
+            end
+          end
+          # Generate shell script
+          sh_content = DatabaseLoader::Template.new(File.read(DatabaseLoader.template_path)).render({
+            "cm_number" => pkg_name.gsub(/[^\d]/, ''),
+            "application" => connection_config[:username]
+          })
+          File.open(File.join(pkg_dir, pkg_name, "#{pkg_name}.sh"), "wb") { |f| f.write(sh_content) }
+          # Generate summary
+          if ENV['COMMIT']
+            File.open(File.join(pkg_dir, pkg_name, "#{pkg_name}.txt"), "wb") do |f|
+              f.write(`git diff --stat #{ENV['COMMIT']} ./db/sql/#{schema}`)
+            end
+            File.open(File.join(pkg_dir, pkg_name, "#{pkg_name}.diff"), "wb") do |f|
+              f.write(`git diff #{ENV['COMMIT']} ./db/sql/#{schema}`)
+            end
+          end
+          # Try to open the folder for manual reviewing
+          system `open #{File.join(pkg_dir, pkg_name)}`
+          # Cleanup & compress
+          puts "Press any key to continue"; STDIN.getc
+          Dir.chdir(pkg_dir)
+          sh "tar cf #{pkg_name}.tar #{pkg_name}"
+          ### print
+          puts
+          # puts "# export TWO_TASK=..."
+          puts "Instructions for #{pkg_name}:"
+          puts "$ tar vxf #{pkg_name}.tar"
+          puts "$ cd #{pkg_name}"
+          puts "$ sh #{pkg_name}.sh <#{connection_config[:username]}_user> <#{connection_config[:username]}_pwd> <#{schema}_user> <#{schema}_pwd>"
+        end
+      end
+    end
+  end
+end

data/lib/database_loader/template.rb

@@ -0,0 +1,27 @@
+module DatabaseLoader
+  class Template
+    attr_accessor :contents
+
+    def initialize(contents)
+      self.contents = contents
+    end
+
+    def render(options = {})
+      case DatabaseLoader.template_engine
+      when :liquid
+        require "liquid"
+        Liquid::Template.parse(contents).render(options)
+      when :erb
+        require "erb"
+        struct = OpenStruct.new(options)
+        ERB.new(contents).result(struct.send(:binding))
+      when :erubis
+        require "erubis"
+        struct = OpenStruct.new(options)
+        Erubis::Eruby.new(contents).result(struct.send(:binding))
+      else
+        contents
+      end
+    end
+  end
+end

data/lib/database_loader/version.rb

@@ -0,0 +1,3 @@
+module DatabaseLoader
+  VERSION = "0.1.0"
+end

data/lib/database_loader.rb

@@ -0,0 +1,123 @@
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__))) unless $LOAD_PATH.include?(File.expand_path(File.dirname(__FILE__)))
+
+require "tmpdir"
+require "ostruct"
+
+require "database_loader/version"
+
+module DatabaseLoader
+  def self.set_schemas
+    if Rails.root
+      self.schemas = Dir[File.expand_path("#{Rails.root}/db/sql/*")].select { |d| File.directory?(d) }.map { |d| File.basename(d).to_sym }
+    else
+      self.schemas = [:local]
+    end
+  end
+
+  # Schemas that contain SQL files. :local will be used to load in SQL files in current database.
+  mattr_accessor :schemas
+  self.set_schemas
+
+  # Folders that contain SQL files.
+  mattr_accessor :types
+  self.types = [ :views, :materialized_views, :indexes, :packages, :functions, :scripts, :grants ]
+
+  # Use :erb, :erubis or :liquid to render SQL files.
+  mattr_accessor :template_engine
+  self.template_engine = :erb
+
+  # Path to store generated deployement packages.
+  mattr_accessor :package_path
+  self.package_path = Dir.tmpdir
+
+  # Path to SH template that will be used to generate deployement script.
+  mattr_accessor :template_path
+  self.template_path = File.dirname(__FILE__) + "/../../examples"
+
+  def self.files(schema, type = nil, name = nil)
+    type ||= "**"
+    name ||= "*"
+    files = []
+    Dir[File.expand_path("#{Rails.root}/db/sql/#{schema}/#{type}")].each do |dir_path|
+      if DatabaseLoader.types.include?(File.basename(dir_path).to_sym)
+        Dir["#{dir_path}/#{name}.sql"].sort.each do |path|
+          files << SqlFile.new(path)
+        end
+        Dir["#{dir_path}/#{name}.rb"].sort.each do |path|
+          files << RubyFile.new(path)
+        end
+      end
+    end
+    files
+  end
+
+  def self.invalid_objects
+    ActiveRecord::Base.connection.select_all(%{
+      SELECT object_name AS name, object_type AS type, status
+      FROM user_objects
+      WHERE object_name LIKE '#{ActiveRecord::Base.table_name_prefix[0..-2].upcase}%'
+        AND status = 'INVALID'
+        AND object_type like 'PACKAGE%'
+    })
+  end
+
+  def self.errors
+    ActiveRecord::Base.connection.select_all(%{
+      SELECT *
+      FROM USER_ERRORS
+      WHERE name LIKE '#{ActiveRecord::Base.table_name_prefix[0..-2].upcase}%'
+      ORDER BY name, type, sequence
+    })
+  end
+
+  def self.enable_dbms_output
+    plsql.dbms_output.enable(10_000)
+  end
+
+  def self.dbms_output
+    loop do
+      result = plsql.dbms_output.get_line(:line => '', :status => 0)
+      break unless result[:status] == 0
+      yield result[:line]
+    end
+  end
+
+  def self.connect_as(schema)
+    begin
+      connection_config = ActiveRecord::Base.connection.raw_connection.instance_variable_get("@config").symbolize_keys
+      connection_name = nil
+      # When running rake test|spec RAILS_ENV is development by default,
+      # so we need to guess config from current connection.
+      ActiveRecord::Base.configurations.each do |name, config|
+        # current configuration?
+        if connection_config.symbolize_keys == config.symbolize_keys
+          # current configuration has also _apps configuration?
+          if ActiveRecord::Base.configurations.any? { |other_name, _| other_name.to_s == "#{name}_#{schema}" }
+            connection_name = name
+            break
+          end
+        end
+      end
+      if schema != :local
+        unless connection_name
+          raise "Missing database.yml configuration for <RAILS_ENV>_#{schema}."
+        end
+        puts "* Loading SQL statements in #{connection_name.downcase}_apps"
+        ActiveRecord::Base.establish_connection("#{connection_name}_#{schema}".to_sym)
+      end
+      yield connection_config
+    ensure
+      if schema != :local
+        # Set back the original connection
+        ActiveRecord::Base.establish_connection(connection_name)
+      end
+    end
+  end
+
+  autoload :SqlFile,      'database_loader/sql_file'
+  autoload :RubyFile,     'database_loader/ruby_file'
+  autoload :SqlStatement, 'database_loader/sql_statement'
+  autoload :Template,     'database_loader/template'
+end
+
+require "database_loader/railtie" if defined?(::Rails::Railtie)

metadata

@@ -0,0 +1,81 @@
+--- !ruby/object:Gem::Specification 
+name: database_loader
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Edgars Beigarts
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-01-18 00:00:00 +02:00
+default_executable: 
+dependencies: []
+
+description: Load SQL views, materialized views, grants, etc. into database
+email: 
+- 1@wb4.lv
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- database_loader.gemspec
+- examples/template.sh
+- lib/database_loader.rb
+- lib/database_loader/railtie.rb
+- lib/database_loader/ruby_file.rb
+- lib/database_loader/sql_file.rb
+- lib/database_loader/sql_statement.rb
+- lib/database_loader/tasks.rb
+- lib/database_loader/template.rb
+- lib/database_loader/version.rb
+has_rdoc: true
+homepage: http://github.com/ebeigarts/database_loader
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.4.1
+signing_key: 
+specification_version: 3
+summary: Load SQL views, materialized views, grants, etc. into database
+test_files: []
+