db_dumper 0.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a7bcc6d6d41f50f4034725ff07c8986e2e6e1f0016725f2cd0787348eb4f5a84
+  data.tar.gz: c7ac55eed36664fa46d3ecfc3e5f098be027207727d3748581450a1f57239f7d
+SHA512:
+  metadata.gz: 1fefff8a2c3150b49b7625a1aeae6eaa2ba2a2cd786e2f166977fcf6350d09b17f4093afeac5e4a5c2e8c343b8bbd384c14e2481f99265e4249b1bd788e80ea8
+  data.tar.gz: 23a304879c10ffebb575300751966adbd425ae7c2405e36cf2f6eb42919972efe7b0d20d22c246c57b0663aa8e1855b956099972e50691757621ffb6ca3596be

data/README.md

@@ -0,0 +1 @@
+db_dumper gem readme

data/lib/db_dumper.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+require 'net/ssh'
+require 'net/scp'
+require 'active_record'
+
+require_relative 'db_dumper/configuration'
+require_relative 'db_dumper/query_builder'
+require_relative 'db_dumper/remote_machine'
+
+module DbDumper
+  module_function def dump(config_file_path = 'config/application.yml', &block)
+    config = Configuration.new(config_file_path)
+    sql = QueryBuilder.to_sql(config, &block)
+    RemoteMachine.new(config, sql).dump
+  end
+end

data/lib/db_dumper/configuration.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative 'configuration/postgres'
+
+module DbDumper
+  class Configuration
+    SshUser = Struct.new(:name, :host, :ssh_keys, :passphrase, keyword_init: true)
+    RemoteDB = Struct.new(:adapter, :host, :port, :database, :username, :password,
+                          :dump_schema_options, :dump_data_options,
+                          keyword_init: true)
+    RemoteMachine = Struct.new(:dest_path, keyword_init: true) do
+      def data_path
+        "#{dest_path}/csv"
+      end
+    end
+    LocalMachine = Struct.new(:dest_path, keyword_init: true)
+
+    DB_UTILS = {
+      'postgres' => Postgres
+    }
+
+    attr_reader :config_file_path
+
+    def initialize(config_file_path)
+      @config_file_path = config_file_path
+    end
+
+    def log(message)
+      logger.info(message)
+    end
+
+    def db_utils
+      @db_utils ||= begin
+        utils = DB_UTILS[remote_db.adapter]
+        raise 'Unknown adapter for remote_db:adapter: check application.yml' unless utils
+        utils.new(remote_db)
+      end
+    end
+
+    def ssh_user
+      @ssh_user ||= SshUser.new(loaded_file['ssh_user'])
+    end
+
+    def remote_machine
+      @remote_machine ||= RemoteMachine.new(loaded_file['remote_machine'])
+    end
+
+    def remote_db
+      @remote_db ||= RemoteDB.new(loaded_file['remote_db'])
+    end
+
+    def local_machine
+      @local_machine ||= LocalMachine.new(loaded_file['local_machine'])
+    end
+
+    private
+
+    def logger
+      @logger ||= Logger.new(STDOUT)
+    end
+
+    def loaded_file
+      @loaded_file ||= YAML.load_file(config_file_path)
+    end
+  end
+end

data/lib/db_dumper/configuration/postgres.rb

@@ -0,0 +1,31 @@
+module DbDumper
+  class Configuration
+    class Postgres
+      attr_reader :db_config
+
+      def initialize(db_config)
+        @db_config = db_config
+      end
+
+      def dump_schema_command(dump_schema_file_path)
+        util_command('pg_dump', "#{db_config.dump_schema_options} -f #{dump_schema_file_path}")
+      end
+
+      def dump_data_command(dump_data_file_path)
+        util_command('psql', "#{db_config.dump_data_options} -f #{dump_data_file_path}")
+      end
+
+      private
+
+      def util_command(util, command)
+        <<-CMD.squish
+          PGPASSWORD=#{db_config.password} #{util}
+            -h #{db_config.host}
+            -U #{db_config.username}
+            -d #{db_config.database}
+          #{command}
+        CMD
+      end
+    end
+  end
+end

data/lib/db_dumper/query_builder.rb

@@ -0,0 +1,52 @@
+require_relative 'query_builder/table'
+require_relative 'query_builder/query'
+
+module DbDumper
+  class QueryBuilder
+    attr_reader :config
+
+    def self.to_sql(config, &block)
+      establish_connection
+
+      instance = new(config)
+      instance.instance_eval(&block)
+      instance.to_sql
+    end
+
+    def initialize(config)
+      @config = config
+    end
+
+    def self.establish_connection
+      return if ActiveRecord::Base.connected?
+
+      ActiveRecord::Migration.verbose = false
+      ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: ':memory:')
+    end
+
+    def dump(query)
+      query = q(query) if query.is_a?(String)
+      query.tap { |q| queries << q }
+    end
+
+    def q(table_name)
+      Query.new Table.from(table_name.to_s)
+    end
+
+    def to_sql
+      queries.map do |query|
+        "\\COPY (#{query.to_sql}) TO '#{copy_path}/#{query.table_name}.csv';"
+      end.join($RS)
+    end
+
+    private
+
+    def queries
+      @queries ||= []
+    end
+
+    def copy_path
+      config.remote_machine.dest_path
+    end
+  end
+end

data/lib/db_dumper/query_builder/query.rb

@@ -0,0 +1,31 @@
+module DbDumper
+  class QueryBuilder
+    class Query
+      attr_reader :table, :ar
+
+      delegate :table_name, to: :table
+
+      def initialize(table)
+        @table  = table
+        @ar     = @table.ar.all
+      end
+
+      def where(*args)
+        tap { @ar = @ar.where(*args) }
+      end
+
+      def joins(*args)
+        raise 'Only simple string for joins supported' unless args.size == 1 && args[0].is_a?(String)
+        tap { @ar = @ar.joins(*args) }
+      end
+
+      def select(*args)
+        tap { @ar = @ar.select(*args) }
+      end
+
+      def to_sql
+        @ar.to_sql
+      end
+    end
+  end
+end

data/lib/db_dumper/query_builder/table.rb

@@ -0,0 +1,28 @@
+module DbDumper
+  class QueryBuilder
+    class Table
+      attr_reader :table_name
+
+      @tables = {}
+
+      def self.from(table_name)
+        @tables[table_name] ||= begin
+          ActiveRecord::Migration.create_table(table_name)
+          new(table_name)
+        end
+      end
+
+      def ar
+        @ar ||= Class.new(ActiveRecord::Base).tap do |klass|
+          klass.table_name = table_name
+        end
+      end
+
+      private
+
+      def initialize(table_name)
+        @table_name = table_name
+      end
+    end
+  end
+end

data/lib/db_dumper/remote_machine.rb

@@ -0,0 +1,125 @@
+module DbDumper
+  class RemoteMachine
+    attr_reader :config, :sql
+
+    def initialize(config, sql)
+      @config = config
+      @sql    = sql
+    end
+
+    def dump
+      config.log('save sql commands to local machine')
+      save_commands_sql_to_tmp_file
+
+      Net::SSH.start(*ssh_credentials) do |ssh|
+        config.log('upload commands to remote machine')
+        upload_commands_sql_to_remote_machine(ssh)
+
+        config.log('dump schema on remote machine')
+        dump_schema(ssh)
+
+        config.log('dump data on remote machine')
+        dump_data(ssh)
+
+        config.log('download schema to local machine')
+        download_schema(ssh)
+
+        config.log('download data to local machine')
+        download_data(ssh)
+
+        config.log('clean remote machine')
+        clean_remote_machine(ssh)
+      end
+    end
+
+    private
+
+    # Upload
+    def save_commands_sql_to_tmp_file
+      File.open(local_commands_sql_file_path, 'w') { |f| f.write(sql) }
+    end
+
+    def upload_commands_sql_to_remote_machine(ssh)
+      ssh.scp.upload!(local_commands_sql_file_path, remote_commands_sql_file_path)
+      File.delete(local_commands_sql_file_path)
+    end
+
+    def remote_commands_sql_file_path
+      "#{remote_machine_dest_path}/#{commands_sql_fname}"
+    end
+
+    def local_commands_sql_file_path
+      "#{dest_path}/#{commands_sql_fname}"
+    end
+
+    def commands_sql_fname
+      @commands_sql_fname ||= "#{Digest::MD5.hexdigest(sql)}.sql"
+    end
+
+    # Schema
+
+    def dump_schema(ssh)
+      ssh.exec! db_utils.dump_schema_command(remote_machine_schema_file_path)
+    end
+
+    def download_schema(ssh)
+      ssh.scp.download!(remote_machine_schema_file_path, dest_path)
+    end
+
+    def remote_machine_schema_file_path
+      "#{remote_machine_dest_path}/#{dump_schema_fname}"
+    end
+
+    def dump_schema_fname
+      'schema_dump.sql'
+    end
+
+    # Data
+
+    def dump_data(ssh)
+      ssh.exec!("mkdir -p #{remote_machine_data_path}")
+      ssh.exec! db_utils.dump_data_command(remote_commands_sql_file_path)
+    end
+
+    def download_data(ssh)
+      ssh.scp.download!(remote_machine_data_path, dest_path, recursive: true)
+    end
+
+    def clean_remote_machine(ssh)
+      ssh.exec! "rm #{remote_commands_sql_file_path}"
+      ssh.exec! "rm #{remote_machine_schema_file_path}"
+      ssh.exec! "rm -rf #{remote_machine_data_path}"
+    end
+
+    #
+
+    def db_utils
+      config.db_utils
+    end
+
+    def remote_machine_dest_path
+      config.remote_machine.dest_path
+    end
+
+    def remote_machine_data_path
+      config.remote_machine.data_path
+    end
+
+    def ssh_credentials
+      [
+        ssh_user.host,
+        ssh_user.name,
+        keys:       ssh_user.ssh_keys,
+        passphrase: ssh_user.passphrase
+      ]
+    end
+
+    def ssh_user
+      config.ssh_user
+    end
+
+    def dest_path
+      config.local_machine.dest_path
+    end
+  end
+end

data/lib/db_dumper/version.rb

@@ -0,0 +1,3 @@
+module DbDumper
+  VERSION = '0.5'
+end

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification
+name: db_dumper
+version: !ruby/object:Gem::Version
+  version: '0.5'
+platform: ruby
+authors:
+- Alexey Lukyanov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-10-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.1
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-scp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Dump database schema and chosen data from remote machine
+email: alukyanov.dev@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/db_dumper.rb
+- lib/db_dumper/configuration.rb
+- lib/db_dumper/configuration/postgres.rb
+- lib/db_dumper/query_builder.rb
+- lib/db_dumper/query_builder/query.rb
+- lib/db_dumper/query_builder/table.rb
+- lib/db_dumper/remote_machine.rb
+- lib/db_dumper/version.rb
+homepage: http://rubygems.org/gems/db_dumper
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.3
+signing_key: 
+specification_version: 4
+summary: Dump database schema and chosen data from remote machine
+test_files: []