db_dumper 0.5 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a7bcc6d6d41f50f4034725ff07c8986e2e6e1f0016725f2cd0787348eb4f5a84
-  data.tar.gz: c7ac55eed36664fa46d3ecfc3e5f098be027207727d3748581450a1f57239f7d
+  metadata.gz: ffa5f19bbf8e2ae8ff7df6413b88ad23eb1ab4aeb702af65150eed75aaa3394c
+  data.tar.gz: ae7a26c5be8a10c80af089b02f38af86e3fdaf6907a6060e4badf2cffbf9a50b
 SHA512:
-  metadata.gz: 1fefff8a2c3150b49b7625a1aeae6eaa2ba2a2cd786e2f166977fcf6350d09b17f4093afeac5e4a5c2e8c343b8bbd384c14e2481f99265e4249b1bd788e80ea8
-  data.tar.gz: 23a304879c10ffebb575300751966adbd425ae7c2405e36cf2f6eb42919972efe7b0d20d22c246c57b0663aa8e1855b956099972e50691757621ffb6ca3596be
+  metadata.gz: 2335003bbacde8b9c6a9784d753863dee4ab7f800e10ac68cc4df7633af8b5cabca0cb1b3ef149735fabba8361a0bf507f9512e5d389d0d99658e61dcb86b5dc
+  data.tar.gz: eff420c0c74e6e665f24b2a886abe547095c04f3446cd7f3ea81d73d56b0dd46d9161c179c47268888444dddd715fa09ef2071e21d52ff2a4d8ef5e17ded6c65

data/CHANGELOG.md

@@ -0,0 +1,10 @@
+# CHANGELOG
+
+## unreleased
+
+## 0.5.1
+
+* Fix gemspec
+* More logs
+* Add Changelog
+* Update README

data/README.md

@@ -1 +1,125 @@
-db_dumper gem readme
+# DB dumper - configurable SQL database data copying util from remote to local machine.
+
+[![Gem Version](https://badge.fury.io/rb/db_dumper.svg)](https://badge.fury.io/rb/db_dumper)
+
+## Index
+- [Usage](#usage)
+- [Config](#config)
+- [Installation](#installation)
+  - [Manually from RubyGems.org](#manually-from-rubygemsorg)
+  - [Bundler](#or-if-you-are-using-bundler)
+
+## Usage
+
+Example for simple Ruby file:
+
+```ruby
+
+require 'rubygems/package'
+require 'bundler/inline'
+
+gemfile do
+  source 'https://rubygems.org'
+  gem 'db_dumper'
+end
+
+DbDumper.dump do
+  user_id = 1
+  dump('users').where(id: user_id)
+  campaigns_q = q('campaigns').where('user_id = ? OR for_all IS TRUE', user_id)
+  dump(campaigns_q)
+  dump('offices').where(campaign_id: campaigns_q.ar)
+end
+```
+
+After executing you will see:
+- schema sql at local_machine/dest_path/schema_dump.sql.
+- csv files at local_machine/dest_path/csv/*.csv
+ 
+
+### Config
+
+Config file example at config/application.yml
+
+```
+remote_db:
+  adapter: postgres
+  host: host
+  port: 5432
+  database: dockerdb
+  username: docker
+  password: pass
+
+remote_user:
+  name: docker
+  host: beta.staging.com
+  ssh_keys: ['/.ssh/docker']
+  passphrase: pass
+
+remote_machine:
+  dest_path: /tmp
+
+local_machine:
+  dest_path: /tmp
+```
+
+### Testing
+
+No tests fo now
+
+## <a id="installation">Installation ##
+
+### Manually from RubyGems.org ###
+
+```sh
+% gem install db_dumper
+```
+
+### Or if you are using Bundler ###
+
+```ruby
+# Gemfile
+gem 'db_dumper'
+```
+
+## Latest changes ##
+
+Take a look at the [CHANGELOG](https://github.com/alukyanov/db_dumper/blob/master/CHANGELOG.md) for details about recent changes to the current version.
+
+## Questions? ##
+
+Feel free to
+
+* [create an issue on GitHub](https://github.com/alukyanov/db_dumper/issues)
+
+## Maintainers ##
+
+* [Alexey Lukyanov](https://github.com/alukyanov)
+
+## Warranty ##
+
+This software is provided "as is" and without any express or
+implied warranties, including, without limitation, the implied
+warranties of merchantibility and fitness for a particular
+purpose.
+
+## License ##
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/db_dumper/configuration.rb

@@ -1,8 +1,12 @@
 # frozen_string_literal: true
 
+require_relative 'configuration/base'
 require_relative 'configuration/postgres'
 
 module DbDumper
+
+  # Configuration class, by default loads from config/application.yml file
+  # see config/application.sample.yml for format details
   class Configuration
     SshUser = Struct.new(:name, :host, :ssh_keys, :passphrase, keyword_init: true)
     RemoteDB = Struct.new(:adapter, :host, :port, :database, :username, :password,
@@ -32,7 +36,7 @@ module DbDumper
     def db_utils
       @db_utils ||= begin
         utils = DB_UTILS[remote_db.adapter]
-        raise 'Unknown adapter for remote_db:adapter: check application.yml' unless utils
+        raise 'Unknown adapter for remote_db:adapter check application.yml' unless utils
         utils.new(remote_db)
       end
     end
@@ -53,12 +57,12 @@ module DbDumper
       @local_machine ||= LocalMachine.new(loaded_file['local_machine'])
     end
 
-    private
-
     def logger
       @logger ||= Logger.new(STDOUT)
     end
 
+    private
+
     def loaded_file
       @loaded_file ||= YAML.load_file(config_file_path)
     end

data/lib/db_dumper/configuration/base.rb

@@ -0,0 +1,23 @@
+module DbDumper
+  class Configuration
+    class Base
+      attr_reader :db_config
+
+      def initialize(db_config)
+        @db_config = db_config
+      end
+
+      def copy_data_command(sql, file_path)
+        raise NotImplementedError
+      end
+
+      def dump_schema_command(dump_schema_file_path)
+        raise NotImplementedError
+      end
+
+      def dump_data_command(dump_data_file_path)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/db_dumper/configuration/postgres.rb

@@ -1,10 +1,8 @@
 module DbDumper
   class Configuration
-    class Postgres
-      attr_reader :db_config
-
-      def initialize(db_config)
-        @db_config = db_config
+    class Postgres < Base
+      def copy_data_command(sql, file_path)
+        "\\COPY (#{sql}) TO '#{file_path}';"
       end
 
       def dump_schema_command(dump_schema_file_path)

data/lib/db_dumper/query_builder.rb

@@ -2,51 +2,62 @@ require_relative 'query_builder/table'
 require_relative 'query_builder/query'
 
 module DbDumper
+
+  # Generates queries for copying data
   class QueryBuilder
     attr_reader :config
 
     def self.to_sql(config, &block)
       establish_connection
 
-      instance = new(config)
-      instance.instance_eval(&block)
-      instance.to_sql
+      new(config).yield_self do |instance|
+        instance.instance_eval(&block)
+        instance.send(:to_sql)
+      end
     end
 
-    def initialize(config)
-      @config = config
-    end
-
-    def self.establish_connection
-      return if ActiveRecord::Base.connected?
-
-      ActiveRecord::Migration.verbose = false
-      ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: ':memory:')
-    end
+    # DSL start
 
+    # creates new query and add it to resulting queries array
     def dump(query)
       query = q(query) if query.is_a?(String)
       query.tap { |q| queries << q }
     end
 
+    # creates new query
     def q(table_name)
       Query.new Table.from(table_name.to_s)
     end
 
+    # DSL end
+
+    private
+
     def to_sql
-      queries.map do |query|
-        "\\COPY (#{query.to_sql}) TO '#{copy_path}/#{query.table_name}.csv';"
-      end.join($RS)
+      queries.map(&method(:build_copy_command)).join("\n")
     end
 
-    private
+    def initialize(config)
+      @config = config
+    end
+
+    def self.establish_connection
+      return if ActiveRecord::Base.connected?
+
+      ActiveRecord::Migration.verbose = false
+      ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: ':memory:')
+    end
 
     def queries
       @queries ||= []
     end
 
+    def build_copy_command(query)
+      config.db_utils.copy_data_command(query.to_sql, "#{copy_path}/#{query.table_name}.csv")
+    end
+
     def copy_path
-      config.remote_machine.dest_path
+      config.remote_machine.data_path
     end
   end
 end

data/lib/db_dumper/query_builder/query.rb

@@ -1,5 +1,7 @@
 module DbDumper
   class QueryBuilder
+
+    # Wrapper under ActiveRecord::Relation
     class Query
       attr_reader :table, :ar
 

data/lib/db_dumper/query_builder/table.rb

@@ -1,5 +1,7 @@
 module DbDumper
   class QueryBuilder
+
+    # Storage for generated classes inherited from ActiveRecord::Base
     class Table
       attr_reader :table_name
 

data/lib/db_dumper/remote_machine.rb

@@ -1,96 +1,80 @@
+require_relative 'remote_machine/ssh_agent'
+
 module DbDumper
   class RemoteMachine
-    attr_reader :config, :sql
+    attr_reader :config, :copy_commands_sql
 
-    def initialize(config, sql)
+    def initialize(config, copy_commands_sql)
       @config = config
-      @sql    = sql
+      @copy_commands_sql = copy_commands_sql
     end
 
     def dump
       config.log('save sql commands to local machine')
       save_commands_sql_to_tmp_file
 
-      Net::SSH.start(*ssh_credentials) do |ssh|
-        config.log('upload commands to remote machine')
+      with_ssh do |ssh|
         upload_commands_sql_to_remote_machine(ssh)
-
-        config.log('dump schema on remote machine')
         dump_schema(ssh)
-
-        config.log('dump data on remote machine')
         dump_data(ssh)
-
-        config.log('download schema to local machine')
         download_schema(ssh)
-
-        config.log('download data to local machine')
         download_data(ssh)
-
-        config.log('clean remote machine')
         clean_remote_machine(ssh)
       end
     end
 
     private
 
+    def with_ssh
+      ssh_agent = SshAgent.new(ssh_credentials, config)
+      yield(ssh_agent)
+    end
+
     # Upload
     def save_commands_sql_to_tmp_file
-      File.open(local_commands_sql_file_path, 'w') { |f| f.write(sql) }
+      File.open(local_commands_sql_file_path, 'w') { |f| f.write(copy_commands_sql) }
     end
 
     def upload_commands_sql_to_remote_machine(ssh)
-      ssh.scp.upload!(local_commands_sql_file_path, remote_commands_sql_file_path)
-      File.delete(local_commands_sql_file_path)
-    end
-
-    def remote_commands_sql_file_path
-      "#{remote_machine_dest_path}/#{commands_sql_fname}"
-    end
-
-    def local_commands_sql_file_path
-      "#{dest_path}/#{commands_sql_fname}"
+      ssh.upload!(local_commands_sql_file_path, remote_commands_sql_file_path)
     end
 
     def commands_sql_fname
-      @commands_sql_fname ||= "#{Digest::MD5.hexdigest(sql)}.sql"
+      @commands_sql_fname ||= "#{Digest::MD5.hexdigest(copy_commands_sql)}.sql"
     end
 
     # Schema
 
     def dump_schema(ssh)
-      ssh.exec! db_utils.dump_schema_command(remote_machine_schema_file_path)
+      ssh.exec!(db_utils.dump_schema_command(remote_machine_schema_file_path))
     end
 
     def download_schema(ssh)
-      ssh.scp.download!(remote_machine_schema_file_path, dest_path)
-    end
-
-    def remote_machine_schema_file_path
-      "#{remote_machine_dest_path}/#{dump_schema_fname}"
-    end
-
-    def dump_schema_fname
-      'schema_dump.sql'
+      ssh.download!(remote_machine_schema_file_path, dest_path)
     end
 
     # Data
 
     def dump_data(ssh)
       ssh.exec!("mkdir -p #{remote_machine_data_path}")
-      ssh.exec! db_utils.dump_data_command(remote_commands_sql_file_path)
+      ssh.exec!(db_utils.dump_data_command(remote_commands_sql_file_path))
     end
 
     def download_data(ssh)
-      ssh.scp.download!(remote_machine_data_path, dest_path, recursive: true)
+      ssh.download!(remote_machine_data_path, dest_path, recursive: true)
     end
 
+    # Clean
     def clean_remote_machine(ssh)
       ssh.exec! "rm #{remote_commands_sql_file_path}"
       ssh.exec! "rm #{remote_machine_schema_file_path}"
       ssh.exec! "rm -rf #{remote_machine_data_path}"
     end
 
+    def clean_local_machine
+      File.delete(local_commands_sql_file_path)
+    end
+
     #
 
     def db_utils
@@ -105,6 +89,22 @@ module DbDumper
       config.remote_machine.data_path
     end
 
+    def remote_machine_schema_file_path
+      "#{remote_machine_dest_path}/#{dump_schema_fname}"
+    end
+
+    def remote_commands_sql_file_path
+      "#{remote_machine_dest_path}/#{commands_sql_fname}"
+    end
+
+    def local_commands_sql_file_path
+      "#{dest_path}/#{commands_sql_fname}"
+    end
+
+    def dump_schema_fname
+      'schema_dump.sql'
+    end
+
     def ssh_credentials
       [
         ssh_user.host,

data/lib/db_dumper/remote_machine/ssh_agent.rb

@@ -0,0 +1,46 @@
+module DbDumper
+  class RemoteMachine
+
+    # Wrapper around Net::SSH, Net:SCP
+    class SshAgent
+      attr_reader :credentials, :config
+      attr_reader :block, :ssh
+      attr_reader :ssh_user_name, :ssh_host_name
+
+      def initialize(credentials, config, &block)
+        @credentials  = credentials
+        @config       = config
+        @block        = block
+        @ssh          = Net::SSH.start(*credentials)
+
+        @ssh_host_name = credentials[0]
+        @ssh_user_name = credentials[1]
+      end
+
+      def exec!(command)
+        log("EXECUTING on #{ssh_machine_name}: #{command}")
+        log ssh.exec!(command)
+      end
+
+      def download!(remote_source, local_dest, options = {})
+        log("DOWNLOADING from #{ssh_machine_name}:#{remote_source} to #{local_dest}")
+        log ssh.scp.download!(remote_source, local_dest, options)
+      end
+
+      def upload!(local_source, remote_dest, options = {})
+        log("uploading from #{ssh_machine_name}:#{local_source} to #{remote_dest}")
+        log ssh.scp.upload!(local_source, remote_dest, options)
+      end
+
+      private
+
+      def ssh_machine_name
+        "#{ssh_user_name}@#{ssh_host_name}"
+      end
+
+      def log(message)
+        config.log(message)
+      end
+    end
+  end
+end

data/lib/db_dumper/version.rb

@@ -1,3 +1,3 @@
 module DbDumper
-  VERSION = '0.5'
+  VERSION = '0.5.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: db_dumper
 version: !ruby/object:Gem::Version
-  version: '0.5'
+  version: 0.5.1
 platform: ruby
 authors:
 - Alexey Lukyanov
@@ -31,21 +31,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 5.2.1
 - !ruby/object:Gem::Dependency
-  name: sqlite3
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: pg
+  name: net-scp
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -73,7 +59,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: net-scp
+  name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -92,19 +78,28 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG.md
 - README.md
 - lib/db_dumper.rb
 - lib/db_dumper/configuration.rb
+- lib/db_dumper/configuration/base.rb
 - lib/db_dumper/configuration/postgres.rb
 - lib/db_dumper/query_builder.rb
 - lib/db_dumper/query_builder/query.rb
 - lib/db_dumper/query_builder/table.rb
 - lib/db_dumper/remote_machine.rb
+- lib/db_dumper/remote_machine/ssh_agent.rb
 - lib/db_dumper/version.rb
 homepage: http://rubygems.org/gems/db_dumper
 licenses:
 - MIT
-metadata: {}
+metadata:
+  bug_tracker_uri: https://github.com/alukyanov/db_dumper/issues
+  changelog_uri: https://github.com/alukyanov/db_dumper/CHANGELOG.md
+  documentation_uri: http://www.rubydoc.info/gems/db_dumper/0.5.1
+  homepage_uri: https://github.com/alukyanov/db_dumper
+  source_code_uri: https://github.com/alukyanov/db_dumper
+  wiki_uri: https://github.com/alukyanov/db_dumper/wiki
 post_install_message: 
 rdoc_options: []
 require_paths: