cloner 0.5.7 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 16802f5e33d2ec7ee1125de6181c768acf050e05
-  data.tar.gz: 102b5732b777d4aa5c2a36d9cfed1be97dcf4be6
+SHA256:
+  metadata.gz: 652e0cb307e408f626c4376e97480acb8bb830f3463b7272befe0eb8b6cd9b2c
+  data.tar.gz: 825b8ea077f10294d0a561ef75d751e4f85e0109935cafb51ec92de7bd1d74e2
 SHA512:
-  metadata.gz: ed39fa7e86356730b021924cda9e07f040e9e1108bfa57b32aa78eb9e26d6da82b65a99a6504c29a04ba53b198497bb2e48ccc4aa89e7c1356c4f5c438ebc13b
-  data.tar.gz: 999f75fb8d067c7470ccb1d7f35a5a6511b1058bafa0c9d6d6b83bbe7357766af7b1a5f7f8c5a6944d59b12319283ee1cf6ae80c390e49a076089b753d3fbc24
+  metadata.gz: 87ab8f908f2c9428f39bf8a2d536a54b647b4eac6393aa07c9ebe4db2149226f0d80bed75f46eb25c1359f03e097ac036cde83b847bed9e21d62c26be724d23c
+  data.tar.gz: 5b172cfb19ad692494218221ac0ca0dae3ebbabab08978fdaf314e911bcf334d2fd6db47ce600ebf426f0df6ff8409a9542058e9d36461d269a349b57e416b2f

data/.ruby-version

@@ -1 +1 @@
-2.1.5
+ruby-2.6.6

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2014 glebtv
+Copyright (c) 2014-2020 glebtv
 
 MIT License
 

data/README.md

@@ -1,6 +1,9 @@
 # Cloner
 
-Easily clone your production Mongoid or PostgreSQL database and files for local development or staging area.
+Easily clone your production Mongoid or PostgreSQL / MySQL database and files for local development or staging area.
+
+Uses rsync and database-specific default dump/restore tools (pg_dump/pg_restore, mysqldump/mysql, mongodump/mongorestore)
+
 
 ## Installation
 
@@ -20,37 +23,44 @@ Or install it yourself as:
 
 ## Usage
 
-create ```lib/tasks/dl.thor``` with following content:
-
-    require 'cloner'
-
-    class Dl < Cloner::Base
-      no_commands do
-        def rails_path
-          File.expand_path("../../../config/environment", __FILE__)
-        end
-        def ssh_host
-          'hottea.ru'
-        end
-        def ssh_user
-          'tea'
-        end
-        def remote_dump_path
-          '/data/tea/dump'
-        end
-        def remote_app_path
-          "/data/tea/app/current"
-        end
-      end
-
-      desc "download", "clone files and DB from production"
-      def download
-        load_env
-        clone_db
-        rsync_public("ckeditor_assets")
-        rsync_public("uploads")
-      end
+For generate cloner base template, run:
+
+```
+bundle exec rails generate cloner
+```
+
+This is create `lib/tasks/dl.thor` file with following content:
+```ruby
+require 'cloner'
+
+class Dl < Cloner::Base
+  no_commands do
+    def rails_path
+      File.expand_path("../../../config/environment", __FILE__)
     end
+    def ssh_host
+      'hottea.ru'
+    end
+    def ssh_user
+      'tea'
+    end
+    def remote_dump_path
+      '/data/tea/dump'
+    end
+    def remote_app_path
+      "/data/tea/app/current"
+    end
+  end
+
+  desc "download", "clone files and DB from production"
+  def download
+    load_env
+    clone_db
+    rsync_public("ckeditor_assets")
+    rsync_public("uploads")
+  end
+end
+```
 
 Adjust it to your project and deployment.
 
@@ -58,6 +68,30 @@ Run it:
 
     thor dl
 
+
+If you generate extended cloner template as: `rails g cloner -e`,
+you can run `thor dl` with additional parameters, for example:
+```
+bundle exec thor dl -D # For skip clone database
+bundle exec thor dl -F # For skip clone files
+```
+
+For details see help:
+```
+bundle exec thor help dl:download
+
+Usage:
+  thor dl:download
+
+Options:
+      [--from=FROM]                            # stage name where cloner get data
+                                               # Default: production
+  -D, [--skip-database], [--no-skip-database]  # skip clone database
+  -F, [--skip-files], [--no-skip-files]        # skip clone files
+
+clone files and DB from production
+```
+
 ## Additional
 
 All functions from cloner/internal.rb can be overriden, for example:
@@ -73,6 +107,26 @@ All functions from cloner/internal.rb can be overriden, for example:
       {}
     end
 
+## Changelog
+
+### 0.9.0
+
+- Add option to rsync to allow sync one file (thx @AnatolyShirykalov)
+- Add env_database to allow overriding database env (thx @Yarroo)
+
+### 0.8.0
+
+- Change default rsync flags - -z to -zz to support newer versions of rsync
+- Allow overriding rsync flags via ```rsync_flags``` and ```rsync_compression```
+
+### 0.7.0
+
+- Add thor file generators
+
+### 0.6.0
+
+- Support MySQL
+
 ## Contributing
 
 1. Fork it ( https://github.com/[my-github-username]/cloner/fork )

data/cloner.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "activesupport"
   spec.add_dependency 'net-ssh'
 
-  spec.add_development_dependency "bundler", "~> 1.7"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
 end
 

data/lib/cloner.rb

@@ -1,10 +1,14 @@
 require "cloner/version"
 require 'thor'
 require 'active_support/concern'
-require "cloner/mongodb"
-require "cloner/postgres"
-require "cloner/ssh"
-require "cloner/rsync"
-require "cloner/internal"
-require "cloner/base"
+module Cloner
+  autoload :Base, "cloner/base"
+  autoload :Internal, "cloner/internal"
+  autoload :Ar, "cloner/ar"
+  autoload :MongoDB, "cloner/mongodb"
+  autoload :Postgres, "cloner/postgres"
+  autoload :MySQL, "cloner/mysql"
+  autoload :SSH, "cloner/ssh"
+  autoload :RSync, "cloner/rsync"
+end
 

data/lib/cloner/ar.rb

@@ -0,0 +1,54 @@
+module Cloner::Ar
+  def ar_conf
+    @conf ||= begin
+      YAML.load_file(Rails.root.join('config', 'database.yml'))[env_database]
+    end
+  end
+
+  def env_database
+    Rails.env
+  end
+
+  def ar_to
+    ar_conf['database']
+  end
+
+  def ar_r_conf
+    @ar_r_conf ||= begin
+      do_ssh do |ssh|
+        ret = ssh_exec!(ssh, "cat #{e(remote_app_path + '/config/database.yml')}")
+        check_ssh_err(ret)
+        begin
+          res = YAML.load(ret[0])[env_from]
+          raise 'no data' if res.blank?
+          #res['host'] ||= '127.0.0.1'
+        rescue Exception => e
+          puts "unable to read remote database.yml for env #{env_from}."
+          puts "Remote file contents:"
+          puts ret[0]
+        end
+        res
+      end
+    end
+  end
+
+  def clone_ar
+    if ar_conf["adapter"] != ar_r_conf["adapter"]
+      puts "Error: ActiveRecord adapter mismatch: local #{ar_conf["adapter"]}, remote #{ar_r_conf["adapter"]}"
+      puts "it is not possible to convert from one database to another via this tool."
+      exit
+    end
+
+    case ar_conf["adapter"]
+    when 'postgresql'
+      clone_pg
+    when 'mysql2'
+      clone_my
+    else
+      puts "unknown activerecord adapter: #{ar_conf["adapter"]}"
+      puts "currently supported adapters: mysql2, postgresql"
+      exit
+    end
+  end
+end
+

data/lib/cloner/internal.rb

@@ -1,7 +1,9 @@
 module Cloner::Internal
   extend ActiveSupport::Concern
   include Cloner::MongoDB
+  include Cloner::Ar
   include Cloner::Postgres
+  include Cloner::MySQL
   include Cloner::SSH
   include Cloner::RSync
 
@@ -27,7 +29,7 @@ module Cloner::Internal
     if defined?(Mongoid)
       clone_mongodb
     else
-      clone_pg
+      clone_ar
     end
   end
 end

data/lib/cloner/mysql.rb

@@ -0,0 +1,77 @@
+module Cloner::MySQL
+  def my_local_auth
+    if ar_conf['password'].blank?
+      ""
+    else
+      "--password='#{ar_conf['password']}'"
+    end
+  end
+
+  def my_remote_auth
+    if ar_r_conf['password'].blank?
+      ""
+    else
+      "--password='#{ar_r_conf['password']}'"
+    end
+  end
+
+  def my_dump_param
+    "--add-drop-table"
+  end
+
+  def my_restore_param
+    ""
+  end
+
+  def my_bin_path(util)
+    util
+  end
+
+  def my_dump_remote
+    puts "backup remote DB via ssh"
+    do_ssh do |ssh|
+      ssh.exec!("rm -R #{e remote_dump_path}")
+      ret = ssh_exec!(ssh, "mkdir -p #{e remote_dump_path}")
+      check_ssh_err(ret)
+      host = ar_r_conf['host'].present? ? " --host #{e ar_r_conf['host']}" : ""
+      port = ar_r_conf['port'].present? ? " --port #{e ar_r_conf['port']}" : ""
+      dump = "#{my_bin_path 'mysqldump'} #{my_dump_param} --user #{e ar_r_conf['username']} #{my_remote_auth}#{host}#{port} #{e ar_r_conf['database']} > #{e(remote_dump_path + '/cloner.sql')}"
+      puts dump if verbose?
+      ret = ssh_exec!(ssh, dump)
+      check_ssh_err(ret)
+    end
+  end
+
+  def my_dump_restore
+    puts "restoring DB"
+    host = ar_conf['host'].present? ? " --host #{e ar_conf['host']}" : ""
+    port = ar_conf['port'].present? ? " --port #{e ar_conf['port']}" : ""
+    restore = "#{my_bin_path 'mysql'} #{my_restore_param} --user #{e ar_conf['username']} #{my_local_auth}#{host}#{port} #{e ar_to} < #{e(my_path + '/cloner.sql')}"
+    puts restore if verbose?
+    pipe = IO.popen(restore)
+    while (line = pipe.gets)
+      print line if verbose?
+    end
+    ret = $?.to_i
+    if ret != 0
+      puts "Error: local command exited with #{ret}"
+    end
+  end
+
+  def my_path
+    Rails.root.join("tmp", "dump").to_s
+  end
+
+  def my_dump_copy
+    FileUtils.mkdir_p(my_path)
+    `mkdir -p #{e my_path}`
+    rsync(remote_dump_path, my_path)
+  end
+
+  def clone_my
+    my_dump_remote()
+    my_dump_copy()
+    my_dump_restore()
+  end
+end
+

data/lib/cloner/postgres.rb

@@ -1,37 +1,8 @@
 module Cloner::Postgres
   extend ActiveSupport::Concern
 
-  def ar_conf
-    @conf ||= begin
-      YAML.load_file(Rails.root.join('config', 'database.yml'))[Rails.env]
-    end
-  end
-
-  def ar_to
-    ar_conf['database']
-  end
-
-  def ar_r_conf
-    @ar_r_conf ||= begin
-      do_ssh do |ssh|
-        ret = ssh_exec!(ssh, "cat #{e(remote_app_path + '/config/database.yml')}")
-        check_ssh_err(ret)
-        begin
-          res = YAML.load(ret[0])[env_from]
-          raise 'no data' if res.blank?
-          #res['host'] ||= '127.0.0.1'
-        rescue Exception => e
-          puts "unable to read remote database.yml for env #{env_from}."
-          puts "Remote file contents:"
-          puts ret[0]
-        end
-        res
-      end
-    end
-  end
-
   def pg_local_auth
-    if ar_conf['password'].nil?
+    if ar_conf['password'].blank?
       ""
     else
       "PGPASSWORD='#{ar_conf['password']}' "
@@ -39,7 +10,7 @@ module Cloner::Postgres
   end
 
   def pg_remote_auth
-    if ar_r_conf['password'].nil?
+    if ar_r_conf['password'].blank?
       ""
     else
       "PGPASSWORD='#{ar_r_conf['password']}' "
@@ -50,10 +21,29 @@ module Cloner::Postgres
     ""
   end
 
+  def pg_dump_param
+    if pg_dump_extra != ""
+      puts "WARN pg_dump_extra is deprecated, use def pg_dump_param; super + ' extra'"
+    end
+    "-Fc #{pg_dump_extra}"
+  end
+
+  def pg_restore_param
+    "--no-owner -Fc -c"
+  end
+  
   def pg_bin_path(util)
     util
   end
 
+  def pg_local_bin_path(util)
+    pg_bin_path(util)
+  end
+  
+  def pg_remote_bin_path(util)
+    pg_bin_path(util)
+  end 
+
   def pg_dump_remote
     puts "backup remote DB via ssh"
     do_ssh do |ssh|
@@ -62,7 +52,7 @@ module Cloner::Postgres
       check_ssh_err(ret)
       host = ar_r_conf['host'].present? ? " -h #{e ar_r_conf['host']}" : ""
       port = ar_r_conf['port'].present? ? " -p #{e ar_r_conf['port']}" : ""
-      dump = pg_remote_auth + "#{pg_bin_path 'pg_dump'} -Fc #{pg_dump_extra} -U #{e ar_r_conf['username']}#{host}#{port} #{e ar_r_conf['database']} > #{e(remote_dump_path + '/tmp.bak')}"
+      dump = pg_remote_auth + "#{pg_remote_bin_path 'pg_dump'} #{pg_dump_param} -U #{e ar_r_conf['username']}#{host}#{port} #{e ar_r_conf['database']} > #{e(remote_dump_path + '/tmp.bak')}"
       puts dump if verbose?
       ret = ssh_exec!(ssh, dump)
       check_ssh_err(ret)
@@ -73,14 +63,14 @@ module Cloner::Postgres
     puts "restoring DB"
     host = ar_conf['host'].present? ? " -h #{e ar_conf['host']}" : ""
     port = ar_conf['port'].present? ? " -p #{e ar_conf['port']}" : ""
-    restore = pg_local_auth + "#{pg_bin_path 'pg_restore'} --no-owner -Fc -c -U #{e ar_conf['username']}#{host}#{port} -d #{e ar_to} #{e(pg_path + '/tmp.bak')}"
+    restore = pg_local_auth + "#{pg_local_bin_path 'pg_restore'} #{pg_restore_param} -U #{e ar_conf['username']}#{host}#{port} -d #{e ar_to} #{e(pg_path + '/tmp.bak')}"
     puts restore if verbose?
     pipe = IO.popen(restore)
     while (line = pipe.gets)
       print line if verbose?
     end
     ret = $?.to_i
-    if ret != 0 
+    if ret != 0
       puts "Error: local command exited with #{ret}"
     end
   end
@@ -92,7 +82,7 @@ module Cloner::Postgres
   def pg_dump_copy
     FileUtils.mkdir_p(pg_path)
     `mkdir -p #{e pg_path}`
-    rsync(remote_dump_path + '/', pg_path)
+    rsync(remote_dump_path, pg_path)
   end
 
   def clone_pg

data/lib/cloner/rsync.rb

@@ -1,8 +1,21 @@
 module Cloner::RSync
   extend ActiveSupport::Concern
-  def rsync(from, to)
+
+  def rsync_compression
+    "-zz"
+  end
+
+  def rsync_flags
     port = ssh_opts[:port] || 22
-    cmd = "rsync -e ssh -zutvr --checksum -e \"ssh -p #{port}\" #{e ssh_user}@#{e ssh_host}:#{e from}/ #{e to}/"
+    "#{rsync_compression} -utvr --checksum -e \"ssh -p #{port}\""
+  end
+
+  def rsync(from, to, directory: true, raise_on_error: false)
+    if directory
+      from = "#{from}/" unless from.ends_with?('/')
+      to = "#{to}/" unless to.ends_with?('/')
+    end
+    cmd = "rsync #{rsync_flags} #{e ssh_user}@#{e ssh_host}:#{e from} #{e to}"
     puts "Running RSync: #{cmd}"
     pipe = IO.popen(cmd)
     while (line = pipe.gets)
@@ -10,7 +23,10 @@ module Cloner::RSync
     end
     pipe.close
     ret = $?.to_i
-    if ret != 0 
+    if ret != 0
+      if raise_on_error
+        raise "Error: local command exited with #{ret}"
+      end
       puts "Error: local command exited with #{ret}"
     end
   end

data/lib/cloner/version.rb

@@ -1,4 +1,4 @@
 module Cloner
-  VERSION = "0.5.7"
+  VERSION = "0.9.0"
 end
 

data/lib/generators/cloner_generator.rb

@@ -0,0 +1,25 @@
+class ClonerGenerator < Rails::Generators::Base
+  source_root File.expand_path('templates', __dir__)
+
+  class_option :extend, default: false, type: :boolean, aliases: '-e'
+
+  desc "This generator create lib/tasks/dl.thor"
+  def create_task_file
+    unless options[:extend]
+      create_default_task_file
+    else
+      create_extended_task_file
+    end
+  end
+
+  private
+  def create_default_task_file
+    copy_file 'cloner_base.template', 'lib/tasks/dl.thor'
+  end
+
+  def create_extended_task_file
+    say 'Create extend file'
+    @username = Rails.application.class.parent_name.downcase
+    template 'cloner_extend.thor.erb', 'lib/tasks/dl.thor'
+  end
+end

data/lib/generators/templates/cloner_base.template

@@ -0,0 +1,29 @@
+require 'cloner'
+
+class Dl < Cloner::Base
+  no_commands do
+    def rails_path
+      File.expand_path("../../../config/environment", __FILE__)
+    end
+    def ssh_host
+      'hottea.ru'
+    end
+    def ssh_user
+      'tea'
+    end
+    def remote_dump_path
+      '/data/tea/dump'
+    end
+    def remote_app_path
+      "/data/tea/app/current"
+    end
+  end
+
+  desc "download", "clone files and DB from production"
+  def download
+    load_env
+    clone_db
+    rsync_public("ckeditor_assets")
+    rsync_public("uploads")
+  end
+end

data/lib/generators/templates/cloner_extend.thor.erb

@@ -0,0 +1,72 @@
+require 'cloner'
+
+class Dl < Cloner::Base
+  no_commands do
+    def rails_path
+      File.expand_path("../../../config/environment", __FILE__)
+    end
+    def stages
+      @_stages ||= {
+        # TODO: Add new stages here if you needed
+        production: {
+          # TODO: Fix production settings
+          ssh_host: 'production.example.com',
+          ssh_user: '<%= @username %>'
+        },
+        staging: {
+          # TODO: Fix staging settings
+          ssh_host: 'production.example.com',
+          ssh_user: '<%= @username %>'
+        }
+      }
+    end
+    def ssh_host
+      stages.dig(options[:from].to_sym, :ssh_host)
+    end
+    def ssh_user
+      stages.dig(options[:from].to_sym, :ssh_user)
+    end
+    def remote_dump_path
+      # TODO: Fix remote dump path
+      '/data/<%= @username %>/dump'
+    end
+    def remote_app_path
+      # TODO: Fix remote app path
+      '/data/<%= @username %>/app/current'
+    end
+  end
+
+  class_option :from,
+               default: 'production',
+               type: :string,
+               desc: 'stage name where cloner get data'
+  class_option :skip_database,
+               default: false,
+               type: :boolean,
+               aliases: '-D',
+               desc: 'skip clone database'
+  class_option :skip_files,
+               default: false,
+               type: :boolean,
+               aliases: '-F',
+               desc: 'skip clone files'
+
+  desc "download", "clone files and DB from production"
+  def download
+    load_env
+    say "Clone from: #{options[:from]}", :green
+    if options[:skip_database]
+      say "Skip clone database!", :yellow
+    else
+      clone_db
+    end
+
+    if options[:skip_files]
+      say "Skip clone files!", :yellow
+    else
+      # TODO: Fix folders for synchronization here
+      rsync_public("ckeditor_assets")
+      rsync_public("uploads")
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloner
 version: !ruby/object:Gem::Version
-  version: 0.5.7
+  version: 0.9.0
 platform: ruby
 authors:
 - glebtv
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-12-20 00:00:00.000000000 Z
+date: 2020-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -56,30 +56,30 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 description: ''
 email:
 - glebtv@gmail.com
@@ -88,7 +88,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".ruby-gemset"
 - ".ruby-version"
 - Gemfile
 - LICENSE.txt
@@ -96,18 +95,23 @@ files:
 - Rakefile
 - cloner.gemspec
 - lib/cloner.rb
+- lib/cloner/ar.rb
 - lib/cloner/base.rb
 - lib/cloner/internal.rb
 - lib/cloner/mongodb.rb
+- lib/cloner/mysql.rb
 - lib/cloner/postgres.rb
 - lib/cloner/rsync.rb
 - lib/cloner/ssh.rb
 - lib/cloner/version.rb
+- lib/generators/cloner_generator.rb
+- lib/generators/templates/cloner_base.template
+- lib/generators/templates/cloner_extend.thor.erb
 homepage: https://github.com/rs-pro/cloner
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -122,9 +126,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
-signing_key: 
+rubygems_version: 3.1.3
+signing_key:
 specification_version: 4
 summary: Easily clone your production Mongoid database and files for local development
 test_files: []

data/.ruby-gemset

@@ -1 +0,0 @@
-cloner