cloner 0.6.0 → 0.9.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.ruby-version +1 -1
- data/LICENSE.txt +1 -1
- data/README.md +85 -31
- data/cloner.gemspec +2 -2
- data/lib/cloner/ar.rb +5 -1
- data/lib/cloner/mysql.rb +3 -3
- data/lib/cloner/postgres.rb +12 -4
- data/lib/cloner/rsync.rb +18 -2
- data/lib/cloner/version.rb +1 -1
- data/lib/generators/cloner_generator.rb +25 -0
- data/lib/generators/templates/cloner_base.template +29 -0
- data/lib/generators/templates/cloner_extend.thor.erb +72 -0
- metadata +17 -17
- data/.ruby-gemset +0 -1
- data/HISTORY.md +0 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 48065947b30b56516063f5d9e83b20f365ff9c706471629db00263926a22f398
|
4
|
+
data.tar.gz: 3baca4e368a95a2dbe0b29d2197169f512e3d0fe7d9ecf2d6576f10590530847
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 21c0b290b324a34524427ce8bd2f0b1f2ddd1034a925384c3e2a7aae36c951dbce630ba2a8335ef55b8524f197e2eff8c4097a236186907a328b91e914318ccc
|
7
|
+
data.tar.gz: 34aa3d7f03dd927193e734e7eb79a99ed4dc5577dd018c873b503f22048b20045c61bdea3361e74042241af8ab3983b11964c12f21bd28a3d4a8f3bf2f13be34
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.
|
1
|
+
ruby-2.6.6
|
data/LICENSE.txt
CHANGED
data/README.md
CHANGED
@@ -1,6 +1,9 @@
|
|
1
1
|
# Cloner
|
2
2
|
|
3
|
-
Easily clone your production Mongoid or PostgreSQL database and files for local development or staging area.
|
3
|
+
Easily clone your production Mongoid or PostgreSQL / MySQL database and files for local development or staging area.
|
4
|
+
|
5
|
+
Uses rsync and database-specific default dump/restore tools (pg_dump/pg_restore, mysqldump/mysql, mongodump/mongorestore)
|
6
|
+
|
4
7
|
|
5
8
|
## Installation
|
6
9
|
|
@@ -20,37 +23,44 @@ Or install it yourself as:
|
|
20
23
|
|
21
24
|
## Usage
|
22
25
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
end
|
38
|
-
def remote_dump_path
|
39
|
-
'/data/tea/dump'
|
40
|
-
end
|
41
|
-
def remote_app_path
|
42
|
-
"/data/tea/app/current"
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
desc "download", "clone files and DB from production"
|
47
|
-
def download
|
48
|
-
load_env
|
49
|
-
clone_db
|
50
|
-
rsync_public("ckeditor_assets")
|
51
|
-
rsync_public("uploads")
|
52
|
-
end
|
26
|
+
For generate cloner base template, run:
|
27
|
+
|
28
|
+
```
|
29
|
+
bundle exec rails generate cloner
|
30
|
+
```
|
31
|
+
|
32
|
+
This is create `lib/tasks/dl.thor` file with following content:
|
33
|
+
```ruby
|
34
|
+
require 'cloner'
|
35
|
+
|
36
|
+
class Dl < Cloner::Base
|
37
|
+
no_commands do
|
38
|
+
def rails_path
|
39
|
+
File.expand_path("../../../config/environment", __FILE__)
|
53
40
|
end
|
41
|
+
def ssh_host
|
42
|
+
'hottea.ru'
|
43
|
+
end
|
44
|
+
def ssh_user
|
45
|
+
'tea'
|
46
|
+
end
|
47
|
+
def remote_dump_path
|
48
|
+
'/data/tea/dump'
|
49
|
+
end
|
50
|
+
def remote_app_path
|
51
|
+
"/data/tea/app/current"
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
desc "download", "clone files and DB from production"
|
56
|
+
def download
|
57
|
+
load_env
|
58
|
+
clone_db
|
59
|
+
rsync_public("ckeditor_assets")
|
60
|
+
rsync_public("uploads")
|
61
|
+
end
|
62
|
+
end
|
63
|
+
```
|
54
64
|
|
55
65
|
Adjust it to your project and deployment.
|
56
66
|
|
@@ -58,6 +68,30 @@ Run it:
|
|
58
68
|
|
59
69
|
thor dl
|
60
70
|
|
71
|
+
|
72
|
+
If you generate extended cloner template as: `rails g cloner -e`,
|
73
|
+
you can run `thor dl` with additional parameters, for example:
|
74
|
+
```
|
75
|
+
bundle exec thor dl -D # For skip clone database
|
76
|
+
bundle exec thor dl -F # For skip clone files
|
77
|
+
```
|
78
|
+
|
79
|
+
For details see help:
|
80
|
+
```
|
81
|
+
bundle exec thor help dl:download
|
82
|
+
|
83
|
+
Usage:
|
84
|
+
thor dl:download
|
85
|
+
|
86
|
+
Options:
|
87
|
+
[--from=FROM] # stage name where cloner get data
|
88
|
+
# Default: production
|
89
|
+
-D, [--skip-database], [--no-skip-database] # skip clone database
|
90
|
+
-F, [--skip-files], [--no-skip-files] # skip clone files
|
91
|
+
|
92
|
+
clone files and DB from production
|
93
|
+
```
|
94
|
+
|
61
95
|
## Additional
|
62
96
|
|
63
97
|
All functions from cloner/internal.rb can be overriden, for example:
|
@@ -73,6 +107,26 @@ All functions from cloner/internal.rb can be overriden, for example:
|
|
73
107
|
{}
|
74
108
|
end
|
75
109
|
|
110
|
+
## Changelog
|
111
|
+
|
112
|
+
### 0.9.0
|
113
|
+
|
114
|
+
- Add option to rsync to allow sync one file (thx @AnatolyShirykalov)
|
115
|
+
- Add env_database to allow overriding database env (thx @Yarroo)
|
116
|
+
|
117
|
+
### 0.8.0
|
118
|
+
|
119
|
+
- Change default rsync flags - -z to -zz to support newer versions of rsync
|
120
|
+
- Allow overriding rsync flags via ```rsync_flags``` and ```rsync_compression```
|
121
|
+
|
122
|
+
### 0.7.0
|
123
|
+
|
124
|
+
- Add thor file generators
|
125
|
+
|
126
|
+
### 0.6.0
|
127
|
+
|
128
|
+
- Support MySQL
|
129
|
+
|
76
130
|
## Contributing
|
77
131
|
|
78
132
|
1. Fork it ( https://github.com/[my-github-username]/cloner/fork )
|
data/cloner.gemspec
CHANGED
@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
|
|
22
22
|
spec.add_dependency "activesupport"
|
23
23
|
spec.add_dependency 'net-ssh'
|
24
24
|
|
25
|
-
spec.add_development_dependency "bundler"
|
26
|
-
spec.add_development_dependency "rake"
|
25
|
+
spec.add_development_dependency "bundler"
|
26
|
+
spec.add_development_dependency "rake"
|
27
27
|
end
|
28
28
|
|
data/lib/cloner/ar.rb
CHANGED
@@ -1,10 +1,14 @@
|
|
1
1
|
module Cloner::Ar
|
2
2
|
def ar_conf
|
3
3
|
@conf ||= begin
|
4
|
-
YAML.load_file(Rails.root.join('config', 'database.yml'))[
|
4
|
+
YAML.load_file(Rails.root.join('config', 'database.yml'))[env_database]
|
5
5
|
end
|
6
6
|
end
|
7
7
|
|
8
|
+
def env_database
|
9
|
+
Rails.env
|
10
|
+
end
|
11
|
+
|
8
12
|
def ar_to
|
9
13
|
ar_conf['database']
|
10
14
|
end
|
data/lib/cloner/mysql.rb
CHANGED
@@ -44,8 +44,8 @@ module Cloner::MySQL
|
|
44
44
|
|
45
45
|
def my_dump_restore
|
46
46
|
puts "restoring DB"
|
47
|
-
host = ar_conf['host'].present? ? "
|
48
|
-
port = ar_conf['port'].present? ? "
|
47
|
+
host = ar_conf['host'].present? ? " --host #{e ar_conf['host']}" : ""
|
48
|
+
port = ar_conf['port'].present? ? " --port #{e ar_conf['port']}" : ""
|
49
49
|
restore = "#{my_bin_path 'mysql'} #{my_restore_param} --user #{e ar_conf['username']} #{my_local_auth}#{host}#{port} #{e ar_to} < #{e(my_path + '/cloner.sql')}"
|
50
50
|
puts restore if verbose?
|
51
51
|
pipe = IO.popen(restore)
|
@@ -65,7 +65,7 @@ module Cloner::MySQL
|
|
65
65
|
def my_dump_copy
|
66
66
|
FileUtils.mkdir_p(my_path)
|
67
67
|
`mkdir -p #{e my_path}`
|
68
|
-
rsync(remote_dump_path
|
68
|
+
rsync(remote_dump_path, my_path)
|
69
69
|
end
|
70
70
|
|
71
71
|
def clone_my
|
data/lib/cloner/postgres.rb
CHANGED
@@ -31,11 +31,19 @@ module Cloner::Postgres
|
|
31
31
|
def pg_restore_param
|
32
32
|
"--no-owner -Fc -c"
|
33
33
|
end
|
34
|
-
|
34
|
+
|
35
35
|
def pg_bin_path(util)
|
36
36
|
util
|
37
37
|
end
|
38
38
|
|
39
|
+
def pg_local_bin_path(util)
|
40
|
+
pg_bin_path(util)
|
41
|
+
end
|
42
|
+
|
43
|
+
def pg_remote_bin_path(util)
|
44
|
+
pg_bin_path(util)
|
45
|
+
end
|
46
|
+
|
39
47
|
def pg_dump_remote
|
40
48
|
puts "backup remote DB via ssh"
|
41
49
|
do_ssh do |ssh|
|
@@ -44,7 +52,7 @@ module Cloner::Postgres
|
|
44
52
|
check_ssh_err(ret)
|
45
53
|
host = ar_r_conf['host'].present? ? " -h #{e ar_r_conf['host']}" : ""
|
46
54
|
port = ar_r_conf['port'].present? ? " -p #{e ar_r_conf['port']}" : ""
|
47
|
-
dump = pg_remote_auth + "#{
|
55
|
+
dump = pg_remote_auth + "#{pg_remote_bin_path 'pg_dump'} #{pg_dump_param} -U #{e ar_r_conf['username']}#{host}#{port} #{e ar_r_conf['database']} > #{e(remote_dump_path + '/tmp.bak')}"
|
48
56
|
puts dump if verbose?
|
49
57
|
ret = ssh_exec!(ssh, dump)
|
50
58
|
check_ssh_err(ret)
|
@@ -55,7 +63,7 @@ module Cloner::Postgres
|
|
55
63
|
puts "restoring DB"
|
56
64
|
host = ar_conf['host'].present? ? " -h #{e ar_conf['host']}" : ""
|
57
65
|
port = ar_conf['port'].present? ? " -p #{e ar_conf['port']}" : ""
|
58
|
-
restore = pg_local_auth + "#{
|
66
|
+
restore = pg_local_auth + "#{pg_local_bin_path 'pg_restore'} #{pg_restore_param} -U #{e ar_conf['username']}#{host}#{port} -d #{e ar_to} #{e(pg_path + '/tmp.bak')}"
|
59
67
|
puts restore if verbose?
|
60
68
|
pipe = IO.popen(restore)
|
61
69
|
while (line = pipe.gets)
|
@@ -74,7 +82,7 @@ module Cloner::Postgres
|
|
74
82
|
def pg_dump_copy
|
75
83
|
FileUtils.mkdir_p(pg_path)
|
76
84
|
`mkdir -p #{e pg_path}`
|
77
|
-
rsync(remote_dump_path
|
85
|
+
rsync(remote_dump_path, pg_path)
|
78
86
|
end
|
79
87
|
|
80
88
|
def clone_pg
|
data/lib/cloner/rsync.rb
CHANGED
@@ -1,8 +1,21 @@
|
|
1
1
|
module Cloner::RSync
|
2
2
|
extend ActiveSupport::Concern
|
3
|
-
|
3
|
+
|
4
|
+
def rsync_compression
|
5
|
+
"-zz"
|
6
|
+
end
|
7
|
+
|
8
|
+
def rsync_flags
|
4
9
|
port = ssh_opts[:port] || 22
|
5
|
-
|
10
|
+
"#{rsync_compression} -utvr --checksum -e \"ssh -p #{port}\""
|
11
|
+
end
|
12
|
+
|
13
|
+
def rsync(from, to, directory: true, raise_on_error: false)
|
14
|
+
if directory
|
15
|
+
from = "#{from}/" unless from.end_with?('/')
|
16
|
+
to = "#{to}/" unless to.end_with?('/')
|
17
|
+
end
|
18
|
+
cmd = "rsync #{rsync_flags} #{e ssh_user}@#{e ssh_host}:#{e from} #{e to}"
|
6
19
|
puts "Running RSync: #{cmd}"
|
7
20
|
pipe = IO.popen(cmd)
|
8
21
|
while (line = pipe.gets)
|
@@ -11,6 +24,9 @@ module Cloner::RSync
|
|
11
24
|
pipe.close
|
12
25
|
ret = $?.to_i
|
13
26
|
if ret != 0
|
27
|
+
if raise_on_error
|
28
|
+
raise "Error: local command exited with #{ret}"
|
29
|
+
end
|
14
30
|
puts "Error: local command exited with #{ret}"
|
15
31
|
end
|
16
32
|
end
|
data/lib/cloner/version.rb
CHANGED
@@ -0,0 +1,25 @@
|
|
1
|
+
class ClonerGenerator < Rails::Generators::Base
|
2
|
+
source_root File.expand_path('templates', __dir__)
|
3
|
+
|
4
|
+
class_option :extend, default: false, type: :boolean, aliases: '-e'
|
5
|
+
|
6
|
+
desc "This generator create lib/tasks/dl.thor"
|
7
|
+
def create_task_file
|
8
|
+
unless options[:extend]
|
9
|
+
create_default_task_file
|
10
|
+
else
|
11
|
+
create_extended_task_file
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
def create_default_task_file
|
17
|
+
copy_file 'cloner_base.template', 'lib/tasks/dl.thor'
|
18
|
+
end
|
19
|
+
|
20
|
+
def create_extended_task_file
|
21
|
+
say 'Create extend file'
|
22
|
+
@username = Rails.application.class.parent_name.downcase
|
23
|
+
template 'cloner_extend.thor.erb', 'lib/tasks/dl.thor'
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'cloner'
|
2
|
+
|
3
|
+
class Dl < Cloner::Base
|
4
|
+
no_commands do
|
5
|
+
def rails_path
|
6
|
+
File.expand_path("../../../config/environment", __FILE__)
|
7
|
+
end
|
8
|
+
def ssh_host
|
9
|
+
'hottea.ru'
|
10
|
+
end
|
11
|
+
def ssh_user
|
12
|
+
'tea'
|
13
|
+
end
|
14
|
+
def remote_dump_path
|
15
|
+
'/data/tea/dump'
|
16
|
+
end
|
17
|
+
def remote_app_path
|
18
|
+
"/data/tea/app/current"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
desc "download", "clone files and DB from production"
|
23
|
+
def download
|
24
|
+
load_env
|
25
|
+
clone_db
|
26
|
+
rsync_public("ckeditor_assets")
|
27
|
+
rsync_public("uploads")
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
require 'cloner'
|
2
|
+
|
3
|
+
class Dl < Cloner::Base
|
4
|
+
no_commands do
|
5
|
+
def rails_path
|
6
|
+
File.expand_path("../../../config/environment", __FILE__)
|
7
|
+
end
|
8
|
+
def stages
|
9
|
+
@_stages ||= {
|
10
|
+
# TODO: Add new stages here if you needed
|
11
|
+
production: {
|
12
|
+
# TODO: Fix production settings
|
13
|
+
ssh_host: 'production.example.com',
|
14
|
+
ssh_user: '<%= @username %>'
|
15
|
+
},
|
16
|
+
staging: {
|
17
|
+
# TODO: Fix staging settings
|
18
|
+
ssh_host: 'production.example.com',
|
19
|
+
ssh_user: '<%= @username %>'
|
20
|
+
}
|
21
|
+
}
|
22
|
+
end
|
23
|
+
def ssh_host
|
24
|
+
stages.dig(options[:from].to_sym, :ssh_host)
|
25
|
+
end
|
26
|
+
def ssh_user
|
27
|
+
stages.dig(options[:from].to_sym, :ssh_user)
|
28
|
+
end
|
29
|
+
def remote_dump_path
|
30
|
+
# TODO: Fix remote dump path
|
31
|
+
'/data/<%= @username %>/dump'
|
32
|
+
end
|
33
|
+
def remote_app_path
|
34
|
+
# TODO: Fix remote app path
|
35
|
+
'/data/<%= @username %>/app/current'
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
class_option :from,
|
40
|
+
default: 'production',
|
41
|
+
type: :string,
|
42
|
+
desc: 'stage name where cloner get data'
|
43
|
+
class_option :skip_database,
|
44
|
+
default: false,
|
45
|
+
type: :boolean,
|
46
|
+
aliases: '-D',
|
47
|
+
desc: 'skip clone database'
|
48
|
+
class_option :skip_files,
|
49
|
+
default: false,
|
50
|
+
type: :boolean,
|
51
|
+
aliases: '-F',
|
52
|
+
desc: 'skip clone files'
|
53
|
+
|
54
|
+
desc "download", "clone files and DB from production"
|
55
|
+
def download
|
56
|
+
load_env
|
57
|
+
say "Clone from: #{options[:from]}", :green
|
58
|
+
if options[:skip_database]
|
59
|
+
say "Skip clone database!", :yellow
|
60
|
+
else
|
61
|
+
clone_db
|
62
|
+
end
|
63
|
+
|
64
|
+
if options[:skip_files]
|
65
|
+
say "Skip clone files!", :yellow
|
66
|
+
else
|
67
|
+
# TODO: Fix folders for synchronization here
|
68
|
+
rsync_public("ckeditor_assets")
|
69
|
+
rsync_public("uploads")
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cloner
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- glebtv
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-08-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -56,30 +56,30 @@ dependencies:
|
|
56
56
|
name: bundler
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - "
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
61
|
+
version: '0'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - "
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
68
|
+
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: rake
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- - "
|
73
|
+
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: '
|
75
|
+
version: '0'
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- - "
|
80
|
+
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: '
|
82
|
+
version: '0'
|
83
83
|
description: ''
|
84
84
|
email:
|
85
85
|
- glebtv@gmail.com
|
@@ -88,10 +88,8 @@ extensions: []
|
|
88
88
|
extra_rdoc_files: []
|
89
89
|
files:
|
90
90
|
- ".gitignore"
|
91
|
-
- ".ruby-gemset"
|
92
91
|
- ".ruby-version"
|
93
92
|
- Gemfile
|
94
|
-
- HISTORY.md
|
95
93
|
- LICENSE.txt
|
96
94
|
- README.md
|
97
95
|
- Rakefile
|
@@ -106,11 +104,14 @@ files:
|
|
106
104
|
- lib/cloner/rsync.rb
|
107
105
|
- lib/cloner/ssh.rb
|
108
106
|
- lib/cloner/version.rb
|
107
|
+
- lib/generators/cloner_generator.rb
|
108
|
+
- lib/generators/templates/cloner_base.template
|
109
|
+
- lib/generators/templates/cloner_extend.thor.erb
|
109
110
|
homepage: https://github.com/rs-pro/cloner
|
110
111
|
licenses:
|
111
112
|
- MIT
|
112
113
|
metadata: {}
|
113
|
-
post_install_message:
|
114
|
+
post_install_message:
|
114
115
|
rdoc_options: []
|
115
116
|
require_paths:
|
116
117
|
- lib
|
@@ -125,9 +126,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
125
126
|
- !ruby/object:Gem::Version
|
126
127
|
version: '0'
|
127
128
|
requirements: []
|
128
|
-
|
129
|
-
|
130
|
-
signing_key:
|
129
|
+
rubygems_version: 3.1.4
|
130
|
+
signing_key:
|
131
131
|
specification_version: 4
|
132
132
|
summary: Easily clone your production Mongoid database and files for local development
|
133
133
|
test_files: []
|
data/.ruby-gemset
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
cloner
|
data/HISTORY.md
DELETED