duple 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +19 -0
- data/.rspec +2 -0
- data/.simplecov +14 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +64 -0
- data/Rakefile +21 -0
- data/bin/duple +11 -0
- data/duple.gemspec +30 -0
- data/lib/duple.rb +3 -0
- data/lib/duple/cli/config.rb +63 -0
- data/lib/duple/cli/copy.rb +29 -0
- data/lib/duple/cli/helpers.rb +192 -0
- data/lib/duple/cli/init.rb +15 -0
- data/lib/duple/cli/refresh.rb +75 -0
- data/lib/duple/cli/root.rb +41 -0
- data/lib/duple/cli/structure.rb +23 -0
- data/lib/duple/configuration.rb +203 -0
- data/lib/duple/heroku_runner.rb +25 -0
- data/lib/duple/pg_runner.rb +27 -0
- data/lib/duple/runner.rb +63 -0
- data/lib/duple/version.rb +3 -0
- data/spec/duple/cli/config_spec.rb +16 -0
- data/spec/duple/cli/copy_spec.rb +112 -0
- data/spec/duple/cli/init_spec.rb +19 -0
- data/spec/duple/cli/refresh_spec.rb +413 -0
- data/spec/duple/cli/structure_spec.rb +133 -0
- data/spec/duple/configuration_spec.rb +166 -0
- data/spec/duple/runner_spec.rb +65 -0
- data/spec/spec_helper.rb +18 -0
- data/spec/support/cli_spec_helpers.rb +85 -0
- data/spec/support/duple.rb +5 -0
- data/spec/support/io_spec_helpers.rb +33 -0
- data/spec/support/rspec_fire.rb +0 -0
- metadata +190 -0
@@ -0,0 +1,15 @@
|
|
1
|
+
module Duple
|
2
|
+
module CLI
|
3
|
+
class Init < Thor::Group
|
4
|
+
include Duple::CLI::Helpers
|
5
|
+
|
6
|
+
config_option
|
7
|
+
|
8
|
+
def create_sample_config
|
9
|
+
config_path = app_config_path(false)
|
10
|
+
empty_directory(File.dirname(config_path))
|
11
|
+
copy_file('templates/config/duple.yml', config_path)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
module Duple
|
2
|
+
module CLI
|
3
|
+
class Refresh < Thor::Group
|
4
|
+
include Duple::CLI::Helpers
|
5
|
+
|
6
|
+
config_option
|
7
|
+
source_option
|
8
|
+
target_option
|
9
|
+
group_option
|
10
|
+
capture_option
|
11
|
+
tables_option
|
12
|
+
|
13
|
+
no_tasks do
|
14
|
+
def capture_snapshot?
|
15
|
+
config.capture? && config.heroku_source?
|
16
|
+
end
|
17
|
+
|
18
|
+
def fetch_snapshot_url?
|
19
|
+
config.heroku_source? && !config.filtered_tables?
|
20
|
+
end
|
21
|
+
|
22
|
+
def download_snapshot?
|
23
|
+
config.heroku_source? && config.local_target? && !config.filtered_tables?
|
24
|
+
end
|
25
|
+
|
26
|
+
def dump_data?
|
27
|
+
config.local_source? || config.filtered_tables?
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def capture_snapshot
|
32
|
+
return unless capture_snapshot?
|
33
|
+
|
34
|
+
heroku.run(source_appname, 'pgbackups:capture')
|
35
|
+
end
|
36
|
+
|
37
|
+
def fetch_snapshot_url
|
38
|
+
return unless fetch_snapshot_url?
|
39
|
+
|
40
|
+
@source_snapshot_url = heroku.capture(source_appname, 'pgbackups:url').strip
|
41
|
+
end
|
42
|
+
|
43
|
+
def download_snapshot
|
44
|
+
return unless download_snapshot?
|
45
|
+
|
46
|
+
timestamp = fetch_latest_snapshot_time(source_appname)
|
47
|
+
|
48
|
+
@snapshot_path = snapshot_file_path(timestamp.strftime('%Y-%m-%d-%H-%M-%S'))
|
49
|
+
unless File.exists?(@snapshot_path)
|
50
|
+
runner.run("curl -o #{@snapshot_path} #{@source_snapshot_url}")
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def dump_data
|
55
|
+
return unless dump_data?
|
56
|
+
|
57
|
+
postgres.pg_dump(dump_flags, data_file_path, source_credentials)
|
58
|
+
end
|
59
|
+
|
60
|
+
def reset_target
|
61
|
+
reset_database(config.target_environment)
|
62
|
+
end
|
63
|
+
|
64
|
+
def restore_database
|
65
|
+
if download_snapshot?
|
66
|
+
postgres.pg_restore('-e -v --no-acl -O -a', @snapshot_path, target_credentials)
|
67
|
+
elsif dump_data?
|
68
|
+
postgres.pg_restore('-e -v --no-acl -O -a', data_file_path, target_credentials)
|
69
|
+
else
|
70
|
+
heroku.run(target_appname, "pgbackups:restore DATABASE #{@source_snapshot_url}")
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'thor/group'
|
3
|
+
require 'duple/cli/helpers'
|
4
|
+
require 'duple/cli/init'
|
5
|
+
require 'duple/cli/copy'
|
6
|
+
require 'duple/cli/config'
|
7
|
+
require 'duple/cli/structure'
|
8
|
+
require 'duple/cli/refresh'
|
9
|
+
require 'duple/runner'
|
10
|
+
require 'duple/pg_runner'
|
11
|
+
require 'duple/heroku_runner'
|
12
|
+
|
13
|
+
module Duple
|
14
|
+
module CLI
|
15
|
+
class Root < Thor
|
16
|
+
include Duple::CLI::Helpers
|
17
|
+
|
18
|
+
# HACK Override register to handle class_options for groups properly.
|
19
|
+
def self.register(klass, task_name, description)
|
20
|
+
super(klass, task_name, task_name, description)
|
21
|
+
tasks[task_name].options = klass.class_options
|
22
|
+
end
|
23
|
+
|
24
|
+
register Duple::CLI::Init, 'init',
|
25
|
+
'Generates a sample configuration file.'
|
26
|
+
|
27
|
+
register Duple::CLI::Copy, 'copy',
|
28
|
+
'Copies data from a source to a target database.'
|
29
|
+
|
30
|
+
register Duple::CLI::Structure, 'structure',
|
31
|
+
'Copies structure from a source to a target database.'
|
32
|
+
|
33
|
+
register Duple::CLI::Refresh, 'refresh',
|
34
|
+
'Resets and copies schema and data from a source to a target database'
|
35
|
+
|
36
|
+
register Duple::CLI::Config,
|
37
|
+
'config [COMMAND]', 'Manage your configuration.'
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Duple
|
2
|
+
module CLI
|
3
|
+
class Structure < Thor::Group
|
4
|
+
include Duple::CLI::Helpers
|
5
|
+
|
6
|
+
config_option
|
7
|
+
source_option
|
8
|
+
target_option
|
9
|
+
|
10
|
+
def dump_structure
|
11
|
+
postgres.pg_dump('-Fc --no-acl -O -s', structure_file_path, source_credentials)
|
12
|
+
end
|
13
|
+
|
14
|
+
def reset_target
|
15
|
+
reset_database(config.target_environment)
|
16
|
+
end
|
17
|
+
|
18
|
+
def load_structure
|
19
|
+
postgres.pg_restore('-v --no-acl -O -s', structure_file_path, target_credentials)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,203 @@
|
|
1
|
+
module Duple
|
2
|
+
|
3
|
+
# Represents the configuration that will be used to perform the data
|
4
|
+
# operations.
|
5
|
+
#
|
6
|
+
# This class should be the only place in the system that knows about the
|
7
|
+
# structure of the config file.
|
8
|
+
#
|
9
|
+
# This class should not have any knowledge of any particular database
|
10
|
+
# system. For example, this class can know about the concept of a "tables",
|
11
|
+
# but it should know nothing about flags for PostgreSQL commands.
|
12
|
+
class Configuration
|
13
|
+
HEROKU = 'heroku'
|
14
|
+
LOCAL = 'local'
|
15
|
+
VALID_TYPES = [HEROKU, LOCAL]
|
16
|
+
|
17
|
+
attr_reader :options, :raw_config
|
18
|
+
|
19
|
+
def initialize(config_hash, options)
|
20
|
+
@raw_config = config_hash
|
21
|
+
@options = options
|
22
|
+
end
|
23
|
+
|
24
|
+
def default_target_name
|
25
|
+
env_names_by_flag('default_target', true).first
|
26
|
+
end
|
27
|
+
|
28
|
+
def target_name
|
29
|
+
options[:target] || default_target_name
|
30
|
+
end
|
31
|
+
|
32
|
+
def target_environment
|
33
|
+
invalid_target_names = env_names_by_flag('allow_target', false, true)
|
34
|
+
if invalid_target_names.include?(target_name)
|
35
|
+
raise ArgumentError.new("Invalid target: #{target_name} is not allowed to be a target.")
|
36
|
+
end
|
37
|
+
environment(target_name)
|
38
|
+
end
|
39
|
+
|
40
|
+
def heroku_target?
|
41
|
+
heroku?(target_environment)
|
42
|
+
end
|
43
|
+
|
44
|
+
def local_target?
|
45
|
+
local?(target_environment)
|
46
|
+
end
|
47
|
+
|
48
|
+
def default_source_name
|
49
|
+
env_names_by_flag('default_source', true).first
|
50
|
+
end
|
51
|
+
|
52
|
+
def source_name
|
53
|
+
options[:source] || default_source_name
|
54
|
+
end
|
55
|
+
|
56
|
+
def source_environment
|
57
|
+
environment(source_name)
|
58
|
+
end
|
59
|
+
|
60
|
+
def heroku_source?
|
61
|
+
heroku?(source_environment)
|
62
|
+
end
|
63
|
+
|
64
|
+
def local_source?
|
65
|
+
local?(source_environment)
|
66
|
+
end
|
67
|
+
|
68
|
+
def heroku?(env)
|
69
|
+
env['type'] == Duple::Configuration::HEROKU
|
70
|
+
end
|
71
|
+
|
72
|
+
def local?(env)
|
73
|
+
env['type'] == Duple::Configuration::LOCAL
|
74
|
+
end
|
75
|
+
|
76
|
+
def heroku_name(env)
|
77
|
+
env['appname']
|
78
|
+
end
|
79
|
+
|
80
|
+
def dry_run?
|
81
|
+
options[:dry_run]
|
82
|
+
end
|
83
|
+
|
84
|
+
def capture?
|
85
|
+
options[:capture]
|
86
|
+
end
|
87
|
+
|
88
|
+
def group_name
|
89
|
+
options[:group]
|
90
|
+
end
|
91
|
+
|
92
|
+
def table_names
|
93
|
+
options[:tables] || []
|
94
|
+
end
|
95
|
+
|
96
|
+
def filtered_tables?
|
97
|
+
included_tables.size > 0 || excluded_tables.size > 0
|
98
|
+
end
|
99
|
+
|
100
|
+
# Returns an array of tables to include, based on the group config and the
|
101
|
+
# --tables option. An empty array indicates that ALL tables should be
|
102
|
+
# included. If the group has the include_all flag, an empty array will be
|
103
|
+
# returned.
|
104
|
+
def included_tables
|
105
|
+
tables = table_names
|
106
|
+
if group_name
|
107
|
+
g = group(group_name)
|
108
|
+
return [] if g['include_all']
|
109
|
+
tables += (g['include_tables'] || [])
|
110
|
+
end
|
111
|
+
tables.uniq.sort
|
112
|
+
end
|
113
|
+
|
114
|
+
# Returns an array of tables to exclude, based on the group config and the
|
115
|
+
# --tables option. The --tables option takes precedence over the --group
|
116
|
+
# option, so if a table is excluded from a group, but specified in the
|
117
|
+
# --tables option the table will NOT be excluded.
|
118
|
+
def excluded_tables
|
119
|
+
tables = []
|
120
|
+
if group_name
|
121
|
+
g = group(group_name)
|
122
|
+
tables += (g['exclude_tables'] || [])
|
123
|
+
end
|
124
|
+
tables -= table_names
|
125
|
+
tables
|
126
|
+
end
|
127
|
+
|
128
|
+
def dry_run_credentials(envname)
|
129
|
+
{
|
130
|
+
user: "[#{envname}.USER]",
|
131
|
+
password: "[#{envname}.PASS]",
|
132
|
+
host: "[#{envname}.HOST]",
|
133
|
+
port: "[#{envname}.PORT]",
|
134
|
+
db: "[#{envname}.DB]"
|
135
|
+
}
|
136
|
+
end
|
137
|
+
|
138
|
+
def local_credentials
|
139
|
+
# TODO: Override these defaults from the config
|
140
|
+
{
|
141
|
+
user: 'postgres',
|
142
|
+
password: '',
|
143
|
+
host: 'localhost',
|
144
|
+
port: '5432',
|
145
|
+
db: 'duple_development'
|
146
|
+
}
|
147
|
+
end
|
148
|
+
|
149
|
+
def environments
|
150
|
+
raw_config['environments'] || {}
|
151
|
+
end
|
152
|
+
|
153
|
+
def environment(env_name)
|
154
|
+
env = environments[env_name]
|
155
|
+
raise ArgumentError.new("Invalid environment: #{env_name}") if env.nil?
|
156
|
+
env
|
157
|
+
end
|
158
|
+
|
159
|
+
def groups
|
160
|
+
raw_config['groups'] || {}
|
161
|
+
end
|
162
|
+
|
163
|
+
def group(group_name)
|
164
|
+
group = groups[group_name]
|
165
|
+
raise ArgumentError.new("Invalid group: #{group_name}") if group.nil?
|
166
|
+
group
|
167
|
+
end
|
168
|
+
|
169
|
+
def pre_refresh_tasks
|
170
|
+
raw_config['pre_refresh'] || {}
|
171
|
+
end
|
172
|
+
|
173
|
+
def pre_refresh_task(task_name)
|
174
|
+
task = pre_refresh_tasks[task_name]
|
175
|
+
raise ArgumentError.new("Invalid pre_refresh task: #{task_name}") if task.nil?
|
176
|
+
task
|
177
|
+
end
|
178
|
+
|
179
|
+
def post_refresh_tasks
|
180
|
+
raw_config['post_refresh'] || {}
|
181
|
+
end
|
182
|
+
|
183
|
+
def post_refresh_task(task_name)
|
184
|
+
task = post_refresh_tasks[task_name]
|
185
|
+
raise ArgumentError.new("Invalid post_refresh task: #{task_name}") if task.nil?
|
186
|
+
task
|
187
|
+
end
|
188
|
+
|
189
|
+
def other_options
|
190
|
+
raw_config.reject { |k,v| %w{environments groups pre_refresh post_refresh}.include?(k) }
|
191
|
+
end
|
192
|
+
|
193
|
+
protected
|
194
|
+
|
195
|
+
def env_names_by_flag(flag_name, flag_value, allow_multiple = false)
|
196
|
+
matching_envs = environments.select { |n, c| c[flag_name] == flag_value }
|
197
|
+
if matching_envs.size > 1 && !allow_multiple
|
198
|
+
raise ArgumentError.new("Only a single environment can be #{flag_name}.")
|
199
|
+
end
|
200
|
+
matching_envs.keys
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Duple
|
2
|
+
class HerokuRunner
|
3
|
+
def initialize(runner)
|
4
|
+
@runner = runner
|
5
|
+
end
|
6
|
+
|
7
|
+
def run(appname, args, tail = nil)
|
8
|
+
@runner.run(heroku_command(appname, args, tail))
|
9
|
+
end
|
10
|
+
|
11
|
+
def capture(appname, args, tail = nil)
|
12
|
+
@runner.capture(heroku_command(appname, args, tail))
|
13
|
+
end
|
14
|
+
|
15
|
+
protected
|
16
|
+
|
17
|
+
def heroku_command(appname, args, tail = nil)
|
18
|
+
command = []
|
19
|
+
command << %{heroku #{args}}
|
20
|
+
command << %{-a #{appname}}
|
21
|
+
command << %{tail} if tail
|
22
|
+
command.join(' ')
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Duple
|
2
|
+
class PGRunner
|
3
|
+
def initialize(runner)
|
4
|
+
@runner = runner
|
5
|
+
end
|
6
|
+
|
7
|
+
def pg_dump(flags, dump_file, credentials)
|
8
|
+
pg_command('pg_dump', flags, credentials, nil, "> #{dump_file}")
|
9
|
+
end
|
10
|
+
|
11
|
+
def pg_restore(flags, dump_file, credentials)
|
12
|
+
pg_command('pg_restore', flags, credentials, '-d', "< #{dump_file}")
|
13
|
+
end
|
14
|
+
|
15
|
+
def pg_command(pg_command, flags, credentials, db_flag, tail = nil)
|
16
|
+
command = []
|
17
|
+
command << %{PGPASSWORD="#{credentials[:password]}"}
|
18
|
+
command << pg_command
|
19
|
+
command << flags
|
20
|
+
command << %{-h #{credentials[:host]} -U #{credentials[:user]} -p #{credentials[:port]}}
|
21
|
+
command << db_flag if db_flag
|
22
|
+
command << credentials[:db]
|
23
|
+
command << tail if tail
|
24
|
+
@runner.run(command.join(' '))
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
data/lib/duple/runner.rb
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
|
3
|
+
module Duple
|
4
|
+
class Runner
|
5
|
+
|
6
|
+
def initialize(options = nil)
|
7
|
+
options ||= {}
|
8
|
+
@options = {
|
9
|
+
log: STDOUT,
|
10
|
+
log_format: ' * Running: %s',
|
11
|
+
dry_run: false,
|
12
|
+
recorder: nil
|
13
|
+
}.merge(options)
|
14
|
+
|
15
|
+
if recorder? && !valid_recorder?
|
16
|
+
raise ArgumentError.new("Invalid :recorder option: #{@options[:recorder]}")
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def run(command, capture = false)
|
21
|
+
log_command(command)
|
22
|
+
record_command(command) if recorder?
|
23
|
+
|
24
|
+
return if dry_run?
|
25
|
+
|
26
|
+
result = capture ? `#{command}` : system(command)
|
27
|
+
raise RuntimeError.new("Command failed: #{$?}") unless $?.success?
|
28
|
+
result
|
29
|
+
end
|
30
|
+
|
31
|
+
def capture(command)
|
32
|
+
run(command, true)
|
33
|
+
end
|
34
|
+
|
35
|
+
def record_command(command)
|
36
|
+
@options[:recorder].puts command
|
37
|
+
end
|
38
|
+
|
39
|
+
def log_command(command)
|
40
|
+
return unless @options[:log]
|
41
|
+
|
42
|
+
formatted = @options[:log_format] % command
|
43
|
+
@options[:log].puts formatted
|
44
|
+
end
|
45
|
+
|
46
|
+
def valid_recorder?
|
47
|
+
@options[:recorder].respond_to?(:puts)
|
48
|
+
end
|
49
|
+
|
50
|
+
def recorder?
|
51
|
+
@options[:recorder]
|
52
|
+
end
|
53
|
+
|
54
|
+
def live?
|
55
|
+
!@options[:dry_run]
|
56
|
+
end
|
57
|
+
|
58
|
+
def dry_run?
|
59
|
+
@options[:dry_run]
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|