duple 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,15 @@
1
+ module Duple
2
+ module CLI
3
+ class Init < Thor::Group
4
+ include Duple::CLI::Helpers
5
+
6
+ config_option
7
+
8
+ def create_sample_config
9
+ config_path = app_config_path(false)
10
+ empty_directory(File.dirname(config_path))
11
+ copy_file('templates/config/duple.yml', config_path)
12
+ end
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,75 @@
1
+ module Duple
2
+ module CLI
3
+ class Refresh < Thor::Group
4
+ include Duple::CLI::Helpers
5
+
6
+ config_option
7
+ source_option
8
+ target_option
9
+ group_option
10
+ capture_option
11
+ tables_option
12
+
13
+ no_tasks do
14
+ def capture_snapshot?
15
+ config.capture? && config.heroku_source?
16
+ end
17
+
18
+ def fetch_snapshot_url?
19
+ config.heroku_source? && !config.filtered_tables?
20
+ end
21
+
22
+ def download_snapshot?
23
+ config.heroku_source? && config.local_target? && !config.filtered_tables?
24
+ end
25
+
26
+ def dump_data?
27
+ config.local_source? || config.filtered_tables?
28
+ end
29
+ end
30
+
31
+ def capture_snapshot
32
+ return unless capture_snapshot?
33
+
34
+ heroku.run(source_appname, 'pgbackups:capture')
35
+ end
36
+
37
+ def fetch_snapshot_url
38
+ return unless fetch_snapshot_url?
39
+
40
+ @source_snapshot_url = heroku.capture(source_appname, 'pgbackups:url').strip
41
+ end
42
+
43
+ def download_snapshot
44
+ return unless download_snapshot?
45
+
46
+ timestamp = fetch_latest_snapshot_time(source_appname)
47
+
48
+ @snapshot_path = snapshot_file_path(timestamp.strftime('%Y-%m-%d-%H-%M-%S'))
49
+ unless File.exists?(@snapshot_path)
50
+ runner.run("curl -o #{@snapshot_path} #{@source_snapshot_url}")
51
+ end
52
+ end
53
+
54
+ def dump_data
55
+ return unless dump_data?
56
+
57
+ postgres.pg_dump(dump_flags, data_file_path, source_credentials)
58
+ end
59
+
60
+ def reset_target
61
+ reset_database(config.target_environment)
62
+ end
63
+
64
+ def restore_database
65
+ if download_snapshot?
66
+ postgres.pg_restore('-e -v --no-acl -O -a', @snapshot_path, target_credentials)
67
+ elsif dump_data?
68
+ postgres.pg_restore('-e -v --no-acl -O -a', data_file_path, target_credentials)
69
+ else
70
+ heroku.run(target_appname, "pgbackups:restore DATABASE #{@source_snapshot_url}")
71
+ end
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,41 @@
1
+ require 'thor'
2
+ require 'thor/group'
3
+ require 'duple/cli/helpers'
4
+ require 'duple/cli/init'
5
+ require 'duple/cli/copy'
6
+ require 'duple/cli/config'
7
+ require 'duple/cli/structure'
8
+ require 'duple/cli/refresh'
9
+ require 'duple/runner'
10
+ require 'duple/pg_runner'
11
+ require 'duple/heroku_runner'
12
+
13
+ module Duple
14
+ module CLI
15
+ class Root < Thor
16
+ include Duple::CLI::Helpers
17
+
18
+ # HACK Override register to handle class_options for groups properly.
19
+ def self.register(klass, task_name, description)
20
+ super(klass, task_name, task_name, description)
21
+ tasks[task_name].options = klass.class_options
22
+ end
23
+
24
+ register Duple::CLI::Init, 'init',
25
+ 'Generates a sample configuration file.'
26
+
27
+ register Duple::CLI::Copy, 'copy',
28
+ 'Copies data from a source to a target database.'
29
+
30
+ register Duple::CLI::Structure, 'structure',
31
+ 'Copies structure from a source to a target database.'
32
+
33
+ register Duple::CLI::Refresh, 'refresh',
34
+ 'Resets and copies schema and data from a source to a target database'
35
+
36
+ register Duple::CLI::Config,
37
+ 'config [COMMAND]', 'Manage your configuration.'
38
+ end
39
+ end
40
+ end
41
+
@@ -0,0 +1,23 @@
1
+ module Duple
2
+ module CLI
3
+ class Structure < Thor::Group
4
+ include Duple::CLI::Helpers
5
+
6
+ config_option
7
+ source_option
8
+ target_option
9
+
10
+ def dump_structure
11
+ postgres.pg_dump('-Fc --no-acl -O -s', structure_file_path, source_credentials)
12
+ end
13
+
14
+ def reset_target
15
+ reset_database(config.target_environment)
16
+ end
17
+
18
+ def load_structure
19
+ postgres.pg_restore('-v --no-acl -O -s', structure_file_path, target_credentials)
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,203 @@
1
+ module Duple
2
+
3
+ # Represents the configuration that will be used to perform the data
4
+ # operations.
5
+ #
6
+ # This class should be the only place in the system that knows about the
7
+ # structure of the config file.
8
+ #
9
+ # This class should not have any knowledge of any particular database
10
+ # system. For example, this class can know about the concept of a "tables",
11
+ # but it should know nothing about flags for PostgreSQL commands.
12
+ class Configuration
13
+ HEROKU = 'heroku'
14
+ LOCAL = 'local'
15
+ VALID_TYPES = [HEROKU, LOCAL]
16
+
17
+ attr_reader :options, :raw_config
18
+
19
+ def initialize(config_hash, options)
20
+ @raw_config = config_hash
21
+ @options = options
22
+ end
23
+
24
+ def default_target_name
25
+ env_names_by_flag('default_target', true).first
26
+ end
27
+
28
+ def target_name
29
+ options[:target] || default_target_name
30
+ end
31
+
32
+ def target_environment
33
+ invalid_target_names = env_names_by_flag('allow_target', false, true)
34
+ if invalid_target_names.include?(target_name)
35
+ raise ArgumentError.new("Invalid target: #{target_name} is not allowed to be a target.")
36
+ end
37
+ environment(target_name)
38
+ end
39
+
40
+ def heroku_target?
41
+ heroku?(target_environment)
42
+ end
43
+
44
+ def local_target?
45
+ local?(target_environment)
46
+ end
47
+
48
+ def default_source_name
49
+ env_names_by_flag('default_source', true).first
50
+ end
51
+
52
+ def source_name
53
+ options[:source] || default_source_name
54
+ end
55
+
56
+ def source_environment
57
+ environment(source_name)
58
+ end
59
+
60
+ def heroku_source?
61
+ heroku?(source_environment)
62
+ end
63
+
64
+ def local_source?
65
+ local?(source_environment)
66
+ end
67
+
68
+ def heroku?(env)
69
+ env['type'] == Duple::Configuration::HEROKU
70
+ end
71
+
72
+ def local?(env)
73
+ env['type'] == Duple::Configuration::LOCAL
74
+ end
75
+
76
+ def heroku_name(env)
77
+ env['appname']
78
+ end
79
+
80
+ def dry_run?
81
+ options[:dry_run]
82
+ end
83
+
84
+ def capture?
85
+ options[:capture]
86
+ end
87
+
88
+ def group_name
89
+ options[:group]
90
+ end
91
+
92
+ def table_names
93
+ options[:tables] || []
94
+ end
95
+
96
+ def filtered_tables?
97
+ included_tables.size > 0 || excluded_tables.size > 0
98
+ end
99
+
100
+ # Returns an array of tables to include, based on the group config and the
101
+ # --tables option. An empty array indicates that ALL tables should be
102
+ # included. If the group has the include_all flag, an empty array will be
103
+ # returned.
104
+ def included_tables
105
+ tables = table_names
106
+ if group_name
107
+ g = group(group_name)
108
+ return [] if g['include_all']
109
+ tables += (g['include_tables'] || [])
110
+ end
111
+ tables.uniq.sort
112
+ end
113
+
114
+ # Returns an array of tables to exclude, based on the group config and the
115
+ # --tables option. The --tables option takes precedence over the --group
116
+ # option, so if a table is excluded from a group, but specified in the
117
+ # --tables option the table will NOT be excluded.
118
+ def excluded_tables
119
+ tables = []
120
+ if group_name
121
+ g = group(group_name)
122
+ tables += (g['exclude_tables'] || [])
123
+ end
124
+ tables -= table_names
125
+ tables
126
+ end
127
+
128
+ def dry_run_credentials(envname)
129
+ {
130
+ user: "[#{envname}.USER]",
131
+ password: "[#{envname}.PASS]",
132
+ host: "[#{envname}.HOST]",
133
+ port: "[#{envname}.PORT]",
134
+ db: "[#{envname}.DB]"
135
+ }
136
+ end
137
+
138
+ def local_credentials
139
+ # TODO: Override these defaults from the config
140
+ {
141
+ user: 'postgres',
142
+ password: '',
143
+ host: 'localhost',
144
+ port: '5432',
145
+ db: 'duple_development'
146
+ }
147
+ end
148
+
149
+ def environments
150
+ raw_config['environments'] || {}
151
+ end
152
+
153
+ def environment(env_name)
154
+ env = environments[env_name]
155
+ raise ArgumentError.new("Invalid environment: #{env_name}") if env.nil?
156
+ env
157
+ end
158
+
159
+ def groups
160
+ raw_config['groups'] || {}
161
+ end
162
+
163
+ def group(group_name)
164
+ group = groups[group_name]
165
+ raise ArgumentError.new("Invalid group: #{group_name}") if group.nil?
166
+ group
167
+ end
168
+
169
+ def pre_refresh_tasks
170
+ raw_config['pre_refresh'] || {}
171
+ end
172
+
173
+ def pre_refresh_task(task_name)
174
+ task = pre_refresh_tasks[task_name]
175
+ raise ArgumentError.new("Invalid pre_refresh task: #{task_name}") if task.nil?
176
+ task
177
+ end
178
+
179
+ def post_refresh_tasks
180
+ raw_config['post_refresh'] || {}
181
+ end
182
+
183
+ def post_refresh_task(task_name)
184
+ task = post_refresh_tasks[task_name]
185
+ raise ArgumentError.new("Invalid post_refresh task: #{task_name}") if task.nil?
186
+ task
187
+ end
188
+
189
+ def other_options
190
+ raw_config.reject { |k,v| %w{environments groups pre_refresh post_refresh}.include?(k) }
191
+ end
192
+
193
+ protected
194
+
195
+ def env_names_by_flag(flag_name, flag_value, allow_multiple = false)
196
+ matching_envs = environments.select { |n, c| c[flag_name] == flag_value }
197
+ if matching_envs.size > 1 && !allow_multiple
198
+ raise ArgumentError.new("Only a single environment can be #{flag_name}.")
199
+ end
200
+ matching_envs.keys
201
+ end
202
+ end
203
+ end
@@ -0,0 +1,25 @@
1
+ module Duple
2
+ class HerokuRunner
3
+ def initialize(runner)
4
+ @runner = runner
5
+ end
6
+
7
+ def run(appname, args, tail = nil)
8
+ @runner.run(heroku_command(appname, args, tail))
9
+ end
10
+
11
+ def capture(appname, args, tail = nil)
12
+ @runner.capture(heroku_command(appname, args, tail))
13
+ end
14
+
15
+ protected
16
+
17
+ def heroku_command(appname, args, tail = nil)
18
+ command = []
19
+ command << %{heroku #{args}}
20
+ command << %{-a #{appname}}
21
+ command << %{tail} if tail
22
+ command.join(' ')
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,27 @@
1
+ module Duple
2
+ class PGRunner
3
+ def initialize(runner)
4
+ @runner = runner
5
+ end
6
+
7
+ def pg_dump(flags, dump_file, credentials)
8
+ pg_command('pg_dump', flags, credentials, nil, "> #{dump_file}")
9
+ end
10
+
11
+ def pg_restore(flags, dump_file, credentials)
12
+ pg_command('pg_restore', flags, credentials, '-d', "< #{dump_file}")
13
+ end
14
+
15
+ def pg_command(pg_command, flags, credentials, db_flag, tail = nil)
16
+ command = []
17
+ command << %{PGPASSWORD="#{credentials[:password]}"}
18
+ command << pg_command
19
+ command << flags
20
+ command << %{-h #{credentials[:host]} -U #{credentials[:user]} -p #{credentials[:port]}}
21
+ command << db_flag if db_flag
22
+ command << credentials[:db]
23
+ command << tail if tail
24
+ @runner.run(command.join(' '))
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,63 @@
1
+ require 'fileutils'
2
+
3
+ module Duple
4
+ class Runner
5
+
6
+ def initialize(options = nil)
7
+ options ||= {}
8
+ @options = {
9
+ log: STDOUT,
10
+ log_format: ' * Running: %s',
11
+ dry_run: false,
12
+ recorder: nil
13
+ }.merge(options)
14
+
15
+ if recorder? && !valid_recorder?
16
+ raise ArgumentError.new("Invalid :recorder option: #{@options[:recorder]}")
17
+ end
18
+ end
19
+
20
+ def run(command, capture = false)
21
+ log_command(command)
22
+ record_command(command) if recorder?
23
+
24
+ return if dry_run?
25
+
26
+ result = capture ? `#{command}` : system(command)
27
+ raise RuntimeError.new("Command failed: #{$?}") unless $?.success?
28
+ result
29
+ end
30
+
31
+ def capture(command)
32
+ run(command, true)
33
+ end
34
+
35
+ def record_command(command)
36
+ @options[:recorder].puts command
37
+ end
38
+
39
+ def log_command(command)
40
+ return unless @options[:log]
41
+
42
+ formatted = @options[:log_format] % command
43
+ @options[:log].puts formatted
44
+ end
45
+
46
+ def valid_recorder?
47
+ @options[:recorder].respond_to?(:puts)
48
+ end
49
+
50
+ def recorder?
51
+ @options[:recorder]
52
+ end
53
+
54
+ def live?
55
+ !@options[:dry_run]
56
+ end
57
+
58
+ def dry_run?
59
+ @options[:dry_run]
60
+ end
61
+ end
62
+ end
63
+