duple 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,19 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+ scratch
19
+ config
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --color
2
+ --format documentation
@@ -0,0 +1,14 @@
1
+ class SimpleCov::Formatter::QualityFormatter
2
+ def format(result)
3
+ SimpleCov::Formatter::HTMLFormatter.new.format(result)
4
+ File.open('coverage/covered_percent', 'w') do |f|
5
+ f.puts result.source_files.covered_percent.to_f
6
+ end
7
+ end
8
+ end
9
+ SimpleCov.formatter = SimpleCov::Formatter::QualityFormatter
10
+
11
+ SimpleCov.start do
12
+ add_filter '/spec/'
13
+ add_group 'CLI', 'lib/duple/cli'
14
+ end
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in duple.gemspec
4
+ gemspec
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2012 Jason Wadsworth
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,64 @@
1
+ # Duple
2
+
3
+ Duple makes it easy to move PostgreSQL data around between your deployment
4
+ environments. Duple knows how to move data from one heroku environment to
5
+ another and how to load it into your local database. It can execute rake or
6
+ Heroku commands before and after your loading data into your target
7
+ environment. This is great for scrubbing, replacing, trimming or generating
8
+ data for your test environments.
9
+
10
+ ## Installation
11
+
12
+ Install the gem:
13
+
14
+ $ gem install duple
15
+
16
+ Generate a config file:
17
+
18
+ $ duple init
19
+
20
+ ## Configuration
21
+
22
+ The generated config file contains samples of all the different ways you can
23
+ figure the application. Read and modify it, or clear it out and write your own.
24
+
25
+ ## Usage
26
+
27
+ # Resets the stage database
28
+ # Loads the latest production snapshot into the stage database
29
+ $ duple refresh production stage
30
+
31
+ # Downloads the latest full snapshot from production
32
+ # Resets the development database
33
+ # Loads the snapshot into the development database
34
+ $ duple refresh production development
35
+
36
+ # Captures a new production database snapshot
37
+ # Downloads the latest full snapshot from production
38
+ # Resets the development database
39
+ # Loads the snapshot into the development database
40
+ $ duple refresh production development --capture
41
+
42
+ # Downloads the schema and a subset of data from stage
43
+ # Resets the backstage database
44
+ # Loads the structure and the subset into the backstage database
45
+ $ duple refresh stage backstage --group minimal
46
+
47
+ # Downloads the data from the specified tables from the stage database
48
+ # Loads the data into the backstage database
49
+ $ duple copy stage backstage --tables products categories
50
+
51
+ ## Future
52
+
53
+ * Everything.
54
+ * Support for skipping pre- and post- refresh steps.
55
+ * Support for running pre- and post- refresh steps by themselves.
56
+ * Support for other data stores.
57
+
58
+ ## Contributing
59
+
60
+ 1. Fork it
61
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
62
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
63
+ 4. Push to the branch (`git push origin my-new-feature`)
64
+ 5. Create new Pull Request
@@ -0,0 +1,21 @@
1
+ #!/usr/bin/env rake
2
+
3
+ require 'bundler/gem_tasks'
4
+
5
+ require 'rspec/core/rake_task'
6
+ RSpec::Core::RakeTask.new(:spec)
7
+
8
+ begin
9
+ require 'cane/rake_task'
10
+
11
+ desc "Run cane to check quality metrics"
12
+ Cane::RakeTask.new(:cane) do |cane|
13
+ cane.style_measure = 100
14
+ cane.style_glob = '{lib}/**/*.rb'
15
+ cane.gte = {'coverage/covered_percent' => 95}
16
+ end
17
+ rescue LoadError
18
+ warn "cane not available, quality task not provided."
19
+ end
20
+
21
+ task :default => [:spec, :cane]
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env ruby
2
+ require "rubygems"
3
+ require "thor"
4
+
5
+ if File.exists?(File.join(File.expand_path('../..', __FILE__), '.git'))
6
+ lib_path = File.expand_path('../../lib', __FILE__)
7
+ $:.unshift(lib_path)
8
+ end
9
+
10
+ require 'duple'
11
+ Duple::CLI::Root.start
@@ -0,0 +1,30 @@
1
+ # -*- encoding: utf-8 -*-
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'duple/version'
5
+
6
+ Gem::Specification.new do |gem|
7
+ gem.name = 'duple'
8
+ gem.version = Duple::VERSION
9
+ gem.authors = ['Jason Wadsworth']
10
+ gem.email = ['jdwadsworth@gmail.com']
11
+ gem.description = %q{
12
+ Duple simplifies moving and processing data snapshots between development,
13
+ testing and production environements.
14
+ }
15
+ gem.summary = %q{Moves PostgreSQL data from here to there.}
16
+ gem.homepage = 'https://github.com/subakva/duple'
17
+
18
+ gem.files = `git ls-files`.split($/)
19
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
20
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
21
+ gem.require_paths = ['lib']
22
+
23
+ gem.add_dependency('thor')
24
+ gem.add_development_dependency('rake', ['~> 0.9.2'])
25
+ gem.add_development_dependency('rspec', ['~> 2.11.0'])
26
+ gem.add_development_dependency('rspec-fire', ['~> 1.1.3'])
27
+ gem.add_development_dependency('cane', ['~> 2.3.0'])
28
+ gem.add_development_dependency('simplecov', ['~> 0.7.1'])
29
+
30
+ end
@@ -0,0 +1,3 @@
1
+ require 'duple/version'
2
+ require 'duple/configuration'
3
+ require 'duple/cli/root'
@@ -0,0 +1,63 @@
1
+ module Duple
2
+ module CLI
3
+ class Config < Thor
4
+ include Duple::CLI::Helpers
5
+
6
+ config_option
7
+
8
+ no_tasks do
9
+ def print_hash(header, values)
10
+ say header
11
+ say '-' * 80
12
+ print_table values, indent: 2
13
+ say
14
+ end
15
+
16
+ def print_tasks(header, task_list)
17
+ say header
18
+ say '-' * 80
19
+ task_list.each do |task_name, commands|
20
+ say
21
+ say ' ' + task_name
22
+ print_table commands, indent: 4
23
+ end
24
+ say
25
+ end
26
+ end
27
+
28
+ desc 'environments', 'Prints the environment configurations.'
29
+ def environments
30
+ print_hash('Environments', config.environments)
31
+ end
32
+
33
+ desc 'groups', 'Prints the group configurations.'
34
+ def groups
35
+ print_hash('Groups', config.groups)
36
+ end
37
+
38
+ desc 'pre-refresh', 'Prints the pre-refresh tasks.'
39
+ def pre_refresh
40
+ print_tasks('Pre-Refresh Tasks', config.pre_refresh_tasks)
41
+ end
42
+
43
+ desc 'post-refresh', 'Prints the post-refresh tasks.'
44
+ def post_refresh
45
+ print_tasks('Post-Refresh Tasks', config.post_refresh_tasks)
46
+ end
47
+
48
+ desc 'other', 'Prints other options.'
49
+ def other
50
+ print_hash('Other Options', config.other_options)
51
+ end
52
+
53
+ desc 'all', 'Prints the current configuration.'
54
+ def all
55
+ environments
56
+ groups
57
+ pre_refresh
58
+ post_refresh
59
+ other
60
+ end
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,29 @@
1
+ module Duple
2
+ module CLI
3
+ class Copy < Thor::Group
4
+ include Duple::CLI::Helpers
5
+
6
+ config_option
7
+ source_option
8
+ target_option
9
+ group_option
10
+ capture_option
11
+ dry_run_option
12
+ tables_option
13
+
14
+ def require_included_tables
15
+ unless config.included_tables.size > 0
16
+ raise ArgumentError.new('One of --group or --tables options is required.')
17
+ end
18
+ end
19
+
20
+ def dump_data
21
+ postgres.pg_dump(dump_flags, data_file_path, source_credentials)
22
+ end
23
+
24
+ def restore_data
25
+ postgres.pg_restore('-e -v --no-acl -O -a', data_file_path, target_credentials)
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,192 @@
1
+ require 'yaml'
2
+
3
+ module Duple
4
+ module CLI
5
+ module Helpers
6
+
7
+ def self.included(base)
8
+ base.send(:include, Thor::Actions)
9
+ base.send(:include, InstanceMethods)
10
+ base.send(:extend, ClassMethods)
11
+ end
12
+
13
+ module ClassMethods
14
+ def source_root
15
+ File.expand_path(File.join(File.dirname(__FILE__), '../../..'))
16
+ end
17
+
18
+ def config_option
19
+ class_option :config,
20
+ desc: 'The location of the config file.',
21
+ type: :string,
22
+ aliases: '-c'
23
+ end
24
+
25
+ def source_option
26
+ class_option :source,
27
+ desc: 'The name of the source environment.',
28
+ type: :string,
29
+ aliases: '-s'
30
+ end
31
+
32
+ def target_option
33
+ class_option :target,
34
+ desc: 'The name of the target environment.',
35
+ type: :string,
36
+ aliases: '-t'
37
+ end
38
+
39
+ def group_option
40
+ class_option :group,
41
+ desc: 'Name of the group configuration to use when dumping source data.',
42
+ type: :string,
43
+ aliases: '-g'
44
+ end
45
+
46
+ def capture_option
47
+ class_option :capture,
48
+ desc: 'Capture a new source snapshot before refreshing.',
49
+ type: :boolean
50
+ end
51
+
52
+ def dry_run_option
53
+ class_option :dry_run,
54
+ desc: 'Perform a dry run of the command. No data will be moved.',
55
+ type: :boolean,
56
+ aliases: '--dry-run'
57
+ end
58
+
59
+ def tables_option(opts = nil)
60
+ opts ||= {}
61
+ opts = {
62
+ desc: 'A list of tables to include when dumping source data.',
63
+ required: false,
64
+ type: :array,
65
+ aliases: '-t'
66
+ }.merge(opts)
67
+ class_option :tables, opts
68
+ end
69
+ end
70
+
71
+ module InstanceMethods
72
+ def default_config_path
73
+ File.join('config', 'duple.yml')
74
+ end
75
+
76
+ def app_config_path(verify_file = true)
77
+ config_path = options[:config] || default_config_path
78
+ if verify_file && !File.exists?(config_path)
79
+ raise ArgumentError.new("Missing config file: #{config_path}")
80
+ end
81
+ config_path
82
+ end
83
+
84
+ def runner
85
+ @runner ||= Duple::Runner.new(dry_run: config.dry_run?)
86
+ end
87
+
88
+ def postgres
89
+ @pg_runner ||= Duple::PGRunner.new(runner)
90
+ end
91
+
92
+ def heroku
93
+ @heroku ||= Duple::HerokuRunner.new(runner)
94
+ end
95
+
96
+ def source_appname
97
+ @source_appname ||= config.heroku_name(config.source_environment)
98
+ end
99
+
100
+ def target_appname
101
+ @target_appname ||= config.heroku_name(config.target_environment)
102
+ end
103
+
104
+ def dump_dir_path
105
+ File.join('tmp', 'duple')
106
+ end
107
+
108
+ def data_file_path
109
+ @data_file_path ||= File.join(dump_dir_path, "#{config.source_name}-data.dump")
110
+ end
111
+
112
+ def structure_file_path
113
+ filename = "#{config.source_name}-structure.dump"
114
+ @structure_file_path ||= File.join(dump_dir_path, filename)
115
+ end
116
+
117
+ def snapshot_file_path(timestamp)
118
+ filename = "#{config.source_name}-#{timestamp}.dump"
119
+ @structure_file_path ||= File.join(dump_dir_path, filename)
120
+ end
121
+
122
+ def fetch_heroku_credentials(appname)
123
+ config_vars = heroku.capture(appname, "config")
124
+
125
+ return config.dry_run_credentials(appname) if config.dry_run?
126
+
127
+ db_url = config_vars.split("\n").detect { |l| l =~ /DATABASE_URL/ }
128
+ raise ArgumentError.new("Missing DATABASE_URL variable for #{appname}") if db_url.nil?
129
+
130
+ db_url.match(
131
+ /postgres:\/\/(?<user>.*):(?<password>.*)@(?<host>.*):(?<port>\d*)\/(?<db>.*)/
132
+ )
133
+ end
134
+
135
+ def fetch_latest_snapshot_time(appname)
136
+ response = heroku.capture(appname, 'pgbackups')
137
+ last_line = response.split("\n").last
138
+ timestring = last_line.match(/\w+\s+(?<timestamp>[\d\s\/\:\.]+)\s+.*/)[:timestamp]
139
+ DateTime.strptime(timestring, '%Y/%m/%d %H:%M.%S')
140
+ end
141
+
142
+ def reset_database(env)
143
+ if config.heroku?(env)
144
+ appname = config.heroku_name(env)
145
+ heroku.run(appname, 'pg:reset')
146
+ else
147
+ # if yes?("Are you sure you want to reset the #{config.target_name} database?", :red)
148
+ runner.run('bundle exec rake db:drop db:create')
149
+ end
150
+ end
151
+
152
+ def dump_flags
153
+ include_tables = config.included_tables
154
+ include_flags = include_tables.map { |t| "-t #{t}" }
155
+
156
+ exclude_tables = config.excluded_tables
157
+ exclude_flags = exclude_tables.map { |t| "-T #{t}" }
158
+
159
+ flags = [ '-Fc -a', include_flags, exclude_flags ].flatten.compact.join(' ')
160
+ end
161
+
162
+ def source_credentials
163
+ @source_credentials ||= if config.heroku_source?
164
+ fetch_heroku_credentials(source_appname)
165
+ else
166
+ config.local_credentials
167
+ end
168
+ end
169
+
170
+ def target_credentials
171
+ @target_credentials ||= if config.heroku_target?
172
+ fetch_heroku_credentials(target_appname)
173
+ else
174
+ config.local_credentials
175
+ end
176
+ end
177
+
178
+ def config
179
+ @config ||= parse_config
180
+ end
181
+
182
+ def parse_config
183
+ config_path = File.join(destination_root, app_config_path)
184
+ config_data = File.read(config_path)
185
+ erbed = ERB.new(config_data).result
186
+ config_hash = YAML.load(erbed) || {}
187
+ Duple::Configuration.new(config_hash, options)
188
+ end
189
+ end
190
+ end
191
+ end
192
+ end