duple 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +19 -0
- data/.rspec +2 -0
- data/.simplecov +14 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +64 -0
- data/Rakefile +21 -0
- data/bin/duple +11 -0
- data/duple.gemspec +30 -0
- data/lib/duple.rb +3 -0
- data/lib/duple/cli/config.rb +63 -0
- data/lib/duple/cli/copy.rb +29 -0
- data/lib/duple/cli/helpers.rb +192 -0
- data/lib/duple/cli/init.rb +15 -0
- data/lib/duple/cli/refresh.rb +75 -0
- data/lib/duple/cli/root.rb +41 -0
- data/lib/duple/cli/structure.rb +23 -0
- data/lib/duple/configuration.rb +203 -0
- data/lib/duple/heroku_runner.rb +25 -0
- data/lib/duple/pg_runner.rb +27 -0
- data/lib/duple/runner.rb +63 -0
- data/lib/duple/version.rb +3 -0
- data/spec/duple/cli/config_spec.rb +16 -0
- data/spec/duple/cli/copy_spec.rb +112 -0
- data/spec/duple/cli/init_spec.rb +19 -0
- data/spec/duple/cli/refresh_spec.rb +413 -0
- data/spec/duple/cli/structure_spec.rb +133 -0
- data/spec/duple/configuration_spec.rb +166 -0
- data/spec/duple/runner_spec.rb +65 -0
- data/spec/spec_helper.rb +18 -0
- data/spec/support/cli_spec_helpers.rb +85 -0
- data/spec/support/duple.rb +5 -0
- data/spec/support/io_spec_helpers.rb +33 -0
- data/spec/support/rspec_fire.rb +0 -0
- metadata +190 -0
data/.gitignore
ADDED
data/.rspec
ADDED
data/.simplecov
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
class SimpleCov::Formatter::QualityFormatter
|
2
|
+
def format(result)
|
3
|
+
SimpleCov::Formatter::HTMLFormatter.new.format(result)
|
4
|
+
File.open('coverage/covered_percent', 'w') do |f|
|
5
|
+
f.puts result.source_files.covered_percent.to_f
|
6
|
+
end
|
7
|
+
end
|
8
|
+
end
|
9
|
+
SimpleCov.formatter = SimpleCov::Formatter::QualityFormatter
|
10
|
+
|
11
|
+
SimpleCov.start do
|
12
|
+
add_filter '/spec/'
|
13
|
+
add_group 'CLI', 'lib/duple/cli'
|
14
|
+
end
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Jason Wadsworth
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
# Duple
|
2
|
+
|
3
|
+
Duple makes it easy to move PostgreSQL data around between your deployment
|
4
|
+
environments. Duple knows how to move data from one heroku environment to
|
5
|
+
another and how to load it into your local database. It can execute rake or
|
6
|
+
Heroku commands before and after your loading data into your target
|
7
|
+
environment. This is great for scrubbing, replacing, trimming or generating
|
8
|
+
data for your test environments.
|
9
|
+
|
10
|
+
## Installation
|
11
|
+
|
12
|
+
Install the gem:
|
13
|
+
|
14
|
+
$ gem install duple
|
15
|
+
|
16
|
+
Generate a config file:
|
17
|
+
|
18
|
+
$ duple init
|
19
|
+
|
20
|
+
## Configuration
|
21
|
+
|
22
|
+
The generated config file contains samples of all the different ways you can
|
23
|
+
figure the application. Read and modify it, or clear it out and write your own.
|
24
|
+
|
25
|
+
## Usage
|
26
|
+
|
27
|
+
# Resets the stage database
|
28
|
+
# Loads the latest production snapshot into the stage database
|
29
|
+
$ duple refresh production stage
|
30
|
+
|
31
|
+
# Downloads the latest full snapshot from production
|
32
|
+
# Resets the development database
|
33
|
+
# Loads the snapshot into the development database
|
34
|
+
$ duple refresh production development
|
35
|
+
|
36
|
+
# Captures a new production database snapshot
|
37
|
+
# Downloads the latest full snapshot from production
|
38
|
+
# Resets the development database
|
39
|
+
# Loads the snapshot into the development database
|
40
|
+
$ duple refresh production development --capture
|
41
|
+
|
42
|
+
# Downloads the schema and a subset of data from stage
|
43
|
+
# Resets the backstage database
|
44
|
+
# Loads the structure and the subset into the backstage database
|
45
|
+
$ duple refresh stage backstage --group minimal
|
46
|
+
|
47
|
+
# Downloads the data from the specified tables from the stage database
|
48
|
+
# Loads the data into the backstage database
|
49
|
+
$ duple copy stage backstage --tables products categories
|
50
|
+
|
51
|
+
## Future
|
52
|
+
|
53
|
+
* Everything.
|
54
|
+
* Support for skipping pre- and post- refresh steps.
|
55
|
+
* Support for running pre- and post- refresh steps by themselves.
|
56
|
+
* Support for other data stores.
|
57
|
+
|
58
|
+
## Contributing
|
59
|
+
|
60
|
+
1. Fork it
|
61
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
62
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
63
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
64
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
|
3
|
+
require 'bundler/gem_tasks'
|
4
|
+
|
5
|
+
require 'rspec/core/rake_task'
|
6
|
+
RSpec::Core::RakeTask.new(:spec)
|
7
|
+
|
8
|
+
begin
|
9
|
+
require 'cane/rake_task'
|
10
|
+
|
11
|
+
desc "Run cane to check quality metrics"
|
12
|
+
Cane::RakeTask.new(:cane) do |cane|
|
13
|
+
cane.style_measure = 100
|
14
|
+
cane.style_glob = '{lib}/**/*.rb'
|
15
|
+
cane.gte = {'coverage/covered_percent' => 95}
|
16
|
+
end
|
17
|
+
rescue LoadError
|
18
|
+
warn "cane not available, quality task not provided."
|
19
|
+
end
|
20
|
+
|
21
|
+
task :default => [:spec, :cane]
|
data/bin/duple
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require "rubygems"
|
3
|
+
require "thor"
|
4
|
+
|
5
|
+
if File.exists?(File.join(File.expand_path('../..', __FILE__), '.git'))
|
6
|
+
lib_path = File.expand_path('../../lib', __FILE__)
|
7
|
+
$:.unshift(lib_path)
|
8
|
+
end
|
9
|
+
|
10
|
+
require 'duple'
|
11
|
+
Duple::CLI::Root.start
|
data/duple.gemspec
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'duple/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |gem|
|
7
|
+
gem.name = 'duple'
|
8
|
+
gem.version = Duple::VERSION
|
9
|
+
gem.authors = ['Jason Wadsworth']
|
10
|
+
gem.email = ['jdwadsworth@gmail.com']
|
11
|
+
gem.description = %q{
|
12
|
+
Duple simplifies moving and processing data snapshots between development,
|
13
|
+
testing and production environements.
|
14
|
+
}
|
15
|
+
gem.summary = %q{Moves PostgreSQL data from here to there.}
|
16
|
+
gem.homepage = 'https://github.com/subakva/duple'
|
17
|
+
|
18
|
+
gem.files = `git ls-files`.split($/)
|
19
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
20
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
21
|
+
gem.require_paths = ['lib']
|
22
|
+
|
23
|
+
gem.add_dependency('thor')
|
24
|
+
gem.add_development_dependency('rake', ['~> 0.9.2'])
|
25
|
+
gem.add_development_dependency('rspec', ['~> 2.11.0'])
|
26
|
+
gem.add_development_dependency('rspec-fire', ['~> 1.1.3'])
|
27
|
+
gem.add_development_dependency('cane', ['~> 2.3.0'])
|
28
|
+
gem.add_development_dependency('simplecov', ['~> 0.7.1'])
|
29
|
+
|
30
|
+
end
|
data/lib/duple.rb
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
module Duple
|
2
|
+
module CLI
|
3
|
+
class Config < Thor
|
4
|
+
include Duple::CLI::Helpers
|
5
|
+
|
6
|
+
config_option
|
7
|
+
|
8
|
+
no_tasks do
|
9
|
+
def print_hash(header, values)
|
10
|
+
say header
|
11
|
+
say '-' * 80
|
12
|
+
print_table values, indent: 2
|
13
|
+
say
|
14
|
+
end
|
15
|
+
|
16
|
+
def print_tasks(header, task_list)
|
17
|
+
say header
|
18
|
+
say '-' * 80
|
19
|
+
task_list.each do |task_name, commands|
|
20
|
+
say
|
21
|
+
say ' ' + task_name
|
22
|
+
print_table commands, indent: 4
|
23
|
+
end
|
24
|
+
say
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
desc 'environments', 'Prints the environment configurations.'
|
29
|
+
def environments
|
30
|
+
print_hash('Environments', config.environments)
|
31
|
+
end
|
32
|
+
|
33
|
+
desc 'groups', 'Prints the group configurations.'
|
34
|
+
def groups
|
35
|
+
print_hash('Groups', config.groups)
|
36
|
+
end
|
37
|
+
|
38
|
+
desc 'pre-refresh', 'Prints the pre-refresh tasks.'
|
39
|
+
def pre_refresh
|
40
|
+
print_tasks('Pre-Refresh Tasks', config.pre_refresh_tasks)
|
41
|
+
end
|
42
|
+
|
43
|
+
desc 'post-refresh', 'Prints the post-refresh tasks.'
|
44
|
+
def post_refresh
|
45
|
+
print_tasks('Post-Refresh Tasks', config.post_refresh_tasks)
|
46
|
+
end
|
47
|
+
|
48
|
+
desc 'other', 'Prints other options.'
|
49
|
+
def other
|
50
|
+
print_hash('Other Options', config.other_options)
|
51
|
+
end
|
52
|
+
|
53
|
+
desc 'all', 'Prints the current configuration.'
|
54
|
+
def all
|
55
|
+
environments
|
56
|
+
groups
|
57
|
+
pre_refresh
|
58
|
+
post_refresh
|
59
|
+
other
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Duple
|
2
|
+
module CLI
|
3
|
+
class Copy < Thor::Group
|
4
|
+
include Duple::CLI::Helpers
|
5
|
+
|
6
|
+
config_option
|
7
|
+
source_option
|
8
|
+
target_option
|
9
|
+
group_option
|
10
|
+
capture_option
|
11
|
+
dry_run_option
|
12
|
+
tables_option
|
13
|
+
|
14
|
+
def require_included_tables
|
15
|
+
unless config.included_tables.size > 0
|
16
|
+
raise ArgumentError.new('One of --group or --tables options is required.')
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def dump_data
|
21
|
+
postgres.pg_dump(dump_flags, data_file_path, source_credentials)
|
22
|
+
end
|
23
|
+
|
24
|
+
def restore_data
|
25
|
+
postgres.pg_restore('-e -v --no-acl -O -a', data_file_path, target_credentials)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,192 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
module Duple
|
4
|
+
module CLI
|
5
|
+
module Helpers
|
6
|
+
|
7
|
+
def self.included(base)
|
8
|
+
base.send(:include, Thor::Actions)
|
9
|
+
base.send(:include, InstanceMethods)
|
10
|
+
base.send(:extend, ClassMethods)
|
11
|
+
end
|
12
|
+
|
13
|
+
module ClassMethods
|
14
|
+
def source_root
|
15
|
+
File.expand_path(File.join(File.dirname(__FILE__), '../../..'))
|
16
|
+
end
|
17
|
+
|
18
|
+
def config_option
|
19
|
+
class_option :config,
|
20
|
+
desc: 'The location of the config file.',
|
21
|
+
type: :string,
|
22
|
+
aliases: '-c'
|
23
|
+
end
|
24
|
+
|
25
|
+
def source_option
|
26
|
+
class_option :source,
|
27
|
+
desc: 'The name of the source environment.',
|
28
|
+
type: :string,
|
29
|
+
aliases: '-s'
|
30
|
+
end
|
31
|
+
|
32
|
+
def target_option
|
33
|
+
class_option :target,
|
34
|
+
desc: 'The name of the target environment.',
|
35
|
+
type: :string,
|
36
|
+
aliases: '-t'
|
37
|
+
end
|
38
|
+
|
39
|
+
def group_option
|
40
|
+
class_option :group,
|
41
|
+
desc: 'Name of the group configuration to use when dumping source data.',
|
42
|
+
type: :string,
|
43
|
+
aliases: '-g'
|
44
|
+
end
|
45
|
+
|
46
|
+
def capture_option
|
47
|
+
class_option :capture,
|
48
|
+
desc: 'Capture a new source snapshot before refreshing.',
|
49
|
+
type: :boolean
|
50
|
+
end
|
51
|
+
|
52
|
+
def dry_run_option
|
53
|
+
class_option :dry_run,
|
54
|
+
desc: 'Perform a dry run of the command. No data will be moved.',
|
55
|
+
type: :boolean,
|
56
|
+
aliases: '--dry-run'
|
57
|
+
end
|
58
|
+
|
59
|
+
def tables_option(opts = nil)
|
60
|
+
opts ||= {}
|
61
|
+
opts = {
|
62
|
+
desc: 'A list of tables to include when dumping source data.',
|
63
|
+
required: false,
|
64
|
+
type: :array,
|
65
|
+
aliases: '-t'
|
66
|
+
}.merge(opts)
|
67
|
+
class_option :tables, opts
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
module InstanceMethods
|
72
|
+
def default_config_path
|
73
|
+
File.join('config', 'duple.yml')
|
74
|
+
end
|
75
|
+
|
76
|
+
def app_config_path(verify_file = true)
|
77
|
+
config_path = options[:config] || default_config_path
|
78
|
+
if verify_file && !File.exists?(config_path)
|
79
|
+
raise ArgumentError.new("Missing config file: #{config_path}")
|
80
|
+
end
|
81
|
+
config_path
|
82
|
+
end
|
83
|
+
|
84
|
+
def runner
|
85
|
+
@runner ||= Duple::Runner.new(dry_run: config.dry_run?)
|
86
|
+
end
|
87
|
+
|
88
|
+
def postgres
|
89
|
+
@pg_runner ||= Duple::PGRunner.new(runner)
|
90
|
+
end
|
91
|
+
|
92
|
+
def heroku
|
93
|
+
@heroku ||= Duple::HerokuRunner.new(runner)
|
94
|
+
end
|
95
|
+
|
96
|
+
def source_appname
|
97
|
+
@source_appname ||= config.heroku_name(config.source_environment)
|
98
|
+
end
|
99
|
+
|
100
|
+
def target_appname
|
101
|
+
@target_appname ||= config.heroku_name(config.target_environment)
|
102
|
+
end
|
103
|
+
|
104
|
+
def dump_dir_path
|
105
|
+
File.join('tmp', 'duple')
|
106
|
+
end
|
107
|
+
|
108
|
+
def data_file_path
|
109
|
+
@data_file_path ||= File.join(dump_dir_path, "#{config.source_name}-data.dump")
|
110
|
+
end
|
111
|
+
|
112
|
+
def structure_file_path
|
113
|
+
filename = "#{config.source_name}-structure.dump"
|
114
|
+
@structure_file_path ||= File.join(dump_dir_path, filename)
|
115
|
+
end
|
116
|
+
|
117
|
+
def snapshot_file_path(timestamp)
|
118
|
+
filename = "#{config.source_name}-#{timestamp}.dump"
|
119
|
+
@structure_file_path ||= File.join(dump_dir_path, filename)
|
120
|
+
end
|
121
|
+
|
122
|
+
def fetch_heroku_credentials(appname)
|
123
|
+
config_vars = heroku.capture(appname, "config")
|
124
|
+
|
125
|
+
return config.dry_run_credentials(appname) if config.dry_run?
|
126
|
+
|
127
|
+
db_url = config_vars.split("\n").detect { |l| l =~ /DATABASE_URL/ }
|
128
|
+
raise ArgumentError.new("Missing DATABASE_URL variable for #{appname}") if db_url.nil?
|
129
|
+
|
130
|
+
db_url.match(
|
131
|
+
/postgres:\/\/(?<user>.*):(?<password>.*)@(?<host>.*):(?<port>\d*)\/(?<db>.*)/
|
132
|
+
)
|
133
|
+
end
|
134
|
+
|
135
|
+
def fetch_latest_snapshot_time(appname)
|
136
|
+
response = heroku.capture(appname, 'pgbackups')
|
137
|
+
last_line = response.split("\n").last
|
138
|
+
timestring = last_line.match(/\w+\s+(?<timestamp>[\d\s\/\:\.]+)\s+.*/)[:timestamp]
|
139
|
+
DateTime.strptime(timestring, '%Y/%m/%d %H:%M.%S')
|
140
|
+
end
|
141
|
+
|
142
|
+
def reset_database(env)
|
143
|
+
if config.heroku?(env)
|
144
|
+
appname = config.heroku_name(env)
|
145
|
+
heroku.run(appname, 'pg:reset')
|
146
|
+
else
|
147
|
+
# if yes?("Are you sure you want to reset the #{config.target_name} database?", :red)
|
148
|
+
runner.run('bundle exec rake db:drop db:create')
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def dump_flags
|
153
|
+
include_tables = config.included_tables
|
154
|
+
include_flags = include_tables.map { |t| "-t #{t}" }
|
155
|
+
|
156
|
+
exclude_tables = config.excluded_tables
|
157
|
+
exclude_flags = exclude_tables.map { |t| "-T #{t}" }
|
158
|
+
|
159
|
+
flags = [ '-Fc -a', include_flags, exclude_flags ].flatten.compact.join(' ')
|
160
|
+
end
|
161
|
+
|
162
|
+
def source_credentials
|
163
|
+
@source_credentials ||= if config.heroku_source?
|
164
|
+
fetch_heroku_credentials(source_appname)
|
165
|
+
else
|
166
|
+
config.local_credentials
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
def target_credentials
|
171
|
+
@target_credentials ||= if config.heroku_target?
|
172
|
+
fetch_heroku_credentials(target_appname)
|
173
|
+
else
|
174
|
+
config.local_credentials
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
def config
|
179
|
+
@config ||= parse_config
|
180
|
+
end
|
181
|
+
|
182
|
+
def parse_config
|
183
|
+
config_path = File.join(destination_root, app_config_path)
|
184
|
+
config_data = File.read(config_path)
|
185
|
+
erbed = ERB.new(config_data).result
|
186
|
+
config_hash = YAML.load(erbed) || {}
|
187
|
+
Duple::Configuration.new(config_hash, options)
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|