databender 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +18 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/Gemfile +4 -0
- data/LICENSE +22 -0
- data/LICENSE.txt +22 -0
- data/README.md +31 -0
- data/Rakefile +9 -0
- data/bin/databender +6 -0
- data/databender.gemspec +32 -0
- data/lib/databender.rb +4 -0
- data/lib/databender/cli/main.rb +40 -0
- data/lib/databender/config.rb +40 -0
- data/lib/databender/connection.rb +52 -0
- data/lib/databender/foreign_constraint.rb +12 -0
- data/lib/databender/runner.rb +87 -0
- data/lib/databender/sql_helper.rb +50 -0
- data/lib/databender/table.rb +15 -0
- data/lib/databender/table_order.rb +36 -0
- data/lib/databender/templates/database.yml +8 -0
- data/lib/databender/templates/filter.yml +18 -0
- data/lib/databender/version.rb +3 -0
- data/lib/subset.sh.mustache +40 -0
- metadata +213 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 4f0224e71e9970b7d51d20dc020ceb5ddb449f37
|
4
|
+
data.tar.gz: 5fe6a3f2af057f4d523b8bbc3bf9c0672a943833
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 5a88848c0ee536aacbc32914c7fa84a95c70475ac10c3d0c679fdf47b3a7be4f66279bca268397377fe4e31a54a91702ab7ad4597d317cc2a3649b15f81b7937
|
7
|
+
data.tar.gz: 90f1af10b4950336312cb406e3b7c5e1f60de5fd2906bcb6c71d76d8c34b063c7aad6016a464a96b114896aa9fc2abe54fbb82c65d0cd1b82826ed2707ab67ca
|
data/.gitignore
ADDED
data/.ruby-gemset
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
databender
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
ruby-2.4.1
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2015 RC
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE.
|
22
|
+
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2015 RC
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
# Databender
|
2
|
+
|
3
|
+
Database Subset Generator
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'databender'
|
11
|
+
```
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle
|
16
|
+
|
17
|
+
Or install it yourself as:
|
18
|
+
|
19
|
+
$ gem install databender
|
20
|
+
|
21
|
+
## Usage
|
22
|
+
|
23
|
+
TODO: Write usage instructions here
|
24
|
+
|
25
|
+
## Contributing
|
26
|
+
|
27
|
+
1. Fork it ( https://github.com/[my-github-username]/databender/fork )
|
28
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
29
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
30
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
31
|
+
5. Create a new Pull Request
|
data/Rakefile
ADDED
data/bin/databender
ADDED
data/databender.gemspec
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'databender/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'databender'
|
8
|
+
spec.version = Databender::VERSION
|
9
|
+
spec.authors = ['RC']
|
10
|
+
spec.email = ['rc.chandru@gmail.com']
|
11
|
+
spec.summary = %q{Database subset generator}
|
12
|
+
spec.description = %q{Database subset generator}
|
13
|
+
spec.homepage = ''
|
14
|
+
spec.license = 'MIT'
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = ['databender']
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ['lib']
|
20
|
+
|
21
|
+
spec.add_dependency 'thor', '~> 0.20.0'
|
22
|
+
spec.add_dependency 'activerecord', '=5.1.4'
|
23
|
+
spec.add_dependency 'mysql2', '=0.4.9'
|
24
|
+
spec.add_dependency 'mustache', '~> 1.0', '>= 1.0.5'
|
25
|
+
spec.add_dependency 'configatron', '~> 4.5', '>= 4.5.1'
|
26
|
+
spec.add_dependency 'terminal-table', '~> 1.8', '>= 1.8.0'
|
27
|
+
|
28
|
+
spec.add_development_dependency 'bundler', '~> 1.7'
|
29
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
30
|
+
spec.add_development_dependency 'pry', '~> 0'
|
31
|
+
|
32
|
+
end
|
data/lib/databender.rb
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
require_relative '../../../lib/databender/runner'
|
2
|
+
|
3
|
+
module Databender
|
4
|
+
module Cli
|
5
|
+
class Main < Thor
|
6
|
+
include Thor::Actions
|
7
|
+
|
8
|
+
|
9
|
+
source_root File.expand_path('../../templates', __FILE__)
|
10
|
+
|
11
|
+
option :db_name, required: true, desc: 'Name of the database'
|
12
|
+
option :driver, required: false, desc: 'Driver: mysql|postgres', default: 'mysql'
|
13
|
+
desc 'init', 'Initialize configuration and filters'
|
14
|
+
def init
|
15
|
+
say 'Creating baseline configuration and filter...', :green
|
16
|
+
template 'database.yml', 'config/database.yml'
|
17
|
+
filter_path = "config/filters/#{options[:db_name]}.yml"
|
18
|
+
template 'filter.yml', filter_path
|
19
|
+
say "Please review #{filter_path} to verify initial settings.", :green
|
20
|
+
end
|
21
|
+
|
22
|
+
option :db_name, required: true, desc: 'Name of the database'
|
23
|
+
desc 'dry_run', 'Perform a dry-run of the subset script without importing the data'
|
24
|
+
def dry_run
|
25
|
+
say "Analyzing #{options[:db_name]}", :green
|
26
|
+
end
|
27
|
+
|
28
|
+
option :db_name, required: true, desc: 'Name of the database'
|
29
|
+
desc 'generate', 'Generate subset given a database'
|
30
|
+
def generate
|
31
|
+
say "Creating subset for #{options[:db_name]}", :green
|
32
|
+
Databender::Runner.process! options[:db_name]
|
33
|
+
puts ''
|
34
|
+
run 'sh subset.sh', verbose: false
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'configatron'
|
2
|
+
|
3
|
+
module Databender
|
4
|
+
class Config
|
5
|
+
class << self
|
6
|
+
|
7
|
+
def load!(db_name, config_path = 'config/database.yml')
|
8
|
+
db_yml = config_path
|
9
|
+
db_config = YAML::load(IO.read(db_yml))
|
10
|
+
filter_config = YAML::load(IO.read("config/filters/#{db_name}.yml"))
|
11
|
+
configatron.configure_from_hash(filter_config.merge({source: db_config[db_name]}))
|
12
|
+
end
|
13
|
+
|
14
|
+
def target_db
|
15
|
+
"#{configatron.source.database}_subset"
|
16
|
+
end
|
17
|
+
|
18
|
+
def table_filters
|
19
|
+
configatron.tables.filters || {}
|
20
|
+
end
|
21
|
+
|
22
|
+
def max_rows
|
23
|
+
configatron.tables.max_row_count
|
24
|
+
end
|
25
|
+
|
26
|
+
def column_filters
|
27
|
+
configatron.columns.filters || {}
|
28
|
+
end
|
29
|
+
|
30
|
+
def method_missing(method)
|
31
|
+
if method == :configatron
|
32
|
+
super(method)
|
33
|
+
else
|
34
|
+
configatron[method]
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'databender/foreign_constraint'
|
3
|
+
|
4
|
+
module Databender
|
5
|
+
class Connection
|
6
|
+
|
7
|
+
def initialize(connection_params)
|
8
|
+
@conn = ActiveRecord::Base.establish_connection(connection_params).connection
|
9
|
+
end
|
10
|
+
|
11
|
+
def execute(sql)
|
12
|
+
@conn.execute(sql).entries.flatten.compact.map(&:to_sym)
|
13
|
+
end
|
14
|
+
|
15
|
+
def execute_count(sql)
|
16
|
+
@conn.execute(sql)
|
17
|
+
end
|
18
|
+
|
19
|
+
def tables_for(db_name)
|
20
|
+
execute(%[
|
21
|
+
SELECT table_name
|
22
|
+
FROM information_schema.tables
|
23
|
+
WHERE table_schema = '#{db_name}';
|
24
|
+
])
|
25
|
+
end
|
26
|
+
|
27
|
+
def columns_for(db_name, table_name)
|
28
|
+
execute(%[
|
29
|
+
SELECT column_name
|
30
|
+
FROM information_schema.columns
|
31
|
+
WHERE table_schema = '#{db_name}'
|
32
|
+
and table_name = '#{table_name}';
|
33
|
+
])
|
34
|
+
end
|
35
|
+
|
36
|
+
def foreign_key_dependency_map_for(db_name)
|
37
|
+
rows = @conn.execute(%[
|
38
|
+
SELECT table_name, column_name, referenced_table_name, referenced_column_name
|
39
|
+
FROM information_schema.key_column_usage
|
40
|
+
WHERE table_schema = '#{db_name}' AND referenced_table_name is not null
|
41
|
+
AND table_name != referenced_table_name
|
42
|
+
ORDER BY table_name;
|
43
|
+
])
|
44
|
+
rows.each_with_object({}) do |row, map|
|
45
|
+
table, column, ref_table_name, ref_column_name = row
|
46
|
+
parent = ForeignConstraint.new(table, column, ref_table_name, ref_column_name)
|
47
|
+
map.has_key?(table) ? map[table] << parent : map[table] = [parent]
|
48
|
+
end.symbolize_keys
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
module Databender
|
2
|
+
class ForeignConstraint
|
3
|
+
attr_accessor :table_name, :column_name, :ref_table_name, :ref_column_name
|
4
|
+
|
5
|
+
def initialize(table_name, column_name, ref_table_name, ref_column_name)
|
6
|
+
@table_name = table_name
|
7
|
+
@column_name = column_name
|
8
|
+
@ref_table_name = ref_table_name
|
9
|
+
@ref_column_name = ref_column_name
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
require 'databender/version'
|
2
|
+
require 'yaml'
|
3
|
+
require 'mustache'
|
4
|
+
require 'terminal-table'
|
5
|
+
require 'databender/config'
|
6
|
+
require 'databender/sql_helper'
|
7
|
+
require 'databender/table'
|
8
|
+
require 'databender/connection'
|
9
|
+
require 'databender/table_order'
|
10
|
+
|
11
|
+
GEM_ROOT = File.expand_path("../..", __FILE__)
|
12
|
+
|
13
|
+
module Databender
|
14
|
+
class Runner
|
15
|
+
extend Databender::SQLHelper
|
16
|
+
|
17
|
+
|
18
|
+
|
19
|
+
def self.generate_script(params)
|
20
|
+
template = File.read("#{GEM_ROOT}/subset.sh.mustache")
|
21
|
+
File.write('subset.sh', Mustache.render(template, params))
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.apply_column_filters(table, source, source_db, target_db)
|
25
|
+
columns = source.columns_for(source_db, table.name)
|
26
|
+
overlapping_filters = Databender::Config.column_filters.keys & columns
|
27
|
+
if overlapping_filters.present?
|
28
|
+
column_filter = Databender::Config.column_filters[overlapping_filters.first]
|
29
|
+
resolve_column_filter(target_db, overlapping_filters.first, column_filter)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.process!(db_name)
|
34
|
+
unless db_name
|
35
|
+
puts 'Parameter db_name not specified. Terminating!'
|
36
|
+
exit(1)
|
37
|
+
end
|
38
|
+
|
39
|
+
report = []
|
40
|
+
|
41
|
+
Databender::Config.load!(db_name)
|
42
|
+
target_db = Databender::Config.target_db
|
43
|
+
source_db = Databender::Config.source.database
|
44
|
+
|
45
|
+
source = Databender::Connection.new(Databender::Config.source.to_h)
|
46
|
+
|
47
|
+
tables = source.tables_for(source_db)
|
48
|
+
|
49
|
+
## process tables with filters first
|
50
|
+
tables.sort_by! {|table| [Databender::Config.table_filters.keys.include?(table.to_sym) ? 0 : 1, table]}
|
51
|
+
|
52
|
+
all_tables = tables.collect {|table| Databender::Table.new(table)}
|
53
|
+
|
54
|
+
ordered_tables = Databender::TableOrder.order_by_foreign_key_dependency(source, source_db, all_tables)
|
55
|
+
|
56
|
+
entries = ordered_tables.collect do |table|
|
57
|
+
column_filters = apply_column_filters table, source, source_db, target_db
|
58
|
+
sql = count_all_query source_db, table.name
|
59
|
+
all_count = source.execute_count(sql).first.first
|
60
|
+
sql, count_query, filter = if Databender::Config.table_filters.keys.include?(table.name)
|
61
|
+
conditions = merge_filters(Databender::Config.table_filters[table.name], column_filters)
|
62
|
+
[insert_into_select(source_db, table.name, conditions),
|
63
|
+
count_filtered_query(source_db, table.name, conditions), conditions]
|
64
|
+
else
|
65
|
+
if column_filters.present?
|
66
|
+
[insert_into_select(source_db, table.name, column_filters), count_filtered_query(source_db, table.name, column_filters), column_filters]
|
67
|
+
else
|
68
|
+
parents = Databender::TableOrder.parent_tables_for(table.name)
|
69
|
+
condition = parents.present? ? where_clause_by_reference(target_db, parents) : nil
|
70
|
+
[insert_into_select(source_db, table.name, condition), count_filtered_query(source_db, table.name, condition), nil]
|
71
|
+
end
|
72
|
+
end
|
73
|
+
subset_count = source.execute_count(count_query).count
|
74
|
+
report << [table.name, all_count, subset_count, filter && filter]
|
75
|
+
{sql: sql, table: table.name}
|
76
|
+
end
|
77
|
+
|
78
|
+
headings = ['Table Name', 'Total Rows', 'Fetched Rows', 'Filter(s)']
|
79
|
+
tty = Terminal::Table.new headings: headings, rows: report
|
80
|
+
puts tty
|
81
|
+
|
82
|
+
|
83
|
+
self.generate_script source: Databender::Config.source, target_db: target_db, entries: entries, database: db_name
|
84
|
+
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'databender/config'
|
2
|
+
|
3
|
+
module Databender
|
4
|
+
module SQLHelper
|
5
|
+
|
6
|
+
def insert_into_select(source_db, table, condition = nil)
|
7
|
+
sql = 'INSERT INTO %s SELECT * FROM %s.%s' % [table, source_db, table]
|
8
|
+
sql = apply_condition(condition, sql)
|
9
|
+
sql
|
10
|
+
end
|
11
|
+
|
12
|
+
def resolve_column_filter(target_db, column, filter)
|
13
|
+
if filter.starts_with? 'refers'
|
14
|
+
match = filter.match(/refers\((\w+),\s*(\w+)/)
|
15
|
+
'%s in (select %s from %s.%s)' % [column, match[2], target_db, match[1]]
|
16
|
+
else
|
17
|
+
filter
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def merge_filters(filter, additional_filter)
|
22
|
+
additional_filter ? [filter, 'and', additional_filter].join(' ') : filter
|
23
|
+
end
|
24
|
+
|
25
|
+
def where_clause_by_reference(target_db, parents)
|
26
|
+
parents.collect do |parent|
|
27
|
+
'%s in (select %s from %s.%s)' % [parent.column_name, parent.ref_column_name, target_db, parent.ref_table_name]
|
28
|
+
end.join(' and ')
|
29
|
+
end
|
30
|
+
|
31
|
+
def count_all_query(source_db, table)
|
32
|
+
'SELECT count(1) cnt from %s.%s' % [source_db, table]
|
33
|
+
end
|
34
|
+
|
35
|
+
def count_filtered_query(source_db, table, condition = nil)
|
36
|
+
sql = 'SELECT 1 from %s.%s' % [source_db, table]
|
37
|
+
sql = apply_condition(condition, sql)
|
38
|
+
sql
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def apply_condition(condition, sql)
|
44
|
+
sql += condition ? ' WHERE %s' % [condition] : " limit #{Databender::Config.max_rows}"
|
45
|
+
sql
|
46
|
+
end
|
47
|
+
|
48
|
+
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module Databender
|
2
|
+
class TableOrder
|
3
|
+
|
4
|
+
class << self
|
5
|
+
|
6
|
+
def order_by_foreign_key_dependency(source, db_name, tables_with_rank)
|
7
|
+
@tables_with_rank = unique_group_by(tables_with_rank, :name).with_indifferent_access
|
8
|
+
@dependency_map = source.foreign_key_dependency_map_for(db_name).with_indifferent_access
|
9
|
+
tables_with_no_dependencies = @tables_with_rank.keys - @dependency_map.keys
|
10
|
+
tables_with_no_dependencies.each { |table_name| @tables_with_rank[table_name].rank = 0 }
|
11
|
+
@dependency_map.keys.each do |table|
|
12
|
+
rank_for(table)
|
13
|
+
end
|
14
|
+
@tables_with_rank.values.flatten.sort {|x,y| x.rank <=> y.rank}
|
15
|
+
end
|
16
|
+
|
17
|
+
def rank_for(table)
|
18
|
+
@tables_with_rank[table].rank ||= @dependency_map[table].collect do |parent|
|
19
|
+
@tables_with_rank[parent.ref_table_name].rank ||= rank_for(parent.ref_table_name)
|
20
|
+
end.max + 1
|
21
|
+
end
|
22
|
+
|
23
|
+
def unique_group_by(array, key)
|
24
|
+
array.each_with_object({}) do |element, hash|
|
25
|
+
hash[element.send(key)] = element
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def parent_tables_for(table_name)
|
30
|
+
@dependency_map[table_name]
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
tables:
|
2
|
+
# Tables with rows lesser than min_row_count will be fully imported with no filters applied
|
3
|
+
min_row_count: 20
|
4
|
+
|
5
|
+
# For tables with no filters, the maximum number of rows to import
|
6
|
+
max_row_count: 1000
|
7
|
+
|
8
|
+
# specify table specific filters here
|
9
|
+
filters:
|
10
|
+
#users: email_id in ('a@b.com', 'c@d.com')
|
11
|
+
#employees: age > 30
|
12
|
+
|
13
|
+
columns:
|
14
|
+
# specify column filters applicable to all tables that contain that column
|
15
|
+
filters:
|
16
|
+
#status: status = 'ACTIVE'
|
17
|
+
#created_at: created_at > '2017-01-01 00:00:00'
|
18
|
+
|
@@ -0,0 +1,40 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
set -e
|
3
|
+
|
4
|
+
#Utility function to fire queries opening a connection with database
|
5
|
+
function target_execute() {
|
6
|
+
MYSQL_PWD='{{source.password}}' mysql -u{{source.username}} -h{{source.host}} -P{{source.port}} {{target_db}} -e "$1"
|
7
|
+
}
|
8
|
+
|
9
|
+
function execute() {
|
10
|
+
MYSQL_PWD='{{source.password}}' mysql -u{{source.username}} -h{{source.host}} -P{{source.port}} -e "$1"
|
11
|
+
}
|
12
|
+
|
13
|
+
#Pull schema from source database
|
14
|
+
echo "Pulling schema from {{source.database}}.."
|
15
|
+
MYSQL_PWD='{{source.password}}' mysqldump -u{{source.username}} -h{{source.host}} -P{{source.port}} --no-data {{source.database}} > /tmp/databender_schema.sql
|
16
|
+
|
17
|
+
#Drop & create destination database. Load source schema.
|
18
|
+
echo "Creating and loading schema to {{target_db}}.."
|
19
|
+
execute 'drop database if exists {{target_db}};'
|
20
|
+
execute 'create database {{target_db}};'
|
21
|
+
MYSQL_PWD='{{source.password}}' mysql -u{{source.username}} -h{{source.host}} -P{{source.port}} {{target_db}} < /tmp/databender_schema.sql
|
22
|
+
|
23
|
+
echo ""
|
24
|
+
echo "Generating subset database..."
|
25
|
+
|
26
|
+
## Process each table
|
27
|
+
{{#entries}}
|
28
|
+
echo "\033[36mProcessing \033[32m{{{table}}}\033[0m..."
|
29
|
+
target_execute "{{{sql}}}"
|
30
|
+
{{/entries}}
|
31
|
+
|
32
|
+
echo "Done!"
|
33
|
+
echo ""
|
34
|
+
|
35
|
+
mkdir -p dumps
|
36
|
+
|
37
|
+
MYSQL_PWD='{{source.password}}' mysqldump -u{{source.username}} -h{{source.host}} -P{{source.port}} {{target_db}} | gzip > dumps/{{database}}.sql.gz
|
38
|
+
|
39
|
+
echo "\033[92mThe dump of db_subset is available at dumps/{{database}}.sql.gz. Use gunzip to extract followed by mysql command to load. The subset database is intact in the db server too!"
|
40
|
+
|
metadata
ADDED
@@ -0,0 +1,213 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: databender
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- RC
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-10-01 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: thor
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.20.0
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.20.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: activerecord
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 5.1.4
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 5.1.4
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: mysql2
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.4.9
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.4.9
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: mustache
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.0'
|
62
|
+
- - ">="
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: 1.0.5
|
65
|
+
type: :runtime
|
66
|
+
prerelease: false
|
67
|
+
version_requirements: !ruby/object:Gem::Requirement
|
68
|
+
requirements:
|
69
|
+
- - "~>"
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
version: '1.0'
|
72
|
+
- - ">="
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: 1.0.5
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: configatron
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - "~>"
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '4.5'
|
82
|
+
- - ">="
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
version: 4.5.1
|
85
|
+
type: :runtime
|
86
|
+
prerelease: false
|
87
|
+
version_requirements: !ruby/object:Gem::Requirement
|
88
|
+
requirements:
|
89
|
+
- - "~>"
|
90
|
+
- !ruby/object:Gem::Version
|
91
|
+
version: '4.5'
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: 4.5.1
|
95
|
+
- !ruby/object:Gem::Dependency
|
96
|
+
name: terminal-table
|
97
|
+
requirement: !ruby/object:Gem::Requirement
|
98
|
+
requirements:
|
99
|
+
- - "~>"
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '1.8'
|
102
|
+
- - ">="
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
version: 1.8.0
|
105
|
+
type: :runtime
|
106
|
+
prerelease: false
|
107
|
+
version_requirements: !ruby/object:Gem::Requirement
|
108
|
+
requirements:
|
109
|
+
- - "~>"
|
110
|
+
- !ruby/object:Gem::Version
|
111
|
+
version: '1.8'
|
112
|
+
- - ">="
|
113
|
+
- !ruby/object:Gem::Version
|
114
|
+
version: 1.8.0
|
115
|
+
- !ruby/object:Gem::Dependency
|
116
|
+
name: bundler
|
117
|
+
requirement: !ruby/object:Gem::Requirement
|
118
|
+
requirements:
|
119
|
+
- - "~>"
|
120
|
+
- !ruby/object:Gem::Version
|
121
|
+
version: '1.7'
|
122
|
+
type: :development
|
123
|
+
prerelease: false
|
124
|
+
version_requirements: !ruby/object:Gem::Requirement
|
125
|
+
requirements:
|
126
|
+
- - "~>"
|
127
|
+
- !ruby/object:Gem::Version
|
128
|
+
version: '1.7'
|
129
|
+
- !ruby/object:Gem::Dependency
|
130
|
+
name: rake
|
131
|
+
requirement: !ruby/object:Gem::Requirement
|
132
|
+
requirements:
|
133
|
+
- - "~>"
|
134
|
+
- !ruby/object:Gem::Version
|
135
|
+
version: '10.0'
|
136
|
+
type: :development
|
137
|
+
prerelease: false
|
138
|
+
version_requirements: !ruby/object:Gem::Requirement
|
139
|
+
requirements:
|
140
|
+
- - "~>"
|
141
|
+
- !ruby/object:Gem::Version
|
142
|
+
version: '10.0'
|
143
|
+
- !ruby/object:Gem::Dependency
|
144
|
+
name: pry
|
145
|
+
requirement: !ruby/object:Gem::Requirement
|
146
|
+
requirements:
|
147
|
+
- - "~>"
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: '0'
|
150
|
+
type: :development
|
151
|
+
prerelease: false
|
152
|
+
version_requirements: !ruby/object:Gem::Requirement
|
153
|
+
requirements:
|
154
|
+
- - "~>"
|
155
|
+
- !ruby/object:Gem::Version
|
156
|
+
version: '0'
|
157
|
+
description: Database subset generator
|
158
|
+
email:
|
159
|
+
- rc.chandru@gmail.com
|
160
|
+
executables:
|
161
|
+
- databender
|
162
|
+
extensions: []
|
163
|
+
extra_rdoc_files: []
|
164
|
+
files:
|
165
|
+
- ".gitignore"
|
166
|
+
- ".ruby-gemset"
|
167
|
+
- ".ruby-version"
|
168
|
+
- Gemfile
|
169
|
+
- LICENSE
|
170
|
+
- LICENSE.txt
|
171
|
+
- README.md
|
172
|
+
- Rakefile
|
173
|
+
- bin/databender
|
174
|
+
- databender.gemspec
|
175
|
+
- dumps/magic_list_service_test.sql.gz
|
176
|
+
- lib/databender.rb
|
177
|
+
- lib/databender/cli/main.rb
|
178
|
+
- lib/databender/config.rb
|
179
|
+
- lib/databender/connection.rb
|
180
|
+
- lib/databender/foreign_constraint.rb
|
181
|
+
- lib/databender/runner.rb
|
182
|
+
- lib/databender/sql_helper.rb
|
183
|
+
- lib/databender/table.rb
|
184
|
+
- lib/databender/table_order.rb
|
185
|
+
- lib/databender/templates/database.yml
|
186
|
+
- lib/databender/templates/filter.yml
|
187
|
+
- lib/databender/version.rb
|
188
|
+
- lib/subset.sh.mustache
|
189
|
+
homepage: ''
|
190
|
+
licenses:
|
191
|
+
- MIT
|
192
|
+
metadata: {}
|
193
|
+
post_install_message:
|
194
|
+
rdoc_options: []
|
195
|
+
require_paths:
|
196
|
+
- lib
|
197
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
198
|
+
requirements:
|
199
|
+
- - ">="
|
200
|
+
- !ruby/object:Gem::Version
|
201
|
+
version: '0'
|
202
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
203
|
+
requirements:
|
204
|
+
- - ">="
|
205
|
+
- !ruby/object:Gem::Version
|
206
|
+
version: '0'
|
207
|
+
requirements: []
|
208
|
+
rubyforge_project:
|
209
|
+
rubygems_version: 2.6.12
|
210
|
+
signing_key:
|
211
|
+
specification_version: 4
|
212
|
+
summary: Database subset generator
|
213
|
+
test_files: []
|