database_sanitizer 0.0.13 → 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/database_sanitizer +2 -2
- data/database_sanitizer.gemspec +1 -1
- data/lib/database_sanitizer.rb +18 -13
- data/lib/database_sanitizer/version.rb +1 -1
- data/spec/{database_exporter_spec.rb → database_sanitizer_spec.rb} +20 -17
- data/spec/spec_helper.rb +12 -10
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 14b6f7afbfbfb4760626dfad121e423a2d9e323b
|
4
|
+
data.tar.gz: e50230516670cd61e8dcf7a5e0cc8564c8f649c8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b0e653000e3e6be25c6361b46d263fce5fc8bd3fde2165f0591e397a3ba1a3630b09b3e1fd0955220a0961793703d08c2ba230bc72593703bb5f460f481ec792
|
7
|
+
data.tar.gz: 21d96d72a6527adbd16d03a03b9515fbe4323d1069941febce96a76a8ad0db84096ba7489fb5ca4c28d5582011ea3fd3b8f72f34eb3ce05cc80a9f1a89e36704
|
data/bin/database_sanitizer
CHANGED
@@ -28,10 +28,10 @@ begin
|
|
28
28
|
db = ENV['SOURCE']
|
29
29
|
DatabaseSanitizer::Source.establish_connection(DBCONF[db]).connection
|
30
30
|
db = ENV['TARGET']
|
31
|
-
|
31
|
+
DatabaseSanitizer::Destination.establish_connection(DBCONF[db]).connection
|
32
32
|
rescue StandardError
|
33
33
|
abort "Couldn't connect to #{options[:dbconf]}[#{db}]: #{$!}"
|
34
34
|
end
|
35
35
|
|
36
|
-
DatabaseSanitizer.export
|
36
|
+
DatabaseSanitizer.export tables: options[:tables], exclude: options[:exclude_tables], schema: options[:schema]
|
37
37
|
|
data/database_sanitizer.gemspec
CHANGED
@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.add_development_dependency 'rake', '~> 10'
|
21
21
|
spec.add_development_dependency 'pg', '~> 0.17'
|
22
22
|
|
23
|
-
spec.add_runtime_dependency 'activerecord', '>= 3
|
23
|
+
spec.add_runtime_dependency 'activerecord', '>= 3'
|
24
24
|
spec.add_runtime_dependency 'activerecord-comments', '~> 0'
|
25
25
|
spec.add_runtime_dependency 'progress', '~> 3'
|
26
26
|
spec.add_runtime_dependency 'iconv', '~> 1'
|
data/lib/database_sanitizer.rb
CHANGED
@@ -6,6 +6,8 @@ module DatabaseSanitizer
|
|
6
6
|
CHUNK_SIZE = (ENV['CHUNK_SIZE'] || "1000").to_i
|
7
7
|
class Source < ActiveRecord::Base
|
8
8
|
end
|
9
|
+
class Destination < ActiveRecord::Base
|
10
|
+
end
|
9
11
|
end
|
10
12
|
|
11
13
|
require 'database_sanitizer/transformers'
|
@@ -16,9 +18,9 @@ module DatabaseSanitizer
|
|
16
18
|
|
17
19
|
def extract_order comment; comment ? comment[/order_by: ?(\w+)/,1] : nil; end
|
18
20
|
|
19
|
-
def read_comments
|
21
|
+
def read_comments tables
|
20
22
|
tables.inject({}) do |transformers, table|
|
21
|
-
transformers[table.to_sym] =
|
23
|
+
transformers[table.to_sym] = Source.connection.retrieve_column_comments(table.to_sym).inject({}) do |table_transformers, column|
|
22
24
|
transformer_key = extract_transformer column[1]
|
23
25
|
unless transformer_key.nil? || Transformers.include?(transformer_key)
|
24
26
|
abort "Transformer '#{transformer_key}' not found (#{table}.#{column[0]})"
|
@@ -41,12 +43,11 @@ module DatabaseSanitizer
|
|
41
43
|
else
|
42
44
|
puts 'Reading schema SQL...'
|
43
45
|
schema_src = IO.read File.expand_path(schema, Dir.pwd)
|
44
|
-
ActiveRecord::Migration.suppress_messages {
|
46
|
+
ActiveRecord::Migration.suppress_messages { Destination.connection.execute schema_src }
|
45
47
|
end
|
46
48
|
end
|
47
49
|
|
48
|
-
def get_chunks table
|
49
|
-
conn = Source.connection
|
50
|
+
def get_chunks conn, table
|
50
51
|
query = "SELECT count(*) FROM #{conn.quote_table_name table}"
|
51
52
|
pg_query = "SELECT reltuples::bigint FROM pg_class WHERE relname=#{conn.quote table}"
|
52
53
|
res = conn.adapter_name == 'PostgreSQL' ? (conn.exec_query(pg_query) rescue false) : false
|
@@ -57,17 +58,19 @@ module DatabaseSanitizer
|
|
57
58
|
res.rows[0][0].to_i / CHUNK_SIZE + 1
|
58
59
|
end
|
59
60
|
|
60
|
-
def export
|
61
|
+
def export opts={}
|
62
|
+
src = Source.connection
|
63
|
+
dest = Destination.connection
|
61
64
|
duplicate_schema opts[:schema]
|
62
65
|
tables = (opts[:tables] || src.tables.collect(&:to_s)) - (opts[:exclude] || [])
|
63
|
-
transformers = read_comments
|
66
|
+
transformers = read_comments tables
|
64
67
|
max_tbl_name_len = transformers.keys.map(&:length).sort.last || 0
|
65
68
|
|
66
69
|
tables.with_progress('Exporting').each do |table|
|
67
70
|
q_table = dest.quote_table_name table
|
68
71
|
s_table = table.to_sym
|
69
72
|
|
70
|
-
get_chunks(table).times_with_progress(table.rjust max_tbl_name_len) do |chunk_i|
|
73
|
+
get_chunks(src, table).times_with_progress(table.rjust max_tbl_name_len) do |chunk_i|
|
71
74
|
result = src.exec_query select_query q_table, s_table, (chunk_i * CHUNK_SIZE)
|
72
75
|
dest.exec_query insert_query q_table, s_table, transformers, result
|
73
76
|
end
|
@@ -75,6 +78,7 @@ module DatabaseSanitizer
|
|
75
78
|
end
|
76
79
|
|
77
80
|
def insert_query q_table, s_table, transformers, result
|
81
|
+
dest = Destination.connection
|
78
82
|
cols = result.columns.map { |col| dest.quote_column_name col }.join ','
|
79
83
|
ins_query_part = "INSERT INTO #{q_table} (#{cols}) VALUES ("
|
80
84
|
ins_query = StringIO.new
|
@@ -85,19 +89,20 @@ module DatabaseSanitizer
|
|
85
89
|
end
|
86
90
|
ins_query << ins_query_part << values.join(',') << '); '
|
87
91
|
end
|
92
|
+
ins_query.string
|
88
93
|
end
|
89
94
|
|
90
95
|
def select_query q_table, s_table, offset
|
91
|
-
"SELECT * FROM #{q_table} #{
|
96
|
+
"SELECT * FROM #{q_table} #{order_clause s_table} LIMIT #{CHUNK_SIZE} OFFSET #{offset}"
|
92
97
|
end
|
93
98
|
|
94
99
|
def order_clause s_table
|
95
|
-
conn = Source.connection
|
96
100
|
order_sql = 'ORDER BY '
|
97
|
-
|
101
|
+
src = Source.connection
|
102
|
+
order_by = extract_order src.retrieve_table_comment s_table
|
98
103
|
if order_by
|
99
|
-
order_sql +
|
100
|
-
elsif
|
104
|
+
order_sql + src.quote_table_name(order_by)
|
105
|
+
elsif src.column_exists? s_table, :id
|
101
106
|
order_sql + 'id'
|
102
107
|
else
|
103
108
|
nil
|
@@ -43,25 +43,23 @@ describe DatabaseSanitizer do
|
|
43
43
|
|
44
44
|
describe '#read_comments' do
|
45
45
|
before do
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
end
|
53
|
-
end
|
46
|
+
DatabaseSanitizer::Source.connection.execute <<-SQL
|
47
|
+
ALTER TABLE test
|
48
|
+
ADD COLUMN field3 character varying(255),
|
49
|
+
ADD COLUMN field4 character varying(255)
|
50
|
+
;
|
51
|
+
SQL
|
54
52
|
comments = {
|
55
53
|
field1: 'comment no tag',
|
56
54
|
field2: nil,
|
57
55
|
field3: 'comment sanitize: name',
|
58
56
|
field4: 'sanitize:email'
|
59
|
-
}.each { |col, com|
|
57
|
+
}.each { |col, com| DatabaseSanitizer::Source.connection.set_column_comment :test, col, com }
|
60
58
|
end
|
61
59
|
|
62
60
|
context 'some defined' do
|
63
61
|
it 'should get transformers' do
|
64
|
-
transformers = described_class.read_comments(
|
62
|
+
transformers = described_class.read_comments([:test])[:test]
|
65
63
|
expect(transformers[:field1]).to be_nil
|
66
64
|
expect(transformers[:field2]).to be_nil
|
67
65
|
expect(transformers[:field3]).to be_kind_of(Proc)
|
@@ -70,20 +68,25 @@ describe DatabaseSanitizer do
|
|
70
68
|
end
|
71
69
|
|
72
70
|
context 'some undefined' do
|
73
|
-
before {
|
71
|
+
before { DatabaseSanitizer::Source.connection.set_column_comment :test, :field2, 'sanitize:undef' }
|
74
72
|
it 'should abort' do
|
75
|
-
expect(lambda {described_class.read_comments
|
73
|
+
expect(lambda {described_class.read_comments [:test]}).to raise_error(SystemExit)
|
76
74
|
end
|
77
75
|
end
|
78
76
|
end
|
79
77
|
|
78
|
+
describe '#insert_query' do
|
79
|
+
context 'empty db' do
|
80
|
+
let(:result) { Struct.new(:rows, :columns).new([], []) }
|
81
|
+
it { expect(described_class.insert_query '"test"', :test, {}, result).to eq('') }
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
80
85
|
describe '#order_clause' do
|
81
86
|
context 'no order comment' do
|
82
87
|
context 'and no id' do
|
83
88
|
before do
|
84
|
-
|
85
|
-
ActiveRecord::Schema.define { remove_column :test, :id }
|
86
|
-
end
|
89
|
+
DatabaseSanitizer::Source.connection.execute 'ALTER TABLE test DROP COLUMN IF EXISTS id;'
|
87
90
|
end
|
88
91
|
|
89
92
|
it 'should not order' do
|
@@ -100,10 +103,10 @@ describe DatabaseSanitizer do
|
|
100
103
|
end
|
101
104
|
|
102
105
|
context 'order comment' do
|
103
|
-
before {
|
106
|
+
before { DatabaseSanitizer::Source.connection.set_table_comment :test, 'order_by: field2' }
|
104
107
|
|
105
108
|
it 'should order by comment' do
|
106
|
-
expect(described_class.order_clause :test).to end_with(
|
109
|
+
expect(described_class.order_clause :test).to end_with(DatabaseSanitizer::Source.connection.quote_table_name 'field2')
|
107
110
|
end
|
108
111
|
end
|
109
112
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -7,22 +7,24 @@ require 'database_sanitizer'
|
|
7
7
|
|
8
8
|
DBCONF = YAML::load(IO.read(File.expand_path('../config/database.yml', __FILE__)))
|
9
9
|
ENV['DB'] ||= 'postgres'
|
10
|
-
|
10
|
+
src_conf = DBCONF[ENV['DB']]
|
11
|
+
dest_conf = src_conf.update({'datbase' => "#{src_conf['database']}_sanitized"})
|
12
|
+
DatabaseSanitizer::Source.establish_connection(src_conf).connection
|
13
|
+
DatabaseSanitizer::Destination.establish_connection(dest_conf).connection
|
11
14
|
|
12
15
|
RSpec.configure do |c|
|
13
16
|
c.around(:each) do |ex|
|
14
17
|
unless ex.metadata[:nodb]
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
end
|
18
|
+
DatabaseSanitizer::Source.connection.execute <<-SQL
|
19
|
+
CREATE TABLE test (
|
20
|
+
id integer NOT NULL,
|
21
|
+
field1 character varying(255),
|
22
|
+
field2 integer
|
23
|
+
);
|
24
|
+
SQL
|
23
25
|
end
|
24
26
|
ex.run
|
25
|
-
|
27
|
+
DatabaseSanitizer::Source.connection.drop_table :test unless ex.metadata[:nodb]
|
26
28
|
end
|
27
29
|
end
|
28
30
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: database_sanitizer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.14
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Marton Somogyi
|
@@ -58,14 +58,14 @@ dependencies:
|
|
58
58
|
requirements:
|
59
59
|
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '3
|
61
|
+
version: '3'
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '3
|
68
|
+
version: '3'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: activerecord-comments
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -127,7 +127,7 @@ files:
|
|
127
127
|
- lib/database_sanitizer/transformers.rb
|
128
128
|
- lib/database_sanitizer/version.rb
|
129
129
|
- spec/config/database.yml
|
130
|
-
- spec/
|
130
|
+
- spec/database_sanitizer_spec.rb
|
131
131
|
- spec/spec_helper.rb
|
132
132
|
homepage: https://github.com/mcbuddha/database_sanitizer
|
133
133
|
licenses:
|
@@ -155,5 +155,5 @@ specification_version: 4
|
|
155
155
|
summary: Export your SQL database
|
156
156
|
test_files:
|
157
157
|
- spec/config/database.yml
|
158
|
-
- spec/
|
158
|
+
- spec/database_sanitizer_spec.rb
|
159
159
|
- spec/spec_helper.rb
|