database_sanitizer 0.0.13 → 0.0.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 816ccf39b00e6e58b01d27e9641101df1be308d9
4
- data.tar.gz: 8527b8bb3d4ae9939e97004d7c11d5c4df953076
3
+ metadata.gz: 14b6f7afbfbfb4760626dfad121e423a2d9e323b
4
+ data.tar.gz: e50230516670cd61e8dcf7a5e0cc8564c8f649c8
5
5
  SHA512:
6
- metadata.gz: c5e5a21fd47e697898a02b865d252c838119ddaec369e541ed17a25ac4695ddddb5f4b8554a589ef9141e41b21fb67012b2fd9fbeda4d9da469daa6077220641
7
- data.tar.gz: 9110e79e2c1bf8251f6ba44665c622391a1aef4443e1a20c5bcdc26ef9ee98558147976cb2140c468f8e2d16bb9c77cd5c643d9dce979f342ed5186ae8ebb1a9
6
+ metadata.gz: b0e653000e3e6be25c6361b46d263fce5fc8bd3fde2165f0591e397a3ba1a3630b09b3e1fd0955220a0961793703d08c2ba230bc72593703bb5f460f481ec792
7
+ data.tar.gz: 21d96d72a6527adbd16d03a03b9515fbe4323d1069941febce96a76a8ad0db84096ba7489fb5ca4c28d5582011ea3fd3b8f72f34eb3ce05cc80a9f1a89e36704
@@ -28,10 +28,10 @@ begin
28
28
  db = ENV['SOURCE']
29
29
  DatabaseSanitizer::Source.establish_connection(DBCONF[db]).connection
30
30
  db = ENV['TARGET']
31
- ActiveRecord::Base.establish_connection(DBCONF[db]).connection
31
+ DatabaseSanitizer::Destination.establish_connection(DBCONF[db]).connection
32
32
  rescue StandardError
33
33
  abort "Couldn't connect to #{options[:dbconf]}[#{db}]: #{$!}"
34
34
  end
35
35
 
36
- DatabaseSanitizer.export DatabaseSanitizer::Source.connection, ActiveRecord::Base.connection, tables: options[:tables], exclude: options[:exclude_tables], schema: options[:schema]
36
+ DatabaseSanitizer.export tables: options[:tables], exclude: options[:exclude_tables], schema: options[:schema]
37
37
 
@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
20
20
  spec.add_development_dependency 'rake', '~> 10'
21
21
  spec.add_development_dependency 'pg', '~> 0.17'
22
22
 
23
- spec.add_runtime_dependency 'activerecord', '>= 3.0'
23
+ spec.add_runtime_dependency 'activerecord', '>= 3'
24
24
  spec.add_runtime_dependency 'activerecord-comments', '~> 0'
25
25
  spec.add_runtime_dependency 'progress', '~> 3'
26
26
  spec.add_runtime_dependency 'iconv', '~> 1'
@@ -6,6 +6,8 @@ module DatabaseSanitizer
6
6
  CHUNK_SIZE = (ENV['CHUNK_SIZE'] || "1000").to_i
7
7
  class Source < ActiveRecord::Base
8
8
  end
9
+ class Destination < ActiveRecord::Base
10
+ end
9
11
  end
10
12
 
11
13
  require 'database_sanitizer/transformers'
@@ -16,9 +18,9 @@ module DatabaseSanitizer
16
18
 
17
19
  def extract_order comment; comment ? comment[/order_by: ?(\w+)/,1] : nil; end
18
20
 
19
- def read_comments conn, tables
21
+ def read_comments tables
20
22
  tables.inject({}) do |transformers, table|
21
- transformers[table.to_sym] = conn.retrieve_column_comments(table.to_sym).inject({}) do |table_transformers, column|
23
+ transformers[table.to_sym] = Source.connection.retrieve_column_comments(table.to_sym).inject({}) do |table_transformers, column|
22
24
  transformer_key = extract_transformer column[1]
23
25
  unless transformer_key.nil? || Transformers.include?(transformer_key)
24
26
  abort "Transformer '#{transformer_key}' not found (#{table}.#{column[0]})"
@@ -41,12 +43,11 @@ module DatabaseSanitizer
41
43
  else
42
44
  puts 'Reading schema SQL...'
43
45
  schema_src = IO.read File.expand_path(schema, Dir.pwd)
44
- ActiveRecord::Migration.suppress_messages { ActiveRecord::Base.connection.exec_query schema_src }
46
+ ActiveRecord::Migration.suppress_messages { Destination.connection.execute schema_src }
45
47
  end
46
48
  end
47
49
 
48
- def get_chunks table
49
- conn = Source.connection
50
+ def get_chunks conn, table
50
51
  query = "SELECT count(*) FROM #{conn.quote_table_name table}"
51
52
  pg_query = "SELECT reltuples::bigint FROM pg_class WHERE relname=#{conn.quote table}"
52
53
  res = conn.adapter_name == 'PostgreSQL' ? (conn.exec_query(pg_query) rescue false) : false
@@ -57,17 +58,19 @@ module DatabaseSanitizer
57
58
  res.rows[0][0].to_i / CHUNK_SIZE + 1
58
59
  end
59
60
 
60
- def export src, dest, opts={}
61
+ def export opts={}
62
+ src = Source.connection
63
+ dest = Destination.connection
61
64
  duplicate_schema opts[:schema]
62
65
  tables = (opts[:tables] || src.tables.collect(&:to_s)) - (opts[:exclude] || [])
63
- transformers = read_comments dest, tables
66
+ transformers = read_comments tables
64
67
  max_tbl_name_len = transformers.keys.map(&:length).sort.last || 0
65
68
 
66
69
  tables.with_progress('Exporting').each do |table|
67
70
  q_table = dest.quote_table_name table
68
71
  s_table = table.to_sym
69
72
 
70
- get_chunks(table).times_with_progress(table.rjust max_tbl_name_len) do |chunk_i|
73
+ get_chunks(src, table).times_with_progress(table.rjust max_tbl_name_len) do |chunk_i|
71
74
  result = src.exec_query select_query q_table, s_table, (chunk_i * CHUNK_SIZE)
72
75
  dest.exec_query insert_query q_table, s_table, transformers, result
73
76
  end
@@ -75,6 +78,7 @@ module DatabaseSanitizer
75
78
  end
76
79
 
77
80
  def insert_query q_table, s_table, transformers, result
81
+ dest = Destination.connection
78
82
  cols = result.columns.map { |col| dest.quote_column_name col }.join ','
79
83
  ins_query_part = "INSERT INTO #{q_table} (#{cols}) VALUES ("
80
84
  ins_query = StringIO.new
@@ -85,19 +89,20 @@ module DatabaseSanitizer
85
89
  end
86
90
  ins_query << ins_query_part << values.join(',') << '); '
87
91
  end
92
+ ins_query.string
88
93
  end
89
94
 
90
95
  def select_query q_table, s_table, offset
91
- "SELECT * FROM #{q_table} #{order_clause_for_table(s_table)} LIMIT #{CHUNK_SIZE} OFFSET #{offset}"
96
+ "SELECT * FROM #{q_table} #{order_clause s_table} LIMIT #{CHUNK_SIZE} OFFSET #{offset}"
92
97
  end
93
98
 
94
99
  def order_clause s_table
95
- conn = Source.connection
96
100
  order_sql = 'ORDER BY '
97
- order_by = extract_order conn.retrieve_table_comment s_table
101
+ src = Source.connection
102
+ order_by = extract_order src.retrieve_table_comment s_table
98
103
  if order_by
99
- order_sql + conn.quote_table_name(order_by)
100
- elsif conn.column_exists? s_table, :id
104
+ order_sql + src.quote_table_name(order_by)
105
+ elsif src.column_exists? s_table, :id
101
106
  order_sql + 'id'
102
107
  else
103
108
  nil
@@ -1,3 +1,3 @@
1
1
  module DatabaseSanitizer
2
- VERSION = '0.0.13'
2
+ VERSION = '0.0.14'
3
3
  end
@@ -43,25 +43,23 @@ describe DatabaseSanitizer do
43
43
 
44
44
  describe '#read_comments' do
45
45
  before do
46
- ActiveRecord::Migration.suppress_messages do
47
- ActiveRecord::Schema.define do
48
- change_table :test do |t|
49
- t.string :field3
50
- t.string :field4
51
- end
52
- end
53
- end
46
+ DatabaseSanitizer::Source.connection.execute <<-SQL
47
+ ALTER TABLE test
48
+ ADD COLUMN field3 character varying(255),
49
+ ADD COLUMN field4 character varying(255)
50
+ ;
51
+ SQL
54
52
  comments = {
55
53
  field1: 'comment no tag',
56
54
  field2: nil,
57
55
  field3: 'comment sanitize: name',
58
56
  field4: 'sanitize:email'
59
- }.each { |col, com| ActiveRecord::Base.connection.set_column_comment :test, col, com }
57
+ }.each { |col, com| DatabaseSanitizer::Source.connection.set_column_comment :test, col, com }
60
58
  end
61
59
 
62
60
  context 'some defined' do
63
61
  it 'should get transformers' do
64
- transformers = described_class.read_comments(ActiveRecord::Base.connection, [:test])[:test]
62
+ transformers = described_class.read_comments([:test])[:test]
65
63
  expect(transformers[:field1]).to be_nil
66
64
  expect(transformers[:field2]).to be_nil
67
65
  expect(transformers[:field3]).to be_kind_of(Proc)
@@ -70,20 +68,25 @@ describe DatabaseSanitizer do
70
68
  end
71
69
 
72
70
  context 'some undefined' do
73
- before { ActiveRecord::Base.connection.set_column_comment :test, :field2, 'sanitize:undef' }
71
+ before { DatabaseSanitizer::Source.connection.set_column_comment :test, :field2, 'sanitize:undef' }
74
72
  it 'should abort' do
75
- expect(lambda {described_class.read_comments ActiveRecord::Base.connection, [:test]}).to raise_error(SystemExit)
73
+ expect(lambda {described_class.read_comments [:test]}).to raise_error(SystemExit)
76
74
  end
77
75
  end
78
76
  end
79
77
 
78
+ describe '#insert_query' do
79
+ context 'empty db' do
80
+ let(:result) { Struct.new(:rows, :columns).new([], []) }
81
+ it { expect(described_class.insert_query '"test"', :test, {}, result).to eq('') }
82
+ end
83
+ end
84
+
80
85
  describe '#order_clause' do
81
86
  context 'no order comment' do
82
87
  context 'and no id' do
83
88
  before do
84
- ActiveRecord::Migration.suppress_messages do
85
- ActiveRecord::Schema.define { remove_column :test, :id }
86
- end
89
+ DatabaseSanitizer::Source.connection.execute 'ALTER TABLE test DROP COLUMN IF EXISTS id;'
87
90
  end
88
91
 
89
92
  it 'should not order' do
@@ -100,10 +103,10 @@ describe DatabaseSanitizer do
100
103
  end
101
104
 
102
105
  context 'order comment' do
103
- before { ActiveRecord::Base.connection.set_table_comment :test, 'order_by: field2' }
106
+ before { DatabaseSanitizer::Source.connection.set_table_comment :test, 'order_by: field2' }
104
107
 
105
108
  it 'should order by comment' do
106
- expect(described_class.order_clause :test).to end_with(ActiveRecord::Base.connection.quote_table_name 'field2')
109
+ expect(described_class.order_clause :test).to end_with(DatabaseSanitizer::Source.connection.quote_table_name 'field2')
107
110
  end
108
111
  end
109
112
  end
data/spec/spec_helper.rb CHANGED
@@ -7,22 +7,24 @@ require 'database_sanitizer'
7
7
 
8
8
  DBCONF = YAML::load(IO.read(File.expand_path('../config/database.yml', __FILE__)))
9
9
  ENV['DB'] ||= 'postgres'
10
- ActiveRecord::Base.establish_connection DBCONF[ENV['DB']]
10
+ src_conf = DBCONF[ENV['DB']]
11
+ dest_conf = src_conf.update({'datbase' => "#{src_conf['database']}_sanitized"})
12
+ DatabaseSanitizer::Source.establish_connection(src_conf).connection
13
+ DatabaseSanitizer::Destination.establish_connection(dest_conf).connection
11
14
 
12
15
  RSpec.configure do |c|
13
16
  c.around(:each) do |ex|
14
17
  unless ex.metadata[:nodb]
15
- ActiveRecord::Migration.suppress_messages do
16
- ActiveRecord::Schema.define do
17
- create_table :test do |t|
18
- t.string :field1
19
- t.integer :field2
20
- end
21
- end
22
- end
18
+ DatabaseSanitizer::Source.connection.execute <<-SQL
19
+ CREATE TABLE test (
20
+ id integer NOT NULL,
21
+ field1 character varying(255),
22
+ field2 integer
23
+ );
24
+ SQL
23
25
  end
24
26
  ex.run
25
- ActiveRecord::Migration.suppress_messages { ActiveRecord::Schema.define { drop_table :test } } unless ex.metadata[:nodb]
27
+ DatabaseSanitizer::Source.connection.drop_table :test unless ex.metadata[:nodb]
26
28
  end
27
29
  end
28
30
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: database_sanitizer
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.13
4
+ version: 0.0.14
5
5
  platform: ruby
6
6
  authors:
7
7
  - Marton Somogyi
@@ -58,14 +58,14 @@ dependencies:
58
58
  requirements:
59
59
  - - ">="
60
60
  - !ruby/object:Gem::Version
61
- version: '3.0'
61
+ version: '3'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - ">="
67
67
  - !ruby/object:Gem::Version
68
- version: '3.0'
68
+ version: '3'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: activerecord-comments
71
71
  requirement: !ruby/object:Gem::Requirement
@@ -127,7 +127,7 @@ files:
127
127
  - lib/database_sanitizer/transformers.rb
128
128
  - lib/database_sanitizer/version.rb
129
129
  - spec/config/database.yml
130
- - spec/database_exporter_spec.rb
130
+ - spec/database_sanitizer_spec.rb
131
131
  - spec/spec_helper.rb
132
132
  homepage: https://github.com/mcbuddha/database_sanitizer
133
133
  licenses:
@@ -155,5 +155,5 @@ specification_version: 4
155
155
  summary: Export your SQL database
156
156
  test_files:
157
157
  - spec/config/database.yml
158
- - spec/database_exporter_spec.rb
158
+ - spec/database_sanitizer_spec.rb
159
159
  - spec/spec_helper.rb