dbx 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/exe/dbx +6 -2
- data/lib/dbx/differ.rb +66 -15
- data/lib/dbx/version.rb +1 -1
- data/lib/dbx.rb +3 -3
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c52bc907eb88537752612d4c288654e737048082
|
4
|
+
data.tar.gz: d4fb5d7f94bf24c126cd896b142eb17fc585fd93
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1755fa1dd6c372745ed3f9234fe1918bfb01f3ef76bff45d5626ff1e0be9300f6ab21fadb096cac23950a27bd2461d2f5130b28c3367e4ed5a7eb5293db0cb1b
|
7
|
+
data.tar.gz: 45cd468ce1ead1093b22fc8fb50171c45fc5f51e7c18ff3bd55b4c703f764d35d1cc950535cc53491238bd47139c92182d89819489db923e6e9c20b14791a113
|
data/Gemfile.lock
CHANGED
data/exe/dbx
CHANGED
@@ -49,6 +49,7 @@ class CLI < Thor
|
|
49
49
|
option :force, type: :boolean, banner: 'remove diff_ table if it exists'
|
50
50
|
option :using, type: :array, banner: 'JOIN USING the columns list here. Ex: id'
|
51
51
|
option :exclude_columns, type: :array, banner: 'Exclude columns from comparison and selection'
|
52
|
+
option :no_a_b, type: :boolean, banner: 'Disable the *_a and *_b columns from *_diff table'
|
52
53
|
def diff(table_a, table_b)
|
53
54
|
handle_global_options
|
54
55
|
DBX::Differ.diff(
|
@@ -56,7 +57,8 @@ class CLI < Thor
|
|
56
57
|
table_b: table_b,
|
57
58
|
using: options[:using],
|
58
59
|
exclude_columns: options[:exclude_columns],
|
59
|
-
force: options[:force]
|
60
|
+
force: options[:force],
|
61
|
+
no_a_b: options[:no_a_b]
|
60
62
|
)
|
61
63
|
end
|
62
64
|
|
@@ -64,6 +66,7 @@ class CLI < Thor
|
|
64
66
|
option :force, type: :boolean, banner: 'remove diff_ table if it exists'
|
65
67
|
option :using, type: :array, banner: 'JOIN USING the columns list here. Ex: id'
|
66
68
|
option :exclude_columns, type: :array, banner: 'Exclude columns from comparison and selection'
|
69
|
+
option :no_a_b, type: :boolean, banner: 'Disable the *_a and *_b columns from *_diff table'
|
67
70
|
def import_diff(src_a, src_b)
|
68
71
|
handle_global_options
|
69
72
|
DBX::Differ.import_and_diff(
|
@@ -71,7 +74,8 @@ class CLI < Thor
|
|
71
74
|
src_b: src_b,
|
72
75
|
using: options[:using],
|
73
76
|
exclude_columns: options[:exclude_columns],
|
74
|
-
force: options[:force]
|
77
|
+
force: options[:force],
|
78
|
+
no_a_b: options[:no_a_b]
|
75
79
|
)
|
76
80
|
end
|
77
81
|
|
data/lib/dbx/differ.rb
CHANGED
@@ -13,7 +13,7 @@ module DBX
|
|
13
13
|
# @param [String] table B Should be newer than table A, but doesn't have to be.
|
14
14
|
# @param [Array<String>] using is the join criteria between the 2 tables.
|
15
15
|
# @param [Array<String>] exclude_columns are excluded from the diff comparison.
|
16
|
-
def diff(table_a:, table_b:, force: false, using: ['id'], exclude_columns: nil)
|
16
|
+
def diff(table_a:, table_b:, force: false, using: ['id'], exclude_columns: nil, no_a_b: false)
|
17
17
|
table_diff = "diff_#{table_a}_#{table_b}"
|
18
18
|
exclude_columns ||= []
|
19
19
|
DBX.info("Creating diff table #{table_diff}")
|
@@ -23,7 +23,7 @@ module DBX
|
|
23
23
|
CREATE TABLE #{table_diff} AS
|
24
24
|
SELECT
|
25
25
|
#{using.join(', ')},
|
26
|
-
#{select_columns(table_a, exclude_columns: using + exclude_columns)}
|
26
|
+
#{select_columns(table_a, exclude_columns: using + exclude_columns, no_a_b: no_a_b)}
|
27
27
|
FROM #{table_a} AS a
|
28
28
|
FULL OUTER JOIN #{table_b} b USING (#{using.join(',')})
|
29
29
|
WHERE
|
@@ -36,10 +36,13 @@ module DBX
|
|
36
36
|
(SELECT COUNT(*) FROM #{table_diff}) diffs
|
37
37
|
SQL
|
38
38
|
end
|
39
|
+
DBX.info("Creating diff stats: #{table_diff}")
|
40
|
+
create_diff_stats(table_diff, force: force)
|
41
|
+
|
39
42
|
DBX.info("Diff complete. Results details in: #{table_diff}")
|
40
43
|
end
|
41
44
|
|
42
|
-
def import_and_diff(src_a:, src_b:, force: false, using: ['id'], exclude_columns: nil)
|
45
|
+
def import_and_diff(src_a:, src_b:, force: false, using: ['id'], exclude_columns: nil, no_a_b: false)
|
43
46
|
DBX.info("Importing #{src_a}")
|
44
47
|
table_a = DBX.import_table(src_a, force: force)
|
45
48
|
|
@@ -47,10 +50,39 @@ module DBX
|
|
47
50
|
DBX.info("Importing #{src_b}")
|
48
51
|
table_b = DBX.import_table(src_b, force: force)
|
49
52
|
|
50
|
-
diff(table_a: table_a, table_b: table_b, force: force, using: using, exclude_columns: exclude_columns)
|
53
|
+
diff(table_a: table_a, table_b: table_b, force: force, using: using, exclude_columns: exclude_columns, no_a_b: no_a_b)
|
51
54
|
end
|
52
55
|
|
53
|
-
def
|
56
|
+
def create_diff_stats(diff_table, force: false)
|
57
|
+
DBX.connection do |conn|
|
58
|
+
diff_stats = "#{diff_table}_stats"
|
59
|
+
conn.execute("DROP TABLE IF EXISTS #{diff_stats}") if force
|
60
|
+
selects = conn.columns(diff_table).map do |column|
|
61
|
+
header, type = column.name, column.type
|
62
|
+
col = header[/(.*)_diff$/, 1]
|
63
|
+
next unless col
|
64
|
+
|
65
|
+
if column.sql_type == 'interval'
|
66
|
+
%{SUM(#{header}) AS #{col}_sum}
|
67
|
+
else
|
68
|
+
case type
|
69
|
+
when :decimal, :integer, :date, :datetime
|
70
|
+
%{SUM(#{header}) AS #{col}_sum}
|
71
|
+
else
|
72
|
+
%{COUNT(#{header}) AS #{col}_count}
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end.compact.join(",\n")
|
76
|
+
conn.execute(<<-SQL)
|
77
|
+
CREATE TABLE #{diff_stats} AS
|
78
|
+
SELECT
|
79
|
+
#{selects}
|
80
|
+
FROM #{diff_table}
|
81
|
+
SQL
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def select_columns(table, exclude_columns: nil, no_a_b: false)
|
54
86
|
exclude_columns ||= []
|
55
87
|
DBX.connection do |conn|
|
56
88
|
conn.columns(table).map do |column|
|
@@ -58,11 +90,13 @@ module DBX
|
|
58
90
|
next if exclude_columns.include?(header)
|
59
91
|
case type
|
60
92
|
when :decimal, :integer
|
61
|
-
select_difference(header)
|
62
|
-
when :date
|
63
|
-
|
93
|
+
select_difference(header, no_a_b: no_a_b)
|
94
|
+
when :date
|
95
|
+
select_difference_as_int(header, no_a_b: no_a_b)
|
96
|
+
when :datetime
|
97
|
+
select_difference_as_interval(header, no_a_b: no_a_b)
|
64
98
|
else
|
65
|
-
select_boolean(header)
|
99
|
+
select_boolean(header, no_a_b: no_a_b)
|
66
100
|
end
|
67
101
|
end.compact.join(',')
|
68
102
|
end
|
@@ -79,22 +113,39 @@ module DBX
|
|
79
113
|
end.compact.join('OR')
|
80
114
|
end
|
81
115
|
|
82
|
-
def select_difference(column)
|
116
|
+
def select_difference(column, no_a_b: false)
|
117
|
+
a = "a.#{column}"
|
118
|
+
b = "b.#{column}"
|
119
|
+
a_b = no_a_b ? '' : "#{a} AS #{column}_a, #{b} AS #{column}_b, "
|
120
|
+
%(#{a_b}(CASE WHEN #{a} = #{b} THEN NULL WHEN #{a} IS NULL THEN #{b} WHEN #{b} IS NULL THEN #{a} ELSE NULLIF(#{b} - #{a}, 0) END) AS #{column}_diff)
|
121
|
+
end
|
122
|
+
|
123
|
+
def select_difference_as_int(column, no_a_b: false)
|
124
|
+
a = "a.#{column}"
|
125
|
+
b = "b.#{column}"
|
126
|
+
a_b = no_a_b ? '' : "#{a} AS #{column}_a, #{b} AS #{column}_b, "
|
127
|
+
%(#{a_b}(CASE WHEN #{a} = #{b} THEN NULL::bigint WHEN #{a} IS NULL THEN 1 WHEN #{b} IS NULL THEN -1 ELSE (#{b} - #{a}) END) AS #{column}_diff)
|
128
|
+
end
|
129
|
+
|
130
|
+
def select_difference_as_interval(column, no_a_b: false)
|
83
131
|
a = "a.#{column}"
|
84
132
|
b = "b.#{column}"
|
85
|
-
|
133
|
+
a_b = no_a_b ? '' : "#{a} AS #{column}_a, #{b} AS #{column}_b, "
|
134
|
+
%(#{a_b}(CASE WHEN #{a} = #{b} THEN NULL::interval WHEN #{a} IS NULL THEN '1 day'::interval WHEN #{b} IS NULL THEN '-1 day'::interval ELSE (#{b} - #{a})::interval END) AS #{column}_diff)
|
86
135
|
end
|
87
136
|
|
88
|
-
def select_difference_as_text(column)
|
137
|
+
def select_difference_as_text(column, no_a_b: false)
|
89
138
|
a = "a.#{column}"
|
90
139
|
b = "b.#{column}"
|
91
|
-
|
140
|
+
a_b = no_a_b ? '' : "#{a} AS #{column}_a, #{b} AS #{column}_b, "
|
141
|
+
%(#{a_b}(CASE WHEN #{a} = #{b} THEN NULL WHEN #{a} IS NULL THEN #{b}::text WHEN #{b} IS NULL THEN #{a}::text ELSE (#{b} - #{a})::text END) AS #{column}_diff)
|
92
142
|
end
|
93
143
|
|
94
|
-
def select_boolean(column)
|
144
|
+
def select_boolean(column, no_a_b: false)
|
95
145
|
a = "a.#{column}"
|
96
146
|
b = "b.#{column}"
|
97
|
-
|
147
|
+
a_b = no_a_b ? '' : "#{a} AS #{column}_a, #{b} AS #{column}_b, "
|
148
|
+
%(#{a_b}NULLIF(#{a} <> #{b}, FALSE) AS #{column}_diff)
|
98
149
|
end
|
99
150
|
end
|
100
151
|
end
|
data/lib/dbx/version.rb
CHANGED
data/lib/dbx.rb
CHANGED
@@ -58,7 +58,7 @@ module DBX
|
|
58
58
|
end
|
59
59
|
|
60
60
|
def parse_table_name(src)
|
61
|
-
File.basename(src).sub(File.extname(src), '')
|
61
|
+
File.basename(src).sub(File.extname(src), '').downcase
|
62
62
|
end
|
63
63
|
|
64
64
|
def create_table(src, name: nil, force: false, sample_rows: config_sample_rows, csv_options: {})
|
@@ -77,12 +77,12 @@ module DBX
|
|
77
77
|
def import_table(src, name: nil, force: false, sample_rows: config_sample_rows, csv_options: {})
|
78
78
|
name ||= parse_table_name(src)
|
79
79
|
connection do |conn|
|
80
|
-
create_table(src, force: force, sample_rows: sample_rows, csv_options: csv_options)
|
80
|
+
create_table(src, name: name, force: force, sample_rows: sample_rows, csv_options: csv_options)
|
81
81
|
# TODO only postgres is support at the moment
|
82
82
|
pg = conn.instance_variable_get(:@connection)
|
83
83
|
types = column_types(src).keys.map{|m| %("#{m}")}
|
84
84
|
|
85
|
-
pg_stmt =
|
85
|
+
pg_stmt = %{COPY "#{name}"(#{types.join(',')}) FROM STDIN CSV}
|
86
86
|
conn.logger.debug(pg_stmt)
|
87
87
|
pg.copy_data(pg_stmt) do
|
88
88
|
first = true
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dbx
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Scott Pierce
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-08-
|
11
|
+
date: 2018-08-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|