dbx 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3a9156b9797726754b8258e171e2bfd19de4ccd7
-  data.tar.gz: 2886905cce5b47d0b2675cf4249659de21666b22
+  metadata.gz: c52bc907eb88537752612d4c288654e737048082
+  data.tar.gz: d4fb5d7f94bf24c126cd896b142eb17fc585fd93
 SHA512:
-  metadata.gz: 35784e53ee334916019dfc1df50a95a5f224767242418e8f8ff2247e778699fddf3287416dc467d738325d86aa4e09d662fe8aefd4be91aa6da50d33a09ecc5b
-  data.tar.gz: b396f106e73027fff0a3dd6a314899bb6996bc6ef82a93c127cae5f9af8f8483fa4e7ffa134c112c68b129d13d65ff2b28f90d139711132c1b84c51fecf1a936
+  metadata.gz: 1755fa1dd6c372745ed3f9234fe1918bfb01f3ef76bff45d5626ff1e0be9300f6ab21fadb096cac23950a27bd2461d2f5130b28c3367e4ed5a7eb5293db0cb1b
+  data.tar.gz: 45cd468ce1ead1093b22fc8fb50171c45fc5f51e7c18ff3bd55b4c703f764d35d1cc950535cc53491238bd47139c92182d89819489db923e6e9c20b14791a113

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    dbx (0.1.0)
+    dbx (0.1.1)
       activerecord (~> 4.0)
       activesupport (~> 4.0)
       thor

data/exe/dbx

@@ -49,6 +49,7 @@ class CLI < Thor
   option :force, type: :boolean, banner: 'remove diff_ table if it exists'
   option :using, type: :array, banner: 'JOIN USING the columns list here. Ex: id'
   option :exclude_columns, type: :array, banner: 'Exclude columns from comparison and selection'
+  option :no_a_b, type: :boolean, banner: 'Disable the *_a and *_b columns from *_diff table'
   def diff(table_a, table_b)
     handle_global_options
     DBX::Differ.diff(
@@ -56,7 +57,8 @@ class CLI < Thor
       table_b: table_b,
       using: options[:using],
       exclude_columns: options[:exclude_columns],
-      force: options[:force]
+      force: options[:force],
+      no_a_b: options[:no_a_b]
     )
   end
 
@@ -64,6 +66,7 @@ class CLI < Thor
   option :force, type: :boolean, banner: 'remove diff_ table if it exists'
   option :using, type: :array, banner: 'JOIN USING the columns list here. Ex: id'
   option :exclude_columns, type: :array, banner: 'Exclude columns from comparison and selection'
+  option :no_a_b, type: :boolean, banner: 'Disable the *_a and *_b columns from *_diff table'
   def import_diff(src_a, src_b)
     handle_global_options
     DBX::Differ.import_and_diff(
@@ -71,7 +74,8 @@ class CLI < Thor
       src_b: src_b,
       using: options[:using],
       exclude_columns: options[:exclude_columns],
-      force: options[:force]
+      force: options[:force],
+      no_a_b: options[:no_a_b]
     )
   end
 

data/lib/dbx/differ.rb

@@ -13,7 +13,7 @@ module DBX
     # @param [String] table B Should be newer than table A, but doesn't have to be.
     # @param [Array<String>] using is the join criteria between the 2 tables.
     # @param [Array<String>] exclude_columns are excluded from the diff comparison.
-    def diff(table_a:, table_b:, force: false, using: ['id'], exclude_columns: nil)
+    def diff(table_a:, table_b:, force: false, using: ['id'], exclude_columns: nil, no_a_b: false)
       table_diff = "diff_#{table_a}_#{table_b}"
       exclude_columns ||= []
       DBX.info("Creating diff table #{table_diff}")
@@ -23,7 +23,7 @@ module DBX
         CREATE TABLE #{table_diff} AS
         SELECT
           #{using.join(', ')},
-          #{select_columns(table_a, exclude_columns: using + exclude_columns)}
+          #{select_columns(table_a, exclude_columns: using + exclude_columns, no_a_b: no_a_b)}
         FROM #{table_a} AS a
         FULL OUTER JOIN #{table_b} b USING (#{using.join(',')})
         WHERE
@@ -36,10 +36,13 @@ module DBX
           (SELECT COUNT(*) FROM #{table_diff}) diffs
         SQL
       end
+      DBX.info("Creating diff stats: #{table_diff}")
+      create_diff_stats(table_diff, force: force)
+
       DBX.info("Diff complete. Results details in:   #{table_diff}")
     end
 
-    def import_and_diff(src_a:, src_b:, force: false, using: ['id'], exclude_columns: nil)
+    def import_and_diff(src_a:, src_b:, force: false, using: ['id'], exclude_columns: nil, no_a_b: false)
       DBX.info("Importing #{src_a}")
       table_a = DBX.import_table(src_a, force: force)
 
@@ -47,10 +50,39 @@ module DBX
       DBX.info("Importing #{src_b}")
       table_b = DBX.import_table(src_b, force: force)
 
-      diff(table_a: table_a, table_b: table_b, force: force, using: using, exclude_columns: exclude_columns)
+      diff(table_a: table_a, table_b: table_b, force: force, using: using, exclude_columns: exclude_columns, no_a_b: no_a_b)
     end
 
-    def select_columns(table, exclude_columns: nil)
+    def create_diff_stats(diff_table, force: false)
+      DBX.connection do |conn|
+        diff_stats = "#{diff_table}_stats"
+        conn.execute("DROP TABLE IF EXISTS #{diff_stats}") if force
+        selects = conn.columns(diff_table).map do |column|
+          header, type = column.name, column.type
+          col = header[/(.*)_diff$/, 1]
+          next unless col
+
+          if column.sql_type == 'interval'
+            %{SUM(#{header}) AS #{col}_sum}
+          else
+            case type
+            when :decimal, :integer, :date, :datetime
+              %{SUM(#{header}) AS #{col}_sum}
+            else
+              %{COUNT(#{header}) AS #{col}_count}
+            end
+          end
+        end.compact.join(",\n")
+        conn.execute(<<-SQL)
+          CREATE TABLE #{diff_stats} AS
+          SELECT
+            #{selects}
+          FROM #{diff_table}
+        SQL
+      end
+    end
+
+    def select_columns(table, exclude_columns: nil, no_a_b: false)
       exclude_columns ||= []
       DBX.connection do |conn|
         conn.columns(table).map do |column|
@@ -58,11 +90,13 @@ module DBX
           next if exclude_columns.include?(header)
           case type
           when :decimal, :integer
-            select_difference(header)
-          when :date, :datetime
-            select_difference_as_text(header)
+            select_difference(header, no_a_b: no_a_b)
+          when :date
+            select_difference_as_int(header, no_a_b: no_a_b)
+          when :datetime
+            select_difference_as_interval(header, no_a_b: no_a_b)
           else
-            select_boolean(header)
+            select_boolean(header, no_a_b: no_a_b)
           end
         end.compact.join(',')
       end
@@ -79,22 +113,39 @@ module DBX
       end.compact.join('OR')
     end
 
-    def select_difference(column)
+    def select_difference(column, no_a_b: false)
+      a = "a.#{column}"
+      b = "b.#{column}"
+      a_b = no_a_b ? '' : "#{a} AS #{column}_a, #{b} AS #{column}_b, "
+      %(#{a_b}(CASE WHEN #{a} = #{b} THEN NULL WHEN #{a} IS NULL THEN #{b} WHEN #{b} IS NULL THEN #{a} ELSE NULLIF(#{b} - #{a}, 0) END) AS #{column}_diff)
+    end
+
+    def select_difference_as_int(column, no_a_b: false)
+      a = "a.#{column}"
+      b = "b.#{column}"
+      a_b = no_a_b ? '' : "#{a} AS #{column}_a, #{b} AS #{column}_b, "
+      %(#{a_b}(CASE WHEN #{a} = #{b} THEN NULL::bigint WHEN #{a} IS NULL THEN 1 WHEN #{b} IS NULL THEN -1 ELSE (#{b} - #{a}) END) AS #{column}_diff)
+    end
+
+    def select_difference_as_interval(column, no_a_b: false)
       a = "a.#{column}"
       b = "b.#{column}"
-      %(#{a} AS #{column}_a, #{b} AS #{column}_b, (CASE WHEN #{a} IS NULL THEN #{b} WHEN #{b} IS NULL THEN #{a} ELSE NULLIF(#{b} - #{a}, 0) END) AS #{column}_diff)
+      a_b = no_a_b ? '' : "#{a} AS #{column}_a, #{b} AS #{column}_b, "
+      %(#{a_b}(CASE WHEN #{a} = #{b} THEN NULL::interval WHEN #{a} IS NULL THEN '1 day'::interval WHEN #{b} IS NULL THEN '-1 day'::interval ELSE (#{b} - #{a})::interval END) AS #{column}_diff)
     end
 
-    def select_difference_as_text(column)
+    def select_difference_as_text(column, no_a_b: false)
       a = "a.#{column}"
       b = "b.#{column}"
-      %(#{a} AS #{column}_a, #{b} AS #{column}_b, (CASE WHEN #{a} IS NULL THEN #{b}::text WHEN #{b} IS NULL THEN #{a}::text ELSE NULLIF((#{b} - #{a})::text, '0') END) AS #{column}_diff)
+      a_b = no_a_b ? '' : "#{a} AS #{column}_a, #{b} AS #{column}_b, "
+      %(#{a_b}(CASE WHEN #{a} = #{b} THEN NULL WHEN #{a} IS NULL THEN #{b}::text WHEN #{b} IS NULL THEN #{a}::text ELSE (#{b} - #{a})::text END) AS #{column}_diff)
     end
 
-    def select_boolean(column)
+    def select_boolean(column, no_a_b: false)
       a = "a.#{column}"
       b = "b.#{column}"
-      %(#{a} AS #{column}_a, #{b} AS #{column}_b, NULLIF(#{a} <> #{b}, FALSE) AS #{column}_diff)
+      a_b = no_a_b ? '' : "#{a} AS #{column}_a, #{b} AS #{column}_b, "
+      %(#{a_b}NULLIF(#{a} <> #{b}, FALSE) AS #{column}_diff)
     end
   end
 end

data/lib/dbx/version.rb

@@ -1,3 +1,3 @@
 module Dbx
-  VERSION = '0.1.1'.freeze
+  VERSION = '0.1.2'.freeze
 end

data/lib/dbx.rb

@@ -58,7 +58,7 @@ module DBX
   end
 
   def parse_table_name(src)
-    File.basename(src).sub(File.extname(src), '')
+    File.basename(src).sub(File.extname(src), '').downcase
   end
 
   def create_table(src, name: nil, force: false, sample_rows: config_sample_rows, csv_options: {})
@@ -77,12 +77,12 @@ module DBX
   def import_table(src, name: nil, force: false, sample_rows: config_sample_rows, csv_options: {})
     name ||= parse_table_name(src)
     connection do |conn|
-      create_table(src, force: force, sample_rows: sample_rows, csv_options: csv_options)
+      create_table(src, name: name, force: force, sample_rows: sample_rows, csv_options: csv_options)
       # TODO only postgres is support at the moment
       pg = conn.instance_variable_get(:@connection)
       types = column_types(src).keys.map{|m| %("#{m}")}
 
-      pg_stmt = "COPY #{name}(#{types.join(',')}) FROM STDIN CSV"
+      pg_stmt = %{COPY "#{name}"(#{types.join(',')}) FROM STDIN CSV}
       conn.logger.debug(pg_stmt)
       pg.copy_data(pg_stmt) do
         first = true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dbx
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Scott Pierce
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-22 00:00:00.000000000 Z
+date: 2018-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler