data_verifier 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de8e7ec5961659d85a8c7fa92ae6e460ac62acddcf2699962f594cb6a3815014
-  data.tar.gz: 13d7a34878d79d2abb31b94112c266c8cae8f96ed40c2c8fe05d25cc003a73a3
+  metadata.gz: 0f8aaa56a85ac1b493591067a74994a348f336b20b541149951d1fb61abcb5bf
+  data.tar.gz: ee108239cb4359371ba74eb6b7ad0bf3a29ee353e65cc718578da49ab9e32da0
 SHA512:
-  metadata.gz: 7e50ce971d0c01a5ebc1661cd09a3a35e057bfa3472c8d2e154fbb51bac431e745ddcde3b6e21c50de484fbc11cc568963ab4e60354b16bbc687ca45294e7786
-  data.tar.gz: 7f346802c64d4e0682a8fabf8530ee83cb1a1c920ec0f0c32cae50ff554405d98d250a8c0b439c17b87e603e24be5a179916835e4b5acbb5fc79115688d2d432
+  metadata.gz: f96afe9be08894e1a399f157e0481d4de1ae6e0ef37b6f104b8e664d7c9344a37195fe70e930bfe37043360eff83afec3f18231f5017f9b67f7879088acd214f
+  data.tar.gz: 576a9b67c836258bde3b18de79333f4d0adc22d0deb123100af5cf77b663a17613faf818fcf59218bb4d45b5c5e99197dcb9bd54da758b0caf0e25c37d361516

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Ajit Singh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,92 @@
+# data_verifier
+
+There are times when we change the approach of modifying data and want to verify
+that the data modified by the new approach is same as the old approach.
+This gem is build for such verifications.
+
+## Motivation
+
+In my project, we replicate the data from other systems. There was a requirement where we changed the source of the data
+and code to replicate the data to our system. In testing phase we wanted to create a validation sheet,
+which can show the difference b/w the data in our system before and after the new source.
+
+## Installation
+Add this line to your application's Gemfile:
+```ruby
+gem 'data_verifier'
+```
+
+## Usage
+This gem has to be used in two phases:
+
+1. Build Phase: Use this phase to build the baseline of the data before changing the code/approach
+2. Verification Phase: Use this phase to verify the new data against the baseline data
+
+#### Config:
+
+```ruby
+require 'data_verifier'
+
+QUERIES = {
+    users_table: "select * from users where id=100",
+    phones_table: "select * from phones where user_id=100",
+    addresses_table: "select * from addresses where user_id=100",
+}
+
+config = DataVerifier::Config.new do |c|
+  c.db_adapter = :oracle
+  c.db_user = 'my_db_user'
+  c.db_password = 'my_pass'
+  c.db_host = 'localhost'
+  c.db_name = 'test_db'
+  c.db_port = '1521'
+  c.data_identifier = 'user_id_100'
+  c.queries = QUERIES
+end
+```
+
+#### Building Baseline Data:
+
+The below code will execute all the queries and store their result in json files.
+
+```ruby
+DataVerifier::BaselineBuilder.new(config).build
+```
+
+#### Verification:
+
+Below code will again execute the queries and will compare it with the baseline data.
+After comparision it will create an excel file of the result.
+
+```ruby
+DataVerifier::Validator.new(config).generate_validation_file
+```
+
+#### Result:
+
+<img src="https://github.com/ajitsing/ScreenShots/blob/master/data_verifier/data_verifier_result.png"/>
+
+## License
+```license
+MIT License
+
+Copyright (c) 2019 Ajit Singh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+```

data/lib/data_verifier/baseline_builder.rb

@@ -3,32 +3,44 @@ require 'sequel'
 
 module DataVerifier
   class BaselineBuilder
-    def initialize(config)
-      @config = config
-      @db = Sequel.connect(adapter: @config.db_adapter,
-                           user: @config.db_user,
-                           password: @config.db_password,
-                           host: @config.db_host,
-                           port: @config.db_port,
-                           database: @config.db_name,
-                           max_connections: @config.db_max_connections)
+    def initialize
+      @configs = []
+    end
+
+    def with(config)
+      @configs << config
+      self
     end
 
     def build
-      @config.queries.each do |tag, query|
-        puts "Executing => #{query}\n"
-        data = @db.fetch(query)
+      @configs.each do |config|
+        db = create_db_connection(config)
+
+        config.queries.each do |tag, query|
+          puts "Executing => #{query}\n"
+          data = db.fetch(query)
 
-        File.open(data_file_name(tag), 'w') do |file|
-          file.write JSON.dump(data.all)
+          File.open(data_file_name(config, tag), 'w') do |file|
+            file.write JSON.dump(data.all)
+          end
         end
       end
     end
 
     private
-    def data_file_name(tag)
-      identifier = @config.data_identifier.nil? ? '' : "#{@config.data_identifier}_"
+    def data_file_name(config, tag)
+      identifier = config.data_identifier.nil? ? '' : "#{config.data_identifier}_"
       "#{identifier}#{tag}.json"
     end
+
+    def create_db_connection(config)
+      Sequel.connect(adapter: config.db_adapter,
+                     user: config.db_user,
+                     password: config.db_password,
+                     host: config.db_host,
+                     port: config.db_port,
+                     database: config.db_name,
+                     max_connections: config.db_max_connections)
+    end
   end
 end

data/lib/data_verifier/validator.rb

@@ -4,71 +4,99 @@ require 'axlsx'
 
 module DataVerifier
   class Validator
-    def initialize(config)
-      @config = config
-      @db = Sequel.connect(adapter: @config.db_adapter,
-                           user: @config.db_user,
-                           password: @config.db_password,
-                           host: @config.db_host,
-                           port: @config.db_port,
-                           database: @config.db_name,
-                           max_connections: @config.db_max_connections)
+    def initialize(report_name = 'data_verifier')
+      @report_name = report_name
+      @excel = Axlsx::Package.new
     end
 
-    def generate_validation_file
-      excel = Axlsx::Package.new
+    def validate_using(config)
+      db = create_db_connection(config)
 
-      @config.queries.each do |tag, query|
+      config.queries.each do |tag, query|
         puts "Executing => #{query}\n"
-        data = @db.fetch(query)
-        update_excel(excel, tag, data)
+
+        new_data = db.fetch(query)
+        old_data = JSON.parse(File.read(data_file_name(config, tag)))
+
+        update_excel(tag, new_data, old_data)
       end
 
-      excel.serialize("#{identifier}data_verifier_result.xlsx")
+      self
     end
 
-    private
-    def data_file_name(tag)
-      "#{identifier}#{tag}.json"
+    def generate_report
+      @excel.serialize("#{@report_name}.xlsx")
     end
 
-    def identifier
-      @config.data_identifier.nil? ? '' : "#{@config.data_identifier}_"
+    private
+    def create_db_connection(config)
+      Sequel.connect(adapter: config.db_adapter,
+                     user: config.db_user,
+                     password: config.db_password,
+                     host: config.db_host,
+                     port: config.db_port,
+                     database: config.db_name,
+                     max_connections: config.db_max_connections)
     end
 
-    def update_excel(excel, sheet_name, data)
-      data_before_replication = JSON.parse(File.read(data_file_name(sheet_name)))
-
-      excel.workbook.add_worksheet(name: sheet_name.to_s) do |s|
-        red_color = Axlsx::Color.new
-        red_color.rgb = 'C40101'
+    def update_excel(sheet_name, new_data, old_data)
+      header_color = "43B275"
+      header = %w(Field Before After)
 
-        white_color = Axlsx::Color.new
-        white_color.rgb = 'FFFFFF'
+      header_style_opts = {bg_color: header_color, b: true, sz: 16, alignment: {horizontal: :center}, color: white_color}
+      data_style_opts   = {sz: 13, alignment: {horizontal: :left}}
+      error_style_opts  = {sz: 12, alignment: {horizontal: :left}, color: red_color, b: true}
 
-        header_style = s.styles.add_style bg_color: "43B275", b: true, sz: 16, alignment: {horizontal: :center}, color: white_color
-        data_style = s.styles.add_style sz: 13, alignment: {:horizontal => :left}
-        error_data_style = s.styles.add_style sz: 12, alignment: {:horizontal => :left}, color: red_color, b: true
+      @excel.workbook.add_worksheet(name: sheet_name.to_s) do |s|
+        header_style      = s.styles.add_style header_style_opts
+        data_style        = s.styles.add_style data_style_opts
+        error_data_style  = s.styles.add_style error_style_opts
 
-        s.add_row ["Field", "Before", "After"], style: header_style
+        s.add_row header, style: header_style
         s.row_style(0, header_style)
 
-        data.each_with_index do |db_row, index|
-          stored_result = data_before_replication[index]
+        new_data.each_with_index do |db_row, index|
+          baseline_row = old_data[index]
 
-          db_row.each do |column, value|
-            value_not_eql = stored_result[column.to_s].to_s != value.to_s
-            data = [column.to_s.upcase, stored_result[column.to_s].to_s, value.to_s]
+          db_row.each do |db_field, new_value|
+            field = db_field.to_s.upcase
+            old_value = baseline_row.nil? ? "" : baseline_row[db_field.to_s].to_s
+            cell_style = old_value == new_value.to_s ? data_style : error_data_style
 
-            conditional_formatting = value_not_eql ? error_data_style : data_style
-            s.add_row data, style: [conditional_formatting, data_style, data_style]
+            s.add_row [field, old_value, new_value.to_s], style: [cell_style, cell_style, cell_style]
           end
 
-          s.add_row ["", "", ""]
-          s.add_row ["", "", ""], style: [header_style, header_style, header_style]
-          s.add_row ["", "", ""]
+          s.add_row empty_row
+          s.add_row empty_row, style: [header_style, header_style, header_style]
+          s.add_row empty_row
         end
       end
     end
+
+    def data_file_name(config, tag)
+      "#{identifier(config)}#{tag}.json"
+    end
+
+    def identifier(config)
+      config.data_identifier.nil? ? '' : "#{config.data_identifier}_"
+    end
+
+    def empty_row
+      ["", "", ""]
+    end
+
+    def red_color
+      red_color = Axlsx::Color.new
+      red_color.rgb = 'C40101'
+
+      red_color
+    end
+
+    def white_color
+      white_color = Axlsx::Color.new
+      white_color.rgb = 'FFFFFF'
+
+      white_color
+    end
   end
 end

data/lib/data_verifier/version.rb

@@ -1,3 +1,3 @@
 module DataVerifier
-  VERSION = '0.0.1'
+  VERSION = '0.1.0'
 end

data/spec/lib/data_verifier/baseline_builder_spec.rb

@@ -0,0 +1,50 @@
+require 'data_verifier'
+
+describe DataVerifier::BaselineBuilder do
+  describe '#build' do
+    let(:db) {double(:db)}
+
+    before(:each) do
+      expect(Sequel).to receive(:connect).and_return(db)
+    end
+
+    it 'should execute queries and store result in json files' do
+      queries = {
+          users_table: 'select * from users where id=100',
+          addresses_table: 'select * from addresses where user_id=100'
+      }
+
+      config = DataVerifier::Config.new do |c|
+        c.data_identifier = '100'
+        c.queries = queries
+      end
+
+      builder = DataVerifier::BaselineBuilder.new
+
+      expect(db).to receive(:fetch).with(queries[:users_table]).and_return(double(:users))
+      expect(db).to receive(:fetch).with(queries[:addresses_table]).and_return(double(:addresses))
+
+      expect(File).to receive(:open).with('100_users_table.json', 'w')
+      expect(File).to receive(:open).with('100_addresses_table.json', 'w')
+
+      builder.with(config).build
+    end
+
+    it 'should not add prefix to json files when identifier is not given' do
+      queries = {
+          users_table: 'select * from users where id=100'
+      }
+
+      config = DataVerifier::Config.new do |c|
+        c.queries = queries
+      end
+
+      builder = DataVerifier::BaselineBuilder.new
+
+      expect(db).to receive(:fetch).with(queries[:users_table]).and_return(double(:users))
+      expect(File).to receive(:open).with('users_table.json', 'w')
+
+      builder.with(config).build
+    end
+  end
+end

data/spec/lib/data_verifier/validator_spec.rb

@@ -0,0 +1,44 @@
+require 'data_verifier'
+
+describe DataVerifier::Validator do
+  describe '#generate_validation_file' do
+    let(:db) {double(:db)}
+
+    before(:each) do
+      expect(Sequel).to receive(:connect).and_return(db)
+    end
+
+    it 'should create result excel file' do
+      queries = {
+          users_table: 'select * from users where id=100',
+          addresses_table: 'select * from addresses where user_id=100'
+      }
+
+      config = DataVerifier::Config.new do |c|
+        c.data_identifier = '100'
+        c.queries = queries
+      end
+
+      excel = double(:excel)
+      expect(Axlsx::Package).to receive(:new).and_return(excel)
+
+      validator = DataVerifier::Validator.new('test_result')
+
+      expect(db).to receive(:fetch).with(queries[:users_table]).and_return(double(:users))
+      expect(db).to receive(:fetch).with(queries[:addresses_table]).and_return(double(:addresses))
+
+      expect(File).to receive(:read).with('100_users_table.json').and_return("{}")
+      expect(File).to receive(:read).with('100_addresses_table.json').and_return("{}")
+
+      work_book = double(:workbook)
+      expect(excel).to receive(:workbook).and_return(work_book).twice
+
+      expect(work_book).to receive(:add_worksheet).with(name: 'users_table')
+      expect(work_book).to receive(:add_worksheet).with(name: 'addresses_table')
+
+      expect(excel).to receive(:serialize).with('test_result.xlsx')
+
+      validator.validate_using(config).generate_report
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_verifier
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Ajit Singh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-16 00:00:00.000000000 Z
+date: 2019-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: axlsx
@@ -75,12 +75,16 @@ files:
 - ".gitignore"
 - Gemfile
 - Gemfile.lock
+- LICENSE
+- README.md
 - data_verifier.gemspec
 - lib/data_verifier.rb
 - lib/data_verifier/baseline_builder.rb
 - lib/data_verifier/config.rb
 - lib/data_verifier/validator.rb
 - lib/data_verifier/version.rb
+- spec/lib/data_verifier/baseline_builder_spec.rb
+- spec/lib/data_verifier/validator_spec.rb
 homepage: https://github.com/ajitsing/data_verifier
 licenses:
 - MIT
@@ -105,4 +109,6 @@ rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Verify data after performing series of operations
-test_files: []
+test_files:
+- spec/lib/data_verifier/baseline_builder_spec.rb
+- spec/lib/data_verifier/validator_spec.rb