lx_data_validation 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 89ff1b56686f16ee995ad1f43e95a6dfe41d8160
-  data.tar.gz: 4f2ae259e53910bdf95432f97033a58473908320
+  metadata.gz: 3e194bf49037ed8424cf3dcf33f3fabf5cb52760
+  data.tar.gz: 0647669d8cf15cb8f21e1c817c149ce4925f4225
 SHA512:
-  metadata.gz: 66022de0f498b75a19d6c582aa55ef6ea1093905483bf7a65ba7d8103d33bd86f52c71150ec88f80e9c3e9785b50207862f970262418100de436931745d00199
-  data.tar.gz: f3fddb6357cabcc1c664b6fb5105f040b8c4cd053d67dcdadab9eb3c0575c951e3bf32fb6b870740aafaca20591b6dba7d4ec27b7cbaf1fda7a2b8dd4517ec91
+  metadata.gz: b165e44eea570728fb66fd5156eaf4136857ca877d6515279e90543dc70135935b55782231bd37cdab2c67b2da82453ac576cc3d791302d2f0974717e39e88e2
+  data.tar.gz: 7671e8e28c54168f2a659b661f118a4900be3b4208049b7af391d94453ee776e7542eb7825a7d2157c91e874c5688fccd7e1a994c2ceb30154ada490ab8389ae

data/.gitignore

@@ -8,3 +8,5 @@
 /spec/reports/
 /tmp/
 .idea/
+log/
+*.gem

data/README.md

@@ -1,14 +1,17 @@
 # DataValidation
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/data_validation`. To experiment with that code, run `bin/console` for an interactive prompt.
+This gem validate the data between web page items and mobile api response.
 
+It will validate stat pages of PRO, men, woman, boys and girls.
+
+e.g. http://php.admin.laxpower.com/update17/binmen/rating01.php
 
 ## Installation
 
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'data_validation'
+gem 'lx_data_validation'
 ```
 
 And then execute:
@@ -20,7 +23,16 @@ Or install it yourself as:
     $ gem install lx_data_validation
 
 ## Usage
+  
+Validate the data of 2017 season (the default season)
+
+    $ compare
+
+Validate the data of 2016 season
+
+    $ compare 16
 
+It will generate a new folder named log under your current directory. All mismatched data will be recorded into the log files in the sub-directories
 
 ## Development
 

data/bin/compare

@@ -2,4 +2,8 @@
 
 require 'data_validation'
 
-DataValidation.compare_with_season ARGV[0]
+season = '17'
+if ARGV[0]
+  season = ARGV[0]
+end
+DataValidation.compare_with_season season

data/lib/data_validation.rb

@@ -21,6 +21,8 @@ module DataValidation
     end
 
     def compare_with_season(season = '17')
+      compare_with_url(web_request_host + '/mll')
+      compare_with_url(web_request_host + '/nll')
       %w|men women boys girls|.each { |category| compare_with_season_and_category season, category }
     end
 

data/lib/data_validation/comparison/comparison_base.rb

@@ -56,7 +56,7 @@ module DataValidation
               (team_value.is_a?(Float) && team_value == item[i].to_f) ||
               (team_value.is_a?(Integer) && team_value == item[i].to_i)
 
-              logger.error "team #{item[0]}'s #{filed_name} is not correct: #{item[i]} -- #{team_value}"
+              logger.error "team #{item[0]}'s #{filed_name}: #{item[i]} -- #{team_value}"
             end
 
             i += 1
@@ -91,23 +91,23 @@ module DataValidation
               record.each_with_index do |item, index|
                 if index == 0 && item.to_i != team_data['rank']
                 elsif index == 1 && item.to_f != team_data['powerRating']
-                  logger.error "team #{i} has incorrect powerRating in #{full_conf}: #{item.to_f} -- #{team_data['powerRating']}"
+                  logger.error "team #{i} powerRating in #{full_conf}: #{item.to_f} -- #{team_data['powerRating']}"
                 elsif index == 2 && item.to_i != team_data['conference']['wins']
-                  logger.error "team #{i} has incorrect conference wins in #{full_conf}: #{item.to_i} -- #{team_data['conference']['wins']}"
+                  logger.error "team #{i} conference wins in #{full_conf}: #{item.to_i} -- #{team_data['conference']['wins']}"
                 elsif index == 3 && item.to_i != team_data['conference']['losses']
-                  logger.error "team #{i} has incorrect conference losses in #{full_conf}: #{item.to_i} -- #{team_data['conference']['losses']}"
+                  logger.error "team #{i} conference losses in #{full_conf}: #{item.to_i} -- #{team_data['conference']['losses']}"
                 elsif index == 4 && item.to_i != team_data['conference']['ties']
-                  logger.error "team #{i} has incorrect conference ties in #{full_conf}: #{item.to_i} -- #{team_data['conference']['ties']}"
+                  logger.error "team #{i} conference ties in #{full_conf}: #{item.to_i} -- #{team_data['conference']['ties']}"
                 elsif index == 5 && item.to_f != team_data['winLossConf']
-                  logger.error "team #{i} has incorrect winLossConf in #{full_conf}: #{item.to_f} -- #{team_data['winLossConf']}"
+                  logger.error "team #{i} winLossConf in #{full_conf}: #{item.to_f} -- #{team_data['winLossConf']}"
                 elsif index == 6 && item.to_i != team_data['total']['wins']
-                  logger.error "team #{i} has incorrect total wins in #{full_conf}: #{item.to_i} -- #{team_data['total']['wins']}"
+                  logger.error "team #{i} total wins in #{full_conf}: #{item.to_i} -- #{team_data['total']['wins']}"
                 elsif index == 7 && item.to_i != team_data['total']['losses']
-                  logger.error "team #{i} has incorrect total losses in #{full_conf}: #{item.to_i} -- #{team_data['total']['losses']}"
+                  logger.error "team #{i} total losses in #{full_conf}: #{item.to_i} -- #{team_data['total']['losses']}"
                 elsif index == 8 && item.to_i != team_data['total']['ties']
-                  logger.error "team #{i} has incorrect total ties in #{full_conf}: #{item.to_i} -- #{team_data['total']['ties']}"
+                  logger.error "team #{i} total ties in #{full_conf}: #{item.to_i} -- #{team_data['total']['ties']}"
                 elsif index == 9 && item.to_f != team_data['winLossTotal']
-                  logger.error "team #{i} has incorrect winLossTotal in #{full_conf}: #{item.to_f} -- #{team_data['winLossTotal']}"
+                  logger.error "team #{i} winLossTotal in #{full_conf}: #{item.to_f} -- #{team_data['winLossTotal']}"
                 end
               end
             end
@@ -134,12 +134,14 @@ module DataValidation
 
       def logger
         @logger ||= begin
-                      tokens = logger_name.split('/')
-                      1.upto(tokens.size - 1) do |n|
-                        dir = tokens[0...n].join('/')
-                        Dir.mkdir(dir) unless Dir.exist?(dir)
-                      end
-                      Logger.new File.new(logger_name, 'w')
+          tokens = logger_name.split('/')
+          1.upto(tokens.size - 1) do |n|
+            dir = tokens[0...n].join('/')
+            Dir.mkdir(dir) unless Dir.exist?(dir)
+          end
+          logger = Logger.new(File.new(logger_name, 'w'))
+          logger.info(web_url)
+          logger
         end
       end
 

data/lib/data_validation/comparison/comparison_factory.rb

@@ -7,6 +7,7 @@ require 'data_validation/comparison/ranking_table_mw_poll_comparison'
 require 'data_validation/comparison/ranking_table_mw_trend_comparison'
 require 'data_validation/comparison/ranking_table_mw_tsi_comparison'
 require 'data_validation/comparison/ranking_table_mw_pr_comparison'
+require 'data_validation/comparison/ranking_table_pro_pr_comparison'
 
 module DataValidation
   module Comparison
@@ -30,6 +31,8 @@ module DataValidation
           return RankingTableMwTrendComparison.new url
         elsif url.match /update\d\d\/bin\S+\/tsi\d+.php/ # Trend page for collage division
           return RankingTableMwTsiComparison.new url
+        elsif url.match /\.com\/(mll|nll)/ # Pro ranking page
+          return RankingTableProPrComparison.new url
         end
       end
     end

data/lib/data_validation/comparison/div_ranking_pr_comparison.rb

@@ -55,7 +55,7 @@ module DataValidation
               (team_value.is_a?(Float) && team_value == item[i].to_f) ||
               (team_value.is_a?(Integer) && team_value == item[i].to_i)
 
-              logger.error "team #{item[0]}'s #{filed_name} is not correct: #{item[i]} -- #{team_value}"
+              logger.error "team #{item[0]}'s #{filed_name}: #{item[i]} -- #{team_value}"
             end
 
             i += 1

data/lib/data_validation/comparison/ranking_table_mw_pr_comparison.rb

@@ -54,7 +54,7 @@ module DataValidation
               (team_value.is_a?(Float) && team_value == item[i].to_f) ||
               (team_value.is_a?(Integer) && team_value == item[i].to_i)
 
-              logger.error "team #{item[0]}'s #{filed_name} is not correct: #{item[i]} -- #{team_value}"
+              logger.error "team #{item[0]}'s #{filed_name}: #{item[i]} -- #{team_value}"
             end
 
             i += 1

data/lib/data_validation/comparison/ranking_table_pro_pr_comparison.rb

@@ -0,0 +1,73 @@
+module DataValidation
+  module Comparison
+    class RankingTableProPrComparison
+
+      attr_reader :web_url
+
+      def initialize(web_url)
+        @web_url = web_url
+      end
+
+      def api_path
+        '/rest/LaxPower/rankingTableProPR'
+      end
+
+      def field_array
+        ['rank', 'teamName', 'powerRating', %w|record wins|, %w|record losses|]
+      end
+
+      def compare
+        div_id = web_url.match(/mll/) ? '12' : '11'
+        season = 2017
+        is_old_season = false
+
+        web_data = DataValidation::DataAccess.get_pro_pr_table web_url
+
+        api_data = (DataValidation::DataAccess.get_pr_for_pro season, 'PRO', div_id)
+        if api_data.nil?
+          season = 2016
+          api_data = DataValidation::DataAccess.get_pr_for_pro season, 'PRO', div_id
+          is_old_season = true
+        end
+
+        @logger_name = "log/#{season}/pro/pr/#{season}_pro_#{div_id}.log"
+        if is_old_season
+          puts 'This PR ranking is still for 2016'
+        end
+
+        if web_data.size != api_data.size
+          logger.error 'team size is incorrect'
+        else
+          web_data.each_with_index do |web_record, index|
+            api_record = api_data[index]
+            field_array.each_with_index do |field_name, i|
+              api_record_item = api_record[field_name]
+              web_record_item = web_record[i]
+
+              if (api_record_item.is_a?(String) && !api_record_item.include?(web_record_item.strip)) ||
+                ((api_record_item.is_a?(Float) && api_record_item != web_record_item.to_f)) ||
+                ((api_record_item.is_a?(Integer) && api_record_item != web_record_item.to_i))
+                logger.error "team #{index + 1}'s #{field_name}: #{api_record_item} -- #{web_record_item}"
+              end
+            end
+          end
+
+        end
+      end
+
+      def logger
+        @logger ||= begin
+          tokens = @logger_name.split('/')
+          1.upto(tokens.size - 1) do |n|
+            dir = tokens[0...n].join('/')
+            Dir.mkdir(dir) unless Dir.exist?(dir)
+          end
+          logger = Logger.new(File.new(@logger_name, 'w'))
+          logger.info(web_url)
+          logger
+        end
+      end
+
+    end
+  end
+end

data/lib/data_validation/data_access.rb

@@ -88,6 +88,8 @@ module DataValidation
       end
     end
 
+
+
     def get_response_from_mobile_api
       HTTParty.post(api_url, headers: {'Content-Type' => 'application/json'}, body: api_body)
     end
@@ -103,6 +105,32 @@ module DataValidation
                     body: %Q|{"season": "#{season}","category": "#{category}", "divisionId": #{div_id}}|)
     end
 
+    # for pro only
+    def self.get_pro_pr_table html_url
+      table = []
+      doc = Nokogiri::HTML(open(html_url))
+      doc.search('div#content_well td:nth-of-type(3) > div:nth-of-type(2) tr:not(:first-of-type)').map do |team|
+        team_data = []
+        team.children.each_with_index do |item, index|
+          value = item.text
+          if index == 3 # number of win and loss
+            value.split('-').each { |n| team_data << n }
+          else
+            team_data << value
+          end
+        end
+        table << team_data
+      end
+      table
+    end
+
+    def self.get_pr_for_pro(season, category, div_id)
+      puts "get PR ranking for #{season}, #{category}, #{div_id} from " + DataValidation.api_request_host + '/rest/LaxPower/rankingTableProPR'
+      HTTParty.post(DataValidation.api_request_host + '/rest/LaxPower/rankingTableProPR',
+                    headers: {'Content-Type' => 'application/json'},
+                    body: %Q|{"season": "#{season}","category": "#{category}", "divisionId": #{div_id}, "pageSize": 1000, "currPage": 1}|)['results']['results']['teams']
+    end
+
     def self.get_high_school_div_stat_urls(season='17', category='boys')
       is_for_hs = category == 'boys' || category == 'girls'
       high_school_div_stat_main_url = if is_for_hs

data/lib/data_validation/version.rb

@@ -1,3 +1,3 @@
 module DataValidation
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lx_data_validation
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Roger Tong
@@ -70,6 +70,7 @@ files:
 - lib/data_validation/comparison/ranking_table_mw_sos_comparison.rb
 - lib/data_validation/comparison/ranking_table_mw_trend_comparison.rb
 - lib/data_validation/comparison/ranking_table_mw_tsi_comparison.rb
+- lib/data_validation/comparison/ranking_table_pro_pr_comparison.rb
 - lib/data_validation/data_access.rb
 - lib/data_validation/version.rb
 homepage: https://gitlab.dev.activenetwork.com/rtong/data-validation-for-laxpower