lx_data_validation 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5d57c15e760b4b13cd67d124d6e0c66c1f5d603c
+  data.tar.gz: 05207e5b3e1d45257a60e02cf4da04bf8b88eab0
+SHA512:
+  metadata.gz: 7183dc0078c905252f2a9179ff5a05637fbeb863e3de117c672f74bec84ce522574a55d30b7b32588dcb9f604ccc276df709ca53ffb6752d1af09bc8211a4901
+  data.tar.gz: 4473bce485ba3e274b347ead5e9a0e0f1c47f6f3e73c6058649ca400b07fe5f8470ac6cf523199519de291bf8e1b3debff8b41001dd5268fc2dd56a62823d821

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.idea/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.0.0
+before_install: gem install bundler -v 1.12.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in data_validation.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Roger Tong
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# DataValidation
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/data_validation`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'data_validation'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install lx_data_validation
+
+## Usage
+
+
+## Development
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/data_validation.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/compare

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'data_validation'
+
+DataValidation.compare_with_season ARGV[0]

data/config/hosts.yml

@@ -0,0 +1,8 @@
+int:
+  web_request_host: http://int-laxpower-phpadmin-01w.dev.activenetwork.com
+  api_request_host: https://awmobile-vip.qa.aw.dev.activenetwork.com
+
+prod:
+  web_request_host: http://php.admin.laxpower.com
+  api_request_host: https://awmobile.active.com
+

data/data_validation.gemspec

@@ -0,0 +1,39 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'data_validation/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "lx_data_validation"
+  spec.version       = DataValidation::VERSION
+  spec.authors       = ["Roger Tong"]
+  spec.email         = ["Roger.Tong@activenetwork.com", "tomtongt@live.com"]
+
+  spec.summary       = "Data validation for LaxPower"
+  spec.description   = "The validation is to compare values between web page values and mobile api responses"
+  spec.homepage      = "https://gitlab.dev.activenetwork.com/rtong/data-validation-for-laxpower"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RupwdbyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  # if spec.respond_to?(:metadata)
+  #   spec.metadata['allowed_push_host'] = "input later"
+  # else
+  #   raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  # end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "bin"
+  spec.executables   = ["compare"]
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '>= 2.1.4'
+
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+
+  spec.add_runtime_dependency 'nokogiri', '1.6.6.2'
+  spec.add_runtime_dependency 'httparty', '0.8.1'
+
+end

data/lib/data_validation/comparison/comparison_base.rb

@@ -0,0 +1,169 @@
+require 'data_validation/data_access'
+require 'logger'
+
+module DataValidation
+  module Comparison
+    class ComparisonBase
+      attr_reader :web_url, :logger_name, :category, :season, :division_id
+
+      def initialize(web_url)
+        @web_url = web_url
+      end
+
+      def compare
+        if api_path.nil?
+          raise 'Please define api_path'
+        end
+
+        if field_array.nil?
+          raise 'Please define field_array'
+        end
+
+        data_access = DataValidation::DataAccess.new(web_url, DataValidation.api_request_host + api_path, determine_request_body_and_logger_name)
+
+        puts 'requesting web page...'
+        web_data = data_access.get_main_table_data
+        puts 'requesting mobile api data: ' + DataValidation.api_request_host + api_path
+        api_data = data_access.get_response_from_mobile_api
+
+        if web_data.length != api_data['results']['results']['total']
+          logger.error "The record size between web page and api data is not the same: #{web_data.length} -- #{api_data['results']['results']['total']}"
+          return
+        end
+
+        web_data.each_with_index do |item, index|
+          team = api_data['results']['results']['teams'][index]
+
+          i = 0
+          field_array.each do |filed_name|
+
+            if filed_name.is_a? String
+              team_value = team[filed_name]
+            elsif filed_name.is_a? Array
+              team_value = team[filed_name[0]][filed_name[1]]
+            else
+              i += 1
+              next
+            end
+
+            if item.length < field_array.length && empty_field && empty_field == filed_name ## when the field is empty in the table
+              logger.error "team #{item[0]}'s #{filed_name} is not empty: #{team_value}" unless (team_value == 0 || team_value.nil?)
+              next
+            end
+
+
+            unless (team_value.is_a?(String) && team_value.include?(item[i])) ||
+              (team_value.is_a?(Float) && team_value == item[i].to_f) ||
+              (team_value.is_a?(Integer) && team_value == item[i].to_i)
+
+              logger.error "team #{item[0]}'s #{filed_name} is not correct: #{item[i]} -- #{team_value}"
+            end
+
+            i += 1
+          end
+
+        end
+      end
+
+      def compare_conf_standings(api_data, data_access)
+        puts 'Start to compare conf ranking tables'
+        conf_standing_data = data_access.conf_standing_data
+        conf_standing_data.each do |standing|
+          full_conf = ''
+          # puts standing.to_s
+          standing.each_with_index do |record, i|
+            if i == 0
+              state = record[0]
+              team_class = record[1]
+              conf = record[2]
+              full_conf = "#{state}, #{team_class}, #{conf}"
+
+              api_data = data_access.get_response_from_mobile_api_with_params(conf_rank_api_url, %Q|{"season": "#{season}", "conference": {"category": "#{category}", "conference": "#{conf}", "divisionId": #{division_id.to_i}, "state": "#{state}", "teamClass": "#{team_class}"}, "currPage": 1, "pageSize": 1000}|)
+
+              if api_data['results']['results']['total'] != standing.length - 1
+                logger.error "standing for #{full_conf} has incorrect length: #{standing.length - 1} : #{api_data['results']['results']['total']}"
+                break
+              end
+            else
+
+              team_data = api_data['results']['results']['teams'][i - 1]
+
+              record.each_with_index do |item, index|
+                if index == 0 && item.to_i != team_data['rank']
+                elsif index == 1 && item.to_f != team_data['powerRating']
+                  logger.error "team #{i} has incorrect powerRating in #{full_conf}: #{item.to_f} -- #{team_data['powerRating']}"
+                elsif index == 2 && item.to_i != team_data['conference']['wins']
+                  logger.error "team #{i} has incorrect conference wins in #{full_conf}: #{item.to_i} -- #{team_data['conference']['wins']}"
+                elsif index == 3 && item.to_i != team_data['conference']['losses']
+                  logger.error "team #{i} has incorrect conference losses in #{full_conf}: #{item.to_i} -- #{team_data['conference']['losses']}"
+                elsif index == 4 && item.to_i != team_data['conference']['ties']
+                  logger.error "team #{i} has incorrect conference ties in #{full_conf}: #{item.to_i} -- #{team_data['conference']['ties']}"
+                elsif index == 5 && item.to_f != team_data['winLossConf']
+                  logger.error "team #{i} has incorrect winLossConf in #{full_conf}: #{item.to_f} -- #{team_data['winLossConf']}"
+                elsif index == 6 && item.to_i != team_data['total']['wins']
+                  logger.error "team #{i} has incorrect total wins in #{full_conf}: #{item.to_i} -- #{team_data['total']['wins']}"
+                elsif index == 7 && item.to_i != team_data['total']['losses']
+                  logger.error "team #{i} has incorrect total losses in #{full_conf}: #{item.to_i} -- #{team_data['total']['losses']}"
+                elsif index == 8 && item.to_i != team_data['total']['ties']
+                  logger.error "team #{i} has incorrect total ties in #{full_conf}: #{item.to_i} -- #{team_data['total']['ties']}"
+                elsif index == 9 && item.to_f != team_data['winLossTotal']
+                  logger.error "team #{i} has incorrect winLossTotal in #{full_conf}: #{item.to_f} -- #{team_data['winLossTotal']}"
+                end
+              end
+            end
+
+          end
+        end
+      end
+
+      protected
+
+      def api_path
+
+      end
+
+      def field_array
+
+      end
+
+      def empty_field
+
+      end
+
+      private
+
+      def logger
+        @logger ||= begin
+                      tokens = logger_name.split('/')
+                      1.upto(tokens.size - 1) do |n|
+                        dir = tokens[0...n].join('/')
+                        Dir.mkdir(dir) unless Dir.exist?(dir)
+                      end
+                      Logger.new File.new(logger_name, 'w')
+        end
+      end
+
+      def determine_request_body_and_logger_name
+        @category = if web_url.match /bingrl/
+                     'GIRLS'
+                   elsif web_url.match /binboy/
+                     'BOYS'
+                   elsif web_url.match /binmen/
+                     'MEN'
+                   elsif web_url.match /binwom/
+                     'WOMEN'
+                   end
+
+        @season = "20#{web_url.match(/update(..)/)[1]}"
+
+        @division_id = web_url.match(/(\d+)\.php/)[1].to_i
+
+        operation = web_url.match(/\/([a-z]+)\d+.php/)[1]
+
+        @logger_name = "log/#{season}/#{category.downcase}/#{operation}/#{operation}_#{season}_#{category}_#{division_id}.log"
+
+        %Q|{"category":"#{category}","season":#{season},"divisionId":#{division_id},"currPage":1,"pageSize":1000}|
+      end
+    end
+  end
+end

data/lib/data_validation/comparison/comparison_factory.rb

@@ -0,0 +1,37 @@
+require 'data_validation/comparison/div_ranking_pr_comparison'
+require 'data_validation/comparison/ranking_table_mw_sos_comparison'
+require 'data_validation/comparison/ranking_table_mw_rpi_comparison'
+require 'data_validation/comparison/div_ranking_wl_comparison'
+require 'data_validation/comparison/ranking_table_mw_qw_comparison'
+require 'data_validation/comparison/ranking_table_mw_poll_comparison'
+require 'data_validation/comparison/ranking_table_mw_trend_comparison'
+require 'data_validation/comparison/ranking_table_mw_tsi_comparison'
+require 'data_validation/comparison/ranking_table_mw_pr_comparison'
+
+module DataValidation
+  module Comparison
+    class ComparisonFactory
+      def self.get_comparison(url)
+        if url.match /update\d\d\/bin(boy|grl)\/rating\d+.php/ # division Computer Rating page for boys/girls
+          return DivRankingPrComparison.new url
+        elsif url.match /update\d\d\/bin\S+\/rating\d+.php/ # division Computer Rating page for men and women
+          return RankingTableMwPrComparison.new url
+        elsif url.match /update\d\d\/bin\S+\/sos\d+.php/ # division Strength of Schedule page for boys/girls
+          return RankingTableMwSosComparison.new url
+        elsif url.match /update\d\d\/bin\S+\/rpi\d+.php/ # division RPI Rankings page for boys/girls
+          return RankingTableMwRpiComparison.new url
+        elsif url.match /update\d\d\/bin\S+\/wl\d+.php/ # division win-loss factor page for boys/girls
+          return DivRankingWlComparison.new url
+        elsif url.match /update\d\d\/bin\S+\/qwf\d+.php/ # division Quality Win Factor page for boys/girls
+          return RankingTableMwQwComparison.new url
+        elsif url.match /update\d\d\/bin\S+\/poll\d+.php/ # Poll page for collage division
+          return RankingTableMwPollComparison.new url
+        elsif url.match /update\d\d\/bin\S+\/trend\d+.php/ # Trend page for collage division
+          return RankingTableMwTrendComparison.new url
+        elsif url.match /update\d\d\/bin\S+\/tsi\d+.php/ # Trend page for collage division
+          return RankingTableMwTsiComparison.new url
+        end
+      end
+    end
+  end
+end

data/lib/data_validation/comparison/conference_ranking_table_comparison.rb

@@ -0,0 +1,72 @@
+require 'data_validation/comparison/comparison_base'
+
+module DataValidation
+  module Comparison
+    class RankingTableMwPollComparison < ComparisonBase
+
+      def compare
+
+        if api_path.nil?
+          raise 'Please define api_path'
+        end
+
+        data_access = DataValidation::DataAccess.new(web_url, DataValidation.api_request_host + api_path, determine_request_body_and_logger_name)
+
+        puts 'requesting web page...'
+        web_data = data_access.get_main_table_data
+        puts 'requesting mobile api data: ' + DataValidation.api_request_host + api_path
+        api_data = data_access.get_response_from_mobile_api
+
+        if web_data.length != api_data['results']['results']['total']
+          logger.error "The record size between web page and api data is not the same: #{web_data.length} -- #{api_data['results']['results']['total']}"
+          return
+        end
+
+        web_data.each_with_index do |item, index|
+          team = api_data['results']['results']['teams'][index]
+
+          field_array.each_with_index do |filed_name, index|
+            if filed_name.is_a? String
+              team_value = team[filed_name]
+            elsif filed_name.is_a? Array
+              team_value = team[filed_name[0]][filed_name[1]]
+            else
+              next
+            end
+
+            unless (team_value.is_a?(String) && team_value.include?(item[index])) ||
+              (team_value.is_a?(Float) && team_value == item[index].to_f) ||
+              (team_value.is_a?(Integer) && team_value == item[index].to_i)
+
+              logger.error "team #{item[0]}'s #{filed_name} is not correct: #{item[index]} -- #{team_value}"
+            end
+          end
+
+        end
+      end
+
+    end
+  end
+
+  def determine_request_body_and_logger_name
+    category = if web_url.match /bingrl/
+                 'GIRLS'
+               elsif web_url.match /binboy/
+                 'BOYS'
+               elsif web_url.match /binmen/
+                 'MEN'
+               elsif web_url.match /binwom/
+                 'WOMEN'
+               end
+
+    season = "20#{web_url.match(/update(..)/)[1]}"
+
+    division_id = web_url.match(/(\d+)\.php/)[1].to_i
+
+    operation = web_url.match(/\/([a-z]+)\d+.php/)[1]
+
+    @logger_name = "log/#{season}/#{category.downcase}/#{operation}/#{operation}_#{season}_#{category}_#{division_id}.log"
+
+    %Q|{"season":"#{season}", "conference": {"category": "#{category}",} "divisionId":#{division_id},"currPage":1,"pageSize":1000}|
+  end
+end

data/lib/data_validation/comparison/div_ranking_pr_comparison.rb

@@ -0,0 +1,78 @@
+require 'data_validation/comparison/comparison_base'
+
+module DataValidation
+  module Comparison
+    class DivRankingPrComparison < ComparisonBase
+
+      def api_path
+        '/rest/LaxPower/divisionRankingPr'
+      end
+
+      def field_array
+        ['rank', 'teamName', 'powerRating', 'regInPr', 'correction', 'championship', 'winLossTotal', %w|record wins|, %w|record losses|, %w|record ties|]
+      end
+
+      def empty_field
+        ''
+      end
+
+      def compare
+
+        data_access = DataValidation::DataAccess.new(web_url, DataValidation.api_request_host + api_path, determine_request_body_and_logger_name)
+
+        puts 'requesting web page...'
+        web_data = data_access.get_main_table_data
+        puts 'requesting mobile api data: ' + DataValidation.api_request_host + api_path
+        api_data = data_access.get_response_from_mobile_api
+
+        if web_data.length != api_data['results']['results']['total']
+          logger.error "The record size between web page and api data is not the same: #{web_data.length} -- #{api_data['results']['results']['total']}"
+          return
+        end
+
+        ## compare first table
+        web_data.each_with_index do |item, index|
+          team = api_data['results']['results']['teams'][index]
+
+          i = 0
+          field_array.each do |filed_name|
+
+            if filed_name.is_a? String
+              team_value = team[filed_name]
+            elsif filed_name.is_a? Array
+              team_value = team[filed_name[0]][filed_name[1]]
+            else
+              next
+            end
+
+            if item.length < field_array.length && empty_field && empty_field == filed_name ## when the field is possibly empty in the table
+              logger.error "team #{item[0]}'s #{filed_name} is not empty: #{team_value}" unless (team_value == 0 || team_value.nil?)
+              next
+            end
+
+
+            unless (team_value.is_a?(String) && team_value.include?(item[i])) ||
+              (team_value.is_a?(Float) && team_value == item[i].to_f) ||
+              (team_value.is_a?(Integer) && team_value == item[i].to_i)
+
+              logger.error "team #{item[0]}'s #{filed_name} is not correct: #{item[i]} -- #{team_value}"
+            end
+
+            i += 1
+          end
+
+        end
+
+        ## compare standing tables
+        compare_conf_standings(api_data, data_access)
+      end
+
+
+
+      def conf_rank_api_url
+        DataValidation.api_request_host + '/rest/LaxPower/conferenceRankingTable'
+      end
+
+    end
+  end
+end

data/lib/data_validation/comparison/div_ranking_wl_comparison.rb

@@ -0,0 +1,17 @@
+require 'data_validation/comparison/comparison_base'
+
+module DataValidation
+  module Comparison
+    class DivRankingWlComparison < ComparisonBase
+
+      def api_path
+        '/rest/LaxPower/divisionRankingWl'
+      end
+
+      def field_array
+        ['rank', 'teamName', 'rank', 'winLossTotal', 'rankDivision', 'winLossDivision', 'rankDate', 'winLossDate', 'rankRoad', 'winLossRoad', %w|record wins|, %w|record losses|, %w|record ties|]
+      end
+
+     end
+  end
+end

data/lib/data_validation/comparison/ranking_table_mw_poll_comparison.rb

@@ -0,0 +1,21 @@
+require 'data_validation/comparison/comparison_base'
+
+module DataValidation
+  module Comparison
+    class RankingTableMwPollComparison < ComparisonBase
+
+      def api_path
+        '/rest/LaxPower/rankingTableMWpoll'
+      end
+
+      def field_array
+        ['rank', 'teamName', %w|record wins|, %w|record losses|, %w|record ties|, 'pollPoint', 'pollFirst', 'lastWeek']
+      end
+
+      def empty_field
+        'pollFirst'
+      end
+
+    end
+  end
+end

data/lib/data_validation/comparison/ranking_table_mw_pr_comparison.rb

@@ -0,0 +1,74 @@
+require 'data_validation/comparison/comparison_base'
+
+module DataValidation
+  module Comparison
+    class RankingTableMwPrComparison < ComparisonBase
+
+      def api_path
+        '/rest/LaxPower/rankingTableMWpr'
+      end
+
+      def field_array
+        ['rank', 'teamName', 'powerRating', 'pollRank', 'sosRank', 'rpiRank', 'qwfRank', 'tsiRank', %w|record wins|, %w|record losses|, %w|record ties|]
+      end
+
+      def empty_field
+        'pollRank'
+      end
+
+      def compare
+        data_access = DataValidation::DataAccess.new(web_url, DataValidation.api_request_host + api_path, determine_request_body_and_logger_name)
+
+        puts 'requesting web page...'
+        web_data = data_access.get_main_table_data
+        puts 'requesting mobile api data: ' + DataValidation.api_request_host + api_path
+        api_data = data_access.get_response_from_mobile_api
+
+        if web_data.length != api_data['results']['results']['total']
+          logger.error "The record size between web page and api data is not the same: #{web_data.length} -- #{api_data['results']['results']['total']}"
+          return
+        end
+
+        ## compare first table
+        web_data.each_with_index do |item, index|
+          team = api_data['results']['results']['teams'][index]
+
+          i = 0
+          field_array.each do |filed_name|
+
+            if filed_name.is_a? String
+              team_value = team[filed_name]
+            elsif filed_name.is_a? Array
+              team_value = team[filed_name[0]][filed_name[1]]
+            else
+              next
+            end
+
+            if item.length < field_array.length && empty_field && empty_field == filed_name ## when the field is possibly empty in the table
+              logger.error "team #{item[0]}'s #{filed_name} is not empty: #{team_value}" unless (team_value == 0 || team_value.nil?)
+              next
+            end
+
+
+            unless (team_value.is_a?(String) && team_value.include?(item[i])) ||
+              (team_value.is_a?(Float) && team_value == item[i].to_f) ||
+              (team_value.is_a?(Integer) && team_value == item[i].to_i)
+
+              logger.error "team #{item[0]}'s #{filed_name} is not correct: #{item[i]} -- #{team_value}"
+            end
+
+            i += 1
+          end
+
+        end
+
+        compare_conf_standings api_data, data_access
+      end
+
+      def conf_rank_api_url
+        DataValidation.api_request_host + '/rest/LaxPower/conferenceRankingTable'
+      end
+
+    end
+  end
+end

data/lib/data_validation/comparison/ranking_table_mw_qw_comparison.rb

@@ -0,0 +1,17 @@
+require 'data_validation/comparison/comparison_base'
+
+module DataValidation
+  module Comparison
+    class RankingTableMwQwComparison < ComparisonBase
+
+      def api_path
+        '/rest/LaxPower/rankingTableMWqw'
+      end
+
+      def field_array
+        ['rank', 'teamName', 'qwPrRank', 'qualityWinsPr', nil, 'qualityWinsPoll', 'qwRpiRank', 'qualityWinsRpi', %w|record wins|, %w|record losses|, %w|record ties|]
+      end
+
+    end
+  end
+end

data/lib/data_validation/comparison/ranking_table_mw_rpi_comparison.rb

@@ -0,0 +1,21 @@
+require 'data_validation/comparison/comparison_base'
+
+module DataValidation
+  module Comparison
+    class RankingTableMwRpiComparison < ComparisonBase
+
+      def api_path
+        '/rest/LaxPower/rankingTableMWrpi'
+      end
+
+      def field_array
+        ['rank', 'teamName', nil, 'rpi', 'signWinsRank', 'signficantWins', 'signLossRank', 'signficantLosses',
+         %w|recordTotal wins|, %w|recordTotal losses|, %w|recordTotal ties|,
+         %w|recordRoad wins|, %w|recordRoad losses|, %w|recordRoad ties|,
+         %w|recordNeutral wins|, %w|recordNeutral losses|, %w|recordNeutral ties|,
+         %w|recordHome wins|, %w|recordHome losses|, %w|recordHome ties|]
+      end
+
+    end
+  end
+end

data/lib/data_validation/comparison/ranking_table_mw_sos_comparison.rb

@@ -0,0 +1,17 @@
+require 'data_validation/comparison/comparison_base'
+
+module DataValidation
+  module Comparison
+    class RankingTableMwSosComparison < ComparisonBase
+
+      def api_path
+        '/rest/LaxPower/rankingTableMWsos'
+      end
+
+      def field_array
+        ['rank', 'teamName', 'totalGames', nil,  'sosAverage', 'sosWeightedRank', 'sosWeighted', 'sosRpiRank', 'sosRpi', %w|record wins|, %w|record losses|, %w|record ties|]
+      end
+
+    end
+  end
+end

data/lib/data_validation/comparison/ranking_table_mw_trend_comparison.rb

@@ -0,0 +1,18 @@
+require 'data_validation/comparison/comparison_base'
+
+module DataValidation
+  module Comparison
+    class RankingTableMwTrendComparison < ComparisonBase
+
+      def api_path
+        '/rest/LaxPower/rankingTableMWtrend
+'
+      end
+
+      def field_array
+        ['rank', 'teamName', %w|record wins|, %w|record losses|, %w|record ties|, 'trend']
+      end
+
+    end
+  end
+end

data/lib/data_validation/comparison/ranking_table_mw_tsi_comparison.rb

@@ -0,0 +1,21 @@
+require 'data_validation/comparison/comparison_base'
+
+module DataValidation
+  module Comparison
+    class RankingTableMwTsiComparison < ComparisonBase
+
+      def api_path
+        '/rest/LaxPower/rankingTableMWtsi'
+      end
+
+      def field_array
+        ['rank', 'teamName', 'tsi', 'pollRank', 'prRank', 'rpiRank', 'sosRank', 'qwf', 'loss', 'trend', %w|record wins|, %w|record losses|, %w|record ties|]
+      end
+
+      def empty_field
+        'pollRank'
+      end
+
+    end
+  end
+end

data/lib/data_validation/data_access.rb

@@ -0,0 +1,140 @@
+require 'nokogiri'
+require 'httparty'
+require 'open-uri'
+
+module DataValidation
+  class DataAccess
+
+    attr_reader :html_url, :api_url, :api_body, :conf_standing_data
+
+    def initialize(html_url, api_url, api_body)
+      @html_url = html_url
+      @api_url = api_url
+      @api_body = api_body
+    end
+
+    def get_main_table_data
+      doc = Nokogiri::HTML(open(html_url))
+
+      if html_url.match /rating/
+        extract_conf_standing_data(doc)
+      end
+
+      team_names = doc.search('div.cs1 div.cs1 a').map { |node| node.text if /^[A-Z\d]+.PHP$/.match(node.attr('href')) }.compact ## extract team names
+
+      doc.search('div.cs1 div.cs1 a').remove ## then remove all links from the doc
+
+      index = 0
+      doc.search('div.cs1 div.cs1').text.lines.map do |line|
+        if /^\s*(?<rank>\d+)\s+(?<rest>.*)/ =~ line
+          data_array = ([rank] << team_names[index]) + rest.split(' ')
+          index += 1
+          data_array
+        end
+      end.compact
+    end
+
+    def extract_conf_standing_data(doc)
+      category = if html_url.match /bingrl/
+                   'GIRLS'
+                 elsif html_url.match /binboy/
+                   'BOYS'
+                 elsif html_url.match /binmen/
+                   'MEN'
+                 elsif html_url.match /binwom/
+                   'WOMEN'
+                 end
+
+      season = "20#{html_url.match(/update(..)/)[1]}"
+
+      div_id = html_url.match(/(\d+)\.php/)[1].to_i
+
+      conf_data = get_conference(season, category, div_id)['results']['results']['conferences']
+
+      # is_for_mw_pr = html_url.include?('binmen') || html_url.include?('binwomen')
+
+      ## only rating page has conf standing tables
+      doc.search('div.laxcss a').remove ## remove team names from conf standing tables
+      @conf_standing_data = []
+      conf_standing_record = []
+      lines = doc.search('div.laxcs1, div.laxcss').text.lines
+
+      conf_data_index = 0
+      lines.each_with_index do |line, index|
+        if line.match /^\s[A-Z]/
+          if conf_standing_record.length > 0 ## it's a new conf standing table
+            @conf_standing_data << conf_standing_record
+            conf_standing_record = []
+          end
+
+          conf = conf_data[conf_data_index]
+
+          if conf.nil?
+            puts 'No independent conf data'
+            @conf_standing_data << conf_standing_record
+            break
+          end
+
+          conf_standing_record << [conf['state'], conf['teamClass'], conf['conference']]
+
+          conf_data_index += 1
+        elsif /^\s*\d+\s+/.match line
+          conf_standing_record << line.split(' ')
+        end
+
+        if (index == lines.length - 1)
+          @conf_standing_data << conf_standing_record
+        end
+      end
+    end
+
+    def get_response_from_mobile_api
+      HTTParty.post(api_url, headers: {'Content-Type' => 'application/json'}, body: api_body)
+    end
+
+    def get_response_from_mobile_api_with_params(url, body)
+      HTTParty.post(url, headers: {'Content-Type' => 'application/json'}, body: body)
+    end
+
+    def get_conference(season, category, div_id)
+      puts "get conference data for #{season}, #{category}, #{div_id} from " + DataValidation.api_request_host + '/rest/LaxPower/getConferences'
+      HTTParty.post(DataValidation.api_request_host + '/rest/LaxPower/getConferences',
+                    headers: {'Content-Type' => 'application/json'},
+                    body: %Q|{"season": "#{season}","category": "#{category}", "divisionId": #{div_id}}|)
+    end
+
+    def self.get_high_school_div_stat_urls(season='17', category='boys')
+      is_for_hs = category == 'boys' || category == 'girls'
+      high_school_div_stat_main_url = if is_for_hs
+                                        "#{DataValidation.web_request_host}/common/hs_#{category}.php"
+                                      else
+                                        "#{DataValidation.web_request_host}/common/college_#{category}.php"
+                                      end
+      doc = Nokogiri::HTML(open(high_school_div_stat_main_url))
+      urls = doc.search('div#content_well a[href*=rating]').map do |item|
+        href = item.attr('href')
+        if href.match /rating\d\d/
+          href.sub!('..', DataValidation.web_request_host)
+          href.sub!('update17', "update#{season}") if season!= '17'
+          href
+        end
+      end.compact
+
+      final_urls = [] + urls
+      ## append other metrics e.g. sos, rpi
+      if is_for_hs
+        %w|sos rpi wl qwf|.each { |stat_name| final_urls += replace_with_stat_name(urls, stat_name) }
+      else
+        %w|tsi poll rpi sos qwf trend|.each { |stat_name| final_urls += replace_with_stat_name(urls, stat_name) }
+      end
+      puts "totally #{final_urls.length} urls for #{season}, #{category}"
+      final_urls
+    end
+
+    def self.replace_with_stat_name urls, stat_name
+      urls.map do |url|
+        url.sub 'rating', stat_name
+      end
+    end
+  end
+end

data/lib/data_validation/version.rb

@@ -0,0 +1,3 @@
+module DataValidation
+  VERSION = "0.0.1"
+end

data/lib/data_validation.rb

@@ -0,0 +1,49 @@
+require 'data_validation/comparison/comparison_factory'
+require 'data_validation/data_access'
+require 'yaml'
+
+module DataValidation
+  class << self
+
+    def web_request_host
+      @web_request_host ||= begin
+        YAML.load_file('config/hosts.yml')['prod']['web_request_host']
+      end
+    end
+
+    def api_request_host
+      @api_request_host ||= begin
+        YAML.load_file('config/hosts.yml')['prod']['api_request_host']
+      end
+    end
+
+    def configure_with_env(env)
+      yaml_config = YAML.load_file('config/hosts.yml')
+      hash_config = yaml_config[env]
+      @web_request_host = hash_config['web_request_host']
+      @api_request_host = hash_config['api_request_host']
+    end
+
+    def compare_with_season_and_category(season = '17', category = 'boys')
+      web_urls = DataAccess.get_high_school_div_stat_urls season, category
+      web_urls.each do |url|
+        compare_with_url url
+      end
+    end
+
+    def compare_with_season(season = '17')
+      %w|men women boys girls|.each { |category| compare_with_season_and_category season, category }
+    end
+
+    def compare_with_url(url)
+      puts "Start validation of web url: #{url}"
+
+      comparison = DataValidation::Comparison::ComparisonFactory.get_comparison(url)
+      comparison.compare
+
+      puts "Complete validation of web url: #{url}"
+      puts '=============================='
+    end
+  end
+
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: lx_data_validation
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Roger Tong
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-02-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.6.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.6.2
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+description: The validation is to compare values between web page values and mobile
+  api responses
+email:
+- Roger.Tong@activenetwork.com
+- tomtongt@live.com
+executables:
+- compare
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/compare
+- config/hosts.yml
+- data_validation.gemspec
+- lib/data_validation.rb
+- lib/data_validation/comparison/comparison_base.rb
+- lib/data_validation/comparison/comparison_factory.rb
+- lib/data_validation/comparison/conference_ranking_table_comparison.rb
+- lib/data_validation/comparison/div_ranking_pr_comparison.rb
+- lib/data_validation/comparison/div_ranking_wl_comparison.rb
+- lib/data_validation/comparison/ranking_table_mw_poll_comparison.rb
+- lib/data_validation/comparison/ranking_table_mw_pr_comparison.rb
+- lib/data_validation/comparison/ranking_table_mw_qw_comparison.rb
+- lib/data_validation/comparison/ranking_table_mw_rpi_comparison.rb
+- lib/data_validation/comparison/ranking_table_mw_sos_comparison.rb
+- lib/data_validation/comparison/ranking_table_mw_trend_comparison.rb
+- lib/data_validation/comparison/ranking_table_mw_tsi_comparison.rb
+- lib/data_validation/data_access.rb
+- lib/data_validation/version.rb
+homepage: https://gitlab.dev.activenetwork.com/rtong/data-validation-for-laxpower
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.1.4
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Data validation for LaxPower
+test_files: []
+has_rdoc: