ae_easy-qa 0.0.20 → 0.0.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c72d5444ae19255110461dc0c1ec0ce4651511f82310098b7824aff7e59b2ead
-  data.tar.gz: 7bec17a2a260cae2aadf815fa6ecb36bed838aa2170899848fea1b23096ab671
+  metadata.gz: b37b9eefd295fd1816c02b906a56eeb0ed2d0a8108c3e93992e4953f36786387
+  data.tar.gz: f8e58457cf18340e6879d088e94031cdc869c05866324474a68465b9aa9318f7
 SHA512:
-  metadata.gz: 81a5b662767655e43daa1d20f7d3f9734b2015ff2e88eb1ce04b7e075ed46300f31d6f1c3ead94461ce276c126d3259b271d01217c312158dfa6814804b171a0
-  data.tar.gz: 0c33dd4a14929970df2d5d7c40121dc3877eddf781c93e1406c66625d6ae4c2df2b3dedfbaec1d7cff7dcb0954473ad413c0e10d4cd1a2304f14993437756164
+  metadata.gz: e2ed8e4b43bef024e4543255123a3df6423ffd6a3c3a72d4bb0b440bcc6eb92c86591ddcf9dbccf0ebcc53b42756c0cf6b191ac4ffa9c2f130a3964d47dbda4d
+  data.tar.gz: 284d9e40adb10130d153a6758714af314e228352e805d6882dbf5acb695cd59ea41692db999305d3c294b93ee3e39f41832b63a9f2c7a05ee1567fb83747d398

data/lib/ae_easy/qa.rb

@@ -14,10 +14,11 @@ module AeEasy
   module Qa
     class Validator
       attr_accessor :config
-      attr_reader :data, :errors
+      attr_reader :data, :options, :errors
 
       def initialize(data=nil, options={})
         load_config
+        @options = options
         @data = data
       end
 
@@ -26,7 +27,7 @@ module AeEasy
       end
 
       def validate_external(outputs, collection_name)
-        ValidateExternal.new(data, config, outputs, collection_name).run
+        ValidateExternal.new(data, config, outputs, collection_name, options).run
       end
 
       private

data/lib/ae_easy/qa/save_output.rb

@@ -2,17 +2,20 @@ module AeEasy
   module Qa
     class SaveOutput
       attr_reader :total_items, :rules, :errors, :collection_name,
-                  :outputs, :summary, :error_totals, :fields_to_ignore
+                  :outputs, :summary, :error_totals, :fields_to_ignore,
+                  :specific_validations_to_ignore, :options
 
-      def initialize(total_items, rules, errors, collection_name, outputs)
+      def initialize(total_items, rules, errors, collection_name, outputs, options)
         @total_items = total_items
         @rules = rules
         @errors = errors
         @collection_name = collection_name
         @outputs = outputs
+        @options = options
         @summary = Hash.new(0)
         @error_totals = {}
         @fields_to_ignore = []
+        @specific_validations_to_ignore = []
       end
 
       def run
@@ -26,29 +29,66 @@ module AeEasy
       private
 
       def gather_threshold_totals
-        rules.each{|field_to_validate, options|
-          if options['threshold']
+        rules.each{|field_to_validate, field_options|
+          field_threshold = return_threshold(field_to_validate, field_options)
+          if field_threshold
             error_total = errors[:errored_items].inject(0){|total, errored_item|
               failed_fields = errored_item[:failures].keys.collect{|failure_key|
                 extract_field(failure_key)
               }.uniq
-              total + 1 if failed_fields.include?(field_to_validate)
+              total += 1 if failed_fields.include?(field_to_validate)
+              total
             }
             error_totals[field_to_validate] = error_total
+          else
+            field_options.each do |validation|
+              potential_failure_name = "#{field_to_validate}_#{validation[0]}_fail"
+              if options['thresholds'][potential_failure_name]
+                error_total = errors[:errored_items].inject(0){|total, errored_item|
+                  failed_validations = errored_item[:failures].keys.collect{|failure_key|
+                    "#{failure_key}_fail"
+                  }
+                  total += 1 if failed_validations.include?(potential_failure_name)
+                  total
+                }
+                error_totals[potential_failure_name] = error_total
+              end
+            end
           end
         }
       end
 
       def gather_fields_to_ignore
-        rules.each{|field_to_validate, options|
-          if options['threshold']
+        rules.each{|field_to_validate, field_options|
+          field_threshold = return_threshold(field_to_validate, field_options)
+          if field_threshold
             total_errors = error_totals[field_to_validate]
-            success_ratio = (total_items - total_errors).to_f / total_items
-            fields_to_ignore.push(field_to_validate) if success_ratio > options['threshold']
+            if total_errors
+              success_ratio = (total_items - total_errors).to_f / total_items
+              fields_to_ignore.push(field_to_validate) if success_ratio > field_threshold
+            end
+          else
+            field_options.each do |validation|
+              potential_failure_name = "#{field_to_validate}_#{validation[0]}_fail"
+              total_errors = error_totals[potential_failure_name]
+              if total_errors
+                specific_validation_threshold = options['thresholds'][potential_failure_name].to_f
+                success_ratio = (total_items - total_errors).to_f / total_items
+                specific_validations_to_ignore.push(potential_failure_name) if success_ratio > specific_validation_threshold
+              end
+            end
           end
         }
       end
 
+      def return_threshold(field_to_validate, field_options)
+        if options['thresholds']
+          options['thresholds'][field_to_validate]
+        else
+          field_options['threshold'] || options['threshold']
+        end
+      end
+
       def save_group_errors
         errors.each{|error_name, output_hash|
           if error_name != :errored_items
@@ -60,7 +100,7 @@ module AeEasy
 
       def save_errors
         errors[:errored_items].each do |errored_item|
-          remove_threshold_failures(errored_item) if fields_to_ignore.any?
+          remove_threshold_failures(errored_item) if fields_to_ignore.any? || specific_validations_to_ignore.any?
           errored_item[:failures].each do |error_key, value|
             key = "#{error_key.to_s}_#{value.to_s}"
             summary[key] += 1
@@ -72,8 +112,9 @@ module AeEasy
 
       def remove_threshold_failures(errored_item)
         errored_item[:failures].delete_if{|error_name, fail|
+          specific_validation_name = "#{error_name}_fail"
           field_name = extract_field(error_name)
-          fields_to_ignore.include?(field_name)
+          fields_to_ignore.include?(field_name) || specific_validations_to_ignore.include?(specific_validation_name)
         }
       end
 

data/lib/ae_easy/qa/validate_external.rb

@@ -1,13 +1,14 @@
 module AeEasy
   module Qa
     class ValidateExternal
-      attr_reader :data, :errors, :rules, :outputs, :collection_name
+      attr_reader :data, :errors, :rules, :outputs, :collection_name, :options
 
-      def initialize(data, config, outputs, collection_name)
+      def initialize(data, config, outputs, collection_name, options)
         @data = data
         @rules = config['individual_validations'] if config
         @outputs = outputs
         @collection_name = collection_name
+        @options = options
         @errors = { errored_items: [] }
       end
 
@@ -17,7 +18,7 @@ module AeEasy
             ValidateGroups.new(data, nil, collection_name, errors).run
             ValidateRules.new(data, errors, rules).run if rules
           end
-          SaveOutput.new(data.count, rules, errors, collection_name, outputs).run
+          SaveOutput.new(data.count, rules, errors, collection_name, outputs, options).run
           return errors
         rescue StandardError => e
           puts "An error has occurred: #{e}"

data/lib/ae_easy/qa/validate_internal.rb

@@ -88,7 +88,7 @@ module AeEasy
           ValidateGroups.new(data, scraper_name, collection_name, errors).run
           ValidateRules.new(data, errors, rules).run if rules
         end
-        SaveOutput.new(data.count, rules, errors, outputs_collection_name, outputs).run
+        SaveOutput.new(data.count, rules, errors, outputs_collection_name, outputs, {}).run
       end
 
       private

data/lib/ae_easy/qa/version.rb

@@ -1,5 +1,5 @@
 module AeEasy
   module Qa
-    VERSION = "0.0.20"
+    VERSION = "0.0.21"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ae_easy-qa
 version: !ruby/object:Gem::Version
-  version: 0.0.20
+  version: 0.0.21
 platform: ruby
 authors:
 - David Lynam
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-07-25 00:00:00.000000000 Z
+date: 2019-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: answersengine