ae_easy-qa 0.0.32 → 0.0.34

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4950f51c210ef1ba1a6efca63cf44d13e422b7063e49be2fb4ddea58bef304a5
-  data.tar.gz: 4204239f672a33aa1c280dd420824e033066abdc4897e3269037fb97701f5901
+  metadata.gz: 6d82799c2aade7ae53677871d20deb99b3bb3319ba380fcc88acaa4ac18bdb81
+  data.tar.gz: 6fb00dca45293956ff594875df6cba899dbde0b28e0065c4e3f8debbbe659497
 SHA512:
-  metadata.gz: 93ecb76f504b23d8213a73d9cd0d1c1cb8a217c36423b6d375ddee202f6bbb42f11eb808cf27ecd4f00f270afed67ef7e69fdf85f40712064b0ae62b15c77b3c
-  data.tar.gz: 42860dbd7c1406045eb6d64aefbfc21fdadb8ade8f7d09f574c0086d0607fc54cc0480303d5a4222aeb06d22ac582a89a863ad8ba83a771714b1e233f9036643
+  metadata.gz: d996f0fc651d37e19a9fa49ac199ac1cd37bbebf8a50b6d649bbbf167e10ab5fdbfdb6bafdda29901800367703309aec9117e373fd92b59e2690434e562aa92c
+  data.tar.gz: 8c44426a128b3e2c44596bf48ed6d10c2e17f701070fc2f153c1bf771b277f664416e908b465302574f572c1ddc6b2c04093238c65647c085cef5864f1200647

data/lib/ae_easy/qa/save_output.rb

@@ -28,6 +28,7 @@ module AeEasy
 
       private
 
+      #thresholds are a setting where you can ignore errors if they are under a specific error rate
       def gather_threshold_totals
         rules.each{|field_to_validate, field_options|
           field_threshold = return_threshold(field_to_validate, field_options)

data/lib/ae_easy/qa/validate_internal.rb

@@ -1,18 +1,19 @@
 module AeEasy
   module Qa
     class ValidateInternal
-      attr_reader :scraper_name, :collections, :rules, :outputs
+      attr_reader :scraper_name, :collections, :rules, :outputs, :data
 
       def initialize(vars, config, outputs)
         @scraper_name = vars['scraper_name']
         @collections = vars['collections']
         @rules = config['individual_validations']
         @outputs = outputs
+        @data = vars['data']
       end
 
       def run
         begin
-          ValidateScraper.new(scraper_name, collections, rules, outputs, thresholds).run
+          ValidateScraper.new(scraper_name, collections, rules, outputs, thresholds, data).run
         rescue StandardError => e
           puts "An error has occurred: #{e}"
           return nil
@@ -21,6 +22,7 @@ module AeEasy
 
       private
 
+      #thresholds are a setting where you can suppress errors if they are under a specific error rate
       def thresholds
         @thresholds ||= begin
                           file_path = File.expand_path('thresholds.yaml', Dir.pwd)
@@ -34,14 +36,15 @@ module AeEasy
     end
 
     class ValidateScraper
-      attr_reader :scraper_name, :collections, :rules, :outputs, :options
+      attr_reader :scraper_name, :collections, :rules, :outputs, :options, :data
 
-      def initialize(scraper_name, collections, rules, outputs, thresholds)
+      def initialize(scraper_name, collections, rules, outputs, thresholds, data)
         @scraper_name = scraper_name
         @collections = collections
         @rules = rules
         @outputs = outputs
         @options = {}
+        @data = data
         options['thresholds'] = thresholds[scraper_name] if thresholds && thresholds[scraper_name]
       end
 
@@ -67,14 +70,14 @@ module AeEasy
       end
 
       def status_ok?
-        !collection_response.parsed_response.nil? && collection_response.code == 200
+        !old_collection_response.parsed_response.nil? && old_collection_response.code == 200
       end
 
       def validate_collections
         collections.each do |collection_name|
           collection = collection_counts.find{|collection_hash| collection_hash['collection'] == collection_name }
           if collection
-            ValidateCollection.new(scraper_name, collection_name, collection['outputs'], rules, outputs, options).run
+            ValidateCollection.new(scraper_name, collection_name, collection['outputs'], rules, outputs, options.merge({'data' => @data})).run
           else
             puts "collection #{collection_name} is missing"
           end
@@ -82,19 +85,29 @@ module AeEasy
       end
 
       def output_response
-        if collection_response.parsed_response.nil?
+        if old_collection_response.parsed_response.nil?
           puts "collection response is null"
         else
-          puts collection_response.parsed_response['message']
+          puts old_collection_response.parsed_response['message']
         end
       end
 
       def collection_counts
-        @collection_counts ||= collection_response.parsed_response
+        @collection_counts ||= collection_response
       end
 
       def collection_response
-        @collection_response || AnswersEngine::Client::ScraperJobOutput.new.collections(scraper_name)
+        json = JSON.parse(AnswersEngine::Client::ScraperJobOutput.new.collections(scraper_name).body)
+        if json['error'] == ""
+          @collection_response || json["data"]
+        else 
+          puts "There was an error: #{JSON.pretty_generate(json['error'])}"
+          @collection_response
+        end
+      end
+
+      def old_collection_response
+        @old_collection_response || AnswersEngine::Client::ScraperJobOutput.new.collections(scraper_name)
       end
     end
 
@@ -107,6 +120,7 @@ module AeEasy
         @total_records = total_records
         @rules = rules
         @outputs = outputs
+        @data = options['data']
         @options = options
         @errors = { errored_items: [] }
       end
@@ -147,6 +161,7 @@ module AeEasy
                     data = []
                     page = 1
                     while data.count < total_records
+                      # if there is an issue maybe due to this method response due to new stream data response AnswersEngine::Client::JobOutput.new(per_page:500, page: page).all
                       records = AnswersEngine::Client::JobOutput.new(per_page:500, page: page).all(most_recent_finished_job['id'], collection_name).parsed_response
                       sleep 1
                       if records

data/lib/ae_easy/qa/version.rb

@@ -1,5 +1,5 @@
 module AeEasy
   module Qa
-    VERSION = "0.0.32"
+    VERSION = "0.0.34"
   end
 end

data/lib/ae_easy/qa.rb

@@ -22,10 +22,12 @@ module AeEasy
         @data = data
       end
 
+      #this method is for validating "internal" scrapers that run on AnswersEngine
       def validate_internal(vars, outputs)
         ValidateInternal.new(vars, config, outputs).run
       end
 
+      #this method is for validating data from "external" sources
       def validate_external(outputs, collection_name)
         ValidateExternal.new(data, config, outputs, collection_name, options).run
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ae_easy-qa
 version: !ruby/object:Gem::Version
-  version: 0.0.32
+  version: 0.0.34
 platform: ruby
 authors:
 - David Lynam
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-10-03 00:00:00.000000000 Z
+date: 2024-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: answersengine
@@ -92,7 +92,7 @@ licenses:
 metadata:
   homepage_uri: https://answersengine.com
   source_code_uri: https://github.com/answersengine/ae_easy-qa
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -107,9 +107,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.8
-signing_key: 
+rubygems_version: 3.2.15
+signing_key:
 specification_version: 4
 summary: AnswersEngine Easy Quality Assurance gem
 test_files: []