dh_easy-qa 0.0.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 5f50c7fbde345982f84159b95991622284552a7c879cef3c41433afb40bd297b
4
+ data.tar.gz: a528d2eab5ad5ab35cae524ba67a10b07c4a2f58d60bdddd2f11b11af8276bd3
5
+ SHA512:
6
+ metadata.gz: 1cc9e15f9e2284744af36d311e185c8ee0df773e76e5b1d6d61a267158264efa63340c0227bdb68e2c1dac2e22a528cdc025204ccd669f140a9daad7f839cdd5
7
+ data.tar.gz: 62ddd23ffdb3fe48ba7380f6ba7ff5196e80ce6c3c72d79b4c3ae4640a797c70fef2a38b10e0047758f648f53eae2fc40fb6ceeefed60250cf584920e741a493
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "dh_easy/qa"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,17 @@
1
+ require 'time'
2
+ require 'dh_easy/config'
3
+ require 'dh_easy/qa/helpers'
4
+ require 'dh_easy/qa/version'
5
+ require 'dh_easy/qa/validate_internal'
6
+ require 'dh_easy/qa/validate_external'
7
+ require 'dh_easy/qa/validate_rules'
8
+ require 'dh_easy/qa/validate_type'
9
+ require 'dh_easy/qa/validate_value'
10
+ require 'dh_easy/qa/validate_groups'
11
+ require 'dh_easy/qa/save_output'
12
+ require 'dh_easy/qa/validator'
13
+
14
+ module DhEasy
15
+ module Qa
16
+ end
17
+ end
@@ -0,0 +1,46 @@
1
+ module DhEasy
2
+ module Qa
3
+ module Helpers
4
+ def add_errored_item(data_hash, field_to_validate, validation)
5
+ error_name = "#{field_to_validate}_#{validation}"
6
+ errored_item[:failures][error_name.to_sym] = 'fail'
7
+ errored_item[:item] = data_hash if errored_item[:data].nil?
8
+ end
9
+
10
+ def pass_if?(if_params, data_hash)
11
+ case if_params.keys.first
12
+ when 'and', 'or'
13
+ operator = if_params.keys.first
14
+ evaluations = if_params[operator].collect{|child_if_params|
15
+ field_name = child_if_params.keys.first
16
+ condition_hash = child_if_params[field_name]
17
+ evaluate_condition(field_name, condition_hash, data_hash)
18
+ }
19
+ operator == 'and' ? evaluations.all? : evaluations.any?
20
+ else
21
+ field_name = if_params.keys.first
22
+ condition_hash = if_params[field_name]
23
+ evaluate_condition(field_name, condition_hash, data_hash)
24
+ end
25
+ end
26
+
27
+ def evaluate_condition(field_name, condition_hash, data_hash)
28
+ case condition_hash.keys.first
29
+ when 'value'
30
+ value_hash = condition_hash['value'] #Ex: {"equal"=>"A"}
31
+ if value_hash['equal']
32
+ data_hash[field_name] == value_hash['equal']
33
+ elsif value_hash['regex']
34
+ !(data_hash[field_name].to_s =~ Regexp.new(value_hash['regex'], true)).nil?
35
+ elsif value_hash['less_than']
36
+ data_hash[field_name].to_i < value_hash['less_than'].to_i
37
+ elsif value_hash['greater_than']
38
+ data_hash[field_name].to_i > value_hash['greater_than'].to_i
39
+ else
40
+ raise StandardError.new("The if condition '#{value_hash}' is unknown.")
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,158 @@
1
+ module DhEasy
2
+ module Qa
3
+ class SaveOutput
4
+ attr_reader :total_items, :rules, :errors, :collection_name,
5
+ :outputs, :summary, :error_totals, :fields_to_ignore,
6
+ :specific_validations_to_ignore, :options
7
+
8
+ def initialize(total_items, rules, errors, collection_name, outputs, options)
9
+ @total_items = total_items
10
+ @rules = rules
11
+ @errors = errors
12
+ @collection_name = collection_name
13
+ @outputs = outputs
14
+ @options = options
15
+ @summary = Hash.new(0)
16
+ @error_totals = {}
17
+ @fields_to_ignore = []
18
+ @specific_validations_to_ignore = []
19
+ end
20
+
21
+ def run
22
+ gather_threshold_totals
23
+ gather_validations_to_ignore
24
+ save_group_errors
25
+ save_errors
26
+ save_summary
27
+ end
28
+
29
+ private
30
+
31
+ #thresholds are a setting where you can ignore errors if they are under a specific error rate
32
+ def gather_threshold_totals
33
+ rules.each{|field_to_validate, field_options|
34
+ field_threshold = return_threshold(field_to_validate, field_options)
35
+ if field_threshold
36
+ gather_field_threshold_totals(field_to_validate, field_options)
37
+ else
38
+ gather_specific_validation_totals(field_to_validate, field_options)
39
+ end
40
+ }
41
+ end
42
+
43
+ def gather_field_threshold_totals(field_to_validate, field_options)
44
+ error_total = errors[:errored_items].inject(0){|total, errored_item|
45
+ failed_fields = errored_item[:failures].keys.collect{|failure_key|
46
+ extract_field(failure_key)
47
+ }.uniq
48
+ total += 1 if failed_fields.include?(field_to_validate)
49
+ total
50
+ }
51
+ error_totals[field_to_validate] = error_total
52
+ end
53
+
54
+ def gather_specific_validation_totals(field_to_validate, field_options)
55
+ field_options.each do |validation|
56
+ potential_failure_name = "#{field_to_validate}_#{validation[0]}_fail"
57
+ if options['thresholds'] && options['thresholds'][potential_failure_name]
58
+ error_total = errors[:errored_items].inject(0){|total, errored_item|
59
+ failed_validations = errored_item[:failures].keys.collect{|failure_key|
60
+ "#{failure_key}_fail"
61
+ }
62
+ total += 1 if failed_validations.include?(potential_failure_name)
63
+ total
64
+ }
65
+ error_totals[potential_failure_name] = error_total
66
+ end
67
+ end
68
+ end
69
+
70
+ def gather_validations_to_ignore
71
+ rules.each{|field_to_validate, field_options|
72
+ field_threshold = return_threshold(field_to_validate, field_options)
73
+ if field_threshold
74
+ gather_fields_to_ignore(field_to_validate, field_threshold)
75
+ else
76
+ gather_specific_validations_to_ignore(field_to_validate, field_options)
77
+ end
78
+ }
79
+ end
80
+
81
+ def gather_fields_to_ignore(field_to_validate, field_threshold)
82
+ total_errors = error_totals[field_to_validate]
83
+ if total_errors
84
+ success_ratio = (total_items - total_errors).to_f / total_items
85
+ if success_ratio > field_threshold.to_f
86
+ puts "Ignoring #{field_to_validate}"
87
+ fields_to_ignore.push(field_to_validate)
88
+ end
89
+ end
90
+ end
91
+
92
+ def gather_specific_validations_to_ignore(field_to_validate, field_options)
93
+ field_options.each do |validation|
94
+ potential_failure_name = "#{field_to_validate}_#{validation[0]}_fail"
95
+ total_errors = error_totals[potential_failure_name]
96
+ if total_errors
97
+ specific_validation_threshold = options['thresholds'][potential_failure_name].to_f
98
+ success_ratio = (total_items - total_errors).to_f / total_items
99
+ if success_ratio > specific_validation_threshold || (specific_validation_threshold == 0.0 && success_ratio == 0.0)
100
+ puts "Ignoring #{potential_failure_name}"
101
+ specific_validations_to_ignore.push(potential_failure_name)
102
+ end
103
+ end
104
+ end
105
+ end
106
+
107
+ def return_threshold(field_to_validate, field_options)
108
+ if options['thresholds']
109
+ options['thresholds'][field_to_validate]
110
+ else
111
+ field_options['threshold'] || options['threshold']
112
+ end
113
+ end
114
+
115
+ def save_group_errors
116
+ errors.each{|error_name, output_hash|
117
+ if error_name != :errored_items
118
+ error_name = output_hash.delete(:failure)
119
+ summary["#{error_name}_fail"] = output_hash
120
+ end
121
+ }
122
+ end
123
+
124
+ def save_errors
125
+ errors[:errored_items].each do |errored_item|
126
+ remove_threshold_failures(errored_item) if fields_to_ignore.any? || specific_validations_to_ignore.any?
127
+ errored_item[:failures].each do |error_key, value|
128
+ key = "#{error_key.to_s}_#{value.to_s}"
129
+ summary[key] += 1
130
+ end
131
+ errored_item['_collection'] = collection_name
132
+ outputs << errored_item if errored_item[:failures].any?
133
+ end
134
+ end
135
+
136
+ def remove_threshold_failures(errored_item)
137
+ errored_item[:failures].delete_if{|error_name, fail|
138
+ specific_validation_name = "#{error_name}_fail"
139
+ field_name = extract_field(error_name)
140
+ fields_to_ignore.include?(field_name) || specific_validations_to_ignore.include?(specific_validation_name)
141
+ }
142
+ end
143
+
144
+ def save_summary
145
+ summary['pass'] = 'true' if summary.empty?
146
+ summary['_collection'] = "#{collection_name}_summary"
147
+ summary['total_items'] = total_items
148
+ outputs << summary
149
+ end
150
+
151
+ def extract_field(field_sym)
152
+ a = field_sym.to_s.split('_')
153
+ a.pop
154
+ a.join('_')
155
+ end
156
+ end
157
+ end
158
+ end
@@ -0,0 +1,30 @@
1
+ module DhEasy
2
+ module Qa
3
+ class ValidateExternal
4
+ attr_reader :data, :errors, :rules, :outputs, :collection_name, :options
5
+
6
+ def initialize(data, config, outputs, collection_name, options)
7
+ @data = data
8
+ @rules = config['individual_validations'] if config
9
+ @outputs = outputs
10
+ @collection_name = collection_name
11
+ @options = options
12
+ @errors = { errored_items: [] }
13
+ end
14
+
15
+ def run
16
+ begin
17
+ if data.any?
18
+ ValidateGroups.new(data, nil, collection_name, errors).run
19
+ ValidateRules.new(data, errors, rules).run if rules
20
+ end
21
+ SaveOutput.new(data.count, rules, errors, collection_name, outputs, options).run
22
+ return errors
23
+ rescue StandardError => e
24
+ puts "An error has occurred: #{e}"
25
+ return nil
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,51 @@
1
+ module DhEasy
2
+ module Qa
3
+ class ValidateGroups
4
+ attr_reader :data, :scraper_name, :collection_name, :errors
5
+
6
+ def initialize(data, scraper_name, collection_name, errors)
7
+ @data = data
8
+ @scraper_name = scraper_name
9
+ @collection_name = collection_name
10
+ @errors = errors
11
+ end
12
+
13
+ def run
14
+ puts "Running Group Validations"
15
+ if group_validations_present?
16
+ load_module
17
+ include_module
18
+ call_validation_methods
19
+ end
20
+ end
21
+
22
+ private
23
+
24
+ def fail_validation(name)
25
+ errors[name.to_sym] = 'fail'
26
+ end
27
+
28
+ def load_module
29
+ load group_validations_path
30
+ end
31
+
32
+ def include_module
33
+ self.class.send(:include, GroupValidations)
34
+ end
35
+
36
+ def call_validation_methods
37
+ GroupValidations.public_instance_methods.each do |method|
38
+ self.send(method)
39
+ end
40
+ end
41
+
42
+ def group_validations_present?
43
+ File.exists?(group_validations_path)
44
+ end
45
+
46
+ def group_validations_path
47
+ @group_validations_path ||= File.expand_path('group_validations.rb', Dir.pwd)
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,172 @@
1
+ module DhEasy
2
+ module Qa
3
+ class ValidateInternal
4
+ attr_reader :scraper_name, :collections, :rules, :outputs
5
+
6
+ def initialize(vars, config, outputs)
7
+ @scraper_name = vars['scraper_name']
8
+ @collections = vars['collections']
9
+ @rules = config['individual_validations']
10
+ @outputs = outputs
11
+ end
12
+
13
+ def run
14
+ begin
15
+ ValidateScraper.new(scraper_name, collections, rules, outputs, thresholds).run
16
+ rescue StandardError => e
17
+ puts "An error has occurred: #{e}"
18
+ return nil
19
+ end
20
+ end
21
+
22
+ private
23
+
24
+ #thresholds are a setting where you can suppress errors if they are under a specific error rate
25
+ def thresholds
26
+ @thresholds ||= begin
27
+ file_path = File.expand_path('thresholds.yaml', Dir.pwd)
28
+ if File.exists? file_path
29
+ YAML.load(File.open(file_path))
30
+ else
31
+ nil
32
+ end
33
+ end
34
+ end
35
+ end
36
+
37
+ class ValidateScraper
38
+ attr_reader :scraper_name, :collections, :rules, :outputs, :options
39
+
40
+ def initialize(scraper_name, collections, rules, outputs, thresholds)
41
+ @scraper_name = scraper_name
42
+ @collections = collections
43
+ @rules = rules
44
+ @outputs = outputs
45
+ @options = {}
46
+ options['thresholds'] = thresholds[scraper_name] if thresholds && thresholds[scraper_name]
47
+ end
48
+
49
+ def run
50
+ begin
51
+ output_scraper
52
+ if status_ok?
53
+ validate_collections if collections && collections.any?
54
+ else
55
+ output_response
56
+ return nil
57
+ end
58
+ rescue StandardError => e
59
+ puts "An error has occurred for the scraper named '#{scraper_name}': #{e}"
60
+ return nil
61
+ end
62
+ end
63
+
64
+ private
65
+
66
+ def output_scraper
67
+ puts "validating scraper: #{scraper_name}"
68
+ end
69
+
70
+ def status_ok?
71
+ !collection_response.parsed_response.nil? && collection_response.code == 200
72
+ end
73
+
74
+ def validate_collections
75
+ collections.each do |collection_name|
76
+ collection = collection_counts.find{|collection_hash| collection_hash['collection'] == collection_name }
77
+ if collection
78
+ ValidateCollection.new(scraper_name, collection_name, collection['outputs'], rules, outputs, options).run
79
+ else
80
+ puts "collection #{collection_name} is missing"
81
+ end
82
+ end
83
+ end
84
+
85
+ def output_response
86
+ if collection_response.parsed_response.nil?
87
+ puts "collection response is null"
88
+ else
89
+ puts collection_response.parsed_response['message']
90
+ end
91
+ end
92
+
93
+ def collection_counts
94
+ @collection_counts ||= collection_response.parsed_response
95
+ end
96
+
97
+ def collection_response
98
+ @collection_response || Datahen::Client::ScraperJobOutput.new.collections(scraper_name)
99
+ end
100
+ end
101
+
102
+ class ValidateCollection
103
+ attr_reader :scraper_name, :collection_name, :total_records, :rules, :errors, :outputs, :options
104
+
105
+ def initialize(scraper_name, collection_name, total_records, rules, outputs, options)
106
+ @scraper_name = scraper_name
107
+ @collection_name = collection_name
108
+ @total_records = total_records
109
+ @rules = rules
110
+ @outputs = outputs
111
+ @options = options
112
+ @errors = { errored_items: [] }
113
+ end
114
+
115
+ def run
116
+ output_collection
117
+ if most_recent_finished_job
118
+ puts "data count #{data.count}"
119
+ if data.any?
120
+ ValidateGroups.new(data, scraper_name, collection_name, errors).run
121
+ ValidateRules.new(data, errors, rules).run if rules
122
+ end
123
+ SaveOutput.new(data.count, rules, errors, outputs_collection_name, outputs, options).run
124
+ else
125
+ puts "No job with status 'done' available"
126
+ end
127
+ end
128
+
129
+ private
130
+
131
+ def output_collection
132
+ puts "Validating collection: #{collection_name}"
133
+ end
134
+
135
+ def most_recent_finished_job
136
+ @most_recent_finished_job ||= begin
137
+ jobs_response = Datahen::Client::ScraperJob.new.all(scraper_name)
138
+ if jobs_response.code == 200
139
+ jobs_response.parsed_response.sort_by { |job| job['created_at'] }.reverse.find{|job| job['status'] == 'active' || job['status'] == 'done' }
140
+ else
141
+ nil
142
+ end
143
+ end
144
+ end
145
+
146
+ def data
147
+ @data ||= begin
148
+ data = []
149
+ page = 1
150
+ while data.count < total_records
151
+ records = Datahen::Client::JobOutput.new(per_page:500, page: page).all(most_recent_finished_job['id'], collection_name).parsed_response
152
+ sleep 1
153
+ if records
154
+ records.each do |record|
155
+ data << record
156
+ end
157
+ else
158
+ puts "All ScraperJobOutput request was nil. Total records: #{total_records}, data count: #{data.count}, page: #{page}"
159
+ break
160
+ end
161
+ page += 1
162
+ end
163
+ data
164
+ end
165
+ end
166
+
167
+ def outputs_collection_name
168
+ @outputs_collection_name ||= "#{scraper_name}_#{collection_name}"
169
+ end
170
+ end
171
+ end
172
+ end
@@ -0,0 +1,89 @@
1
+ module DhEasy
2
+ module Qa
3
+ class ValidateRules
4
+ include Helpers
5
+
6
+ attr_reader :data, :errors, :rules
7
+ attr_accessor :errored_item
8
+
9
+ def initialize(data, errors, rules)
10
+ @data = data
11
+ @errors = errors
12
+ @rules = rules
13
+ end
14
+
15
+ def run
16
+ puts "Validating Rules"
17
+ handle_rules
18
+ end
19
+
20
+ private
21
+
22
+ def handle_rules
23
+ data.each do |data_hash|
24
+ reset_errored_item
25
+ rules.each{|field_to_validate, options|
26
+ if passes_required_check?(options, data_hash, field_to_validate)
27
+ options.each{|validation, value|
28
+ case validation
29
+ when 'type'
30
+ ValidateType.new(data_hash, field_to_validate, value, rules, errored_item).run
31
+ when 'value'
32
+ ValidateValue.new(data_hash, field_to_validate, value, errored_item).run
33
+ when 'length'
34
+ validate_length(data_hash, field_to_validate, value)
35
+ when /required|threshold|if/
36
+ nil
37
+ else
38
+ unknown_validation_error(validation) if validation !~ /format/
39
+ end
40
+ }
41
+ end
42
+ }
43
+ errors[:errored_items].push(errored_item) if errored_item && !errored_item[:failures].empty?
44
+ end
45
+ end
46
+
47
+ def passes_required_check?(options, data_hash, field_to_validate)
48
+ if options['required'] == true
49
+ if options['if']
50
+ if pass_if?(options['if'], data_hash)
51
+ check_for_required_failure(data_hash, field_to_validate)
52
+ else
53
+ false
54
+ end
55
+ else
56
+ check_for_required_failure(data_hash, field_to_validate)
57
+ end
58
+ else
59
+ false
60
+ end
61
+ end
62
+
63
+ def check_for_required_failure(data_hash, field_to_validate)
64
+ if fails_required_check?(data_hash, field_to_validate)
65
+ add_errored_item(data_hash, field_to_validate, 'required')
66
+ false
67
+ else
68
+ true
69
+ end
70
+ end
71
+
72
+ def fails_required_check?(data_hash, field_to_validate)
73
+ data_hash[field_to_validate].nil? || (data_hash[field_to_validate].class == String && data_hash[field_to_validate].empty?)
74
+ end
75
+
76
+ def validate_length(data_hash, field_to_validate, length)
77
+ add_errored_item(data_hash, field_to_validate, 'length') if data_hash[field_to_validate].to_s.length != length
78
+ end
79
+
80
+ def reset_errored_item
81
+ self.errored_item = { failures: {} }
82
+ end
83
+
84
+ def unknown_validation_error(validation)
85
+ raise StandardError.new("The validation '#{validation}' is unknown.")
86
+ end
87
+ end
88
+ end
89
+ end
@@ -0,0 +1,64 @@
1
+ module DhEasy
2
+ module Qa
3
+ class ValidateType
4
+ include Helpers
5
+
6
+ attr_reader :data_hash, :field_to_validate, :desired_type, :rules, :errored_item
7
+
8
+ def initialize(data_hash, field_to_validate, desired_type, rules, errored_item)
9
+ @data_hash = data_hash
10
+ @field_to_validate = field_to_validate
11
+ @desired_type = desired_type
12
+ @rules = rules
13
+ @errored_item = errored_item
14
+ end
15
+
16
+ def run
17
+ handle_types
18
+ end
19
+
20
+ private
21
+
22
+ def handle_types
23
+ case desired_type
24
+ when 'String'
25
+ add_errored_item(data_hash, field_to_validate, 'type') if data_hash[field_to_validate].class != String
26
+ when 'Integer'
27
+ add_errored_item(data_hash, field_to_validate, 'type') unless data_hash[field_to_validate].class == Fixnum || data_hash[field_to_validate].to_s.strip =~ /\A\d+(\,\d+)?\z/
28
+ when 'Float'
29
+ add_errored_item(data_hash, field_to_validate, 'type') unless data_hash[field_to_validate].class == Float || data_hash[field_to_validate].to_s.strip =~ /\A\.?\d+(\,\d+)?(\.\d+)?\z/
30
+ when 'Date'
31
+ validate_date_type
32
+ when 'Url'
33
+ add_errored_item(data_hash, field_to_validate, 'type') if data_hash[field_to_validate] !~ /^(http|https):\/\//
34
+ else
35
+ unknown_type_error(desired_type)
36
+ end
37
+ end
38
+
39
+ def validate_date_type
40
+ format = rules[field_to_validate]['format']
41
+ missing_date_format_error if format.nil?
42
+ date_str = data_hash[field_to_validate]
43
+ begin
44
+ date = Time.strptime(date_str, format.gsub('%-m', '%m').gsub('%-d', '%d'))
45
+ add_errored_item(data_hash, field_to_validate, 'type') if date.strftime(format) != date_str
46
+ rescue ArgumentError => e
47
+ if e.to_s =~ /^invalid strptime format/
48
+ add_errored_item(data_hash, field_to_validate, 'type')
49
+ else
50
+ raise StandardError.new(e.to_s)
51
+ end
52
+ end
53
+ end
54
+
55
+ def unknown_type_error(desired_type)
56
+ raise StandardError.new("The validation type '#{desired_type}' is unknown.")
57
+ end
58
+
59
+ def missing_date_format_error
60
+ raise StandardError.new("Date validation is missing a format.")
61
+ end
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,75 @@
1
+ module DhEasy
2
+ module Qa
3
+ class ValidateValue
4
+ include Helpers
5
+
6
+ attr_reader :data_hash, :field_to_validate, :params, :errored_item
7
+
8
+ def initialize(data_hash, field_to_validate, params, errored_item)
9
+ @data_hash = data_hash
10
+ @field_to_validate = field_to_validate
11
+ @params = params
12
+ @errored_item = errored_item
13
+ end
14
+
15
+ def run
16
+ if_exists? ? handle_if : main_value_check
17
+ end
18
+
19
+ private
20
+
21
+ def if_exists?
22
+ !params['if'].nil?
23
+ end
24
+
25
+ def handle_if
26
+ main_value_check if pass_if?(params['if'], data_hash)
27
+ end
28
+
29
+ def main_value_check
30
+ if params['equal']
31
+ if equal_with_operators?
32
+ equal_with_operators
33
+ else
34
+ add_errored_item(data_hash, field_to_validate, 'value') if (data_hash[field_to_validate] != params['equal'])
35
+ end
36
+ elsif params['regex']
37
+ add_errored_item(data_hash, field_to_validate, 'value') if (data_hash[field_to_validate].to_s !~ Regexp.new(params['regex'], true))
38
+ elsif params['less_than']
39
+ add_errored_item(data_hash, field_to_validate, 'value') if !(data_hash[field_to_validate].to_i < params['less_than'].to_i)
40
+ elsif params['greater_than']
41
+ add_errored_item(data_hash, field_to_validate, 'value') if !(data_hash[field_to_validate].to_i > params['greater_than'].to_i)
42
+ else
43
+ unknown_value_error
44
+ end
45
+ end
46
+
47
+ def equal_with_operators
48
+ case equal_operator
49
+ when 'or'
50
+ add_errored_item(data_hash, field_to_validate, 'value') if !or_statment
51
+ end
52
+ end
53
+
54
+ def or_statment
55
+ eval or_vals.map{|val| "data_hash[field_to_validate] == #{val}" }.join(' || ')
56
+ end
57
+
58
+ def equal_with_operators?
59
+ params['equal'].class == Hash
60
+ end
61
+
62
+ def equal_operator
63
+ params['equal'].keys.first
64
+ end
65
+
66
+ def or_vals
67
+ params['equal']['or']
68
+ end
69
+
70
+ def unknown_value_error
71
+ raise StandardError.new("The value rule '#{params}' is unknown.")
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,49 @@
1
+ module DhEasy
2
+ module Qa
3
+ class Validator
4
+ attr_reader :local_config, :data, :options, :errors
5
+
6
+ def initialize(data=nil, options={})
7
+ load_config
8
+ @options = options
9
+ @data = data
10
+ end
11
+
12
+ #this method is for validating "internal" scrapers that run on Datahen
13
+ def validate_internal(vars, outputs)
14
+ ValidateInternal.new(vars, config, outputs).run
15
+ end
16
+
17
+ #this method is for validating data from "external" sources
18
+ def validate_external(outputs, collection_name)
19
+ ValidateExternal.new(data, config, outputs, collection_name, options).run
20
+ end
21
+
22
+ # Configuration.
23
+ #
24
+ # @return [Hash]
25
+ def config
26
+ @config ||= local_config['qa']
27
+ end
28
+
29
+ # Configuration.
30
+ #
31
+ # @param [Hash] value Configuration.
32
+ #
33
+ # @return [Hash]
34
+ def config=value
35
+ @config = value
36
+ end
37
+
38
+ private
39
+
40
+ def load_config
41
+ @local_config ||= DhEasy::Config::Local.new
42
+ end
43
+
44
+ def config_path
45
+ local_config.file_path
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,5 @@
1
+ module DhEasy
2
+ module Qa
3
+ VERSION = "0.0.33"
4
+ end
5
+ end
metadata ADDED
@@ -0,0 +1,143 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: dh_easy-qa
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.33
5
+ platform: ruby
6
+ authors:
7
+ - David Lynam
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2019-12-04 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: dh_easy-config
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: bundler
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '1.16'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '1.16'
41
+ - !ruby/object:Gem::Dependency
42
+ name: minitest
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '5'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '5'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rake
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '10.0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '10.0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: simplecov
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: simplecov-console
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ description: DataHen Easy QA gem allows you to ensure the quality of output on Fetch
98
+ email:
99
+ - dlynam@gmail.com
100
+ executables: []
101
+ extensions: []
102
+ extra_rdoc_files: []
103
+ files:
104
+ - bin/console
105
+ - bin/setup
106
+ - lib/dh_easy/qa.rb
107
+ - lib/dh_easy/qa/helpers.rb
108
+ - lib/dh_easy/qa/save_output.rb
109
+ - lib/dh_easy/qa/validate_external.rb
110
+ - lib/dh_easy/qa/validate_groups.rb
111
+ - lib/dh_easy/qa/validate_internal.rb
112
+ - lib/dh_easy/qa/validate_rules.rb
113
+ - lib/dh_easy/qa/validate_type.rb
114
+ - lib/dh_easy/qa/validate_value.rb
115
+ - lib/dh_easy/qa/validator.rb
116
+ - lib/dh_easy/qa/version.rb
117
+ homepage: https://datahen.com
118
+ licenses:
119
+ - MIT
120
+ metadata:
121
+ homepage_uri: https://datahen.com
122
+ source_code_uri: https://github.com/DataHenOfficial/dh_easy-qa
123
+ post_install_message:
124
+ rdoc_options: []
125
+ require_paths:
126
+ - lib
127
+ required_ruby_version: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ">="
130
+ - !ruby/object:Gem::Version
131
+ version: 2.2.2
132
+ required_rubygems_version: !ruby/object:Gem::Requirement
133
+ requirements:
134
+ - - ">="
135
+ - !ruby/object:Gem::Version
136
+ version: '0'
137
+ requirements: []
138
+ rubyforge_project:
139
+ rubygems_version: 2.7.6
140
+ signing_key:
141
+ specification_version: 4
142
+ summary: DataHen Easy Quality Assurance gem
143
+ test_files: []