data_sanity 0.2.2 → 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +13 -6
- data/data_sanity.gemspec +2 -0
- data/lib/data_sanity.rb +1 -0
- data/lib/data_sanity/inspector.rb +5 -9
- data/lib/data_sanity/output.rb +37 -0
- data/lib/data_sanity/tasks.rb +2 -2
- data/lib/data_sanity/version.rb +1 -1
- data/spec/data_sanity/inspector_spec.rb +14 -0
- data/spec/data_sanity/output_spec.rb +45 -0
- metadata +23 -5
data/README.md
CHANGED
@@ -6,8 +6,13 @@
|
|
6
6
|
# Random data in all models :
|
7
7
|
* specify n number of random records,
|
8
8
|
* specify criteria
|
9
|
+
It allows you to see details in two strategies - table/csv
|
10
|
+
# output to a model named DataInspector
|
11
|
+
* For this you need to run migrations
|
12
|
+
* You get an advantage of using variable from command line
|
13
|
+
# output to a csv file in tmp folder as data_inspector.csv
|
9
14
|
|
10
|
-
# It would output result in a table with following details
|
15
|
+
# It would output result in a table/csv with following details
|
11
16
|
|
12
17
|
+----+------------+-------------+-------------------+-----------------------+
|
13
18
|
| id | table_name | primary_key | primary_key_value | errors |
|
@@ -16,15 +21,17 @@
|
|
16
21
|
+----+------------+-------------+-------------------+-----------------------+
|
17
22
|
|
18
23
|
# HOW TO USE
|
19
|
-
|
20
|
-
*
|
21
|
-
*
|
24
|
+
|
25
|
+
* adding migrations : rake data_sanity:db:migrate [for strategy table]
|
26
|
+
* removing migrations : rake data_sanity:db:rollback [for strategy table]
|
22
27
|
* You can add a sample criteria file using rake data_sanity:criteria
|
23
28
|
* To Investigate your data use rake data_sanity:investigate
|
24
|
-
- By default it runs for all data
|
29
|
+
- By default it runs for all data with strategy table
|
25
30
|
: If criteria file is specifies then picks model from file and validates only those
|
26
31
|
: Note: For any criteria mentioned except model names is ignored
|
27
|
-
- rake data_sanity:investigate[random,2]
|
32
|
+
- rake data_sanity:investigate[table,random,2]
|
28
33
|
: parameter 1: random: show you want random selection
|
29
34
|
: parameter 2: lets you add as many random records you want to verify [default is 1]
|
30
35
|
: Note: if you have a criteria file it works only for models in criteria file
|
36
|
+
- rake data_sanity:investigate[csv,random,2]
|
37
|
+
|
data/data_sanity.gemspec
CHANGED
data/lib/data_sanity.rb
CHANGED
@@ -3,9 +3,10 @@ module DataSanity
|
|
3
3
|
|
4
4
|
CONSIDERED_RECORDS = 1
|
5
5
|
ALLOWED_VALIDATION = [:all, :random, :criteria]
|
6
|
-
attr_accessor :all, :random, :criteria, :models, :records_per_model
|
6
|
+
attr_accessor :all, :random, :criteria, :models, :records_per_model, :output
|
7
7
|
|
8
8
|
def initialize options = {}
|
9
|
+
@output = DataSanity::Output.new :option => ( options[:strategy] || :table ).to_sym
|
9
10
|
options[:validate] == "random" ? @random = true : @all = true
|
10
11
|
@records_per_model = options[:records_per_model].to_i == 0 ? CONSIDERED_RECORDS : options[:records_per_model].to_i
|
11
12
|
@models = load_models
|
@@ -35,6 +36,7 @@ module DataSanity
|
|
35
36
|
log_end(model_string)
|
36
37
|
end
|
37
38
|
end
|
39
|
+
@output.close
|
38
40
|
end
|
39
41
|
|
40
42
|
private
|
@@ -71,15 +73,9 @@ module DataSanity
|
|
71
73
|
end
|
72
74
|
|
73
75
|
def populate_if_invalid_record(instance, model)
|
74
|
-
|
75
|
-
:table_primary_key => model.primary_key,
|
76
|
-
:primary_key_value => instance.send(model.primary_key),
|
77
|
-
:validation_errors => instance.errors.full_messages.to_yaml) unless instance.valid?
|
76
|
+
@output.create_from model, instance unless instance.valid?
|
78
77
|
rescue Exception => exp
|
79
|
-
|
80
|
-
:table_primary_key => model.primary_key,
|
81
|
-
:primary_key_value => instance.send(model.primary_key),
|
82
|
-
:validation_errors => exp.to_yaml)
|
78
|
+
@output.create_from model, instance, exp
|
83
79
|
end
|
84
80
|
|
85
81
|
def load_models
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'fastercsv'
|
2
|
+
|
3
|
+
module DataSanity
|
4
|
+
class Output
|
5
|
+
OPTIONS = [:table, :csv]
|
6
|
+
|
7
|
+
attr_accessor :option, :csv_file
|
8
|
+
def initialize options = {}
|
9
|
+
raise Exception.new("This options to output the result of inspector is not valid") unless OPTIONS.include?(options[:option])
|
10
|
+
self.option = options[:option]
|
11
|
+
if self.option == :csv
|
12
|
+
self.csv_file = FasterCSV.open("#{Rails.root}/tmp/data_inspector.csv", 'w')
|
13
|
+
self.csv_file.add_row ['table_name', 'table_primary_key', 'primary_key_value', 'validation_errors']
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def create_from model, instance, exception = nil
|
18
|
+
send("store_to_#{self.option}", model, instance, exception)
|
19
|
+
end
|
20
|
+
|
21
|
+
def close
|
22
|
+
self.csv_file.close_write if self.csv_file
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
def store_to_table model, instance, exception
|
27
|
+
DataInspector.create(:table_name => model.to_s,
|
28
|
+
:table_primary_key => model.primary_key,
|
29
|
+
:primary_key_value => instance.send(model.primary_key),
|
30
|
+
:validation_errors => (exception || instance.errors.full_messages.to_yaml))
|
31
|
+
end
|
32
|
+
|
33
|
+
def store_to_csv model, instance, exception
|
34
|
+
self.csv_file.add_row [model.to_s, model.primary_key, instance.send(model.primary_key), ( exception || instance.errors.full_messages.to_sentence) ]
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/lib/data_sanity/tasks.rb
CHANGED
@@ -26,8 +26,8 @@ namespace :data_sanity do
|
|
26
26
|
end
|
27
27
|
end
|
28
28
|
|
29
|
-
desc 'Data Sanity run investigation'
|
30
|
-
task :investigate, [:validate, :records_per_model] => :environment do |t, args|
|
29
|
+
desc 'Data Sanity run investigation - params[strategy,validate,records_per_model]'
|
30
|
+
task :investigate, [:strategy, :validate, :records_per_model] => :environment do |t, args|
|
31
31
|
DataSanity::Inspector.new(args).investigate
|
32
32
|
end
|
33
33
|
end
|
data/lib/data_sanity/version.rb
CHANGED
@@ -50,6 +50,20 @@ describe "DataSanity::Inspector" do
|
|
50
50
|
YAML.load(DataInspector.first.validation_errors).should == first_person.errors.full_messages
|
51
51
|
end
|
52
52
|
|
53
|
+
it "should check for errors on the model and populate fields in DataInspector" do
|
54
|
+
inspector = DataSanity::Inspector.new :strategy => :csv
|
55
|
+
|
56
|
+
Person.new(:age => 1).save(:validate => false)
|
57
|
+
|
58
|
+
inspector.investigate
|
59
|
+
|
60
|
+
first_person = Person.find_by_age(1)
|
61
|
+
first_person.valid?
|
62
|
+
path = "#{Rails.root}/tmp/data_inspector.csv"
|
63
|
+
FasterCSV.read(path).should == [["table_name", "table_primary_key", "primary_key_value", "validation_errors"],
|
64
|
+
["Person", "id", "#{first_person.id}", "#{first_person.errors.full_messages.to_sentence}"]]
|
65
|
+
end
|
66
|
+
|
53
67
|
it "should check for errors on the model and populate fields in DataInspector even if valid? raises exception" do
|
54
68
|
inspector = DataSanity::Inspector.new
|
55
69
|
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe "DataSanity::Output" do
|
4
|
+
|
5
|
+
describe "initialize" do
|
6
|
+
it "should initialize the method of output of inspection" do
|
7
|
+
output = DataSanity::Output.new :option => :table
|
8
|
+
output.option.should == :table
|
9
|
+
end
|
10
|
+
|
11
|
+
it "should throw an exception if method not among specified options" do
|
12
|
+
lambda { DataSanity::Output.new :option => :invalid }.should raise_error("This options to output the result of inspector is not valid")
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
describe "store" do
|
17
|
+
it "should store to data_inspector model when option is database" do
|
18
|
+
output = DataSanity::Output.new :option => :table
|
19
|
+
errors = ["error1", "error2"]
|
20
|
+
instance = mock(:id => 2, :errors => mock(:full_messages => errors))
|
21
|
+
model = mock(:to_s => "model", :primary_key => "id")
|
22
|
+
output.create_from(model, instance)
|
23
|
+
|
24
|
+
DataInspector.count.should == 1
|
25
|
+
di = DataInspector.first
|
26
|
+
di.table_name.should == "model"
|
27
|
+
di.table_primary_key.should == "id"
|
28
|
+
di.primary_key_value.should == "2"
|
29
|
+
di.validation_errors.should == errors.to_yaml
|
30
|
+
end
|
31
|
+
|
32
|
+
it "should store to csv file in tmp folder of app" do
|
33
|
+
output = DataSanity::Output.new :option => :csv
|
34
|
+
errors = ["error1", "error2"]
|
35
|
+
instance = mock(:id => 2, :errors => mock(:full_messages => errors))
|
36
|
+
model = mock(:to_s => "model", :primary_key => "id")
|
37
|
+
output.create_from(model, instance)
|
38
|
+
output.close
|
39
|
+
|
40
|
+
path = "#{Rails.root}/tmp/data_inspector.csv"
|
41
|
+
FasterCSV.read(path).should == [["table_name", "table_primary_key", "primary_key_value", "validation_errors"],
|
42
|
+
["model", "id", "2", "error1 and error2"]]
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 2
|
8
|
-
-
|
9
|
-
version: 0.2.
|
8
|
+
- 3
|
9
|
+
version: 0.2.3
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Habibullah, Rahul, Jigyasa, Jyotsna, Hephzibah, Garima
|
@@ -14,10 +14,23 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2012-06-
|
17
|
+
date: 2012-06-15 00:00:00 +05:30
|
18
18
|
default_executable:
|
19
|
-
dependencies:
|
20
|
-
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: fastercsv
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - "="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
segments:
|
28
|
+
- 1
|
29
|
+
- 5
|
30
|
+
- 5
|
31
|
+
version: 1.5.5
|
32
|
+
type: :runtime
|
33
|
+
version_requirements: *id001
|
21
34
|
description: Gem for checking data sanity
|
22
35
|
email:
|
23
36
|
- igarimasingh@gmail.com
|
@@ -37,11 +50,13 @@ files:
|
|
37
50
|
- data_sanity.gemspec
|
38
51
|
- lib/data_sanity.rb
|
39
52
|
- lib/data_sanity/inspector.rb
|
53
|
+
- lib/data_sanity/output.rb
|
40
54
|
- lib/data_sanity/railtie.rb
|
41
55
|
- lib/data_sanity/tasks.rb
|
42
56
|
- lib/data_sanity/templates/data_sanity_criteria.yml
|
43
57
|
- lib/data_sanity/version.rb
|
44
58
|
- spec/data_sanity/inspector_spec.rb
|
59
|
+
- spec/data_sanity/output_spec.rb
|
45
60
|
- spec/spec_helper.rb
|
46
61
|
- spec/support/helper.rb
|
47
62
|
- spec/support/sample_app/.gitignore
|
@@ -90,6 +105,7 @@ files:
|
|
90
105
|
- spec/support/sample_app/script/rails
|
91
106
|
- spec/support/sample_app/test/performance/browsing_test.rb
|
92
107
|
- spec/support/sample_app/test/test_helper.rb
|
108
|
+
- spec/support/sample_app/tmp/tmp.txt
|
93
109
|
- spec/support/sample_app/vendor/plugins/.gitkeep
|
94
110
|
has_rdoc: true
|
95
111
|
homepage: https://github.com/singhgarima/data_sanity
|
@@ -123,6 +139,7 @@ specification_version: 3
|
|
123
139
|
summary: Gem for checking data sanity
|
124
140
|
test_files:
|
125
141
|
- spec/data_sanity/inspector_spec.rb
|
142
|
+
- spec/data_sanity/output_spec.rb
|
126
143
|
- spec/spec_helper.rb
|
127
144
|
- spec/support/helper.rb
|
128
145
|
- spec/support/sample_app/.gitignore
|
@@ -171,4 +188,5 @@ test_files:
|
|
171
188
|
- spec/support/sample_app/script/rails
|
172
189
|
- spec/support/sample_app/test/performance/browsing_test.rb
|
173
190
|
- spec/support/sample_app/test/test_helper.rb
|
191
|
+
- spec/support/sample_app/tmp/tmp.txt
|
174
192
|
- spec/support/sample_app/vendor/plugins/.gitkeep
|