csvlint 0.4.0 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (53) hide show
  1. checksums.yaml +5 -5
  2. data/.github/dependabot.yml +11 -0
  3. data/.github/workflows/push.yml +35 -0
  4. data/.gitignore +1 -0
  5. data/.ruby-version +1 -1
  6. data/.standard_todo.yml +43 -0
  7. data/CHANGELOG.md +38 -0
  8. data/Dockerfile +16 -0
  9. data/Gemfile +2 -2
  10. data/README.md +13 -10
  11. data/Rakefile +7 -7
  12. data/bin/create_schema +2 -2
  13. data/csvlint.gemspec +19 -22
  14. data/docker_notes_for_windows.txt +20 -0
  15. data/features/step_definitions/cli_steps.rb +11 -11
  16. data/features/step_definitions/information_steps.rb +4 -4
  17. data/features/step_definitions/parse_csv_steps.rb +11 -11
  18. data/features/step_definitions/schema_validation_steps.rb +10 -10
  19. data/features/step_definitions/sources_steps.rb +1 -1
  20. data/features/step_definitions/validation_errors_steps.rb +19 -19
  21. data/features/step_definitions/validation_info_steps.rb +9 -9
  22. data/features/step_definitions/validation_warnings_steps.rb +11 -11
  23. data/features/support/aruba.rb +10 -9
  24. data/features/support/earl_formatter.rb +39 -39
  25. data/features/support/env.rb +10 -11
  26. data/features/support/load_tests.rb +109 -105
  27. data/features/support/webmock.rb +3 -1
  28. data/lib/csvlint/cli.rb +136 -142
  29. data/lib/csvlint/csvw/column.rb +279 -280
  30. data/lib/csvlint/csvw/date_format.rb +90 -92
  31. data/lib/csvlint/csvw/metadata_error.rb +1 -3
  32. data/lib/csvlint/csvw/number_format.rb +40 -32
  33. data/lib/csvlint/csvw/property_checker.rb +714 -717
  34. data/lib/csvlint/csvw/table.rb +49 -52
  35. data/lib/csvlint/csvw/table_group.rb +24 -23
  36. data/lib/csvlint/error_collector.rb +2 -0
  37. data/lib/csvlint/error_message.rb +0 -1
  38. data/lib/csvlint/field.rb +153 -141
  39. data/lib/csvlint/schema.rb +35 -43
  40. data/lib/csvlint/validate.rb +173 -151
  41. data/lib/csvlint/version.rb +1 -1
  42. data/lib/csvlint.rb +22 -23
  43. data/spec/csvw/column_spec.rb +15 -16
  44. data/spec/csvw/date_format_spec.rb +5 -7
  45. data/spec/csvw/number_format_spec.rb +2 -4
  46. data/spec/csvw/table_group_spec.rb +103 -105
  47. data/spec/csvw/table_spec.rb +71 -73
  48. data/spec/field_spec.rb +116 -121
  49. data/spec/schema_spec.rb +131 -141
  50. data/spec/spec_helper.rb +6 -6
  51. data/spec/validator_spec.rb +167 -203
  52. metadata +41 -85
  53. data/.travis.yml +0 -37
data/spec/field_spec.rb CHANGED
@@ -1,110 +1,108 @@
1
- require 'spec_helper'
1
+ require "spec_helper"
2
2
 
3
3
  describe Csvlint::Field do
4
-
5
4
  it "should validate required fields" do
6
- field = Csvlint::Field.new("test", { "required" => true } )
7
- expect( field.validate_column( nil ) ).to be(false)
8
- expect( field.errors.first.category ).to be(:schema)
9
- expect( field.validate_column( "" ) ).to be(false)
10
- expect( field.validate_column( "data" ) ).to be(true)
5
+ field = Csvlint::Field.new("test", {"required" => true})
6
+ expect(field.validate_column(nil)).to be(false)
7
+ expect(field.errors.first.category).to be(:schema)
8
+ expect(field.validate_column("")).to be(false)
9
+ expect(field.validate_column("data")).to be(true)
11
10
  end
12
11
 
13
12
  it "should include the failed constraints" do
14
- field = Csvlint::Field.new("test", { "required" => true } )
15
- expect( field.validate_column( nil ) ).to be(false)
16
- expect( field.errors.first.constraints ).to eql( { "required" => true } )
13
+ field = Csvlint::Field.new("test", {"required" => true})
14
+ expect(field.validate_column(nil)).to be(false)
15
+ expect(field.errors.first.constraints).to eql({"required" => true})
17
16
  end
18
17
 
19
18
  it "should validate minimum length" do
20
- field = Csvlint::Field.new("test", { "minLength" => 3 } )
21
- expect( field.validate_column( nil ) ).to be(false)
22
- expect( field.validate_column( "" ) ).to be(false)
23
- expect( field.validate_column( "ab" ) ).to be(false)
24
- expect( field.validate_column( "abc" ) ).to be(true)
25
- expect( field.validate_column( "abcd" ) ).to be(true)
19
+ field = Csvlint::Field.new("test", {"minLength" => 3})
20
+ expect(field.validate_column(nil)).to be(false)
21
+ expect(field.validate_column("")).to be(false)
22
+ expect(field.validate_column("ab")).to be(false)
23
+ expect(field.validate_column("abc")).to be(true)
24
+ expect(field.validate_column("abcd")).to be(true)
26
25
  end
27
26
 
28
27
  it "should validate maximum length" do
29
- field = Csvlint::Field.new("test", { "maxLength" => 3 } )
30
- expect( field.validate_column( nil ) ).to be(true)
31
- expect( field.validate_column( "" ) ).to be(true)
32
- expect( field.validate_column( "ab" ) ).to be(true)
33
- expect( field.validate_column( "abc" ) ).to be(true)
34
- expect( field.validate_column( "abcd" ) ).to be(false)
28
+ field = Csvlint::Field.new("test", {"maxLength" => 3})
29
+ expect(field.validate_column(nil)).to be(true)
30
+ expect(field.validate_column("")).to be(true)
31
+ expect(field.validate_column("ab")).to be(true)
32
+ expect(field.validate_column("abc")).to be(true)
33
+ expect(field.validate_column("abcd")).to be(false)
35
34
  end
36
35
 
37
36
  it "should validate against regex" do
38
- field = Csvlint::Field.new("test", { "pattern" => "\{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}\}"} )
39
- expect( field.validate_column( "abc") ).to be(false)
40
- expect( field.validate_column( "{3B0DA29C-C89A-4FAA-918A-0000074FA0E0}") ).to be(true)
37
+ field = Csvlint::Field.new("test", {"pattern" => "{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}}"})
38
+ expect(field.validate_column("abc")).to be(false)
39
+ expect(field.validate_column("{3B0DA29C-C89A-4FAA-918A-0000074FA0E0}")).to be(true)
41
40
  end
42
41
 
43
42
  it "should apply combinations of constraints" do
44
- field = Csvlint::Field.new("test", { "required"=>true, "pattern" => "\{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}\}"} )
45
- expect( field.validate_column( "abc") ).to be(false)
46
- expect( field.errors.first.constraints ).to eql( { "pattern" => "\{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}\}" } )
47
-
48
- expect( field.validate_column( nil ) ).to be(false)
49
- expect( field.errors.first.constraints ).to eql( { "required"=>true } )
43
+ field = Csvlint::Field.new("test", {"required" => true, "pattern" => "{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}}"})
44
+ expect(field.validate_column("abc")).to be(false)
45
+ expect(field.errors.first.constraints).to eql({"pattern" => "{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}}"})
50
46
 
51
- expect( field.validate_column( "{3B0DA29C-C89A-4FAA-918A-0000074FA0E0}") ).to be(true)
47
+ expect(field.validate_column(nil)).to be(false)
48
+ expect(field.errors.first.constraints).to eql({"required" => true})
52
49
 
50
+ expect(field.validate_column("{3B0DA29C-C89A-4FAA-918A-0000074FA0E0}")).to be(true)
53
51
  end
54
52
 
55
53
  it "should enforce uniqueness for a column" do
56
- field = Csvlint::Field.new("test", { "unique" => true } )
57
- expect( field.validate_column( "abc") ).to be(true)
58
- expect( field.validate_column( "abc") ).to be(false)
59
- expect( field.errors.first.category ).to be(:schema)
60
- expect( field.errors.first.type ).to be(:unique)
54
+ field = Csvlint::Field.new("test", {"unique" => true})
55
+ expect(field.validate_column("abc")).to be(true)
56
+ expect(field.validate_column("abc")).to be(false)
57
+ expect(field.errors.first.category).to be(:schema)
58
+ expect(field.errors.first.type).to be(:unique)
61
59
  end
62
60
 
63
61
  context "it should validate correct types" do
64
62
  it "skips empty fields" do
65
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#int" })
66
- expect( field.validate_column("")).to be(true)
63
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#int"})
64
+ expect(field.validate_column("")).to be(true)
67
65
  end
68
66
 
69
67
  it "validates strings" do
70
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#string" })
71
- expect( field.validate_column("42")).to be(true)
72
- expect( field.validate_column("forty-two")).to be(true)
68
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#string"})
69
+ expect(field.validate_column("42")).to be(true)
70
+ expect(field.validate_column("forty-two")).to be(true)
73
71
  end
74
72
 
75
73
  it "validates ints" do
76
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#int" })
77
- expect( field.validate_column("42")).to be(true)
78
- expect( field.validate_column("forty-two")).to be(false)
74
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#int"})
75
+ expect(field.validate_column("42")).to be(true)
76
+ expect(field.validate_column("forty-two")).to be(false)
79
77
  end
80
78
 
81
79
  it "validates integers" do
82
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#integer" })
83
- expect( field.validate_column("42")).to be(true)
84
- expect( field.validate_column("forty-two")).to be(false)
80
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#integer"})
81
+ expect(field.validate_column("42")).to be(true)
82
+ expect(field.validate_column("forty-two")).to be(false)
85
83
  end
86
84
 
87
85
  it "validates floats" do
88
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#float" })
86
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#float"})
89
87
  expect(field.validate_column("42.0")).to be(true)
90
88
  expect(field.validate_column("42")).to be(true)
91
89
  expect(field.validate_column("forty-two")).to be(false)
92
90
  end
93
91
 
94
92
  it "validates URIs" do
95
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#anyURI" })
93
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#anyURI"})
96
94
  expect(field.validate_column("http://theodi.org/team")).to be(true)
97
95
  expect(field.validate_column("https://theodi.org/team")).to be(true)
98
96
  expect(field.validate_column("42.0")).to be(false)
99
97
  end
100
98
 
101
99
  it "works with invalid URIs" do
102
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#anyURI" })
100
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#anyURI"})
103
101
  expect(field.validate_column("£123")).to be(false)
104
102
  end
105
103
 
106
104
  it "validates booleans" do
107
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#boolean" })
105
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#boolean"})
108
106
  expect(field.validate_column("true")).to be(true)
109
107
  expect(field.validate_column("1")).to be(true)
110
108
  expect(field.validate_column("false")).to be(true)
@@ -114,138 +112,135 @@ describe Csvlint::Field do
114
112
 
115
113
  context "it should validate all kinds of integers" do
116
114
  it "validates a non-positive integer" do
117
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#nonPositiveInteger" })
115
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#nonPositiveInteger"})
118
116
  expect(field.validate_column("0")).to be(true)
119
117
  expect(field.validate_column("-1")).to be(true)
120
118
  expect(field.validate_column("1")).to be(false)
121
119
  end
122
120
 
123
121
  it "validates a negative integer" do
124
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#negativeInteger" })
122
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#negativeInteger"})
125
123
  expect(field.validate_column("0")).to be(false)
126
124
  expect(field.validate_column("-1")).to be(true)
127
125
  expect(field.validate_column("1")).to be(false)
128
126
  end
129
127
 
130
128
  it "validates a non-negative integer" do
131
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#nonNegativeInteger" })
129
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#nonNegativeInteger"})
132
130
  expect(field.validate_column("0")).to be(true)
133
131
  expect(field.validate_column("-1")).to be(false)
134
132
  expect(field.validate_column("1")).to be(true)
135
133
  end
136
134
 
137
135
  it "validates a positive integer" do
138
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#positiveInteger" })
136
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#positiveInteger"})
139
137
  expect(field.validate_column("0")).to be(false)
140
138
  expect(field.validate_column("-1")).to be(false)
141
- expect(field.errors.first.constraints).to eql( { "type" => "http://www.w3.org/2001/XMLSchema#positiveInteger" } )
139
+ expect(field.errors.first.constraints).to eql({"type" => "http://www.w3.org/2001/XMLSchema#positiveInteger"})
142
140
  expect(field.validate_column("1")).to be(true)
143
141
  end
144
142
  end
145
143
 
146
144
  context "when validating ranges" do
147
-
148
145
  it "should enforce minimum values" do
149
146
  field = Csvlint::Field.new("test", {
150
- "type" => "http://www.w3.org/2001/XMLSchema#int",
151
- "minimum" => "40"
147
+ "type" => "http://www.w3.org/2001/XMLSchema#int",
148
+ "minimum" => "40"
152
149
  })
153
- expect( field.validate_column("42")).to be(true)
150
+ expect(field.validate_column("42")).to be(true)
154
151
 
155
152
  field = Csvlint::Field.new("test", {
156
- "type" => "http://www.w3.org/2001/XMLSchema#int",
157
- "minimum" => "40"
153
+ "type" => "http://www.w3.org/2001/XMLSchema#int",
154
+ "minimum" => "40"
158
155
  })
159
- expect( field.validate_column("39")).to be(false)
160
- expect( field.errors.first.type ).to eql(:below_minimum)
156
+ expect(field.validate_column("39")).to be(false)
157
+ expect(field.errors.first.type).to eql(:below_minimum)
161
158
  end
162
159
 
163
160
  it "should enforce maximum values" do
164
161
  field = Csvlint::Field.new("test", {
165
- "type" => "http://www.w3.org/2001/XMLSchema#int",
166
- "maximum" => "40"
162
+ "type" => "http://www.w3.org/2001/XMLSchema#int",
163
+ "maximum" => "40"
167
164
  })
168
- expect( field.validate_column("39")).to be(true)
165
+ expect(field.validate_column("39")).to be(true)
169
166
 
170
167
  field = Csvlint::Field.new("test", {
171
- "type" => "http://www.w3.org/2001/XMLSchema#int",
172
- "maximum" => "40"
168
+ "type" => "http://www.w3.org/2001/XMLSchema#int",
169
+ "maximum" => "40"
173
170
  })
174
- expect( field.validate_column("41")).to be(false)
175
- expect( field.errors.first.type ).to eql(:above_maximum)
176
-
171
+ expect(field.validate_column("41")).to be(false)
172
+ expect(field.errors.first.type).to eql(:above_maximum)
177
173
  end
178
174
  end
179
175
 
180
176
  context "when validating dates" do
181
177
  it "should validate a date time" do
182
178
  field = Csvlint::Field.new("test", {
183
- "type" => "http://www.w3.org/2001/XMLSchema#dateTime"
184
- })
185
- expect( field.validate_column("2014-02-17T11:09:00Z")).to be(true)
186
- expect( field.validate_column("invalid-date")).to be(false)
187
- expect( field.validate_column("2014-02-17")).to be(false)
179
+ "type" => "http://www.w3.org/2001/XMLSchema#dateTime"
180
+ })
181
+ expect(field.validate_column("2014-02-17T11:09:00Z")).to be(true)
182
+ expect(field.validate_column("invalid-date")).to be(false)
183
+ expect(field.validate_column("2014-02-17")).to be(false)
188
184
  end
189
185
  it "should validate a date" do
190
186
  field = Csvlint::Field.new("test", {
191
- "type" => "http://www.w3.org/2001/XMLSchema#date"
192
- })
193
- expect( field.validate_column("2014-02-17T11:09:00Z")).to be(false)
194
- expect( field.validate_column("invalid-date")).to be(false)
195
- expect( field.validate_column("2014-02-17")).to be(true)
187
+ "type" => "http://www.w3.org/2001/XMLSchema#date"
188
+ })
189
+ expect(field.validate_column("2014-02-17T11:09:00Z")).to be(false)
190
+ expect(field.validate_column("invalid-date")).to be(false)
191
+ expect(field.validate_column("2014-02-17")).to be(true)
196
192
  end
197
193
  it "should validate a time" do
198
194
  field = Csvlint::Field.new("test", {
199
- "type" => "http://www.w3.org/2001/XMLSchema#time"
200
- })
201
- expect( field.validate_column("11:09:00")).to be(true)
202
- expect( field.validate_column("2014-02-17T11:09:00Z")).to be(false)
203
- expect( field.validate_column("not-a-time")).to be(false)
204
- expect( field.validate_column("27:97:00")).to be(false)
195
+ "type" => "http://www.w3.org/2001/XMLSchema#time"
196
+ })
197
+ expect(field.validate_column("11:09:00")).to be(true)
198
+ expect(field.validate_column("2014-02-17T11:09:00Z")).to be(false)
199
+ expect(field.validate_column("not-a-time")).to be(false)
200
+ expect(field.validate_column("27:97:00")).to be(false)
205
201
  end
206
202
  it "should validate a year" do
207
203
  field = Csvlint::Field.new("test", {
208
- "type" => "http://www.w3.org/2001/XMLSchema#gYear"
209
- })
210
- expect( field.validate_column("1999")).to be(true)
211
- expect( field.validate_column("2525")).to be(true)
212
- expect( field.validate_column("0001")).to be(true)
213
- expect( field.validate_column("2014-02-17T11:09:00Z")).to be(false)
214
- expect( field.validate_column("not-a-time")).to be(false)
215
- expect( field.validate_column("27:97:00")).to be(false)
204
+ "type" => "http://www.w3.org/2001/XMLSchema#gYear"
205
+ })
206
+ expect(field.validate_column("1999")).to be(true)
207
+ expect(field.validate_column("2525")).to be(true)
208
+ expect(field.validate_column("0001")).to be(true)
209
+ expect(field.validate_column("2014-02-17T11:09:00Z")).to be(false)
210
+ expect(field.validate_column("not-a-time")).to be(false)
211
+ expect(field.validate_column("27:97:00")).to be(false)
216
212
  end
217
213
  it "should validate a year-month" do
218
214
  field = Csvlint::Field.new("test", {
219
- "type" => "http://www.w3.org/2001/XMLSchema#gYearMonth"
220
- })
221
- expect( field.validate_column("1999-12")).to be(true)
222
- expect( field.validate_column("2525-01")).to be(true)
223
- expect( field.validate_column("2014-02-17T11:09:00Z")).to be(false)
224
- expect( field.validate_column("not-a-time")).to be(false)
225
- expect( field.validate_column("27:97:00")).to be(false)
215
+ "type" => "http://www.w3.org/2001/XMLSchema#gYearMonth"
216
+ })
217
+ expect(field.validate_column("1999-12")).to be(true)
218
+ expect(field.validate_column("2525-01")).to be(true)
219
+ expect(field.validate_column("2014-02-17T11:09:00Z")).to be(false)
220
+ expect(field.validate_column("not-a-time")).to be(false)
221
+ expect(field.validate_column("27:97:00")).to be(false)
226
222
  end
227
223
  it "should allow user to specify custom date time pattern" do
228
224
  field = Csvlint::Field.new("test", {
229
- "type" => "http://www.w3.org/2001/XMLSchema#dateTime",
230
- "datePattern" => "%Y-%m-%d %H:%M:%S"
231
- })
232
- expect( field.validate_column("1999-12-01 10:00:00")).to be(true)
233
- expect( field.validate_column("invalid-date")).to be(false)
234
- expect( field.validate_column("2014-02-17")).to be(false)
235
- expect( field.errors.first.constraints ).to eql( {
236
- "type" => "http://www.w3.org/2001/XMLSchema#dateTime",
237
- "datePattern" => "%Y-%m-%d %H:%M:%S"
238
- })
239
-
225
+ "type" => "http://www.w3.org/2001/XMLSchema#dateTime",
226
+ "datePattern" => "%Y-%m-%d %H:%M:%S"
227
+ })
228
+ expect(field.validate_column("1999-12-01 10:00:00")).to be(true)
229
+ expect(field.validate_column("invalid-date")).to be(false)
230
+ expect(field.validate_column("2014-02-17")).to be(false)
231
+ expect(field.errors.first.constraints).to eql({
232
+ "type" => "http://www.w3.org/2001/XMLSchema#dateTime",
233
+ "datePattern" => "%Y-%m-%d %H:%M:%S"
234
+ })
240
235
  end
241
236
  it "should allow user to compare dates" do
242
237
  field = Csvlint::Field.new("test", {
243
- "type" => "http://www.w3.org/2001/XMLSchema#dateTime",
244
- "datePattern" => "%Y-%m-%d %H:%M:%S",
245
- "minimum" => "1990-01-01 10:00:00"
246
- })
247
- expect( field.validate_column("1999-12-01 10:00:00")).to be(true)
248
- expect( field.validate_column("1989-12-01 10:00:00")).to be(false)
238
+ "type" => "http://www.w3.org/2001/XMLSchema#dateTime",
239
+ "datePattern" => "%Y-%m-%d %H:%M:%S",
240
+ "minimum" => "1990-01-01 10:00:00"
241
+ })
242
+ expect(field.validate_column("1999-12-01 10:00:00")).to be(true)
243
+ expect(field.validate_column("1989-12-01 10:00:00")).to be(false)
249
244
  end
250
245
  end
251
246
  end