csvlint 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.github/dependabot.yml +4 -0
  3. data/.github/workflows/push.yml +14 -2
  4. data/.pre-commit-hooks.yaml +5 -0
  5. data/.ruby-version +1 -1
  6. data/.standard_todo.yml +43 -0
  7. data/CHANGELOG.md +84 -32
  8. data/Dockerfile +16 -0
  9. data/Gemfile +2 -2
  10. data/README.md +30 -9
  11. data/Rakefile +7 -7
  12. data/csvlint.gemspec +14 -16
  13. data/docker_notes_for_windows.txt +20 -0
  14. data/features/step_definitions/cli_steps.rb +11 -11
  15. data/features/step_definitions/information_steps.rb +4 -4
  16. data/features/step_definitions/parse_csv_steps.rb +11 -11
  17. data/features/step_definitions/schema_validation_steps.rb +10 -10
  18. data/features/step_definitions/sources_steps.rb +1 -1
  19. data/features/step_definitions/validation_errors_steps.rb +19 -19
  20. data/features/step_definitions/validation_info_steps.rb +9 -9
  21. data/features/step_definitions/validation_warnings_steps.rb +11 -11
  22. data/features/support/aruba.rb +6 -6
  23. data/features/support/earl_formatter.rb +39 -39
  24. data/features/support/env.rb +10 -11
  25. data/features/support/load_tests.rb +107 -103
  26. data/features/support/webmock.rb +2 -2
  27. data/lib/csvlint/cli.rb +133 -130
  28. data/lib/csvlint/csvw/column.rb +279 -280
  29. data/lib/csvlint/csvw/date_format.rb +90 -92
  30. data/lib/csvlint/csvw/metadata_error.rb +1 -3
  31. data/lib/csvlint/csvw/number_format.rb +40 -32
  32. data/lib/csvlint/csvw/property_checker.rb +714 -717
  33. data/lib/csvlint/csvw/table.rb +49 -52
  34. data/lib/csvlint/csvw/table_group.rb +24 -23
  35. data/lib/csvlint/error_collector.rb +2 -0
  36. data/lib/csvlint/error_message.rb +0 -1
  37. data/lib/csvlint/field.rb +153 -141
  38. data/lib/csvlint/schema.rb +34 -42
  39. data/lib/csvlint/validate.rb +161 -143
  40. data/lib/csvlint/version.rb +1 -1
  41. data/lib/csvlint.rb +22 -23
  42. data/spec/csvw/column_spec.rb +15 -16
  43. data/spec/csvw/date_format_spec.rb +5 -7
  44. data/spec/csvw/number_format_spec.rb +2 -4
  45. data/spec/csvw/table_group_spec.rb +103 -105
  46. data/spec/csvw/table_spec.rb +71 -73
  47. data/spec/field_spec.rb +116 -121
  48. data/spec/schema_spec.rb +129 -139
  49. data/spec/spec_helper.rb +6 -6
  50. data/spec/validator_spec.rb +167 -190
  51. metadata +23 -55
data/spec/field_spec.rb CHANGED
@@ -1,110 +1,108 @@
1
- require 'spec_helper'
1
+ require "spec_helper"
2
2
 
3
3
  describe Csvlint::Field do
4
-
5
4
  it "should validate required fields" do
6
- field = Csvlint::Field.new("test", { "required" => true } )
7
- expect( field.validate_column( nil ) ).to be(false)
8
- expect( field.errors.first.category ).to be(:schema)
9
- expect( field.validate_column( "" ) ).to be(false)
10
- expect( field.validate_column( "data" ) ).to be(true)
5
+ field = Csvlint::Field.new("test", {"required" => true})
6
+ expect(field.validate_column(nil)).to be(false)
7
+ expect(field.errors.first.category).to be(:schema)
8
+ expect(field.validate_column("")).to be(false)
9
+ expect(field.validate_column("data")).to be(true)
11
10
  end
12
11
 
13
12
  it "should include the failed constraints" do
14
- field = Csvlint::Field.new("test", { "required" => true } )
15
- expect( field.validate_column( nil ) ).to be(false)
16
- expect( field.errors.first.constraints ).to eql( { "required" => true } )
13
+ field = Csvlint::Field.new("test", {"required" => true})
14
+ expect(field.validate_column(nil)).to be(false)
15
+ expect(field.errors.first.constraints).to eql({"required" => true})
17
16
  end
18
17
 
19
18
  it "should validate minimum length" do
20
- field = Csvlint::Field.new("test", { "minLength" => 3 } )
21
- expect( field.validate_column( nil ) ).to be(false)
22
- expect( field.validate_column( "" ) ).to be(false)
23
- expect( field.validate_column( "ab" ) ).to be(false)
24
- expect( field.validate_column( "abc" ) ).to be(true)
25
- expect( field.validate_column( "abcd" ) ).to be(true)
19
+ field = Csvlint::Field.new("test", {"minLength" => 3})
20
+ expect(field.validate_column(nil)).to be(false)
21
+ expect(field.validate_column("")).to be(false)
22
+ expect(field.validate_column("ab")).to be(false)
23
+ expect(field.validate_column("abc")).to be(true)
24
+ expect(field.validate_column("abcd")).to be(true)
26
25
  end
27
26
 
28
27
  it "should validate maximum length" do
29
- field = Csvlint::Field.new("test", { "maxLength" => 3 } )
30
- expect( field.validate_column( nil ) ).to be(true)
31
- expect( field.validate_column( "" ) ).to be(true)
32
- expect( field.validate_column( "ab" ) ).to be(true)
33
- expect( field.validate_column( "abc" ) ).to be(true)
34
- expect( field.validate_column( "abcd" ) ).to be(false)
28
+ field = Csvlint::Field.new("test", {"maxLength" => 3})
29
+ expect(field.validate_column(nil)).to be(true)
30
+ expect(field.validate_column("")).to be(true)
31
+ expect(field.validate_column("ab")).to be(true)
32
+ expect(field.validate_column("abc")).to be(true)
33
+ expect(field.validate_column("abcd")).to be(false)
35
34
  end
36
35
 
37
36
  it "should validate against regex" do
38
- field = Csvlint::Field.new("test", { "pattern" => "\{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}\}"} )
39
- expect( field.validate_column( "abc") ).to be(false)
40
- expect( field.validate_column( "{3B0DA29C-C89A-4FAA-918A-0000074FA0E0}") ).to be(true)
37
+ field = Csvlint::Field.new("test", {"pattern" => "{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}}"})
38
+ expect(field.validate_column("abc")).to be(false)
39
+ expect(field.validate_column("{3B0DA29C-C89A-4FAA-918A-0000074FA0E0}")).to be(true)
41
40
  end
42
41
 
43
42
  it "should apply combinations of constraints" do
44
- field = Csvlint::Field.new("test", { "required"=>true, "pattern" => "\{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}\}"} )
45
- expect( field.validate_column( "abc") ).to be(false)
46
- expect( field.errors.first.constraints ).to eql( { "pattern" => "\{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}\}" } )
47
-
48
- expect( field.validate_column( nil ) ).to be(false)
49
- expect( field.errors.first.constraints ).to eql( { "required"=>true } )
43
+ field = Csvlint::Field.new("test", {"required" => true, "pattern" => "{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}}"})
44
+ expect(field.validate_column("abc")).to be(false)
45
+ expect(field.errors.first.constraints).to eql({"pattern" => "{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}}"})
50
46
 
51
- expect( field.validate_column( "{3B0DA29C-C89A-4FAA-918A-0000074FA0E0}") ).to be(true)
47
+ expect(field.validate_column(nil)).to be(false)
48
+ expect(field.errors.first.constraints).to eql({"required" => true})
52
49
 
50
+ expect(field.validate_column("{3B0DA29C-C89A-4FAA-918A-0000074FA0E0}")).to be(true)
53
51
  end
54
52
 
55
53
  it "should enforce uniqueness for a column" do
56
- field = Csvlint::Field.new("test", { "unique" => true } )
57
- expect( field.validate_column( "abc") ).to be(true)
58
- expect( field.validate_column( "abc") ).to be(false)
59
- expect( field.errors.first.category ).to be(:schema)
60
- expect( field.errors.first.type ).to be(:unique)
54
+ field = Csvlint::Field.new("test", {"unique" => true})
55
+ expect(field.validate_column("abc")).to be(true)
56
+ expect(field.validate_column("abc")).to be(false)
57
+ expect(field.errors.first.category).to be(:schema)
58
+ expect(field.errors.first.type).to be(:unique)
61
59
  end
62
60
 
63
61
  context "it should validate correct types" do
64
62
  it "skips empty fields" do
65
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#int" })
66
- expect( field.validate_column("")).to be(true)
63
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#int"})
64
+ expect(field.validate_column("")).to be(true)
67
65
  end
68
66
 
69
67
  it "validates strings" do
70
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#string" })
71
- expect( field.validate_column("42")).to be(true)
72
- expect( field.validate_column("forty-two")).to be(true)
68
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#string"})
69
+ expect(field.validate_column("42")).to be(true)
70
+ expect(field.validate_column("forty-two")).to be(true)
73
71
  end
74
72
 
75
73
  it "validates ints" do
76
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#int" })
77
- expect( field.validate_column("42")).to be(true)
78
- expect( field.validate_column("forty-two")).to be(false)
74
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#int"})
75
+ expect(field.validate_column("42")).to be(true)
76
+ expect(field.validate_column("forty-two")).to be(false)
79
77
  end
80
78
 
81
79
  it "validates integers" do
82
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#integer" })
83
- expect( field.validate_column("42")).to be(true)
84
- expect( field.validate_column("forty-two")).to be(false)
80
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#integer"})
81
+ expect(field.validate_column("42")).to be(true)
82
+ expect(field.validate_column("forty-two")).to be(false)
85
83
  end
86
84
 
87
85
  it "validates floats" do
88
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#float" })
86
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#float"})
89
87
  expect(field.validate_column("42.0")).to be(true)
90
88
  expect(field.validate_column("42")).to be(true)
91
89
  expect(field.validate_column("forty-two")).to be(false)
92
90
  end
93
91
 
94
92
  it "validates URIs" do
95
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#anyURI" })
93
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#anyURI"})
96
94
  expect(field.validate_column("http://theodi.org/team")).to be(true)
97
95
  expect(field.validate_column("https://theodi.org/team")).to be(true)
98
96
  expect(field.validate_column("42.0")).to be(false)
99
97
  end
100
98
 
101
99
  it "works with invalid URIs" do
102
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#anyURI" })
100
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#anyURI"})
103
101
  expect(field.validate_column("£123")).to be(false)
104
102
  end
105
103
 
106
104
  it "validates booleans" do
107
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#boolean" })
105
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#boolean"})
108
106
  expect(field.validate_column("true")).to be(true)
109
107
  expect(field.validate_column("1")).to be(true)
110
108
  expect(field.validate_column("false")).to be(true)
@@ -114,138 +112,135 @@ describe Csvlint::Field do
114
112
 
115
113
  context "it should validate all kinds of integers" do
116
114
  it "validates a non-positive integer" do
117
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#nonPositiveInteger" })
115
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#nonPositiveInteger"})
118
116
  expect(field.validate_column("0")).to be(true)
119
117
  expect(field.validate_column("-1")).to be(true)
120
118
  expect(field.validate_column("1")).to be(false)
121
119
  end
122
120
 
123
121
  it "validates a negative integer" do
124
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#negativeInteger" })
122
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#negativeInteger"})
125
123
  expect(field.validate_column("0")).to be(false)
126
124
  expect(field.validate_column("-1")).to be(true)
127
125
  expect(field.validate_column("1")).to be(false)
128
126
  end
129
127
 
130
128
  it "validates a non-negative integer" do
131
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#nonNegativeInteger" })
129
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#nonNegativeInteger"})
132
130
  expect(field.validate_column("0")).to be(true)
133
131
  expect(field.validate_column("-1")).to be(false)
134
132
  expect(field.validate_column("1")).to be(true)
135
133
  end
136
134
 
137
135
  it "validates a positive integer" do
138
- field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#positiveInteger" })
136
+ field = Csvlint::Field.new("test", {"type" => "http://www.w3.org/2001/XMLSchema#positiveInteger"})
139
137
  expect(field.validate_column("0")).to be(false)
140
138
  expect(field.validate_column("-1")).to be(false)
141
- expect(field.errors.first.constraints).to eql( { "type" => "http://www.w3.org/2001/XMLSchema#positiveInteger" } )
139
+ expect(field.errors.first.constraints).to eql({"type" => "http://www.w3.org/2001/XMLSchema#positiveInteger"})
142
140
  expect(field.validate_column("1")).to be(true)
143
141
  end
144
142
  end
145
143
 
146
144
  context "when validating ranges" do
147
-
148
145
  it "should enforce minimum values" do
149
146
  field = Csvlint::Field.new("test", {
150
- "type" => "http://www.w3.org/2001/XMLSchema#int",
151
- "minimum" => "40"
147
+ "type" => "http://www.w3.org/2001/XMLSchema#int",
148
+ "minimum" => "40"
152
149
  })
153
- expect( field.validate_column("42")).to be(true)
150
+ expect(field.validate_column("42")).to be(true)
154
151
 
155
152
  field = Csvlint::Field.new("test", {
156
- "type" => "http://www.w3.org/2001/XMLSchema#int",
157
- "minimum" => "40"
153
+ "type" => "http://www.w3.org/2001/XMLSchema#int",
154
+ "minimum" => "40"
158
155
  })
159
- expect( field.validate_column("39")).to be(false)
160
- expect( field.errors.first.type ).to eql(:below_minimum)
156
+ expect(field.validate_column("39")).to be(false)
157
+ expect(field.errors.first.type).to eql(:below_minimum)
161
158
  end
162
159
 
163
160
  it "should enforce maximum values" do
164
161
  field = Csvlint::Field.new("test", {
165
- "type" => "http://www.w3.org/2001/XMLSchema#int",
166
- "maximum" => "40"
162
+ "type" => "http://www.w3.org/2001/XMLSchema#int",
163
+ "maximum" => "40"
167
164
  })
168
- expect( field.validate_column("39")).to be(true)
165
+ expect(field.validate_column("39")).to be(true)
169
166
 
170
167
  field = Csvlint::Field.new("test", {
171
- "type" => "http://www.w3.org/2001/XMLSchema#int",
172
- "maximum" => "40"
168
+ "type" => "http://www.w3.org/2001/XMLSchema#int",
169
+ "maximum" => "40"
173
170
  })
174
- expect( field.validate_column("41")).to be(false)
175
- expect( field.errors.first.type ).to eql(:above_maximum)
176
-
171
+ expect(field.validate_column("41")).to be(false)
172
+ expect(field.errors.first.type).to eql(:above_maximum)
177
173
  end
178
174
  end
179
175
 
180
176
  context "when validating dates" do
181
177
  it "should validate a date time" do
182
178
  field = Csvlint::Field.new("test", {
183
- "type" => "http://www.w3.org/2001/XMLSchema#dateTime"
184
- })
185
- expect( field.validate_column("2014-02-17T11:09:00Z")).to be(true)
186
- expect( field.validate_column("invalid-date")).to be(false)
187
- expect( field.validate_column("2014-02-17")).to be(false)
179
+ "type" => "http://www.w3.org/2001/XMLSchema#dateTime"
180
+ })
181
+ expect(field.validate_column("2014-02-17T11:09:00Z")).to be(true)
182
+ expect(field.validate_column("invalid-date")).to be(false)
183
+ expect(field.validate_column("2014-02-17")).to be(false)
188
184
  end
189
185
  it "should validate a date" do
190
186
  field = Csvlint::Field.new("test", {
191
- "type" => "http://www.w3.org/2001/XMLSchema#date"
192
- })
193
- expect( field.validate_column("2014-02-17T11:09:00Z")).to be(false)
194
- expect( field.validate_column("invalid-date")).to be(false)
195
- expect( field.validate_column("2014-02-17")).to be(true)
187
+ "type" => "http://www.w3.org/2001/XMLSchema#date"
188
+ })
189
+ expect(field.validate_column("2014-02-17T11:09:00Z")).to be(false)
190
+ expect(field.validate_column("invalid-date")).to be(false)
191
+ expect(field.validate_column("2014-02-17")).to be(true)
196
192
  end
197
193
  it "should validate a time" do
198
194
  field = Csvlint::Field.new("test", {
199
- "type" => "http://www.w3.org/2001/XMLSchema#time"
200
- })
201
- expect( field.validate_column("11:09:00")).to be(true)
202
- expect( field.validate_column("2014-02-17T11:09:00Z")).to be(false)
203
- expect( field.validate_column("not-a-time")).to be(false)
204
- expect( field.validate_column("27:97:00")).to be(false)
195
+ "type" => "http://www.w3.org/2001/XMLSchema#time"
196
+ })
197
+ expect(field.validate_column("11:09:00")).to be(true)
198
+ expect(field.validate_column("2014-02-17T11:09:00Z")).to be(false)
199
+ expect(field.validate_column("not-a-time")).to be(false)
200
+ expect(field.validate_column("27:97:00")).to be(false)
205
201
  end
206
202
  it "should validate a year" do
207
203
  field = Csvlint::Field.new("test", {
208
- "type" => "http://www.w3.org/2001/XMLSchema#gYear"
209
- })
210
- expect( field.validate_column("1999")).to be(true)
211
- expect( field.validate_column("2525")).to be(true)
212
- expect( field.validate_column("0001")).to be(true)
213
- expect( field.validate_column("2014-02-17T11:09:00Z")).to be(false)
214
- expect( field.validate_column("not-a-time")).to be(false)
215
- expect( field.validate_column("27:97:00")).to be(false)
204
+ "type" => "http://www.w3.org/2001/XMLSchema#gYear"
205
+ })
206
+ expect(field.validate_column("1999")).to be(true)
207
+ expect(field.validate_column("2525")).to be(true)
208
+ expect(field.validate_column("0001")).to be(true)
209
+ expect(field.validate_column("2014-02-17T11:09:00Z")).to be(false)
210
+ expect(field.validate_column("not-a-time")).to be(false)
211
+ expect(field.validate_column("27:97:00")).to be(false)
216
212
  end
217
213
  it "should validate a year-month" do
218
214
  field = Csvlint::Field.new("test", {
219
- "type" => "http://www.w3.org/2001/XMLSchema#gYearMonth"
220
- })
221
- expect( field.validate_column("1999-12")).to be(true)
222
- expect( field.validate_column("2525-01")).to be(true)
223
- expect( field.validate_column("2014-02-17T11:09:00Z")).to be(false)
224
- expect( field.validate_column("not-a-time")).to be(false)
225
- expect( field.validate_column("27:97:00")).to be(false)
215
+ "type" => "http://www.w3.org/2001/XMLSchema#gYearMonth"
216
+ })
217
+ expect(field.validate_column("1999-12")).to be(true)
218
+ expect(field.validate_column("2525-01")).to be(true)
219
+ expect(field.validate_column("2014-02-17T11:09:00Z")).to be(false)
220
+ expect(field.validate_column("not-a-time")).to be(false)
221
+ expect(field.validate_column("27:97:00")).to be(false)
226
222
  end
227
223
  it "should allow user to specify custom date time pattern" do
228
224
  field = Csvlint::Field.new("test", {
229
- "type" => "http://www.w3.org/2001/XMLSchema#dateTime",
230
- "datePattern" => "%Y-%m-%d %H:%M:%S"
231
- })
232
- expect( field.validate_column("1999-12-01 10:00:00")).to be(true)
233
- expect( field.validate_column("invalid-date")).to be(false)
234
- expect( field.validate_column("2014-02-17")).to be(false)
235
- expect( field.errors.first.constraints ).to eql( {
236
- "type" => "http://www.w3.org/2001/XMLSchema#dateTime",
237
- "datePattern" => "%Y-%m-%d %H:%M:%S"
238
- })
239
-
225
+ "type" => "http://www.w3.org/2001/XMLSchema#dateTime",
226
+ "datePattern" => "%Y-%m-%d %H:%M:%S"
227
+ })
228
+ expect(field.validate_column("1999-12-01 10:00:00")).to be(true)
229
+ expect(field.validate_column("invalid-date")).to be(false)
230
+ expect(field.validate_column("2014-02-17")).to be(false)
231
+ expect(field.errors.first.constraints).to eql({
232
+ "type" => "http://www.w3.org/2001/XMLSchema#dateTime",
233
+ "datePattern" => "%Y-%m-%d %H:%M:%S"
234
+ })
240
235
  end
241
236
  it "should allow user to compare dates" do
242
237
  field = Csvlint::Field.new("test", {
243
- "type" => "http://www.w3.org/2001/XMLSchema#dateTime",
244
- "datePattern" => "%Y-%m-%d %H:%M:%S",
245
- "minimum" => "1990-01-01 10:00:00"
246
- })
247
- expect( field.validate_column("1999-12-01 10:00:00")).to be(true)
248
- expect( field.validate_column("1989-12-01 10:00:00")).to be(false)
238
+ "type" => "http://www.w3.org/2001/XMLSchema#dateTime",
239
+ "datePattern" => "%Y-%m-%d %H:%M:%S",
240
+ "minimum" => "1990-01-01 10:00:00"
241
+ })
242
+ expect(field.validate_column("1999-12-01 10:00:00")).to be(true)
243
+ expect(field.validate_column("1989-12-01 10:00:00")).to be(false)
249
244
  end
250
245
  end
251
246
  end