csvlint 1.0.0 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/.github/dependabot.yml +4 -0
  3. data/.github/workflows/push.yml +14 -2
  4. data/.ruby-version +1 -1
  5. data/.standard_todo.yml +43 -0
  6. data/Dockerfile +16 -0
  7. data/Gemfile +2 -2
  8. data/README.md +9 -9
  9. data/Rakefile +7 -7
  10. data/csvlint.gemspec +14 -16
  11. data/docker_notes_for_windows.txt +20 -0
  12. data/features/step_definitions/cli_steps.rb +11 -11
  13. data/features/step_definitions/information_steps.rb +4 -4
  14. data/features/step_definitions/parse_csv_steps.rb +11 -11
  15. data/features/step_definitions/schema_validation_steps.rb +10 -10
  16. data/features/step_definitions/sources_steps.rb +1 -1
  17. data/features/step_definitions/validation_errors_steps.rb +19 -19
  18. data/features/step_definitions/validation_info_steps.rb +9 -9
  19. data/features/step_definitions/validation_warnings_steps.rb +11 -11
  20. data/features/support/aruba.rb +6 -6
  21. data/features/support/earl_formatter.rb +39 -39
  22. data/features/support/env.rb +10 -11
  23. data/features/support/load_tests.rb +107 -103
  24. data/features/support/webmock.rb +2 -2
  25. data/lib/csvlint/cli.rb +133 -130
  26. data/lib/csvlint/csvw/column.rb +279 -280
  27. data/lib/csvlint/csvw/date_format.rb +90 -92
  28. data/lib/csvlint/csvw/metadata_error.rb +1 -3
  29. data/lib/csvlint/csvw/number_format.rb +40 -32
  30. data/lib/csvlint/csvw/property_checker.rb +714 -717
  31. data/lib/csvlint/csvw/table.rb +49 -52
  32. data/lib/csvlint/csvw/table_group.rb +24 -23
  33. data/lib/csvlint/error_collector.rb +2 -0
  34. data/lib/csvlint/error_message.rb +0 -1
  35. data/lib/csvlint/field.rb +153 -141
  36. data/lib/csvlint/schema.rb +34 -42
  37. data/lib/csvlint/validate.rb +161 -143
  38. data/lib/csvlint/version.rb +1 -1
  39. data/lib/csvlint.rb +22 -23
  40. data/spec/csvw/column_spec.rb +15 -16
  41. data/spec/csvw/date_format_spec.rb +5 -7
  42. data/spec/csvw/number_format_spec.rb +2 -4
  43. data/spec/csvw/table_group_spec.rb +103 -105
  44. data/spec/csvw/table_spec.rb +71 -73
  45. data/spec/field_spec.rb +116 -121
  46. data/spec/schema_spec.rb +129 -139
  47. data/spec/spec_helper.rb +6 -6
  48. data/spec/validator_spec.rb +167 -190
  49. metadata +22 -55
data/spec/schema_spec.rb CHANGED
@@ -1,150 +1,143 @@
1
- require 'spec_helper'
1
+ require "spec_helper"
2
2
 
3
3
  describe Csvlint::Schema do
4
-
5
4
  it "should tolerate missing fields" do
6
5
  schema = Csvlint::Schema.from_json_table("http://example.org", {})
7
- expect( schema ).to_not be(nil)
8
- expect( schema.fields.empty? ).to eql(true)
6
+ expect(schema).to_not be(nil)
7
+ expect(schema.fields.empty?).to eql(true)
9
8
  end
10
9
 
11
10
  it "should tolerate fields with no constraints" do
12
11
  schema = Csvlint::Schema.from_json_table("http://example.org", {
13
- "fields" => [ { "name" => "test" } ]
12
+ "fields" => [{"name" => "test"}]
14
13
  })
15
- expect( schema ).to_not be(nil)
16
- expect( schema.fields[0].name ).to eql("test")
17
- expect( schema.fields[0].constraints ).to eql({})
14
+ expect(schema).to_not be(nil)
15
+ expect(schema.fields[0].name).to eql("test")
16
+ expect(schema.fields[0].constraints).to eql({})
18
17
  end
19
18
 
20
19
  it "should validate against the schema" do
21
- field = Csvlint::Field.new("test", { "required" => true } )
22
- field2 = Csvlint::Field.new("test", { "minLength" => 3 } )
23
- schema = Csvlint::Schema.new("http://example.org", [field, field2] )
24
-
25
- expect( schema.validate_row( ["", "x"] ) ).to eql(false)
26
- expect( schema.errors.size ).to eql(2)
27
- expect( schema.errors.first.type).to eql(:missing_value)
28
- expect( schema.errors.first.category).to eql(:schema)
29
- expect( schema.errors.first.column).to eql(1)
30
- expect( schema.validate_row( ["abc", "1234"] ) ).to eql(true)
31
-
20
+ field = Csvlint::Field.new("test", {"required" => true})
21
+ field2 = Csvlint::Field.new("test", {"minLength" => 3})
22
+ schema = Csvlint::Schema.new("http://example.org", [field, field2])
23
+
24
+ expect(schema.validate_row(["", "x"])).to eql(false)
25
+ expect(schema.errors.size).to eql(2)
26
+ expect(schema.errors.first.type).to eql(:missing_value)
27
+ expect(schema.errors.first.category).to eql(:schema)
28
+ expect(schema.errors.first.column).to eql(1)
29
+ expect(schema.validate_row(["abc", "1234"])).to eql(true)
32
30
  end
33
31
 
34
32
  it "should include validations for missing columns" do
35
- minimum = Csvlint::Field.new("test", { "minLength" => 3 } )
36
- required = Csvlint::Field.new("test2", { "required" => true } )
37
- schema = Csvlint::Schema.new("http://example.org", [minimum, required] )
33
+ minimum = Csvlint::Field.new("test", {"minLength" => 3})
34
+ required = Csvlint::Field.new("test2", {"required" => true})
35
+ schema = Csvlint::Schema.new("http://example.org", [minimum, required])
38
36
 
39
- expect( schema.validate_row( ["abc", "x"] ) ).to eql(true)
37
+ expect(schema.validate_row(["abc", "x"])).to eql(true)
40
38
 
41
- expect( schema.validate_row( ["abc"] ) ).to eql(false)
42
- expect( schema.errors.size ).to eql(1)
43
- expect( schema.errors.first.type).to eql(:missing_value)
39
+ expect(schema.validate_row(["abc"])).to eql(false)
40
+ expect(schema.errors.size).to eql(1)
41
+ expect(schema.errors.first.type).to eql(:missing_value)
44
42
 
45
- schema = Csvlint::Schema.new("http://example.org", [required, minimum] )
46
- expect( schema.validate_row( ["abc"] ) ).to eql(false)
47
- expect( schema.errors.size ).to eql(1)
48
- expect( schema.errors.first.type).to eql(:min_length)
43
+ schema = Csvlint::Schema.new("http://example.org", [required, minimum])
44
+ expect(schema.validate_row(["abc"])).to eql(false)
45
+ expect(schema.errors.size).to eql(1)
46
+ expect(schema.errors.first.type).to eql(:min_length)
49
47
  end
50
48
 
51
49
  it "should warn if the data has fewer columns" do
52
- minimum = Csvlint::Field.new("test", { "minLength" => 3 } )
53
- required = Csvlint::Field.new("test2", { "maxLength" => 5 } )
54
- schema = Csvlint::Schema.new("http://example.org", [minimum, required] )
55
-
56
- expect( schema.validate_row( ["abc"], 1 ) ).to eql(true)
57
- expect( schema.warnings.size ).to eql(1)
58
- expect( schema.warnings.first.type).to eql(:missing_column)
59
- expect( schema.warnings.first.category).to eql(:schema)
60
- expect( schema.warnings.first.row).to eql(1)
61
- expect( schema.warnings.first.column).to eql(2)
62
-
63
- #no ragged row error
64
- expect( schema.errors.size ).to eql(0)
50
+ minimum = Csvlint::Field.new("test", {"minLength" => 3})
51
+ required = Csvlint::Field.new("test2", {"maxLength" => 5})
52
+ schema = Csvlint::Schema.new("http://example.org", [minimum, required])
53
+
54
+ expect(schema.validate_row(["abc"], 1)).to eql(true)
55
+ expect(schema.warnings.size).to eql(1)
56
+ expect(schema.warnings.first.type).to eql(:missing_column)
57
+ expect(schema.warnings.first.category).to eql(:schema)
58
+ expect(schema.warnings.first.row).to eql(1)
59
+ expect(schema.warnings.first.column).to eql(2)
60
+
61
+ # no ragged row error
62
+ expect(schema.errors.size).to eql(0)
65
63
  end
66
64
 
67
65
  it "should warn if the data has additional columns" do
68
- minimum = Csvlint::Field.new("test", { "minLength" => 3 } )
69
- required = Csvlint::Field.new("test2", { "required" => true } )
70
- schema = Csvlint::Schema.new("http://example.org", [minimum, required] )
71
-
72
- expect( schema.validate_row( ["abc", "x", "more", "columns"], 1 ) ).to eql(true)
73
- expect( schema.warnings.size ).to eql(2)
74
- expect( schema.warnings.first.type).to eql(:extra_column)
75
- expect( schema.warnings.first.category).to eql(:schema)
76
- expect( schema.warnings.first.row).to eql(1)
77
- expect( schema.warnings.first.column).to eql(3)
78
-
79
- expect( schema.warnings[1].type).to eql(:extra_column)
80
- expect( schema.warnings[1].column).to eql(4)
81
-
82
- #no ragged row error
83
- expect( schema.errors.size ).to eql(0)
66
+ minimum = Csvlint::Field.new("test", {"minLength" => 3})
67
+ required = Csvlint::Field.new("test2", {"required" => true})
68
+ schema = Csvlint::Schema.new("http://example.org", [minimum, required])
69
+
70
+ expect(schema.validate_row(["abc", "x", "more", "columns"], 1)).to eql(true)
71
+ expect(schema.warnings.size).to eql(2)
72
+ expect(schema.warnings.first.type).to eql(:extra_column)
73
+ expect(schema.warnings.first.category).to eql(:schema)
74
+ expect(schema.warnings.first.row).to eql(1)
75
+ expect(schema.warnings.first.column).to eql(3)
76
+
77
+ expect(schema.warnings[1].type).to eql(:extra_column)
78
+ expect(schema.warnings[1].column).to eql(4)
79
+
80
+ # no ragged row error
81
+ expect(schema.errors.size).to eql(0)
84
82
  end
85
83
 
86
84
  context "when validating header" do
87
85
  it "should warn if column names are different to field names" do
88
- minimum = Csvlint::Field.new("minimum", { "minLength" => 3 } )
89
- required = Csvlint::Field.new("required", { "required" => true } )
90
- schema = Csvlint::Schema.new("http://example.org", [minimum, required] )
91
-
92
- expect( schema.validate_header(["minimum", "required"]) ).to eql(true)
93
- expect( schema.warnings.size ).to eql(0)
94
-
95
- expect( schema.validate_header(["wrong", "required"]) ).to eql(true)
96
- expect( schema.warnings.size ).to eql(1)
97
- expect( schema.warnings.first.row ).to eql(1)
98
- expect( schema.warnings.first.type ).to eql(:malformed_header)
99
- expect( schema.warnings.first.content ).to eql('wrong,required')
100
- expect( schema.warnings.first.column ).to eql(nil)
101
- expect( schema.warnings.first.category ).to eql(:schema)
102
- expect schema.warnings.first.constraints.has_value?('minimum,required')
86
+ minimum = Csvlint::Field.new("minimum", {"minLength" => 3})
87
+ required = Csvlint::Field.new("required", {"required" => true})
88
+ schema = Csvlint::Schema.new("http://example.org", [minimum, required])
89
+
90
+ expect(schema.validate_header(["minimum", "required"])).to eql(true)
91
+ expect(schema.warnings.size).to eql(0)
92
+
93
+ expect(schema.validate_header(["wrong", "required"])).to eql(true)
94
+ expect(schema.warnings.size).to eql(1)
95
+ expect(schema.warnings.first.row).to eql(1)
96
+ expect(schema.warnings.first.type).to eql(:malformed_header)
97
+ expect(schema.warnings.first.content).to eql("wrong,required")
98
+ expect(schema.warnings.first.column).to eql(nil)
99
+ expect(schema.warnings.first.category).to eql(:schema)
100
+ expect schema.warnings.first.constraints.has_value?("minimum,required")
103
101
  # expect( schema.warnings.first.constraints.values ).to eql(['minimum,required'])
104
- expect( schema.validate_header(["minimum", "Required"]) ).to eql(true)
105
- expect( schema.warnings.size ).to eql(1)
106
-
102
+ expect(schema.validate_header(["minimum", "Required"])).to eql(true)
103
+ expect(schema.warnings.size).to eql(1)
107
104
  end
108
105
 
109
106
  it "should warn if column count is less than field count" do
110
- minimum = Csvlint::Field.new("minimum", { "minLength" => 3 } )
111
- required = Csvlint::Field.new("required", { "required" => true } )
112
- schema = Csvlint::Schema.new("http://example.org", [minimum, required] )
113
-
114
- expect( schema.validate_header(["minimum"]) ).to eql(true)
115
- expect( schema.warnings.size ).to eql(1)
116
- expect( schema.warnings.first.row ).to eql(1)
117
- expect( schema.warnings.first.type ).to eql(:malformed_header)
118
- expect( schema.warnings.first.content ).to eql("minimum")
119
- expect( schema.warnings.first.column ).to eql(nil)
120
- expect( schema.warnings.first.category ).to eql(:schema)
121
- expect schema.warnings.first.constraints.has_value?('minimum,required')
107
+ minimum = Csvlint::Field.new("minimum", {"minLength" => 3})
108
+ required = Csvlint::Field.new("required", {"required" => true})
109
+ schema = Csvlint::Schema.new("http://example.org", [minimum, required])
110
+
111
+ expect(schema.validate_header(["minimum"])).to eql(true)
112
+ expect(schema.warnings.size).to eql(1)
113
+ expect(schema.warnings.first.row).to eql(1)
114
+ expect(schema.warnings.first.type).to eql(:malformed_header)
115
+ expect(schema.warnings.first.content).to eql("minimum")
116
+ expect(schema.warnings.first.column).to eql(nil)
117
+ expect(schema.warnings.first.category).to eql(:schema)
118
+ expect schema.warnings.first.constraints.has_value?("minimum,required")
122
119
  # expect( schema.warnings.first.constraints.values ).to eql(['minimum,required'])
123
-
124
120
  end
125
121
 
126
122
  it "should warn if column count is more than field count" do
127
- minimum = Csvlint::Field.new("minimum", { "minLength" => 3 } )
128
- schema = Csvlint::Schema.new("http://example.org", [minimum] )
129
-
130
- expect( schema.validate_header(["wrong", "required"]) ).to eql(true)
131
- expect( schema.warnings.size ).to eql(1)
132
- expect( schema.warnings.first.row ).to eql(1)
133
- expect( schema.warnings.first.type ).to eql(:malformed_header)
134
- expect( schema.warnings.first.content ).to eql("wrong,required")
135
- expect( schema.warnings.first.column ).to eql(nil)
136
- expect( schema.warnings.first.category ).to eql(:schema)
123
+ minimum = Csvlint::Field.new("minimum", {"minLength" => 3})
124
+ schema = Csvlint::Schema.new("http://example.org", [minimum])
125
+
126
+ expect(schema.validate_header(["wrong", "required"])).to eql(true)
127
+ expect(schema.warnings.size).to eql(1)
128
+ expect(schema.warnings.first.row).to eql(1)
129
+ expect(schema.warnings.first.type).to eql(:malformed_header)
130
+ expect(schema.warnings.first.content).to eql("wrong,required")
131
+ expect(schema.warnings.first.column).to eql(nil)
132
+ expect(schema.warnings.first.category).to eql(:schema)
137
133
  # expect( schema.warnings.first.constraints.values ).to eql('minimum')
138
- expect( schema.warnings.first.constraints.has_value?('minimum'))
139
-
134
+ expect(schema.warnings.first.constraints.has_value?("minimum"))
140
135
  end
141
-
142
136
  end
143
137
 
144
138
  context "when parsing JSON Tables" do
145
-
146
139
  before(:each) do
147
- @example=<<-EOL
140
+ @example = <<-EOL
148
141
  {
149
142
  "title": "Schema title",
150
143
  "description": "schema",
@@ -154,58 +147,55 @@ describe Csvlint::Schema do
154
147
  { "name": "Postcode", "constraints": { "required": true, "pattern": "[A-Z]{1,2}[0-9][0-9A-Z]? ?[0-9][A-Z]{2}" } }
155
148
  ]
156
149
  }
157
- EOL
158
- stub_request(:get, "http://example.com/example.json").to_return(:status => 200, :body => @example)
150
+ EOL
151
+ stub_request(:get, "http://example.com/example.json").to_return(status: 200, body: @example)
159
152
  end
160
153
 
161
154
  it "should create a schema from a pre-parsed JSON table" do
162
- json = JSON.parse( @example )
155
+ json = JSON.parse(@example)
163
156
  schema = Csvlint::Schema.from_json_table("http://example.org", json)
164
157
 
165
- expect( schema.uri ).to eql("http://example.org")
166
- expect( schema.title ).to eql("Schema title")
167
- expect( schema.description ).to eql("schema")
168
- expect( schema.fields.length ).to eql(3)
169
- expect( schema.fields[0].name ).to eql("ID")
170
- expect( schema.fields[0].constraints["required"] ).to eql(true)
171
- expect( schema.fields[0].title ).to eql("id")
172
- expect( schema.fields[0].description ).to eql("house identifier")
173
- expect( schema.fields[2].constraints["pattern"]).to eql("[A-Z]{1,2}[0-9][0-9A-Z]? ?[0-9][A-Z]{2}")
158
+ expect(schema.uri).to eql("http://example.org")
159
+ expect(schema.title).to eql("Schema title")
160
+ expect(schema.description).to eql("schema")
161
+ expect(schema.fields.length).to eql(3)
162
+ expect(schema.fields[0].name).to eql("ID")
163
+ expect(schema.fields[0].constraints["required"]).to eql(true)
164
+ expect(schema.fields[0].title).to eql("id")
165
+ expect(schema.fields[0].description).to eql("house identifier")
166
+ expect(schema.fields[2].constraints["pattern"]).to eql("[A-Z]{1,2}[0-9][0-9A-Z]? ?[0-9][A-Z]{2}")
174
167
  end
175
168
 
176
169
  it "should create a schema from a JSON Table URL" do
177
170
  schema = Csvlint::Schema.load_from_uri("http://example.com/example.json")
178
- expect( schema.uri ).to eql("http://example.com/example.json")
179
- expect( schema.fields.length ).to eql(3)
180
- expect( schema.fields[0].name ).to eql("ID")
181
- expect( schema.fields[0].constraints["required"] ).to eql(true)
182
-
171
+ expect(schema.uri).to eql("http://example.com/example.json")
172
+ expect(schema.fields.length).to eql(3)
173
+ expect(schema.fields[0].name).to eql("ID")
174
+ expect(schema.fields[0].constraints["required"]).to eql(true)
183
175
  end
184
176
  end
185
177
 
186
178
  context "when parsing CSVW metadata" do
187
-
188
179
  before(:each) do
189
- @example=<<-EOL
190
- {
191
- "@context": "http://www.w3.org/ns/csvw",
192
- "url": "http://example.com/example1.csv",
193
- "tableSchema": {
194
- "columns": [
195
- { "name": "Name", "required": true, "datatype": { "base": "string", "format": ".+" } },
196
- { "name": "Id", "required": true, "datatype": { "base": "string", "minLength": 3 } },
197
- { "name": "Email", "required": true }
198
- ]
199
- }
200
- }
201
- EOL
202
- stub_request(:get, "http://example.com/metadata.json").to_return(:status => 200, :body => @example)
180
+ @example = <<~EOL
181
+ {
182
+ "@context": "http://www.w3.org/ns/csvw",
183
+ "url": "http://example.com/example1.csv",
184
+ "tableSchema": {
185
+ "columns": [
186
+ { "name": "Name", "required": true, "datatype": { "base": "string", "format": ".+" } },
187
+ { "name": "Id", "required": true, "datatype": { "base": "string", "minLength": 3 } },
188
+ { "name": "Email", "required": true }
189
+ ]
190
+ }
191
+ }
192
+ EOL
193
+ stub_request(:get, "http://example.com/metadata.json").to_return(status: 200, body: @example)
203
194
  end
204
195
 
205
196
  it "should create a table group from a CSVW metadata URL" do
206
197
  schema = Csvlint::Schema.load_from_uri("http://example.com/metadata.json")
207
- expect( schema.class ).to eq(Csvlint::Csvw::TableGroup)
198
+ expect(schema.class).to eq(Csvlint::Csvw::TableGroup)
208
199
  end
209
200
  end
210
-
211
201
  end
data/spec/spec_helper.rb CHANGED
@@ -1,9 +1,9 @@
1
- require 'coveralls'
2
- Coveralls.wear_merged!('test_frameworks')
1
+ require "coveralls"
2
+ Coveralls.wear_merged!("test_frameworks")
3
3
 
4
- require 'csvlint'
5
- require 'byebug'
6
- require 'webmock/rspec'
4
+ require "csvlint"
5
+ require "byebug"
6
+ require "webmock/rspec"
7
7
 
8
8
  RSpec.configure do |config|
9
9
  config.run_all_when_everything_filtered = true
@@ -13,5 +13,5 @@ RSpec.configure do |config|
13
13
  # order dependency and want to debug it, you can fix the order by providing
14
14
  # the seed, which is printed after each run.
15
15
  # --seed 1234
16
- config.order = 'random'
16
+ config.order = "random"
17
17
  end