csvlint 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.github/dependabot.yml +4 -0
  3. data/.github/workflows/push.yml +14 -2
  4. data/.pre-commit-hooks.yaml +5 -0
  5. data/.ruby-version +1 -1
  6. data/.standard_todo.yml +43 -0
  7. data/CHANGELOG.md +84 -32
  8. data/Dockerfile +16 -0
  9. data/Gemfile +2 -2
  10. data/README.md +30 -9
  11. data/Rakefile +7 -7
  12. data/csvlint.gemspec +14 -16
  13. data/docker_notes_for_windows.txt +20 -0
  14. data/features/step_definitions/cli_steps.rb +11 -11
  15. data/features/step_definitions/information_steps.rb +4 -4
  16. data/features/step_definitions/parse_csv_steps.rb +11 -11
  17. data/features/step_definitions/schema_validation_steps.rb +10 -10
  18. data/features/step_definitions/sources_steps.rb +1 -1
  19. data/features/step_definitions/validation_errors_steps.rb +19 -19
  20. data/features/step_definitions/validation_info_steps.rb +9 -9
  21. data/features/step_definitions/validation_warnings_steps.rb +11 -11
  22. data/features/support/aruba.rb +6 -6
  23. data/features/support/earl_formatter.rb +39 -39
  24. data/features/support/env.rb +10 -11
  25. data/features/support/load_tests.rb +107 -103
  26. data/features/support/webmock.rb +2 -2
  27. data/lib/csvlint/cli.rb +133 -130
  28. data/lib/csvlint/csvw/column.rb +279 -280
  29. data/lib/csvlint/csvw/date_format.rb +90 -92
  30. data/lib/csvlint/csvw/metadata_error.rb +1 -3
  31. data/lib/csvlint/csvw/number_format.rb +40 -32
  32. data/lib/csvlint/csvw/property_checker.rb +714 -717
  33. data/lib/csvlint/csvw/table.rb +49 -52
  34. data/lib/csvlint/csvw/table_group.rb +24 -23
  35. data/lib/csvlint/error_collector.rb +2 -0
  36. data/lib/csvlint/error_message.rb +0 -1
  37. data/lib/csvlint/field.rb +153 -141
  38. data/lib/csvlint/schema.rb +34 -42
  39. data/lib/csvlint/validate.rb +161 -143
  40. data/lib/csvlint/version.rb +1 -1
  41. data/lib/csvlint.rb +22 -23
  42. data/spec/csvw/column_spec.rb +15 -16
  43. data/spec/csvw/date_format_spec.rb +5 -7
  44. data/spec/csvw/number_format_spec.rb +2 -4
  45. data/spec/csvw/table_group_spec.rb +103 -105
  46. data/spec/csvw/table_spec.rb +71 -73
  47. data/spec/field_spec.rb +116 -121
  48. data/spec/schema_spec.rb +129 -139
  49. data/spec/spec_helper.rb +6 -6
  50. data/spec/validator_spec.rb +167 -190
  51. metadata +23 -55
data/spec/schema_spec.rb CHANGED
@@ -1,150 +1,143 @@
1
- require 'spec_helper'
1
+ require "spec_helper"
2
2
 
3
3
  describe Csvlint::Schema do
4
-
5
4
  it "should tolerate missing fields" do
6
5
  schema = Csvlint::Schema.from_json_table("http://example.org", {})
7
- expect( schema ).to_not be(nil)
8
- expect( schema.fields.empty? ).to eql(true)
6
+ expect(schema).to_not be(nil)
7
+ expect(schema.fields.empty?).to eql(true)
9
8
  end
10
9
 
11
10
  it "should tolerate fields with no constraints" do
12
11
  schema = Csvlint::Schema.from_json_table("http://example.org", {
13
- "fields" => [ { "name" => "test" } ]
12
+ "fields" => [{"name" => "test"}]
14
13
  })
15
- expect( schema ).to_not be(nil)
16
- expect( schema.fields[0].name ).to eql("test")
17
- expect( schema.fields[0].constraints ).to eql({})
14
+ expect(schema).to_not be(nil)
15
+ expect(schema.fields[0].name).to eql("test")
16
+ expect(schema.fields[0].constraints).to eql({})
18
17
  end
19
18
 
20
19
  it "should validate against the schema" do
21
- field = Csvlint::Field.new("test", { "required" => true } )
22
- field2 = Csvlint::Field.new("test", { "minLength" => 3 } )
23
- schema = Csvlint::Schema.new("http://example.org", [field, field2] )
24
-
25
- expect( schema.validate_row( ["", "x"] ) ).to eql(false)
26
- expect( schema.errors.size ).to eql(2)
27
- expect( schema.errors.first.type).to eql(:missing_value)
28
- expect( schema.errors.first.category).to eql(:schema)
29
- expect( schema.errors.first.column).to eql(1)
30
- expect( schema.validate_row( ["abc", "1234"] ) ).to eql(true)
31
-
20
+ field = Csvlint::Field.new("test", {"required" => true})
21
+ field2 = Csvlint::Field.new("test", {"minLength" => 3})
22
+ schema = Csvlint::Schema.new("http://example.org", [field, field2])
23
+
24
+ expect(schema.validate_row(["", "x"])).to eql(false)
25
+ expect(schema.errors.size).to eql(2)
26
+ expect(schema.errors.first.type).to eql(:missing_value)
27
+ expect(schema.errors.first.category).to eql(:schema)
28
+ expect(schema.errors.first.column).to eql(1)
29
+ expect(schema.validate_row(["abc", "1234"])).to eql(true)
32
30
  end
33
31
 
34
32
  it "should include validations for missing columns" do
35
- minimum = Csvlint::Field.new("test", { "minLength" => 3 } )
36
- required = Csvlint::Field.new("test2", { "required" => true } )
37
- schema = Csvlint::Schema.new("http://example.org", [minimum, required] )
33
+ minimum = Csvlint::Field.new("test", {"minLength" => 3})
34
+ required = Csvlint::Field.new("test2", {"required" => true})
35
+ schema = Csvlint::Schema.new("http://example.org", [minimum, required])
38
36
 
39
- expect( schema.validate_row( ["abc", "x"] ) ).to eql(true)
37
+ expect(schema.validate_row(["abc", "x"])).to eql(true)
40
38
 
41
- expect( schema.validate_row( ["abc"] ) ).to eql(false)
42
- expect( schema.errors.size ).to eql(1)
43
- expect( schema.errors.first.type).to eql(:missing_value)
39
+ expect(schema.validate_row(["abc"])).to eql(false)
40
+ expect(schema.errors.size).to eql(1)
41
+ expect(schema.errors.first.type).to eql(:missing_value)
44
42
 
45
- schema = Csvlint::Schema.new("http://example.org", [required, minimum] )
46
- expect( schema.validate_row( ["abc"] ) ).to eql(false)
47
- expect( schema.errors.size ).to eql(1)
48
- expect( schema.errors.first.type).to eql(:min_length)
43
+ schema = Csvlint::Schema.new("http://example.org", [required, minimum])
44
+ expect(schema.validate_row(["abc"])).to eql(false)
45
+ expect(schema.errors.size).to eql(1)
46
+ expect(schema.errors.first.type).to eql(:min_length)
49
47
  end
50
48
 
51
49
  it "should warn if the data has fewer columns" do
52
- minimum = Csvlint::Field.new("test", { "minLength" => 3 } )
53
- required = Csvlint::Field.new("test2", { "maxLength" => 5 } )
54
- schema = Csvlint::Schema.new("http://example.org", [minimum, required] )
55
-
56
- expect( schema.validate_row( ["abc"], 1 ) ).to eql(true)
57
- expect( schema.warnings.size ).to eql(1)
58
- expect( schema.warnings.first.type).to eql(:missing_column)
59
- expect( schema.warnings.first.category).to eql(:schema)
60
- expect( schema.warnings.first.row).to eql(1)
61
- expect( schema.warnings.first.column).to eql(2)
62
-
63
- #no ragged row error
64
- expect( schema.errors.size ).to eql(0)
50
+ minimum = Csvlint::Field.new("test", {"minLength" => 3})
51
+ required = Csvlint::Field.new("test2", {"maxLength" => 5})
52
+ schema = Csvlint::Schema.new("http://example.org", [minimum, required])
53
+
54
+ expect(schema.validate_row(["abc"], 1)).to eql(true)
55
+ expect(schema.warnings.size).to eql(1)
56
+ expect(schema.warnings.first.type).to eql(:missing_column)
57
+ expect(schema.warnings.first.category).to eql(:schema)
58
+ expect(schema.warnings.first.row).to eql(1)
59
+ expect(schema.warnings.first.column).to eql(2)
60
+
61
+ # no ragged row error
62
+ expect(schema.errors.size).to eql(0)
65
63
  end
66
64
 
67
65
  it "should warn if the data has additional columns" do
68
- minimum = Csvlint::Field.new("test", { "minLength" => 3 } )
69
- required = Csvlint::Field.new("test2", { "required" => true } )
70
- schema = Csvlint::Schema.new("http://example.org", [minimum, required] )
71
-
72
- expect( schema.validate_row( ["abc", "x", "more", "columns"], 1 ) ).to eql(true)
73
- expect( schema.warnings.size ).to eql(2)
74
- expect( schema.warnings.first.type).to eql(:extra_column)
75
- expect( schema.warnings.first.category).to eql(:schema)
76
- expect( schema.warnings.first.row).to eql(1)
77
- expect( schema.warnings.first.column).to eql(3)
78
-
79
- expect( schema.warnings[1].type).to eql(:extra_column)
80
- expect( schema.warnings[1].column).to eql(4)
81
-
82
- #no ragged row error
83
- expect( schema.errors.size ).to eql(0)
66
+ minimum = Csvlint::Field.new("test", {"minLength" => 3})
67
+ required = Csvlint::Field.new("test2", {"required" => true})
68
+ schema = Csvlint::Schema.new("http://example.org", [minimum, required])
69
+
70
+ expect(schema.validate_row(["abc", "x", "more", "columns"], 1)).to eql(true)
71
+ expect(schema.warnings.size).to eql(2)
72
+ expect(schema.warnings.first.type).to eql(:extra_column)
73
+ expect(schema.warnings.first.category).to eql(:schema)
74
+ expect(schema.warnings.first.row).to eql(1)
75
+ expect(schema.warnings.first.column).to eql(3)
76
+
77
+ expect(schema.warnings[1].type).to eql(:extra_column)
78
+ expect(schema.warnings[1].column).to eql(4)
79
+
80
+ # no ragged row error
81
+ expect(schema.errors.size).to eql(0)
84
82
  end
85
83
 
86
84
  context "when validating header" do
87
85
  it "should warn if column names are different to field names" do
88
- minimum = Csvlint::Field.new("minimum", { "minLength" => 3 } )
89
- required = Csvlint::Field.new("required", { "required" => true } )
90
- schema = Csvlint::Schema.new("http://example.org", [minimum, required] )
91
-
92
- expect( schema.validate_header(["minimum", "required"]) ).to eql(true)
93
- expect( schema.warnings.size ).to eql(0)
94
-
95
- expect( schema.validate_header(["wrong", "required"]) ).to eql(true)
96
- expect( schema.warnings.size ).to eql(1)
97
- expect( schema.warnings.first.row ).to eql(1)
98
- expect( schema.warnings.first.type ).to eql(:malformed_header)
99
- expect( schema.warnings.first.content ).to eql('wrong,required')
100
- expect( schema.warnings.first.column ).to eql(nil)
101
- expect( schema.warnings.first.category ).to eql(:schema)
102
- expect schema.warnings.first.constraints.has_value?('minimum,required')
86
+ minimum = Csvlint::Field.new("minimum", {"minLength" => 3})
87
+ required = Csvlint::Field.new("required", {"required" => true})
88
+ schema = Csvlint::Schema.new("http://example.org", [minimum, required])
89
+
90
+ expect(schema.validate_header(["minimum", "required"])).to eql(true)
91
+ expect(schema.warnings.size).to eql(0)
92
+
93
+ expect(schema.validate_header(["wrong", "required"])).to eql(true)
94
+ expect(schema.warnings.size).to eql(1)
95
+ expect(schema.warnings.first.row).to eql(1)
96
+ expect(schema.warnings.first.type).to eql(:malformed_header)
97
+ expect(schema.warnings.first.content).to eql("wrong,required")
98
+ expect(schema.warnings.first.column).to eql(nil)
99
+ expect(schema.warnings.first.category).to eql(:schema)
100
+ expect schema.warnings.first.constraints.has_value?("minimum,required")
103
101
  # expect( schema.warnings.first.constraints.values ).to eql(['minimum,required'])
104
- expect( schema.validate_header(["minimum", "Required"]) ).to eql(true)
105
- expect( schema.warnings.size ).to eql(1)
106
-
102
+ expect(schema.validate_header(["minimum", "Required"])).to eql(true)
103
+ expect(schema.warnings.size).to eql(1)
107
104
  end
108
105
 
109
106
  it "should warn if column count is less than field count" do
110
- minimum = Csvlint::Field.new("minimum", { "minLength" => 3 } )
111
- required = Csvlint::Field.new("required", { "required" => true } )
112
- schema = Csvlint::Schema.new("http://example.org", [minimum, required] )
113
-
114
- expect( schema.validate_header(["minimum"]) ).to eql(true)
115
- expect( schema.warnings.size ).to eql(1)
116
- expect( schema.warnings.first.row ).to eql(1)
117
- expect( schema.warnings.first.type ).to eql(:malformed_header)
118
- expect( schema.warnings.first.content ).to eql("minimum")
119
- expect( schema.warnings.first.column ).to eql(nil)
120
- expect( schema.warnings.first.category ).to eql(:schema)
121
- expect schema.warnings.first.constraints.has_value?('minimum,required')
107
+ minimum = Csvlint::Field.new("minimum", {"minLength" => 3})
108
+ required = Csvlint::Field.new("required", {"required" => true})
109
+ schema = Csvlint::Schema.new("http://example.org", [minimum, required])
110
+
111
+ expect(schema.validate_header(["minimum"])).to eql(true)
112
+ expect(schema.warnings.size).to eql(1)
113
+ expect(schema.warnings.first.row).to eql(1)
114
+ expect(schema.warnings.first.type).to eql(:malformed_header)
115
+ expect(schema.warnings.first.content).to eql("minimum")
116
+ expect(schema.warnings.first.column).to eql(nil)
117
+ expect(schema.warnings.first.category).to eql(:schema)
118
+ expect schema.warnings.first.constraints.has_value?("minimum,required")
122
119
  # expect( schema.warnings.first.constraints.values ).to eql(['minimum,required'])
123
-
124
120
  end
125
121
 
126
122
  it "should warn if column count is more than field count" do
127
- minimum = Csvlint::Field.new("minimum", { "minLength" => 3 } )
128
- schema = Csvlint::Schema.new("http://example.org", [minimum] )
129
-
130
- expect( schema.validate_header(["wrong", "required"]) ).to eql(true)
131
- expect( schema.warnings.size ).to eql(1)
132
- expect( schema.warnings.first.row ).to eql(1)
133
- expect( schema.warnings.first.type ).to eql(:malformed_header)
134
- expect( schema.warnings.first.content ).to eql("wrong,required")
135
- expect( schema.warnings.first.column ).to eql(nil)
136
- expect( schema.warnings.first.category ).to eql(:schema)
123
+ minimum = Csvlint::Field.new("minimum", {"minLength" => 3})
124
+ schema = Csvlint::Schema.new("http://example.org", [minimum])
125
+
126
+ expect(schema.validate_header(["wrong", "required"])).to eql(true)
127
+ expect(schema.warnings.size).to eql(1)
128
+ expect(schema.warnings.first.row).to eql(1)
129
+ expect(schema.warnings.first.type).to eql(:malformed_header)
130
+ expect(schema.warnings.first.content).to eql("wrong,required")
131
+ expect(schema.warnings.first.column).to eql(nil)
132
+ expect(schema.warnings.first.category).to eql(:schema)
137
133
  # expect( schema.warnings.first.constraints.values ).to eql('minimum')
138
- expect( schema.warnings.first.constraints.has_value?('minimum'))
139
-
134
+ expect(schema.warnings.first.constraints.has_value?("minimum"))
140
135
  end
141
-
142
136
  end
143
137
 
144
138
  context "when parsing JSON Tables" do
145
-
146
139
  before(:each) do
147
- @example=<<-EOL
140
+ @example = <<-EOL
148
141
  {
149
142
  "title": "Schema title",
150
143
  "description": "schema",
@@ -154,58 +147,55 @@ describe Csvlint::Schema do
154
147
  { "name": "Postcode", "constraints": { "required": true, "pattern": "[A-Z]{1,2}[0-9][0-9A-Z]? ?[0-9][A-Z]{2}" } }
155
148
  ]
156
149
  }
157
- EOL
158
- stub_request(:get, "http://example.com/example.json").to_return(:status => 200, :body => @example)
150
+ EOL
151
+ stub_request(:get, "http://example.com/example.json").to_return(status: 200, body: @example)
159
152
  end
160
153
 
161
154
  it "should create a schema from a pre-parsed JSON table" do
162
- json = JSON.parse( @example )
155
+ json = JSON.parse(@example)
163
156
  schema = Csvlint::Schema.from_json_table("http://example.org", json)
164
157
 
165
- expect( schema.uri ).to eql("http://example.org")
166
- expect( schema.title ).to eql("Schema title")
167
- expect( schema.description ).to eql("schema")
168
- expect( schema.fields.length ).to eql(3)
169
- expect( schema.fields[0].name ).to eql("ID")
170
- expect( schema.fields[0].constraints["required"] ).to eql(true)
171
- expect( schema.fields[0].title ).to eql("id")
172
- expect( schema.fields[0].description ).to eql("house identifier")
173
- expect( schema.fields[2].constraints["pattern"]).to eql("[A-Z]{1,2}[0-9][0-9A-Z]? ?[0-9][A-Z]{2}")
158
+ expect(schema.uri).to eql("http://example.org")
159
+ expect(schema.title).to eql("Schema title")
160
+ expect(schema.description).to eql("schema")
161
+ expect(schema.fields.length).to eql(3)
162
+ expect(schema.fields[0].name).to eql("ID")
163
+ expect(schema.fields[0].constraints["required"]).to eql(true)
164
+ expect(schema.fields[0].title).to eql("id")
165
+ expect(schema.fields[0].description).to eql("house identifier")
166
+ expect(schema.fields[2].constraints["pattern"]).to eql("[A-Z]{1,2}[0-9][0-9A-Z]? ?[0-9][A-Z]{2}")
174
167
  end
175
168
 
176
169
  it "should create a schema from a JSON Table URL" do
177
170
  schema = Csvlint::Schema.load_from_uri("http://example.com/example.json")
178
- expect( schema.uri ).to eql("http://example.com/example.json")
179
- expect( schema.fields.length ).to eql(3)
180
- expect( schema.fields[0].name ).to eql("ID")
181
- expect( schema.fields[0].constraints["required"] ).to eql(true)
182
-
171
+ expect(schema.uri).to eql("http://example.com/example.json")
172
+ expect(schema.fields.length).to eql(3)
173
+ expect(schema.fields[0].name).to eql("ID")
174
+ expect(schema.fields[0].constraints["required"]).to eql(true)
183
175
  end
184
176
  end
185
177
 
186
178
  context "when parsing CSVW metadata" do
187
-
188
179
  before(:each) do
189
- @example=<<-EOL
190
- {
191
- "@context": "http://www.w3.org/ns/csvw",
192
- "url": "http://example.com/example1.csv",
193
- "tableSchema": {
194
- "columns": [
195
- { "name": "Name", "required": true, "datatype": { "base": "string", "format": ".+" } },
196
- { "name": "Id", "required": true, "datatype": { "base": "string", "minLength": 3 } },
197
- { "name": "Email", "required": true }
198
- ]
199
- }
200
- }
201
- EOL
202
- stub_request(:get, "http://example.com/metadata.json").to_return(:status => 200, :body => @example)
180
+ @example = <<~EOL
181
+ {
182
+ "@context": "http://www.w3.org/ns/csvw",
183
+ "url": "http://example.com/example1.csv",
184
+ "tableSchema": {
185
+ "columns": [
186
+ { "name": "Name", "required": true, "datatype": { "base": "string", "format": ".+" } },
187
+ { "name": "Id", "required": true, "datatype": { "base": "string", "minLength": 3 } },
188
+ { "name": "Email", "required": true }
189
+ ]
190
+ }
191
+ }
192
+ EOL
193
+ stub_request(:get, "http://example.com/metadata.json").to_return(status: 200, body: @example)
203
194
  end
204
195
 
205
196
  it "should create a table group from a CSVW metadata URL" do
206
197
  schema = Csvlint::Schema.load_from_uri("http://example.com/metadata.json")
207
- expect( schema.class ).to eq(Csvlint::Csvw::TableGroup)
198
+ expect(schema.class).to eq(Csvlint::Csvw::TableGroup)
208
199
  end
209
200
  end
210
-
211
201
  end
data/spec/spec_helper.rb CHANGED
@@ -1,9 +1,9 @@
1
- require 'coveralls'
2
- Coveralls.wear_merged!('test_frameworks')
1
+ require "coveralls"
2
+ Coveralls.wear_merged!("test_frameworks")
3
3
 
4
- require 'csvlint'
5
- require 'byebug'
6
- require 'webmock/rspec'
4
+ require "csvlint"
5
+ require "byebug"
6
+ require "webmock/rspec"
7
7
 
8
8
  RSpec.configure do |config|
9
9
  config.run_all_when_everything_filtered = true
@@ -13,5 +13,5 @@ RSpec.configure do |config|
13
13
  # order dependency and want to debug it, you can fix the order by providing
14
14
  # the seed, which is printed after each run.
15
15
  # --seed 1234
16
- config.order = 'random'
16
+ config.order = "random"
17
17
  end