csvlint 1.0.0 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/.github/dependabot.yml +4 -0
  3. data/.github/workflows/push.yml +14 -2
  4. data/.ruby-version +1 -1
  5. data/.standard_todo.yml +43 -0
  6. data/Dockerfile +16 -0
  7. data/Gemfile +2 -2
  8. data/README.md +9 -9
  9. data/Rakefile +7 -7
  10. data/csvlint.gemspec +14 -16
  11. data/docker_notes_for_windows.txt +20 -0
  12. data/features/step_definitions/cli_steps.rb +11 -11
  13. data/features/step_definitions/information_steps.rb +4 -4
  14. data/features/step_definitions/parse_csv_steps.rb +11 -11
  15. data/features/step_definitions/schema_validation_steps.rb +10 -10
  16. data/features/step_definitions/sources_steps.rb +1 -1
  17. data/features/step_definitions/validation_errors_steps.rb +19 -19
  18. data/features/step_definitions/validation_info_steps.rb +9 -9
  19. data/features/step_definitions/validation_warnings_steps.rb +11 -11
  20. data/features/support/aruba.rb +6 -6
  21. data/features/support/earl_formatter.rb +39 -39
  22. data/features/support/env.rb +10 -11
  23. data/features/support/load_tests.rb +107 -103
  24. data/features/support/webmock.rb +2 -2
  25. data/lib/csvlint/cli.rb +133 -130
  26. data/lib/csvlint/csvw/column.rb +279 -280
  27. data/lib/csvlint/csvw/date_format.rb +90 -92
  28. data/lib/csvlint/csvw/metadata_error.rb +1 -3
  29. data/lib/csvlint/csvw/number_format.rb +40 -32
  30. data/lib/csvlint/csvw/property_checker.rb +714 -717
  31. data/lib/csvlint/csvw/table.rb +49 -52
  32. data/lib/csvlint/csvw/table_group.rb +24 -23
  33. data/lib/csvlint/error_collector.rb +2 -0
  34. data/lib/csvlint/error_message.rb +0 -1
  35. data/lib/csvlint/field.rb +153 -141
  36. data/lib/csvlint/schema.rb +34 -42
  37. data/lib/csvlint/validate.rb +161 -143
  38. data/lib/csvlint/version.rb +1 -1
  39. data/lib/csvlint.rb +22 -23
  40. data/spec/csvw/column_spec.rb +15 -16
  41. data/spec/csvw/date_format_spec.rb +5 -7
  42. data/spec/csvw/number_format_spec.rb +2 -4
  43. data/spec/csvw/table_group_spec.rb +103 -105
  44. data/spec/csvw/table_spec.rb +71 -73
  45. data/spec/field_spec.rb +116 -121
  46. data/spec/schema_spec.rb +129 -139
  47. data/spec/spec_helper.rb +6 -6
  48. data/spec/validator_spec.rb +167 -190
  49. metadata +22 -55
@@ -1,49 +1,45 @@
1
- require 'spec_helper'
1
+ require "spec_helper"
2
2
 
3
3
  describe Csvlint::Validator do
4
-
5
4
  before do
6
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :body => "")
7
- stub_request(:get, "http://example.com/.well-known/csvm").to_return(:status => 404)
8
- stub_request(:get, "http://example.com/example.csv-metadata.json").to_return(:status => 404)
9
- stub_request(:get, "http://example.com/csv-metadata.json").to_return(:status => 404)
5
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, body: "")
6
+ stub_request(:get, "http://example.com/.well-known/csvm").to_return(status: 404)
7
+ stub_request(:get, "http://example.com/example.csv-metadata.json").to_return(status: 404)
8
+ stub_request(:get, "http://example.com/csv-metadata.json").to_return(status: 404)
10
9
  end
11
10
 
12
11
  it "should validate from a URL" do
13
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
12
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
14
13
  validator = Csvlint::Validator.new("http://example.com/example.csv")
15
14
 
16
15
  expect(validator.valid?).to eql(true)
17
- expect(validator.instance_variable_get("@expected_columns")).to eql(3)
18
- expect(validator.instance_variable_get("@col_counts").count).to eql(3)
16
+ expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
17
+ expect(validator.instance_variable_get(:@col_counts).count).to eql(3)
19
18
  expect(validator.data.size).to eql(3)
20
19
  end
21
20
 
22
21
  it "should validate from a file path" do
23
- validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
22
+ validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
24
23
 
25
24
  expect(validator.valid?).to eql(true)
26
- expect(validator.instance_variable_get("@expected_columns")).to eql(3)
27
- expect(validator.instance_variable_get("@col_counts").count).to eql(3)
25
+ expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
26
+ expect(validator.instance_variable_get(:@col_counts).count).to eql(3)
28
27
  expect(validator.data.size).to eql(3)
29
28
  end
30
29
 
31
30
  it "should validate from a file path including whitespace" do
32
- validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),'..','features','fixtures','white space in filename.csv')))
31
+ validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "white space in filename.csv")))
33
32
 
34
33
  expect(validator.valid?).to eql(true)
35
34
  end
36
35
 
37
36
  context "multi line CSV validation with included schema" do
38
-
39
37
  end
40
38
 
41
39
  context "single line row validation with included schema" do
42
-
43
40
  end
44
41
 
45
42
  context "validation with multiple lines: " do
46
-
47
43
  # TODO multiple lines permits testing of warnings
48
44
  # TODO need more assertions in each test IE @formats
49
45
  # TODO the phrasing of col_counts if only consulting specs might be confusing
@@ -52,22 +48,21 @@ describe Csvlint::Validator do
52
48
 
53
49
  it ".each() -> parse_contents method validates a well formed CSV" do
54
50
  # when invoking parse contents
55
- data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"})
51
+ data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"))
56
52
 
57
53
  validator = Csvlint::Validator.new(data)
58
54
 
59
55
  expect(validator.valid?).to eql(true)
60
56
  # TODO would be beneficial to know how formats functions WRT to headers - check_format.feature:17 returns 3 rows total
61
57
  # TODO in its formats object but is provided with 5 rows (with one nil row) [uses validation_warnings_steps.rb]
62
- expect(validator.instance_variable_get("@expected_columns")).to eql(3)
63
- expect(validator.instance_variable_get("@col_counts").count).to eql(4)
58
+ expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
59
+ expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
64
60
  expect(validator.data.size).to eql(4)
65
-
66
61
  end
67
62
 
68
63
  it ".each() -> `parse_contents` parses malformed CSV and catches unclosed quote" do
69
64
  # doesn't build warnings because check_consistency isn't invoked
70
- data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1})
65
+ data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1))
71
66
 
72
67
  validator = Csvlint::Validator.new(data)
73
68
 
@@ -79,7 +74,7 @@ describe Csvlint::Validator do
79
74
  it ".each() -> `parse_contents` parses malformed CSV and catches whitespace and edge case" do
80
75
  # when this data gets passed the header it rescues a whitespace error, resulting in the header row being discarded
81
76
  # TODO - check if this is an edge case, currently passing because it requires advice on how to specify
82
- data = StringIO.new(%Q{ "Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1" })
77
+ data = StringIO.new(%( "Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1" ))
83
78
 
84
79
  validator = Csvlint::Validator.new(data)
85
80
 
@@ -89,13 +84,13 @@ describe Csvlint::Validator do
89
84
  end
90
85
 
91
86
  it "handles line breaks within a cell" do
92
- data = StringIO.new(%Q{"a","b","c"\r\n"d","e","this is\r\nvalid"\r\n"a","b","c"})
87
+ data = StringIO.new(%("a","b","c"\r\n"d","e","this is\r\nvalid"\r\n"a","b","c"))
93
88
  validator = Csvlint::Validator.new(data)
94
89
  expect(validator.valid?).to eql(true)
95
90
  end
96
91
 
97
92
  it "handles multiple line breaks within a cell" do
98
- data = StringIO.new(%Q{"a","b","c"\r\n"d","this is\r\n valid","as is this\r\n too"})
93
+ data = StringIO.new(%("a","b","c"\r\n"d","this is\r\n valid","as is this\r\n too"))
99
94
  validator = Csvlint::Validator.new(data)
100
95
  expect(validator.valid?).to eql(true)
101
96
  end
@@ -104,50 +99,50 @@ describe Csvlint::Validator do
104
99
  context "csv dialect" do
105
100
  it "should provide sensible defaults for CSV parsing" do
106
101
  validator = Csvlint::Validator.new("http://example.com/example.csv")
107
- opts = validator.instance_variable_get("@csv_options")
102
+ opts = validator.instance_variable_get(:@csv_options)
108
103
  expect(opts).to include({
109
- :col_sep => ",",
110
- :row_sep => :auto,
111
- :quote_char => '"',
112
- :skip_blanks => false
104
+ col_sep: ",",
105
+ row_sep: :auto,
106
+ quote_char: '"',
107
+ skip_blanks: false
113
108
  })
114
109
  end
115
110
 
116
111
  it "should map CSV DDF to correct values" do
117
112
  validator = Csvlint::Validator.new("http://example.com/example.csv")
118
- opts = validator.dialect_to_csv_options( {
113
+ opts = validator.dialect_to_csv_options({
119
114
  "lineTerminator" => "\n",
120
115
  "delimiter" => "\t",
121
116
  "quoteChar" => "'"
122
117
  })
123
118
  expect(opts).to include({
124
- :col_sep => "\t",
125
- :row_sep => "\n",
126
- :quote_char => "'",
127
- :skip_blanks => false
119
+ col_sep: "\t",
120
+ row_sep: "\n",
121
+ quote_char: "'",
122
+ skip_blanks: false
128
123
  })
129
124
  end
130
125
 
131
126
  it ".each() -> `validate` to pass input in streaming fashion" do
132
127
  # warnings are built when validate is used to call all three methods
133
- data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"})
128
+ data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"))
134
129
  validator = Csvlint::Validator.new(data)
135
130
 
136
131
  expect(validator.valid?).to eql(true)
137
- expect(validator.instance_variable_get("@expected_columns")).to eql(3)
138
- expect(validator.instance_variable_get("@col_counts").count).to eql(4)
132
+ expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
133
+ expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
139
134
  expect(validator.data.size).to eql(4)
140
135
  expect(validator.info_messages.count).to eql(1)
141
136
  end
142
137
 
143
138
  it ".each() -> `validate` parses malformed CSV, populates errors, warnings & info_msgs,invokes finish()" do
144
- data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"1","two","3"\r\n"3","2", "1"})
139
+ data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"1","two","3"\r\n"3","2", "1"))
145
140
 
146
141
  validator = Csvlint::Validator.new(data)
147
142
 
148
143
  expect(validator.valid?).to eql(false)
149
- expect(validator.instance_variable_get("@expected_columns")).to eql(3)
150
- expect(validator.instance_variable_get("@col_counts").count).to eql(4)
144
+ expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
145
+ expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
151
146
  expect(validator.data.size).to eql(5)
152
147
  expect(validator.info_messages.count).to eql(1)
153
148
  expect(validator.errors.count).to eql(1)
@@ -157,7 +152,7 @@ describe Csvlint::Validator do
157
152
  end
158
153
 
159
154
  it "File.open.each_line -> `validate` passes a valid csv" do
160
- filename = 'valid_many_rows.csv'
155
+ filename = "valid_many_rows.csv"
161
156
  file = File.join(File.expand_path(Dir.pwd), "features", "fixtures", filename)
162
157
  validator = Csvlint::Validator.new(File.new(file))
163
158
 
@@ -166,11 +161,9 @@ describe Csvlint::Validator do
166
161
  expect(validator.info_messages.first.type).to eql(:assumed_header)
167
162
  expect(validator.info_messages.first.category).to eql(:structure)
168
163
  end
169
-
170
164
  end
171
165
 
172
166
  context "with a single row" do
173
-
174
167
  it "validates correctly" do
175
168
  stream = "\"a\",\"b\",\"c\"\r\n"
176
169
  validator = Csvlint::Validator.new(StringIO.new(stream), "header" => false)
@@ -204,10 +197,10 @@ describe Csvlint::Validator do
204
197
  stream = "1,2,3\r\n"
205
198
  validator = Csvlint::Validator.new(StringIO.new(stream))
206
199
 
207
- expect( validator.valid? ).to eql(true)
208
- expect( validator.info_messages.size ).to eql(1)
209
- expect( validator.info_messages.first.type).to eql(:assumed_header)
210
- expect( validator.info_messages.first.category).to eql(:structure)
200
+ expect(validator.valid?).to eql(true)
201
+ expect(validator.info_messages.size).to eql(1)
202
+ expect(validator.info_messages.first.type).to eql(:assumed_header)
203
+ expect(validator.info_messages.first.category).to eql(:structure)
211
204
  end
212
205
 
213
206
  it "should evaluate the row as 'row 2' when stipulated" do
@@ -217,11 +210,9 @@ describe Csvlint::Validator do
217
210
  expect(validator.valid?).to eql(true)
218
211
  expect(validator.info_messages.size).to eql(0)
219
212
  end
220
-
221
213
  end
222
214
 
223
215
  context "it returns the correct error from ERROR_MATCHES" do
224
-
225
216
  it "checks for unclosed quotes" do
226
217
  stream = "\"a,\"b\",\"c\"\n"
227
218
  validator = Csvlint::Validator.new(StringIO.new(stream))
@@ -230,7 +221,6 @@ describe Csvlint::Validator do
230
221
  expect(validator.errors.first.type).to eql(:unclosed_quote)
231
222
  end
232
223
 
233
-
234
224
  # TODO stray quotes is not covered in any spec in this library
235
225
  # it "checks for stray quotes" do
236
226
  # stream = "\"a\",“b“,\"c\"" "\r\n"
@@ -258,64 +248,61 @@ describe Csvlint::Validator do
258
248
  expect(validator.errors.count).to eq(1)
259
249
  expect(validator.errors.first.type).to eql(:line_breaks)
260
250
  end
261
-
262
251
  end
263
252
 
264
253
  context "when validating headers" do
265
-
266
254
  it "should warn if column names aren't unique" do
267
- data = StringIO.new( "minimum, minimum" )
255
+ data = StringIO.new("minimum, minimum")
268
256
  validator = Csvlint::Validator.new(data)
269
257
  validator.reset
270
- expect( validator.validate_header(["minimum", "minimum"]) ).to eql(true)
271
- expect( validator.warnings.size ).to eql(1)
272
- expect( validator.warnings.first.type).to eql(:duplicate_column_name)
273
- expect( validator.warnings.first.category).to eql(:schema)
258
+ expect(validator.validate_header(["minimum", "minimum"])).to eql(true)
259
+ expect(validator.warnings.size).to eql(1)
260
+ expect(validator.warnings.first.type).to eql(:duplicate_column_name)
261
+ expect(validator.warnings.first.category).to eql(:schema)
274
262
  end
275
263
 
276
264
  it "should warn if column names are blank" do
277
- data = StringIO.new( "minimum," )
265
+ data = StringIO.new("minimum,")
278
266
  validator = Csvlint::Validator.new(data)
279
267
 
280
- expect( validator.validate_header(["minimum", ""]) ).to eql(true)
281
- expect( validator.warnings.size ).to eql(1)
282
- expect( validator.warnings.first.type).to eql(:empty_column_name)
283
- expect( validator.warnings.first.category).to eql(:schema)
268
+ expect(validator.validate_header(["minimum", ""])).to eql(true)
269
+ expect(validator.warnings.size).to eql(1)
270
+ expect(validator.warnings.first.type).to eql(:empty_column_name)
271
+ expect(validator.warnings.first.category).to eql(:schema)
284
272
  end
285
273
 
286
274
  it "should include info message about missing header when we have assumed a header" do
287
- data = StringIO.new( "1,2,3\r\n" )
275
+ data = StringIO.new("1,2,3\r\n")
288
276
  validator = Csvlint::Validator.new(data)
289
- expect( validator.valid? ).to eql(true)
290
- expect( validator.info_messages.size ).to eql(1)
291
- expect( validator.info_messages.first.type).to eql(:assumed_header)
292
- expect( validator.info_messages.first.category).to eql(:structure)
277
+ expect(validator.valid?).to eql(true)
278
+ expect(validator.info_messages.size).to eql(1)
279
+ expect(validator.info_messages.first.type).to eql(:assumed_header)
280
+ expect(validator.info_messages.first.category).to eql(:structure)
293
281
  end
294
282
 
295
283
  it "should not include info message about missing header when we are told about the header" do
296
- data = StringIO.new( "1,2,3\r\n" )
284
+ data = StringIO.new("1,2,3\r\n")
297
285
  validator = Csvlint::Validator.new(data, "header" => false)
298
- expect( validator.valid? ).to eql(true)
299
- expect( validator.info_messages.size ).to eql(0)
286
+ expect(validator.valid?).to eql(true)
287
+ expect(validator.info_messages.size).to eql(0)
300
288
  end
301
289
  end
302
290
 
303
291
  context "build_formats" do
304
-
305
292
  {
306
- :string => "foo",
307
- :numeric => "1",
308
- :uri => "http://www.example.com",
309
- :dateTime_iso8601 => "2013-01-01T13:00:00Z",
310
- :date_db => "2013-01-01",
311
- :dateTime_hms => "13:00:00"
293
+ string: "foo",
294
+ numeric: "1",
295
+ uri: "http://www.example.com",
296
+ dateTime_iso8601: "2013-01-01T13:00:00Z",
297
+ date_db: "2013-01-01",
298
+ dateTime_hms: "13:00:00"
312
299
  }.each do |type, content|
313
300
  it "should return the format of #{type} correctly" do
314
301
  row = [content]
315
302
 
316
303
  validator = Csvlint::Validator.new("http://example.com/example.csv")
317
304
  validator.build_formats(row)
318
- formats = validator.instance_variable_get("@formats")
305
+ formats = validator.instance_variable_get(:@formats)
319
306
 
320
307
  expect(formats[0].keys.first).to eql type
321
308
  end
@@ -326,7 +313,7 @@ describe Csvlint::Validator do
326
313
 
327
314
  validator = Csvlint::Validator.new("http://example.com/example.csv")
328
315
  validator.build_formats(row)
329
- formats = validator.instance_variable_get("@formats")
316
+ formats = validator.instance_variable_get(:@formats)
330
317
 
331
318
  expect(formats[0].keys.first).to eql :numeric
332
319
  expect(formats[1].keys.first).to eql :numeric
@@ -338,15 +325,15 @@ describe Csvlint::Validator do
338
325
  validator = Csvlint::Validator.new("http://example.com/example.csv")
339
326
  validator.build_formats(row)
340
327
 
341
- formats = validator.instance_variable_get("@formats")
328
+ formats = validator.instance_variable_get(:@formats)
342
329
  expect(formats).to eql []
343
330
  end
344
331
 
345
332
  it "should work correctly for single columns" do
346
333
  rows = [
347
- ["foo"],
348
- ["bar"],
349
- ["baz"]
334
+ ["foo"],
335
+ ["bar"],
336
+ ["baz"]
350
337
  ]
351
338
 
352
339
  validator = Csvlint::Validator.new("http://example.com/example.csv")
@@ -355,14 +342,14 @@ describe Csvlint::Validator do
355
342
  validator.build_formats(row)
356
343
  end
357
344
 
358
- formats = validator.instance_variable_get("@formats")
359
- expect(formats).to eql [{:string => 3}]
345
+ formats = validator.instance_variable_get(:@formats)
346
+ expect(formats).to eql [{string: 3}]
360
347
  end
361
348
 
362
349
  it "should return formats correctly if a row is blank" do
363
350
  rows = [
364
- [],
365
- ["foo", "1", "$2345"]
351
+ [],
352
+ ["foo", "1", "$2345"]
366
353
  ]
367
354
 
368
355
  validator = Csvlint::Validator.new("http://example.com/example.csv")
@@ -371,72 +358,68 @@ describe Csvlint::Validator do
371
358
  validator.build_formats(row)
372
359
  end
373
360
 
374
- formats = validator.instance_variable_get("@formats")
361
+ formats = validator.instance_variable_get(:@formats)
375
362
 
376
363
  expect(formats).to eql [
377
- {:string => 1},
378
- {:numeric => 1},
379
- {:string => 1},
364
+ {string: 1},
365
+ {numeric: 1},
366
+ {string: 1}
380
367
  ]
381
368
  end
382
-
383
369
  end
384
370
 
385
371
  context "csv dialect" do
386
372
  it "should provide sensible defaults for CSV parsing" do
387
373
  validator = Csvlint::Validator.new("http://example.com/example.csv")
388
- opts = validator.instance_variable_get("@csv_options")
374
+ opts = validator.instance_variable_get(:@csv_options)
389
375
  expect(opts).to include({
390
- :col_sep => ",",
391
- :row_sep => :auto,
392
- :quote_char => '"',
393
- :skip_blanks => false
394
- })
376
+ col_sep: ",",
377
+ row_sep: :auto,
378
+ quote_char: '"',
379
+ skip_blanks: false
380
+ })
395
381
  end
396
382
 
397
383
  it "should map CSV DDF to correct values" do
398
384
  validator = Csvlint::Validator.new("http://example.com/example.csv")
399
385
  opts = validator.dialect_to_csv_options({
400
- "lineTerminator" => "\n",
401
- "delimiter" => "\t",
402
- "quoteChar" => "'"
403
- })
386
+ "lineTerminator" => "\n",
387
+ "delimiter" => "\t",
388
+ "quoteChar" => "'"
389
+ })
404
390
  expect(opts).to include({
405
- :col_sep => "\t",
406
- :row_sep => "\n",
407
- :quote_char => "'",
408
- :skip_blanks => false
409
- })
391
+ col_sep: "\t",
392
+ row_sep: "\n",
393
+ quote_char: "'",
394
+ skip_blanks: false
395
+ })
410
396
  end
411
-
412
397
  end
413
398
 
414
399
  context "check_consistency" do
415
-
416
400
  it "should return a warning if columns have inconsistent values" do
417
401
  formats = [
418
- {:string => 3},
419
- {:string => 2, :numeric => 1},
420
- {:numeric => 3},
402
+ {string: 3},
403
+ {string: 2, numeric: 1},
404
+ {numeric: 3}
421
405
  ]
422
406
 
423
407
  validator = Csvlint::Validator.new("http://example.com/example.csv")
424
- validator.instance_variable_set("@formats", formats)
408
+ validator.instance_variable_set(:@formats, formats)
425
409
  validator.check_consistency
426
410
 
427
- warnings = validator.instance_variable_get("@warnings")
411
+ warnings = validator.instance_variable_get(:@warnings)
428
412
  warnings.delete_if { |h| h.type != :inconsistent_values }
429
413
 
430
414
  expect(warnings.count).to eql 1
431
415
  end
432
-
433
416
  end
434
417
 
435
- #TODO the below tests are all the remaining tests from validator_spec.rb, annotations indicate their status HOWEVER these tests may be best refactored into client specs
418
+ # TODO the below tests are all the remaining tests from validator_spec.rb, annotations indicate their status HOWEVER these tests may be best refactored into client specs
436
419
  context "when detecting headers" do
437
420
  it "should default to expecting a header" do
438
421
  validator = Csvlint::Validator.new("http://example.com/example.csv")
439
- expect( validator.header? ).to eql(true)
422
+ expect(validator.header?).to eql(true)
440
423
  end
441
424
 
442
425
  it "should look in CSV options to detect header" do
@@ -444,171 +427,165 @@ describe Csvlint::Validator do
444
427
  "header" => true
445
428
  }
446
429
  validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
447
- expect( validator.header? ).to eql(true)
430
+ expect(validator.header?).to eql(true)
448
431
  opts = {
449
432
  "header" => false
450
433
  }
451
434
  validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
452
- expect( validator.header? ).to eql(false)
435
+ expect(validator.header?).to eql(false)
453
436
  end
454
437
 
455
438
  it "should look in content-type for header=absent" do
456
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv; header=absent"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
439
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv; header=absent"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
457
440
  validator = Csvlint::Validator.new("http://example.com/example.csv")
458
- expect( validator.header? ).to eql(false)
459
- expect( validator.errors.size ).to eql(0)
441
+ expect(validator.header?).to eql(false)
442
+ expect(validator.errors.size).to eql(0)
460
443
  end
461
444
 
462
445
  it "should look in content-type for header=present" do
463
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv; header=present"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
446
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv; header=present"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
464
447
  validator = Csvlint::Validator.new("http://example.com/example.csv")
465
- expect( validator.header? ).to eql(true)
466
- expect( validator.errors.size ).to eql(0)
448
+ expect(validator.header?).to eql(true)
449
+ expect(validator.errors.size).to eql(0)
467
450
  end
468
451
 
469
452
  it "assume header present if not specified in content type" do
470
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
453
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
471
454
  validator = Csvlint::Validator.new("http://example.com/example.csv")
472
- expect( validator.header? ).to eql(true)
473
- expect( validator.errors.size ).to eql(0)
474
- expect( validator.info_messages.size ).to eql(1)
475
- expect( validator.info_messages.first.type).to eql(:assumed_header)
455
+ expect(validator.header?).to eql(true)
456
+ expect(validator.errors.size).to eql(0)
457
+ expect(validator.info_messages.size).to eql(1)
458
+ expect(validator.info_messages.first.type).to eql(:assumed_header)
476
459
  end
477
460
 
478
461
  it "give wrong content type error if content type is wrong" do
479
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/html"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
462
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/html"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
480
463
  validator = Csvlint::Validator.new("http://example.com/example.csv")
481
- expect( validator.header? ).to eql(true)
482
- expect( validator.errors.size ).to eql(1)
483
- expect( validator.errors[0].type).to eql(:wrong_content_type)
464
+ expect(validator.header?).to eql(true)
465
+ expect(validator.errors.size).to eql(1)
466
+ expect(validator.errors[0].type).to eql(:wrong_content_type)
484
467
  end
485
-
486
468
  end
487
469
 
488
470
  context "when validating headers" do
489
471
  it "should warn if column names aren't unique" do
490
- data = StringIO.new( "minimum, minimum" )
472
+ data = StringIO.new("minimum, minimum")
491
473
  validator = Csvlint::Validator.new(data)
492
- expect( validator.warnings.size ).to eql(1)
493
- expect( validator.warnings.first.type).to eql(:duplicate_column_name)
494
- expect( validator.warnings.first.category).to eql(:schema)
474
+ expect(validator.warnings.size).to eql(1)
475
+ expect(validator.warnings.first.type).to eql(:duplicate_column_name)
476
+ expect(validator.warnings.first.category).to eql(:schema)
495
477
  end
496
478
 
497
479
  it "should warn if column names are blank" do
498
- data = StringIO.new( "minimum," )
480
+ data = StringIO.new("minimum,")
499
481
  validator = Csvlint::Validator.new(data)
500
482
 
501
- expect( validator.validate_header(["minimum", ""]) ).to eql(true)
502
- expect( validator.warnings.size ).to eql(1)
503
- expect( validator.warnings.first.type).to eql(:empty_column_name)
504
- expect( validator.warnings.first.category).to eql(:schema)
483
+ expect(validator.validate_header(["minimum", ""])).to eql(true)
484
+ expect(validator.warnings.size).to eql(1)
485
+ expect(validator.warnings.first.type).to eql(:empty_column_name)
486
+ expect(validator.warnings.first.category).to eql(:schema)
505
487
  end
506
488
 
507
489
  it "should include info message about missing header when we have assumed a header" do
508
- data = StringIO.new( "1,2,3\r\n" )
490
+ data = StringIO.new("1,2,3\r\n")
509
491
  validator = Csvlint::Validator.new(data)
510
492
 
511
- expect( validator.valid? ).to eql(true)
512
- expect( validator.info_messages.size ).to eql(1)
513
- expect( validator.info_messages.first.type).to eql(:assumed_header)
514
- expect( validator.info_messages.first.category).to eql(:structure)
493
+ expect(validator.valid?).to eql(true)
494
+ expect(validator.info_messages.size).to eql(1)
495
+ expect(validator.info_messages.first.type).to eql(:assumed_header)
496
+ expect(validator.info_messages.first.category).to eql(:structure)
515
497
  end
516
498
 
517
499
  it "should not include info message about missing header when we are told about the header" do
518
- data = StringIO.new( "1,2,3\r\n" )
519
- validator = Csvlint::Validator.new(data, "header"=>false)
520
- expect( validator.valid? ).to eql(true)
521
- expect( validator.info_messages.size ).to eql(0)
500
+ data = StringIO.new("1,2,3\r\n")
501
+ validator = Csvlint::Validator.new(data, "header" => false)
502
+ expect(validator.valid?).to eql(true)
503
+ expect(validator.info_messages.size).to eql(0)
522
504
  end
523
505
 
524
506
  it "should not be an error if we have assumed a header, there is no dialect and content-type doesn't declare header, as we assume header=present" do
525
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
507
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
526
508
  validator = Csvlint::Validator.new("http://example.com/example.csv")
527
- expect( validator.valid? ).to eql(true)
509
+ expect(validator.valid?).to eql(true)
528
510
  end
529
511
 
530
512
  it "should be valid if we have a dialect and the data is from the web" do
531
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
532
- #header defaults to true in csv dialect, so this is valid
513
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
514
+ # header defaults to true in csv dialect, so this is valid
533
515
  validator = Csvlint::Validator.new("http://example.com/example.csv", {})
534
- expect( validator.valid? ).to eql(true)
516
+ expect(validator.valid?).to eql(true)
535
517
 
536
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
537
- validator = Csvlint::Validator.new("http://example.com/example.csv", {"header"=>true})
538
- expect( validator.valid? ).to eql(true)
518
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
519
+ validator = Csvlint::Validator.new("http://example.com/example.csv", {"header" => true})
520
+ expect(validator.valid?).to eql(true)
539
521
 
540
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
541
- validator = Csvlint::Validator.new("http://example.com/example.csv", {"header"=>false})
542
- expect( validator.valid? ).to eql(true)
522
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
523
+ validator = Csvlint::Validator.new("http://example.com/example.csv", {"header" => false})
524
+ expect(validator.valid?).to eql(true)
543
525
  end
544
-
545
526
  end
546
527
 
547
528
  context "accessing metadata" do
548
-
549
529
  before :all do
550
- stub_request(:get, "http://example.com/crlf.csv").to_return(:status => 200, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','windows-line-endings.csv')))
551
- stub_request(:get, "http://example.com/crlf.csv-metadata.json").to_return(:status => 404)
530
+ stub_request(:get, "http://example.com/crlf.csv").to_return(status: 200, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "windows-line-endings.csv")))
531
+ stub_request(:get, "http://example.com/crlf.csv-metadata.json").to_return(status: 404)
552
532
  end
553
533
 
554
534
  it "can get line break symbol" do
555
535
  validator = Csvlint::Validator.new("http://example.com/crlf.csv")
556
536
  expect(validator.line_breaks).to eql "\r\n"
557
537
  end
558
-
559
538
  end
560
539
 
561
540
  it "should give access to the complete CSV data file" do
562
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200,
563
- :headers=>{"Content-Type" => "text/csv; header=present"},
564
- :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
541
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
542
+ headers: {"Content-Type" => "text/csv; header=present"},
543
+ body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
565
544
  validator = Csvlint::Validator.new("http://example.com/example.csv")
566
- expect( validator.valid? ).to eql(true)
545
+ expect(validator.valid?).to eql(true)
567
546
  data = validator.data
568
547
 
569
- expect( data.count ).to eql 3
570
- expect( data[0] ).to eql ['Foo','Bar','Baz']
571
- expect( data[2] ).to eql ['3','2','1']
548
+ expect(data.count).to eql 3
549
+ expect(data[0]).to eql ["Foo", "Bar", "Baz"]
550
+ expect(data[2]).to eql ["3", "2", "1"]
572
551
  end
573
552
 
574
553
  it "should count the total number of rows read" do
575
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200,
576
- :headers=>{"Content-Type" => "text/csv; header=present"},
577
- :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
554
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
555
+ headers: {"Content-Type" => "text/csv; header=present"},
556
+ body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
578
557
  validator = Csvlint::Validator.new("http://example.com/example.csv")
579
558
  expect(validator.row_count).to eq(3)
580
559
  end
581
560
 
582
561
  it "should limit number of lines read" do
583
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200,
584
- :headers=>{"Content-Type" => "text/csv; header=present"},
585
- :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
562
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
563
+ headers: {"Content-Type" => "text/csv; header=present"},
564
+ body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
586
565
  validator = Csvlint::Validator.new("http://example.com/example.csv", {}, nil, limit_lines: 2)
587
- expect( validator.valid? ).to eql(true)
566
+ expect(validator.valid?).to eql(true)
588
567
  data = validator.data
589
- expect( data.count ).to eql 2
590
- expect( data[0] ).to eql ['Foo','Bar','Baz']
568
+ expect(data.count).to eql 2
569
+ expect(data[0]).to eql ["Foo", "Bar", "Baz"]
591
570
  end
592
571
 
593
572
  context "with a lambda" do
594
-
595
573
  it "should call a lambda for each line" do
596
574
  @count = 0
597
- mylambda = lambda { |row| @count = @count + 1 }
598
- validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')), {}, nil, { lambda: mylambda })
575
+ mylambda = lambda { |row| @count += 1 }
576
+ validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")), {}, nil, {lambda: mylambda})
599
577
  expect(@count).to eq(3)
600
578
  end
601
579
 
602
580
  it "reports back the status of each line" do
603
581
  @results = []
604
582
  mylambda = lambda { |row| @results << row.current_line }
605
- validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')), {}, nil, { lambda: mylambda })
583
+ validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")), {}, nil, {lambda: mylambda})
606
584
  expect(@results.count).to eq(3)
607
585
  expect(@results[0]).to eq(1)
608
586
  expect(@results[1]).to eq(2)
609
587
  expect(@results[2]).to eq(3)
610
588
  end
611
-
612
589
  end
613
590
 
614
591
  # Commented out because there is currently no way to mock redirects with Typhoeus and WebMock - see https://github.com/bblimke/webmock/issues/237