csvlint 0.4.0 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (53) hide show
  1. checksums.yaml +5 -5
  2. data/.github/dependabot.yml +11 -0
  3. data/.github/workflows/push.yml +35 -0
  4. data/.gitignore +1 -0
  5. data/.ruby-version +1 -1
  6. data/.standard_todo.yml +43 -0
  7. data/CHANGELOG.md +38 -0
  8. data/Dockerfile +16 -0
  9. data/Gemfile +2 -2
  10. data/README.md +13 -10
  11. data/Rakefile +7 -7
  12. data/bin/create_schema +2 -2
  13. data/csvlint.gemspec +19 -22
  14. data/docker_notes_for_windows.txt +20 -0
  15. data/features/step_definitions/cli_steps.rb +11 -11
  16. data/features/step_definitions/information_steps.rb +4 -4
  17. data/features/step_definitions/parse_csv_steps.rb +11 -11
  18. data/features/step_definitions/schema_validation_steps.rb +10 -10
  19. data/features/step_definitions/sources_steps.rb +1 -1
  20. data/features/step_definitions/validation_errors_steps.rb +19 -19
  21. data/features/step_definitions/validation_info_steps.rb +9 -9
  22. data/features/step_definitions/validation_warnings_steps.rb +11 -11
  23. data/features/support/aruba.rb +10 -9
  24. data/features/support/earl_formatter.rb +39 -39
  25. data/features/support/env.rb +10 -11
  26. data/features/support/load_tests.rb +109 -105
  27. data/features/support/webmock.rb +3 -1
  28. data/lib/csvlint/cli.rb +136 -142
  29. data/lib/csvlint/csvw/column.rb +279 -280
  30. data/lib/csvlint/csvw/date_format.rb +90 -92
  31. data/lib/csvlint/csvw/metadata_error.rb +1 -3
  32. data/lib/csvlint/csvw/number_format.rb +40 -32
  33. data/lib/csvlint/csvw/property_checker.rb +714 -717
  34. data/lib/csvlint/csvw/table.rb +49 -52
  35. data/lib/csvlint/csvw/table_group.rb +24 -23
  36. data/lib/csvlint/error_collector.rb +2 -0
  37. data/lib/csvlint/error_message.rb +0 -1
  38. data/lib/csvlint/field.rb +153 -141
  39. data/lib/csvlint/schema.rb +35 -43
  40. data/lib/csvlint/validate.rb +173 -151
  41. data/lib/csvlint/version.rb +1 -1
  42. data/lib/csvlint.rb +22 -23
  43. data/spec/csvw/column_spec.rb +15 -16
  44. data/spec/csvw/date_format_spec.rb +5 -7
  45. data/spec/csvw/number_format_spec.rb +2 -4
  46. data/spec/csvw/table_group_spec.rb +103 -105
  47. data/spec/csvw/table_spec.rb +71 -73
  48. data/spec/field_spec.rb +116 -121
  49. data/spec/schema_spec.rb +131 -141
  50. data/spec/spec_helper.rb +6 -6
  51. data/spec/validator_spec.rb +167 -203
  52. metadata +41 -85
  53. data/.travis.yml +0 -37
@@ -1,49 +1,45 @@
1
- require 'spec_helper'
1
+ require "spec_helper"
2
2
 
3
3
  describe Csvlint::Validator do
4
-
5
4
  before do
6
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :body => "")
7
- stub_request(:get, "http://example.com/.well-known/csvm").to_return(:status => 404)
8
- stub_request(:get, "http://example.com/example.csv-metadata.json").to_return(:status => 404)
9
- stub_request(:get, "http://example.com/csv-metadata.json").to_return(:status => 404)
5
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, body: "")
6
+ stub_request(:get, "http://example.com/.well-known/csvm").to_return(status: 404)
7
+ stub_request(:get, "http://example.com/example.csv-metadata.json").to_return(status: 404)
8
+ stub_request(:get, "http://example.com/csv-metadata.json").to_return(status: 404)
10
9
  end
11
10
 
12
11
  it "should validate from a URL" do
13
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
12
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
14
13
  validator = Csvlint::Validator.new("http://example.com/example.csv")
15
14
 
16
15
  expect(validator.valid?).to eql(true)
17
- expect(validator.instance_variable_get("@expected_columns")).to eql(3)
18
- expect(validator.instance_variable_get("@col_counts").count).to eql(3)
16
+ expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
17
+ expect(validator.instance_variable_get(:@col_counts).count).to eql(3)
19
18
  expect(validator.data.size).to eql(3)
20
19
  end
21
20
 
22
21
  it "should validate from a file path" do
23
- validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
22
+ validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
24
23
 
25
24
  expect(validator.valid?).to eql(true)
26
- expect(validator.instance_variable_get("@expected_columns")).to eql(3)
27
- expect(validator.instance_variable_get("@col_counts").count).to eql(3)
25
+ expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
26
+ expect(validator.instance_variable_get(:@col_counts).count).to eql(3)
28
27
  expect(validator.data.size).to eql(3)
29
28
  end
30
29
 
31
30
  it "should validate from a file path including whitespace" do
32
- validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),'..','features','fixtures','white space in filename.csv')))
31
+ validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "white space in filename.csv")))
33
32
 
34
33
  expect(validator.valid?).to eql(true)
35
34
  end
36
35
 
37
36
  context "multi line CSV validation with included schema" do
38
-
39
37
  end
40
38
 
41
39
  context "single line row validation with included schema" do
42
-
43
40
  end
44
41
 
45
42
  context "validation with multiple lines: " do
46
-
47
43
  # TODO multiple lines permits testing of warnings
48
44
  # TODO need more assertions in each test IE @formats
49
45
  # TODO the phrasing of col_counts if only consulting specs might be confusing
@@ -52,22 +48,21 @@ describe Csvlint::Validator do
52
48
 
53
49
  it ".each() -> parse_contents method validates a well formed CSV" do
54
50
  # when invoking parse contents
55
- data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"})
51
+ data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"))
56
52
 
57
53
  validator = Csvlint::Validator.new(data)
58
54
 
59
55
  expect(validator.valid?).to eql(true)
60
56
  # TODO would be beneficial to know how formats functions WRT to headers - check_format.feature:17 returns 3 rows total
61
57
  # TODO in its formats object but is provided with 5 rows (with one nil row) [uses validation_warnings_steps.rb]
62
- expect(validator.instance_variable_get("@expected_columns")).to eql(3)
63
- expect(validator.instance_variable_get("@col_counts").count).to eql(4)
58
+ expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
59
+ expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
64
60
  expect(validator.data.size).to eql(4)
65
-
66
61
  end
67
62
 
68
63
  it ".each() -> `parse_contents` parses malformed CSV and catches unclosed quote" do
69
64
  # doesn't build warnings because check_consistency isn't invoked
70
- data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1})
65
+ data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1))
71
66
 
72
67
  validator = Csvlint::Validator.new(data)
73
68
 
@@ -76,23 +71,10 @@ describe Csvlint::Validator do
76
71
  expect(validator.errors.first.type).to eql(:unclosed_quote)
77
72
  end
78
73
 
79
- it ".each() -> `parse_contents` parses malformed CSV and catches stray quote" do
80
- pending "cannot make Ruby generate a stray quote error"
81
- # doesn't build warnings because check_consistency isn't invoked
82
- # TODO below is trailing whitespace but is interpreted as a stray quote
83
- data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1""})
84
-
85
- validator = Csvlint::Validator.new(data)
86
-
87
- expect(validator.valid?).to eql(false)
88
- expect(validator.errors.first.type).to eql(:stray_quote)
89
- expect(validator.errors.count).to eql(1)
90
- end
91
-
92
74
  it ".each() -> `parse_contents` parses malformed CSV and catches whitespace and edge case" do
93
75
  # when this data gets passed the header it rescues a whitespace error, resulting in the header row being discarded
94
76
  # TODO - check if this is an edge case, currently passing because it requires advice on how to specify
95
- data = StringIO.new(%Q{ "Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1" })
77
+ data = StringIO.new(%( "Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1" ))
96
78
 
97
79
  validator = Csvlint::Validator.new(data)
98
80
 
@@ -102,13 +84,13 @@ describe Csvlint::Validator do
102
84
  end
103
85
 
104
86
  it "handles line breaks within a cell" do
105
- data = StringIO.new(%Q{"a","b","c"\r\n"d","e","this is\r\nvalid"\r\n"a","b","c"})
87
+ data = StringIO.new(%("a","b","c"\r\n"d","e","this is\r\nvalid"\r\n"a","b","c"))
106
88
  validator = Csvlint::Validator.new(data)
107
89
  expect(validator.valid?).to eql(true)
108
90
  end
109
91
 
110
92
  it "handles multiple line breaks within a cell" do
111
- data = StringIO.new(%Q{"a","b","c"\r\n"d","this is\r\n valid","as is this\r\n too"})
93
+ data = StringIO.new(%("a","b","c"\r\n"d","this is\r\n valid","as is this\r\n too"))
112
94
  validator = Csvlint::Validator.new(data)
113
95
  expect(validator.valid?).to eql(true)
114
96
  end
@@ -117,50 +99,50 @@ describe Csvlint::Validator do
117
99
  context "csv dialect" do
118
100
  it "should provide sensible defaults for CSV parsing" do
119
101
  validator = Csvlint::Validator.new("http://example.com/example.csv")
120
- opts = validator.instance_variable_get("@csv_options")
102
+ opts = validator.instance_variable_get(:@csv_options)
121
103
  expect(opts).to include({
122
- :col_sep => ",",
123
- :row_sep => :auto,
124
- :quote_char => '"',
125
- :skip_blanks => false
104
+ col_sep: ",",
105
+ row_sep: :auto,
106
+ quote_char: '"',
107
+ skip_blanks: false
126
108
  })
127
109
  end
128
110
 
129
111
  it "should map CSV DDF to correct values" do
130
112
  validator = Csvlint::Validator.new("http://example.com/example.csv")
131
- opts = validator.dialect_to_csv_options( {
113
+ opts = validator.dialect_to_csv_options({
132
114
  "lineTerminator" => "\n",
133
115
  "delimiter" => "\t",
134
116
  "quoteChar" => "'"
135
117
  })
136
118
  expect(opts).to include({
137
- :col_sep => "\t",
138
- :row_sep => "\n",
139
- :quote_char => "'",
140
- :skip_blanks => false
119
+ col_sep: "\t",
120
+ row_sep: "\n",
121
+ quote_char: "'",
122
+ skip_blanks: false
141
123
  })
142
124
  end
143
125
 
144
126
  it ".each() -> `validate` to pass input in streaming fashion" do
145
127
  # warnings are built when validate is used to call all three methods
146
- data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"})
128
+ data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"))
147
129
  validator = Csvlint::Validator.new(data)
148
130
 
149
131
  expect(validator.valid?).to eql(true)
150
- expect(validator.instance_variable_get("@expected_columns")).to eql(3)
151
- expect(validator.instance_variable_get("@col_counts").count).to eql(4)
132
+ expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
133
+ expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
152
134
  expect(validator.data.size).to eql(4)
153
135
  expect(validator.info_messages.count).to eql(1)
154
136
  end
155
137
 
156
138
  it ".each() -> `validate` parses malformed CSV, populates errors, warnings & info_msgs,invokes finish()" do
157
- data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"1","two","3"\r\n"3","2", "1"})
139
+ data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"1","two","3"\r\n"3","2", "1"))
158
140
 
159
141
  validator = Csvlint::Validator.new(data)
160
142
 
161
143
  expect(validator.valid?).to eql(false)
162
- expect(validator.instance_variable_get("@expected_columns")).to eql(3)
163
- expect(validator.instance_variable_get("@col_counts").count).to eql(4)
144
+ expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
145
+ expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
164
146
  expect(validator.data.size).to eql(5)
165
147
  expect(validator.info_messages.count).to eql(1)
166
148
  expect(validator.errors.count).to eql(1)
@@ -170,7 +152,7 @@ describe Csvlint::Validator do
170
152
  end
171
153
 
172
154
  it "File.open.each_line -> `validate` passes a valid csv" do
173
- filename = 'valid_many_rows.csv'
155
+ filename = "valid_many_rows.csv"
174
156
  file = File.join(File.expand_path(Dir.pwd), "features", "fixtures", filename)
175
157
  validator = Csvlint::Validator.new(File.new(file))
176
158
 
@@ -179,11 +161,9 @@ describe Csvlint::Validator do
179
161
  expect(validator.info_messages.first.type).to eql(:assumed_header)
180
162
  expect(validator.info_messages.first.category).to eql(:structure)
181
163
  end
182
-
183
164
  end
184
165
 
185
166
  context "with a single row" do
186
-
187
167
  it "validates correctly" do
188
168
  stream = "\"a\",\"b\",\"c\"\r\n"
189
169
  validator = Csvlint::Validator.new(StringIO.new(stream), "header" => false)
@@ -217,10 +197,10 @@ describe Csvlint::Validator do
217
197
  stream = "1,2,3\r\n"
218
198
  validator = Csvlint::Validator.new(StringIO.new(stream))
219
199
 
220
- expect( validator.valid? ).to eql(true)
221
- expect( validator.info_messages.size ).to eql(1)
222
- expect( validator.info_messages.first.type).to eql(:assumed_header)
223
- expect( validator.info_messages.first.category).to eql(:structure)
200
+ expect(validator.valid?).to eql(true)
201
+ expect(validator.info_messages.size).to eql(1)
202
+ expect(validator.info_messages.first.type).to eql(:assumed_header)
203
+ expect(validator.info_messages.first.category).to eql(:structure)
224
204
  end
225
205
 
226
206
  it "should evaluate the row as 'row 2' when stipulated" do
@@ -230,11 +210,9 @@ describe Csvlint::Validator do
230
210
  expect(validator.valid?).to eql(true)
231
211
  expect(validator.info_messages.size).to eql(0)
232
212
  end
233
-
234
213
  end
235
214
 
236
215
  context "it returns the correct error from ERROR_MATCHES" do
237
-
238
216
  it "checks for unclosed quotes" do
239
217
  stream = "\"a,\"b\",\"c\"\n"
240
218
  validator = Csvlint::Validator.new(StringIO.new(stream))
@@ -243,7 +221,6 @@ describe Csvlint::Validator do
243
221
  expect(validator.errors.first.type).to eql(:unclosed_quote)
244
222
  end
245
223
 
246
-
247
224
  # TODO stray quotes is not covered in any spec in this library
248
225
  # it "checks for stray quotes" do
249
226
  # stream = "\"a\",“b“,\"c\"" "\r\n"
@@ -271,64 +248,61 @@ describe Csvlint::Validator do
271
248
  expect(validator.errors.count).to eq(1)
272
249
  expect(validator.errors.first.type).to eql(:line_breaks)
273
250
  end
274
-
275
251
  end
276
252
 
277
253
  context "when validating headers" do
278
-
279
254
  it "should warn if column names aren't unique" do
280
- data = StringIO.new( "minimum, minimum" )
255
+ data = StringIO.new("minimum, minimum")
281
256
  validator = Csvlint::Validator.new(data)
282
257
  validator.reset
283
- expect( validator.validate_header(["minimum", "minimum"]) ).to eql(true)
284
- expect( validator.warnings.size ).to eql(1)
285
- expect( validator.warnings.first.type).to eql(:duplicate_column_name)
286
- expect( validator.warnings.first.category).to eql(:schema)
258
+ expect(validator.validate_header(["minimum", "minimum"])).to eql(true)
259
+ expect(validator.warnings.size).to eql(1)
260
+ expect(validator.warnings.first.type).to eql(:duplicate_column_name)
261
+ expect(validator.warnings.first.category).to eql(:schema)
287
262
  end
288
263
 
289
264
  it "should warn if column names are blank" do
290
- data = StringIO.new( "minimum," )
265
+ data = StringIO.new("minimum,")
291
266
  validator = Csvlint::Validator.new(data)
292
267
 
293
- expect( validator.validate_header(["minimum", ""]) ).to eql(true)
294
- expect( validator.warnings.size ).to eql(1)
295
- expect( validator.warnings.first.type).to eql(:empty_column_name)
296
- expect( validator.warnings.first.category).to eql(:schema)
268
+ expect(validator.validate_header(["minimum", ""])).to eql(true)
269
+ expect(validator.warnings.size).to eql(1)
270
+ expect(validator.warnings.first.type).to eql(:empty_column_name)
271
+ expect(validator.warnings.first.category).to eql(:schema)
297
272
  end
298
273
 
299
274
  it "should include info message about missing header when we have assumed a header" do
300
- data = StringIO.new( "1,2,3\r\n" )
275
+ data = StringIO.new("1,2,3\r\n")
301
276
  validator = Csvlint::Validator.new(data)
302
- expect( validator.valid? ).to eql(true)
303
- expect( validator.info_messages.size ).to eql(1)
304
- expect( validator.info_messages.first.type).to eql(:assumed_header)
305
- expect( validator.info_messages.first.category).to eql(:structure)
277
+ expect(validator.valid?).to eql(true)
278
+ expect(validator.info_messages.size).to eql(1)
279
+ expect(validator.info_messages.first.type).to eql(:assumed_header)
280
+ expect(validator.info_messages.first.category).to eql(:structure)
306
281
  end
307
282
 
308
283
  it "should not include info message about missing header when we are told about the header" do
309
- data = StringIO.new( "1,2,3\r\n" )
284
+ data = StringIO.new("1,2,3\r\n")
310
285
  validator = Csvlint::Validator.new(data, "header" => false)
311
- expect( validator.valid? ).to eql(true)
312
- expect( validator.info_messages.size ).to eql(0)
286
+ expect(validator.valid?).to eql(true)
287
+ expect(validator.info_messages.size).to eql(0)
313
288
  end
314
289
  end
315
290
 
316
291
  context "build_formats" do
317
-
318
292
  {
319
- :string => "foo",
320
- :numeric => "1",
321
- :uri => "http://www.example.com",
322
- :dateTime_iso8601 => "2013-01-01T13:00:00Z",
323
- :date_db => "2013-01-01",
324
- :dateTime_hms => "13:00:00"
293
+ string: "foo",
294
+ numeric: "1",
295
+ uri: "http://www.example.com",
296
+ dateTime_iso8601: "2013-01-01T13:00:00Z",
297
+ date_db: "2013-01-01",
298
+ dateTime_hms: "13:00:00"
325
299
  }.each do |type, content|
326
300
  it "should return the format of #{type} correctly" do
327
301
  row = [content]
328
302
 
329
303
  validator = Csvlint::Validator.new("http://example.com/example.csv")
330
304
  validator.build_formats(row)
331
- formats = validator.instance_variable_get("@formats")
305
+ formats = validator.instance_variable_get(:@formats)
332
306
 
333
307
  expect(formats[0].keys.first).to eql type
334
308
  end
@@ -339,7 +313,7 @@ describe Csvlint::Validator do
339
313
 
340
314
  validator = Csvlint::Validator.new("http://example.com/example.csv")
341
315
  validator.build_formats(row)
342
- formats = validator.instance_variable_get("@formats")
316
+ formats = validator.instance_variable_get(:@formats)
343
317
 
344
318
  expect(formats[0].keys.first).to eql :numeric
345
319
  expect(formats[1].keys.first).to eql :numeric
@@ -351,15 +325,15 @@ describe Csvlint::Validator do
351
325
  validator = Csvlint::Validator.new("http://example.com/example.csv")
352
326
  validator.build_formats(row)
353
327
 
354
- formats = validator.instance_variable_get("@formats")
328
+ formats = validator.instance_variable_get(:@formats)
355
329
  expect(formats).to eql []
356
330
  end
357
331
 
358
332
  it "should work correctly for single columns" do
359
333
  rows = [
360
- ["foo"],
361
- ["bar"],
362
- ["baz"]
334
+ ["foo"],
335
+ ["bar"],
336
+ ["baz"]
363
337
  ]
364
338
 
365
339
  validator = Csvlint::Validator.new("http://example.com/example.csv")
@@ -368,14 +342,14 @@ describe Csvlint::Validator do
368
342
  validator.build_formats(row)
369
343
  end
370
344
 
371
- formats = validator.instance_variable_get("@formats")
372
- expect(formats).to eql [{:string => 3}]
345
+ formats = validator.instance_variable_get(:@formats)
346
+ expect(formats).to eql [{string: 3}]
373
347
  end
374
348
 
375
349
  it "should return formats correctly if a row is blank" do
376
350
  rows = [
377
- [],
378
- ["foo", "1", "$2345"]
351
+ [],
352
+ ["foo", "1", "$2345"]
379
353
  ]
380
354
 
381
355
  validator = Csvlint::Validator.new("http://example.com/example.csv")
@@ -384,72 +358,68 @@ describe Csvlint::Validator do
384
358
  validator.build_formats(row)
385
359
  end
386
360
 
387
- formats = validator.instance_variable_get("@formats")
361
+ formats = validator.instance_variable_get(:@formats)
388
362
 
389
363
  expect(formats).to eql [
390
- {:string => 1},
391
- {:numeric => 1},
392
- {:string => 1},
364
+ {string: 1},
365
+ {numeric: 1},
366
+ {string: 1}
393
367
  ]
394
368
  end
395
-
396
369
  end
397
370
 
398
371
  context "csv dialect" do
399
372
  it "should provide sensible defaults for CSV parsing" do
400
373
  validator = Csvlint::Validator.new("http://example.com/example.csv")
401
- opts = validator.instance_variable_get("@csv_options")
374
+ opts = validator.instance_variable_get(:@csv_options)
402
375
  expect(opts).to include({
403
- :col_sep => ",",
404
- :row_sep => :auto,
405
- :quote_char => '"',
406
- :skip_blanks => false
407
- })
376
+ col_sep: ",",
377
+ row_sep: :auto,
378
+ quote_char: '"',
379
+ skip_blanks: false
380
+ })
408
381
  end
409
382
 
410
383
  it "should map CSV DDF to correct values" do
411
384
  validator = Csvlint::Validator.new("http://example.com/example.csv")
412
385
  opts = validator.dialect_to_csv_options({
413
- "lineTerminator" => "\n",
414
- "delimiter" => "\t",
415
- "quoteChar" => "'"
416
- })
386
+ "lineTerminator" => "\n",
387
+ "delimiter" => "\t",
388
+ "quoteChar" => "'"
389
+ })
417
390
  expect(opts).to include({
418
- :col_sep => "\t",
419
- :row_sep => "\n",
420
- :quote_char => "'",
421
- :skip_blanks => false
422
- })
391
+ col_sep: "\t",
392
+ row_sep: "\n",
393
+ quote_char: "'",
394
+ skip_blanks: false
395
+ })
423
396
  end
424
-
425
397
  end
426
398
 
427
399
  context "check_consistency" do
428
-
429
400
  it "should return a warning if columns have inconsistent values" do
430
401
  formats = [
431
- {:string => 3},
432
- {:string => 2, :numeric => 1},
433
- {:numeric => 3},
402
+ {string: 3},
403
+ {string: 2, numeric: 1},
404
+ {numeric: 3}
434
405
  ]
435
406
 
436
407
  validator = Csvlint::Validator.new("http://example.com/example.csv")
437
- validator.instance_variable_set("@formats", formats)
408
+ validator.instance_variable_set(:@formats, formats)
438
409
  validator.check_consistency
439
410
 
440
- warnings = validator.instance_variable_get("@warnings")
411
+ warnings = validator.instance_variable_get(:@warnings)
441
412
  warnings.delete_if { |h| h.type != :inconsistent_values }
442
413
 
443
414
  expect(warnings.count).to eql 1
444
415
  end
445
-
446
416
  end
447
417
 
448
- #TODO the below tests are all the remaining tests from validator_spec.rb, annotations indicate their status HOWEVER these tests may be best refactored into client specs
418
+ # TODO the below tests are all the remaining tests from validator_spec.rb, annotations indicate their status HOWEVER these tests may be best refactored into client specs
449
419
  context "when detecting headers" do
450
420
  it "should default to expecting a header" do
451
421
  validator = Csvlint::Validator.new("http://example.com/example.csv")
452
- expect( validator.header? ).to eql(true)
422
+ expect(validator.header?).to eql(true)
453
423
  end
454
424
 
455
425
  it "should look in CSV options to detect header" do
@@ -457,171 +427,165 @@ describe Csvlint::Validator do
457
427
  "header" => true
458
428
  }
459
429
  validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
460
- expect( validator.header? ).to eql(true)
430
+ expect(validator.header?).to eql(true)
461
431
  opts = {
462
432
  "header" => false
463
433
  }
464
434
  validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
465
- expect( validator.header? ).to eql(false)
435
+ expect(validator.header?).to eql(false)
466
436
  end
467
437
 
468
438
  it "should look in content-type for header=absent" do
469
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv; header=absent"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
439
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv; header=absent"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
470
440
  validator = Csvlint::Validator.new("http://example.com/example.csv")
471
- expect( validator.header? ).to eql(false)
472
- expect( validator.errors.size ).to eql(0)
441
+ expect(validator.header?).to eql(false)
442
+ expect(validator.errors.size).to eql(0)
473
443
  end
474
444
 
475
445
  it "should look in content-type for header=present" do
476
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv; header=present"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
446
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv; header=present"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
477
447
  validator = Csvlint::Validator.new("http://example.com/example.csv")
478
- expect( validator.header? ).to eql(true)
479
- expect( validator.errors.size ).to eql(0)
448
+ expect(validator.header?).to eql(true)
449
+ expect(validator.errors.size).to eql(0)
480
450
  end
481
451
 
482
452
  it "assume header present if not specified in content type" do
483
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
453
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
484
454
  validator = Csvlint::Validator.new("http://example.com/example.csv")
485
- expect( validator.header? ).to eql(true)
486
- expect( validator.errors.size ).to eql(0)
487
- expect( validator.info_messages.size ).to eql(1)
488
- expect( validator.info_messages.first.type).to eql(:assumed_header)
455
+ expect(validator.header?).to eql(true)
456
+ expect(validator.errors.size).to eql(0)
457
+ expect(validator.info_messages.size).to eql(1)
458
+ expect(validator.info_messages.first.type).to eql(:assumed_header)
489
459
  end
490
460
 
491
461
  it "give wrong content type error if content type is wrong" do
492
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/html"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
462
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/html"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
493
463
  validator = Csvlint::Validator.new("http://example.com/example.csv")
494
- expect( validator.header? ).to eql(true)
495
- expect( validator.errors.size ).to eql(1)
496
- expect( validator.errors[0].type).to eql(:wrong_content_type)
464
+ expect(validator.header?).to eql(true)
465
+ expect(validator.errors.size).to eql(1)
466
+ expect(validator.errors[0].type).to eql(:wrong_content_type)
497
467
  end
498
-
499
468
  end
500
469
 
501
470
  context "when validating headers" do
502
471
  it "should warn if column names aren't unique" do
503
- data = StringIO.new( "minimum, minimum" )
472
+ data = StringIO.new("minimum, minimum")
504
473
  validator = Csvlint::Validator.new(data)
505
- expect( validator.warnings.size ).to eql(1)
506
- expect( validator.warnings.first.type).to eql(:duplicate_column_name)
507
- expect( validator.warnings.first.category).to eql(:schema)
474
+ expect(validator.warnings.size).to eql(1)
475
+ expect(validator.warnings.first.type).to eql(:duplicate_column_name)
476
+ expect(validator.warnings.first.category).to eql(:schema)
508
477
  end
509
478
 
510
479
  it "should warn if column names are blank" do
511
- data = StringIO.new( "minimum," )
480
+ data = StringIO.new("minimum,")
512
481
  validator = Csvlint::Validator.new(data)
513
482
 
514
- expect( validator.validate_header(["minimum", ""]) ).to eql(true)
515
- expect( validator.warnings.size ).to eql(1)
516
- expect( validator.warnings.first.type).to eql(:empty_column_name)
517
- expect( validator.warnings.first.category).to eql(:schema)
483
+ expect(validator.validate_header(["minimum", ""])).to eql(true)
484
+ expect(validator.warnings.size).to eql(1)
485
+ expect(validator.warnings.first.type).to eql(:empty_column_name)
486
+ expect(validator.warnings.first.category).to eql(:schema)
518
487
  end
519
488
 
520
489
  it "should include info message about missing header when we have assumed a header" do
521
- data = StringIO.new( "1,2,3\r\n" )
490
+ data = StringIO.new("1,2,3\r\n")
522
491
  validator = Csvlint::Validator.new(data)
523
492
 
524
- expect( validator.valid? ).to eql(true)
525
- expect( validator.info_messages.size ).to eql(1)
526
- expect( validator.info_messages.first.type).to eql(:assumed_header)
527
- expect( validator.info_messages.first.category).to eql(:structure)
493
+ expect(validator.valid?).to eql(true)
494
+ expect(validator.info_messages.size).to eql(1)
495
+ expect(validator.info_messages.first.type).to eql(:assumed_header)
496
+ expect(validator.info_messages.first.category).to eql(:structure)
528
497
  end
529
498
 
530
499
  it "should not include info message about missing header when we are told about the header" do
531
- data = StringIO.new( "1,2,3\r\n" )
532
- validator = Csvlint::Validator.new(data, "header"=>false)
533
- expect( validator.valid? ).to eql(true)
534
- expect( validator.info_messages.size ).to eql(0)
500
+ data = StringIO.new("1,2,3\r\n")
501
+ validator = Csvlint::Validator.new(data, "header" => false)
502
+ expect(validator.valid?).to eql(true)
503
+ expect(validator.info_messages.size).to eql(0)
535
504
  end
536
505
 
537
506
  it "should not be an error if we have assumed a header, there is no dialect and content-type doesn't declare header, as we assume header=present" do
538
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
507
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
539
508
  validator = Csvlint::Validator.new("http://example.com/example.csv")
540
- expect( validator.valid? ).to eql(true)
509
+ expect(validator.valid?).to eql(true)
541
510
  end
542
511
 
543
512
  it "should be valid if we have a dialect and the data is from the web" do
544
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
545
- #header defaults to true in csv dialect, so this is valid
513
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
514
+ # header defaults to true in csv dialect, so this is valid
546
515
  validator = Csvlint::Validator.new("http://example.com/example.csv", {})
547
- expect( validator.valid? ).to eql(true)
516
+ expect(validator.valid?).to eql(true)
548
517
 
549
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
550
- validator = Csvlint::Validator.new("http://example.com/example.csv", {"header"=>true})
551
- expect( validator.valid? ).to eql(true)
518
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
519
+ validator = Csvlint::Validator.new("http://example.com/example.csv", {"header" => true})
520
+ expect(validator.valid?).to eql(true)
552
521
 
553
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
554
- validator = Csvlint::Validator.new("http://example.com/example.csv", {"header"=>false})
555
- expect( validator.valid? ).to eql(true)
522
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
523
+ validator = Csvlint::Validator.new("http://example.com/example.csv", {"header" => false})
524
+ expect(validator.valid?).to eql(true)
556
525
  end
557
-
558
526
  end
559
527
 
560
528
  context "accessing metadata" do
561
-
562
529
  before :all do
563
- stub_request(:get, "http://example.com/crlf.csv").to_return(:status => 200, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','windows-line-endings.csv')))
564
- stub_request(:get, "http://example.com/crlf.csv-metadata.json").to_return(:status => 404)
530
+ stub_request(:get, "http://example.com/crlf.csv").to_return(status: 200, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "windows-line-endings.csv")))
531
+ stub_request(:get, "http://example.com/crlf.csv-metadata.json").to_return(status: 404)
565
532
  end
566
533
 
567
534
  it "can get line break symbol" do
568
535
  validator = Csvlint::Validator.new("http://example.com/crlf.csv")
569
536
  expect(validator.line_breaks).to eql "\r\n"
570
537
  end
571
-
572
538
  end
573
539
 
574
540
  it "should give access to the complete CSV data file" do
575
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200,
576
- :headers=>{"Content-Type" => "text/csv; header=present"},
577
- :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
541
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
542
+ headers: {"Content-Type" => "text/csv; header=present"},
543
+ body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
578
544
  validator = Csvlint::Validator.new("http://example.com/example.csv")
579
- expect( validator.valid? ).to eql(true)
545
+ expect(validator.valid?).to eql(true)
580
546
  data = validator.data
581
547
 
582
- expect( data.count ).to eql 3
583
- expect( data[0] ).to eql ['Foo','Bar','Baz']
584
- expect( data[2] ).to eql ['3','2','1']
548
+ expect(data.count).to eql 3
549
+ expect(data[0]).to eql ["Foo", "Bar", "Baz"]
550
+ expect(data[2]).to eql ["3", "2", "1"]
585
551
  end
586
552
 
587
553
  it "should count the total number of rows read" do
588
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200,
589
- :headers=>{"Content-Type" => "text/csv; header=present"},
590
- :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
554
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
555
+ headers: {"Content-Type" => "text/csv; header=present"},
556
+ body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
591
557
  validator = Csvlint::Validator.new("http://example.com/example.csv")
592
558
  expect(validator.row_count).to eq(3)
593
559
  end
594
560
 
595
561
  it "should limit number of lines read" do
596
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200,
597
- :headers=>{"Content-Type" => "text/csv; header=present"},
598
- :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
562
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
563
+ headers: {"Content-Type" => "text/csv; header=present"},
564
+ body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
599
565
  validator = Csvlint::Validator.new("http://example.com/example.csv", {}, nil, limit_lines: 2)
600
- expect( validator.valid? ).to eql(true)
566
+ expect(validator.valid?).to eql(true)
601
567
  data = validator.data
602
- expect( data.count ).to eql 2
603
- expect( data[0] ).to eql ['Foo','Bar','Baz']
568
+ expect(data.count).to eql 2
569
+ expect(data[0]).to eql ["Foo", "Bar", "Baz"]
604
570
  end
605
571
 
606
572
  context "with a lambda" do
607
-
608
573
  it "should call a lambda for each line" do
609
574
  @count = 0
610
- mylambda = lambda { |row| @count = @count + 1 }
611
- validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')), {}, nil, { lambda: mylambda })
575
+ mylambda = lambda { |row| @count += 1 }
576
+ validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")), {}, nil, {lambda: mylambda})
612
577
  expect(@count).to eq(3)
613
578
  end
614
579
 
615
580
  it "reports back the status of each line" do
616
581
  @results = []
617
582
  mylambda = lambda { |row| @results << row.current_line }
618
- validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')), {}, nil, { lambda: mylambda })
583
+ validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")), {}, nil, {lambda: mylambda})
619
584
  expect(@results.count).to eq(3)
620
585
  expect(@results[0]).to eq(1)
621
586
  expect(@results[1]).to eq(2)
622
587
  expect(@results[2]).to eq(3)
623
588
  end
624
-
625
589
  end
626
590
 
627
591
  # Commented out because there is currently no way to mock redirects with Typhoeus and WebMock - see https://github.com/bblimke/webmock/issues/237