csvlint 0.4.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +5 -5
  2. data/.github/dependabot.yml +11 -0
  3. data/.github/workflows/push.yml +35 -0
  4. data/.gitignore +1 -0
  5. data/.ruby-version +1 -1
  6. data/.standard_todo.yml +43 -0
  7. data/CHANGELOG.md +38 -0
  8. data/Dockerfile +16 -0
  9. data/Gemfile +2 -2
  10. data/README.md +13 -10
  11. data/Rakefile +7 -7
  12. data/bin/create_schema +2 -2
  13. data/csvlint.gemspec +19 -22
  14. data/docker_notes_for_windows.txt +20 -0
  15. data/features/step_definitions/cli_steps.rb +11 -11
  16. data/features/step_definitions/information_steps.rb +4 -4
  17. data/features/step_definitions/parse_csv_steps.rb +11 -11
  18. data/features/step_definitions/schema_validation_steps.rb +10 -10
  19. data/features/step_definitions/sources_steps.rb +1 -1
  20. data/features/step_definitions/validation_errors_steps.rb +19 -19
  21. data/features/step_definitions/validation_info_steps.rb +9 -9
  22. data/features/step_definitions/validation_warnings_steps.rb +11 -11
  23. data/features/support/aruba.rb +10 -9
  24. data/features/support/earl_formatter.rb +39 -39
  25. data/features/support/env.rb +10 -11
  26. data/features/support/load_tests.rb +109 -105
  27. data/features/support/webmock.rb +3 -1
  28. data/lib/csvlint/cli.rb +136 -142
  29. data/lib/csvlint/csvw/column.rb +279 -280
  30. data/lib/csvlint/csvw/date_format.rb +90 -92
  31. data/lib/csvlint/csvw/metadata_error.rb +1 -3
  32. data/lib/csvlint/csvw/number_format.rb +40 -32
  33. data/lib/csvlint/csvw/property_checker.rb +714 -717
  34. data/lib/csvlint/csvw/table.rb +49 -52
  35. data/lib/csvlint/csvw/table_group.rb +24 -23
  36. data/lib/csvlint/error_collector.rb +2 -0
  37. data/lib/csvlint/error_message.rb +0 -1
  38. data/lib/csvlint/field.rb +153 -141
  39. data/lib/csvlint/schema.rb +35 -43
  40. data/lib/csvlint/validate.rb +173 -151
  41. data/lib/csvlint/version.rb +1 -1
  42. data/lib/csvlint.rb +22 -23
  43. data/spec/csvw/column_spec.rb +15 -16
  44. data/spec/csvw/date_format_spec.rb +5 -7
  45. data/spec/csvw/number_format_spec.rb +2 -4
  46. data/spec/csvw/table_group_spec.rb +103 -105
  47. data/spec/csvw/table_spec.rb +71 -73
  48. data/spec/field_spec.rb +116 -121
  49. data/spec/schema_spec.rb +131 -141
  50. data/spec/spec_helper.rb +6 -6
  51. data/spec/validator_spec.rb +167 -203
  52. metadata +41 -85
  53. data/.travis.yml +0 -37
@@ -1,49 +1,45 @@
1
- require 'spec_helper'
1
+ require "spec_helper"
2
2
 
3
3
  describe Csvlint::Validator do
4
-
5
4
  before do
6
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :body => "")
7
- stub_request(:get, "http://example.com/.well-known/csvm").to_return(:status => 404)
8
- stub_request(:get, "http://example.com/example.csv-metadata.json").to_return(:status => 404)
9
- stub_request(:get, "http://example.com/csv-metadata.json").to_return(:status => 404)
5
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, body: "")
6
+ stub_request(:get, "http://example.com/.well-known/csvm").to_return(status: 404)
7
+ stub_request(:get, "http://example.com/example.csv-metadata.json").to_return(status: 404)
8
+ stub_request(:get, "http://example.com/csv-metadata.json").to_return(status: 404)
10
9
  end
11
10
 
12
11
  it "should validate from a URL" do
13
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
12
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
14
13
  validator = Csvlint::Validator.new("http://example.com/example.csv")
15
14
 
16
15
  expect(validator.valid?).to eql(true)
17
- expect(validator.instance_variable_get("@expected_columns")).to eql(3)
18
- expect(validator.instance_variable_get("@col_counts").count).to eql(3)
16
+ expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
17
+ expect(validator.instance_variable_get(:@col_counts).count).to eql(3)
19
18
  expect(validator.data.size).to eql(3)
20
19
  end
21
20
 
22
21
  it "should validate from a file path" do
23
- validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
22
+ validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
24
23
 
25
24
  expect(validator.valid?).to eql(true)
26
- expect(validator.instance_variable_get("@expected_columns")).to eql(3)
27
- expect(validator.instance_variable_get("@col_counts").count).to eql(3)
25
+ expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
26
+ expect(validator.instance_variable_get(:@col_counts).count).to eql(3)
28
27
  expect(validator.data.size).to eql(3)
29
28
  end
30
29
 
31
30
  it "should validate from a file path including whitespace" do
32
- validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),'..','features','fixtures','white space in filename.csv')))
31
+ validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "white space in filename.csv")))
33
32
 
34
33
  expect(validator.valid?).to eql(true)
35
34
  end
36
35
 
37
36
  context "multi line CSV validation with included schema" do
38
-
39
37
  end
40
38
 
41
39
  context "single line row validation with included schema" do
42
-
43
40
  end
44
41
 
45
42
  context "validation with multiple lines: " do
46
-
47
43
  # TODO multiple lines permits testing of warnings
48
44
  # TODO need more assertions in each test IE @formats
49
45
  # TODO the phrasing of col_counts if only consulting specs might be confusing
@@ -52,22 +48,21 @@ describe Csvlint::Validator do
52
48
 
53
49
  it ".each() -> parse_contents method validates a well formed CSV" do
54
50
  # when invoking parse contents
55
- data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"})
51
+ data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"))
56
52
 
57
53
  validator = Csvlint::Validator.new(data)
58
54
 
59
55
  expect(validator.valid?).to eql(true)
60
56
  # TODO would be beneficial to know how formats functions WRT to headers - check_format.feature:17 returns 3 rows total
61
57
  # TODO in its formats object but is provided with 5 rows (with one nil row) [uses validation_warnings_steps.rb]
62
- expect(validator.instance_variable_get("@expected_columns")).to eql(3)
63
- expect(validator.instance_variable_get("@col_counts").count).to eql(4)
58
+ expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
59
+ expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
64
60
  expect(validator.data.size).to eql(4)
65
-
66
61
  end
67
62
 
68
63
  it ".each() -> `parse_contents` parses malformed CSV and catches unclosed quote" do
69
64
  # doesn't build warnings because check_consistency isn't invoked
70
- data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1})
65
+ data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1))
71
66
 
72
67
  validator = Csvlint::Validator.new(data)
73
68
 
@@ -76,23 +71,10 @@ describe Csvlint::Validator do
76
71
  expect(validator.errors.first.type).to eql(:unclosed_quote)
77
72
  end
78
73
 
79
- it ".each() -> `parse_contents` parses malformed CSV and catches stray quote" do
80
- pending "cannot make Ruby generate a stray quote error"
81
- # doesn't build warnings because check_consistency isn't invoked
82
- # TODO below is trailing whitespace but is interpreted as a stray quote
83
- data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1""})
84
-
85
- validator = Csvlint::Validator.new(data)
86
-
87
- expect(validator.valid?).to eql(false)
88
- expect(validator.errors.first.type).to eql(:stray_quote)
89
- expect(validator.errors.count).to eql(1)
90
- end
91
-
92
74
  it ".each() -> `parse_contents` parses malformed CSV and catches whitespace and edge case" do
93
75
  # when this data gets passed the header it rescues a whitespace error, resulting in the header row being discarded
94
76
  # TODO - check if this is an edge case, currently passing because it requires advice on how to specify
95
- data = StringIO.new(%Q{ "Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1" })
77
+ data = StringIO.new(%( "Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1" ))
96
78
 
97
79
  validator = Csvlint::Validator.new(data)
98
80
 
@@ -102,13 +84,13 @@ describe Csvlint::Validator do
102
84
  end
103
85
 
104
86
  it "handles line breaks within a cell" do
105
- data = StringIO.new(%Q{"a","b","c"\r\n"d","e","this is\r\nvalid"\r\n"a","b","c"})
87
+ data = StringIO.new(%("a","b","c"\r\n"d","e","this is\r\nvalid"\r\n"a","b","c"))
106
88
  validator = Csvlint::Validator.new(data)
107
89
  expect(validator.valid?).to eql(true)
108
90
  end
109
91
 
110
92
  it "handles multiple line breaks within a cell" do
111
- data = StringIO.new(%Q{"a","b","c"\r\n"d","this is\r\n valid","as is this\r\n too"})
93
+ data = StringIO.new(%("a","b","c"\r\n"d","this is\r\n valid","as is this\r\n too"))
112
94
  validator = Csvlint::Validator.new(data)
113
95
  expect(validator.valid?).to eql(true)
114
96
  end
@@ -117,50 +99,50 @@ describe Csvlint::Validator do
117
99
  context "csv dialect" do
118
100
  it "should provide sensible defaults for CSV parsing" do
119
101
  validator = Csvlint::Validator.new("http://example.com/example.csv")
120
- opts = validator.instance_variable_get("@csv_options")
102
+ opts = validator.instance_variable_get(:@csv_options)
121
103
  expect(opts).to include({
122
- :col_sep => ",",
123
- :row_sep => :auto,
124
- :quote_char => '"',
125
- :skip_blanks => false
104
+ col_sep: ",",
105
+ row_sep: :auto,
106
+ quote_char: '"',
107
+ skip_blanks: false
126
108
  })
127
109
  end
128
110
 
129
111
  it "should map CSV DDF to correct values" do
130
112
  validator = Csvlint::Validator.new("http://example.com/example.csv")
131
- opts = validator.dialect_to_csv_options( {
113
+ opts = validator.dialect_to_csv_options({
132
114
  "lineTerminator" => "\n",
133
115
  "delimiter" => "\t",
134
116
  "quoteChar" => "'"
135
117
  })
136
118
  expect(opts).to include({
137
- :col_sep => "\t",
138
- :row_sep => "\n",
139
- :quote_char => "'",
140
- :skip_blanks => false
119
+ col_sep: "\t",
120
+ row_sep: "\n",
121
+ quote_char: "'",
122
+ skip_blanks: false
141
123
  })
142
124
  end
143
125
 
144
126
  it ".each() -> `validate` to pass input in streaming fashion" do
145
127
  # warnings are built when validate is used to call all three methods
146
- data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"})
128
+ data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1"))
147
129
  validator = Csvlint::Validator.new(data)
148
130
 
149
131
  expect(validator.valid?).to eql(true)
150
- expect(validator.instance_variable_get("@expected_columns")).to eql(3)
151
- expect(validator.instance_variable_get("@col_counts").count).to eql(4)
132
+ expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
133
+ expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
152
134
  expect(validator.data.size).to eql(4)
153
135
  expect(validator.info_messages.count).to eql(1)
154
136
  end
155
137
 
156
138
  it ".each() -> `validate` parses malformed CSV, populates errors, warnings & info_msgs,invokes finish()" do
157
- data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"1","two","3"\r\n"3","2", "1"})
139
+ data = StringIO.new(%("Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"1","two","3"\r\n"3","2", "1"))
158
140
 
159
141
  validator = Csvlint::Validator.new(data)
160
142
 
161
143
  expect(validator.valid?).to eql(false)
162
- expect(validator.instance_variable_get("@expected_columns")).to eql(3)
163
- expect(validator.instance_variable_get("@col_counts").count).to eql(4)
144
+ expect(validator.instance_variable_get(:@expected_columns)).to eql(3)
145
+ expect(validator.instance_variable_get(:@col_counts).count).to eql(4)
164
146
  expect(validator.data.size).to eql(5)
165
147
  expect(validator.info_messages.count).to eql(1)
166
148
  expect(validator.errors.count).to eql(1)
@@ -170,7 +152,7 @@ describe Csvlint::Validator do
170
152
  end
171
153
 
172
154
  it "File.open.each_line -> `validate` passes a valid csv" do
173
- filename = 'valid_many_rows.csv'
155
+ filename = "valid_many_rows.csv"
174
156
  file = File.join(File.expand_path(Dir.pwd), "features", "fixtures", filename)
175
157
  validator = Csvlint::Validator.new(File.new(file))
176
158
 
@@ -179,11 +161,9 @@ describe Csvlint::Validator do
179
161
  expect(validator.info_messages.first.type).to eql(:assumed_header)
180
162
  expect(validator.info_messages.first.category).to eql(:structure)
181
163
  end
182
-
183
164
  end
184
165
 
185
166
  context "with a single row" do
186
-
187
167
  it "validates correctly" do
188
168
  stream = "\"a\",\"b\",\"c\"\r\n"
189
169
  validator = Csvlint::Validator.new(StringIO.new(stream), "header" => false)
@@ -217,10 +197,10 @@ describe Csvlint::Validator do
217
197
  stream = "1,2,3\r\n"
218
198
  validator = Csvlint::Validator.new(StringIO.new(stream))
219
199
 
220
- expect( validator.valid? ).to eql(true)
221
- expect( validator.info_messages.size ).to eql(1)
222
- expect( validator.info_messages.first.type).to eql(:assumed_header)
223
- expect( validator.info_messages.first.category).to eql(:structure)
200
+ expect(validator.valid?).to eql(true)
201
+ expect(validator.info_messages.size).to eql(1)
202
+ expect(validator.info_messages.first.type).to eql(:assumed_header)
203
+ expect(validator.info_messages.first.category).to eql(:structure)
224
204
  end
225
205
 
226
206
  it "should evaluate the row as 'row 2' when stipulated" do
@@ -230,11 +210,9 @@ describe Csvlint::Validator do
230
210
  expect(validator.valid?).to eql(true)
231
211
  expect(validator.info_messages.size).to eql(0)
232
212
  end
233
-
234
213
  end
235
214
 
236
215
  context "it returns the correct error from ERROR_MATCHES" do
237
-
238
216
  it "checks for unclosed quotes" do
239
217
  stream = "\"a,\"b\",\"c\"\n"
240
218
  validator = Csvlint::Validator.new(StringIO.new(stream))
@@ -243,7 +221,6 @@ describe Csvlint::Validator do
243
221
  expect(validator.errors.first.type).to eql(:unclosed_quote)
244
222
  end
245
223
 
246
-
247
224
  # TODO stray quotes is not covered in any spec in this library
248
225
  # it "checks for stray quotes" do
249
226
  # stream = "\"a\",“b“,\"c\"" "\r\n"
@@ -271,64 +248,61 @@ describe Csvlint::Validator do
271
248
  expect(validator.errors.count).to eq(1)
272
249
  expect(validator.errors.first.type).to eql(:line_breaks)
273
250
  end
274
-
275
251
  end
276
252
 
277
253
  context "when validating headers" do
278
-
279
254
  it "should warn if column names aren't unique" do
280
- data = StringIO.new( "minimum, minimum" )
255
+ data = StringIO.new("minimum, minimum")
281
256
  validator = Csvlint::Validator.new(data)
282
257
  validator.reset
283
- expect( validator.validate_header(["minimum", "minimum"]) ).to eql(true)
284
- expect( validator.warnings.size ).to eql(1)
285
- expect( validator.warnings.first.type).to eql(:duplicate_column_name)
286
- expect( validator.warnings.first.category).to eql(:schema)
258
+ expect(validator.validate_header(["minimum", "minimum"])).to eql(true)
259
+ expect(validator.warnings.size).to eql(1)
260
+ expect(validator.warnings.first.type).to eql(:duplicate_column_name)
261
+ expect(validator.warnings.first.category).to eql(:schema)
287
262
  end
288
263
 
289
264
  it "should warn if column names are blank" do
290
- data = StringIO.new( "minimum," )
265
+ data = StringIO.new("minimum,")
291
266
  validator = Csvlint::Validator.new(data)
292
267
 
293
- expect( validator.validate_header(["minimum", ""]) ).to eql(true)
294
- expect( validator.warnings.size ).to eql(1)
295
- expect( validator.warnings.first.type).to eql(:empty_column_name)
296
- expect( validator.warnings.first.category).to eql(:schema)
268
+ expect(validator.validate_header(["minimum", ""])).to eql(true)
269
+ expect(validator.warnings.size).to eql(1)
270
+ expect(validator.warnings.first.type).to eql(:empty_column_name)
271
+ expect(validator.warnings.first.category).to eql(:schema)
297
272
  end
298
273
 
299
274
  it "should include info message about missing header when we have assumed a header" do
300
- data = StringIO.new( "1,2,3\r\n" )
275
+ data = StringIO.new("1,2,3\r\n")
301
276
  validator = Csvlint::Validator.new(data)
302
- expect( validator.valid? ).to eql(true)
303
- expect( validator.info_messages.size ).to eql(1)
304
- expect( validator.info_messages.first.type).to eql(:assumed_header)
305
- expect( validator.info_messages.first.category).to eql(:structure)
277
+ expect(validator.valid?).to eql(true)
278
+ expect(validator.info_messages.size).to eql(1)
279
+ expect(validator.info_messages.first.type).to eql(:assumed_header)
280
+ expect(validator.info_messages.first.category).to eql(:structure)
306
281
  end
307
282
 
308
283
  it "should not include info message about missing header when we are told about the header" do
309
- data = StringIO.new( "1,2,3\r\n" )
284
+ data = StringIO.new("1,2,3\r\n")
310
285
  validator = Csvlint::Validator.new(data, "header" => false)
311
- expect( validator.valid? ).to eql(true)
312
- expect( validator.info_messages.size ).to eql(0)
286
+ expect(validator.valid?).to eql(true)
287
+ expect(validator.info_messages.size).to eql(0)
313
288
  end
314
289
  end
315
290
 
316
291
  context "build_formats" do
317
-
318
292
  {
319
- :string => "foo",
320
- :numeric => "1",
321
- :uri => "http://www.example.com",
322
- :dateTime_iso8601 => "2013-01-01T13:00:00Z",
323
- :date_db => "2013-01-01",
324
- :dateTime_hms => "13:00:00"
293
+ string: "foo",
294
+ numeric: "1",
295
+ uri: "http://www.example.com",
296
+ dateTime_iso8601: "2013-01-01T13:00:00Z",
297
+ date_db: "2013-01-01",
298
+ dateTime_hms: "13:00:00"
325
299
  }.each do |type, content|
326
300
  it "should return the format of #{type} correctly" do
327
301
  row = [content]
328
302
 
329
303
  validator = Csvlint::Validator.new("http://example.com/example.csv")
330
304
  validator.build_formats(row)
331
- formats = validator.instance_variable_get("@formats")
305
+ formats = validator.instance_variable_get(:@formats)
332
306
 
333
307
  expect(formats[0].keys.first).to eql type
334
308
  end
@@ -339,7 +313,7 @@ describe Csvlint::Validator do
339
313
 
340
314
  validator = Csvlint::Validator.new("http://example.com/example.csv")
341
315
  validator.build_formats(row)
342
- formats = validator.instance_variable_get("@formats")
316
+ formats = validator.instance_variable_get(:@formats)
343
317
 
344
318
  expect(formats[0].keys.first).to eql :numeric
345
319
  expect(formats[1].keys.first).to eql :numeric
@@ -351,15 +325,15 @@ describe Csvlint::Validator do
351
325
  validator = Csvlint::Validator.new("http://example.com/example.csv")
352
326
  validator.build_formats(row)
353
327
 
354
- formats = validator.instance_variable_get("@formats")
328
+ formats = validator.instance_variable_get(:@formats)
355
329
  expect(formats).to eql []
356
330
  end
357
331
 
358
332
  it "should work correctly for single columns" do
359
333
  rows = [
360
- ["foo"],
361
- ["bar"],
362
- ["baz"]
334
+ ["foo"],
335
+ ["bar"],
336
+ ["baz"]
363
337
  ]
364
338
 
365
339
  validator = Csvlint::Validator.new("http://example.com/example.csv")
@@ -368,14 +342,14 @@ describe Csvlint::Validator do
368
342
  validator.build_formats(row)
369
343
  end
370
344
 
371
- formats = validator.instance_variable_get("@formats")
372
- expect(formats).to eql [{:string => 3}]
345
+ formats = validator.instance_variable_get(:@formats)
346
+ expect(formats).to eql [{string: 3}]
373
347
  end
374
348
 
375
349
  it "should return formats correctly if a row is blank" do
376
350
  rows = [
377
- [],
378
- ["foo", "1", "$2345"]
351
+ [],
352
+ ["foo", "1", "$2345"]
379
353
  ]
380
354
 
381
355
  validator = Csvlint::Validator.new("http://example.com/example.csv")
@@ -384,72 +358,68 @@ describe Csvlint::Validator do
384
358
  validator.build_formats(row)
385
359
  end
386
360
 
387
- formats = validator.instance_variable_get("@formats")
361
+ formats = validator.instance_variable_get(:@formats)
388
362
 
389
363
  expect(formats).to eql [
390
- {:string => 1},
391
- {:numeric => 1},
392
- {:string => 1},
364
+ {string: 1},
365
+ {numeric: 1},
366
+ {string: 1}
393
367
  ]
394
368
  end
395
-
396
369
  end
397
370
 
398
371
  context "csv dialect" do
399
372
  it "should provide sensible defaults for CSV parsing" do
400
373
  validator = Csvlint::Validator.new("http://example.com/example.csv")
401
- opts = validator.instance_variable_get("@csv_options")
374
+ opts = validator.instance_variable_get(:@csv_options)
402
375
  expect(opts).to include({
403
- :col_sep => ",",
404
- :row_sep => :auto,
405
- :quote_char => '"',
406
- :skip_blanks => false
407
- })
376
+ col_sep: ",",
377
+ row_sep: :auto,
378
+ quote_char: '"',
379
+ skip_blanks: false
380
+ })
408
381
  end
409
382
 
410
383
  it "should map CSV DDF to correct values" do
411
384
  validator = Csvlint::Validator.new("http://example.com/example.csv")
412
385
  opts = validator.dialect_to_csv_options({
413
- "lineTerminator" => "\n",
414
- "delimiter" => "\t",
415
- "quoteChar" => "'"
416
- })
386
+ "lineTerminator" => "\n",
387
+ "delimiter" => "\t",
388
+ "quoteChar" => "'"
389
+ })
417
390
  expect(opts).to include({
418
- :col_sep => "\t",
419
- :row_sep => "\n",
420
- :quote_char => "'",
421
- :skip_blanks => false
422
- })
391
+ col_sep: "\t",
392
+ row_sep: "\n",
393
+ quote_char: "'",
394
+ skip_blanks: false
395
+ })
423
396
  end
424
-
425
397
  end
426
398
 
427
399
  context "check_consistency" do
428
-
429
400
  it "should return a warning if columns have inconsistent values" do
430
401
  formats = [
431
- {:string => 3},
432
- {:string => 2, :numeric => 1},
433
- {:numeric => 3},
402
+ {string: 3},
403
+ {string: 2, numeric: 1},
404
+ {numeric: 3}
434
405
  ]
435
406
 
436
407
  validator = Csvlint::Validator.new("http://example.com/example.csv")
437
- validator.instance_variable_set("@formats", formats)
408
+ validator.instance_variable_set(:@formats, formats)
438
409
  validator.check_consistency
439
410
 
440
- warnings = validator.instance_variable_get("@warnings")
411
+ warnings = validator.instance_variable_get(:@warnings)
441
412
  warnings.delete_if { |h| h.type != :inconsistent_values }
442
413
 
443
414
  expect(warnings.count).to eql 1
444
415
  end
445
-
446
416
  end
447
417
 
448
- #TODO the below tests are all the remaining tests from validator_spec.rb, annotations indicate their status HOWEVER these tests may be best refactored into client specs
418
+ # TODO the below tests are all the remaining tests from validator_spec.rb, annotations indicate their status HOWEVER these tests may be best refactored into client specs
449
419
  context "when detecting headers" do
450
420
  it "should default to expecting a header" do
451
421
  validator = Csvlint::Validator.new("http://example.com/example.csv")
452
- expect( validator.header? ).to eql(true)
422
+ expect(validator.header?).to eql(true)
453
423
  end
454
424
 
455
425
  it "should look in CSV options to detect header" do
@@ -457,171 +427,165 @@ describe Csvlint::Validator do
457
427
  "header" => true
458
428
  }
459
429
  validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
460
- expect( validator.header? ).to eql(true)
430
+ expect(validator.header?).to eql(true)
461
431
  opts = {
462
432
  "header" => false
463
433
  }
464
434
  validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
465
- expect( validator.header? ).to eql(false)
435
+ expect(validator.header?).to eql(false)
466
436
  end
467
437
 
468
438
  it "should look in content-type for header=absent" do
469
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv; header=absent"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
439
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv; header=absent"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
470
440
  validator = Csvlint::Validator.new("http://example.com/example.csv")
471
- expect( validator.header? ).to eql(false)
472
- expect( validator.errors.size ).to eql(0)
441
+ expect(validator.header?).to eql(false)
442
+ expect(validator.errors.size).to eql(0)
473
443
  end
474
444
 
475
445
  it "should look in content-type for header=present" do
476
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv; header=present"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
446
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv; header=present"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
477
447
  validator = Csvlint::Validator.new("http://example.com/example.csv")
478
- expect( validator.header? ).to eql(true)
479
- expect( validator.errors.size ).to eql(0)
448
+ expect(validator.header?).to eql(true)
449
+ expect(validator.errors.size).to eql(0)
480
450
  end
481
451
 
482
452
  it "assume header present if not specified in content type" do
483
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
453
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
484
454
  validator = Csvlint::Validator.new("http://example.com/example.csv")
485
- expect( validator.header? ).to eql(true)
486
- expect( validator.errors.size ).to eql(0)
487
- expect( validator.info_messages.size ).to eql(1)
488
- expect( validator.info_messages.first.type).to eql(:assumed_header)
455
+ expect(validator.header?).to eql(true)
456
+ expect(validator.errors.size).to eql(0)
457
+ expect(validator.info_messages.size).to eql(1)
458
+ expect(validator.info_messages.first.type).to eql(:assumed_header)
489
459
  end
490
460
 
491
461
  it "give wrong content type error if content type is wrong" do
492
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/html"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
462
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/html"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
493
463
  validator = Csvlint::Validator.new("http://example.com/example.csv")
494
- expect( validator.header? ).to eql(true)
495
- expect( validator.errors.size ).to eql(1)
496
- expect( validator.errors[0].type).to eql(:wrong_content_type)
464
+ expect(validator.header?).to eql(true)
465
+ expect(validator.errors.size).to eql(1)
466
+ expect(validator.errors[0].type).to eql(:wrong_content_type)
497
467
  end
498
-
499
468
  end
500
469
 
501
470
  context "when validating headers" do
502
471
  it "should warn if column names aren't unique" do
503
- data = StringIO.new( "minimum, minimum" )
472
+ data = StringIO.new("minimum, minimum")
504
473
  validator = Csvlint::Validator.new(data)
505
- expect( validator.warnings.size ).to eql(1)
506
- expect( validator.warnings.first.type).to eql(:duplicate_column_name)
507
- expect( validator.warnings.first.category).to eql(:schema)
474
+ expect(validator.warnings.size).to eql(1)
475
+ expect(validator.warnings.first.type).to eql(:duplicate_column_name)
476
+ expect(validator.warnings.first.category).to eql(:schema)
508
477
  end
509
478
 
510
479
  it "should warn if column names are blank" do
511
- data = StringIO.new( "minimum," )
480
+ data = StringIO.new("minimum,")
512
481
  validator = Csvlint::Validator.new(data)
513
482
 
514
- expect( validator.validate_header(["minimum", ""]) ).to eql(true)
515
- expect( validator.warnings.size ).to eql(1)
516
- expect( validator.warnings.first.type).to eql(:empty_column_name)
517
- expect( validator.warnings.first.category).to eql(:schema)
483
+ expect(validator.validate_header(["minimum", ""])).to eql(true)
484
+ expect(validator.warnings.size).to eql(1)
485
+ expect(validator.warnings.first.type).to eql(:empty_column_name)
486
+ expect(validator.warnings.first.category).to eql(:schema)
518
487
  end
519
488
 
520
489
  it "should include info message about missing header when we have assumed a header" do
521
- data = StringIO.new( "1,2,3\r\n" )
490
+ data = StringIO.new("1,2,3\r\n")
522
491
  validator = Csvlint::Validator.new(data)
523
492
 
524
- expect( validator.valid? ).to eql(true)
525
- expect( validator.info_messages.size ).to eql(1)
526
- expect( validator.info_messages.first.type).to eql(:assumed_header)
527
- expect( validator.info_messages.first.category).to eql(:structure)
493
+ expect(validator.valid?).to eql(true)
494
+ expect(validator.info_messages.size).to eql(1)
495
+ expect(validator.info_messages.first.type).to eql(:assumed_header)
496
+ expect(validator.info_messages.first.category).to eql(:structure)
528
497
  end
529
498
 
530
499
  it "should not include info message about missing header when we are told about the header" do
531
- data = StringIO.new( "1,2,3\r\n" )
532
- validator = Csvlint::Validator.new(data, "header"=>false)
533
- expect( validator.valid? ).to eql(true)
534
- expect( validator.info_messages.size ).to eql(0)
500
+ data = StringIO.new("1,2,3\r\n")
501
+ validator = Csvlint::Validator.new(data, "header" => false)
502
+ expect(validator.valid?).to eql(true)
503
+ expect(validator.info_messages.size).to eql(0)
535
504
  end
536
505
 
537
506
  it "should not be an error if we have assumed a header, there is no dialect and content-type doesn't declare header, as we assume header=present" do
538
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
507
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
539
508
  validator = Csvlint::Validator.new("http://example.com/example.csv")
540
- expect( validator.valid? ).to eql(true)
509
+ expect(validator.valid?).to eql(true)
541
510
  end
542
511
 
543
512
  it "should be valid if we have a dialect and the data is from the web" do
544
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
545
- #header defaults to true in csv dialect, so this is valid
513
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
514
+ # header defaults to true in csv dialect, so this is valid
546
515
  validator = Csvlint::Validator.new("http://example.com/example.csv", {})
547
- expect( validator.valid? ).to eql(true)
516
+ expect(validator.valid?).to eql(true)
548
517
 
549
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
550
- validator = Csvlint::Validator.new("http://example.com/example.csv", {"header"=>true})
551
- expect( validator.valid? ).to eql(true)
518
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
519
+ validator = Csvlint::Validator.new("http://example.com/example.csv", {"header" => true})
520
+ expect(validator.valid?).to eql(true)
552
521
 
553
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
554
- validator = Csvlint::Validator.new("http://example.com/example.csv", {"header"=>false})
555
- expect( validator.valid? ).to eql(true)
522
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200, headers: {"Content-Type" => "text/csv"}, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
523
+ validator = Csvlint::Validator.new("http://example.com/example.csv", {"header" => false})
524
+ expect(validator.valid?).to eql(true)
556
525
  end
557
-
558
526
  end
559
527
 
560
528
  context "accessing metadata" do
561
-
562
529
  before :all do
563
- stub_request(:get, "http://example.com/crlf.csv").to_return(:status => 200, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','windows-line-endings.csv')))
564
- stub_request(:get, "http://example.com/crlf.csv-metadata.json").to_return(:status => 404)
530
+ stub_request(:get, "http://example.com/crlf.csv").to_return(status: 200, body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "windows-line-endings.csv")))
531
+ stub_request(:get, "http://example.com/crlf.csv-metadata.json").to_return(status: 404)
565
532
  end
566
533
 
567
534
  it "can get line break symbol" do
568
535
  validator = Csvlint::Validator.new("http://example.com/crlf.csv")
569
536
  expect(validator.line_breaks).to eql "\r\n"
570
537
  end
571
-
572
538
  end
573
539
 
574
540
  it "should give access to the complete CSV data file" do
575
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200,
576
- :headers=>{"Content-Type" => "text/csv; header=present"},
577
- :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
541
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
542
+ headers: {"Content-Type" => "text/csv; header=present"},
543
+ body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
578
544
  validator = Csvlint::Validator.new("http://example.com/example.csv")
579
- expect( validator.valid? ).to eql(true)
545
+ expect(validator.valid?).to eql(true)
580
546
  data = validator.data
581
547
 
582
- expect( data.count ).to eql 3
583
- expect( data[0] ).to eql ['Foo','Bar','Baz']
584
- expect( data[2] ).to eql ['3','2','1']
548
+ expect(data.count).to eql 3
549
+ expect(data[0]).to eql ["Foo", "Bar", "Baz"]
550
+ expect(data[2]).to eql ["3", "2", "1"]
585
551
  end
586
552
 
587
553
  it "should count the total number of rows read" do
588
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200,
589
- :headers=>{"Content-Type" => "text/csv; header=present"},
590
- :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
554
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
555
+ headers: {"Content-Type" => "text/csv; header=present"},
556
+ body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
591
557
  validator = Csvlint::Validator.new("http://example.com/example.csv")
592
558
  expect(validator.row_count).to eq(3)
593
559
  end
594
560
 
595
561
  it "should limit number of lines read" do
596
- stub_request(:get, "http://example.com/example.csv").to_return(:status => 200,
597
- :headers=>{"Content-Type" => "text/csv; header=present"},
598
- :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
562
+ stub_request(:get, "http://example.com/example.csv").to_return(status: 200,
563
+ headers: {"Content-Type" => "text/csv; header=present"},
564
+ body: File.read(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")))
599
565
  validator = Csvlint::Validator.new("http://example.com/example.csv", {}, nil, limit_lines: 2)
600
- expect( validator.valid? ).to eql(true)
566
+ expect(validator.valid?).to eql(true)
601
567
  data = validator.data
602
- expect( data.count ).to eql 2
603
- expect( data[0] ).to eql ['Foo','Bar','Baz']
568
+ expect(data.count).to eql 2
569
+ expect(data[0]).to eql ["Foo", "Bar", "Baz"]
604
570
  end
605
571
 
606
572
  context "with a lambda" do
607
-
608
573
  it "should call a lambda for each line" do
609
574
  @count = 0
610
- mylambda = lambda { |row| @count = @count + 1 }
611
- validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')), {}, nil, { lambda: mylambda })
575
+ mylambda = lambda { |row| @count += 1 }
576
+ validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")), {}, nil, {lambda: mylambda})
612
577
  expect(@count).to eq(3)
613
578
  end
614
579
 
615
580
  it "reports back the status of each line" do
616
581
  @results = []
617
582
  mylambda = lambda { |row| @results << row.current_line }
618
- validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')), {}, nil, { lambda: mylambda })
583
+ validator = Csvlint::Validator.new(File.new(File.join(File.dirname(__FILE__), "..", "features", "fixtures", "valid.csv")), {}, nil, {lambda: mylambda})
619
584
  expect(@results.count).to eq(3)
620
585
  expect(@results[0]).to eq(1)
621
586
  expect(@results[1]).to eq(2)
622
587
  expect(@results[2]).to eq(3)
623
588
  end
624
-
625
589
  end
626
590
 
627
591
  # Commented out because there is currently no way to mock redirects with Typhoeus and WebMock - see https://github.com/bblimke/webmock/issues/237