csvlint 0.0.1 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +13 -5
- data/.travis.yml +21 -3
- data/README.md +1 -1
- data/features/step_definitions/validation_info_steps.rb +2 -2
- data/features/validation_info.feature +5 -5
- data/lib/csvlint.rb +0 -1
- data/lib/csvlint/error_collector.rb +15 -33
- data/lib/csvlint/error_message.rb +9 -10
- data/lib/csvlint/types.rb +58 -34
- data/lib/csvlint/validate.rb +32 -32
- data/lib/csvlint/version.rb +1 -1
- data/lib/csvlint/wrapped_io.rb +13 -31
- data/spec/validator_spec.rb +33 -10
- metadata +37 -37
checksums.yaml
CHANGED
@@ -1,7 +1,15 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
NzY4MzE5MGI1OWI4YzkzY2U1MTZlMWJjNDM4YzM5ZjNiMWI5ZWIyMA==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
OTM5MWJmYzNkYTBiMjM4ZGU3Yjc2NTY0Njk4OTEzMDJhNmI2Yjg5Mg==
|
5
7
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
YzU4NzJmZTlkYWE1NTY0MGY4NzY0N2Q1NDYxMzI1N2YxYjZhZThmNDZhMDVj
|
10
|
+
Mjc1YmMyZGZkODhlNDE4OWZiYWM5NmVjNTNiOGRmMjI0OGY4ZjJkY2I4OTNl
|
11
|
+
ZmYwMGMwN2YwOWNlYmUzNjZjNWU3OTEzYTIwZDYzMGMzZDBiOGQ=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
YTkwNWVlNjlmZGZkZDJjNmUyZGQxMzBkYzViOTc1NzlhNjYzYjU3ZDljMmRl
|
14
|
+
YzExNDFkMDUxOThkMjJkNDRkZWYxZjNlMzdmYTRmNjI1MTliNGYzOWQxYzY3
|
15
|
+
MGIxMjA1YzhkYjUwOWZkOGU2NjViYjNlNjBhMTY4NTY0N2E4MmI=
|
data/.travis.yml
CHANGED
@@ -3,8 +3,26 @@ rvm:
|
|
3
3
|
notifications:
|
4
4
|
irc:
|
5
5
|
channels:
|
6
|
-
|
6
|
+
- irc.freenode.net#theodi
|
7
7
|
template:
|
8
|
-
|
8
|
+
- '%{repository} %{branch} - %{message} %{build_url}'
|
9
9
|
on_success: change
|
10
|
-
on_failure: always
|
10
|
+
on_failure: always
|
11
|
+
deploy:
|
12
|
+
provider: rubygems
|
13
|
+
api_key:
|
14
|
+
secure: fjnPS61/skQ1PsRJu1SYWIct0vjdkTyQhj0ria9zfcJfbWbfiHnpehVh1ege3sTUkDSTvoOFT35jzEeozvtKZOlAWMU5QbL8LTXu+JSp9olOdSuGeWRWVuQT3NLgRJW0+2c7N66piZvnRUUTyt2P8VIR8c/Ltuhc32bUGL7X6Gw=
|
15
|
+
gem: csvlint
|
16
|
+
on:
|
17
|
+
tags: true
|
18
|
+
repo: theodi/csvlint.rb
|
19
|
+
all_branches: true
|
20
|
+
after_success:
|
21
|
+
- export GEM_VERSION=`ruby -e "puts Gem::Specification.load(Dir['*.gemspec'].first).version.to_s"`
|
22
|
+
- '[ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ] && curl
|
23
|
+
-v -X POST -d ''{"ref":"refs/tags/''$GEM_VERSION''","sha":"''$TRAVIS_COMMIT''"}'' --header
|
24
|
+
"Content-Type:application/json" -u $GITHUB_USER:$GITHUB_PASSWORD "https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs"'
|
25
|
+
env:
|
26
|
+
global:
|
27
|
+
- secure: hM8Pv/EAZoB4fGGEUwKQQ/2+mgKbUac+PH2cwud25l2MJ64eoKEMZzfy4TF6+XMIzXhlWpsg4s51y6K3+W8pUyRAuwo2MsN5Iee7HnLz+LlYTiT0//iQdAmxQ0JlOyh8vd2SEFBcVfwpp/iZFIHAfbBO73ZDXhtGMEieMk025I0=
|
28
|
+
- secure: hsKbZ0kVKVU/e8oVCPJpRmBxbbieN2tVbyjdhJo/UYchcyLWeGAmAj/ApUDL4SyR0HZxpfi+SIsGFNRy5LnrFeCFgY9aZ6u9ma4n0Y2cTElMRt5kvI/c28FehsRjCzAe6W8dxwhJ8wSkvFDNpLml47/iy8bw4a7aUZDdq3OYXck=
|
data/README.md
CHANGED
@@ -157,7 +157,7 @@ An example schema file is:
|
|
157
157
|
|
158
158
|
Parsing and validating with a schema:
|
159
159
|
|
160
|
-
schema = Schema.load_from_json_table(uri)
|
160
|
+
schema = Csvlint::Schema.load_from_json_table(uri)
|
161
161
|
validator = Csvlint::Validator.new( "http://example.org/data.csv", nil, schema )
|
162
162
|
|
163
163
|
Supported constraints:
|
@@ -13,6 +13,6 @@ Then(/^there should be (\d+) info messages?$/) do |num|
|
|
13
13
|
@info_messages.count.should == num.to_i
|
14
14
|
end
|
15
15
|
|
16
|
-
Then(/^
|
17
|
-
@info_messages.
|
16
|
+
Then(/^one of the messages should have the type "(.*?)"$/) do |msg_type|
|
17
|
+
@info_messages.find{|x| x.type == msg_type.to_sym}.should be_present
|
18
18
|
end
|
@@ -5,20 +5,20 @@ Feature: Get validation information messages
|
|
5
5
|
And it is stored at the url "http://example.com/example1.csv"
|
6
6
|
And I set header to "true"
|
7
7
|
And I ask if there are info messages
|
8
|
-
Then there should be
|
9
|
-
And
|
8
|
+
Then there should be 2 info messages
|
9
|
+
And one of the messages should have the type "nonrfc_line_breaks"
|
10
10
|
|
11
11
|
Scenario: CR line endings in file give an info message
|
12
12
|
Given I have a CSV file called "cr-line-endings.csv"
|
13
13
|
And it is stored at the url "http://example.com/example1.csv"
|
14
14
|
And I set header to "true"
|
15
15
|
And I ask if there are info messages
|
16
|
-
Then there should be
|
17
|
-
And
|
16
|
+
Then there should be 2 info messages
|
17
|
+
And one of the messages should have the type "nonrfc_line_breaks"
|
18
18
|
|
19
19
|
Scenario: CRLF line endings in file produces no info messages
|
20
20
|
Given I have a CSV file called "crlf-line-endings.csv"
|
21
21
|
And it is stored at the url "http://example.com/example1.csv"
|
22
22
|
And I set header to "true"
|
23
23
|
And I ask if there are info messages
|
24
|
-
Then there should be
|
24
|
+
Then there should be 1 info message
|
data/lib/csvlint.rb
CHANGED
@@ -1,43 +1,25 @@
|
|
1
1
|
module Csvlint
|
2
|
-
|
3
2
|
module ErrorCollector
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
:category => category,
|
9
|
-
:row => row,
|
10
|
-
:column => column,
|
11
|
-
:content => content,
|
12
|
-
:constraints => constraints
|
13
|
-
})
|
3
|
+
attr_reader :errors, :warnings, :info_messages
|
4
|
+
|
5
|
+
def build_errors(type, category = nil, row = nil, column = nil, content = nil, constraints = {})
|
6
|
+
@errors << Csvlint::ErrorMessage.new(type, category, row, column, content, constraints)
|
14
7
|
end
|
15
|
-
|
16
|
-
|
17
|
-
:errors,
|
18
|
-
:warnings,
|
19
|
-
:info_messages
|
20
|
-
]
|
21
|
-
|
22
|
-
MESSAGE_LEVELS.each do |level|
|
23
|
-
|
24
|
-
attr_reader level
|
25
|
-
|
26
|
-
define_method "build_#{level}" do |type, category = nil, row = nil, column = nil, content = nil, constraints = {}|
|
27
|
-
instance_variable_get("@#{level}") << build_message(type, category, row, column, content, constraints)
|
28
|
-
end
|
29
|
-
|
8
|
+
def build_warnings(type, category = nil, row = nil, column = nil, content = nil, constraints = {})
|
9
|
+
@warnings << Csvlint::ErrorMessage.new(type, category, row, column, content, constraints)
|
30
10
|
end
|
31
|
-
|
11
|
+
def build_info_messages(type, category = nil, row = nil, column = nil, content = nil, constraints = {})
|
12
|
+
@info_messages << Csvlint::ErrorMessage.new(type, category, row, column, content, constraints)
|
13
|
+
end
|
14
|
+
|
32
15
|
def valid?
|
33
16
|
errors.empty?
|
34
17
|
end
|
35
|
-
|
18
|
+
|
36
19
|
def reset
|
37
|
-
|
38
|
-
|
39
|
-
|
20
|
+
@errors = []
|
21
|
+
@warnings = []
|
22
|
+
@info_messages = []
|
40
23
|
end
|
41
|
-
|
42
24
|
end
|
43
|
-
end
|
25
|
+
end
|
@@ -1,15 +1,14 @@
|
|
1
1
|
module Csvlint
|
2
|
-
|
3
2
|
class ErrorMessage
|
4
|
-
|
5
3
|
attr_reader :type, :category, :row, :column, :content, :constraints
|
6
|
-
|
7
|
-
def initialize(
|
8
|
-
|
9
|
-
|
10
|
-
|
4
|
+
|
5
|
+
def initialize(type, category, row, column, content, constraints)
|
6
|
+
@type = type
|
7
|
+
@category = category
|
8
|
+
@row = row
|
9
|
+
@column = column
|
10
|
+
@content = content
|
11
|
+
@constraints = constraints
|
11
12
|
end
|
12
|
-
|
13
13
|
end
|
14
|
-
|
15
|
-
end
|
14
|
+
end
|
data/lib/csvlint/types.rb
CHANGED
@@ -4,54 +4,79 @@ require 'active_support/core_ext/date/conversions'
|
|
4
4
|
require 'active_support/core_ext/time/conversions'
|
5
5
|
|
6
6
|
module Csvlint
|
7
|
-
|
8
7
|
module Types
|
9
|
-
|
10
8
|
SIMPLE_FORMATS = {
|
11
|
-
'string'
|
12
|
-
'numeric'
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
9
|
+
'string' => lambda { |value| true },
|
10
|
+
'numeric' => lambda { |value| value.strip[/\A[-+]?\d*\.?\d+(?:[eE][-+]?\d+)?\z/] },
|
11
|
+
'uri' => lambda do |value|
|
12
|
+
if value.strip[/\Ahttps?:/]
|
13
|
+
u = URI.parse(value)
|
14
|
+
u.kind_of?(URI::HTTP) || u.kind_of?(URI::HTTPS)
|
17
15
|
end
|
18
|
-
end,
|
19
|
-
'uri' => lambda do |value, constraints|
|
20
|
-
u = URI.parse value
|
21
|
-
raise ArgumentError unless u.kind_of?(URI::HTTP) || u.kind_of?(URI::HTTPS)
|
22
|
-
u
|
23
16
|
end
|
24
17
|
}
|
25
|
-
|
26
|
-
def self.date_format(klass
|
27
|
-
|
28
|
-
|
18
|
+
|
19
|
+
def self.date_format(klass, value, format, pattern)
|
20
|
+
if value[pattern]
|
21
|
+
klass.strptime(value, format).strftime(format) == value
|
22
|
+
end
|
29
23
|
end
|
30
|
-
|
24
|
+
|
31
25
|
def self.included(base)
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
26
|
+
[
|
27
|
+
[ :db, "%Y-%m-%d",
|
28
|
+
/\A\d{4,}-\d\d-\d\d\z/],
|
29
|
+
[ :number, "%Y%m%d",
|
30
|
+
/\A\d{8}\z/],
|
31
|
+
[ :short, "%e %b",
|
32
|
+
/\A[ \d]\d (?:#{Date::ABBR_MONTHNAMES.join('|')})\z/],
|
33
|
+
[ :rfc822, "%e %b %Y",
|
34
|
+
/\A[ \d]\d (?:#{Date::ABBR_MONTHNAMES.join('|')}) \d{4,}\z/],
|
35
|
+
[ :long, "%B %e, %Y",
|
36
|
+
/\A(?:#{Date::MONTHNAMES.join('|')}) [ \d]\d, \d{4,}\z/],
|
37
|
+
].each do |type,format,pattern|
|
38
|
+
SIMPLE_FORMATS["date_#{type}"] = lambda do |value|
|
39
|
+
date_format(Date, value, format, pattern)
|
38
40
|
end
|
39
41
|
end
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
42
|
+
|
43
|
+
# strptime doesn't support widths like %9N, unlike strftime.
|
44
|
+
# @see http://ruby-doc.org/stdlib-2.0/libdoc/date/rdoc/DateTime.html
|
45
|
+
[
|
46
|
+
[ :time, "%H:%M",
|
47
|
+
/\A\d\d:\d\d\z/],
|
48
|
+
[ :hms, "%H:%M:%S",
|
49
|
+
/\A\d\d:\d\d:\d\d\z/],
|
50
|
+
[ :db, "%Y-%m-%d %H:%M:%S",
|
51
|
+
/\A\d{4,}-\d\d-\d\d \d\d:\d\d:\d\d\z/],
|
52
|
+
[ :iso8601, "%Y-%m-%dT%H:%M:%SZ",
|
53
|
+
/\A\d{4,}-\d\d-\d\dT\d\d:\d\d:\d\dZ\z/],
|
54
|
+
[ :number, "%Y%m%d%H%M%S",
|
55
|
+
/\A\d{14}\z/],
|
56
|
+
[ :nsec, "%Y%m%d%H%M%S%N",
|
57
|
+
/\A\d{23}\z/],
|
58
|
+
[ :short, "%d %b %H:%M",
|
59
|
+
/\A\d\d (?:#{Date::ABBR_MONTHNAMES.join('|')}) \d\d:\d\d\z/],
|
60
|
+
[ :long, "%B %d, %Y %H:%M",
|
61
|
+
/\A(?:#{Date::MONTHNAMES.join('|')}) \d\d, \d{4,} \d\d:\d\d\z/],
|
62
|
+
].each do |type,format,pattern|
|
63
|
+
SIMPLE_FORMATS["dateTime_#{type}"] = lambda do |value|
|
64
|
+
date_format(Time, value, format, pattern)
|
44
65
|
end
|
45
66
|
end
|
46
67
|
end
|
47
|
-
|
68
|
+
|
48
69
|
TYPE_VALIDATIONS = {
|
49
|
-
'http://www.w3.org/2001/XMLSchema#string' =>
|
70
|
+
'http://www.w3.org/2001/XMLSchema#string' => lambda { |value, constraints| value },
|
50
71
|
'http://www.w3.org/2001/XMLSchema#int' => lambda { |value, constraints| Integer value },
|
51
72
|
'http://www.w3.org/2001/XMLSchema#integer' => lambda { |value, constraints| Integer value },
|
52
73
|
'http://www.w3.org/2001/XMLSchema#float' => lambda { |value, constraints| Float value },
|
53
74
|
'http://www.w3.org/2001/XMLSchema#double' => lambda { |value, constraints| Float value },
|
54
|
-
'http://www.w3.org/2001/XMLSchema#anyURI' =>
|
75
|
+
'http://www.w3.org/2001/XMLSchema#anyURI' => lambda do |value, constraints|
|
76
|
+
u = URI.parse value
|
77
|
+
raise ArgumentError unless u.kind_of?(URI::HTTP) || u.kind_of?(URI::HTTPS)
|
78
|
+
u
|
79
|
+
end,
|
55
80
|
'http://www.w3.org/2001/XMLSchema#boolean' => lambda do |value, constraints|
|
56
81
|
return true if ['true', '1'].include? value
|
57
82
|
return false if ['false', '0'].include? value
|
@@ -100,14 +125,13 @@ module Csvlint
|
|
100
125
|
d = Date.strptime(value, date_pattern)
|
101
126
|
raise ArgumentError unless d.strftime(date_pattern) == value
|
102
127
|
d
|
103
|
-
end,
|
128
|
+
end,
|
104
129
|
'http://www.w3.org/2001/XMLSchema#gYearMonth' => lambda do |value, constraints|
|
105
130
|
date_pattern = constraints["datePattern"] || "%Y-%m"
|
106
131
|
d = Date.strptime(value, date_pattern)
|
107
132
|
raise ArgumentError unless d.strftime(date_pattern) == value
|
108
133
|
d
|
109
|
-
end
|
134
|
+
end,
|
110
135
|
}
|
111
136
|
end
|
112
|
-
|
113
137
|
end
|
data/lib/csvlint/validate.rb
CHANGED
@@ -20,14 +20,9 @@ module Csvlint
|
|
20
20
|
@formats = []
|
21
21
|
@schema = schema
|
22
22
|
|
23
|
-
@assumed_header = true
|
24
23
|
@supplied_dialect = dialect != nil
|
25
|
-
|
26
|
-
|
27
|
-
@assumed_header = false
|
28
|
-
end
|
29
|
-
|
30
|
-
@dialect = dialect_defaults = {
|
24
|
+
|
25
|
+
@dialect = {
|
31
26
|
"header" => true,
|
32
27
|
"delimiter" => ",",
|
33
28
|
"skipInitialSpace" => true,
|
@@ -50,8 +45,9 @@ module Csvlint
|
|
50
45
|
io = @source.respond_to?(:gets) ? @source : open(@source, :allow_redirections=>:all)
|
51
46
|
validate_metadata(io)
|
52
47
|
parse_csv(io)
|
53
|
-
|
54
|
-
|
48
|
+
sum = @col_counts.inject(:+)
|
49
|
+
unless sum.nil?
|
50
|
+
build_warnings(:title_row, :structure) if @col_counts.first < (sum / @col_counts.size.to_f)
|
55
51
|
end
|
56
52
|
build_warnings(:check_options, :structure) if @expected_columns == 1
|
57
53
|
check_consistency
|
@@ -66,11 +62,18 @@ module Csvlint
|
|
66
62
|
@encoding = io.charset rescue nil
|
67
63
|
@content_type = io.content_type rescue nil
|
68
64
|
@headers = io.meta rescue nil
|
65
|
+
assumed_header = undeclared_header = !@supplied_dialect
|
69
66
|
if @headers
|
67
|
+
if @headers["content-type"] =~ /text\/csv/
|
68
|
+
@csv_header = true
|
69
|
+
undeclared_header = false
|
70
|
+
assumed_header = true
|
71
|
+
end
|
70
72
|
if @headers["content-type"] =~ /header=(present|absent)/
|
71
73
|
@csv_header = true if $1 == "present"
|
72
74
|
@csv_header = false if $1 == "absent"
|
73
|
-
|
75
|
+
undeclared_header = false
|
76
|
+
assumed_header = false
|
74
77
|
end
|
75
78
|
if @headers["content-type"] !~ /charset=/
|
76
79
|
build_warnings(:no_encoding, :context)
|
@@ -81,12 +84,13 @@ module Csvlint
|
|
81
84
|
build_warnings(:excel, :context) if @content_type == nil && @extension =~ /.xls(x)?/
|
82
85
|
build_errors(:wrong_content_type, :context) unless (@content_type && @content_type =~ /text\/csv/)
|
83
86
|
|
84
|
-
if
|
87
|
+
if undeclared_header
|
85
88
|
build_errors(:undeclared_header, :structure)
|
89
|
+
assumed_header = false
|
86
90
|
end
|
87
91
|
|
88
92
|
end
|
89
|
-
build_info_messages(:assumed_header, :structure) if
|
93
|
+
build_info_messages(:assumed_header, :structure) if assumed_header
|
90
94
|
end
|
91
95
|
|
92
96
|
def parse_csv(io)
|
@@ -109,27 +113,27 @@ module Csvlint
|
|
109
113
|
loop do
|
110
114
|
current_line = current_line + 1
|
111
115
|
begin
|
116
|
+
wrapper.reset_line
|
112
117
|
row = csv.shift
|
113
118
|
@data << row
|
114
|
-
wrapper.finished
|
115
119
|
if row
|
116
|
-
if
|
120
|
+
if current_line == 1 && header?
|
117
121
|
row = row.reject {|r| r.blank? }
|
118
122
|
validate_header(row)
|
119
|
-
@col_counts << row.
|
123
|
+
@col_counts << row.size
|
120
124
|
else
|
121
125
|
build_formats(row, current_line)
|
122
|
-
@col_counts << row.reject {|r| r.blank? }.
|
123
|
-
@expected_columns = row.
|
126
|
+
@col_counts << row.reject {|r| r.blank? }.size
|
127
|
+
@expected_columns = row.size unless @expected_columns != 0
|
124
128
|
|
125
|
-
build_errors(:blank_rows, :structure, current_line, nil, wrapper.line) if row.reject{ |c| c.nil? || c.empty? }.
|
129
|
+
build_errors(:blank_rows, :structure, current_line, nil, wrapper.line) if row.reject{ |c| c.nil? || c.empty? }.size == 0
|
126
130
|
|
127
131
|
if @schema
|
128
132
|
@schema.validate_row(row, current_line)
|
129
133
|
@errors += @schema.errors
|
130
134
|
@warnings += @schema.warnings
|
131
135
|
else
|
132
|
-
build_errors(:ragged_rows, :structure, current_line, nil, wrapper.line) if !row.empty? && row.
|
136
|
+
build_errors(:ragged_rows, :structure, current_line, nil, wrapper.line) if !row.empty? && row.size != @expected_columns
|
133
137
|
end
|
134
138
|
|
135
139
|
end
|
@@ -137,7 +141,6 @@ module Csvlint
|
|
137
141
|
break
|
138
142
|
end
|
139
143
|
rescue CSV::MalformedCSVError => e
|
140
|
-
wrapper.finished
|
141
144
|
type = fetch_error(e)
|
142
145
|
if type == :stray_quote && !wrapper.line.match(csv.row_sep)
|
143
146
|
build_errors(:line_breaks, :structure)
|
@@ -147,7 +150,6 @@ module Csvlint
|
|
147
150
|
end
|
148
151
|
end
|
149
152
|
rescue ArgumentError => ae
|
150
|
-
wrapper.finished
|
151
153
|
build_errors(:invalid_encoding, :structure, current_line, wrapper.line) unless reported_invalid_encoding
|
152
154
|
reported_invalid_encoding = true
|
153
155
|
end
|
@@ -172,7 +174,7 @@ module Csvlint
|
|
172
174
|
end
|
173
175
|
|
174
176
|
def header?
|
175
|
-
|
177
|
+
@csv_header
|
176
178
|
end
|
177
179
|
|
178
180
|
def fetch_error(error)
|
@@ -200,10 +202,10 @@ module Csvlint
|
|
200
202
|
|
201
203
|
SIMPLE_FORMATS.each do |type, lambda|
|
202
204
|
begin
|
203
|
-
lambda.call(col
|
204
|
-
|
205
|
-
|
206
|
-
|
205
|
+
if lambda.call(col)
|
206
|
+
@format = type
|
207
|
+
end
|
208
|
+
rescue ArgumentError, URI::InvalidURIError
|
207
209
|
end
|
208
210
|
end
|
209
211
|
|
@@ -214,13 +216,11 @@ module Csvlint
|
|
214
216
|
def check_consistency
|
215
217
|
percentages = []
|
216
218
|
|
217
|
-
|
218
|
-
|
219
|
-
formats.each do |type, regex|
|
220
|
-
@formats.count.times do |i|
|
219
|
+
SIMPLE_FORMATS.keys.each do |type|
|
220
|
+
@formats.each_with_index do |format,i|
|
221
221
|
percentages[i] ||= {}
|
222
|
-
unless
|
223
|
-
percentages[i][type] =
|
222
|
+
unless format.nil?
|
223
|
+
percentages[i][type] = format.count(type) / format.size.to_f
|
224
224
|
end
|
225
225
|
end
|
226
226
|
end
|
data/lib/csvlint/version.rb
CHANGED
data/lib/csvlint/wrapped_io.rb
CHANGED
@@ -1,39 +1,21 @@
|
|
1
1
|
module Csvlint
|
2
|
-
class WrappedIO
|
3
|
-
|
4
|
-
|
5
|
-
|
2
|
+
class WrappedIO < SimpleDelegator
|
3
|
+
attr_reader :line
|
4
|
+
|
5
|
+
def reset_line
|
6
|
+
@line = ''
|
6
7
|
end
|
7
|
-
|
8
|
+
|
8
9
|
def gets(*args)
|
9
|
-
if args.
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
if s != nil
|
14
|
-
@line << s
|
10
|
+
if args.size == 1 && args[0].is_a?(String)
|
11
|
+
s = __getobj__.gets(args[0])
|
12
|
+
if s
|
13
|
+
@line << s
|
15
14
|
end
|
16
|
-
|
15
|
+
s
|
17
16
|
else
|
18
|
-
|
17
|
+
__getobj__.gets(*args)
|
19
18
|
end
|
20
19
|
end
|
21
|
-
|
22
|
-
def eof?
|
23
|
-
@io.eof?
|
24
|
-
end
|
25
|
-
|
26
|
-
def finished
|
27
|
-
@new_line = true
|
28
|
-
end
|
29
|
-
|
30
|
-
def line
|
31
|
-
@line
|
32
|
-
end
|
33
|
-
|
34
|
-
def method_missing(method, *args)
|
35
|
-
@io.send(method, *args)
|
36
|
-
end
|
37
|
-
|
38
20
|
end
|
39
|
-
end
|
21
|
+
end
|
data/spec/validator_spec.rb
CHANGED
@@ -53,17 +53,40 @@ describe Csvlint::Validator do
|
|
53
53
|
validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
|
54
54
|
expect( validator.header? ).to eql(false)
|
55
55
|
end
|
56
|
-
|
57
|
-
it "should look in content-type for header" do
|
56
|
+
|
57
|
+
it "should look in content-type for header=absent" do
|
58
58
|
stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv; header=absent"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
|
59
59
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
60
60
|
expect( validator.header? ).to eql(false)
|
61
|
+
expect( validator.errors.size ).to eql(0)
|
62
|
+
end
|
61
63
|
|
64
|
+
it "should look in content-type for header=present" do
|
62
65
|
stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv; header=present"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
|
63
66
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
64
|
-
expect( validator.header? ).to eql(true)
|
65
|
-
|
66
|
-
|
67
|
+
expect( validator.header? ).to eql(true)
|
68
|
+
expect( validator.errors.size ).to eql(0)
|
69
|
+
end
|
70
|
+
|
71
|
+
it "assume header present if not specified in content type" do
|
72
|
+
stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
|
73
|
+
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
74
|
+
expect( validator.header? ).to eql(true)
|
75
|
+
expect( validator.errors.size ).to eql(0)
|
76
|
+
expect( validator.info_messages.size ).to eql(1)
|
77
|
+
expect( validator.info_messages.first.type).to eql(:assumed_header)
|
78
|
+
end
|
79
|
+
|
80
|
+
it "give undeclared header error if content type is wrong" do
|
81
|
+
stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/html"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
|
82
|
+
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
83
|
+
expect( validator.header? ).to eql(true)
|
84
|
+
expect( validator.errors.size ).to eql(2)
|
85
|
+
expect( validator.errors[0].type).to eql(:wrong_content_type)
|
86
|
+
expect( validator.errors[1].type).to eql(:undeclared_header)
|
87
|
+
expect( validator.info_messages.size ).to eql(0)
|
88
|
+
end
|
89
|
+
|
67
90
|
end
|
68
91
|
|
69
92
|
context "when validating headers" do
|
@@ -109,13 +132,13 @@ describe Csvlint::Validator do
|
|
109
132
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
110
133
|
expect( validator.valid? ).to eql(false)
|
111
134
|
end
|
112
|
-
|
113
|
-
it "should be an error if we have assumed a header, there is no dialect and content-type doesn't declare header" do
|
135
|
+
|
136
|
+
it "should not be an error if we have assumed a header, there is no dialect and content-type doesn't declare header, as we assume header=present" do
|
114
137
|
stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
|
115
138
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
116
|
-
expect( validator.valid? ).to eql(
|
117
|
-
end
|
118
|
-
|
139
|
+
expect( validator.valid? ).to eql(true)
|
140
|
+
end
|
141
|
+
|
119
142
|
it "should be valid if we have a dialect and the data is from the web" do
|
120
143
|
stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
|
121
144
|
#header defaults to true in csv dialect, so this is valid
|
metadata
CHANGED
@@ -1,83 +1,83 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: csvlint
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- pezholio
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-11-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mime-types
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - '>='
|
17
|
+
- - ! '>='
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - '>='
|
24
|
+
- - ! '>='
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: colorize
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - '>='
|
31
|
+
- - ! '>='
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - '>='
|
38
|
+
- - ! '>='
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: open_uri_redirections
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - '>='
|
45
|
+
- - ! '>='
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '0'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - '>='
|
52
|
+
- - ! '>='
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: activesupport
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - '>='
|
59
|
+
- - ! '>='
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - '>='
|
66
|
+
- - ! '>='
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: addressable
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- - '>='
|
73
|
+
- - ! '>='
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
76
|
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- - '>='
|
80
|
+
- - ! '>='
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
83
|
- !ruby/object:Gem::Dependency
|
@@ -98,154 +98,154 @@ dependencies:
|
|
98
98
|
name: rake
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- - '>='
|
101
|
+
- - ! '>='
|
102
102
|
- !ruby/object:Gem::Version
|
103
103
|
version: '0'
|
104
104
|
type: :development
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
|
-
- - '>='
|
108
|
+
- - ! '>='
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
112
|
name: cucumber
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
|
-
- - '>='
|
115
|
+
- - ! '>='
|
116
116
|
- !ruby/object:Gem::Version
|
117
117
|
version: '0'
|
118
118
|
type: :development
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
|
-
- - '>='
|
122
|
+
- - ! '>='
|
123
123
|
- !ruby/object:Gem::Version
|
124
124
|
version: '0'
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
126
|
name: simplecov
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
128
128
|
requirements:
|
129
|
-
- - '>='
|
129
|
+
- - ! '>='
|
130
130
|
- !ruby/object:Gem::Version
|
131
131
|
version: '0'
|
132
132
|
type: :development
|
133
133
|
prerelease: false
|
134
134
|
version_requirements: !ruby/object:Gem::Requirement
|
135
135
|
requirements:
|
136
|
-
- - '>='
|
136
|
+
- - ! '>='
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '0'
|
139
139
|
- !ruby/object:Gem::Dependency
|
140
140
|
name: simplecov-rcov
|
141
141
|
requirement: !ruby/object:Gem::Requirement
|
142
142
|
requirements:
|
143
|
-
- - '>='
|
143
|
+
- - ! '>='
|
144
144
|
- !ruby/object:Gem::Version
|
145
145
|
version: '0'
|
146
146
|
type: :development
|
147
147
|
prerelease: false
|
148
148
|
version_requirements: !ruby/object:Gem::Requirement
|
149
149
|
requirements:
|
150
|
-
- - '>='
|
150
|
+
- - ! '>='
|
151
151
|
- !ruby/object:Gem::Version
|
152
152
|
version: '0'
|
153
153
|
- !ruby/object:Gem::Dependency
|
154
154
|
name: spork
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|
156
156
|
requirements:
|
157
|
-
- - '>='
|
157
|
+
- - ! '>='
|
158
158
|
- !ruby/object:Gem::Version
|
159
159
|
version: '0'
|
160
160
|
type: :development
|
161
161
|
prerelease: false
|
162
162
|
version_requirements: !ruby/object:Gem::Requirement
|
163
163
|
requirements:
|
164
|
-
- - '>='
|
164
|
+
- - ! '>='
|
165
165
|
- !ruby/object:Gem::Version
|
166
166
|
version: '0'
|
167
167
|
- !ruby/object:Gem::Dependency
|
168
168
|
name: webmock
|
169
169
|
requirement: !ruby/object:Gem::Requirement
|
170
170
|
requirements:
|
171
|
-
- - '>='
|
171
|
+
- - ! '>='
|
172
172
|
- !ruby/object:Gem::Version
|
173
173
|
version: '0'
|
174
174
|
type: :development
|
175
175
|
prerelease: false
|
176
176
|
version_requirements: !ruby/object:Gem::Requirement
|
177
177
|
requirements:
|
178
|
-
- - '>='
|
178
|
+
- - ! '>='
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: '0'
|
181
181
|
- !ruby/object:Gem::Dependency
|
182
182
|
name: rspec
|
183
183
|
requirement: !ruby/object:Gem::Requirement
|
184
184
|
requirements:
|
185
|
-
- - '>='
|
185
|
+
- - ! '>='
|
186
186
|
- !ruby/object:Gem::Version
|
187
187
|
version: '0'
|
188
188
|
type: :development
|
189
189
|
prerelease: false
|
190
190
|
version_requirements: !ruby/object:Gem::Requirement
|
191
191
|
requirements:
|
192
|
-
- - '>='
|
192
|
+
- - ! '>='
|
193
193
|
- !ruby/object:Gem::Version
|
194
194
|
version: '0'
|
195
195
|
- !ruby/object:Gem::Dependency
|
196
196
|
name: rspec-pride
|
197
197
|
requirement: !ruby/object:Gem::Requirement
|
198
198
|
requirements:
|
199
|
-
- - '>='
|
199
|
+
- - ! '>='
|
200
200
|
- !ruby/object:Gem::Version
|
201
201
|
version: '0'
|
202
202
|
type: :development
|
203
203
|
prerelease: false
|
204
204
|
version_requirements: !ruby/object:Gem::Requirement
|
205
205
|
requirements:
|
206
|
-
- - '>='
|
206
|
+
- - ! '>='
|
207
207
|
- !ruby/object:Gem::Version
|
208
208
|
version: '0'
|
209
209
|
- !ruby/object:Gem::Dependency
|
210
210
|
name: rspec-expectations
|
211
211
|
requirement: !ruby/object:Gem::Requirement
|
212
212
|
requirements:
|
213
|
-
- - '>='
|
213
|
+
- - ! '>='
|
214
214
|
- !ruby/object:Gem::Version
|
215
215
|
version: '0'
|
216
216
|
type: :development
|
217
217
|
prerelease: false
|
218
218
|
version_requirements: !ruby/object:Gem::Requirement
|
219
219
|
requirements:
|
220
|
-
- - '>='
|
220
|
+
- - ! '>='
|
221
221
|
- !ruby/object:Gem::Version
|
222
222
|
version: '0'
|
223
223
|
- !ruby/object:Gem::Dependency
|
224
224
|
name: coveralls
|
225
225
|
requirement: !ruby/object:Gem::Requirement
|
226
226
|
requirements:
|
227
|
-
- - '>='
|
227
|
+
- - ! '>='
|
228
228
|
- !ruby/object:Gem::Version
|
229
229
|
version: '0'
|
230
230
|
type: :development
|
231
231
|
prerelease: false
|
232
232
|
version_requirements: !ruby/object:Gem::Requirement
|
233
233
|
requirements:
|
234
|
-
- - '>='
|
234
|
+
- - ! '>='
|
235
235
|
- !ruby/object:Gem::Version
|
236
236
|
version: '0'
|
237
237
|
- !ruby/object:Gem::Dependency
|
238
238
|
name: pry
|
239
239
|
requirement: !ruby/object:Gem::Requirement
|
240
240
|
requirements:
|
241
|
-
- - '>='
|
241
|
+
- - ! '>='
|
242
242
|
- !ruby/object:Gem::Version
|
243
243
|
version: '0'
|
244
244
|
type: :development
|
245
245
|
prerelease: false
|
246
246
|
version_requirements: !ruby/object:Gem::Requirement
|
247
247
|
requirements:
|
248
|
-
- - '>='
|
248
|
+
- - ! '>='
|
249
249
|
- !ruby/object:Gem::Version
|
250
250
|
version: '0'
|
251
251
|
description: CSV Validator
|
@@ -318,17 +318,17 @@ require_paths:
|
|
318
318
|
- lib
|
319
319
|
required_ruby_version: !ruby/object:Gem::Requirement
|
320
320
|
requirements:
|
321
|
-
- - '>='
|
321
|
+
- - ! '>='
|
322
322
|
- !ruby/object:Gem::Version
|
323
323
|
version: '0'
|
324
324
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
325
325
|
requirements:
|
326
|
-
- - '>='
|
326
|
+
- - ! '>='
|
327
327
|
- !ruby/object:Gem::Version
|
328
328
|
version: '0'
|
329
329
|
requirements: []
|
330
330
|
rubyforge_project:
|
331
|
-
rubygems_version: 2.
|
331
|
+
rubygems_version: 2.4.2
|
332
332
|
signing_key:
|
333
333
|
specification_version: 4
|
334
334
|
summary: CSV Validator
|