csvlint 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +13 -5
- data/.travis.yml +21 -3
- data/README.md +1 -1
- data/features/step_definitions/validation_info_steps.rb +2 -2
- data/features/validation_info.feature +5 -5
- data/lib/csvlint.rb +0 -1
- data/lib/csvlint/error_collector.rb +15 -33
- data/lib/csvlint/error_message.rb +9 -10
- data/lib/csvlint/types.rb +58 -34
- data/lib/csvlint/validate.rb +32 -32
- data/lib/csvlint/version.rb +1 -1
- data/lib/csvlint/wrapped_io.rb +13 -31
- data/spec/validator_spec.rb +33 -10
- metadata +37 -37
checksums.yaml
CHANGED
@@ -1,7 +1,15 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
NzY4MzE5MGI1OWI4YzkzY2U1MTZlMWJjNDM4YzM5ZjNiMWI5ZWIyMA==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
OTM5MWJmYzNkYTBiMjM4ZGU3Yjc2NTY0Njk4OTEzMDJhNmI2Yjg5Mg==
|
5
7
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
YzU4NzJmZTlkYWE1NTY0MGY4NzY0N2Q1NDYxMzI1N2YxYjZhZThmNDZhMDVj
|
10
|
+
Mjc1YmMyZGZkODhlNDE4OWZiYWM5NmVjNTNiOGRmMjI0OGY4ZjJkY2I4OTNl
|
11
|
+
ZmYwMGMwN2YwOWNlYmUzNjZjNWU3OTEzYTIwZDYzMGMzZDBiOGQ=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
YTkwNWVlNjlmZGZkZDJjNmUyZGQxMzBkYzViOTc1NzlhNjYzYjU3ZDljMmRl
|
14
|
+
YzExNDFkMDUxOThkMjJkNDRkZWYxZjNlMzdmYTRmNjI1MTliNGYzOWQxYzY3
|
15
|
+
MGIxMjA1YzhkYjUwOWZkOGU2NjViYjNlNjBhMTY4NTY0N2E4MmI=
|
data/.travis.yml
CHANGED
@@ -3,8 +3,26 @@ rvm:
|
|
3
3
|
notifications:
|
4
4
|
irc:
|
5
5
|
channels:
|
6
|
-
|
6
|
+
- irc.freenode.net#theodi
|
7
7
|
template:
|
8
|
-
|
8
|
+
- '%{repository} %{branch} - %{message} %{build_url}'
|
9
9
|
on_success: change
|
10
|
-
on_failure: always
|
10
|
+
on_failure: always
|
11
|
+
deploy:
|
12
|
+
provider: rubygems
|
13
|
+
api_key:
|
14
|
+
secure: fjnPS61/skQ1PsRJu1SYWIct0vjdkTyQhj0ria9zfcJfbWbfiHnpehVh1ege3sTUkDSTvoOFT35jzEeozvtKZOlAWMU5QbL8LTXu+JSp9olOdSuGeWRWVuQT3NLgRJW0+2c7N66piZvnRUUTyt2P8VIR8c/Ltuhc32bUGL7X6Gw=
|
15
|
+
gem: csvlint
|
16
|
+
on:
|
17
|
+
tags: true
|
18
|
+
repo: theodi/csvlint.rb
|
19
|
+
all_branches: true
|
20
|
+
after_success:
|
21
|
+
- export GEM_VERSION=`ruby -e "puts Gem::Specification.load(Dir['*.gemspec'].first).version.to_s"`
|
22
|
+
- '[ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ] && curl
|
23
|
+
-v -X POST -d ''{"ref":"refs/tags/''$GEM_VERSION''","sha":"''$TRAVIS_COMMIT''"}'' --header
|
24
|
+
"Content-Type:application/json" -u $GITHUB_USER:$GITHUB_PASSWORD "https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs"'
|
25
|
+
env:
|
26
|
+
global:
|
27
|
+
- secure: hM8Pv/EAZoB4fGGEUwKQQ/2+mgKbUac+PH2cwud25l2MJ64eoKEMZzfy4TF6+XMIzXhlWpsg4s51y6K3+W8pUyRAuwo2MsN5Iee7HnLz+LlYTiT0//iQdAmxQ0JlOyh8vd2SEFBcVfwpp/iZFIHAfbBO73ZDXhtGMEieMk025I0=
|
28
|
+
- secure: hsKbZ0kVKVU/e8oVCPJpRmBxbbieN2tVbyjdhJo/UYchcyLWeGAmAj/ApUDL4SyR0HZxpfi+SIsGFNRy5LnrFeCFgY9aZ6u9ma4n0Y2cTElMRt5kvI/c28FehsRjCzAe6W8dxwhJ8wSkvFDNpLml47/iy8bw4a7aUZDdq3OYXck=
|
data/README.md
CHANGED
@@ -157,7 +157,7 @@ An example schema file is:
|
|
157
157
|
|
158
158
|
Parsing and validating with a schema:
|
159
159
|
|
160
|
-
schema = Schema.load_from_json_table(uri)
|
160
|
+
schema = Csvlint::Schema.load_from_json_table(uri)
|
161
161
|
validator = Csvlint::Validator.new( "http://example.org/data.csv", nil, schema )
|
162
162
|
|
163
163
|
Supported constraints:
|
@@ -13,6 +13,6 @@ Then(/^there should be (\d+) info messages?$/) do |num|
|
|
13
13
|
@info_messages.count.should == num.to_i
|
14
14
|
end
|
15
15
|
|
16
|
-
Then(/^
|
17
|
-
@info_messages.
|
16
|
+
Then(/^one of the messages should have the type "(.*?)"$/) do |msg_type|
|
17
|
+
@info_messages.find{|x| x.type == msg_type.to_sym}.should be_present
|
18
18
|
end
|
@@ -5,20 +5,20 @@ Feature: Get validation information messages
|
|
5
5
|
And it is stored at the url "http://example.com/example1.csv"
|
6
6
|
And I set header to "true"
|
7
7
|
And I ask if there are info messages
|
8
|
-
Then there should be
|
9
|
-
And
|
8
|
+
Then there should be 2 info messages
|
9
|
+
And one of the messages should have the type "nonrfc_line_breaks"
|
10
10
|
|
11
11
|
Scenario: CR line endings in file give an info message
|
12
12
|
Given I have a CSV file called "cr-line-endings.csv"
|
13
13
|
And it is stored at the url "http://example.com/example1.csv"
|
14
14
|
And I set header to "true"
|
15
15
|
And I ask if there are info messages
|
16
|
-
Then there should be
|
17
|
-
And
|
16
|
+
Then there should be 2 info messages
|
17
|
+
And one of the messages should have the type "nonrfc_line_breaks"
|
18
18
|
|
19
19
|
Scenario: CRLF line endings in file produces no info messages
|
20
20
|
Given I have a CSV file called "crlf-line-endings.csv"
|
21
21
|
And it is stored at the url "http://example.com/example1.csv"
|
22
22
|
And I set header to "true"
|
23
23
|
And I ask if there are info messages
|
24
|
-
Then there should be
|
24
|
+
Then there should be 1 info message
|
data/lib/csvlint.rb
CHANGED
@@ -1,43 +1,25 @@
|
|
1
1
|
module Csvlint
|
2
|
-
|
3
2
|
module ErrorCollector
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
:category => category,
|
9
|
-
:row => row,
|
10
|
-
:column => column,
|
11
|
-
:content => content,
|
12
|
-
:constraints => constraints
|
13
|
-
})
|
3
|
+
attr_reader :errors, :warnings, :info_messages
|
4
|
+
|
5
|
+
def build_errors(type, category = nil, row = nil, column = nil, content = nil, constraints = {})
|
6
|
+
@errors << Csvlint::ErrorMessage.new(type, category, row, column, content, constraints)
|
14
7
|
end
|
15
|
-
|
16
|
-
|
17
|
-
:errors,
|
18
|
-
:warnings,
|
19
|
-
:info_messages
|
20
|
-
]
|
21
|
-
|
22
|
-
MESSAGE_LEVELS.each do |level|
|
23
|
-
|
24
|
-
attr_reader level
|
25
|
-
|
26
|
-
define_method "build_#{level}" do |type, category = nil, row = nil, column = nil, content = nil, constraints = {}|
|
27
|
-
instance_variable_get("@#{level}") << build_message(type, category, row, column, content, constraints)
|
28
|
-
end
|
29
|
-
|
8
|
+
def build_warnings(type, category = nil, row = nil, column = nil, content = nil, constraints = {})
|
9
|
+
@warnings << Csvlint::ErrorMessage.new(type, category, row, column, content, constraints)
|
30
10
|
end
|
31
|
-
|
11
|
+
def build_info_messages(type, category = nil, row = nil, column = nil, content = nil, constraints = {})
|
12
|
+
@info_messages << Csvlint::ErrorMessage.new(type, category, row, column, content, constraints)
|
13
|
+
end
|
14
|
+
|
32
15
|
def valid?
|
33
16
|
errors.empty?
|
34
17
|
end
|
35
|
-
|
18
|
+
|
36
19
|
def reset
|
37
|
-
|
38
|
-
|
39
|
-
|
20
|
+
@errors = []
|
21
|
+
@warnings = []
|
22
|
+
@info_messages = []
|
40
23
|
end
|
41
|
-
|
42
24
|
end
|
43
|
-
end
|
25
|
+
end
|
@@ -1,15 +1,14 @@
|
|
1
1
|
module Csvlint
|
2
|
-
|
3
2
|
class ErrorMessage
|
4
|
-
|
5
3
|
attr_reader :type, :category, :row, :column, :content, :constraints
|
6
|
-
|
7
|
-
def initialize(
|
8
|
-
|
9
|
-
|
10
|
-
|
4
|
+
|
5
|
+
def initialize(type, category, row, column, content, constraints)
|
6
|
+
@type = type
|
7
|
+
@category = category
|
8
|
+
@row = row
|
9
|
+
@column = column
|
10
|
+
@content = content
|
11
|
+
@constraints = constraints
|
11
12
|
end
|
12
|
-
|
13
13
|
end
|
14
|
-
|
15
|
-
end
|
14
|
+
end
|
data/lib/csvlint/types.rb
CHANGED
@@ -4,54 +4,79 @@ require 'active_support/core_ext/date/conversions'
|
|
4
4
|
require 'active_support/core_ext/time/conversions'
|
5
5
|
|
6
6
|
module Csvlint
|
7
|
-
|
8
7
|
module Types
|
9
|
-
|
10
8
|
SIMPLE_FORMATS = {
|
11
|
-
'string'
|
12
|
-
'numeric'
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
9
|
+
'string' => lambda { |value| true },
|
10
|
+
'numeric' => lambda { |value| value.strip[/\A[-+]?\d*\.?\d+(?:[eE][-+]?\d+)?\z/] },
|
11
|
+
'uri' => lambda do |value|
|
12
|
+
if value.strip[/\Ahttps?:/]
|
13
|
+
u = URI.parse(value)
|
14
|
+
u.kind_of?(URI::HTTP) || u.kind_of?(URI::HTTPS)
|
17
15
|
end
|
18
|
-
end,
|
19
|
-
'uri' => lambda do |value, constraints|
|
20
|
-
u = URI.parse value
|
21
|
-
raise ArgumentError unless u.kind_of?(URI::HTTP) || u.kind_of?(URI::HTTPS)
|
22
|
-
u
|
23
16
|
end
|
24
17
|
}
|
25
|
-
|
26
|
-
def self.date_format(klass
|
27
|
-
|
28
|
-
|
18
|
+
|
19
|
+
def self.date_format(klass, value, format, pattern)
|
20
|
+
if value[pattern]
|
21
|
+
klass.strptime(value, format).strftime(format) == value
|
22
|
+
end
|
29
23
|
end
|
30
|
-
|
24
|
+
|
31
25
|
def self.included(base)
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
26
|
+
[
|
27
|
+
[ :db, "%Y-%m-%d",
|
28
|
+
/\A\d{4,}-\d\d-\d\d\z/],
|
29
|
+
[ :number, "%Y%m%d",
|
30
|
+
/\A\d{8}\z/],
|
31
|
+
[ :short, "%e %b",
|
32
|
+
/\A[ \d]\d (?:#{Date::ABBR_MONTHNAMES.join('|')})\z/],
|
33
|
+
[ :rfc822, "%e %b %Y",
|
34
|
+
/\A[ \d]\d (?:#{Date::ABBR_MONTHNAMES.join('|')}) \d{4,}\z/],
|
35
|
+
[ :long, "%B %e, %Y",
|
36
|
+
/\A(?:#{Date::MONTHNAMES.join('|')}) [ \d]\d, \d{4,}\z/],
|
37
|
+
].each do |type,format,pattern|
|
38
|
+
SIMPLE_FORMATS["date_#{type}"] = lambda do |value|
|
39
|
+
date_format(Date, value, format, pattern)
|
38
40
|
end
|
39
41
|
end
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
42
|
+
|
43
|
+
# strptime doesn't support widths like %9N, unlike strftime.
|
44
|
+
# @see http://ruby-doc.org/stdlib-2.0/libdoc/date/rdoc/DateTime.html
|
45
|
+
[
|
46
|
+
[ :time, "%H:%M",
|
47
|
+
/\A\d\d:\d\d\z/],
|
48
|
+
[ :hms, "%H:%M:%S",
|
49
|
+
/\A\d\d:\d\d:\d\d\z/],
|
50
|
+
[ :db, "%Y-%m-%d %H:%M:%S",
|
51
|
+
/\A\d{4,}-\d\d-\d\d \d\d:\d\d:\d\d\z/],
|
52
|
+
[ :iso8601, "%Y-%m-%dT%H:%M:%SZ",
|
53
|
+
/\A\d{4,}-\d\d-\d\dT\d\d:\d\d:\d\dZ\z/],
|
54
|
+
[ :number, "%Y%m%d%H%M%S",
|
55
|
+
/\A\d{14}\z/],
|
56
|
+
[ :nsec, "%Y%m%d%H%M%S%N",
|
57
|
+
/\A\d{23}\z/],
|
58
|
+
[ :short, "%d %b %H:%M",
|
59
|
+
/\A\d\d (?:#{Date::ABBR_MONTHNAMES.join('|')}) \d\d:\d\d\z/],
|
60
|
+
[ :long, "%B %d, %Y %H:%M",
|
61
|
+
/\A(?:#{Date::MONTHNAMES.join('|')}) \d\d, \d{4,} \d\d:\d\d\z/],
|
62
|
+
].each do |type,format,pattern|
|
63
|
+
SIMPLE_FORMATS["dateTime_#{type}"] = lambda do |value|
|
64
|
+
date_format(Time, value, format, pattern)
|
44
65
|
end
|
45
66
|
end
|
46
67
|
end
|
47
|
-
|
68
|
+
|
48
69
|
TYPE_VALIDATIONS = {
|
49
|
-
'http://www.w3.org/2001/XMLSchema#string' =>
|
70
|
+
'http://www.w3.org/2001/XMLSchema#string' => lambda { |value, constraints| value },
|
50
71
|
'http://www.w3.org/2001/XMLSchema#int' => lambda { |value, constraints| Integer value },
|
51
72
|
'http://www.w3.org/2001/XMLSchema#integer' => lambda { |value, constraints| Integer value },
|
52
73
|
'http://www.w3.org/2001/XMLSchema#float' => lambda { |value, constraints| Float value },
|
53
74
|
'http://www.w3.org/2001/XMLSchema#double' => lambda { |value, constraints| Float value },
|
54
|
-
'http://www.w3.org/2001/XMLSchema#anyURI' =>
|
75
|
+
'http://www.w3.org/2001/XMLSchema#anyURI' => lambda do |value, constraints|
|
76
|
+
u = URI.parse value
|
77
|
+
raise ArgumentError unless u.kind_of?(URI::HTTP) || u.kind_of?(URI::HTTPS)
|
78
|
+
u
|
79
|
+
end,
|
55
80
|
'http://www.w3.org/2001/XMLSchema#boolean' => lambda do |value, constraints|
|
56
81
|
return true if ['true', '1'].include? value
|
57
82
|
return false if ['false', '0'].include? value
|
@@ -100,14 +125,13 @@ module Csvlint
|
|
100
125
|
d = Date.strptime(value, date_pattern)
|
101
126
|
raise ArgumentError unless d.strftime(date_pattern) == value
|
102
127
|
d
|
103
|
-
end,
|
128
|
+
end,
|
104
129
|
'http://www.w3.org/2001/XMLSchema#gYearMonth' => lambda do |value, constraints|
|
105
130
|
date_pattern = constraints["datePattern"] || "%Y-%m"
|
106
131
|
d = Date.strptime(value, date_pattern)
|
107
132
|
raise ArgumentError unless d.strftime(date_pattern) == value
|
108
133
|
d
|
109
|
-
end
|
134
|
+
end,
|
110
135
|
}
|
111
136
|
end
|
112
|
-
|
113
137
|
end
|
data/lib/csvlint/validate.rb
CHANGED
@@ -20,14 +20,9 @@ module Csvlint
|
|
20
20
|
@formats = []
|
21
21
|
@schema = schema
|
22
22
|
|
23
|
-
@assumed_header = true
|
24
23
|
@supplied_dialect = dialect != nil
|
25
|
-
|
26
|
-
|
27
|
-
@assumed_header = false
|
28
|
-
end
|
29
|
-
|
30
|
-
@dialect = dialect_defaults = {
|
24
|
+
|
25
|
+
@dialect = {
|
31
26
|
"header" => true,
|
32
27
|
"delimiter" => ",",
|
33
28
|
"skipInitialSpace" => true,
|
@@ -50,8 +45,9 @@ module Csvlint
|
|
50
45
|
io = @source.respond_to?(:gets) ? @source : open(@source, :allow_redirections=>:all)
|
51
46
|
validate_metadata(io)
|
52
47
|
parse_csv(io)
|
53
|
-
|
54
|
-
|
48
|
+
sum = @col_counts.inject(:+)
|
49
|
+
unless sum.nil?
|
50
|
+
build_warnings(:title_row, :structure) if @col_counts.first < (sum / @col_counts.size.to_f)
|
55
51
|
end
|
56
52
|
build_warnings(:check_options, :structure) if @expected_columns == 1
|
57
53
|
check_consistency
|
@@ -66,11 +62,18 @@ module Csvlint
|
|
66
62
|
@encoding = io.charset rescue nil
|
67
63
|
@content_type = io.content_type rescue nil
|
68
64
|
@headers = io.meta rescue nil
|
65
|
+
assumed_header = undeclared_header = !@supplied_dialect
|
69
66
|
if @headers
|
67
|
+
if @headers["content-type"] =~ /text\/csv/
|
68
|
+
@csv_header = true
|
69
|
+
undeclared_header = false
|
70
|
+
assumed_header = true
|
71
|
+
end
|
70
72
|
if @headers["content-type"] =~ /header=(present|absent)/
|
71
73
|
@csv_header = true if $1 == "present"
|
72
74
|
@csv_header = false if $1 == "absent"
|
73
|
-
|
75
|
+
undeclared_header = false
|
76
|
+
assumed_header = false
|
74
77
|
end
|
75
78
|
if @headers["content-type"] !~ /charset=/
|
76
79
|
build_warnings(:no_encoding, :context)
|
@@ -81,12 +84,13 @@ module Csvlint
|
|
81
84
|
build_warnings(:excel, :context) if @content_type == nil && @extension =~ /.xls(x)?/
|
82
85
|
build_errors(:wrong_content_type, :context) unless (@content_type && @content_type =~ /text\/csv/)
|
83
86
|
|
84
|
-
if
|
87
|
+
if undeclared_header
|
85
88
|
build_errors(:undeclared_header, :structure)
|
89
|
+
assumed_header = false
|
86
90
|
end
|
87
91
|
|
88
92
|
end
|
89
|
-
build_info_messages(:assumed_header, :structure) if
|
93
|
+
build_info_messages(:assumed_header, :structure) if assumed_header
|
90
94
|
end
|
91
95
|
|
92
96
|
def parse_csv(io)
|
@@ -109,27 +113,27 @@ module Csvlint
|
|
109
113
|
loop do
|
110
114
|
current_line = current_line + 1
|
111
115
|
begin
|
116
|
+
wrapper.reset_line
|
112
117
|
row = csv.shift
|
113
118
|
@data << row
|
114
|
-
wrapper.finished
|
115
119
|
if row
|
116
|
-
if
|
120
|
+
if current_line == 1 && header?
|
117
121
|
row = row.reject {|r| r.blank? }
|
118
122
|
validate_header(row)
|
119
|
-
@col_counts << row.
|
123
|
+
@col_counts << row.size
|
120
124
|
else
|
121
125
|
build_formats(row, current_line)
|
122
|
-
@col_counts << row.reject {|r| r.blank? }.
|
123
|
-
@expected_columns = row.
|
126
|
+
@col_counts << row.reject {|r| r.blank? }.size
|
127
|
+
@expected_columns = row.size unless @expected_columns != 0
|
124
128
|
|
125
|
-
build_errors(:blank_rows, :structure, current_line, nil, wrapper.line) if row.reject{ |c| c.nil? || c.empty? }.
|
129
|
+
build_errors(:blank_rows, :structure, current_line, nil, wrapper.line) if row.reject{ |c| c.nil? || c.empty? }.size == 0
|
126
130
|
|
127
131
|
if @schema
|
128
132
|
@schema.validate_row(row, current_line)
|
129
133
|
@errors += @schema.errors
|
130
134
|
@warnings += @schema.warnings
|
131
135
|
else
|
132
|
-
build_errors(:ragged_rows, :structure, current_line, nil, wrapper.line) if !row.empty? && row.
|
136
|
+
build_errors(:ragged_rows, :structure, current_line, nil, wrapper.line) if !row.empty? && row.size != @expected_columns
|
133
137
|
end
|
134
138
|
|
135
139
|
end
|
@@ -137,7 +141,6 @@ module Csvlint
|
|
137
141
|
break
|
138
142
|
end
|
139
143
|
rescue CSV::MalformedCSVError => e
|
140
|
-
wrapper.finished
|
141
144
|
type = fetch_error(e)
|
142
145
|
if type == :stray_quote && !wrapper.line.match(csv.row_sep)
|
143
146
|
build_errors(:line_breaks, :structure)
|
@@ -147,7 +150,6 @@ module Csvlint
|
|
147
150
|
end
|
148
151
|
end
|
149
152
|
rescue ArgumentError => ae
|
150
|
-
wrapper.finished
|
151
153
|
build_errors(:invalid_encoding, :structure, current_line, wrapper.line) unless reported_invalid_encoding
|
152
154
|
reported_invalid_encoding = true
|
153
155
|
end
|
@@ -172,7 +174,7 @@ module Csvlint
|
|
172
174
|
end
|
173
175
|
|
174
176
|
def header?
|
175
|
-
|
177
|
+
@csv_header
|
176
178
|
end
|
177
179
|
|
178
180
|
def fetch_error(error)
|
@@ -200,10 +202,10 @@ module Csvlint
|
|
200
202
|
|
201
203
|
SIMPLE_FORMATS.each do |type, lambda|
|
202
204
|
begin
|
203
|
-
lambda.call(col
|
204
|
-
|
205
|
-
|
206
|
-
|
205
|
+
if lambda.call(col)
|
206
|
+
@format = type
|
207
|
+
end
|
208
|
+
rescue ArgumentError, URI::InvalidURIError
|
207
209
|
end
|
208
210
|
end
|
209
211
|
|
@@ -214,13 +216,11 @@ module Csvlint
|
|
214
216
|
def check_consistency
|
215
217
|
percentages = []
|
216
218
|
|
217
|
-
|
218
|
-
|
219
|
-
formats.each do |type, regex|
|
220
|
-
@formats.count.times do |i|
|
219
|
+
SIMPLE_FORMATS.keys.each do |type|
|
220
|
+
@formats.each_with_index do |format,i|
|
221
221
|
percentages[i] ||= {}
|
222
|
-
unless
|
223
|
-
percentages[i][type] =
|
222
|
+
unless format.nil?
|
223
|
+
percentages[i][type] = format.count(type) / format.size.to_f
|
224
224
|
end
|
225
225
|
end
|
226
226
|
end
|
data/lib/csvlint/version.rb
CHANGED
data/lib/csvlint/wrapped_io.rb
CHANGED
@@ -1,39 +1,21 @@
|
|
1
1
|
module Csvlint
|
2
|
-
class WrappedIO
|
3
|
-
|
4
|
-
|
5
|
-
|
2
|
+
class WrappedIO < SimpleDelegator
|
3
|
+
attr_reader :line
|
4
|
+
|
5
|
+
def reset_line
|
6
|
+
@line = ''
|
6
7
|
end
|
7
|
-
|
8
|
+
|
8
9
|
def gets(*args)
|
9
|
-
if args.
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
if s != nil
|
14
|
-
@line << s
|
10
|
+
if args.size == 1 && args[0].is_a?(String)
|
11
|
+
s = __getobj__.gets(args[0])
|
12
|
+
if s
|
13
|
+
@line << s
|
15
14
|
end
|
16
|
-
|
15
|
+
s
|
17
16
|
else
|
18
|
-
|
17
|
+
__getobj__.gets(*args)
|
19
18
|
end
|
20
19
|
end
|
21
|
-
|
22
|
-
def eof?
|
23
|
-
@io.eof?
|
24
|
-
end
|
25
|
-
|
26
|
-
def finished
|
27
|
-
@new_line = true
|
28
|
-
end
|
29
|
-
|
30
|
-
def line
|
31
|
-
@line
|
32
|
-
end
|
33
|
-
|
34
|
-
def method_missing(method, *args)
|
35
|
-
@io.send(method, *args)
|
36
|
-
end
|
37
|
-
|
38
20
|
end
|
39
|
-
end
|
21
|
+
end
|
data/spec/validator_spec.rb
CHANGED
@@ -53,17 +53,40 @@ describe Csvlint::Validator do
|
|
53
53
|
validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
|
54
54
|
expect( validator.header? ).to eql(false)
|
55
55
|
end
|
56
|
-
|
57
|
-
it "should look in content-type for header" do
|
56
|
+
|
57
|
+
it "should look in content-type for header=absent" do
|
58
58
|
stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv; header=absent"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
|
59
59
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
60
60
|
expect( validator.header? ).to eql(false)
|
61
|
+
expect( validator.errors.size ).to eql(0)
|
62
|
+
end
|
61
63
|
|
64
|
+
it "should look in content-type for header=present" do
|
62
65
|
stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv; header=present"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
|
63
66
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
64
|
-
expect( validator.header? ).to eql(true)
|
65
|
-
|
66
|
-
|
67
|
+
expect( validator.header? ).to eql(true)
|
68
|
+
expect( validator.errors.size ).to eql(0)
|
69
|
+
end
|
70
|
+
|
71
|
+
it "assume header present if not specified in content type" do
|
72
|
+
stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
|
73
|
+
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
74
|
+
expect( validator.header? ).to eql(true)
|
75
|
+
expect( validator.errors.size ).to eql(0)
|
76
|
+
expect( validator.info_messages.size ).to eql(1)
|
77
|
+
expect( validator.info_messages.first.type).to eql(:assumed_header)
|
78
|
+
end
|
79
|
+
|
80
|
+
it "give undeclared header error if content type is wrong" do
|
81
|
+
stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/html"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
|
82
|
+
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
83
|
+
expect( validator.header? ).to eql(true)
|
84
|
+
expect( validator.errors.size ).to eql(2)
|
85
|
+
expect( validator.errors[0].type).to eql(:wrong_content_type)
|
86
|
+
expect( validator.errors[1].type).to eql(:undeclared_header)
|
87
|
+
expect( validator.info_messages.size ).to eql(0)
|
88
|
+
end
|
89
|
+
|
67
90
|
end
|
68
91
|
|
69
92
|
context "when validating headers" do
|
@@ -109,13 +132,13 @@ describe Csvlint::Validator do
|
|
109
132
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
110
133
|
expect( validator.valid? ).to eql(false)
|
111
134
|
end
|
112
|
-
|
113
|
-
it "should be an error if we have assumed a header, there is no dialect and content-type doesn't declare header" do
|
135
|
+
|
136
|
+
it "should not be an error if we have assumed a header, there is no dialect and content-type doesn't declare header, as we assume header=present" do
|
114
137
|
stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
|
115
138
|
validator = Csvlint::Validator.new("http://example.com/example.csv")
|
116
|
-
expect( validator.valid? ).to eql(
|
117
|
-
end
|
118
|
-
|
139
|
+
expect( validator.valid? ).to eql(true)
|
140
|
+
end
|
141
|
+
|
119
142
|
it "should be valid if we have a dialect and the data is from the web" do
|
120
143
|
stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
|
121
144
|
#header defaults to true in csv dialect, so this is valid
|
metadata
CHANGED
@@ -1,83 +1,83 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: csvlint
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- pezholio
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-11-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mime-types
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - '>='
|
17
|
+
- - ! '>='
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - '>='
|
24
|
+
- - ! '>='
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: colorize
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - '>='
|
31
|
+
- - ! '>='
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - '>='
|
38
|
+
- - ! '>='
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: open_uri_redirections
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - '>='
|
45
|
+
- - ! '>='
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '0'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - '>='
|
52
|
+
- - ! '>='
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: activesupport
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - '>='
|
59
|
+
- - ! '>='
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - '>='
|
66
|
+
- - ! '>='
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: addressable
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- - '>='
|
73
|
+
- - ! '>='
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
76
|
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- - '>='
|
80
|
+
- - ! '>='
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
83
|
- !ruby/object:Gem::Dependency
|
@@ -98,154 +98,154 @@ dependencies:
|
|
98
98
|
name: rake
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- - '>='
|
101
|
+
- - ! '>='
|
102
102
|
- !ruby/object:Gem::Version
|
103
103
|
version: '0'
|
104
104
|
type: :development
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
|
-
- - '>='
|
108
|
+
- - ! '>='
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
112
|
name: cucumber
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
|
-
- - '>='
|
115
|
+
- - ! '>='
|
116
116
|
- !ruby/object:Gem::Version
|
117
117
|
version: '0'
|
118
118
|
type: :development
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
|
-
- - '>='
|
122
|
+
- - ! '>='
|
123
123
|
- !ruby/object:Gem::Version
|
124
124
|
version: '0'
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
126
|
name: simplecov
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
128
128
|
requirements:
|
129
|
-
- - '>='
|
129
|
+
- - ! '>='
|
130
130
|
- !ruby/object:Gem::Version
|
131
131
|
version: '0'
|
132
132
|
type: :development
|
133
133
|
prerelease: false
|
134
134
|
version_requirements: !ruby/object:Gem::Requirement
|
135
135
|
requirements:
|
136
|
-
- - '>='
|
136
|
+
- - ! '>='
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '0'
|
139
139
|
- !ruby/object:Gem::Dependency
|
140
140
|
name: simplecov-rcov
|
141
141
|
requirement: !ruby/object:Gem::Requirement
|
142
142
|
requirements:
|
143
|
-
- - '>='
|
143
|
+
- - ! '>='
|
144
144
|
- !ruby/object:Gem::Version
|
145
145
|
version: '0'
|
146
146
|
type: :development
|
147
147
|
prerelease: false
|
148
148
|
version_requirements: !ruby/object:Gem::Requirement
|
149
149
|
requirements:
|
150
|
-
- - '>='
|
150
|
+
- - ! '>='
|
151
151
|
- !ruby/object:Gem::Version
|
152
152
|
version: '0'
|
153
153
|
- !ruby/object:Gem::Dependency
|
154
154
|
name: spork
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|
156
156
|
requirements:
|
157
|
-
- - '>='
|
157
|
+
- - ! '>='
|
158
158
|
- !ruby/object:Gem::Version
|
159
159
|
version: '0'
|
160
160
|
type: :development
|
161
161
|
prerelease: false
|
162
162
|
version_requirements: !ruby/object:Gem::Requirement
|
163
163
|
requirements:
|
164
|
-
- - '>='
|
164
|
+
- - ! '>='
|
165
165
|
- !ruby/object:Gem::Version
|
166
166
|
version: '0'
|
167
167
|
- !ruby/object:Gem::Dependency
|
168
168
|
name: webmock
|
169
169
|
requirement: !ruby/object:Gem::Requirement
|
170
170
|
requirements:
|
171
|
-
- - '>='
|
171
|
+
- - ! '>='
|
172
172
|
- !ruby/object:Gem::Version
|
173
173
|
version: '0'
|
174
174
|
type: :development
|
175
175
|
prerelease: false
|
176
176
|
version_requirements: !ruby/object:Gem::Requirement
|
177
177
|
requirements:
|
178
|
-
- - '>='
|
178
|
+
- - ! '>='
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: '0'
|
181
181
|
- !ruby/object:Gem::Dependency
|
182
182
|
name: rspec
|
183
183
|
requirement: !ruby/object:Gem::Requirement
|
184
184
|
requirements:
|
185
|
-
- - '>='
|
185
|
+
- - ! '>='
|
186
186
|
- !ruby/object:Gem::Version
|
187
187
|
version: '0'
|
188
188
|
type: :development
|
189
189
|
prerelease: false
|
190
190
|
version_requirements: !ruby/object:Gem::Requirement
|
191
191
|
requirements:
|
192
|
-
- - '>='
|
192
|
+
- - ! '>='
|
193
193
|
- !ruby/object:Gem::Version
|
194
194
|
version: '0'
|
195
195
|
- !ruby/object:Gem::Dependency
|
196
196
|
name: rspec-pride
|
197
197
|
requirement: !ruby/object:Gem::Requirement
|
198
198
|
requirements:
|
199
|
-
- - '>='
|
199
|
+
- - ! '>='
|
200
200
|
- !ruby/object:Gem::Version
|
201
201
|
version: '0'
|
202
202
|
type: :development
|
203
203
|
prerelease: false
|
204
204
|
version_requirements: !ruby/object:Gem::Requirement
|
205
205
|
requirements:
|
206
|
-
- - '>='
|
206
|
+
- - ! '>='
|
207
207
|
- !ruby/object:Gem::Version
|
208
208
|
version: '0'
|
209
209
|
- !ruby/object:Gem::Dependency
|
210
210
|
name: rspec-expectations
|
211
211
|
requirement: !ruby/object:Gem::Requirement
|
212
212
|
requirements:
|
213
|
-
- - '>='
|
213
|
+
- - ! '>='
|
214
214
|
- !ruby/object:Gem::Version
|
215
215
|
version: '0'
|
216
216
|
type: :development
|
217
217
|
prerelease: false
|
218
218
|
version_requirements: !ruby/object:Gem::Requirement
|
219
219
|
requirements:
|
220
|
-
- - '>='
|
220
|
+
- - ! '>='
|
221
221
|
- !ruby/object:Gem::Version
|
222
222
|
version: '0'
|
223
223
|
- !ruby/object:Gem::Dependency
|
224
224
|
name: coveralls
|
225
225
|
requirement: !ruby/object:Gem::Requirement
|
226
226
|
requirements:
|
227
|
-
- - '>='
|
227
|
+
- - ! '>='
|
228
228
|
- !ruby/object:Gem::Version
|
229
229
|
version: '0'
|
230
230
|
type: :development
|
231
231
|
prerelease: false
|
232
232
|
version_requirements: !ruby/object:Gem::Requirement
|
233
233
|
requirements:
|
234
|
-
- - '>='
|
234
|
+
- - ! '>='
|
235
235
|
- !ruby/object:Gem::Version
|
236
236
|
version: '0'
|
237
237
|
- !ruby/object:Gem::Dependency
|
238
238
|
name: pry
|
239
239
|
requirement: !ruby/object:Gem::Requirement
|
240
240
|
requirements:
|
241
|
-
- - '>='
|
241
|
+
- - ! '>='
|
242
242
|
- !ruby/object:Gem::Version
|
243
243
|
version: '0'
|
244
244
|
type: :development
|
245
245
|
prerelease: false
|
246
246
|
version_requirements: !ruby/object:Gem::Requirement
|
247
247
|
requirements:
|
248
|
-
- - '>='
|
248
|
+
- - ! '>='
|
249
249
|
- !ruby/object:Gem::Version
|
250
250
|
version: '0'
|
251
251
|
description: CSV Validator
|
@@ -318,17 +318,17 @@ require_paths:
|
|
318
318
|
- lib
|
319
319
|
required_ruby_version: !ruby/object:Gem::Requirement
|
320
320
|
requirements:
|
321
|
-
- - '>='
|
321
|
+
- - ! '>='
|
322
322
|
- !ruby/object:Gem::Version
|
323
323
|
version: '0'
|
324
324
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
325
325
|
requirements:
|
326
|
-
- - '>='
|
326
|
+
- - ! '>='
|
327
327
|
- !ruby/object:Gem::Version
|
328
328
|
version: '0'
|
329
329
|
requirements: []
|
330
330
|
rubyforge_project:
|
331
|
-
rubygems_version: 2.
|
331
|
+
rubygems_version: 2.4.2
|
332
332
|
signing_key:
|
333
333
|
specification_version: 4
|
334
334
|
summary: CSV Validator
|