csvlint 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +4 -0
- data/.github/workflows/push.yml +14 -2
- data/.ruby-version +1 -1
- data/.standard_todo.yml +43 -0
- data/Dockerfile +16 -0
- data/Gemfile +2 -2
- data/README.md +9 -9
- data/Rakefile +7 -7
- data/csvlint.gemspec +14 -16
- data/docker_notes_for_windows.txt +20 -0
- data/features/step_definitions/cli_steps.rb +11 -11
- data/features/step_definitions/information_steps.rb +4 -4
- data/features/step_definitions/parse_csv_steps.rb +11 -11
- data/features/step_definitions/schema_validation_steps.rb +10 -10
- data/features/step_definitions/sources_steps.rb +1 -1
- data/features/step_definitions/validation_errors_steps.rb +19 -19
- data/features/step_definitions/validation_info_steps.rb +9 -9
- data/features/step_definitions/validation_warnings_steps.rb +11 -11
- data/features/support/aruba.rb +6 -6
- data/features/support/earl_formatter.rb +39 -39
- data/features/support/env.rb +10 -11
- data/features/support/load_tests.rb +107 -103
- data/features/support/webmock.rb +2 -2
- data/lib/csvlint/cli.rb +133 -130
- data/lib/csvlint/csvw/column.rb +279 -280
- data/lib/csvlint/csvw/date_format.rb +90 -92
- data/lib/csvlint/csvw/metadata_error.rb +1 -3
- data/lib/csvlint/csvw/number_format.rb +40 -32
- data/lib/csvlint/csvw/property_checker.rb +714 -717
- data/lib/csvlint/csvw/table.rb +49 -52
- data/lib/csvlint/csvw/table_group.rb +24 -23
- data/lib/csvlint/error_collector.rb +2 -0
- data/lib/csvlint/error_message.rb +0 -1
- data/lib/csvlint/field.rb +153 -141
- data/lib/csvlint/schema.rb +34 -42
- data/lib/csvlint/validate.rb +161 -143
- data/lib/csvlint/version.rb +1 -1
- data/lib/csvlint.rb +22 -23
- data/spec/csvw/column_spec.rb +15 -16
- data/spec/csvw/date_format_spec.rb +5 -7
- data/spec/csvw/number_format_spec.rb +2 -4
- data/spec/csvw/table_group_spec.rb +103 -105
- data/spec/csvw/table_spec.rb +71 -73
- data/spec/field_spec.rb +116 -121
- data/spec/schema_spec.rb +129 -139
- data/spec/spec_helper.rb +6 -6
- data/spec/validator_spec.rb +167 -190
- metadata +22 -55
data/spec/schema_spec.rb
CHANGED
@@ -1,150 +1,143 @@
|
|
1
|
-
require
|
1
|
+
require "spec_helper"
|
2
2
|
|
3
3
|
describe Csvlint::Schema do
|
4
|
-
|
5
4
|
it "should tolerate missing fields" do
|
6
5
|
schema = Csvlint::Schema.from_json_table("http://example.org", {})
|
7
|
-
expect(
|
8
|
-
expect(
|
6
|
+
expect(schema).to_not be(nil)
|
7
|
+
expect(schema.fields.empty?).to eql(true)
|
9
8
|
end
|
10
9
|
|
11
10
|
it "should tolerate fields with no constraints" do
|
12
11
|
schema = Csvlint::Schema.from_json_table("http://example.org", {
|
13
|
-
"fields" => [
|
12
|
+
"fields" => [{"name" => "test"}]
|
14
13
|
})
|
15
|
-
expect(
|
16
|
-
expect(
|
17
|
-
expect(
|
14
|
+
expect(schema).to_not be(nil)
|
15
|
+
expect(schema.fields[0].name).to eql("test")
|
16
|
+
expect(schema.fields[0].constraints).to eql({})
|
18
17
|
end
|
19
18
|
|
20
19
|
it "should validate against the schema" do
|
21
|
-
field = Csvlint::Field.new("test", {
|
22
|
-
field2 = Csvlint::Field.new("test", {
|
23
|
-
schema = Csvlint::Schema.new("http://example.org", [field, field2]
|
24
|
-
|
25
|
-
expect(
|
26
|
-
expect(
|
27
|
-
expect(
|
28
|
-
expect(
|
29
|
-
expect(
|
30
|
-
expect(
|
31
|
-
|
20
|
+
field = Csvlint::Field.new("test", {"required" => true})
|
21
|
+
field2 = Csvlint::Field.new("test", {"minLength" => 3})
|
22
|
+
schema = Csvlint::Schema.new("http://example.org", [field, field2])
|
23
|
+
|
24
|
+
expect(schema.validate_row(["", "x"])).to eql(false)
|
25
|
+
expect(schema.errors.size).to eql(2)
|
26
|
+
expect(schema.errors.first.type).to eql(:missing_value)
|
27
|
+
expect(schema.errors.first.category).to eql(:schema)
|
28
|
+
expect(schema.errors.first.column).to eql(1)
|
29
|
+
expect(schema.validate_row(["abc", "1234"])).to eql(true)
|
32
30
|
end
|
33
31
|
|
34
32
|
it "should include validations for missing columns" do
|
35
|
-
minimum = Csvlint::Field.new("test", {
|
36
|
-
required = Csvlint::Field.new("test2", {
|
37
|
-
schema = Csvlint::Schema.new("http://example.org", [minimum, required]
|
33
|
+
minimum = Csvlint::Field.new("test", {"minLength" => 3})
|
34
|
+
required = Csvlint::Field.new("test2", {"required" => true})
|
35
|
+
schema = Csvlint::Schema.new("http://example.org", [minimum, required])
|
38
36
|
|
39
|
-
expect(
|
37
|
+
expect(schema.validate_row(["abc", "x"])).to eql(true)
|
40
38
|
|
41
|
-
expect(
|
42
|
-
expect(
|
43
|
-
expect(
|
39
|
+
expect(schema.validate_row(["abc"])).to eql(false)
|
40
|
+
expect(schema.errors.size).to eql(1)
|
41
|
+
expect(schema.errors.first.type).to eql(:missing_value)
|
44
42
|
|
45
|
-
schema = Csvlint::Schema.new("http://example.org", [required, minimum]
|
46
|
-
expect(
|
47
|
-
expect(
|
48
|
-
expect(
|
43
|
+
schema = Csvlint::Schema.new("http://example.org", [required, minimum])
|
44
|
+
expect(schema.validate_row(["abc"])).to eql(false)
|
45
|
+
expect(schema.errors.size).to eql(1)
|
46
|
+
expect(schema.errors.first.type).to eql(:min_length)
|
49
47
|
end
|
50
48
|
|
51
49
|
it "should warn if the data has fewer columns" do
|
52
|
-
minimum = Csvlint::Field.new("test", {
|
53
|
-
required = Csvlint::Field.new("test2", {
|
54
|
-
schema = Csvlint::Schema.new("http://example.org", [minimum, required]
|
55
|
-
|
56
|
-
expect(
|
57
|
-
expect(
|
58
|
-
expect(
|
59
|
-
expect(
|
60
|
-
expect(
|
61
|
-
expect(
|
62
|
-
|
63
|
-
#no ragged row error
|
64
|
-
expect(
|
50
|
+
minimum = Csvlint::Field.new("test", {"minLength" => 3})
|
51
|
+
required = Csvlint::Field.new("test2", {"maxLength" => 5})
|
52
|
+
schema = Csvlint::Schema.new("http://example.org", [minimum, required])
|
53
|
+
|
54
|
+
expect(schema.validate_row(["abc"], 1)).to eql(true)
|
55
|
+
expect(schema.warnings.size).to eql(1)
|
56
|
+
expect(schema.warnings.first.type).to eql(:missing_column)
|
57
|
+
expect(schema.warnings.first.category).to eql(:schema)
|
58
|
+
expect(schema.warnings.first.row).to eql(1)
|
59
|
+
expect(schema.warnings.first.column).to eql(2)
|
60
|
+
|
61
|
+
# no ragged row error
|
62
|
+
expect(schema.errors.size).to eql(0)
|
65
63
|
end
|
66
64
|
|
67
65
|
it "should warn if the data has additional columns" do
|
68
|
-
minimum = Csvlint::Field.new("test", {
|
69
|
-
required = Csvlint::Field.new("test2", {
|
70
|
-
schema = Csvlint::Schema.new("http://example.org", [minimum, required]
|
71
|
-
|
72
|
-
expect(
|
73
|
-
expect(
|
74
|
-
expect(
|
75
|
-
expect(
|
76
|
-
expect(
|
77
|
-
expect(
|
78
|
-
|
79
|
-
expect(
|
80
|
-
expect(
|
81
|
-
|
82
|
-
#no ragged row error
|
83
|
-
expect(
|
66
|
+
minimum = Csvlint::Field.new("test", {"minLength" => 3})
|
67
|
+
required = Csvlint::Field.new("test2", {"required" => true})
|
68
|
+
schema = Csvlint::Schema.new("http://example.org", [minimum, required])
|
69
|
+
|
70
|
+
expect(schema.validate_row(["abc", "x", "more", "columns"], 1)).to eql(true)
|
71
|
+
expect(schema.warnings.size).to eql(2)
|
72
|
+
expect(schema.warnings.first.type).to eql(:extra_column)
|
73
|
+
expect(schema.warnings.first.category).to eql(:schema)
|
74
|
+
expect(schema.warnings.first.row).to eql(1)
|
75
|
+
expect(schema.warnings.first.column).to eql(3)
|
76
|
+
|
77
|
+
expect(schema.warnings[1].type).to eql(:extra_column)
|
78
|
+
expect(schema.warnings[1].column).to eql(4)
|
79
|
+
|
80
|
+
# no ragged row error
|
81
|
+
expect(schema.errors.size).to eql(0)
|
84
82
|
end
|
85
83
|
|
86
84
|
context "when validating header" do
|
87
85
|
it "should warn if column names are different to field names" do
|
88
|
-
minimum = Csvlint::Field.new("minimum", {
|
89
|
-
required = Csvlint::Field.new("required", {
|
90
|
-
schema = Csvlint::Schema.new("http://example.org", [minimum, required]
|
91
|
-
|
92
|
-
expect(
|
93
|
-
expect(
|
94
|
-
|
95
|
-
expect(
|
96
|
-
expect(
|
97
|
-
expect(
|
98
|
-
expect(
|
99
|
-
expect(
|
100
|
-
expect(
|
101
|
-
expect(
|
102
|
-
expect schema.warnings.first.constraints.has_value?(
|
86
|
+
minimum = Csvlint::Field.new("minimum", {"minLength" => 3})
|
87
|
+
required = Csvlint::Field.new("required", {"required" => true})
|
88
|
+
schema = Csvlint::Schema.new("http://example.org", [minimum, required])
|
89
|
+
|
90
|
+
expect(schema.validate_header(["minimum", "required"])).to eql(true)
|
91
|
+
expect(schema.warnings.size).to eql(0)
|
92
|
+
|
93
|
+
expect(schema.validate_header(["wrong", "required"])).to eql(true)
|
94
|
+
expect(schema.warnings.size).to eql(1)
|
95
|
+
expect(schema.warnings.first.row).to eql(1)
|
96
|
+
expect(schema.warnings.first.type).to eql(:malformed_header)
|
97
|
+
expect(schema.warnings.first.content).to eql("wrong,required")
|
98
|
+
expect(schema.warnings.first.column).to eql(nil)
|
99
|
+
expect(schema.warnings.first.category).to eql(:schema)
|
100
|
+
expect schema.warnings.first.constraints.has_value?("minimum,required")
|
103
101
|
# expect( schema.warnings.first.constraints.values ).to eql(['minimum,required'])
|
104
|
-
expect(
|
105
|
-
expect(
|
106
|
-
|
102
|
+
expect(schema.validate_header(["minimum", "Required"])).to eql(true)
|
103
|
+
expect(schema.warnings.size).to eql(1)
|
107
104
|
end
|
108
105
|
|
109
106
|
it "should warn if column count is less than field count" do
|
110
|
-
minimum = Csvlint::Field.new("minimum", {
|
111
|
-
required = Csvlint::Field.new("required", {
|
112
|
-
schema = Csvlint::Schema.new("http://example.org", [minimum, required]
|
113
|
-
|
114
|
-
expect(
|
115
|
-
expect(
|
116
|
-
expect(
|
117
|
-
expect(
|
118
|
-
expect(
|
119
|
-
expect(
|
120
|
-
expect(
|
121
|
-
expect schema.warnings.first.constraints.has_value?(
|
107
|
+
minimum = Csvlint::Field.new("minimum", {"minLength" => 3})
|
108
|
+
required = Csvlint::Field.new("required", {"required" => true})
|
109
|
+
schema = Csvlint::Schema.new("http://example.org", [minimum, required])
|
110
|
+
|
111
|
+
expect(schema.validate_header(["minimum"])).to eql(true)
|
112
|
+
expect(schema.warnings.size).to eql(1)
|
113
|
+
expect(schema.warnings.first.row).to eql(1)
|
114
|
+
expect(schema.warnings.first.type).to eql(:malformed_header)
|
115
|
+
expect(schema.warnings.first.content).to eql("minimum")
|
116
|
+
expect(schema.warnings.first.column).to eql(nil)
|
117
|
+
expect(schema.warnings.first.category).to eql(:schema)
|
118
|
+
expect schema.warnings.first.constraints.has_value?("minimum,required")
|
122
119
|
# expect( schema.warnings.first.constraints.values ).to eql(['minimum,required'])
|
123
|
-
|
124
120
|
end
|
125
121
|
|
126
122
|
it "should warn if column count is more than field count" do
|
127
|
-
minimum = Csvlint::Field.new("minimum", {
|
128
|
-
schema = Csvlint::Schema.new("http://example.org", [minimum]
|
129
|
-
|
130
|
-
expect(
|
131
|
-
expect(
|
132
|
-
expect(
|
133
|
-
expect(
|
134
|
-
expect(
|
135
|
-
expect(
|
136
|
-
expect(
|
123
|
+
minimum = Csvlint::Field.new("minimum", {"minLength" => 3})
|
124
|
+
schema = Csvlint::Schema.new("http://example.org", [minimum])
|
125
|
+
|
126
|
+
expect(schema.validate_header(["wrong", "required"])).to eql(true)
|
127
|
+
expect(schema.warnings.size).to eql(1)
|
128
|
+
expect(schema.warnings.first.row).to eql(1)
|
129
|
+
expect(schema.warnings.first.type).to eql(:malformed_header)
|
130
|
+
expect(schema.warnings.first.content).to eql("wrong,required")
|
131
|
+
expect(schema.warnings.first.column).to eql(nil)
|
132
|
+
expect(schema.warnings.first.category).to eql(:schema)
|
137
133
|
# expect( schema.warnings.first.constraints.values ).to eql('minimum')
|
138
|
-
expect(
|
139
|
-
|
134
|
+
expect(schema.warnings.first.constraints.has_value?("minimum"))
|
140
135
|
end
|
141
|
-
|
142
136
|
end
|
143
137
|
|
144
138
|
context "when parsing JSON Tables" do
|
145
|
-
|
146
139
|
before(:each) do
|
147
|
-
@example
|
140
|
+
@example = <<-EOL
|
148
141
|
{
|
149
142
|
"title": "Schema title",
|
150
143
|
"description": "schema",
|
@@ -154,58 +147,55 @@ describe Csvlint::Schema do
|
|
154
147
|
{ "name": "Postcode", "constraints": { "required": true, "pattern": "[A-Z]{1,2}[0-9][0-9A-Z]? ?[0-9][A-Z]{2}" } }
|
155
148
|
]
|
156
149
|
}
|
157
|
-
|
158
|
-
stub_request(:get, "http://example.com/example.json").to_return(:
|
150
|
+
EOL
|
151
|
+
stub_request(:get, "http://example.com/example.json").to_return(status: 200, body: @example)
|
159
152
|
end
|
160
153
|
|
161
154
|
it "should create a schema from a pre-parsed JSON table" do
|
162
|
-
json = JSON.parse(
|
155
|
+
json = JSON.parse(@example)
|
163
156
|
schema = Csvlint::Schema.from_json_table("http://example.org", json)
|
164
157
|
|
165
|
-
expect(
|
166
|
-
expect(
|
167
|
-
expect(
|
168
|
-
expect(
|
169
|
-
expect(
|
170
|
-
expect(
|
171
|
-
expect(
|
172
|
-
expect(
|
173
|
-
expect(
|
158
|
+
expect(schema.uri).to eql("http://example.org")
|
159
|
+
expect(schema.title).to eql("Schema title")
|
160
|
+
expect(schema.description).to eql("schema")
|
161
|
+
expect(schema.fields.length).to eql(3)
|
162
|
+
expect(schema.fields[0].name).to eql("ID")
|
163
|
+
expect(schema.fields[0].constraints["required"]).to eql(true)
|
164
|
+
expect(schema.fields[0].title).to eql("id")
|
165
|
+
expect(schema.fields[0].description).to eql("house identifier")
|
166
|
+
expect(schema.fields[2].constraints["pattern"]).to eql("[A-Z]{1,2}[0-9][0-9A-Z]? ?[0-9][A-Z]{2}")
|
174
167
|
end
|
175
168
|
|
176
169
|
it "should create a schema from a JSON Table URL" do
|
177
170
|
schema = Csvlint::Schema.load_from_uri("http://example.com/example.json")
|
178
|
-
expect(
|
179
|
-
expect(
|
180
|
-
expect(
|
181
|
-
expect(
|
182
|
-
|
171
|
+
expect(schema.uri).to eql("http://example.com/example.json")
|
172
|
+
expect(schema.fields.length).to eql(3)
|
173
|
+
expect(schema.fields[0].name).to eql("ID")
|
174
|
+
expect(schema.fields[0].constraints["required"]).to eql(true)
|
183
175
|
end
|
184
176
|
end
|
185
177
|
|
186
178
|
context "when parsing CSVW metadata" do
|
187
|
-
|
188
179
|
before(:each) do
|
189
|
-
@example
|
190
|
-
{
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
}
|
201
|
-
|
202
|
-
stub_request(:get, "http://example.com/metadata.json").to_return(:
|
180
|
+
@example = <<~EOL
|
181
|
+
{
|
182
|
+
"@context": "http://www.w3.org/ns/csvw",
|
183
|
+
"url": "http://example.com/example1.csv",
|
184
|
+
"tableSchema": {
|
185
|
+
"columns": [
|
186
|
+
{ "name": "Name", "required": true, "datatype": { "base": "string", "format": ".+" } },
|
187
|
+
{ "name": "Id", "required": true, "datatype": { "base": "string", "minLength": 3 } },
|
188
|
+
{ "name": "Email", "required": true }
|
189
|
+
]
|
190
|
+
}
|
191
|
+
}
|
192
|
+
EOL
|
193
|
+
stub_request(:get, "http://example.com/metadata.json").to_return(status: 200, body: @example)
|
203
194
|
end
|
204
195
|
|
205
196
|
it "should create a table group from a CSVW metadata URL" do
|
206
197
|
schema = Csvlint::Schema.load_from_uri("http://example.com/metadata.json")
|
207
|
-
expect(
|
198
|
+
expect(schema.class).to eq(Csvlint::Csvw::TableGroup)
|
208
199
|
end
|
209
200
|
end
|
210
|
-
|
211
201
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
|
-
require
|
2
|
-
Coveralls.wear_merged!(
|
1
|
+
require "coveralls"
|
2
|
+
Coveralls.wear_merged!("test_frameworks")
|
3
3
|
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
4
|
+
require "csvlint"
|
5
|
+
require "byebug"
|
6
|
+
require "webmock/rspec"
|
7
7
|
|
8
8
|
RSpec.configure do |config|
|
9
9
|
config.run_all_when_everything_filtered = true
|
@@ -13,5 +13,5 @@ RSpec.configure do |config|
|
|
13
13
|
# order dependency and want to debug it, you can fix the order by providing
|
14
14
|
# the seed, which is printed after each run.
|
15
15
|
# --seed 1234
|
16
|
-
config.order =
|
16
|
+
config.order = "random"
|
17
17
|
end
|