csvlint 0.4.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/dependabot.yml +7 -0
- data/.github/workflows/push.yml +23 -0
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +38 -0
- data/README.md +5 -2
- data/bin/create_schema +2 -2
- data/csvlint.gemspec +6 -7
- data/features/support/aruba.rb +4 -3
- data/features/support/env.rb +1 -1
- data/features/support/load_tests.rb +12 -12
- data/features/support/webmock.rb +2 -0
- data/lib/csvlint/cli.rb +11 -20
- data/lib/csvlint/csvw/property_checker.rb +1 -1
- data/lib/csvlint/schema.rb +1 -1
- data/lib/csvlint/validate.rb +17 -13
- data/lib/csvlint/version.rb +1 -1
- data/lib/csvlint.rb +1 -1
- data/spec/schema_spec.rb +2 -2
- data/spec/spec_helper.rb +1 -1
- data/spec/validator_spec.rb +0 -13
- metadata +23 -34
- data/.travis.yml +0 -37
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 651e0cab26de36c97d333823f6ac72ad1377397e5e5dbf02397725fd8b42d663
|
|
4
|
+
data.tar.gz: cf81fa72bc040e242e947c8abb7009f7dd8ec6d5c9f08c21c9177eb62ab9c648
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: dbe214a6737375e2e5f4b6c08c63345ce850eaaea30373fab732a8352b84e58f862af5490d39dc18415f0256b92d0625c51c8e61ade9d2ef562050239340d985
|
|
7
|
+
data.tar.gz: 79296f466687b53a26a3e1c402a2ac7a6f2c2f999092a11567c897d71e3b78eca592cba97da211af5226e09ffd7086ed2cf0433cdeb14ad8c0635e5f6b55a7b8
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
branches: [ main ]
|
|
5
|
+
pull_request:
|
|
6
|
+
branches: [ main ]
|
|
7
|
+
jobs:
|
|
8
|
+
test:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
strategy:
|
|
11
|
+
matrix:
|
|
12
|
+
ruby-version: ['2.5', '2.6', '2.7', '3.0', '3.1']
|
|
13
|
+
fail-fast: false
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v2
|
|
16
|
+
- uses: ruby/setup-ruby@v1
|
|
17
|
+
with:
|
|
18
|
+
bundler-cache: true
|
|
19
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
20
|
+
- name: Install dependencies
|
|
21
|
+
run: bundle install
|
|
22
|
+
- name: Run the tests
|
|
23
|
+
run: bundle exec rake
|
data/.gitignore
CHANGED
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
3.1.2
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,43 @@
|
|
|
1
1
|
# Change Log
|
|
2
2
|
|
|
3
|
+
## [v1.0.0](https://github.com/Data-Liberation-Front/csvlint.rb/tree/v1.0.0) (2022-07-13)
|
|
4
|
+
[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.4.0...v1.0.0)
|
|
5
|
+
|
|
6
|
+
Support Ruby 3.x, and DROPPED support for Ruby 2.4 - that's why the major version bump. That and this has been around long enough that it really shouldn't be on a zero version any more :)
|
|
7
|
+
|
|
8
|
+
## What's Changed
|
|
9
|
+
* Don't patch CSV#init_converters for ruby 2.5 compatibility by @rbmrclo in https://github.com/Data-Liberation-Front/csvlint.rb/pull/217
|
|
10
|
+
* correct typos in README by @erikj in https://github.com/Data-Liberation-Front/csvlint.rb/pull/216
|
|
11
|
+
* add info about your PATH by @ftrotter in https://github.com/Data-Liberation-Front/csvlint.rb/pull/222
|
|
12
|
+
* Remove tests on deprecated ruby versions < 2.3 by @Floppy in https://github.com/Data-Liberation-Front/csvlint.rb/pull/234
|
|
13
|
+
* Drop mime-types gem dependency by @ohbarye in https://github.com/Data-Liberation-Front/csvlint.rb/pull/221
|
|
14
|
+
* remove specific version of net-http-persistent in gemspec by @kotaro0522 in https://github.com/Data-Liberation-Front/csvlint.rb/pull/219
|
|
15
|
+
* Replace colorize with rainbow to make licensing consistent. by @cobbr2 in https://github.com/Data-Liberation-Front/csvlint.rb/pull/215
|
|
16
|
+
* Update rdf requirement from < 2.0 to < 4.0 by @dependabot-preview in https://github.com/Data-Liberation-Front/csvlint.rb/pull/231
|
|
17
|
+
* Test on Ruby 2.5 and 2.6 by @Domon in https://github.com/Data-Liberation-Front/csvlint.rb/pull/223
|
|
18
|
+
* Fix load_from_json deprecation warnings. by @jezhiggins in https://github.com/Data-Liberation-Front/csvlint.rb/pull/237
|
|
19
|
+
* Fix csvw tests by @Floppy in https://github.com/Data-Liberation-Front/csvlint.rb/pull/239
|
|
20
|
+
* Test on Ruby 2.6 and 2.7 by @Floppy in https://github.com/Data-Liberation-Front/csvlint.rb/pull/240
|
|
21
|
+
* Create Dependabot config file by @dependabot-preview in https://github.com/Data-Liberation-Front/csvlint.rb/pull/245
|
|
22
|
+
* Include active_support/object to ensure this works in ruby 2.6 by @mseverini in https://github.com/Data-Liberation-Front/csvlint.rb/pull/246
|
|
23
|
+
* add CI workflow for github actions by @Floppy in https://github.com/Data-Liberation-Front/csvlint.rb/pull/255
|
|
24
|
+
* Enable and fix tests for Ruby 2.5 by @Floppy in https://github.com/Data-Liberation-Front/csvlint.rb/pull/259
|
|
25
|
+
* Support Ruby 2.6 by @Floppy in https://github.com/Data-Liberation-Front/csvlint.rb/pull/262
|
|
26
|
+
* Ruby 2.7 support by @Floppy in https://github.com/Data-Liberation-Front/csvlint.rb/pull/263
|
|
27
|
+
* Drop support for Ruby 2.4 by @Floppy in https://github.com/Data-Liberation-Front/csvlint.rb/pull/265
|
|
28
|
+
* Ruby 3.0 by @Floppy in https://github.com/Data-Liberation-Front/csvlint.rb/pull/264
|
|
29
|
+
|
|
30
|
+
## New Contributors
|
|
31
|
+
* @rbmrclo made their first contribution in https://github.com/Data-Liberation-Front/csvlint.rb/pull/217
|
|
32
|
+
* @erikj made their first contribution in https://github.com/Data-Liberation-Front/csvlint.rb/pull/216
|
|
33
|
+
* @ftrotter made their first contribution in https://github.com/Data-Liberation-Front/csvlint.rb/pull/222
|
|
34
|
+
* @ohbarye made their first contribution in https://github.com/Data-Liberation-Front/csvlint.rb/pull/221
|
|
35
|
+
* @kotaro0522 made their first contribution in https://github.com/Data-Liberation-Front/csvlint.rb/pull/219
|
|
36
|
+
* @cobbr2 made their first contribution in https://github.com/Data-Liberation-Front/csvlint.rb/pull/215
|
|
37
|
+
* @dependabot-preview made their first contribution in https://github.com/Data-Liberation-Front/csvlint.rb/pull/231
|
|
38
|
+
* @Domon made their first contribution in https://github.com/Data-Liberation-Front/csvlint.rb/pull/223
|
|
39
|
+
* @mseverini made their first contribution in https://github.com/Data-Liberation-Front/csvlint.rb/pull/246
|
|
40
|
+
|
|
3
41
|
## [0.4.0](https://github.com/theodi/csvlint.rb/tree/0.4.0) (2017-xx-xx)
|
|
4
42
|
[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.3.3...0.4.0)
|
|
5
43
|
|
data/README.md
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
# CSV Lint
|
|
8
8
|
|
|
9
|
-
A ruby gem to support validating CSV files to check their syntax and contents. You can either use this gem within your own Ruby code, or as a
|
|
9
|
+
A ruby gem to support validating CSV files to check their syntax and contents. You can either use this gem within your own Ruby code, or as a standalone command line application
|
|
10
10
|
|
|
11
11
|
## Summary of features
|
|
12
12
|
|
|
@@ -63,7 +63,7 @@ Or install it yourself as:
|
|
|
63
63
|
|
|
64
64
|
## Usage
|
|
65
65
|
|
|
66
|
-
You can either use this gem within your own Ruby code, or as a
|
|
66
|
+
You can either use this gem within your own Ruby code, or as a standalone command line application
|
|
67
67
|
|
|
68
68
|
## On the command line
|
|
69
69
|
|
|
@@ -71,6 +71,9 @@ After installing the gem, you can validate a CSV on the command line like so:
|
|
|
71
71
|
|
|
72
72
|
csvlint myfile.csv
|
|
73
73
|
|
|
74
|
+
You may need to add the gem exectuable directory to your path, by adding '/usr/local/lib/ruby/gems/2.6.0/bin'
|
|
75
|
+
or whatever your version is, to your .bash_profile PATH entry. [like so](https://stackoverflow.com/questions/2392293/ruby-gems-returns-command-not-found)
|
|
76
|
+
|
|
74
77
|
You will then see the validation result, together with any warnings or errors e.g.
|
|
75
78
|
|
|
76
79
|
```
|
data/bin/create_schema
CHANGED
|
@@ -5,7 +5,7 @@ require 'csvlint'
|
|
|
5
5
|
|
|
6
6
|
begin
|
|
7
7
|
puts ARGV[0]
|
|
8
|
-
|
|
8
|
+
csv = CSV.new( URI.open(ARGV[0]) )
|
|
9
9
|
headers = csv.shift
|
|
10
10
|
|
|
11
11
|
name = File.basename( ARGV[0] )
|
|
@@ -29,4 +29,4 @@ rescue => e
|
|
|
29
29
|
puts e
|
|
30
30
|
puts e.backtrace
|
|
31
31
|
puts "Unable to parse CSV file"
|
|
32
|
-
end
|
|
32
|
+
end
|
data/csvlint.gemspec
CHANGED
|
@@ -18,10 +18,9 @@ Gem::Specification.new do |spec|
|
|
|
18
18
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
|
19
19
|
spec.require_paths = ["lib"]
|
|
20
20
|
|
|
21
|
-
spec.required_ruby_version = ['
|
|
21
|
+
spec.required_ruby_version = ['>= 2.5', '< 3.2']
|
|
22
22
|
|
|
23
|
-
spec.add_dependency "
|
|
24
|
-
spec.add_dependency "colorize"
|
|
23
|
+
spec.add_dependency "rainbow"
|
|
25
24
|
spec.add_dependency "open_uri_redirections"
|
|
26
25
|
spec.add_dependency "activesupport"
|
|
27
26
|
spec.add_dependency "addressable"
|
|
@@ -30,9 +29,9 @@ Gem::Specification.new do |spec|
|
|
|
30
29
|
spec.add_dependency "uri_template"
|
|
31
30
|
spec.add_dependency "thor"
|
|
32
31
|
spec.add_dependency "rack"
|
|
33
|
-
spec.add_dependency "net-http-persistent"
|
|
32
|
+
spec.add_dependency "net-http-persistent"
|
|
34
33
|
|
|
35
|
-
spec.add_development_dependency "bundler", "
|
|
34
|
+
spec.add_development_dependency "bundler", ">= 1.3"
|
|
36
35
|
spec.add_development_dependency "rake"
|
|
37
36
|
spec.add_development_dependency "cucumber"
|
|
38
37
|
spec.add_development_dependency "simplecov"
|
|
@@ -43,10 +42,10 @@ Gem::Specification.new do |spec|
|
|
|
43
42
|
spec.add_development_dependency "rspec-pride"
|
|
44
43
|
spec.add_development_dependency "rspec-expectations"
|
|
45
44
|
spec.add_development_dependency "coveralls"
|
|
46
|
-
spec.add_development_dependency "
|
|
45
|
+
spec.add_development_dependency "byebug"
|
|
47
46
|
spec.add_development_dependency "github_changelog_generator"
|
|
48
47
|
spec.add_development_dependency "aruba"
|
|
49
|
-
spec.add_development_dependency "rdf", "<
|
|
48
|
+
spec.add_development_dependency "rdf", "< 4.0"
|
|
50
49
|
spec.add_development_dependency "rdf-turtle"
|
|
51
50
|
spec.add_development_dependency "henry"
|
|
52
51
|
|
data/features/support/aruba.rb
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
require 'aruba'
|
|
2
|
-
require 'aruba/in_process'
|
|
3
2
|
require 'aruba/cucumber'
|
|
4
3
|
|
|
5
4
|
require 'csvlint/cli'
|
|
@@ -52,5 +51,7 @@ module Csvlint
|
|
|
52
51
|
end
|
|
53
52
|
end
|
|
54
53
|
|
|
55
|
-
Aruba.
|
|
56
|
-
|
|
54
|
+
Aruba.configure do |config|
|
|
55
|
+
config.command_launcher = :in_process
|
|
56
|
+
config.main_class = Csvlint::CliRunner
|
|
57
|
+
end
|
data/features/support/env.rb
CHANGED
|
@@ -2,7 +2,7 @@ require 'json'
|
|
|
2
2
|
require 'open-uri'
|
|
3
3
|
require 'uri'
|
|
4
4
|
|
|
5
|
-
BASE_URI = "
|
|
5
|
+
BASE_URI = "https://w3c.github.io/csvw/tests/"
|
|
6
6
|
BASE_PATH = File.join(File.dirname(__FILE__), "..", "fixtures", "csvw")
|
|
7
7
|
FEATURE_BASE_PATH = File.join(File.dirname(__FILE__), "..")
|
|
8
8
|
VALIDATION_FEATURE_FILE_PATH = File.join(FEATURE_BASE_PATH, "csvw_validation_tests.feature")
|
|
@@ -23,7 +23,7 @@ def cache_file(filename)
|
|
|
23
23
|
end
|
|
24
24
|
STDERR.puts("storing #{file} locally")
|
|
25
25
|
File.open(file, 'wb') do |f|
|
|
26
|
-
|
|
26
|
+
f.puts URI.open(uri, 'rb').read
|
|
27
27
|
end
|
|
28
28
|
end
|
|
29
29
|
return uri, file
|
|
@@ -31,14 +31,14 @@ end
|
|
|
31
31
|
|
|
32
32
|
File.open(SCRIPT_FILE_PATH, 'w') do |file|
|
|
33
33
|
File.chmod(0755, SCRIPT_FILE_PATH)
|
|
34
|
-
|
|
34
|
+
manifest = JSON.parse( URI.open("#{BASE_URI}manifest-validation.jsonld").read )
|
|
35
35
|
manifest["entries"].each do |entry|
|
|
36
36
|
type = "valid"
|
|
37
|
-
case entry["type"]
|
|
38
|
-
when "csvt:WarningValidationTest"
|
|
39
|
-
type = "warnings"
|
|
40
|
-
when "csvt:NegativeValidationTest"
|
|
41
|
-
type = "errors"
|
|
37
|
+
case entry["type"]
|
|
38
|
+
when "csvt:WarningValidationTest"
|
|
39
|
+
type = "warnings"
|
|
40
|
+
when "csvt:NegativeValidationTest"
|
|
41
|
+
type = "errors"
|
|
42
42
|
end
|
|
43
43
|
file.puts "echo \"#{entry["id"].split("#")[-1]}: #{entry["name"].gsub("`", "'")}\""
|
|
44
44
|
file.puts "echo \"#{type}: #{entry["comment"].gsub("\"", "\\\"").gsub("`", "'")}\""
|
|
@@ -54,14 +54,14 @@ File.open(SCRIPT_FILE_PATH, 'w') do |file|
|
|
|
54
54
|
end unless File.exist? SCRIPT_FILE_PATH
|
|
55
55
|
|
|
56
56
|
File.open(VALIDATION_FEATURE_FILE_PATH, 'w') do |file|
|
|
57
|
-
file.puts "# Auto-generated file based on standard validation CSVW tests from
|
|
57
|
+
file.puts "# Auto-generated file based on standard validation CSVW tests from #{BASE_URI}manifest-validation.jsonld"
|
|
58
58
|
file.puts ""
|
|
59
59
|
|
|
60
|
-
|
|
60
|
+
manifest = JSON.parse( URI.open("#{BASE_URI}manifest-validation.jsonld").read )
|
|
61
61
|
|
|
62
62
|
file.puts "Feature: #{manifest["label"]}"
|
|
63
63
|
file.puts ""
|
|
64
|
-
|
|
64
|
+
|
|
65
65
|
manifest["entries"].each do |entry|
|
|
66
66
|
action_uri, action_file = cache_file(entry["action"])
|
|
67
67
|
metadata = nil
|
|
@@ -85,7 +85,7 @@ File.open(VALIDATION_FEATURE_FILE_PATH, 'w') do |file|
|
|
|
85
85
|
end
|
|
86
86
|
provided_files << action_uri.to_s
|
|
87
87
|
if entry["name"].include?("/.well-known/csvm")
|
|
88
|
-
file.puts "\t\tAnd I have a file called \"w3.org/.well-known/csvm\" at the url \"
|
|
88
|
+
file.puts "\t\tAnd I have a file called \"w3.org/.well-known/csvm\" at the url \"https://www.w3.org/.well-known/csvm\""
|
|
89
89
|
missing_files << "#{action_uri}.json"
|
|
90
90
|
missing_files << URI.join(action_uri, 'csvm.json').to_s
|
|
91
91
|
else
|
data/features/support/webmock.rb
CHANGED
data/lib/csvlint/cli.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
require 'csvlint'
|
|
2
|
-
require '
|
|
2
|
+
require 'rainbow'
|
|
3
|
+
require 'active_support/json'
|
|
3
4
|
require 'json'
|
|
4
5
|
require 'pp'
|
|
5
6
|
require 'thor'
|
|
@@ -21,6 +22,8 @@ module Csvlint
|
|
|
21
22
|
@schema = get_schema(options[:schema]) if options[:schema]
|
|
22
23
|
fetch_schema_tables(@schema, options) if source.nil?
|
|
23
24
|
|
|
25
|
+
Rainbow.enabled = $stdout.tty?
|
|
26
|
+
|
|
24
27
|
valid = validate_csv(source, @schema, options[:dump_errors], options[:json], options[:werror])
|
|
25
28
|
exit 1 unless valid
|
|
26
29
|
end
|
|
@@ -56,7 +59,7 @@ module Csvlint
|
|
|
56
59
|
|
|
57
60
|
def get_schema(schema)
|
|
58
61
|
begin
|
|
59
|
-
schema = Csvlint::Schema.
|
|
62
|
+
schema = Csvlint::Schema.load_from_uri(schema, false)
|
|
60
63
|
rescue Csvlint::Csvw::MetadataError => e
|
|
61
64
|
return_error "invalid metadata: #{e.message}#{" at " + e.path if e.path}"
|
|
62
65
|
rescue OpenURI::HTTPError, Errno::ENOENT
|
|
@@ -106,11 +109,7 @@ module Csvlint
|
|
|
106
109
|
output_string += ". #{location}" unless location.empty?
|
|
107
110
|
output_string += ". #{error.content}" if error.content
|
|
108
111
|
|
|
109
|
-
|
|
110
|
-
puts output_string.colorize(color)
|
|
111
|
-
else
|
|
112
|
-
puts output_string
|
|
113
|
-
end
|
|
112
|
+
puts Rainbow(output_string).color(color)
|
|
114
113
|
|
|
115
114
|
if dump
|
|
116
115
|
pp error
|
|
@@ -124,11 +123,7 @@ module Csvlint
|
|
|
124
123
|
end
|
|
125
124
|
|
|
126
125
|
def return_error(message)
|
|
127
|
-
|
|
128
|
-
puts message.colorize(:red)
|
|
129
|
-
else
|
|
130
|
-
puts message
|
|
131
|
-
end
|
|
126
|
+
puts Rainbow(message).red
|
|
132
127
|
exit 1
|
|
133
128
|
end
|
|
134
129
|
|
|
@@ -160,15 +155,11 @@ module Csvlint
|
|
|
160
155
|
}.to_json
|
|
161
156
|
print json
|
|
162
157
|
else
|
|
163
|
-
|
|
164
|
-
puts "\r\n#{csv} is #{validator.valid? ? "VALID".green : "INVALID".red}"
|
|
165
|
-
else
|
|
166
|
-
puts "\r\n#{csv} is #{validator.valid? ? "VALID" : "INVALID"}"
|
|
167
|
-
end
|
|
158
|
+
puts "\r\n#{csv} is #{validator.valid? ? Rainbow("VALID").green : Rainbow("INVALID").red}"
|
|
168
159
|
print_errors(validator.errors, dump)
|
|
169
160
|
print_errors(validator.warnings, dump)
|
|
170
161
|
end
|
|
171
|
-
|
|
162
|
+
|
|
172
163
|
return false if werror && validator.warnings.size > 0
|
|
173
164
|
return validator.valid?
|
|
174
165
|
end
|
|
@@ -194,9 +185,9 @@ module Csvlint
|
|
|
194
185
|
lambda do |row|
|
|
195
186
|
new_errors = row.errors.count
|
|
196
187
|
if new_errors > @error_count
|
|
197
|
-
print "!".red
|
|
188
|
+
print Rainbow("!").red
|
|
198
189
|
else
|
|
199
|
-
print ".".green
|
|
190
|
+
print Rainbow(".").green
|
|
200
191
|
end
|
|
201
192
|
@error_count = new_errors
|
|
202
193
|
end
|
|
@@ -455,7 +455,7 @@ module Csvlint
|
|
|
455
455
|
schema_url = URI.join(base_url, value).to_s
|
|
456
456
|
schema_base_url = schema_url
|
|
457
457
|
schema_ref = schema_url.start_with?("file:") ? File.new(schema_url[5..-1]) : schema_url
|
|
458
|
-
schema = JSON.parse( open(schema_ref).read )
|
|
458
|
+
schema = JSON.parse( URI.open(schema_ref).read )
|
|
459
459
|
schema["@id"] = schema["@id"] ? URI.join(schema_url, schema["@id"]).to_s : schema_url
|
|
460
460
|
if schema["@context"]
|
|
461
461
|
if schema["@context"].instance_of?(Array) && schema["@context"].length > 1
|
data/lib/csvlint/schema.rb
CHANGED
|
@@ -38,7 +38,7 @@ module Csvlint
|
|
|
38
38
|
deprecate :load_from_json, :load_from_uri, 2018, 1
|
|
39
39
|
|
|
40
40
|
def load_from_uri(uri, output_errors = true)
|
|
41
|
-
load_from_string(uri, open(uri).read, output_errors)
|
|
41
|
+
load_from_string(uri, URI.open(uri).read, output_errors)
|
|
42
42
|
rescue OpenURI::HTTPError, Errno::ENOENT => e
|
|
43
43
|
raise e
|
|
44
44
|
end
|
data/lib/csvlint/validate.rb
CHANGED
|
@@ -38,13 +38,15 @@ module Csvlint
|
|
|
38
38
|
ESCAPE_RE[@re_chars][@re_esc][str]
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
41
|
+
if RUBY_VERSION < '2.5'
|
|
42
|
+
# Optimization: Disable the CSV library's converters feature.
|
|
43
|
+
# @see https://github.com/ruby/ruby/blob/v2_2_3/lib/csv.rb#L2100
|
|
44
|
+
def init_converters(options, field_name = :converters)
|
|
45
|
+
@converters = []
|
|
46
|
+
@header_converters = []
|
|
47
|
+
options.delete(:unconverted_fields)
|
|
48
|
+
options.delete(field_name)
|
|
49
|
+
end
|
|
48
50
|
end
|
|
49
51
|
end
|
|
50
52
|
|
|
@@ -56,6 +58,7 @@ module Csvlint
|
|
|
56
58
|
"Missing or stray quote" => :stray_quote,
|
|
57
59
|
"Illegal quoting" => :whitespace,
|
|
58
60
|
"Unclosed quoted field" => :unclosed_quote,
|
|
61
|
+
"Any value after quoted field isn't allowed" => :unclosed_quote,
|
|
59
62
|
"Unquoted fields do not allow \\r or \\n" => :line_breaks,
|
|
60
63
|
}
|
|
61
64
|
|
|
@@ -179,9 +182,9 @@ module Csvlint
|
|
|
179
182
|
@csv_options[:encoding] = @encoding
|
|
180
183
|
|
|
181
184
|
begin
|
|
182
|
-
row = LineCSV.parse_line(stream,
|
|
185
|
+
row = LineCSV.parse_line(stream, **@csv_options)
|
|
183
186
|
rescue LineCSV::MalformedCSVError => e
|
|
184
|
-
build_exception_messages(e, stream, current_line)
|
|
187
|
+
build_exception_messages(e, stream, current_line) unless e.message.include?("UTF") && @reported_invalid_encoding
|
|
185
188
|
end
|
|
186
189
|
|
|
187
190
|
if row
|
|
@@ -250,7 +253,7 @@ module Csvlint
|
|
|
250
253
|
if rel == "describedby" && param == "type" && ["application/csvm+json", "application/ld+json", "application/json"].include?(param_value)
|
|
251
254
|
begin
|
|
252
255
|
url = URI.join(@source_url, uri)
|
|
253
|
-
schema = Schema.
|
|
256
|
+
schema = Schema.load_from_uri(url)
|
|
254
257
|
if schema.instance_of? Csvlint::Csvw::TableGroup
|
|
255
258
|
if schema.tables[@source_url]
|
|
256
259
|
@schema = schema
|
|
@@ -434,7 +437,8 @@ module Csvlint
|
|
|
434
437
|
when StringIO
|
|
435
438
|
return
|
|
436
439
|
when File
|
|
437
|
-
|
|
440
|
+
uri_parser = URI::Parser.new
|
|
441
|
+
@source_url = "file:#{uri_parser.escape(File.expand_path(@source))}"
|
|
438
442
|
else
|
|
439
443
|
@source_url = @source
|
|
440
444
|
end
|
|
@@ -449,7 +453,7 @@ module Csvlint
|
|
|
449
453
|
if @source_url =~ /^http(s)?/
|
|
450
454
|
begin
|
|
451
455
|
well_known_uri = URI.join(@source_url, "/.well-known/csvm")
|
|
452
|
-
paths = open(well_known_uri).read.split("\n")
|
|
456
|
+
paths = URI.open(well_known_uri.to_s).read.split("\n")
|
|
453
457
|
rescue OpenURI::HTTPError, URI::BadURIError
|
|
454
458
|
end
|
|
455
459
|
end
|
|
@@ -460,7 +464,7 @@ module Csvlint
|
|
|
460
464
|
path = template.expand('url' => @source_url)
|
|
461
465
|
url = URI.join(@source_url, path)
|
|
462
466
|
url = File.new(url.to_s.sub(/^file:/, "")) if url.to_s =~ /^file:/
|
|
463
|
-
schema = Schema.
|
|
467
|
+
schema = Schema.load_from_uri(url)
|
|
464
468
|
if schema.instance_of? Csvlint::Csvw::TableGroup
|
|
465
469
|
if schema.tables[@source_url]
|
|
466
470
|
@schema = schema
|
data/lib/csvlint/version.rb
CHANGED
data/lib/csvlint.rb
CHANGED
data/spec/schema_spec.rb
CHANGED
|
@@ -174,7 +174,7 @@ describe Csvlint::Schema do
|
|
|
174
174
|
end
|
|
175
175
|
|
|
176
176
|
it "should create a schema from a JSON Table URL" do
|
|
177
|
-
schema = Csvlint::Schema.
|
|
177
|
+
schema = Csvlint::Schema.load_from_uri("http://example.com/example.json")
|
|
178
178
|
expect( schema.uri ).to eql("http://example.com/example.json")
|
|
179
179
|
expect( schema.fields.length ).to eql(3)
|
|
180
180
|
expect( schema.fields[0].name ).to eql("ID")
|
|
@@ -203,7 +203,7 @@ describe Csvlint::Schema do
|
|
|
203
203
|
end
|
|
204
204
|
|
|
205
205
|
it "should create a table group from a CSVW metadata URL" do
|
|
206
|
-
schema = Csvlint::Schema.
|
|
206
|
+
schema = Csvlint::Schema.load_from_uri("http://example.com/metadata.json")
|
|
207
207
|
expect( schema.class ).to eq(Csvlint::Csvw::TableGroup)
|
|
208
208
|
end
|
|
209
209
|
end
|
data/spec/spec_helper.rb
CHANGED
data/spec/validator_spec.rb
CHANGED
|
@@ -76,19 +76,6 @@ describe Csvlint::Validator do
|
|
|
76
76
|
expect(validator.errors.first.type).to eql(:unclosed_quote)
|
|
77
77
|
end
|
|
78
78
|
|
|
79
|
-
it ".each() -> `parse_contents` parses malformed CSV and catches stray quote" do
|
|
80
|
-
pending "cannot make Ruby generate a stray quote error"
|
|
81
|
-
# doesn't build warnings because check_consistency isn't invoked
|
|
82
|
-
# TODO below is trailing whitespace but is interpreted as a stray quote
|
|
83
|
-
data = StringIO.new(%Q{"Foo","Bar","Baz"\r\n"1","2","3"\r\n"1","2","3"\r\n"3","2","1""})
|
|
84
|
-
|
|
85
|
-
validator = Csvlint::Validator.new(data)
|
|
86
|
-
|
|
87
|
-
expect(validator.valid?).to eql(false)
|
|
88
|
-
expect(validator.errors.first.type).to eql(:stray_quote)
|
|
89
|
-
expect(validator.errors.count).to eql(1)
|
|
90
|
-
end
|
|
91
|
-
|
|
92
79
|
it ".each() -> `parse_contents` parses malformed CSV and catches whitespace and edge case" do
|
|
93
80
|
# when this data gets passed the header it rescues a whitespace error, resulting in the header row being discarded
|
|
94
81
|
# TODO - check if this is an edge case, currently passing because it requires advice on how to specify
|
metadata
CHANGED
|
@@ -1,31 +1,17 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: csvlint
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 1.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- pezholio
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-07-13 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
|
-
name:
|
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
|
16
|
-
requirements:
|
|
17
|
-
- - ">="
|
|
18
|
-
- !ruby/object:Gem::Version
|
|
19
|
-
version: '0'
|
|
20
|
-
type: :runtime
|
|
21
|
-
prerelease: false
|
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
-
requirements:
|
|
24
|
-
- - ">="
|
|
25
|
-
- !ruby/object:Gem::Version
|
|
26
|
-
version: '0'
|
|
27
|
-
- !ruby/object:Gem::Dependency
|
|
28
|
-
name: colorize
|
|
14
|
+
name: rainbow
|
|
29
15
|
requirement: !ruby/object:Gem::Requirement
|
|
30
16
|
requirements:
|
|
31
17
|
- - ">="
|
|
@@ -154,28 +140,28 @@ dependencies:
|
|
|
154
140
|
name: net-http-persistent
|
|
155
141
|
requirement: !ruby/object:Gem::Requirement
|
|
156
142
|
requirements:
|
|
157
|
-
- - "
|
|
143
|
+
- - ">="
|
|
158
144
|
- !ruby/object:Gem::Version
|
|
159
|
-
version: '
|
|
145
|
+
version: '0'
|
|
160
146
|
type: :runtime
|
|
161
147
|
prerelease: false
|
|
162
148
|
version_requirements: !ruby/object:Gem::Requirement
|
|
163
149
|
requirements:
|
|
164
|
-
- - "
|
|
150
|
+
- - ">="
|
|
165
151
|
- !ruby/object:Gem::Version
|
|
166
|
-
version: '
|
|
152
|
+
version: '0'
|
|
167
153
|
- !ruby/object:Gem::Dependency
|
|
168
154
|
name: bundler
|
|
169
155
|
requirement: !ruby/object:Gem::Requirement
|
|
170
156
|
requirements:
|
|
171
|
-
- - "
|
|
157
|
+
- - ">="
|
|
172
158
|
- !ruby/object:Gem::Version
|
|
173
159
|
version: '1.3'
|
|
174
160
|
type: :development
|
|
175
161
|
prerelease: false
|
|
176
162
|
version_requirements: !ruby/object:Gem::Requirement
|
|
177
163
|
requirements:
|
|
178
|
-
- - "
|
|
164
|
+
- - ">="
|
|
179
165
|
- !ruby/object:Gem::Version
|
|
180
166
|
version: '1.3'
|
|
181
167
|
- !ruby/object:Gem::Dependency
|
|
@@ -319,7 +305,7 @@ dependencies:
|
|
|
319
305
|
- !ruby/object:Gem::Version
|
|
320
306
|
version: '0'
|
|
321
307
|
- !ruby/object:Gem::Dependency
|
|
322
|
-
name:
|
|
308
|
+
name: byebug
|
|
323
309
|
requirement: !ruby/object:Gem::Requirement
|
|
324
310
|
requirements:
|
|
325
311
|
- - ">="
|
|
@@ -366,14 +352,14 @@ dependencies:
|
|
|
366
352
|
requirements:
|
|
367
353
|
- - "<"
|
|
368
354
|
- !ruby/object:Gem::Version
|
|
369
|
-
version: '
|
|
355
|
+
version: '4.0'
|
|
370
356
|
type: :development
|
|
371
357
|
prerelease: false
|
|
372
358
|
version_requirements: !ruby/object:Gem::Requirement
|
|
373
359
|
requirements:
|
|
374
360
|
- - "<"
|
|
375
361
|
- !ruby/object:Gem::Version
|
|
376
|
-
version: '
|
|
362
|
+
version: '4.0'
|
|
377
363
|
- !ruby/object:Gem::Dependency
|
|
378
364
|
name: rdf-turtle
|
|
379
365
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -415,9 +401,10 @@ files:
|
|
|
415
401
|
- ".gitattributes"
|
|
416
402
|
- ".github/ISSUE_TEMPLATE.md"
|
|
417
403
|
- ".github/PULL_REQUEST_TEMPLATE.md"
|
|
404
|
+
- ".github/dependabot.yml"
|
|
405
|
+
- ".github/workflows/push.yml"
|
|
418
406
|
- ".gitignore"
|
|
419
407
|
- ".ruby-version"
|
|
420
|
-
- ".travis.yml"
|
|
421
408
|
- CHANGELOG.md
|
|
422
409
|
- CODE_OF_CONDUCT.md
|
|
423
410
|
- CONTRIBUTING.md
|
|
@@ -497,24 +484,26 @@ homepage: https://github.com/theodi/csvlint.rb
|
|
|
497
484
|
licenses:
|
|
498
485
|
- MIT
|
|
499
486
|
metadata: {}
|
|
500
|
-
post_install_message:
|
|
487
|
+
post_install_message:
|
|
501
488
|
rdoc_options: []
|
|
502
489
|
require_paths:
|
|
503
490
|
- lib
|
|
504
491
|
required_ruby_version: !ruby/object:Gem::Requirement
|
|
505
492
|
requirements:
|
|
506
|
-
- - "
|
|
493
|
+
- - ">="
|
|
494
|
+
- !ruby/object:Gem::Version
|
|
495
|
+
version: '2.5'
|
|
496
|
+
- - "<"
|
|
507
497
|
- !ruby/object:Gem::Version
|
|
508
|
-
version: '2
|
|
498
|
+
version: '3.2'
|
|
509
499
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
510
500
|
requirements:
|
|
511
501
|
- - ">="
|
|
512
502
|
- !ruby/object:Gem::Version
|
|
513
503
|
version: '0'
|
|
514
504
|
requirements: []
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
signing_key:
|
|
505
|
+
rubygems_version: 3.3.7
|
|
506
|
+
signing_key:
|
|
518
507
|
specification_version: 4
|
|
519
508
|
summary: CSV Validator
|
|
520
509
|
test_files:
|
data/.travis.yml
DELETED
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
rvm:
|
|
3
|
-
- 2.0.0-p648
|
|
4
|
-
- 2.1.10
|
|
5
|
-
- 2.2.7
|
|
6
|
-
- 2.3.4
|
|
7
|
-
- 2.4.1
|
|
8
|
-
sudo: false
|
|
9
|
-
cache: bundler
|
|
10
|
-
notifications:
|
|
11
|
-
irc:
|
|
12
|
-
channels:
|
|
13
|
-
- irc.freenode.net#theodi
|
|
14
|
-
template:
|
|
15
|
-
- "%{repository} %{branch} - %{message} %{build_url}"
|
|
16
|
-
on_success: change
|
|
17
|
-
on_failure: always
|
|
18
|
-
deploy:
|
|
19
|
-
provider: rubygems
|
|
20
|
-
api_key:
|
|
21
|
-
secure: fjnPS61/skQ1PsRJu1SYWIct0vjdkTyQhj0ria9zfcJfbWbfiHnpehVh1ege3sTUkDSTvoOFT35jzEeozvtKZOlAWMU5QbL8LTXu+JSp9olOdSuGeWRWVuQT3NLgRJW0+2c7N66piZvnRUUTyt2P8VIR8c/Ltuhc32bUGL7X6Gw=
|
|
22
|
-
gem: csvlint
|
|
23
|
-
true:
|
|
24
|
-
tags: true
|
|
25
|
-
repo: theodi/csvlint.rb
|
|
26
|
-
all_branches: true
|
|
27
|
-
after_success:
|
|
28
|
-
- export GEM_VERSION=`ruby -e "puts Gem::Specification.load(Dir['*.gemspec'].first).version.to_s"`
|
|
29
|
-
- '[ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ] && curl
|
|
30
|
-
-v -X POST -d ''{"ref":"refs/tags/''$GEM_VERSION''","sha":"''$TRAVIS_COMMIT''"}'' --header
|
|
31
|
-
"Content-Type:application/json" -u $GITHUB_USER:$GITHUB_PASSWORD "https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs"'
|
|
32
|
-
- bundle exec henry deploy
|
|
33
|
-
env:
|
|
34
|
-
global:
|
|
35
|
-
- secure: hM8Pv/EAZoB4fGGEUwKQQ/2+mgKbUac+PH2cwud25l2MJ64eoKEMZzfy4TF6+XMIzXhlWpsg4s51y6K3+W8pUyRAuwo2MsN5Iee7HnLz+LlYTiT0//iQdAmxQ0JlOyh8vd2SEFBcVfwpp/iZFIHAfbBO73ZDXhtGMEieMk025I0=
|
|
36
|
-
- secure: hsKbZ0kVKVU/e8oVCPJpRmBxbbieN2tVbyjdhJo/UYchcyLWeGAmAj/ApUDL4SyR0HZxpfi+SIsGFNRy5LnrFeCFgY9aZ6u9ma4n0Y2cTElMRt5kvI/c28FehsRjCzAe6W8dxwhJ8wSkvFDNpLml47/iy8bw4a7aUZDdq3OYXck=
|
|
37
|
-
- secure: FW0iyoq4yS55yVTxWXv0OrnOQWuUzpsv32w0FsE7E6OgS/UtCSAQeXO+T448+JXVbJClQDQNM9Pimdfzz/znGzHF6//UVmIbxfKwbyGtvzg4dZUexK8KlGnQy1na/4qiOi5xBSQQ5q/CwtblkCkklhDIs8CwV1M7mCIK5Sf0HZ4=
|