csvlint 1.0.0 → 1.2.0

data/features/support/load_tests.rb

@@ -1,6 +1,6 @@
-require 'json'
-require 'open-uri'
-require 'uri'
+require "json"
+require "open-uri"
+require "uri"
 
 BASE_URI = "https://w3c.github.io/csvw/tests/"
 BASE_PATH = File.join(File.dirname(__FILE__), "..", "fixtures", "csvw")
@@ -11,109 +11,113 @@ SCRIPT_FILE_PATH = File.join(File.dirname(__FILE__), "..", "..", "bin", "run-csv
 Dir.mkdir(BASE_PATH) unless Dir.exist?(BASE_PATH)
 
 def cache_file(filename)
-	file = File.join(BASE_PATH, filename)
-	uri = URI.join(BASE_URI, filename)
-	unless File.exist?(file)
-		if filename.include? "/"
-			levels = filename.split("/")[0..-2]
-			for i in 0..levels.length
-				dir = File.join(BASE_PATH, levels[0..i].join("/"))
-				Dir.mkdir(dir) unless Dir.exist?(dir)
-			end
-		end
-		STDERR.puts("storing #{file} locally")
-		File.open(file, 'wb') do |f|
-      f.puts URI.open(uri, 'rb').read
-		end
-	end
-	return uri, file
+  file = File.join(BASE_PATH, filename)
+  uri = URI.join(BASE_URI, filename)
+  unless File.exist?(file)
+    if filename.include? "/"
+      levels = filename.split("/")[0..-2]
+      for i in 0..levels.length
+        dir = File.join(BASE_PATH, levels[0..i].join("/"))
+        Dir.mkdir(dir) unless Dir.exist?(dir)
+      end
+    end
+    warn("storing #{file} locally")
+    File.open(file, "wb") do |f|
+      f.puts URI.open(uri, "rb").read
+    end
+  end
+  [uri, file]
 end
 
-File.open(SCRIPT_FILE_PATH, 'w') do |file|
-	File.chmod(0755, SCRIPT_FILE_PATH)
-  manifest = JSON.parse( URI.open("#{BASE_URI}manifest-validation.jsonld").read )
-	manifest["entries"].each do |entry|
-		type = "valid"
-		case entry["type"]
-		when "csvt:WarningValidationTest"
-			type = "warnings"
-		when "csvt:NegativeValidationTest"
-			type = "errors"
-		end
-		file.puts "echo \"#{entry["id"].split("#")[-1]}: #{entry["name"].gsub("`", "'")}\""
-		file.puts "echo \"#{type}: #{entry["comment"].gsub("\"", "\\\"").gsub("`", "'")}\""
-		if entry["action"].end_with?(".json")
-			file.puts "csvlint --schema=features/fixtures/csvw/#{entry["action"]}"
-		elsif entry["option"] && entry["option"]["metadata"]
-			file.puts "csvlint features/fixtures/csvw/#{entry["action"]} --schema=features/fixtures/csvw/#{entry["option"]["metadata"]}"
-		else
-			file.puts "csvlint features/fixtures/csvw/#{entry["action"]}"
-		end
-		file.puts "echo"
-	end
-end unless File.exist? SCRIPT_FILE_PATH
+unless File.exist? SCRIPT_FILE_PATH
+  File.open(SCRIPT_FILE_PATH, "w") do |file|
+    File.chmod(0o755, SCRIPT_FILE_PATH)
+    manifest = JSON.parse(URI.open("#{BASE_URI}manifest-validation.jsonld").read)
+    manifest["entries"].each do |entry|
+      type = "valid"
+      case entry["type"]
+      when "csvt:WarningValidationTest"
+        type = "warnings"
+      when "csvt:NegativeValidationTest"
+        type = "errors"
+      end
+      file.puts "echo \"#{entry["id"].split("#")[-1]}: #{entry["name"].tr("`", "'")}\""
+      file.puts "echo \"#{type}: #{entry["comment"].gsub("\"", "\\\"").tr("`", "'")}\""
+      if entry["action"].end_with?(".json")
+        file.puts "csvlint --schema=features/fixtures/csvw/#{entry["action"]}"
+      elsif entry["option"] && entry["option"]["metadata"]
+        file.puts "csvlint features/fixtures/csvw/#{entry["action"]} --schema=features/fixtures/csvw/#{entry["option"]["metadata"]}"
+      else
+        file.puts "csvlint features/fixtures/csvw/#{entry["action"]}"
+      end
+      file.puts "echo"
+    end
+  end
+end
 
-File.open(VALIDATION_FEATURE_FILE_PATH, 'w') do |file|
-	file.puts "# Auto-generated file based on standard validation CSVW tests from #{BASE_URI}manifest-validation.jsonld"
-	file.puts ""
+unless File.exist? VALIDATION_FEATURE_FILE_PATH
+  File.open(VALIDATION_FEATURE_FILE_PATH, "w") do |file|
+    file.puts "# Auto-generated file based on standard validation CSVW tests from #{BASE_URI}manifest-validation.jsonld"
+    file.puts ""
 
-  manifest = JSON.parse( URI.open("#{BASE_URI}manifest-validation.jsonld").read )
+    manifest = JSON.parse(URI.open("#{BASE_URI}manifest-validation.jsonld").read)
 
-	file.puts "Feature: #{manifest["label"]}"
-	file.puts ""
+    file.puts "Feature: #{manifest["label"]}"
+    file.puts ""
 
-	manifest["entries"].each do |entry|
-		action_uri, action_file = cache_file(entry["action"])
-		metadata = nil
-		provided_files = []
-		missing_files = []
-		file.puts "\t# #{entry["id"]}"
-		file.puts "\t# #{entry["comment"]}"
-		file.puts "\tScenario: #{entry["id"]} #{entry["name"].gsub("<", "less than")}"
-		if entry["action"].end_with?(".json")
-			file.puts "\t\tGiven I have a metadata file called \"csvw/#{entry["action"]}\""
-			file.puts "\t\tAnd the metadata is stored at the url \"#{action_uri}\""
-		else
-			file.puts "\t\tGiven I have a CSV file called \"csvw/#{entry["action"]}\""
-			file.puts "\t\tAnd it has a Link header holding \"#{entry["httpLink"]}\"" if entry["httpLink"]
-			file.puts "\t\tAnd it is stored at the url \"#{action_uri}\""
-			if entry["option"] && entry["option"]["metadata"]
-				# no need to store the file here, as it will be listed in the 'implicit' list, which all get stored
-				metadata = URI.join(BASE_URI, entry["option"]["metadata"])
-				file.puts "\t\tAnd I have a metadata file called \"csvw/#{entry["option"]["metadata"]}\""
-				file.puts "\t\tAnd the metadata is stored at the url \"#{metadata}\""
-			end
-			provided_files << action_uri.to_s
-			if entry["name"].include?("/.well-known/csvm")
-				file.puts "\t\tAnd I have a file called \"w3.org/.well-known/csvm\" at the url \"https://www.w3.org/.well-known/csvm\""
-	  			missing_files << "#{action_uri}.json"
-	  			missing_files << URI.join(action_uri, 'csvm.json').to_s
-			else
-				missing_files << URI.join(action_uri, '/.well-known/csvm').to_s
-			end
-  			missing_files << "#{action_uri}-metadata.json"
-  			missing_files << URI.join(action_uri, 'csv-metadata.json').to_s
-		end
-		entry["implicit"].each do |implicit|
-			implicit_uri, implicit_file = cache_file(implicit)
-			provided_files << implicit_uri.to_s
-			unless implicit_uri == metadata
-				file.puts "\t\tAnd I have a file called \"csvw/#{implicit}\" at the url \"#{implicit_uri}\""
-			end
-		end if entry["implicit"]
-		missing_files.each do |uri|
-			file.puts "\t\tAnd there is no file at the url \"#{uri}\"" unless provided_files.include? uri
-		end
-		file.puts "\t\tWhen I carry out CSVW validation"
-		if entry["type"] == "csvt:WarningValidationTest"
-			file.puts "\t\tThen there should not be errors"
-			file.puts "\t\tAnd there should be warnings"
-		elsif entry["type"] == "csvt:NegativeValidationTest"
-	    	file.puts "\t\tThen there should be errors"
-		else
-		    file.puts "\t\tThen there should not be errors"
-		    file.puts "\t\tAnd there should not be warnings"
-		end
-		file.puts "\t"
-	end
-end unless File.exist? VALIDATION_FEATURE_FILE_PATH
+    manifest["entries"].each do |entry|
+      action_uri, action_file = cache_file(entry["action"])
+      metadata = nil
+      provided_files = []
+      missing_files = []
+      file.puts "\t# #{entry["id"]}"
+      file.puts "\t# #{entry["comment"]}"
+      file.puts "\tScenario: #{entry["id"]} #{entry["name"].gsub("<", "less than")}"
+      if entry["action"].end_with?(".json")
+        file.puts "\t\tGiven I have a metadata file called \"csvw/#{entry["action"]}\""
+        file.puts "\t\tAnd the metadata is stored at the url \"#{action_uri}\""
+      else
+        file.puts "\t\tGiven I have a CSV file called \"csvw/#{entry["action"]}\""
+        file.puts "\t\tAnd it has a Link header holding \"#{entry["httpLink"]}\"" if entry["httpLink"]
+        file.puts "\t\tAnd it is stored at the url \"#{action_uri}\""
+        if entry["option"] && entry["option"]["metadata"]
+          # no need to store the file here, as it will be listed in the 'implicit' list, which all get stored
+          metadata = URI.join(BASE_URI, entry["option"]["metadata"])
+          file.puts "\t\tAnd I have a metadata file called \"csvw/#{entry["option"]["metadata"]}\""
+          file.puts "\t\tAnd the metadata is stored at the url \"#{metadata}\""
+        end
+        provided_files << action_uri.to_s
+        if entry["name"].include?("/.well-known/csvm")
+          file.puts "\t\tAnd I have a file called \"w3.org/.well-known/csvm\" at the url \"https://www.w3.org/.well-known/csvm\""
+          missing_files << "#{action_uri}.json"
+          missing_files << URI.join(action_uri, "csvm.json").to_s
+        else
+          missing_files << URI.join(action_uri, "/.well-known/csvm").to_s
+        end
+        missing_files << "#{action_uri}-metadata.json"
+        missing_files << URI.join(action_uri, "csv-metadata.json").to_s
+      end
+      entry["implicit"]&.each do |implicit|
+        implicit_uri, implicit_file = cache_file(implicit)
+        provided_files << implicit_uri.to_s
+        unless implicit_uri == metadata
+          file.puts "\t\tAnd I have a file called \"csvw/#{implicit}\" at the url \"#{implicit_uri}\""
+        end
+      end
+      missing_files.each do |uri|
+        file.puts "\t\tAnd there is no file at the url \"#{uri}\"" unless provided_files.include? uri
+      end
+      file.puts "\t\tWhen I carry out CSVW validation"
+      if entry["type"] == "csvt:WarningValidationTest"
+        file.puts "\t\tThen there should not be errors"
+        file.puts "\t\tAnd there should be warnings"
+      elsif entry["type"] == "csvt:NegativeValidationTest"
+        file.puts "\t\tThen there should be errors"
+      else
+        file.puts "\t\tThen there should not be errors"
+        file.puts "\t\tAnd there should not be warnings"
+      end
+      file.puts "\t"
+    end
+  end
+end

data/features/support/webmock.rb

@@ -1,3 +1,3 @@
-require 'webmock/cucumber'
+require "webmock/cucumber"
 
-WebMock.disable_net_connect!(allow: %r{csvw/tests})
+WebMock.disable_net_connect!(allow: %r{csvw/tests})

data/lib/csvlint/cli.rb

@@ -1,15 +1,13 @@
-require 'csvlint'
-require 'rainbow'
-require 'active_support/json'
-require 'json'
-require 'pp'
-require 'thor'
+require "csvlint"
+require "rainbow"
+require "active_support/json"
+require "json"
+require "thor"
 
-require 'active_support/inflector'
+require "active_support/inflector"
 
 module Csvlint
   class Cli < Thor
-
     desc "myfile.csv OR csvlint http://example.com/myfile.csv", "Supports validating CSV files to check their syntax and contents"
 
     option :dump_errors, desc: "Pretty print error and warning objects.", type: :boolean, aliases: :d
@@ -36,162 +34,167 @@ module Csvlint
 
     private
 
-      def read_source(source)
-        if source.nil?
-          # If no source is present, try reading from stdin
-          if !$stdin.tty?
-            source = StringIO.new(STDIN.read) rescue nil
-            return_error "No CSV data to validate" if !options[:schema] && source.nil?
+    def read_source(source)
+      if source.nil?
+        # If no source is present, try reading from stdin
+        if !$stdin.tty?
+          source = begin
+            StringIO.new($stdin.read)
+          rescue
+            nil
           end
-        else
-          # If the source isn't a URL, it's a file
-          unless source =~ /^http(s)?/
-            begin
-              source = File.new( source )
-            rescue Errno::ENOENT
-              return_error "#{source} not found"
-            end
+          return_error "No CSV data to validate" if !options[:schema] && source.nil?
+        end
+      else
+        # If the source isn't a URL, it's a file
+        unless /^http(s)?/.match?(source)
+          begin
+            source = File.new(source)
+          rescue Errno::ENOENT
+            return_error "#{source} not found"
           end
         end
-
-        source
       end
 
-      def get_schema(schema)
-        begin
-          schema = Csvlint::Schema.load_from_uri(schema, false)
-        rescue Csvlint::Csvw::MetadataError => e
-          return_error "invalid metadata: #{e.message}#{" at " + e.path if e.path}"
-        rescue OpenURI::HTTPError, Errno::ENOENT
-          return_error "#{options[:schema]} not found"
-        end
+      source
+    end
 
-        if schema.class == Csvlint::Schema && schema.description == "malformed"
-          return_error "invalid metadata: malformed JSON"
-        end
+    def get_schema(schema)
+      begin
+        schema = Csvlint::Schema.load_from_uri(schema, false)
+      rescue Csvlint::Csvw::MetadataError => e
+        return_error "invalid metadata: #{e.message}#{" at " + e.path if e.path}"
+      rescue OpenURI::HTTPError, Errno::ENOENT
+        return_error "#{options[:schema]} not found"
+      end
 
-        schema
+      if schema.instance_of?(Csvlint::Schema) && schema.description == "malformed"
+        return_error "invalid metadata: malformed JSON"
       end
 
-      def fetch_schema_tables(schema, options)
-        valid = true
+      schema
+    end
 
-        unless schema.instance_of? Csvlint::Csvw::TableGroup
-          return_error "No CSV data to validate."
-        end
-        schema.tables.keys.each do |source|
+    def fetch_schema_tables(schema, options)
+      valid = true
+
+      unless schema.instance_of? Csvlint::Csvw::TableGroup
+        return_error "No CSV data to validate."
+      end
+      schema.tables.keys.each do |source|
+        unless /^http(s)?/.match?(source)
           begin
-            source = source.sub("file:","")
-            source = File.new( source )
+            source = source.sub("file:", "")
+            source = File.new(source)
           rescue Errno::ENOENT
             return_error "#{source} not found"
-          end unless source =~ /^http(s)?/
-          valid &= validate_csv(source, schema, options[:dump_errors], nil, options[:werror])
-        end
-
-        exit 1 unless valid
-      end
-
-      def print_error(index, error, dump, color)
-        location = ""
-        location += error.row.to_s if error.row
-        location += "#{error.row ? "," : ""}#{error.column.to_s}" if error.column
-        if error.row || error.column
-          location = "#{error.row ? "Row" : "Column"}: #{location}"
-        end
-        output_string = "#{index+1}. "
-        if error.column && @schema && @schema.class == Csvlint::Schema
-          if @schema.fields[error.column - 1] != nil
-            output_string += "#{@schema.fields[error.column - 1].name}: "
           end
         end
-        output_string += "#{error.type}"
-        output_string += ". #{location}" unless location.empty?
-        output_string += ". #{error.content}" if error.content
+        valid &= validate_csv(source, schema, options[:dump_errors], nil, options[:werror])
+      end
 
-        puts Rainbow(output_string).color(color)
+      exit 1 unless valid
+    end
 
-        if dump
-          pp error
+    def print_error(index, error, dump, color)
+      location = ""
+      location += error.row.to_s if error.row
+      location += "#{error.row ? "," : ""}#{error.column}" if error.column
+      if error.row || error.column
+        location = "#{error.row ? "Row" : "Column"}: #{location}"
+      end
+      output_string = "#{index + 1}. "
+      if error.column && @schema && @schema.instance_of?(Csvlint::Schema)
+        if @schema.fields[error.column - 1] != nil
+          output_string += "#{@schema.fields[error.column - 1].name}: "
         end
       end
+      output_string += error.type.to_s
+      output_string += ". #{location}" unless location.empty?
+      output_string += ". #{error.content}" if error.content
 
-      def print_errors(errors, dump)
-        if errors.size > 0
-          errors.each_with_index { |error, i| print_error(i, error, dump, :red)  }
-        end
+      puts Rainbow(output_string).color(color)
+
+      if dump
+        pp error
       end
+    end
 
-      def return_error(message)
-        puts Rainbow(message).red
-        exit 1
+    def print_errors(errors, dump)
+      if errors.size > 0
+        errors.each_with_index { |error, i| print_error(i, error, dump, :red) }
       end
+    end
 
-      def validate_csv(source, schema, dump, json, werror)
-        @error_count = 0
+    def return_error(message)
+      puts Rainbow(message).red
+      exit 1
+    end
 
-        if json === true
-          validator = Csvlint::Validator.new( source, {}, schema )
-        else
-          validator = Csvlint::Validator.new( source, {}, schema, { lambda: report_lines } )
-        end
+    def validate_csv(source, schema, dump, json, werror)
+      @error_count = 0
 
-        if source.class == String
-          csv = source
-        elsif source.class == File
-          csv = source.path
-        else
-          csv = "CSV"
-        end
+      validator = if json === true
+        Csvlint::Validator.new(source, {}, schema)
+      else
+        Csvlint::Validator.new(source, {}, schema, {lambda: report_lines})
+      end
 
-        if json === true
-          json = {
-            validation: {
-              state: validator.valid? ? "valid" : "invalid",
-              errors: validator.errors.map { |v| hashify(v) },
-              warnings: validator.warnings.map { |v| hashify(v) },
-              info: validator.info_messages.map { |v| hashify(v) },
-            }
-          }.to_json
-          print json
-        else
-          puts "\r\n#{csv} is #{validator.valid? ? Rainbow("VALID").green : Rainbow("INVALID").red}"
-          print_errors(validator.errors,   dump)
-          print_errors(validator.warnings, dump)
-        end
+      csv = if source.instance_of?(String)
+        source
+      elsif source.instance_of?(File)
+        source.path
+      else
+        "CSV"
+      end
 
-        return false if werror && validator.warnings.size > 0
-        return validator.valid?
+      if json === true
+        json = {
+          validation: {
+            state: validator.valid? ? "valid" : "invalid",
+            errors: validator.errors.map { |v| hashify(v) },
+            warnings: validator.warnings.map { |v| hashify(v) },
+            info: validator.info_messages.map { |v| hashify(v) }
+          }
+        }.to_json
+        print json
+      else
+        puts "\r\n#{csv} is #{validator.valid? ? Rainbow("VALID").green : Rainbow("INVALID").red}"
+        print_errors(validator.errors, dump)
+        print_errors(validator.warnings, dump)
       end
 
-      def hashify(error)
-        h = {
-          type: error.type,
-          category: error.category,
-          row: error.row,
-          col: error.column
-        }
-
-        if error.column && @schema && @schema.class == Csvlint::Schema && @schema.fields[error.column - 1] != nil
-          field = @schema.fields[error.column - 1]
-          h[:header] = field.name
-          h[:constraints] = Hash[field.constraints.map {|k,v| [k.underscore, v] }]
-        end
+      return false if werror && validator.warnings.size > 0
+      validator.valid?
+    end
 
-        h
+    def hashify(error)
+      h = {
+        type: error.type,
+        category: error.category,
+        row: error.row,
+        col: error.column
+      }
+
+      if error.column && @schema && @schema.instance_of?(Csvlint::Schema) && @schema.fields[error.column - 1] != nil
+        field = @schema.fields[error.column - 1]
+        h[:header] = field.name
+        h[:constraints] = field.constraints.map { |k, v| [k.underscore, v] }.to_h
       end
 
-      def report_lines
-        lambda do |row|
-          new_errors = row.errors.count
-          if new_errors > @error_count
-            print Rainbow("!").red
-          else
-            print Rainbow(".").green
-          end
-          @error_count = new_errors
+      h
+    end
+
+    def report_lines
+      lambda do |row|
+        new_errors = row.errors.count
+        if new_errors > @error_count
+          print Rainbow("!").red
+        else
+          print Rainbow(".").green
         end
+        @error_count = new_errors
       end
-
+    end
   end
 end