shipit-csv-validator 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e971afcc4413f5b4bde9b9991bb978de54493677a4949bc3d67bb3a03cc43e17
+  data.tar.gz: 61db75a326805c874644e1f6ce04203fb66cde7cb456b64464cbafb78de453fe
+SHA512:
+  metadata.gz: 1abab5c40be4b160a17e04b0cf8723d04f6e0aca278aaee71079b928da4f8325729edc46345177e366b4fa69dc003215938ed83a455931a6112a1ff265cc6acd
+  data.tar.gz: 14952f95d54ca6bd6eb294946916166adc655aefbfbe194be82bcd1d5d10c3a7a8111995fd8e9f2f29cc6f9248807425f3e55013151e5effb0a6da54a436d30d

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shipit Software
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,44 @@
+# CSV Validator for Ruby
+
+Check UTF-8 CSV structure in Ruby or from a terminal. The library and command
+line tool work locally, report problem locations without printing cell values,
+and leave input files untouched.
+
+Use the [online CSV validator](https://csvfile.tools/validate) in a browser.
+The [CSV Validator source repository](https://github.com/shipit-software/csv-validator)
+contains the Ruby source, shared conformance cases and other language editions.
+
+## Command line
+
+With Ruby 3.0 or newer:
+
+~~~sh
+gem install shipit-csv-validator
+shipit-csv-validate --json data.csv
+~~~
+
+Pass a file, pass "-" or omit the filename to read standard input. Use
+"--no-header" when the first record is data, "--delimiter tab" for tab-separated
+data, and "--max-errors 20" to cap diagnostics. Exit status is 0 for valid CSV,
+1 for invalid CSV, and 2 for an option or input error. The command accepts up to
+32 MiB of UTF-8 input.
+
+## Ruby API
+
+~~~ruby
+require "shipit/csv_validator"
+
+report = Shipit::CsvValidator.validate("id,name\n1,Ada\n", header: true)
+puts report["valid"]      # true
+puts report["columns"]    # 2
+~~~
+
+The API accepts UTF-8 input without the command line byte limit. It reports
+quoting errors, unsupported NUL characters, inconsistent row widths and empty
+or repeated headers. The first complete record sets the expected width. Header
+comparisons are case sensitive. The parser supports escaped quotes, quoted
+delimiters, multiline fields and CRLF, LF or CR line endings. It ignores a
+single leading byte order mark. Diagnostic positions are one-based, and columns
+count UTF-16 code units. It validates structure without inferring data types
+or applying schemas. An invalid delimiter, option or encoding raises
+"ArgumentError". The implementation uses only Ruby standard libraries.

data/exe/shipit-csv-validate

@@ -0,0 +1,120 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "json"
+require "shipit/csv_validator"
+
+module CsvValidatorCommand
+  INPUT_LIMIT = 32 * 1024 * 1024
+  HELP = <<~TEXT
+    Usage: shipit-csv-validate [options] [FILE|-]
+    Validate UTF-8 CSV from a file or standard input.
+
+      --json              Print a JSON report
+      --delimiter CHAR    Use a different delimiter (use "tab" for a tab)
+      --no-header         Treat the first record as data
+      --max-errors N      Cap diagnostics (1-10000, default 100)
+      --help              Show help
+      --version           Show the version
+      --                  End options
+
+    Exit codes: 0 valid, 1 invalid CSV, 2 usage or input error.
+    Input is buffered and limited to 32 MiB. No type or schema inference.
+  TEXT
+
+  def self.parse(arguments)
+    args = arguments.dup
+    options = {}
+    path = nil
+    end_options = false
+    until args.empty?
+      arg = args.shift
+      unless end_options
+        case arg
+        when "--"
+          end_options = true
+          next
+        when "--help", "--version"
+          return [arg, nil, options]
+        when "--json"
+          next
+        when "--no-header"
+          options[:header] = false
+          next
+        when "--delimiter"
+          value = args.shift or raise ArgumentError, "missing option value"
+          options[:delimiter] = value == "tab" ? "\t" : value
+          next
+        when "--max-errors"
+          value = args.shift or raise ArgumentError, "missing option value"
+          unless /\A[1-9][0-9]{0,4}\z/.match?(value) && value.to_i <= 10_000
+            raise ArgumentError, "maximum errors must be 1-10000"
+          end
+          options[:max_errors] = value.to_i
+          next
+        end
+        raise ArgumentError, "unknown option; use --help" if arg.start_with?("-") && arg != "-"
+      end
+      raise ArgumentError, "provide one file, or standard input" if path
+
+      path = arg
+    end
+    [nil, path, options]
+  end
+
+  def self.run(args)
+    as_json = args.take_while { |arg| arg != "--" }.include?("--json")
+    fail_input = lambda do |code, message|
+      if as_json
+        puts JSON.generate(valid: false, error: { code: code, message: message })
+      else
+        warn message
+      end
+      2
+    end
+
+    begin
+      action, path, options = parse(args)
+      if action
+        puts action == "--help" ? HELP : Shipit::CsvValidator::VERSION
+        return 0
+      end
+      # Validate options before reading a file or blocking on standard input.
+      Shipit::CsvValidator.validate("", **options)
+    rescue ArgumentError => error
+      return fail_input.call("USAGE", error.message)
+    end
+
+    begin
+      data = if path.nil? || path == "-"
+               STDIN.binmode.read(INPUT_LIMIT + 1)
+             else
+               File.binread(path, INPUT_LIMIT + 1)
+             end
+    rescue SystemCallError, IOError
+      return fail_input.call("INPUT_READ", "Could not read input")
+    end
+    return fail_input.call("INPUT_LIMIT", "Input exceeds the 32 MiB limit") if data.to_s.bytesize > INPUT_LIMIT
+
+    begin
+      report = Shipit::CsvValidator.validate(data.to_s, **options)
+    rescue ArgumentError => error
+      return fail_input.call("INPUT_ENCODING", error.message)
+    end
+
+    if as_json
+      puts JSON.generate(report)
+    elsif report["valid"]
+      puts "Valid CSV: #{report['dataRecords']} data records, #{report['columns']} columns"
+    else
+      report["diagnostics"].each do |issue|
+        puts format("%<code>s: record %<record>d, field %<field>d, line %<line>d, column %<column>d: %<message>s",
+                    **issue.transform_keys(&:to_sym))
+      end
+      puts "Additional diagnostics omitted" if report["diagnosticsTruncated"]
+    end
+    report["valid"] ? 0 : 1
+  end
+end
+
+exit CsvValidatorCommand.run(ARGV)

data/lib/shipit/csv_validator.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require "strscan"
+
+module Shipit
+  module CsvValidator
+    VERSION = "0.1.0"
+
+    # Return a report with JSON-compatible string keys and one-based positions.
+    # Input is interpreted as UTF-8 bytes. Bad encoding or options raise ArgumentError.
+    def self.validate(input, delimiter: ",", header: true, max_errors: 100)
+      raise ArgumentError, "input must be a UTF-8 string" unless input.is_a?(String)
+
+      input = input.dup.force_encoding(Encoding::UTF_8)
+      raise ArgumentError, "input must be valid UTF-8" unless input.valid_encoding?
+
+      unless delimiter.is_a?(String) && delimiter.encoding.ascii_compatible?
+        raise ArgumentError, "delimiter must be one BMP character other than quote, newline or NUL"
+      end
+      delimiter = delimiter.dup.force_encoding(Encoding::UTF_8)
+      unless delimiter.valid_encoding? && delimiter.length == 1 &&
+             delimiter.ord <= 0xffff && !["\"", "\r", "\n", "\0"].include?(delimiter)
+        raise ArgumentError, "delimiter must be one BMP character other than quote, newline or NUL"
+      end
+      raise ArgumentError, "header must be true or false" unless [true, false].include?(header)
+      unless max_errors.is_a?(Integer) && (1..10_000).cover?(max_errors)
+        raise ArgumentError, "maximum errors must be 1-10000"
+      end
+
+      Parser.new(input.delete_prefix("\uFEFF"), delimiter, header, max_errors).report
+    end
+
+    class Parser
+      def initialize(input, delimiter, header, max_errors)
+        @scanner = StringScanner.new(input)
+        @delimiter, @header, @max_errors = delimiter, header, max_errors
+        @records = @columns = @fields = 0
+        @line = @column = 1
+        @row_line = @field_line = @field_column = 1
+        @state = :start
+        @touched = false
+        @parse_complete = true
+        @diagnostics = []
+        @truncated = false
+        @header_issues = []
+        @header_truncated = false
+        @names = {}
+        @name = +""
+      end
+
+      def report
+        add(issue("EMPTY_INPUT", "Input contains no records", 1, 1, 1, 1)) if @scanner.eos?
+        while (token = @scanner.scan(/\r\n|./m))
+          consume(token)
+          break unless @parse_complete
+        end
+
+        if @parse_complete
+          if @state == :quoted
+            fatal("UNCLOSED_QUOTE", "Quoted field is missing a closing quote", @field_line, @field_column)
+          elsif @touched
+            finish_record
+          end
+        end
+
+        {
+          "valid" => @diagnostics.empty?,
+          "records" => @records,
+          "dataRecords" => @header && @records.positive? ? @records - 1 : @records,
+          "columns" => @columns,
+          "header" => @header,
+          "diagnostics" => @diagnostics,
+          "diagnosticsTruncated" => @truncated,
+          "parseComplete" => @parse_complete
+        }
+      end
+
+      private
+
+      def consume(token)
+        newline = token.start_with?("\r", "\n")
+        if token == "\0"
+          fatal("NUL_CHARACTER", "NUL characters are not supported")
+        elsif @state == :quoted
+          consume_quoted(token, newline)
+        elsif token == @delimiter
+          @touched = true
+          finish_field
+          @column += 1
+          @field_line, @field_column = @line, @column
+        elsif newline
+          finish_record
+          advance_line
+          @row_line = @field_line = @line
+          @field_column = @column
+        elsif @state == :closed
+          fatal("TEXT_AFTER_QUOTE", "Only a delimiter or line ending may follow a closing quote")
+        elsif token == "\""
+          if @state == :start
+            @state = :quoted
+            @touched = true
+            @column += 1
+          else
+            fatal("QUOTE_IN_UNQUOTED_FIELD", "A quote must start at the beginning of a field")
+          end
+        else
+          @state = :unquoted
+          @touched = true
+          append_name(token)
+          @column += token.ord > 0xffff ? 2 : 1
+        end
+      end
+
+      def consume_quoted(token, newline)
+        if token == "\""
+          if @scanner.scan(/"/)
+            append_name(token)
+            @column += 2
+          else
+            @state = :closed
+            @column += 1
+          end
+        else
+          append_name(token)
+          if newline
+            advance_line
+          else
+            @column += token.ord > 0xffff ? 2 : 1
+          end
+        end
+      end
+
+      def advance_line
+        @line += 1
+        @column = 1
+      end
+
+      def append_name(token)
+        @name << token if @header && @records.zero?
+      end
+
+      def finish_field
+        @fields += 1
+        @state = :start
+        return unless @header && @records.zero?
+
+        header_issue("EMPTY_HEADER", "Header name is empty") if @name.empty?
+        if (earlier = @names[@name])
+          header_issue("DUPLICATE_HEADER", "Header repeats field #{earlier}")
+        else
+          @names[@name] = @fields
+        end
+        @name = +""
+      end
+
+      def finish_record
+        finish_field
+        @records += 1
+        if @records == 1
+          @columns = @fields
+          @diagnostics.concat(@header_issues)
+          @truncated = @header_truncated
+          @header_issues.clear
+          @names.clear
+        elsif @fields != @columns
+          add(issue("ROW_WIDTH", "Expected #{@columns} fields; found #{@fields}",
+                    @records, [@fields, @columns].min + 1, @row_line, 1))
+        end
+        @fields = 0
+        @touched = false
+      end
+
+      def issue(code, message, record, field, line, column)
+        { "code" => code, "message" => message, "record" => record, "field" => field,
+          "line" => line, "column" => column }
+      end
+
+      def header_issue(code, message)
+        if @header_issues.length < @max_errors
+          @header_issues << issue(code, message, 1, @fields, @field_line, @field_column)
+        else
+          @header_truncated = true
+        end
+      end
+
+      def add(diagnostic)
+        if @diagnostics.length < @max_errors
+          @diagnostics << diagnostic
+        else
+          @truncated = true
+        end
+      end
+
+      def fatal(code, message, line = @line, column = @column)
+        if @diagnostics.length == @max_errors
+          @diagnostics.pop
+          @truncated = true
+        end
+        @diagnostics.unshift(issue(code, message, @records + 1, @fields + 1, line, column))
+        @parse_complete = false
+      end
+    end
+
+    private_constant :Parser
+  end
+end

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: shipit-csv-validator
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Shipit Software
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2026-11-05 00:00:00.000000000 Z
+dependencies: []
+description: Check UTF-8 CSV for quoting, header and row width problems with precise
+  diagnostics.
+email: 
+executables:
+- shipit-csv-validate
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- exe/shipit-csv-validate
+- lib/shipit/csv_validator.rb
+homepage: https://csvfile.tools/validate
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://csvfile.tools/validate
+  source_code_uri: https://github.com/shipit-software/csv-validator
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.3
+signing_key: 
+specification_version: 4
+summary: Validate CSV structure from Ruby or a terminal
+test_files: []