csv-utils 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: d2ec5906d30d2be8c744e4311fc0c7af0f58a5f4
-  data.tar.gz: bcd816369f3e2698e30b372e3dc9c5a74a30b02f
+SHA256:
+  metadata.gz: ecb75f60c8e9b9db4cc3eb0e4ca3a0ac53aad67726ed995b7e8c341cd0dc76a3
+  data.tar.gz: 5138b5cc82eec0b7667c9e3435c2662bbfa1de51e469582372a158b943e57d7f
 SHA512:
-  metadata.gz: 44b66f060cbba4c1c2d48d62e40f6b17d437334633199e4de22988100dbd74b4bba92b9c03247bd284c562bf57b1ffdc0823e05d4591781f2cb01882fdaecc55
-  data.tar.gz: 14aba3fd327f3f03d9e82a3a189791aa1cc84985cd78fbb752dac8d03373868af92919a8e87e900e6e4765c3e41291a2203515cc4fc365c605fd4639df60fd59
+  metadata.gz: 1a7d685b0db28805833596b32793fca968c6a5a1f57346223b487579622423d0151a122253c69806e377015fe0cf9cf02381bafbee37cb0ba54fa290a857c1cc
+  data.tar.gz: 7c572f9e7c74d626084612afa188bb15b036377bbac16ed5374d7fcff300e58fab8100a77729c2fa428e4294ac6b3b643feb00b2e87e7de78883c8548193de54

data/.ruby-gemset

@@ -0,0 +1 @@
+csv-utils

data/.ruby-version

@@ -0,0 +1 @@
+2.6.3

data/Gemfile

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+source 'http://rubygems.org'
+
+gem 'inheritance-helper'
+
+group :development do
+  gem 'rake'
+  gem 'rubocop'
+end
+
+group :spec do
+  gem 'rspec'
+  gem 'simplecov'
+end

data/Gemfile.lock

@@ -0,0 +1,57 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    ast (2.4.1)
+    diff-lcs (1.3)
+    docile (1.3.2)
+    inheritance-helper (0.1.5)
+    parallel (1.19.2)
+    parser (2.7.1.4)
+      ast (~> 2.4.1)
+    rainbow (3.0.0)
+    rake (13.0.1)
+    regexp_parser (1.7.1)
+    rexml (3.2.4)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    rubocop (0.86.0)
+      parallel (~> 1.10)
+      parser (>= 2.7.0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.7)
+      rexml
+      rubocop-ast (>= 0.0.3, < 1.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (0.0.3)
+      parser (>= 2.7.0.1)
+    ruby-progressbar (1.10.1)
+    simplecov (0.18.5)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+    simplecov-html (0.12.2)
+    unicode-display_width (1.7.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  inheritance-helper
+  rake
+  rspec
+  rubocop
+  simplecov
+
+BUNDLED WITH
+   1.17.3

data/bin/csv-change-eol

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+
+require 'csv'
+
+def bold_string(str)
+  "\033[1m#{str}\033[0m"
+end
+  
+USAGE = "Usage: #{bold_string('csv-change-eol')} <csv_file> <end of line character sequence in hex>"
+
+def exit_on_error(msg)
+  $stderr.print <<STR
+Error: #{bold_string(msg)}
+
+#{USAGE}
+
+  End of line example: '7C5E7C0A' is '|^|\\n'
+   - 0A is new line
+   - 0D is carriage return
+
+  Goto: #{bold_string('http://www.asciitable.com/')} for help with the character sequence
+
+STR
+  exit 1
+end
+
+csv_file = ARGV.shift || exit_on_error('no csv file specified')
+eol_sequence = ARGV.shift || exit_on_error('no EOL character sequence specified')
+
+exit_on_error("file #{csv_file} not found") unless File.exist?(csv_file)
+exit_on_error("not a HEX sequece (#{eol_sequence})") unless eol_sequence =~ /\A[0-9a-f]+\z/i
+exit_on_error("incorrect number of characters in (#{eol_sequence}), should be even") unless eol_sequence.size.even?
+
+eol_sequence = [eol_sequence].pack('H*')
+
+
+escaped_csv_file =
+  if csv_file =~ /\.csv$/i
+    csv_file.sub(/(\.csv)$/i, '.escaped-eol\1')
+  else
+    csv_file + '.escaped-eol'
+  end
+
+
+File.open(escaped_csv_file, 'wb') do |out|
+  CSV.foreach(csv_file) do |row|
+    line = row.to_csv
+    line.rstrip!
+    line.concat(eol_sequence)
+    out.write line
+  end
+end
+
+puts escaped_csv_file

data/bin/csv-find-error

@@ -7,8 +7,7 @@ begin
   CSV.open(ARGV[0], 'rb').each { }
 rescue CSV::MalformedCSVError => e
   puts e.class.to_s + ': ' + e.message
-  if e.message =~ /Missing or stray quote in line (\d+)/ ||
-     e.message =~ /Unclosed quoted field on line (\d+)/
+  if e.message =~ /line (\d+)/
     lineno = $1.to_i
     cmd = "csv-readline #{Shellwords.escape(ARGV[0])} #{lineno}"
     puts "running #{cmd}"

data/bin/csv-readline

@@ -1,5 +1,7 @@
 #!/usr/bin/env ruby
 
+require 'optparse'
+
 BYTE_ORDER_MARKS = {
   "\xEF\xBB\xBF".force_encoding('ASCII-8BIT') => 'UTF-8',
   "\xFE\xFF".force_encoding('ASCII-8BIT') => 'UTF-16',
@@ -25,11 +27,12 @@ def csv_parse_line(line)
   last_comma_pos = -1
   column = 1
 
-  while pos = line.index(/([",])/, pos + 1)
+  while pos = line.index(/([",\n])/, pos + 1)
     case line[pos]
     when '"'
       if opened_quote
         if line[pos+1] == ',' ||
+           line[pos+1] == "\r" ||
            line[pos+1] == "\n" ||
            line[pos+1].nil?
           opened_quote = false
@@ -47,9 +50,22 @@ def csv_parse_line(line)
           # return columns
         end
       else
-        opened_quote = true
+        if (last_comma_pos == -1 && pos != 0) ||
+           (last_comma_pos != -1 && line[pos-1] != ',')
+          col_end = line.index(/,/, pos + 1)
+          col_end ||= line.size - 1
+          # slice out the column value
+          val = line[last_comma_pos + 1, col_end - last_comma_pos - 1]
+          columns << [val, :stray_quote]
+          opened_quote = false
+          last_comma_pos = col_end
+          pos = col_end
+        else
+          opened_quote = true
+        end
       end
-    when ','
+    when ',',
+         "\n"
       if ! opened_quote
         column += 1
         columns << [line[last_comma_pos + 1, pos - last_comma_pos - 1], :ok]
@@ -82,6 +98,22 @@ def parse_csv_row(file, lineno, number_of_lines)
   csv_parse_line(str)
 end
 
+options = {
+  all_columns: false
+}
+OptionParser.new do |opts|
+  opts.banner = 'Usage: ' + File.basename(__FILE__) + ' [options] <csv file> <line number> [<number of lines>]'
+
+  opts.on('-h', '--help', 'Prints this help') do
+    puts opts
+    exit
+  end
+
+  opts.on('-a', '--all', 'Display all columns') do
+    options[:all_columns] = true
+  end
+end.parse!
+
 file = File.open(ARGV[0], 'rb')
 lineno = ARGV[1].to_i
 number_of_lines = (ARGV[2] || 1).to_i
@@ -96,7 +128,7 @@ file.close
 cnt = 0
 data.each do |k, (v, status)|
   cnt += 1
-  next if empty_column?(v)
+  next if !options[:all_columns] && empty_column?(v)
   if status == :ok
     puts sprintf('  %-3d %s: %s', cnt, k, v)
   else

data/csv-utils.gemspec

@@ -2,7 +2,8 @@
 
 Gem::Specification.new do |s|
   s.name        = 'csv-utils'
-  s.version     = '0.1.3'
+  s.version     = '0.2.0'
+  s.licenses    = ['MIT']
   s.summary     = 'CSV Utils'
   s.description = 'Tools for debugging malformed CSV files'
   s.authors     = ['Doug Youch']
@@ -11,4 +12,6 @@ Gem::Specification.new do |s|
   s.files       = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   s.bindir      = 'bin'
   s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+
+  s.add_runtime_dependency 'inheritance-helper'
 end

data/lib/csv-utils.rb

@@ -0,0 +1,8 @@
+require 'csv'
+
+# Collection of tools for working with CSV files.
+module CSVUtils
+  autoload :CSVOptions, 'csv_utils/csv_options'
+  autoload :CSVReport, 'csv_utils/csv_report'
+  autoload :CSVRow, 'csv_utils/csv_row'
+end

data/lib/csv_utils/csv_options.rb

@@ -0,0 +1,87 @@
+# Auto detect a csv files options
+module CSVUtils
+  class CSVOptions
+
+    # this list is from https://en.wikipedia.org/wiki/Byte_order_mark
+    BYTE_ORDER_MARKS = {
+      "\xEF\xBB\xBF".force_encoding('ASCII-8BIT') => 'UTF-8',
+      "\xFE\xFF".force_encoding('ASCII-8BIT') => 'UTF-16',
+      "\xFF\xFE".force_encoding('ASCII-8BIT') => 'UTF-16',
+      "\x00\x00\xFE\xFF".force_encoding('ASCII-8BIT') => 'UTF-32',
+      "\xFF\xFE\x00\x00".force_encoding('ASCII-8BIT') => 'UTF-32'
+    }
+
+    COL_SEPARATORS = [
+      "\x02",
+      "\t",
+      '|',
+      ','
+    ]
+
+    ROW_SEPARATORS = [
+      "\r\n",
+      "\n",
+      "\r"
+    ]
+
+    attr_reader :columns,
+                :byte_order_mark,
+                :encoding,
+                :col_separator,
+                :row_separator
+    
+
+    def initialize(io)
+      line =
+        if io.is_a?(String)
+          File.open(io, 'rb', &:readline)
+        else
+          io.readline
+        end
+
+      @col_separator = auto_detect_col_sep(line)
+      @row_separator = auto_detect_row_sep(line)
+      @byte_order_mark = get_byte_order_mark(line)
+      @encoding = get_character_encoding(@byte_order_mark)
+      @columns = get_number_of_columns(line) if @col_separator
+    end
+
+    def valid?
+      return false if @col_separator.nil? || @row_separator.nil?
+
+      true
+    end
+
+    def auto_detect_col_sep(line)
+      COL_SEPARATORS.detect { |sep| line.include?(sep) }
+    end
+
+    def auto_detect_row_sep(line)
+      ROW_SEPARATORS.detect { |sep| line.include?(sep) }
+    end
+
+    def get_headers(line)
+      headers = line.split(col_separator)
+      headers[0] = strip_byte_order_marks(headers[0])
+      headers
+    end
+
+    def get_number_of_columns(line)
+      get_headers(line).size
+    end
+
+    def get_byte_order_mark(line)
+      BYTE_ORDER_MARKS.keys.detect do |bom|
+        line =~ /\A#{bom}/
+      end
+    end
+
+    def get_character_encoding(bom)
+      BYTE_ORDER_MARKS[bom] || 'UTF-8'
+    end
+
+    def strip_byte_order_marks(header)
+      @byte_order_marks ? header.sub(@byte_order_marks, '') : header
+    end
+  end
+end

data/lib/csv_utils/csv_report.rb

@@ -0,0 +1,40 @@
+# Builds a csv file from csv rows
+module CSVUtils
+  class CSVReport
+    attr_reader :csv,
+                :must_close
+
+    def initialize(csv, csv_options = {}, &block)
+      @csv =
+        if csv.is_a?(String)
+          @must_close = true
+          mode = csv_options.delete(:mode) || 'wb'
+          CSV.open(csv, mode, csv_options)
+        else
+          @must_close = false
+          csv
+        end
+
+      generate(&block) if block
+    end
+
+    def generate
+      yield self
+      @csv.close if @must_close
+    end
+
+    def append(csv_row)
+      @csv <<
+        if csv_row.is_a?(Array)
+          csv_row
+        else
+          csv_row.to_a
+        end
+    end
+    alias << append
+
+    def add_headers(csv_row)
+      append(csv_row.is_a?(Array) ? csv_row : csv_row.csv_headers)
+    end
+  end
+end

data/lib/csv_utils/csv_row.rb

@@ -0,0 +1,51 @@
+require 'inheritance-helper'
+
+module CSVUtils
+  module CSVRow
+    def self.included(base)
+      base.extend InheritanceHelper::Methods
+      base.extend ClassMethods
+    end
+
+    module ClassMethods
+      def csv_columns
+        {}
+      end
+
+      def csv_column(header, options = {}, &block)
+        options[:header] ||= header.to_s
+
+        if block
+          options[:proc] = block
+        elsif options[:proc].nil?
+          options[:method] ||= header
+        end
+
+        add_value_to_class_method(:csv_columns, header => options)
+      end
+    end
+
+    def csv_headers
+      self.class.csv_columns.values.map { |column_options| csv_column_header(column_options) }
+    end
+
+    def csv_row
+      self.class.csv_columns.values.map { |column_options| csv_column_value(column_options) }
+    end
+    alias_method :to_a, :csv_row
+
+    private
+
+    def csv_column_header(column_options)
+      column_options[:header]
+    end
+
+    def csv_column_value(column_options)
+      if column_options[:proc]
+        instance_eval(&column_options[:proc])
+      else
+        send(column_options[:method])
+      end
+    end
+  end
+end

data/script/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH << File.expand_path('../lib', __dir__)
+require 'csv-utils'
+require 'irb'
+IRB.start(__FILE__)

metadata

@@ -1,33 +1,59 @@
 --- !ruby/object:Gem::Specification
 name: csv-utils
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Doug Youch
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-04-19 00:00:00.000000000 Z
-dependencies: []
+date: 2020-06-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: inheritance-helper
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Tools for debugging malformed CSV files
 email: dougyouch@gmail.com
 executables:
+- csv-change-eol
 - csv-find-error
 - csv-readline
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".ruby-gemset"
+- ".ruby-version"
+- Gemfile
+- Gemfile.lock
 - LICENSE
 - README.md
+- bin/csv-change-eol
 - bin/csv-find-error
 - bin/csv-readline
 - csv-utils.gemspec
+- lib/csv-utils.rb
+- lib/csv_utils/csv_options.rb
+- lib/csv_utils/csv_report.rb
+- lib/csv_utils/csv_row.rb
+- script/console
 homepage: https://github.com/dougyouch/csv-utils
-licenses: []
+licenses:
+- MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -42,9 +68,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.3
-signing_key: 
+rubygems_version: 3.0.8
+signing_key:
 specification_version: 4
 summary: CSV Utils
 test_files: []