csv-utils 0.1.7 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5d426e3be79ea7c3424c7778e2eb28ac750a55d0af374115021791506cfe933d
-  data.tar.gz: 00fec1a3995a515fca21f8032d40f9447cdbccfb53e7f22644982ff4c3d703a6
+  metadata.gz: 98303ab9b2df05bc501c1c66b66a62be5ade9d79ab38a5b8bda8eb52d91b26cc
+  data.tar.gz: 8adfd2144220de2cc4f23136ee4eb7314a3c16eeac68be87e1dc19b1ac7dc350
 SHA512:
-  metadata.gz: d560c6b227162d297f396213cb26b277b437dc744a05f072f9f5716ea6920ff35dbedcdfcd544e4b5b8589f0e8e1322e79b5cdcb63d9d9e5f26019c141d77772
-  data.tar.gz: 7fd77c041baefdc87e4166781739bd97dc1183824414cb928c4a369adb972423d460f67562f848ac4373fb1c19cde9d962ee73a554590b0950d0bc9462e42b52
+  metadata.gz: a2a2b2067a9ca06920b171230a122eba479c1f91af3919e2965eaec6d073fff34d544221a92cffaa1b9546078960aee0c9b9031e7b652368e975cff9b196214c
+  data.tar.gz: '0786cfb3e75771ccb68bfa0e2cba42994c7c04a5c8be14432ae6467425536e7dfd4a4ef33403ae5bd129eafd871a07077610c000a78762052e0b055192c0cc16'

data/.ruby-gemset

@@ -0,0 +1 @@
+csv-utils

data/.ruby-version

@@ -0,0 +1 @@
+2.6.3

data/Gemfile

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+source 'http://rubygems.org'
+
+gem 'inheritance-helper'
+
+group :development do
+  gem 'rake'
+  gem 'rubocop'
+end
+
+group :spec do
+  gem 'rspec'
+  gem 'simplecov'
+end

data/Gemfile.lock

@@ -0,0 +1,57 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    ast (2.4.1)
+    diff-lcs (1.3)
+    docile (1.3.2)
+    inheritance-helper (0.1.5)
+    parallel (1.19.2)
+    parser (2.7.1.4)
+      ast (~> 2.4.1)
+    rainbow (3.0.0)
+    rake (13.0.1)
+    regexp_parser (1.7.1)
+    rexml (3.2.4)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    rubocop (0.86.0)
+      parallel (~> 1.10)
+      parser (>= 2.7.0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.7)
+      rexml
+      rubocop-ast (>= 0.0.3, < 1.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (0.0.3)
+      parser (>= 2.7.0.1)
+    ruby-progressbar (1.10.1)
+    simplecov (0.18.5)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+    simplecov-html (0.12.2)
+    unicode-display_width (1.7.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  inheritance-helper
+  rake
+  rspec
+  rubocop
+  simplecov
+
+BUNDLED WITH
+   1.17.3

data/csv-utils.gemspec

@@ -2,7 +2,8 @@
 
 Gem::Specification.new do |s|
   s.name        = 'csv-utils'
-  s.version     = '0.1.7'
+  s.version     = '0.3.0'
+  s.licenses    = ['MIT']
   s.summary     = 'CSV Utils'
   s.description = 'Tools for debugging malformed CSV files'
   s.authors     = ['Doug Youch']
@@ -11,4 +12,6 @@ Gem::Specification.new do |s|
   s.files       = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   s.bindir      = 'bin'
   s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+
+  s.add_runtime_dependency 'inheritance-helper'
 end

data/lib/csv-utils.rb

@@ -0,0 +1,12 @@
+require 'csv'
+
+# Collection of tools for working with CSV files.
+module CSVUtils
+  autoload :CSVExtender, 'csv_utils/csv_extender'
+  autoload :CSVOptions, 'csv_utils/csv_options'
+  autoload :CSVReport, 'csv_utils/csv_report'
+  autoload :CSVRow, 'csv_utils/csv_row'
+  autoload :CSVSort, 'csv_utils/csv_sort'
+  autoload :CSVTransformer, 'csv_utils/csv_transformer'
+  autoload :CSVWrapper, 'csv_utils/csv_wrapper'
+end

data/lib/csv_utils/csv_extender.rb

@@ -0,0 +1,63 @@
+# Utility class for appending data to a csv file.
+class CSVUtils::CSVExtender
+  def initialize(src_csv, dest_csv, csv_options = {})
+    @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', csv_options)
+    @dest_csv = CSVUtils::CSVWrapper.new(dest_csv, 'wb', csv_options)
+  end
+
+  def append(additional_headers)
+    process(additional_headers) do |current_headers|
+      while (row = @src_csv.shift)
+        additional_columns = yield row, current_headers
+        @dest_csv << (row + additional_columns)
+      end
+    end
+  end
+
+  def append_in_batches(additional_headers, batch_size = 1_000)
+    process(additional_headers) do |current_headers|
+      batch = []
+
+      process_batch_proc = Proc.new do
+        additional_rows = yield batch, current_headers
+
+        batch.each_with_index do |row, idx|
+          @dest_csv << (row + additional_rows[idx])
+        end
+
+        batch = []
+      end
+
+      while (row = @src_csv.shift)
+        batch << row
+
+        process_batch_proc.call if batch.size >= batch_size
+      end
+
+      process_batch_proc.call if batch.size > 0
+    end
+  end
+
+  private
+
+  def process(additional_headers)
+    current_headers = append_headers(additional_headers)
+
+    yield current_headers
+
+    close
+  end
+
+  def close
+    @src_csv.close
+    @dest_csv.close
+  end
+
+  def append_headers(additional_headers)
+    return nil unless additional_headers
+
+    current_headers = @src_csv.shift
+    @dest_csv << (current_headers + additional_headers)
+    current_headers
+  end
+end

data/lib/csv_utils/csv_options.rb

@@ -0,0 +1,87 @@
+# Auto detect a csv files options
+module CSVUtils
+  class CSVOptions
+
+    # this list is from https://en.wikipedia.org/wiki/Byte_order_mark
+    BYTE_ORDER_MARKS = {
+      "\xEF\xBB\xBF".force_encoding('ASCII-8BIT') => 'UTF-8',
+      "\xFE\xFF".force_encoding('ASCII-8BIT') => 'UTF-16',
+      "\xFF\xFE".force_encoding('ASCII-8BIT') => 'UTF-16',
+      "\x00\x00\xFE\xFF".force_encoding('ASCII-8BIT') => 'UTF-32',
+      "\xFF\xFE\x00\x00".force_encoding('ASCII-8BIT') => 'UTF-32'
+    }
+
+    COL_SEPARATORS = [
+      "\x02",
+      "\t",
+      '|',
+      ','
+    ]
+
+    ROW_SEPARATORS = [
+      "\r\n",
+      "\n",
+      "\r"
+    ]
+
+    attr_reader :columns,
+                :byte_order_mark,
+                :encoding,
+                :col_separator,
+                :row_separator
+    
+
+    def initialize(io)
+      line =
+        if io.is_a?(String)
+          File.open(io, 'rb', &:readline)
+        else
+          io.readline
+        end
+
+      @col_separator = auto_detect_col_sep(line)
+      @row_separator = auto_detect_row_sep(line)
+      @byte_order_mark = get_byte_order_mark(line)
+      @encoding = get_character_encoding(@byte_order_mark)
+      @columns = get_number_of_columns(line) if @col_separator
+    end
+
+    def valid?
+      return false if @col_separator.nil? || @row_separator.nil?
+
+      true
+    end
+
+    def auto_detect_col_sep(line)
+      COL_SEPARATORS.detect { |sep| line.include?(sep) }
+    end
+
+    def auto_detect_row_sep(line)
+      ROW_SEPARATORS.detect { |sep| line.include?(sep) }
+    end
+
+    def get_headers(line)
+      headers = line.split(col_separator)
+      headers[0] = strip_byte_order_marks(headers[0])
+      headers
+    end
+
+    def get_number_of_columns(line)
+      get_headers(line).size
+    end
+
+    def get_byte_order_mark(line)
+      BYTE_ORDER_MARKS.keys.detect do |bom|
+        line =~ /\A#{bom}/
+      end
+    end
+
+    def get_character_encoding(bom)
+      BYTE_ORDER_MARKS[bom] || 'UTF-8'
+    end
+
+    def strip_byte_order_marks(header)
+      @byte_order_marks ? header.sub(@byte_order_marks, '') : header
+    end
+  end
+end

data/lib/csv_utils/csv_report.rb

@@ -0,0 +1,41 @@
+# Builds a csv file from csv rows
+module CSVUtils
+  class CSVReport
+    attr_reader :csv,
+                :must_close
+
+    def initialize(csv, headers = nil, csv_options = {}, &block)
+      @csv =
+        if csv.is_a?(String)
+          @must_close = true
+          mode = csv_options.delete(:mode) || 'wb'
+          CSV.open(csv, mode, csv_options)
+        else
+          @must_close = false
+          csv
+        end
+
+      generate(headers, &block) if block
+    end
+
+    def generate(headers = nil)
+      add_headers(headers) if headers
+      yield self
+      @csv.close if @must_close
+    end
+
+    def append(csv_row)
+      @csv <<
+        if csv_row.is_a?(Array)
+          csv_row
+        else
+          csv_row.to_a
+        end
+    end
+    alias << append
+
+    def add_headers(csv_row)
+      append(csv_row.is_a?(Array) ? csv_row : csv_row.csv_headers)
+    end
+  end
+end

data/lib/csv_utils/csv_row.rb

@@ -0,0 +1,57 @@
+require 'inheritance-helper'
+
+module CSVUtils
+  module CSVRow
+    def self.included(base)
+      base.extend InheritanceHelper::Methods
+      base.extend ClassMethods
+    end
+
+    module ClassMethods
+      def csv_columns
+        {}
+      end
+
+      def csv_column(header, options = {}, &block)
+        options[:header] ||= header.to_s
+
+        if block
+          options[:proc] = block
+        elsif options[:proc].nil?
+          options[:method] ||= header
+        end
+
+        add_value_to_class_method(:csv_columns, header => options)
+      end
+
+      def csv_headers
+        csv_columns.values.map { |column_options| csv_column_header(column_options) }
+      end
+
+      private
+
+      def csv_column_header(column_options)
+        column_options[:header]
+      end
+    end
+
+    def csv_row
+      self.class.csv_columns.values.map { |column_options| csv_column_value(column_options) }
+    end
+    alias_method :to_a, :csv_row
+
+    def csv_headers
+      self.class.csv_headers
+    end
+
+    private
+
+    def csv_column_value(column_options)
+      if column_options[:proc]
+        instance_eval(&column_options[:proc])
+      else
+        send(column_options[:method])
+      end
+    end
+  end
+end

data/lib/csv_utils/csv_sort.rb

@@ -0,0 +1,112 @@
+require 'fileutils'
+
+# Utility class for sorting the rows for a csv file
+class CSVUtils::CSVSort
+  attr_reader :csv_file,
+              :new_csv_file,
+              :has_headers,
+              :csv_options,
+              :headers
+
+  def initialize(csv_file, new_csv_file, has_headers = true, csv_options = {})
+    @csv_file = csv_file
+    @new_csv_file = new_csv_file
+    @has_headers = has_headers
+    @csv_options = csv_options
+    @csv_part_files = []
+    @files_to_delete = []
+  end
+
+  def sort(batch_size = 100_000, &block)
+    create_sorted_csv_part_files(batch_size, &block)
+    merge_csv_part_files(&block)
+  end
+
+  private
+
+  def merge_sort_csv_files(src_csv_file1, src_csv_file2, dest_csv_file)
+    src1 = CSV.open(src_csv_file1, 'rb', csv_options)
+    src2 = CSV.open(src_csv_file2, 'rb', csv_options)
+    dest = CSV.open(dest_csv_file, 'wb', csv_options)
+
+    if @headers
+      dest << @headers
+      src1.shift
+      src2.shift
+    end
+
+    row1 = src1.shift
+    row2 = src2.shift
+
+    append_row1_proc = Proc.new do
+      dest << row1
+      row1 = src1.shift
+    end
+
+    append_row2_proc = Proc.new do
+      dest << row2
+      row2 = src2.shift
+    end
+
+    while row1 || row2
+      if row1.nil?
+        append_row2_proc.call
+      elsif row2.nil?
+        append_row1_proc.call
+      elsif yield(row1, row2) <= 0
+        append_row1_proc.call
+      else
+        append_row2_proc.call
+      end
+    end
+
+    src1.close
+    src2.close
+    dest.close
+  end
+
+  def create_sorted_csv_part_files(batch_size, &block)
+    src = CSV.open(csv_file, 'rb', csv_options)
+
+    @headers = src.shift if has_headers
+
+    batch = []
+    create_batch_part_proc = Proc.new do
+      batch.sort!(&block)
+      @csv_part_files << "#{new_csv_file}.part.#{@csv_part_files.size}"
+      CSV.open(@csv_part_files.last, 'wb', csv_options) do |csv|
+        csv << @headers if @headers
+        batch.each { |row| csv << row }
+      end
+      batch = []
+    end
+
+    while (row = src.shift)
+      batch << row
+      create_batch_part_proc.call if batch.size >= batch_size
+    end
+
+    create_batch_part_proc.call if batch.size > 0
+
+    src.close
+  end
+
+  def merge_csv_part_files(&block)
+    file_merge_cnt = 0
+
+    while @csv_part_files.size > 1
+      file_merge_cnt += 1
+
+      csv_part_file1 = @csv_part_files.shift
+      csv_part_file2 = @csv_part_files.shift
+      @csv_part_files << "#{new_csv_file}.merge.#{file_merge_cnt}"
+
+      merge_sort_csv_files(csv_part_file1, csv_part_file2, @csv_part_files.last, &block)
+
+      File.unlink(csv_part_file1)
+      File.unlink(csv_part_file2)
+    end
+
+    FileUtils.mv(@csv_part_files.last, new_csv_file)
+  end
+end

data/lib/csv_utils/csv_transformer.rb

@@ -0,0 +1,119 @@
+# Transforms a CSV given a series of steps
+class CSVUtils::CSVTransformer
+  attr_reader :headers
+
+  def initialize(src_csv, dest_csv, csv_options = {})
+    @src_csv = CSVUtils::CSVWrapper.new(src_csv, 'rb', csv_options)
+    @dest_csv = CSVUtils::CSVWrapper.new(dest_csv, 'wb', csv_options)
+  end
+
+  def read_headers
+    @headers = @src_csv.shift
+    self
+  end
+
+  def additional_data(&block)
+    steps << [:additional_data, @headers, block]
+    self
+  end
+
+  def select(&block)
+    steps << [:select, @headers, block]
+    self
+  end
+
+  def reject(&block)
+    steps << [:reject, @headers, block]
+    self
+  end
+
+  def map(new_headers, &block)
+    steps << [:map, @headers, block]
+    @headers = new_headers
+    self
+  end
+
+  def append(additional_headers, &block)
+    steps << [:append, @headers, block]
+
+    if additional_headers
+      @headers += additional_headers
+    else
+      @headers = nil
+    end
+
+    self
+  end
+
+  def each(&block)
+    steps << [:each, @headers, block]
+    self
+  end
+
+  def set_headers(headers)
+    @headers = headers
+    self
+  end
+
+  def process(batch_size = 10_000, &block)
+    batch = []
+
+    @dest_csv << @headers if @headers
+
+    steps_proc = Proc.new do
+      steps.each do |step_type, current_headers, proc|
+        batch = process_step(step_type, current_headers, batch, &proc)
+      end
+
+      batch.each { |row| @dest_csv << row }
+
+      batch = []
+    end
+                   
+    while (row = @src_csv.shift)
+      batch << row
+      steps_proc.call if batch.size >= batch_size
+    end
+
+    steps_proc.call if batch.size > 0
+
+    @src_csv.close
+    @dest_csv.close
+  end
+
+  private
+
+  def steps
+    @steps ||= []
+  end
+
+
+  def process_step(step_type, current_headers, batch, &block)
+    case step_type
+    when :select
+      batch.select! do |row|
+        block.call row, current_headers, @additional_data
+      end
+    when :reject
+      batch.reject! do |row|
+        block.call row, current_headers, @additional_data
+      end
+    when :map
+      batch.map! do |row|
+        block.call row, current_headers, @additional_data
+      end
+    when :append
+      batch.map! do |row|
+        row + block.call(row, current_headers, @additional_data)
+      end
+    when :additional_data
+      @additional_data = block.call(batch, current_headers)
+    when :each
+      batch.each do |row|
+        block.call(row, current_headers, @additional_data)
+      end
+    end
+
+    batch
+  end
+end

data/lib/csv_utils/csv_wrapper.rb

@@ -0,0 +1,47 @@
+# Wraps a CSV object, if wrapper opens the csv file it will close it
+class CSVUtils::CSVWrapper
+  attr_reader :csv
+
+  def initialize(csv, mode, csv_options)
+    open(csv, mode, csv_options)
+  end
+
+  def self.open(file, mode, csv_options = {})
+    csv = new(file, mode, csv_options)
+
+    if block_given?
+      yield csv
+      csv.close
+    else
+      csv
+    end
+  end
+
+  def open(csv, mode, csv_options)
+    if csv.is_a?(String)
+      @close_when_done = true
+      @csv = CSV.open(csv, mode, csv_options)
+    else
+      @close_when_done = false
+      @csv = csv
+    end
+  end
+
+  def <<(row)
+    csv << row
+  end
+
+  def shift
+    csv.shift
+  end
+
+  def close
+    csv.close if close_when_done?
+  end
+
+  private
+
+  def close_when_done?
+    @close_when_done
+  end
+end

data/script/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH << File.expand_path('../lib', __dir__)
+require 'csv-utils'
+require 'irb'
+IRB.start(__FILE__)

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: csv-utils
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.3.0
 platform: ruby
 authors:
 - Doug Youch
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-27 00:00:00.000000000 Z
-dependencies: []
+date: 2020-07-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: inheritance-helper
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Tools for debugging malformed CSV files
 email: dougyouch@gmail.com
 executables:
@@ -20,16 +34,30 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".ruby-gemset"
+- ".ruby-version"
+- Gemfile
+- Gemfile.lock
 - LICENSE
 - README.md
 - bin/csv-change-eol
 - bin/csv-find-error
 - bin/csv-readline
 - csv-utils.gemspec
+- lib/csv-utils.rb
+- lib/csv_utils/csv_extender.rb
+- lib/csv_utils/csv_options.rb
+- lib/csv_utils/csv_report.rb
+- lib/csv_utils/csv_row.rb
+- lib/csv_utils/csv_sort.rb
+- lib/csv_utils/csv_transformer.rb
+- lib/csv_utils/csv_wrapper.rb
+- script/console
 homepage: https://github.com/dougyouch/csv-utils
-licenses: []
+licenses:
+- MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -45,7 +73,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.8
-signing_key: 
+signing_key:
 specification_version: 4
 summary: CSV Utils
 test_files: []