honey_format 0.16.0 → 0.17.0

data/lib/honey_format/cli/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'optparse'
 
 module HoneyFormat
@@ -15,8 +17,11 @@ module HoneyFormat
 
     private
 
-    def puts(*args)
-      @io.puts(*args)
+    # Puts to configured IO
+    # @param [#to_s] object to print
+    # @return [nil] nil
+    def puts(arg)
+      @io.puts(arg)
     end
 
     # Parse command line arguments and return options
@@ -33,26 +38,26 @@ module HoneyFormat
       type_map = {}
 
       OptionParser.new do |parser|
-        parser.banner = "Usage: honey_format [options] <file.csv>"
+        parser.banner = 'Usage: honey_format [options] <file.csv>'
         parser.default_argv = argv
 
-        parser.on("--csv=input.csv", String, "CSV file") do |value|
+        parser.on('--csv=input.csv', String, 'CSV file') do |value|
           input_path = value
         end
 
-        parser.on("--columns=id,name", Array, "Select columns") do |value|
+        parser.on('--columns=id,name', Array, 'Select columns') do |value|
           columns = value&.map(&:to_sym)
         end
 
-        parser.on("--output=output.csv", String, "CSV output (STDOUT otherwise)") do |value|
+        parser.on('--output=output.csv', String, 'CSV output (STDOUT otherwise)') do |value|
           output_path = value
         end
 
-        parser.on("--delimiter=,", String, "CSV delimiter (default: ,)") do |value|
+        parser.on('--delimiter=,', String, 'CSV delimiter (default: ,)') do |value|
           delimiter = value
         end
 
-        parser.on("--skip-lines=,", String, "Skip lines that match this pattern") do |value|
+        parser.on('--skip-lines=,', String, 'Skip lines that match this pattern') do |value|
           skip_lines = value
         end
 
@@ -60,15 +65,15 @@ module HoneyFormat
           type_map = option_to_h(value || [])
         end
 
-        parser.on("--[no-]header-only", "Print only the header") do |value|
+        parser.on('--[no-]header-only', 'Print only the header') do |value|
           header_only = value
         end
 
-        parser.on("--[no-]rows-only", "Print only the rows") do |value|
+        parser.on('--[no-]rows-only', 'Print only the rows') do |value|
           rows_only = value
         end
 
-        parser.on("-h", "--help", "How to use") do
+        parser.on('-h', '--help', 'How to use') do
           puts parser
           exit
         end
@@ -96,7 +101,7 @@ module HoneyFormat
         header_only: header_only,
         rows_only: rows_only,
         skip_lines: skip_lines,
-        type_map: type_map
+        type_map: type_map,
       }
     end
 

data/lib/honey_format/cli/result_writer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module HoneyFormat
   # CLI result writer handles command output
   # @attr_reader [true, false] verbose the writer mode

data/lib/honey_format/configuration.rb

@@ -1,26 +1,129 @@
+# frozen_string_literal: true
+
+require 'honey_format/helpers/helpers'
+
 module HoneyFormat
   # Holds HoneyFormat configuration
-  # @attr_reader [#call] header_converter the configured header converter
-  # @attr_reader [#call] converter the configured value converter
-  # @attr_writer [#call] header_converter to use
-  # @attr_writer [#call] converter the value converter to use
+  # @attr_accessor [String] delimiter the default column delimiter (default: ,)
+  # @attr_accessor [String, Symbol] row_delimiter the default row delimiter (default: :auto)
+  # @attr_accessor [String] quote_character the default quote character (default: ")
+  # @attr_accessor [String, Regexp] skip_lines skip all lines matching pattern (default: nil)
   class Configuration
-    attr_accessor :header_converter, :converter
+    attr_accessor :delimiter, :row_delimiter, :quote_character, :skip_lines
 
     # Instantiate configuration
     def initialize
-      @converter = ValueConverter.new
-      @header_converter = @converter[:header_column]
+      @converter_registry = nil
+      @header_converter = nil
+      @header_deduplicator = nil
+      @delimiter = ','
+      @row_delimiter = :auto
+      @quote_character = '"'
+      @skip_lines = nil
+    end
+
+    # Returns the header converter
+    # @return [#call] header_converter the configured header converter
+    def header_converter
+      @header_converter ||= converter_registry[:header_column]
     end
 
     # Set the header converter
-    # @param [Symbol, #call] converter for registered value converter or object that responds to #call
+    # @param [Symbol, #call] converter for registered converter registry or object that
+    #                        responds to #call
     # @return [#call] the header converter
     def header_converter=(converter)
-      if converter.is_a?(Symbol)
-        return @header_converter = @converter[converter]
+      @header_converter = if converter.is_a?(Symbol)
+                            converter_registry[converter]
+                          else
+                            converter
+                          end
+    end
+
+    # Return the deduplication header strategy
+    # @return [#call] the header deduplication strategy
+    def header_deduplicator
+      @header_deduplicator ||= header_deduplicator_registry[:deduplicate]
+    end
+
+    # Set the deduplication header strategy
+    # @param [Symbol, #call]
+    #   symbol with known strategy identifier or method that responds
+    #   to #call(colums, key_count)
+    # @return [#call] the header deduplication strategy
+    # @raise [UnknownDeduplicationStrategyError]
+    def header_deduplicator=(strategy)
+      if header_deduplicator_registry.type?(strategy)
+        @header_deduplicator = header_deduplicator_registry[strategy]
+      elsif strategy.respond_to?(:call)
+        @header_deduplicator = strategy
+      else
+        message = "unknown deduplication strategy: '#{strategy}'"
+        raise(Errors::UnknownDeduplicationStrategyError, message)
       end
-      @header_converter = converter
+    end
+
+    # Default header deduplicate strategies
+    # @return [Hash] the default header deduplicatation strategies
+    def default_header_deduplicators
+      @default_header_deduplicators ||= {
+        deduplicate: proc do |columns|
+          Helpers.key_count_to_deduplicated_array(columns)
+        end,
+        raise: proc do |columns|
+          duplicates = Helpers.duplicated_items(columns)
+          if duplicates.any?
+            message = "all columns must be unique, duplicates are: #{duplicates}"
+            raise(Errors::DuplicateHeaderColumnError, message)
+          end
+          columns
+        end,
+        none: proc { |columns| columns },
+      }.freeze
+    end
+
+    # Returns the column deduplication registry
+    # @return [#call] column deduplication registry
+    def header_deduplicator_registry
+      @header_deduplicator_registry ||= Registry.new(default_header_deduplicators)
+    end
+
+    # Returns the converter registry
+    # @return [#call] converter the configured converter registry
+    def converter_registry
+      @converter_registry ||= Registry.new(default_converters)
+    end
+
+    # Default converter registry
+    # @return [Hash] hash with default converters
+    def default_converters
+      @default_converters ||= {
+        # strict variants
+        decimal!: StrictConvertDecimal,
+        integer!: StrictConvertInteger,
+        date!: StrictConvertDate,
+        datetime!: StrictConvertDatetime,
+        symbol!: StrictConvertSymbol,
+        downcase!: StrictConvertDowncase,
+        upcase!: StrictConvertUpcase,
+        boolean!: StrictConvertBoolean,
+        # safe variants
+        decimal: ConvertDecimal,
+        decimal_or_zero: ConvertDecimalOrZero,
+        integer: ConvertInteger,
+        integer_or_zero: ConvertIntegerOrZero,
+        date: ConvertDate,
+        datetime: ConvertDatetime,
+        symbol: ConvertSymbol,
+        downcase: ConvertDowncase,
+        upcase: ConvertUpcase,
+        boolean: ConvertBoolean,
+        md5: ConvertMD5,
+        hex: ConvertHex,
+        nil: ConvertNil,
+        blank: ConvertBlank,
+        header_column: ConvertHeaderColumn,
+      }.freeze
     end
   end
 end

data/lib/honey_format/converters/convert_boolean.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module HoneyFormat
+  # String values considered truthy
+  TRUTHY = Set.new(%w[t T 1 y Y true TRUE]).freeze
+  # String values considered falsy
+  FALSY = Set.new(%w[f F 0 n N false FALSE]).freeze
+
+  # Tries to convert value boolean to, returns nil if it can't convert
+  ConvertBoolean = proc do |v|
+    if TRUTHY.include?(v)
+      true
+    elsif FALSY.include?(v)
+      false
+    end
+  end
+
+  # Convert to boolean or raise error
+  StrictConvertBoolean = proc do |v|
+    ConvertBoolean.call(v).tap do |value|
+      raise(ArgumentError, "can't convert #{v} to boolean") if value.nil?
+    end
+  end
+end

data/lib/honey_format/converters/convert_date_and_time.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'date'
+require 'time'
+
+module HoneyFormat
+  # Convert to date
+  ConvertDate = proc do |v|
+    begin
+      StrictConvertDate.call(v)
+    rescue ArgumentError, TypeError
+      nil
+    end
+  end
+
+  # Convert to datetime
+  ConvertDatetime = proc do |v|
+    begin
+      StrictConvertDatetime.call(v)
+    rescue ArgumentError, TypeError
+      nil
+    end
+  end
+
+  # Convert to date or raise error
+  StrictConvertDate = proc { |v| Date.parse(v) }
+
+  # Convert to datetime or raise error
+  StrictConvertDatetime = proc { |v| Time.parse(v) }
+end

data/lib/honey_format/converters/convert_number.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module HoneyFormat
+  # Converts decimal or nil
+  ConvertDecimal = proc do |v|
+    begin
+      Float(v)
+    rescue ArgumentError, TypeError
+      nil
+    end
+  end
+
+  # Converts to decimal or zero
+  ConvertDecimalOrZero = proc { |v| v.to_f }
+
+  # Convert to integer or nil
+  ConvertInteger = proc do |v|
+    begin
+      Integer(v)
+    rescue ArgumentError, TypeError
+      nil
+    end
+  end
+
+  # Convert to integer or zero
+  ConvertIntegerOrZero = proc { |v| v.to_i }
+
+  # Convert to decimal or raise error
+  StrictConvertDecimal = proc { |v| Float(v) }
+
+  # Convert to integer or raise error
+  StrictConvertInteger = proc { |v| Integer(v) }
+end

data/lib/honey_format/converters/convert_string.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'securerandom'
+
+module HoneyFormat
+  # Convert to downcase or nil
+  ConvertDowncase = proc { |v| v&.downcase }
+
+  # Convert to upcase or nil
+  ConvertUpcase = proc { |v| v&.upcase }
+
+  # Convert to symbol or nil
+  ConvertSymbol = proc { |v| v&.to_sym }
+
+  # Convert to md5 or nil
+  ConvertMD5 = proc { |v| Digest::MD5.hexdigest(v) if v }
+
+  # Convert to hex or nil
+  ConvertHex = proc { |v| SecureRandom.hex if v }
+
+  # Convert to blank string
+  ConvertBlank = proc { '' }
+
+  # Convert header column
+  ConvertHeaderColumn = HeaderColumnConverter
+
+  # Convert to upcase or raise error
+  StrictConvertUpcase = proc do |v|
+    ConvertUpcase.call(v) || raise(ArgumentError, "can't convert nil to upcased string")
+  end
+
+  # Convert to downcase or raise error
+  StrictConvertDowncase = proc do |v|
+    ConvertDowncase.call(v) || raise(ArgumentError, "can't convert nil to downcased string")
+  end
+
+  # Convert to symbol or raise error
+  StrictConvertSymbol = proc do |v|
+    ConvertSymbol.call(v) || raise(ArgumentError, "can't convert nil to symbol")
+  end
+end

data/lib/honey_format/converters/converters.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'honey_format/converters/header_column_converter'
+require 'honey_format/converters/convert_boolean'
+require 'honey_format/converters/convert_date_and_time'
+require 'honey_format/converters/convert_number'
+require 'honey_format/converters/convert_string'
+
+module HoneyFormat
+  # Convert to nil
+  ConvertNil = proc {}
+end

data/lib/honey_format/converters/header_column_converter.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module HoneyFormat
+  # Header column converter
+  module HeaderColumnConverter
+    # Bracket character matcher
+    BRACKETS = /\(|\[|\{|\)|\]|\}/
+
+    # Separator characters
+    SEPS = /'|"|\||\*|\^|\&|%|\$|€|#/
+
+    # Replace map
+    REPLACE_MAP = [
+      [/\\/, '/'],     # replace "\" with "/"
+      [/ \(/, '('],    # replace " (" with "("
+      [/ \[/, '['],    # replace " [" with "["
+      [/ \{/, '{'],    # replace " {" with "{"
+      [/ \{/, '{'],    # replace " {" with "{"
+      [/\) /, ')'],    # replace ") " with ")"
+      [/\] /, ']'],    # replace "] " with "]"
+      [/\} /, '}'],    # replace "} " with "}"
+      [BRACKETS, '_'], # replace (, [, {, ), ] and } with "_"
+      [/ +/, '_'],     # replace one or more spaces with "_"
+      [/-/, '_'],      # replace "-" with "("
+      [/::/, '_'],     # replace "::" with "_"
+      [%r{/}, '_'],    # replace "/" with "_"
+      [SEPS, '_'],     # replace separator chars with "_"
+      [/_+/, '_'],     # replace one or more "_" with single "_"
+      [/\A_+/, ''],    # remove leading "_"
+      [/_+\z/, ''],    # remove trailing "_"
+    ].map(&:freeze).freeze
+
+    # Returns converted value and mutates the argument.
+    # @return [Symbol] the cleaned header column.
+    # @param [String] column the string to be cleaned.
+    # @param [Integer] index the column index.
+    # @example Convert simple header
+    #     HeaderColumnConverter.call("  User name ") #=> "user_name"
+    # @example Convert complex header
+    #     HeaderColumnConverter.call(" First name (user)") #=> :'first_name(user)'
+    def self.call(column, index = nil)
+      if column.nil? || column.empty?
+        raise(ArgumentError, "column and column index can't be blank/nil") unless index
+        return :"column#{index}"
+      end
+
+      column = column.dup
+      column.strip!
+      column.downcase!
+      REPLACE_MAP.each do |data|
+        from, to = data
+        column.gsub!(from, to)
+      end
+      column.to_sym
+    end
+  end
+end

data/lib/honey_format/csv.rb

@@ -1,8 +1,7 @@
-require 'csv'
+# frozen_string_literal: true
 
+require 'csv'
 require 'honey_format/matrix'
-# require 'honey_format/rows'
-# require 'honey_format/header'
 
 module HoneyFormat
   # Represents CSV.
@@ -10,14 +9,18 @@ module HoneyFormat
     # Instantiate CSV.
     # @return [CSV] a new instance of CSV.
     # @param [String] csv the CSV string
-    # @param [String] delimiter the CSV column delimiter
-    # @param [String, Symbol] row_delimiter the CSV row delimiter (default: :auto)
-    # @param [String] quote_character the CSV quote character (default: ")
-    # @param [Array<String>] header optional argument that represents CSV header, required if the CSV file lacks a header row.
-    # @param [#call] header_converter converts header columns.
-    # @param [#call] row_builder will be called for each parsed row.
+    # @param delimiter [String] the CSV column delimiter
+    # @param row_delimiter [String, Symbol] the CSV row delimiter (default: :auto)
+    # @param quote_character [String] the CSV quote character (default: ")
+    # @param header [Array<String>]
+    #   header optional argument that represents CSV header, required if the CSV file
+    #   lacks a header row.
+    # @param header_converter [#call] converts header columns.
+    # @param header_deduplicator [#call] deduplicates header columns.
+    # @param row_builder [#call] will be called for each parsed row.
     # @param type_map [Hash] map of column_name => type conversion to perform.
-    # @param skip_lines [Regexp, String] Regexp for determining wheter a line is a comment. See CSV skip_lines option.
+    # @param skip_lines [Regexp, String]
+    #   Regexp for determining wheter a line is a comment. See CSV skip_lines option.
     # @raise [HeaderError] super class of errors raised when there is a CSV header error.
     # @raise [MissingHeaderError] raised when header is missing (empty or nil).
     # @raise [MissingHeaderColumnError] raised when header column is missing.
@@ -40,17 +43,21 @@ module HoneyFormat
     #   rescue HoneyFormat::RowError => e
     #     puts "row error: #{e.class}, #{e.message}"
     #   end
+    # @example Skip lines all lines starting with '#'
+    #   csv = HoneyFormat::CSV.new("name,id\n# some comment\njacob,1", skip_lines: '#')
+    #   csv.rows.length # => 1
     # @see Matrix#new
     def initialize(
       csv,
-      delimiter: ',',
-      row_delimiter: :auto,
-      quote_character: '"',
+      delimiter: HoneyFormat.config.delimiter,
+      row_delimiter: HoneyFormat.config.row_delimiter,
+      quote_character: HoneyFormat.config.quote_character,
       header: nil,
       header_converter: HoneyFormat.header_converter,
+      header_deduplicator: HoneyFormat.config.header_deduplicator,
       row_builder: nil,
       type_map: {},
-      skip_lines: nil
+      skip_lines: HoneyFormat.config.skip_lines
     )
       csv = ::CSV.parse(
         csv,
@@ -64,6 +71,7 @@ module HoneyFormat
         csv,
         header: header,
         header_converter: header_converter,
+        header_deduplicator: header_deduplicator,
         row_builder: row_builder,
         type_map: type_map
       )

data/lib/honey_format/errors.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module HoneyFormat
   # Errors
   module Errors
@@ -8,6 +10,10 @@ module HoneyFormat
     class MissingHeaderError < HeaderError; end
     # Raised when header column is missing
     class MissingHeaderColumnError < HeaderError; end
+    # Raised when header column duplicate is found
+    class DuplicateHeaderColumnError < HeaderError; end
+    # Raised when deduplication strategy is unknown
+    class UnknownDeduplicationStrategyError < HeaderError; end
 
     # Row errors
     # Super class of errors raised when there is a row error
@@ -19,9 +25,9 @@ module HoneyFormat
 
     # Value conversion errors
     # Raised when value type is unknown
-    class UnknownValueTypeError < ArgumentError; end
+    class UnknownTypeError < ArgumentError; end
     # Raised when value type already exists
-    class ValueTypeExistsError < ArgumentError; end
+    class TypeExistsError < ArgumentError; end
   end
 
   include Errors

data/lib/honey_format/helpers/helpers.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module HoneyFormat
+  module Helpers
+    # Converts a Hash with key => count to a deduplicated array.
+    # @param [Hash] data with key => count
+    # @return [Array<Symbol>] an array of symbols
+    # @example
+    #   Helpers.key_count_to_deduplicated_array({ a: 2, b: 1, c: 0})
+    #   # => [:a, :a1, :b]
+    def self.key_count_to_deduplicated_array(data)
+      array = []
+      count_occurences(data).each do |key, value|
+        next array << key if value == 1
+
+        values = Array.new(value) { |i| i }.map do |index|
+          next key if index.zero?
+          :"#{key}#{index}"
+        end
+        array.concat(values)
+      end
+      array
+    end
+
+    # Returns hash with key => occurrences_count
+    # @param [Array<Object>] the array to count occurrences in
+    # @return [Hash] key => occurrences_count
+    def self.count_occurences(array)
+      occurrences = Hash.new(0)
+      array.each { |column| occurrences[column] += 1 }
+      occurrences
+    end
+
+    # Returns array with duplicated objects
+    # @param [Array<Object>] the array to find duplicates in
+    # @return [Array<Object>] array of duplicated objects
+    def self.duplicated_items(array)
+      array.select { |col| array.count(col) > 1 }.uniq
+    end
+  end
+end

data/lib/honey_format/{header.rb → matrix/header.rb}

@@ -1,4 +1,6 @@
-require 'set'
+# frozen_string_literal: true
+
+require 'honey_format/helpers/helpers'
 
 module HoneyFormat
   # Represents a header
@@ -8,28 +10,36 @@ module HoneyFormat
     # Instantiate a Header
     # @return [Header] a new instance of Header.
     # @param [Array<String>] header array of strings.
-    # @param converter [#call, Symbol] header converter that implements a #call method that takes one column (string) argument OR symbol for a registered value converter.
+    # @param converter [#call, Symbol]
+    #   header converter that implements a #call method
+    #   that takes one column (string) argument OR symbol for a registered
+    #   converter registry.
+    # @param deduplicator [#call, Symbol]
+    #   header deduplicator that implements a #call method
+    #   that takes columns Array<String> argument OR symbol for a registered
+    #   deduplicator registry.
     # @raise [HeaderError] super class of errors raised when there is a CSV header error.
     # @raise [MissingHeaderColumnError] raised when header is missing
     # @example Instantiate a header with a custom converter
     #     converter = ->(col) { col == 'username' ? 'handle' : col }
     #     header = HoneyFormat::Header.new(['name', 'username'], converter: converter)
     #     header.to_a # => ['name', 'handle']
-    def initialize(header, converter: HoneyFormat.header_converter)
+    def initialize(
+      header,
+      converter: HoneyFormat.header_converter,
+      deduplicator: HoneyFormat.config.header_deduplicator
+    )
       if header.nil? || header.empty?
         raise(Errors::MissingHeaderError, "CSV header can't be empty.")
       end
 
       @original_header = header
-      @converter = if converter.is_a?(Symbol)
-                     HoneyFormat.value_converter[converter]
-                   else
-                     converter
-                   end
-
+      self.deduplicator = deduplicator
+      self.converter = converter
       @columns = build_columns(@original_header)
     end
 
+    # Returns the original header
     # @return [Array<String>] the original header
     def original
       @original_header
@@ -82,15 +92,41 @@ module HoneyFormat
 
     private
 
+    # Set the header converter
+    # @param [Symbol, #call] symbol to known converter or object that responds to #call
+    # @return [nil]
+    def converter=(object)
+      if object.is_a?(Symbol)
+        @converter = HoneyFormat.converter_registry[object]
+        return
+      end
+
+      @converter = object
+    end
+
+    # Set the header deduplicator
+    # @param [Symbol, #call] symbol to known deduplicator or object that responds to #call
+    # @return [nil]
+    def deduplicator=(object)
+      if object.is_a?(Symbol)
+        @deduplicator = HoneyFormat.header_deduplicator_registry[object]
+        return
+      end
+
+      @deduplicator = object
+    end
+
     # Convert original header
     # @param [Array<String>] header the original header
     # @return [Array<String>] converted columns
     def build_columns(header)
-      header.each_with_index.map do |header_column, index|
+      columns = header.each_with_index.map do |header_column, index|
         convert_column(header_column, index).tap do |column|
           maybe_raise_missing_column!(column)
         end
       end
+
+      @deduplicator.call(columns)
     end
 
     # Convert the column value
@@ -122,8 +158,8 @@ module HoneyFormat
 
       parts = [
         "CSV header column can't be nil or empty!",
-        "When you pass your own converter make sure that it never returns nil or an empty string.",
-        'Instead generate unique columns names.'
+        'When you pass your own converter make sure that it never returns nil or an empty string.', # rubocop:disable Metrics/LineLength
+        'Instead generate unique columns names.',
       ]
       raise(Errors::MissingHeaderColumnError, parts.join(' '))
     end