comma_splice 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f6b56980107ae3e81c5d61f6331f4eedcd21d6120f145f6a3b4e18c79a4d63b
-  data.tar.gz: cf3bac98bb36e2a94f6d939b54496d803cbc7078ee044b1972f66450b667628e
+  metadata.gz: 67cde27bb8ea56782dfe614bb99c593c121c369c01730e211aa7e058cbc22251
+  data.tar.gz: f015c933560a42374d201422780d693ee31ac43394a9d587aa80f582037d647e
 SHA512:
-  metadata.gz: 17025c468ae170c2e7a3a3b8555072208181e1a020dbd835d8ee4ced13d7dd91c8eca4f99b6c27ce25b279e2f6a2354c0af734c42446a2cf31375d572d8c283d
-  data.tar.gz: dd0bf6268d3ee42443a143e6d99df65c56bb65703eddf0e67f10d86651af9c7a56f2ae3199bb31d39a03ae1d7cc673a4de9278abb6922e0fd5ce8dd68e082bf1
+  metadata.gz: e34800ecbdfd7454b83aa6faa34ec48aa72468428df32c11176319ae44cd1b50b6c06195072ad5bec15968394830eb51209795ed4861406468a6384f0ec7f665
+  data.tar.gz: cf77db07813250f91e0dd1e73bd0b11740ffa2fb50b00e5f4bd9ec3e3023c89c34c73afb31d735c00297f04135448c5ca6ff26dcdb00243b2b6bcc1fb5e1fc6f

data/.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "11:00"
+  open-pull-requests-limit: 10
+  ignore:
+  - dependency-name: activesupport
+    versions:
+    - ">= 6.a, < 7"
+  - dependency-name: activesupport
+    versions:
+    - 5.2.4.4
+    - 5.2.4.5

data/.github/workflows/ruby.yml

@@ -0,0 +1,21 @@
+name: Ruby
+on: [push, pull_request]
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, macos-latest ]
+        ruby: [ '2.5', '2.6', '2.7', '3.0', '3.1' ]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: Build and test with Rake
+      run: |
+        gem install bundler
+        bundle install --jobs 4 --retry 3
+        bundle exec rake

data/.gitignore

@@ -9,3 +9,5 @@
 .DS_Store
 .byebug_history
 test-file.txt
+.idea/
+.rspec_status

data/CHANGELOG.md

@@ -1,4 +1,9 @@
 # Changelog
+### 0.3.0 (November 3, 2022)
+- [IMPROVEMENT] Add support for using a separator besides comma, through the `separator: ` argument. Defaults to comma if not supplied.
+
+### 0.2.3 (January 28, 2020)
+- [BUGFIX] Fix another scoring issue
 
 ### 0.2.2 (January 27, 2020)
 - [BUGFIX] Fix another scoring issue

data/Gemfile.lock

@@ -1,47 +1,46 @@
 PATH
   remote: .
   specs:
-    comma_splice (0.2.2)
+    comma_splice (0.2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.2.3)
+    activesupport (7.0.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-    byebug (11.0.1)
-    concurrent-ruby (1.1.5)
-    diff-lcs (1.3)
-    docile (1.3.2)
-    i18n (1.6.0)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.10)
+    diff-lcs (1.5.0)
+    docile (1.4.0)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    json (2.3.0)
-    minitest (5.11.3)
-    rake (10.5.0)
-    rspec (3.8.0)
-      rspec-core (~> 3.8.0)
-      rspec-expectations (~> 3.8.0)
-      rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.2)
-      rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.4)
+    minitest (5.16.3)
+    rake (13.0.6)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.0)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.1)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-support (3.8.2)
-    simplecov (0.17.1)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    simplecov (0.21.2)
       docile (~> 1.1)
-      json (>= 1.8, < 3)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.2)
-    thor (0.20.3)
-    thread_safe (0.3.6)
-    tzinfo (1.2.5)
-      thread_safe (~> 0.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    thor (1.2.1)
+    tzinfo (2.0.5)
+      concurrent-ruby (~> 1.0)
 
 PLATFORMS
   ruby
@@ -51,7 +50,7 @@ DEPENDENCIES
   bundler (~> 2.0)
   byebug
   comma_splice!
-  rake (~> 10.0)
+  rake (~> 13.0)
   rspec
   simplecov
   thor

data/README.md

@@ -87,6 +87,9 @@ You can use this in a ruby program by using installing the `comma_splice` gem, o
 
 ```ruby
   CommaSplice::FileCorrector.new(file_path).bad_lines.size
+
+  #you can specify another separator
+  CommaSplice::FileCorrector.new(file_path, separator: ';').bad_lines.size
 ```
 ```
   comma_splice bad_line_count /path/to/file.csv
@@ -95,6 +98,9 @@ You can use this in a ruby program by using installing the `comma_splice` gem, o
 ##### Display the fixed contents
 ```ruby
   CommaSplice::FileCorrector.new(file_path).corrected
+  
+  #you can specify another separator
+  CommaSplice::FileCorrector.new(file_path, separator: ';').corrected
 ```
 ```bash
   comma_splice correct /path/to/file.csv
@@ -103,6 +109,9 @@ You can use this in a ruby program by using installing the `comma_splice` gem, o
 ##### Process a file and save the fixed version
 ```ruby
   CommaSplice::FileCorrector.new(file_path).save(save_path)
+  
+  #you can specify another separator
+  CommaSplice::FileCorrector.new(file_path, separator: ';').save(save_path)
 ```
 ```bash
   comma_splice fix /path/to/file.csv /path/to/save

data/bin/comma_splice

@@ -8,6 +8,7 @@ class CommaSpliceCLI < Thor
   class_option :start_line, type: :numeric, default: nil
   class_option :end_line, type: :numeric, default: nil
   class_option :debug, type: :boolean, default: false
+  class_option :separator, type: :string, default: ','
 
   desc 'version', 'print the current comma_splice version'
   def version
@@ -21,7 +22,8 @@ class CommaSpliceCLI < Thor
     file_corrector = CommaSplice::FileCorrector.new(
       file_path,
       start_line: options[:start_line],
-      end_line: options[:end_line]
+      end_line: options[:end_line],
+      separator: options[:separator]
     )
 
     puts file_corrector.corrected
@@ -34,7 +36,8 @@ class CommaSpliceCLI < Thor
     file_corrector = CommaSplice::FileCorrector.new(
       file_path,
       start_line: options[:start_line],
-      end_line: options[:end_line]
+      end_line: options[:end_line],
+      separator: options[:separator]
     )
 
     file_corrector.save(fix_path)
@@ -47,7 +50,8 @@ class CommaSpliceCLI < Thor
     file_corrector = CommaSplice::FileCorrector.new(
       file_path,
       start_line: options[:start_line],
-      end_line: options[:end_line]
+      end_line: options[:end_line],
+      separator: options[:separator]
     )
 
     puts file_corrector.bad_lines
@@ -60,7 +64,8 @@ class CommaSpliceCLI < Thor
     file_corrector = CommaSplice::FileCorrector.new(
       file_path,
       start_line: options[:start_line],
-      end_line: options[:end_line]
+      end_line: options[:end_line],
+      separator: options[:separator]
     )
 
     puts file_corrector.bad_lines.size

data/comma_splice.gemspec

@@ -38,7 +38,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "byebug"
   spec.add_development_dependency "activesupport"

data/lib/comma_splice/file_corrector.rb

@@ -1,21 +1,22 @@
 module CommaSplice
   class FileCorrector
-    attr_reader :file_contents, :csv_content, :start_line, :end_line, :start_column, :end_column
+    attr_reader :file_contents, :csv_content, :start_line, :end_line, :start_column, :end_column, :separator
 
-    def initialize(file_path, start_line: nil, end_line:nil, start_column: nil, end_column: nil)
+    def initialize(file_path, start_line: nil, end_line: nil, start_column: nil, end_column: nil, separator: ',')
       @file_path       = file_path
       @file_contents   = File.read(file_path, encoding: 'utf-8')
+      @separator       = separator
 
-      @content_finder = ContentFinder.new(@file_contents, start_line, end_line)
+
+      @content_finder = ContentFinder.new(@file_contents, start_line, end_line, separator)
       @csv_content   = @content_finder.content
       @start_line    = @content_finder.start_line
       @end_line      = @content_finder.end_line
-
       if start_column && end_column
         @start_column = start_column
         @end_column = end_column
       else
-        finder = VariableColumnFinder.new(@csv_content[0], @csv_content[1..-1])
+        finder = VariableColumnFinder.new(@csv_content[0], @csv_content[1..-1], @separator)
         @start_column = finder.start_column
         @end_column = finder.end_column
       end
@@ -24,7 +25,7 @@ module CommaSplice
     end
 
     def header
-      @header ||= Line.new(csv_content.first)
+      @header ||= Line.new(csv_content.first, @separator)
     end
 
     def bad_lines
@@ -60,7 +61,7 @@ module CommaSplice
       end
     end
 
-    def to_json
+    def to_json(*_args)
       @content_finder.parsed.try(:to_json)
     end
 
@@ -68,7 +69,7 @@ module CommaSplice
 
     def line_correctors
       @line_correctors ||= csv_content.collect do |line|
-        LineCorrector.new(header, Line.new(line), @start_column, @end_column)
+        LineCorrector.new(header, Line.new(line, @separator), @start_column, @end_column, @separator)
       end
     end
 

data/lib/comma_splice/helpers/comma_calculator.rb

@@ -1,13 +1,18 @@
 # frozen_string_literal: true
+
 module CommaSplice
   # provide an array of CSV headers and and array of CSV values
   # and this will figure out the best correction and prompt
   # you if it can't find out
 
   class CommaCalculator
-    def initialize(headers, values)
-      raise StandardError, "Determining all the possibilities to fit #{values.size} values into the #{headers.size} headers #{headers.inspect} is computationally expensive. Please specify the columns where commas might be." if headers.size > 10 && values.size > 10
+    def initialize(headers, values, separator = ',')
+      if headers.size > 10 && values.size > 10
+        raise StandardError,
+              "Determining all the possibilities to fit #{values.size} values into the #{headers.size} headers #{headers.inspect} is computationally expensive. Please specify the columns where commas might be."
+      end
 
+      @separator      = separator
       @headers        = headers
       @values         = values
       @longest_header = @headers.max_by(&:length)
@@ -41,12 +46,12 @@ module CommaSplice
       end
 
       @ranked_options ||= @all_options.collect do |option|
-        OptionScorer.new(option)
+        OptionScorer.new(option, separator: @separator)
       end
     end
 
     def score_option(option)
-      OptionScorer.new(option).score
+      OptionScorer.new(option, separator: @separator).score
     end
 
     def best_options
@@ -62,7 +67,13 @@ module CommaSplice
       @headers.size < @values.size
     end
 
-    private
+    def print_all_options
+      ranked_options.each_with_index do |option, index|
+        print_option(option, index)
+      end
+    end
+
+    protected
 
     def join_possibilities
       JoinPossibilities.new(@values.size, @headers.size).possibilities
@@ -73,7 +84,7 @@ module CommaSplice
         print_option(option, index)
       end
 
-      puts "press 0 to see all options" if ranked_options.size != options.size
+      puts 'press 0 to see all options' if ranked_options.size != options.size
 
       selected_option = nil
       until selected_option && selected_option.to_i > -1
@@ -103,9 +114,7 @@ module CommaSplice
                header.ljust(@longest_header.size) + ': ' +
                option.option[i].to_s.ljust(75)
 
-        if CommaSplice.debug
-          line = line + "| " + (score_breakdown.shift || "")
-        end
+        line = line + '| ' + (score_breakdown.shift || '') if CommaSplice.debug
 
         lines << line
       end

data/lib/comma_splice/helpers/content_finder.rb

@@ -4,11 +4,11 @@ module CommaSplice
   # Given a file this will find the CSV content. Some files have some non-csv junk at the top
 
   class ContentFinder
-    attr_reader :start_line, :end_line, :content
+    attr_reader :start_line, :end_line, :content, :separator
 
-    def initialize(file_contents, start_line = nil, end_line = nil)
+    def initialize(file_contents, start_line = nil, end_line = nil, separator = ',')
       @file_contents = file_contents
-
+      @separator = separator
       if start_line && end_line
         # the csvs this was built for have non-csv headers
         @start_line  = start_line
@@ -21,11 +21,11 @@ module CommaSplice
 
     def find_content
       @start_line = @file_contents.lines.find_index do |line|
-        Line.new(line).values.size > 2
+        Line.new(line, separator).values.size > 2
       end
 
       relative_end_line = @file_contents.lines[@start_line..-1].find_index do |line|
-        Line.new(line).values.size < 2
+        Line.new(line, separator).values.size < 2
       end
 
       @end_line = if relative_end_line
@@ -40,7 +40,7 @@ module CommaSplice
     def parsed
       quote_chars = %w[" | ~ ^ & *]
       begin
-        CSV.parse(@content.join("\n"), quote_char: quote_chars.shift, headers: :first_row, liberal_parsing: true)
+        CSV.parse(@content.join('\n'), col_sep: separator, quote_char: quote_chars.shift, headers: :first_row, liberal_parsing: true)
       rescue CSV::MalformedCSVError
         quote_chars.empty? ? raise : retry
       end

data/lib/comma_splice/helpers/line.rb

@@ -1,9 +1,10 @@
 module CommaSplice
   class Line
-    attr_reader :values, :line
+    attr_reader :values, :line, :separator
 
-    def initialize(line)
+    def initialize(line, separator)
       @line = line
+      @separator = separator
       @values = parse_csv_content(line).first
     end
 
@@ -12,7 +13,8 @@ module CommaSplice
     def parse_csv_content(content, headers = false)
       quote_chars = %w[" | ~ ^ & *]
       begin
-        CSV.parse(content.mb_chars.tidy_bytes.to_s, quote_char: quote_chars.shift, headers: headers, liberal_parsing: true)
+        CSV.parse(content.mb_chars.tidy_bytes.to_s, col_sep: @separator, quote_char: quote_chars.shift,
+                                                    headers:, liberal_parsing: true)
       rescue CSV::MalformedCSVError
         quote_chars.empty? ? raise : retry
       end

data/lib/comma_splice/helpers/option_scorer.rb

@@ -3,9 +3,10 @@ module CommaSplice
   class OptionScorer
     attr_reader :option
 
-    def initialize(option)
+    def initialize(option, separator: ',')
       @option = option
       @start_score = 100
+      @separator = separator
     end
 
     def breakdown
@@ -15,9 +16,7 @@ module CommaSplice
       rules.each do |rule|
         rule_score = send(rule.to_sym)
         score += rule_score
-        if rule_score != 0
-          breakdown << "#{rule_score.to_s.ljust(3)} #{rule.to_sym}"
-        end
+        breakdown << "#{rule_score.to_s.ljust(3)} #{rule.to_sym}" if rule_score != 0
       end
 
       breakdown.unshift("score: #{score}")
@@ -43,21 +42,21 @@ module CommaSplice
       end.size * -1
     end
 
-    def options_that_start_with_a_comma
+    def options_that_start_with_a_separator
       option.select do |o|
-        o.to_s.starts_with?(',')
+        o.to_s.starts_with?(@separator)
       end.size * -5
     end
 
-    def options_that_end_with_a_comma
+    def options_that_end_with_a_separator
       option.select do |o|
-        o.to_s.ends_with?(',')
+        o.to_s.ends_with?(@separator)
       end.size * -5
     end
 
-    def options_that_have_words_joined_by_commas
+    def options_that_have_words_joined_by_separators
       option.select do |o|
-        o.to_s.match(/[^0-9],[\w]/) || o.to_s.match(/[^\w],[0-9]/)
+        Regexp.new("[^0-9\\s]#{@separator}\\w").match(o.to_s) || Regexp.new("\\w#{@separator}[^0-9\\s]").match(o.to_s)
       end.compact.size * -5
     end
 
@@ -68,6 +67,8 @@ module CommaSplice
     end
 
     def options_that_have_longest_comma_separated_number
+      # return 0 unless @separator == ','
+
       # favor items that have a longer comma separated number
       # i.e in the following example, option 1 should win
       # (1)    artist    : Half Japanese

data/lib/comma_splice/helpers/variable_column_finder.rb

@@ -18,12 +18,12 @@ module CommaSplice
   #     17385094,,,01-27-2019 @ 13:47:00,KIng Tubby Meets The Upsetter,King And The Upsetter At Spanish Town,KIng Tubby Meets The Upsetter,Celluloid,post,live,y,
 
   class VariableColumnFinder
-    attr_reader :start_column, :end_column
+    attr_reader :start_column, :end_column, :separator
 
-    def initialize(header_line, value_lines)
+    def initialize(header_line, value_lines, separator = ',')
       @values = value_lines
       @header = header_line
-
+      @separator = separator
       find_variable_column_boundaries
     end
 
@@ -44,9 +44,9 @@ module CommaSplice
 
     def left_to_right_index
       left_to_right_index = []
-      @header.split(',').size.times do |time|
+      @header.split(@separator).size.times do |time|
         left_to_right_index.push(@values.map do |value_line|
-          value_line.split(',')[time].to_s.size
+          value_line.split(@separator)[time].to_s.size
         end.uniq.size == 1)
       end
 
@@ -55,9 +55,9 @@ module CommaSplice
 
     def right_to_left_index
       right_to_left_index = []
-      @header.split(',').size.times do |time|
+      @header.split(@separator).size.times do |time|
         right_to_left_index.unshift(@values.map do |value_line|
-          value_line.split(',')[-time].to_s.size
+          value_line.split(@separator)[-time].to_s.size
         end.uniq.size == 1)
       end
 

data/lib/comma_splice/line_corrector.rb

@@ -1,10 +1,10 @@
 module CommaSplice
   class LineCorrector
-    attr_reader :headers, :values, :header_line, :value_line, :right_bounds, :left_bounds
+    attr_reader :headers, :values, :header_line, :value_line, :right_bounds, :left_bounds, :separator
 
-    def initialize(header_line, value_line, left_bounds = 0, right_bounds = -1)
-      header_line = Line.new(header_line) unless header_line.is_a?(Line)
-      value_line  = Line.new(value_line) unless value_line.is_a?(Line)
+    def initialize(header_line, value_line, left_bounds = 0, right_bounds = -1, separator = ',')
+      header_line = Line.new(header_line, separator) unless header_line.is_a?(Line)
+      value_line  = Line.new(value_line, separator) unless value_line.is_a?(Line)
 
       @header_line = header_line
       @value_line = value_line
@@ -12,13 +12,14 @@ module CommaSplice
       @values = value_line.values
       @left_bounds = left_bounds
       @right_bounds = right_bounds
+      @separator = separator
 
       raise 'right bounds must be negative' unless right_bounds.negative?
       raise 'left bounds must be not be negative' if left_bounds.negative?
     end
 
     def needs_correcting?
-      @values && @values.size.positive? && @headers.size != @values.size
+      @values&.size&.positive? && @headers.size != @values.size
     end
 
     def needs_manual_input?
@@ -26,9 +27,14 @@ module CommaSplice
     end
 
     def option_count
+      puts corrector.best_options
       corrector.best_options.size
     end
 
+    def all_options
+      corrector.ranked_options
+    end
+
     def original
       generate_csv_line(@values)
     end
@@ -48,14 +54,18 @@ module CommaSplice
       generate_csv_line([values_before, corrector.correction, values_after].flatten)
     end
 
-    private
-
-    def generate_csv_line(values)
-      CSV.generate_line(values)
+    def print_all_options
+      corrector.print_all_options
     end
 
+    protected
+
     def corrector
-      CommaCalculator.new(selected_headers, selected_values)
+      CommaCalculator.new(selected_headers, selected_values, @separator)
+    end
+
+    def generate_csv_line(values)
+      CSV.generate_line(values, col_sep: @separator)
     end
 
     def selected_headers

data/lib/comma_splice/version.rb

@@ -1,3 +1,3 @@
 module CommaSplice
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: comma_splice
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Jeff Keen
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-01-27 00:00:00.000000000 Z
+date: 2022-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -102,6 +102,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
 - CHANGELOG.md
@@ -129,7 +131,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -144,8 +146,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key: 
+rubygems_version: 3.3.7
+signing_key:
 specification_version: 4
 summary: Fixes CSVs with unescaped commas
 test_files: []