comma_splice 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 74958cacc824b9e475df20acee4cc75ac354738ab4ed475a7ddc5f3ce5c7c553
-  data.tar.gz: 3b7129b58e24be7e9549e2971b994fd159e0d21675118e3c413a38dc1823db9c
+  metadata.gz: bd44bca5b615b5f267ae81b16aaa6b379b53963e637af1cc000d6229996963d4
+  data.tar.gz: bd70a25937f8baf3cec97eeb5276ce050f3898a76f5be9cf8e0c8e726bc5a8fa
 SHA512:
-  metadata.gz: 112fea4670275a7222bbf60d13c8f61cea9386a60eaaf2cf16fd514f5c0ffd69a4e2e0453658408977607608340ca1e94e7f53581565d10014501c18d61a6570
-  data.tar.gz: b62a9835b22555efd4aad03c467c1d24f298d50c34b399171f8fc5bcb0d8c1ddfff1be14268599fe5ce67b88c6b9201c6c3e250e72de34361ba252594a6146e1
+  metadata.gz: 991f2138d4d08941b1a83d231338a5197bce6bcea0e136c3cbb67ca67cf04b31ad425ce148d48f2bf5ca8280bfee2b736d7ace002cd043dac071bdee8ca2ad2f
+  data.tar.gz: 4d9e601ab359fe6511d9c54b3469104e64f057953924a7867c6b7e58df224c9b5d9b279060ff7e67e3d965da7460334ed23f97c1fd2792c32441e2cc9f8161da

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+### 0.2 (January 27, 2020)
+- [IMPROVEMENT] Add scoring model to better handle cases that needed prompting before, like comma-separated numbers
+- [IMPROVEMENT] Correct line escaping even on lines that don't have incorrect commas to ensure correct parsing of generated CSV down the line
+- [IMPROVEMENT] Use ruby csv library to generate lines instead of handling escaping cases manually
+
 ### 0.1.1 (January 24, 2020)
 - [BUGFIX] handle case where all columns are equal widths
 - [BUGFIX] Improve error message

data/Gemfile

@@ -1,4 +1,3 @@
 source "https://rubygems.org"
-
 # Specify your gem's dependencies in comma_splice.gemspec
 gemspec

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    comma_splice (0.1.1)
+    comma_splice (0.2.0)
 
 GEM
   remote: https://rubygems.org/

data/bin/comma_splice

@@ -8,7 +8,12 @@ class CommaSpliceCLI < Thor
   class_option :start_line, type: :numeric, default: nil
   class_option :end_line, type: :numeric, default: nil
 
-  desc 'fix FILE_PATH [SAVE_PATH]', 'return corrected file contents'
+  desc 'version', 'print the current comma_splice version'
+  def version
+    puts CommaSplice::VERSION
+  end
+
+  desc 'correct FILE_PATH', 'return corrected file contents'
   def correct(file_path)
     file_corrector = CommaSplice::FileCorrector.new(
       file_path,

data/lib/comma_splice/file_corrector.rb

@@ -9,7 +9,7 @@ module CommaSplice
       @content_finder = ContentFinder.new(@file_contents, start_line, end_line)
       @csv_content   = @content_finder.content
       @start_line    = @content_finder.start_line
-      @end_line      = @content_finder.start_line
+      @end_line      = @content_finder.end_line
 
       if start_column && end_column
         @start_column = start_column
@@ -35,6 +35,10 @@ module CommaSplice
       bad_lines.size.positive?
     end
 
+    def needs_manual_input?
+      line_correctors.any?(&:needs_manual_input?)
+    end
+
     def corrected
       @corrected ||= [
         @file_contents.lines[0, @start_line],

data/lib/comma_splice/helpers/comma_calculator.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module CommaSplice
   # provide an array of CSV headers and and array of CSV values
   # and this will figure out the best correction and prompt
@@ -5,17 +6,17 @@ module CommaSplice
 
   class CommaCalculator
     def initialize(headers, values)
+      raise StandardError, "Determining all the possibilities to fit #{values.size} values into the #{headers.size} headers #{headers.inspect} is computationally expensive. Please specify the columns where commas might be." if headers.size > 10 && values.size > 10
+
       @headers = headers
       @values  = values
-
-      raise StandardError, "Determining all the possibilities to fit #{@values.size} values into the #{@headers.size} headers #{@headers.inspect} is computationally expensive. Please specify the columns where commas might be." if @headers.size > 10 && @values.size > 10
     end
 
     def correction
-      if @headers.size === @values.size
+      if @headers.size == @values.size
         @values
       elsif best_options.size == 1
-        best_options.first
+        best_options.first.option
       elsif best_options.size > 1
         prompt_for_options(best_options)
       else
@@ -24,12 +25,14 @@ module CommaSplice
     end
 
     def all_options
-      options = join_possibilities.collect do |joins|
+      @all_options ||= join_possibilities.collect do |joins|
         values = @values.dup
         joins.collect do |join_num|
           val = values.shift(join_num)
-          if val.size > 1
-            quoted_values(val)
+          if val.empty?
+            nil
+          elsif val.size == 1
+            val.first
           else
             val.join(',')
           end
@@ -37,14 +40,23 @@ module CommaSplice
       end
     end
 
-    def best_options
-      all_options.select do |option|
-        option.none? { |o| o.starts_with?(' ') || o.starts_with?('" ') }
+    def ranked_options
+      @ranked_options ||= all_options.collect do |option|
+        OptionScorer.new(option)
       end
     end
 
-    def requires_manual_input?
-      needs_correcting? && best_options.many?
+    def score_option(option)
+      OptionScorer.new(option).score
+    end
+
+    def best_options
+      max_score = ranked_options.collect { |o| o.score }.max
+      ranked_options.select { |o| o.score == max_score }
+    end
+
+    def needs_manual_input?
+      !best_options.one?
     end
 
     def needs_correcting?
@@ -53,10 +65,6 @@ module CommaSplice
 
     private
 
-    def quoted_values(values)
-      "\"#{values.join(',').gsub(/(?<!")(?:"{2})*\K\"/, '""')}\"" # escape a double quote if it hasn't been escaped already
-    end
-
     def join_possibilities
       JoinPossibilities.new(@values.size, @headers.size).possibilities
     end
@@ -66,21 +74,34 @@ module CommaSplice
 
       options.each_with_index do |option, index|
         @headers.each_with_index do |header, i|
-          marker = i.zero? ? "(#{index + 1})" : ''
-          puts marker.ljust(5) +
+          marker = if i.zero?
+                     "(#{index + 1})"
+                   else
+                     ''
+                   end
+
+          puts marker.ljust(7) +
                header.ljust(longest_header.size) + ': ' +
-               option[i]
+               option.option[i].to_s
         end
+        puts ''.ljust(7) + "(score = #{option.score})"
+        puts ''.ljust(7) + option.breakdown
         puts "\n"
       end
 
+      puts "press 0 to see all options" if all_options.size != options.size
+
       selected_option = nil
-      until selected_option && selected_option.to_i > 0
+      until selected_option && selected_option.to_i > -1
         puts 'which one is correct?'
         selected_option = STDIN.gets
       end
 
-      options[selected_option.to_i - 1]
+      if selected_option.to_i == 0
+        prompt_for_options(ranked_options.sort_by { |s| s.score.to_i }.reverse)
+      else
+        options[selected_option.to_i - 1].option
+      end
     end
   end
 end

data/lib/comma_splice/helpers/content_finder.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module CommaSplice
   # Given a file this will find the CSV content. Some files have some non-csv junk at the top
 
@@ -26,11 +28,11 @@ module CommaSplice
         Line.new(line).values.size < 2
       end
 
-      if relative_end_line
-        @end_line = @start_line + relative_end_line - 1
-      else
-        @end_line = -1
-      end
+      @end_line = if relative_end_line
+                    @start_line + relative_end_line - 1
+                  else
+                    -1
+                  end
 
       @content = @file_contents.lines[@start_line..@end_line]
     end
@@ -38,7 +40,7 @@ module CommaSplice
     def parsed
       quote_chars = %w[" | ~ ^ & *]
       begin
-        CSV.parse(@content.join('\n'), quote_char: quote_chars.shift, headers: :first_row, liberal_parsing: true)
+        CSV.parse(@content.join("\n"), quote_char: quote_chars.shift, headers: :first_row, liberal_parsing: true)
       rescue CSV::MalformedCSVError
         quote_chars.empty? ? raise : retry
       end

data/lib/comma_splice/helpers/option_scorer.rb

@@ -0,0 +1,114 @@
+module CommaSplice
+  # scores options based on how likely they are to be correct
+
+  class OptionScorer
+    attr_reader :option
+
+    def initialize(option)
+      @option = option
+      @start_score = 100
+    end
+
+    def breakdown
+      score     = @start_score
+      breakdown = []
+
+      rules.each do |rule|
+        rule_score = send(rule.to_sym)
+        score += rule_score
+        if rule_score != 0
+          breakdown << "#{rule_score.to_s.ljust(3)} #{rule.to_sym}"
+        end
+      end
+
+      breakdown.join("\n")
+    end
+
+    def score
+      score = @start_score
+      rules.each do |rule|
+        score += send(rule.to_sym)
+      end
+      score
+    end
+
+    def options_that_start_with_a_space
+      option.select do |o|
+        o.to_s.starts_with?(' ')
+      end.size * -10
+    end
+
+    def options_that_start_with_a_quote_followed_by_a_space
+      option.select do |o|
+        o.to_s.starts_with?('" ')
+      end.size * -1
+    end
+
+    def options_that_start_with_a_comma
+      option.select do |o|
+        o.to_s.starts_with?(',')
+      end.size * -5
+    end
+
+    def options_that_end_with_a_comma
+      option.select do |o|
+        o.to_s.ends_with?(',')
+      end.size * -5
+    end
+
+    def options_that_have_words_joined_by_commas
+      option.select do |o|
+        o.to_s.match(/[A-Za-z],[A-Za-z]/)
+      end.compact.size * -5
+    end
+
+    def options_that_are_blank
+      option.select do |o|
+        o.to_s.strip.blank?
+      end.size * -5
+    end
+
+    def options_that_have_longest_comma_separated_number
+      # favor items that have a longer comma separated number
+      # i.e in the following example, option 1 should win
+      # (1)    artist    : Half Japanese
+      #        title     : 1,000,000,000 Kisses
+      #        albumtitle: Beautiful Songs: The Best of Jad Fair & Half Japanese
+      #        label     : Stillwater/Fire
+      #
+      #
+      # (2)    artist    : Half Japanese,1,000,000
+      #        title     : 000 Kisses
+      #        albumtitle: Beautiful Songs: The Best of Jad Fair & Half Japanese
+      #        label     : Stillwater/Fire
+      #
+      #
+      # (3)    artist    : Half Japanese,1
+      #        title     : 000,000,000 Kisses
+      #        albumtitle: Beautiful Songs: The Best of Jad Fair & Half Japanese
+      #        label     : Stillwater/Fire
+      #
+      #
+      # (4)    artist    : Half Japanese,1,000
+      #        title     : 000,000 Kisses
+      #        albumtitle: Beautiful Songs: The Best of Jad Fair & Half Japanese
+      #        label     : Stillwater/Fire
+
+      option.collect do |o|
+        result = o.to_s.scan(/\d{1,3}(?:,\d{1,3})*(?:\.\d+)?/)
+        if result.first && result.first.index(',')
+          result.first.size
+        else
+          0
+        end
+      end.max.to_i
+    end
+
+    private
+
+    def rules
+      methods.grep(/options_that/)
+    end
+
+  end
+end

data/lib/comma_splice/helpers/variable_column_finder.rb

@@ -46,7 +46,7 @@ module CommaSplice
       left_to_right_index = []
       @header.split(',').size.times do |time|
         left_to_right_index.push(@values.map do |value_line|
-          value_line.split(',')[time].size
+          value_line.split(',')[time].to_s.size
         end.uniq.size == 1)
       end
 
@@ -57,7 +57,7 @@ module CommaSplice
       right_to_left_index = []
       @header.split(',').size.times do |time|
         right_to_left_index.unshift(@values.map do |value_line|
-          value_line.split(',')[-time].size
+          value_line.split(',')[-time].to_s.size
         end.uniq.size == 1)
       end
 

data/lib/comma_splice/line_corrector.rb

@@ -21,8 +21,12 @@ module CommaSplice
       @values && @values.size > 0 && @headers.size != @values.size
     end
 
+    def needs_manual_input?
+      corrector.needs_manual_input?
+    end
+
     def original
-      @values.join(',')
+      generate_csv_line(@values)
     end
 
     def corrected
@@ -37,11 +41,15 @@ module CommaSplice
 
       values_before = values[0...left_bounds]
       values_after = values.slice(right_bounds + 1, -(right_bounds + 1))
-      [values_before, corrector.correction, values_after].flatten.join(',')
+      generate_csv_line([values_before, corrector.correction, values_after].flatten)
     end
 
     private
 
+    def generate_csv_line(values)
+      CSV.generate_line(values)
+    end
+
     def corrector
       CommaCalculator.new(selected_headers, selected_values)
     end

data/lib/comma_splice/version.rb

@@ -1,3 +1,3 @@
 module CommaSplice
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/lib/comma_splice.rb

@@ -8,6 +8,7 @@ require 'comma_splice/helpers/variable_column_finder'
 require 'comma_splice/helpers/line'
 require 'comma_splice/helpers/join_possibilities'
 require 'comma_splice/helpers/comma_calculator'
+require 'comma_splice/helpers/option_scorer'
 
 require 'comma_splice/line_corrector'
 require 'comma_splice/file_corrector'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: comma_splice
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Jeff Keen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-24 00:00:00.000000000 Z
+date: 2020-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -120,6 +120,7 @@ files:
 - lib/comma_splice/helpers/content_finder.rb
 - lib/comma_splice/helpers/join_possibilities.rb
 - lib/comma_splice/helpers/line.rb
+- lib/comma_splice/helpers/option_scorer.rb
 - lib/comma_splice/helpers/variable_column_finder.rb
 - lib/comma_splice/line_corrector.rb
 - lib/comma_splice/version.rb