comma_splice 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e0a958c45c6fe94fbc6a4af1cb5fc2a0c20cf7a68728520d227b62a79e2078a5
+  data.tar.gz: 7e4e09b4a3308a2ccb2c5a58d1b56ced9f6f5927bf41d89680c4401b5aad2b5d
+SHA512:
+  metadata.gz: 1daff9cfdd5f54c37bd47edc83b02308a0f3fe7b0d8af86dd0a910e264929a5b5b1191386b3b06df42f157770c4bc733c656e3beb4d9bafe10d7bc65a5483185
+  data.tar.gz: f6e4c32e2e03d3bf915a9b8360146eaa34ed9b34a954cb4b9a74b08c68ef4f8ae15cba83c854f96b88b098b791f802200992e46194a373e27cdd388c40311136

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.DS_Store
+.byebug_history

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in comma_splice.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,52 @@
+PATH
+  remote: .
+  specs:
+    comma_splice (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.2.3)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    byebug (11.0.1)
+    concurrent-ruby (1.1.5)
+    diff-lcs (1.3)
+    i18n (1.6.0)
+      concurrent-ruby (~> 1.0)
+    minitest (5.11.3)
+    rake (10.5.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.2)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.4)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.2)
+    thor (0.20.3)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport
+  bundler (~> 2.0)
+  byebug
+  comma_splice!
+  rake (~> 10.0)
+  rspec
+  thor
+
+BUNDLED WITH
+   2.0.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Jeff Keen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,139 @@
+# Comma Splice
+
+This gem tackles one very specific problem: when CSVs have commas in the values and the values haven't been quoted. This determines which commas separate fields and which commas are part of a value, and corrects the file.
+
+For example, given the following CSV
+
+```
+timestamp,artist,title,albumtitle,label
+01-27-2019 @ 12:34:00,Lester Sterling, Lynn Taitt & The Jets,Check Point Charlie,Merritone Rock Steady 3: Bang Bang Rock Steady 1966-1968,Dub Store,
+01-27-2019 @ 12:31:00,Lester Sterling,Lester Sterling Special,Merritone Rock Steady 2: This Music Got Soul 1966-1967,Dub Store,
+
+```
+
+which parses incorrectly as:
+
+| timestamp             | artist          | title       | albumtitle      | label                                                      |
+|-----------------------|-----------------|-------------|-----------------|------------------------------------------------------------|
+| 01-27-2019 @ 12:34:00 | Lester Sterling | Lynn Taitt & The Jets   | Check Point Charlie                                    | Merritone Rock Steady 3: Bang Bang Rock Steady 1966-1968
+| 01-27-2019 @ 12:31:00 | Lester Sterling | Lester Sterling Special | Merritone Rock Steady 2: This Music Got Soul 1966-1967 | Dub Store   |
+
+
+Running this through `comma_splice fix /path/to/file` will return this corrected content:
+
+```
+timestamp,artist,title,albumtitle,label
+01-27-2019 @ 12:34:00,"Lester Sterling, Lynn Taitt & The Jets",Check Point Charlie,Merritone Rock Steady 3: Bang Bang Rock Steady 1966-1968,Dub Store,
+01-27-2019 @ 12:31:00,Lester Sterling,Lester Sterling Special,Merritone Rock Steady 2: This Music Got Soul 1966-1967,Dub Store,
+```
+
+| timestamp             | artist          | title       | albumtitle      | label                                                      |
+|-----------------------|-----------------|-------------|-----------------|------------------------------------------------------------|
+| 01-27-2019 @ 12:34:00 | Lester Sterling, Lynn Taitt & The Jets   | Check Point Charlie | Merritone Rock Steady 3: Bang Bang Rock Steady 1966-1968 | Dub Store |
+| 01-27-2019 @ 12:31:00 | Lester Sterling | Lester Sterling Special | Merritone Rock Steady 2: This Music Got Soul 1966-1967 | Dub Store   |
+
+
+If it can't determine where the comma should go, it prompts you for the possible options
+
+
+given the following CSV:
+
+```
+playid,playtype,genre,timestamp,artist,title,albumtitle,label,prepost,programtype,iswebcast,isrequest
+16851097,,,12-09-2017 @ 09:57:00,10,000 Maniacs and Michael Stipe,To Sir with Love,Campfire Songs,Rhino,post,live,y,
+16851096,,,12-09-2017 @ 09:44:00,Fran Jeffries,Mine Eyes,Fran Can Really Hang You Up the Most,Warwick,post,live,y,
+```
+
+It prompts:
+
+```
+Which one of these is correct?
+
+(1)  artist    : 10
+     title     : 000 Maniacs and Michael Stipe
+     albumtitle: To Sir with Love
+     label     : "Campfire Songs,Rhino"
+
+(2)  artist    : 10
+     title     : 000 Maniacs and Michael Stipe
+     albumtitle: "To Sir with Love,Campfire Songs"
+     label     : Rhino
+
+(3)  artist    : 10
+     title     : "000 Maniacs and Michael Stipe,To Sir with Love"
+     albumtitle: Campfire Songs
+     label     : Rhino
+
+(4)  artist    : "10,000 Maniacs and Michael Stipe"
+     title     : To Sir with Love
+     albumtitle: Campfire Songs
+     label     : Rhino
+```
+
+Select an option (4), and it returns:
+
+```
+playid,playtype,genre,timestamp,artist,title,albumtitle,label,prepost,programtype,iswebcast,isrequest
+16851097,,,12-09-2017 @ 09:57:00,"10,000 Maniacs and Michael Stipe",To Sir with Love,Campfire Songs,Rhino,post,live,y,
+16851096,,,12-09-2017 @ 09:44:00,Fran Jeffries,Mine Eyes,Fran Can Really Hang You Up the Most,Warwick,post,live,y,
+```
+
+## Usage
+
+You can use this in a ruby program by using installing the `comma_splice` gem, or you can install it on your system and use the `comma_splice` command line utility.
+
+
+##### Return the number of bad lines in a file
+
+```ruby
+  CommaSplice::FileCorrector.new(file_path).bad_lines.size
+```
+```
+  comma_splice bad_line_count /path/to/file.csv
+```
+
+##### Display the fixed contents
+```ruby
+  CommaSplice::FileCorrector.new(file_path).corrected
+```
+```bash
+  comma_splice correct /path/to/file.csv
+```
+
+##### Process a file and save the fixed version
+```ruby
+  CommaSplice::FileCorrector.new(file_path).save(save_path)
+```
+```bash
+  comma_splice fix /path/to/file.csv /path/to/save
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'comma_splice'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install comma_splice
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/jkeen/comma_splice.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/comma_splice

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'comma_splice'
+require 'thor'
+
+class CommaSpliceCLI < Thor
+  class_option :start_line, type: :numeric, default: nil
+  class_option :end_line, type: :numeric, default: nil
+
+  desc 'fix FILE_PATH [SAVE_PATH]', 'return corrected file contents'
+  def correct(file_path)
+    file_corrector = CommaSplice::FileCorrector.new(
+      file_path,
+      start_line: options[:start_line],
+      end_line: options[:end_line]
+    )
+
+    puts file_corrector.corrected
+  end
+
+  desc 'fix FILE_PATH [SAVE_PATH]', 'return corrected file contents'
+  def fix(file_path, fix_path)
+    file_corrector = CommaSplice::FileCorrector.new(
+      file_path,
+      start_line: options[:start_line],
+      end_line: options[:end_line]
+    )
+
+    file_corrector.save(fix_path)
+  end
+
+  desc 'bad_lines FILE_PATH', 'show bad lines'
+  def bad_lines(file_path)
+    file_corrector = CommaSplice::FileCorrector.new(
+      file_path,
+      start_line: options[:start_line],
+      end_line: options[:end_line]
+    )
+
+    puts file_corrector.bad_lines
+  end
+
+  desc 'bad_line_count FILE_PATH', 'check file contents for needed corrections'
+  def bad_line_count(file_path)
+    file_corrector = CommaSplice::FileCorrector.new(
+      file_path,
+      start_line: options[:start_line],
+      end_line: options[:end_line]
+    )
+
+    puts file_corrector.bad_lines.size
+  end
+end
+
+CommaSpliceCLI.start(ARGV)

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "comma_splice"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/comma_splice.gemspec

@@ -0,0 +1,46 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "comma_splice/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "comma_splice"
+  spec.version       = CommaSplice::VERSION
+  spec.authors       = ["Jeff Keen"]
+  spec.email         = ["jeff@keen.me"]
+
+  spec.summary       = %q{Fixes CSVs with unescaped commas}
+  spec.description   = %q{}
+  spec.homepage      = "http://github.com/jkeen/comma_splice"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata["allowed_push_host"] = 'https://rubygems.org'
+
+    # spec.metadata["homepage_uri"] = spec.homepage
+    # spec.metadata["source_code_uri"] = "http://github.com/jkeen/comma_splice"
+    # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "bin"
+  spec.executables   << 'comma_splice'
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "byebug"
+  spec.add_development_dependency "activesupport"
+  spec.add_development_dependency "thor"
+end

data/lib/comma_splice.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'csv'
+require 'active_support/core_ext/string'
+require 'comma_splice/version'
+require 'comma_splice/helpers/content_finder'
+require 'comma_splice/helpers/variable_column_finder'
+require 'comma_splice/helpers/line'
+require 'comma_splice/helpers/join_possibilities'
+require 'comma_splice/helpers/comma_calculator'
+
+require 'comma_splice/line_corrector'
+require 'comma_splice/file_corrector'
+require 'byebug'
+
+module CommaSplice
+  class Error < StandardError; end
+end

data/lib/comma_splice/file_corrector.rb

@@ -0,0 +1,81 @@
+module CommaSplice
+  class FileCorrector
+    attr_reader :file_contents, :csv_content, :start_line, :end_line, :start_column, :end_column
+
+    def initialize(file_path, start_line: nil, end_line:nil, start_column: nil, end_column: nil)
+      @file_path       = file_path
+      @file_contents   = File.read(file_path, encoding: 'utf-8')
+
+      @content_finder = ContentFinder.new(@file_contents, start_line, end_line)
+      @csv_content   = @content_finder.content
+      @start_line    = @content_finder.start_line
+      @end_line      = @content_finder.start_line
+
+      if start_column && end_column
+        @start_column = start_column
+        @end_column = end_column
+      else
+        finder = VariableColumnFinder.new(@csv_content[0], @csv_content[1..-1])
+        @start_column = finder.start_column
+        @end_column = finder.end_column
+      end
+
+      raise CommaSplice::Error, "empty contents #{file_path}" unless @csv_content.present?
+    end
+
+    def header
+      @header ||= Line.new(csv_content.first)
+    end
+
+    def bad_lines
+      line_correctors.select(&:needs_correcting?).collect(&:original)
+    end
+
+    def needs_correcting?
+      bad_lines.size.positive?
+    end
+
+    def corrected
+      @corrected ||= [
+        @file_contents.lines[0, @start_line],
+        corrected_lines,
+        @file_contents.lines[@end_line, -1]
+      ].flatten
+    end
+
+    def save!
+      save(@file_path)
+    end
+
+    def save(path)
+      File.open(path, 'w+') do |f|
+        corrected.each_with_index do |line, index|
+          # don't add an extra line break at the end
+          f.puts line if corrected.size > index && line
+        end
+      end
+    end
+
+    def to_json
+      @content_finder.parsed.try(:to_json)
+    end
+
+    private
+
+    def line_correctors
+      @line_correctors ||= csv_content.collect do |line|
+        LineCorrector.new(header, Line.new(line), @start_column, @end_column)
+      end
+    end
+
+    def corrected_lines
+      line_correctors.collect do |line|
+        if line.needs_correcting?
+          line.corrected
+        else
+          line.original
+        end
+      end
+    end
+  end
+end

data/lib/comma_splice/helpers/comma_calculator.rb

@@ -0,0 +1,86 @@
+module CommaSplice
+  # provide an array of CSV headers and and array of CSV values
+  # and this will figure out the best correction and prompt
+  # you if it can't find out
+
+  class CommaCalculator
+    def initialize(headers, values)
+      @headers = headers
+      @values  = values
+
+      raise StandardError, "Determining all the possibilities to fit #{@values.size} values into the #{@headers.size} headers #{@headers.inspect} is computationally expensive. Please specify the columns where commas might be." if @headers.size > 10 && @values.size > 10
+    end
+
+    def correction
+      if @headers.size === @values.size
+        @values
+      elsif best_options.size == 1
+        best_options.first
+      elsif best_options.size > 1
+        prompt_for_options(best_options)
+      else
+        prompt_for_options(all_options)
+      end
+    end
+
+    def all_options
+      options = join_possibilities.collect do |joins|
+        values = @values.dup
+        joins.collect do |join_num|
+          val = values.shift(join_num)
+          if val.size > 1
+            quoted_values(val)
+          else
+            val.join(',')
+          end
+        end
+      end
+    end
+
+    def best_options
+      all_options.select do |option|
+        option.none? { |o| o.starts_with?(' ') || o.starts_with?('" ') }
+      end
+    end
+
+    def requires_manual_input?
+      needs_correcting? && best_options.many?
+    end
+
+    def needs_correcting?
+      @headers.size < @values.size
+    end
+
+    private
+
+    def quoted_values(values)
+      "\"#{values.join(',').gsub(/(?<!")(?:"{2})*\K\"/, '""')}\"" # escape a double quote if it hasn't been escaped already
+    end
+
+    def join_possibilities
+      JoinPossibilities.new(@values.size, @headers.size).possibilities
+    end
+
+    def prompt_for_options(options)
+      longest_header = @headers.max_by(&:length)
+
+      options.each_with_index do |option, index|
+        @headers.each_with_index do |header, i|
+          marker = i.zero? ? "(#{index + 1})" : ''
+          puts marker.ljust(5) +
+               header.ljust(longest_header.size) + ': ' +
+               option[i]
+        end
+        puts "\n"
+      end
+
+      selected_option = nil
+      until selected_option && selected_option.to_i > 0
+        puts 'which one is correct?'
+        selected_option = STDIN.gets
+      end
+
+      options[selected_option.to_i - 1]
+    end
+  end
+end

data/lib/comma_splice/helpers/content_finder.rb

@@ -0,0 +1,47 @@
+module CommaSplice
+  # Given a file this will find the CSV content. Some files have some non-csv junk at the top
+
+  class ContentFinder
+    attr_reader :start_line, :end_line, :content
+
+    def initialize(file_contents, start_line = nil, end_line = nil)
+      @file_contents = file_contents
+
+      if start_line && end_line
+        # the csvs this was built for have non-csv headers
+        @start_line  = start_line
+        @end_line    = end_line
+        @content     = @file_contents.lines[@start_line..@end_line]
+      else
+        find_content
+      end
+    end
+
+    def find_content
+      @start_line = @file_contents.lines.find_index do |line|
+        Line.new(line).values.size > 2
+      end
+
+      relative_end_line = @file_contents.lines[@start_line..-1].find_index do |line|
+        Line.new(line).values.size < 2
+      end
+
+      if relative_end_line
+        @end_line = @start_line + relative_end_line - 1
+      else
+        @end_line = -1
+      end
+
+      @content = @file_contents.lines[@start_line..@end_line]
+    end
+
+    def parsed
+      quote_chars = %w[" | ~ ^ & *]
+      begin
+        CSV.parse(@content.join('\n'), quote_char: quote_chars.shift, headers: :first_row, liberal_parsing: true)
+      rescue CSV::MalformedCSVError
+        quote_chars.empty? ? raise : retry
+      end
+    end
+  end
+end

data/lib/comma_splice/helpers/join_possibilities.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module CommaSplice
+  class JoinPossibilities
+    attr_reader :from_size, :to_size
+
+    def initialize(value_count, header_count)
+      @from_size = value_count
+      @to_size   = header_count
+    end
+
+    def possibilities
+      @possibilities ||= permutations(combos(from_size, to_size))
+    end
+
+    private
+
+    def permutations(combinations)
+      # get all permutations of those combinations
+      # to determine every possibility of join
+
+      all_permutations = combinations.collect do |combo|
+        combo.permutation(to_size).to_a
+      end
+
+      # flatten down to a list of arrays
+      all_permutations.flatten(1).uniq
+    end
+
+    def combos(desired_size, count, minimum = 1)
+      # determine all combinations of [count] numbers that add up to [desired_size]
+      # e.g if we have an array of 6 items and want an array of 4 items
+      # we need 4 numbers that add up to 6, => [[1, 1, 1, 3], [1, 1, 2, 2]]
+
+      return [] if desired_size < count || desired_size < minimum
+      return [desired_size] if count == 1
+
+      (minimum..desired_size - 1).flat_map do |i|
+        combos(desired_size - i, count - 1, i).map { |r| [i, *r] }
+      end
+    end
+  end
+end

data/lib/comma_splice/helpers/line.rb

@@ -0,0 +1,21 @@
+module CommaSplice
+  class Line
+    attr_reader :values, :line
+
+    def initialize(line)
+      @line = line
+      @values = parse_csv_content(line).first
+    end
+
+    private
+
+    def parse_csv_content(content, headers = false)
+      quote_chars = %w[" | ~ ^ & *]
+      begin
+        CSV.parse(content.mb_chars.tidy_bytes.to_s, quote_char: quote_chars.shift, headers: headers, liberal_parsing: true)
+      rescue CSV::MalformedCSVError
+        quote_chars.empty? ? raise : retry
+      end
+    end
+  end
+end

data/lib/comma_splice/helpers/variable_column_finder.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module CommaSplice
+  # Given a header line and some value lines this will try to figure out the columns
+  # where it's likely an error might be.
+
+  # Columns on the left and right bounds will be ignored if each line has the same length
+
+  # For example, the following CSV will evaluate with @start_column = 5 and @end_column = -4
+  # since in this example playid, playtime, genre, and timestamp are all non-variable on the left,
+  # and prepost, programtype, iswebcast, and isrequest are non-variable on the right
+
+  #     playid,playtype,genre,timestamp,artist,title,albumtitle,label,prepost,programtype,iswebcast,isrequest
+  #     17385098,,,01-27-2019 @ 13:58:00,Niney & Soul Syndicate,So Long Dub,Dub Box Set Vol. 2,Trojan,post,live,y,
+  #     17385097,,,01-27-2019 @ 13:57:00,King Tubby,Love Thy Neighbor,Jesus Dread,Blood & Fire,post,live,y,
+  #     17385096,,,01-27-2019 @ 13:53:00,King Tubby / The Aggrovators,Declaration Of Dub,Dub From The Roots,Charly,post,live,y,
+  #     17385095,,,01-27-2019 @ 13:50:00,Harry Mudie / King Tubby,Dub With A Difference,In Dub Conference Vol. 1,Moodisc,post,live,y,
+  #     17385094,,,01-27-2019 @ 13:47:00,KIng Tubby Meets The Upsetter,King And The Upsetter At Spanish Town,KIng Tubby Meets The Upsetter,Celluloid,post,live,y,
+
+  class VariableColumnFinder
+    attr_reader :start_column, :end_column
+
+    def initialize(header_line, value_lines)
+      @values = value_lines
+      @header = header_line
+
+      find_variable_column_boundaries
+    end
+
+    def find_variable_column_boundaries
+      # Now given both of these, we can eliminate some columns on the left and right
+      variables = left_to_right_index.zip(right_to_left_index).map do |pair|
+        pair == [false, false]
+      end
+
+      start_column = variables.find_index(true)
+      end_column = variables.reverse.find_index(true) * -1
+
+      @start_column = start_column
+      @end_column = end_column
+    end
+
+    private
+
+    def left_to_right_index
+      left_to_right_index = []
+      @header.split(',').size.times do |time|
+        left_to_right_index.push(@values.map do |value_line|
+          value_line.split(',')[time].size
+        end.uniq.size == 1)
+      end
+
+      left_to_right_index
+    end
+
+    def right_to_left_index
+      right_to_left_index = []
+      @header.split(',').size.times do |time|
+        right_to_left_index.unshift(@values.map do |value_line|
+          value_line.split(',')[-time].size
+        end.uniq.size == 1)
+      end
+
+      right_to_left_index
+    end
+  end
+end

data/lib/comma_splice/line_corrector.rb

@@ -0,0 +1,66 @@
+module CommaSplice
+  class LineCorrector
+    attr_reader :headers, :values, :header_line, :value_line, :right_bounds, :left_bounds
+
+    def initialize(header_line, value_line, left_bounds = 0, right_bounds = -1)
+      header_line = Line.new(header_line) unless header_line.is_a?(Line)
+      value_line  = Line.new(value_line) unless value_line.is_a?(Line)
+
+      @header_line = header_line
+      @value_line = value_line
+      @headers = header_line.values
+      @values = value_line.values
+      @left_bounds = left_bounds
+      @right_bounds = right_bounds
+
+      raise 'right bounds must be less than -1' unless right_bounds < 0
+      raise 'left bounds must be greater than zero' unless left_bounds >= 0
+    end
+
+    def needs_correcting?
+      @values && @values.size > 0 && @headers.size != @values.size
+    end
+
+    def original
+      @values.join(',')
+    end
+
+    def corrected
+      # you want to provide this with the smallest set of possibilities
+      # for performance reasons. Left and right bounds limit the values
+      # where the comma error could be
+
+      # For instance, with the following headers:
+      # [playid,playtype,genre,timestamp,artist,title,albumtitle,label,prepost,programtype,iswebcast,isrequest]
+      # the only values that could contain an extra comma are "artist,title,albumtitle,label"
+      # therefore our left_bounds = 4, right_bounds = -5
+
+      values_before = if left_bounds > 0
+                        values[0..(left_bounds - 1)]
+                      else
+                        []
+                      end
+
+      values_after =  if right_bounds < -1
+                        values[(right_bounds + 1)..-1]
+                      else
+                        []
+                      end
+      [values_before, corrector.correction, values_after].flatten.join(',')
+    end
+
+    private
+
+    def corrector
+      CommaCalculator.new(selected_headers, selected_values)
+    end
+
+    def selected_headers
+      headers[left_bounds..right_bounds]
+    end
+
+    def selected_values
+      values[left_bounds..right_bounds]
+    end
+  end
+end

data/lib/comma_splice/version.rb

@@ -0,0 +1,3 @@
+module CommaSplice
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,150 @@
+--- !ruby/object:Gem::Specification
+name: comma_splice
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jeff Keen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-08-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- jeff@keen.me
+executables:
+- comma_splice
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/comma_splice
+- bin/console
+- bin/setup
+- comma_splice.gemspec
+- lib/comma_splice.rb
+- lib/comma_splice/file_corrector.rb
+- lib/comma_splice/helpers/comma_calculator.rb
+- lib/comma_splice/helpers/content_finder.rb
+- lib/comma_splice/helpers/join_possibilities.rb
+- lib/comma_splice/helpers/line.rb
+- lib/comma_splice/helpers/variable_column_finder.rb
+- lib/comma_splice/line_corrector.rb
+- lib/comma_splice/version.rb
+homepage: http://github.com/jkeen/comma_splice
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.8
+signing_key: 
+specification_version: 4
+summary: Fixes CSVs with unescaped commas
+test_files: []