csv2avro 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6eae97d5b2bf7476331128770ffee4d3b6d69d7a
-  data.tar.gz: 554e64338b5950de37ccaad44927176f6922f94c
+  metadata.gz: 4102328d73046f50036d1a35848142d0d23e50dc
+  data.tar.gz: d7f59e943fd02106579e9f25b6bc5480e5243b03
 SHA512:
-  metadata.gz: 3a70f269a7337d6dad0bd24528e9092b897217254610a8257db0fecc72d55dada505c68040b3a1309b3e8266473257f118a88231a54b5627c74ffb63c998d49c
-  data.tar.gz: 01cc32197d34410522aed53d4682aa9c91b20a63ad9e09a80831cc7e6af6d5bfd1a972488bb7b52f263451222a0363f10e5ab8a07e09ab38a5154b46496ff93e
+  metadata.gz: e1e6e26a916c6beb65c25395e062fa8f45037171fcfc624798fb68417f257ac761fdb6ffd560dda251bee89953be59399f3e66ee0ce612e405cf352862cd4d30
+  data.tar.gz: e94c31792b3113edb64dfc1e72cb24868af3a446469df2ada734d6ef0210992c5739ad26ba485832fcfc37204193d1bbde59155df98a18c34b644cf65aac50f6

data/.gitignore

@@ -4,7 +4,9 @@
 /_yardoc/
 /coverage/
 /doc/
+/feeds/
 /pkg/
+/schema/
 /spec/reports/
 /tmp/
 *.bundle

data/CHANGELOG.md

@@ -3,6 +3,11 @@
 All notable changes to this project are documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## 1.0.1 (2015-06-12; [compare](https://github.com/sspinc/csv2avro/compare/1.0.0...1.0.1))
+
+### Fixed
+ * CSV parsing issues
+
 ## 1.0.0 (2015-06-05; [compare](https://github.com/sspinc/csv2avro/compare/0.4.0...1.0.0))
 
 ### Added

data/README.md

@@ -8,12 +8,6 @@ Convert CSV files to Avro like a boss.
 
 or if you prefer to live on the edge, just clone this repository and build it from scratch.
 
-You can run the converter within a **Docker** container, you just need to pull the `sspinc/csv2avro` image.
-
-```
-$ docker pull sspinc/csv2avro
-```
-
 ## Usage
 
 ### Basic
@@ -24,12 +18,6 @@ This will process the data.csv file and creates a *data.avro* file and a *data.b
 
 You can override the bad-rows file location with the `--bad-rows [BAD_ROWS]` option.
 
-### CSV2Avro in Docker
-
-```
-$ docker run sspinc/csv2avro --help
-```
-
 ### Streaming
 ```
 $ cat ./spec/support/data.csv | csv2avro --schema ./spec/support/schema.avsc --bad-rows ./spec/support/data.bad.csv > ./spec/support/data.avro
@@ -59,7 +47,7 @@ This will uncompress the file and converts it to avro, leaving the original file
 For a full list of available options, run `csv2avro --help`
 ```
 $ csv2avro --help
-Version 1.0.0 of CSV2Avro
+Version 1.0.1 of CSV2Avro
 Usage: csv2avro [options] [file]
     -s, --schema SCHEMA              A file containing the Avro schema. This value is required.
     -b, --bad-rows [BAD_ROWS]        The output location of the bad rows file.

data/Rakefile

@@ -16,26 +16,3 @@ RSpec::Core::RakeTask.new(:spec) do |task|
 end
 
 task :default => :spec
-
-namespace :docker do
-  desc "Build docker image"
-  task :build do
-    sh "docker build -t sspinc/csv2avro:#{CSV2Avro::VERSION} ."
-    minor_version = CSV2Avro::VERSION.sub(/\.[0-9]+$/, '')
-    sh "docker tag -f sspinc/csv2avro:#{CSV2Avro::VERSION} sspinc/csv2avro:#{minor_version}"
-    major_version = minor_version.sub(/\.[0-9]+$/, '')
-    sh "docker tag -f sspinc/csv2avro:#{CSV2Avro::VERSION} sspinc/csv2avro:#{major_version}"
-
-    sh "docker tag -f sspinc/csv2avro:#{CSV2Avro::VERSION} sspinc/csv2avro:latest"
-  end
-
-  desc "Run specs inside docker image"
-  task :spec => :build do
-    sh "docker run -t --entrypoint=rake sspinc/csv2avro:#{CSV2Avro::VERSION} spec"
-  end
-
-  desc "Push docker image"
-  task :push => :spec do
-    sh "docker push sspinc/csv2avro"
-  end
-end

data/csv2avro.gemspec

@@ -24,5 +24,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry", "~> 0.10"
   spec.add_development_dependency "bump", "~> 0.5"
 
-  spec.add_dependency "avro", "~> 1.7"
+  spec.add_runtime_dependency "avro", "~> 1.7"
 end

data/lib/avro_schema.rb

@@ -47,7 +47,7 @@ module Avro
         if datum.nil? && expected_type != :null
           @errors << "Missing value at #{name}"
         else
-          @errors << "'#{datum}' at #{name} does'n match the type '#{expected_schema.to_s}'"
+          @errors << "'#{datum}' at #{name} doesn't match the type '#{expected_schema.to_s}'"
         end
       end
 

data/lib/csv2avro/avro_writer.rb

@@ -17,11 +17,11 @@ class CSV2Avro
     end
 
     def writer_schema
-      avro_writer.datum_writer.writers_schema
+      @avro_writer.datum_writer.writers_schema
     end
 
     def write(hash)
-      avro_writer << hash
+      @avro_writer << hash
     end
   end
 end

data/lib/csv2avro/converter.rb

@@ -4,122 +4,120 @@ require 'csv'
 
 class CSV2Avro
   class Converter
-    attr_reader :writer, :bad_rows_writer, :error_writer, :schema, :reader, :csv_options, :converter_options, :header_row, :column_separator
-
     def initialize(reader, writer, bad_rows_writer, error_writer, options, schema: schema)
+      @reader = reader
       @writer = writer
       @bad_rows_writer = bad_rows_writer
       @error_writer = error_writer
+      @options = options
       @schema = schema
 
-      @column_separator = options[:delimiter] || ','
-
-      @reader = reader
-      @header_row = reader.readline.strip
-      header = header_row.split(column_separator)
-
-      init_header_converter
-      @csv_options = {
-        headers: header,
-        skip_blanks: true,
-        col_sep: column_separator,
-        header_converters: :aliases
-      }
-
-      @converter_options = options
+      # read header row explicitly
+      @header = @reader.readline.strip.split(col_sep)
     end
 
     def convert
-      defaults = schema.defaults if converter_options[:write_defaults]
+      csv.each do |row|
+        hash = row.to_hash
 
-      fields_to_convert = schema.types.reject{ |key, value| value == :string }
+        add_defaults_to_hash!(hash) if @options[:write_defaults]
+        convert_fields!(hash)
 
-      reader.each do |line|
-        CSV.parse(line, csv_options) do |row|
-          row = row.to_hash
+        begin
+          @writer.write(hash)
+        rescue Avro::IO::AvroTypeError
+          bad_rows_csv << row
 
-          if converter_options[:write_defaults]
-            add_defaults_to_row!(row, defaults)
+          until Avro::Schema.errors.empty? do
+            @error_writer.puts("line #{line_number}: #{Avro::Schema.errors.shift}")
           end
+        end
+      end
 
-          convert_fields!(row, fields_to_convert)
+      @writer.flush
+    rescue CSV::MalformedCSVError
+      @error_writer.puts("line #{line_number}: Unable to parse")
+    end
 
-          begin
-            writer.write(row)
-            writer.flush
-          rescue
-            if bad_rows_writer.size == 0
-              bad_rows_writer << header_row + "\n"
-            end
+    private
 
-            bad_rows_writer << line
-            bad_rows_writer.flush
+    def array_delimiter
+      @options[:array_delimiter] || ','
+    end
 
-            until Avro::Schema.errors.empty? do
-              error_writer << "line #{reader.lineno}: #{Avro::Schema.errors.shift}\n"
-            end
-          end
-        end
-      end
+    def col_sep
+      @options[:delimiter] || ','
     end
 
-    private
+    def csv_options
+      {
+        col_sep: col_sep,
+        headers: @header,
+        header_converters: :aliases,
+        skip_blanks: true,
+        write_headers: true
+      }
+    end
 
-    def convert_fields!(row, fields_to_convert)
-      fields_to_convert.each do |key, value|
-        row[key] = begin
-          case value
-            when :int
-              Integer(row[key])
-            when :float, :double
-              Float(row[key])
-            when :boolean
-              parse_boolean(row[key])
-            when :array
-              parse_array(row[key])
-            when :enum
-              row[key].downcase.tr(" ", "_")
-          end
-        rescue
-          row[key]
-        end
+    def csv
+      # Initialize header converter
+      CSV::HeaderConverters[:aliases] = lambda do |header|
+        @schema.aliases[header] || header
       end
 
-      row
+      @csv ||= CSV.new(@reader, csv_options)
     end
 
-    def parse_boolean(value)
-      return true  if value == true  || value =~ (/^(true|t|yes|y|1)$/i)
-      return false if value == false || value =~ (/^(false|f|no|n|0)$/i)
-      nil
+    def bad_rows_csv
+      options = csv_options.tap { |hash| hash.delete(:header_converters) }
+      @bad_rows_csv ||= CSV.new(@bad_rows_writer, options)
     end
 
-    def parse_array(value)
-      delimiter = converter_options[:array_delimiter] || ','
-
-      value.split(delimiter) if value
+    def line_number
+      @reader.lineno + 1
     end
 
-    def add_defaults_to_row!(row, defaults)
-      # Add default values to nil cells
-      row.each do |key, value|
-        row[key] = defaults[key] if value.nil?
+    def add_defaults_to_hash!(hash)
+      # Add default values to empty/missing fields
+      @schema.defaults.each  do |key, value|
+        hash[key] = @schema.defaults[key] if hash[key].nil? or !hash.has_key?(key)
       end
+    end
 
-      # Add default values to missing columns
-      defaults.each  do |key, value|
-        row[key] = defaults[key]  unless row.has_key?(key)
+    def convert_fields!(hash)
+      @schema.types.each do |key, value|
+        hash[key] = begin
+                      case value
+                      when :int
+                        Integer(hash[key])
+                      when :float, :double
+                        Float(hash[key])
+                      when :boolean
+                        parse_boolean(hash[key])
+                      when :array
+                        parse_array(hash[key])
+                      when :enum
+                        hash[key].downcase.tr(" ", "_")
+                      else
+                        hash[key]
+                      end
+                    rescue
+                      hash[key]
+                    end
       end
-
-      row
     end
 
-    def init_header_converter
-      aliases = schema.aliases
-
-      CSV::HeaderConverters[:aliases] = lambda do |header|
-          aliases[header] || header
+    def parse_boolean(value)
+      case
+      when value == true  || value =~ (/^(true|t|yes|y|1)$/i) then true
+      when value == false || value =~ (/^(false|f|no|n|0)$/i) then false
+      else
+        nil
       end
     end
+
+    def parse_array(value)
+      value.split(array_delimiter) if value
+    end
   end
 end

data/lib/csv2avro/version.rb

@@ -1,3 +1,3 @@
 class CSV2Avro
-  VERSION = "1.0.0"
+  VERSION = "1.0.1"
 end

data/lib/csv2avro.rb

@@ -17,12 +17,7 @@ class CSV2Avro
     Converter.new(reader, writer, bad_rows_writer, error_writer, options, schema: schema).convert
   ensure
     writer.close if writer
-
-    if bad_rows_writer.size == 0
-      File.delete(bad_rows_uri)
-    elsif bad_rows_writer
-      bad_rows_writer.close
-    end
+    bad_rows_writer.close
   end
 
   private
@@ -34,7 +29,7 @@ class CSV2Avro
   end
 
   def reader
-    ARGF.lineno = 0
+    ARGF.lineno = -1
     ARGF
   end
 

data/spec/csv2avro/converter_spec.rb

@@ -1,9 +1,15 @@
 require 'spec_helper'
 
 RSpec.describe CSV2Avro::Converter do
-  describe '#read' do
+  describe '#convert' do
+    let(:schema) { CSV2Avro::Schema.new(schema_reader) }
+    let(:writer) { StringIO.new }
+    let(:avro_writer) { CSV2Avro::AvroWriter.new(writer, schema) }
+    let(:bad_rows_writer) { StringIO.new }
+    let(:error_writer) { StringIO.new }
+
     context 'schema with string and integer columns' do
-      let(:schema_io) do
+      let(:schema_reader) do
         StringIO.new(
           {
             name: 'categories',
@@ -20,7 +26,7 @@ RSpec.describe CSV2Avro::Converter do
       context 'separated with commas (csv)' do
         let(:reader) do
           StringIO.new(
-            csv_string = CSV.generate do |csv|
+            CSV.generate do |csv|
               csv << %w[id name description]
               csv << %w[1 dresses Dresses]
               csv << %w[2 female-tops]
@@ -28,28 +34,20 @@ RSpec.describe CSV2Avro::Converter do
             )
         end
 
-        let(:schema) { CSV2Avro::Schema.new(schema_io) }
-
-        let(:writer) { CSV2Avro::AvroWriter.new(StringIO.new, schema) }
-
-        let(:bad_rows_writer) { StringIO.new }
-
-        let(:error_writer) { StringIO.new }
-
         before do
-          CSV2Avro::Converter.new(reader, writer, bad_rows_writer, error_writer, {}, schema: schema).convert
+          CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, error_writer, {}, schema: schema).convert
         end
 
         it 'should not have any bad rows' do
-          expect(bad_rows_writer.read).to eq("")
+          expect(bad_rows_writer.read).to be_empty
         end
 
         it 'should not have any errors' do
-          expect(error_writer.read).to eq("")
+          expect(error_writer.read).to be_empty
         end
 
         it 'should store the data with the given schema' do
-          expect(AvroReader.new(writer).read).to eq(
+          expect(AvroReader.new(avro_writer).read).to eq(
             [
               { 'id'=>1, 'name'=>'dresses',     'description'=>'Dresses' },
               { 'id'=>2, 'name'=>'female-tops', 'description'=>nil }
@@ -61,7 +59,7 @@ RSpec.describe CSV2Avro::Converter do
       context 'separated with tabs (tsv)' do
         let(:reader) do
           StringIO.new(
-            csv_string = CSV.generate({col_sep: "\t"}) do |csv|
+            CSV.generate({col_sep: "\t"}) do |csv|
               csv << %w[id name description]
               csv << %w[1 dresses Dresses]
               csv << %w[2 female-tops]
@@ -69,24 +67,16 @@ RSpec.describe CSV2Avro::Converter do
           )
         end
 
-        let(:schema) { CSV2Avro::Schema.new(schema_io) }
-
-        let(:writer) { CSV2Avro::AvroWriter.new(StringIO.new, schema) }
-
-        let(:bad_rows_writer) { StringIO.new }
-
-        let(:error_writer) { StringIO.new }
-
         before do
-          CSV2Avro::Converter.new(reader, writer, bad_rows_writer, error_writer, { delimiter: "\t" }, schema: schema).convert
+          CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, error_writer, { delimiter: "\t" }, schema: schema).convert
         end
 
         it 'should not have any bad rows' do
-          expect(bad_rows_writer.read).to eq("")
+          expect(bad_rows_writer.read).to be_empty
         end
 
         it 'should not have any errors' do
-          expect(error_writer.read).to eq("")
+          expect(error_writer.read).to be_empty
         end
 
         it 'should store the data with the given schema' do
@@ -101,7 +91,7 @@ RSpec.describe CSV2Avro::Converter do
     end
 
     context 'schema with boolean and array columns' do
-      let(:schema_io) do
+      let(:schema_reader) do
         StringIO.new(
           {
             name: 'categories',
@@ -118,7 +108,7 @@ RSpec.describe CSV2Avro::Converter do
       context 'separated with commas (default)' do
         let(:reader) do
           StringIO.new(
-            csv_string = CSV.generate({col_sep: "\t"}) do |csv|
+            CSV.generate do |csv|
               csv << %w[id enabled image_links]
               csv << %w[1 true http://www.images.com/dresses.jpeg]
               csv << %w[2 false http://www.images.com/bras1.jpeg,http://www.images.com/bras2.jpeg]
@@ -126,24 +116,16 @@ RSpec.describe CSV2Avro::Converter do
           )
         end
 
-        let(:schema) { CSV2Avro::Schema.new(schema_io) }
-
-        let(:writer) { CSV2Avro::AvroWriter.new(StringIO.new, schema) }
-
-        let(:bad_rows_writer) { StringIO.new }
-
-        let(:error_writer) { StringIO.new }
-
         before do
-          CSV2Avro::Converter.new(reader, writer, bad_rows_writer, error_writer, { delimiter: "\t" }, schema: schema).convert
+          CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, error_writer, {}, schema: schema).convert
         end
 
         it 'should not have any bad rows' do
-          expect(bad_rows_writer.read).to eq("")
+          expect(bad_rows_writer.read).to be_empty
         end
 
         it 'should not have any errors' do
-          expect(error_writer.read).to eq("")
+          expect(error_writer.read).to be_empty
         end
 
         it 'should store the data with the given schema' do
@@ -159,7 +141,7 @@ RSpec.describe CSV2Avro::Converter do
       context 'separated with semicolons' do
         let(:reader) do
           StringIO.new(
-            csv_string = CSV.generate({col_sep: "\t"}) do |csv|
+            CSV.generate({col_sep: "\t"}) do |csv|
               csv << %w[id enabled image_links]
               csv << %w[1 true http://www.images.com/dresses.jpeg]
               csv << %w[2 false http://www.images.com/bras1.jpeg;http://www.images.com/bras2.jpeg]
@@ -167,24 +149,16 @@ RSpec.describe CSV2Avro::Converter do
           )
         end
 
-        let(:schema) { CSV2Avro::Schema.new(schema_io) }
-
-        let(:writer) { CSV2Avro::AvroWriter.new(StringIO.new, schema) }
-
-        let(:bad_rows_writer) { StringIO.new }
-
-        let(:error_writer) { StringIO.new }
-
         before do
-          CSV2Avro::Converter.new(reader, writer, bad_rows_writer, error_writer, { delimiter: "\t", array_delimiter: ';' }, schema: schema).convert
+          CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, error_writer, { delimiter: "\t", array_delimiter: ';' }, schema: schema).convert
         end
 
         it 'should not have any bad rows' do
-          expect(bad_rows_writer.read).to eq("")
+          expect(bad_rows_writer.read).to be_empty
         end
 
         it 'should not have any errors' do
-          expect(error_writer.read).to eq("")
+          expect(error_writer.read).to be_empty
         end
 
         it 'should store the data with the given schema' do
@@ -198,8 +172,8 @@ RSpec.describe CSV2Avro::Converter do
       end
     end
 
-    context 'shema with default vaules' do
-      let(:schema_io) do
+    context 'schema with default vaules' do
+      let(:schema_reader) do
         StringIO.new(
           {
             name: 'product',
@@ -216,7 +190,7 @@ RSpec.describe CSV2Avro::Converter do
 
       let(:reader) do
         StringIO.new(
-          csv_string = CSV.generate do |csv|
+          CSV.generate do |csv|
             csv << %w[id category enabled]
             csv << %w[1 dresses true]
             csv << %w[2  ]
@@ -224,24 +198,16 @@ RSpec.describe CSV2Avro::Converter do
         )
       end
 
-      let(:schema) { CSV2Avro::Schema.new(schema_io) }
-
-      let(:writer) { CSV2Avro::AvroWriter.new(StringIO.new, schema) }
-
-      let(:bad_rows_writer) { StringIO.new }
-
-      let(:error_writer) { StringIO.new }
-
       before do
-        CSV2Avro::Converter.new(reader, writer, bad_rows_writer, error_writer, { write_defaults: true }, schema: schema).convert
+        CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, error_writer, { write_defaults: true }, schema: schema).convert
       end
 
       it 'should not have any bad rows' do
-        expect(bad_rows_writer.read).to eq("")
+        expect(bad_rows_writer.read).to be_empty
       end
 
       it 'should not have any errors' do
-        expect(error_writer.read).to eq("")
+        expect(error_writer.read).to be_empty
       end
 
       it 'should store the defaults data' do
@@ -257,7 +223,7 @@ RSpec.describe CSV2Avro::Converter do
     context 'schema with aliased fields' do
       let(:reader) do
         StringIO.new(
-          csv_string = CSV.generate do |csv|
+          CSV.generate do |csv|
             csv << %w[id color_id]
             csv << %w[1 1_red]
             csv << %w[2 2_blue]
@@ -265,7 +231,7 @@ RSpec.describe CSV2Avro::Converter do
         )
       end
 
-      let(:schema_io) do
+      let(:schema_reader) do
         StringIO.new(
           {
             name: 'product',
@@ -278,24 +244,16 @@ RSpec.describe CSV2Avro::Converter do
         )
       end
 
-      let(:schema) { CSV2Avro::Schema.new(schema_io) }
-
-      let(:writer) { CSV2Avro::AvroWriter.new(StringIO.new, schema) }
-
-      let(:bad_rows_writer) { StringIO.new }
-
-      let(:error_writer) { StringIO.new }
-
       before do
-        CSV2Avro::Converter.new(reader, writer, bad_rows_writer, error_writer, {}, schema: schema).convert
+        CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, error_writer, {}, schema: schema).convert
       end
 
       it 'should not have any bad rows' do
-        expect(bad_rows_writer.read).to eq("")
+        expect(bad_rows_writer.read).to be_empty
       end
 
       it 'should not have any errors' do
-        expect(error_writer.read).to eq("")
+        expect(error_writer.read).to be_empty
       end
 
       it 'should store the data with the given schema' do
@@ -309,7 +267,7 @@ RSpec.describe CSV2Avro::Converter do
     end
 
     context 'schema with enum column' do
-      let(:schema_io) do
+      let(:schema_reader) do
         StringIO.new(
           {
             name: 'product',
@@ -330,7 +288,7 @@ RSpec.describe CSV2Avro::Converter do
 
       let(:reader) do
         StringIO.new(
-          csv_string = CSV.generate do |csv|
+          CSV.generate do |csv|
             csv << %w[id size_type]
             csv << %w[1 regular]
             csv << %W[2 big\sand\stall]
@@ -339,24 +297,16 @@ RSpec.describe CSV2Avro::Converter do
         )
       end
 
-      let(:schema) { CSV2Avro::Schema.new(schema_io) }
-
-      let(:writer) { CSV2Avro::AvroWriter.new(StringIO.new, schema) }
-
-      let(:bad_rows_writer) { StringIO.new }
-
-      let(:error_writer) { StringIO.new }
-
       before do
-        CSV2Avro::Converter.new(reader, writer, bad_rows_writer, error_writer, { write_defaults: true }, schema: schema).convert
+        CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, error_writer, { write_defaults: true }, schema: schema).convert
       end
 
       it 'should not have any bad rows' do
-        expect(bad_rows_writer.read).to eq("")
+        expect(bad_rows_writer.read).to be_empty
       end
 
       it 'should not have any errors' do
-        expect(error_writer.read).to eq("")
+        expect(error_writer.read).to be_empty
       end
 
       it 'should store the data with the given schema' do
@@ -371,7 +321,7 @@ RSpec.describe CSV2Avro::Converter do
     end
 
     context 'data with bad rows' do
-      let(:schema_io) do
+      let(:schema_reader) do
         StringIO.new(
           {
             name: 'categories',
@@ -387,7 +337,7 @@ RSpec.describe CSV2Avro::Converter do
 
       let(:reader) do
         StringIO.new(
-          csv_string = CSV.generate({col_sep: "\t"}) do |csv|
+          CSV.generate({col_sep: "\t"}) do |csv|
             csv << %w[id title description]
             csv << ['1', nil, 'dresses']
             csv << %w[2 female-tops]
@@ -397,16 +347,8 @@ RSpec.describe CSV2Avro::Converter do
           )
       end
 
-      let(:schema) { CSV2Avro::Schema.new(schema_io) }
-
-      let(:writer) { CSV2Avro::AvroWriter.new(StringIO.new, schema) }
-
-      let(:bad_rows_writer) { StringIO.new }
-
-      let(:error_writer) { StringIO.new }
-
       before do
-        CSV2Avro::Converter.new(reader, writer, bad_rows_writer, error_writer, { delimiter: "\t" }, schema: schema).convert
+        CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, error_writer, { delimiter: "\t" }, schema: schema).convert
       end
 
       it 'should have the bad data in the original form' do

data/spec/csv2avro_spec.rb

@@ -2,20 +2,15 @@ require 'spec_helper'
 
 RSpec.describe CSV2Avro do
   describe '#convert' do
-    let(:options) do
-      {
-        schema: './spec/support/schema.avsc'
-      }
-    end
+    let(:options) { { schema: './spec/support/schema.avsc' } }
 
-    subject(:converter) do
+    before do
       ARGV.replace ['./spec/support/data.csv']
-
-      CSV2Avro.new(options)
     end
+    subject(:converter) { CSV2Avro.new(options) }
 
-    it 'should write the problems to STDERR' do
-      expect { converter.convert }.to output("line 4: Missing value at name\n").to_stderr
+    it 'should write errors to STDERR' do
+      expect { converter.convert }.to output("line 4: Missing value at name\nline 5: Unable to parse\n").to_stderr
     end
 
     it 'should have a bad row' do

data/spec/support/data.csv

@@ -1,4 +1,5 @@
 id,name,description
-1,dresses,Dresses
+1,dresses,"Dresses"
 2,female-tops,
-3,,Bras
+3,,"Bras"
+4,male-tops,"Male Tops""

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csv2avro
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Peter Ableda
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-11 00:00:00.000000000 Z
+date: 2015-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -102,11 +102,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".dockerignore"
 - ".gitignore"
 - ".travis.yml"
 - CHANGELOG.md
-- Dockerfile
 - Gemfile
 - LICENSE.txt
 - README.md

data/.dockerignore

@@ -1 +0,0 @@
-.git

data/Dockerfile

@@ -1,23 +0,0 @@
-FROM ruby:2.1
-MAINTAINER Secret Sauce Partners, Inc. <dev@sspinc.io>
-
-RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
-    python2.7 get-pip.py && \
-    pip install awscli
-
-# throw errors if Gemfile has been modified since Gemfile.lock
-RUN bundle config --global frozen 1
-
-RUN mkdir -p /srv/csv2avro
-WORKDIR /srv/csv2avro
-
-RUN mkdir -p /srv/csv2avro/lib/csv2avro
-
-COPY lib/csv2avro/version.rb /srv/csv2avro/lib/csv2avro/version.rb
-COPY csv2avro.gemspec Gemfile Gemfile.lock /srv/csv2avro/
-
-RUN bundle install
-
-COPY . /srv/csv2avro
-
-ENTRYPOINT ["./bin/csv2avro"]