csv2avro 1.0.0

data/spec/csv2avro/converter_spec.rb

@@ -0,0 +1,434 @@
+require 'spec_helper'
+
+RSpec.describe CSV2Avro::Converter do
+  describe '#read' do
+    context 'schema with string and integer columns' do
+      let(:schema_io) do
+        StringIO.new(
+          {
+            name: 'categories',
+            type: 'record',
+            fields: [
+              { name: 'id', type: 'int' },
+              { name: 'name', type: 'string' },
+              { name: 'description', type: ['string', 'null'] }
+            ]
+          }.to_json
+        )
+      end
+
+      context 'separated with commas (csv)' do
+        let(:reader) do
+          StringIO.new(
+            csv_string = CSV.generate do |csv|
+              csv << %w[id name description]
+              csv << %w[1 dresses Dresses]
+              csv << %w[2 female-tops]
+            end
+            )
+        end
+
+        let(:schema) { CSV2Avro::Schema.new(schema_io) }
+
+        let(:writer) { CSV2Avro::AvroWriter.new(StringIO.new, schema) }
+
+        let(:bad_rows_writer) { StringIO.new }
+
+        let(:error_writer) { StringIO.new }
+
+        before do
+          CSV2Avro::Converter.new(reader, writer, bad_rows_writer, error_writer, {}, schema: schema).convert
+        end
+
+        it 'should not have any bad rows' do
+          expect(bad_rows_writer.read).to eq("")
+        end
+
+        it 'should not have any errors' do
+          expect(error_writer.read).to eq("")
+        end
+
+        it 'should store the data with the given schema' do
+          expect(AvroReader.new(writer).read).to eq(
+            [
+              { 'id'=>1, 'name'=>'dresses',     'description'=>'Dresses' },
+              { 'id'=>2, 'name'=>'female-tops', 'description'=>nil }
+            ]
+          )
+        end
+      end
+
+      context 'separated with tabs (tsv)' do
+        let(:reader) do
+          StringIO.new(
+            csv_string = CSV.generate({col_sep: "\t"}) do |csv|
+              csv << %w[id name description]
+              csv << %w[1 dresses Dresses]
+              csv << %w[2 female-tops]
+            end
+          )
+        end
+
+        let(:schema) { CSV2Avro::Schema.new(schema_io) }
+
+        let(:writer) { CSV2Avro::AvroWriter.new(StringIO.new, schema) }
+
+        let(:bad_rows_writer) { StringIO.new }
+
+        let(:error_writer) { StringIO.new }
+
+        before do
+          CSV2Avro::Converter.new(reader, writer, bad_rows_writer, error_writer, { delimiter: "\t" }, schema: schema).convert
+        end
+
+        it 'should not have any bad rows' do
+          expect(bad_rows_writer.read).to eq("")
+        end
+
+        it 'should not have any errors' do
+          expect(error_writer.read).to eq("")
+        end
+
+        it 'should store the data with the given schema' do
+          expect(AvroReader.new(writer).read).to eq(
+            [
+              { 'id'=>1, 'name'=>'dresses',     'description'=>'Dresses' },
+              { 'id'=>2, 'name'=>'female-tops', 'description'=>nil }
+            ]
+          )
+        end
+      end
+    end
+
+    context 'schema with boolean and array columns' do
+      let(:schema_io) do
+        StringIO.new(
+          {
+            name: 'categories',
+            type: 'record',
+            fields: [
+              { name: 'id', type: 'int' },
+              { name: 'enabled', type: ['boolean', 'null'] },
+              { name: 'image_links', type: [{ type: 'array', items: 'string' }, 'null'] }
+            ]
+          }.to_json
+        )
+      end
+
+      context 'separated with commas (default)' do
+        let(:reader) do
+          StringIO.new(
+            csv_string = CSV.generate({col_sep: "\t"}) do |csv|
+              csv << %w[id enabled image_links]
+              csv << %w[1 true http://www.images.com/dresses.jpeg]
+              csv << %w[2 false http://www.images.com/bras1.jpeg,http://www.images.com/bras2.jpeg]
+            end
+          )
+        end
+
+        let(:schema) { CSV2Avro::Schema.new(schema_io) }
+
+        let(:writer) { CSV2Avro::AvroWriter.new(StringIO.new, schema) }
+
+        let(:bad_rows_writer) { StringIO.new }
+
+        let(:error_writer) { StringIO.new }
+
+        before do
+          CSV2Avro::Converter.new(reader, writer, bad_rows_writer, error_writer, { delimiter: "\t" }, schema: schema).convert
+        end
+
+        it 'should not have any bad rows' do
+          expect(bad_rows_writer.read).to eq("")
+        end
+
+        it 'should not have any errors' do
+          expect(error_writer.read).to eq("")
+        end
+
+        it 'should store the data with the given schema' do
+          expect(AvroReader.new(writer).read).to eq(
+            [
+              { 'id'=>1, 'enabled'=>true,  'image_links'=>['http://www.images.com/dresses.jpeg'] },
+              { 'id'=>2, 'enabled'=>false, 'image_links'=>['http://www.images.com/bras1.jpeg', 'http://www.images.com/bras2.jpeg'] }
+            ]
+          )
+        end
+      end
+
+      context 'separated with semicolons' do
+        let(:reader) do
+          StringIO.new(
+            csv_string = CSV.generate({col_sep: "\t"}) do |csv|
+              csv << %w[id enabled image_links]
+              csv << %w[1 true http://www.images.com/dresses.jpeg]
+              csv << %w[2 false http://www.images.com/bras1.jpeg;http://www.images.com/bras2.jpeg]
+            end
+          )
+        end
+
+        let(:schema) { CSV2Avro::Schema.new(schema_io) }
+
+        let(:writer) { CSV2Avro::AvroWriter.new(StringIO.new, schema) }
+
+        let(:bad_rows_writer) { StringIO.new }
+
+        let(:error_writer) { StringIO.new }
+
+        before do
+          CSV2Avro::Converter.new(reader, writer, bad_rows_writer, error_writer, { delimiter: "\t", array_delimiter: ';' }, schema: schema).convert
+        end
+
+        it 'should not have any bad rows' do
+          expect(bad_rows_writer.read).to eq("")
+        end
+
+        it 'should not have any errors' do
+          expect(error_writer.read).to eq("")
+        end
+
+        it 'should store the data with the given schema' do
+          expect(AvroReader.new(writer).read).to eq(
+            [
+              { 'id'=>1, 'enabled'=>true,  'image_links'=>['http://www.images.com/dresses.jpeg'] },
+              { 'id'=>2, 'enabled'=>false, 'image_links'=>['http://www.images.com/bras1.jpeg', 'http://www.images.com/bras2.jpeg'] }
+            ]
+          )
+        end
+      end
+    end
+
+    context 'shema with default vaules' do
+      let(:schema_io) do
+        StringIO.new(
+          {
+            name: 'product',
+            type: 'record',
+            fields: [
+              { name: 'id', type: 'int' },
+              { name: 'category',  type: 'string', default: 'unknown' },
+              { name: 'size_type', type: 'string', default: 'regular' },
+              { name: 'enabled',   type: ['boolean', 'null'], default: false }
+            ]
+          }.to_json
+        )
+      end
+
+      let(:reader) do
+        StringIO.new(
+          csv_string = CSV.generate do |csv|
+            csv << %w[id category enabled]
+            csv << %w[1 dresses true]
+            csv << %w[2  ]
+          end
+        )
+      end
+
+      let(:schema) { CSV2Avro::Schema.new(schema_io) }
+
+      let(:writer) { CSV2Avro::AvroWriter.new(StringIO.new, schema) }
+
+      let(:bad_rows_writer) { StringIO.new }
+
+      let(:error_writer) { StringIO.new }
+
+      before do
+        CSV2Avro::Converter.new(reader, writer, bad_rows_writer, error_writer, { write_defaults: true }, schema: schema).convert
+      end
+
+      it 'should not have any bad rows' do
+        expect(bad_rows_writer.read).to eq("")
+      end
+
+      it 'should not have any errors' do
+        expect(error_writer.read).to eq("")
+      end
+
+      it 'should store the defaults data' do
+        expect(AvroReader.new(writer).read).to eq(
+          [
+            { 'id'=>1, 'category'=>'dresses', 'size_type'=> 'regular' ,'enabled'=>true },
+            { 'id'=>2, 'category'=>'unknown', 'size_type'=> 'regular' ,'enabled'=>false }
+          ]
+        )
+      end
+    end
+
+    context 'schema with aliased fields' do
+      let(:reader) do
+        StringIO.new(
+          csv_string = CSV.generate do |csv|
+            csv << %w[id color_id]
+            csv << %w[1 1_red]
+            csv << %w[2 2_blue]
+          end
+        )
+      end
+
+      let(:schema_io) do
+        StringIO.new(
+          {
+            name: 'product',
+            type: 'record',
+            fields: [
+              { name: 'id', type: 'int' },
+              { name: 'look_id', type: 'string', aliases: ['color_id', 'photo_group_id'] }
+            ]
+          }.to_json
+        )
+      end
+
+      let(:schema) { CSV2Avro::Schema.new(schema_io) }
+
+      let(:writer) { CSV2Avro::AvroWriter.new(StringIO.new, schema) }
+
+      let(:bad_rows_writer) { StringIO.new }
+
+      let(:error_writer) { StringIO.new }
+
+      before do
+        CSV2Avro::Converter.new(reader, writer, bad_rows_writer, error_writer, {}, schema: schema).convert
+      end
+
+      it 'should not have any bad rows' do
+        expect(bad_rows_writer.read).to eq("")
+      end
+
+      it 'should not have any errors' do
+        expect(error_writer.read).to eq("")
+      end
+
+      it 'should store the data with the given schema' do
+        expect(AvroReader.new(writer).read).to eq(
+          [
+            {'id'=>1, 'look_id'=>'1_red'},
+            {'id'=>2, 'look_id'=>'2_blue'}
+          ]
+        )
+      end
+    end
+
+    context 'schema with enum column' do
+      let(:schema_io) do
+        StringIO.new(
+          {
+            name: 'product',
+            type: 'record',
+            fields: [
+              { name: 'id', type: 'int' },
+              { name: 'size_type', type:
+                [
+                  {
+                    type:'enum', name:'size_type_values', symbols:['regular', 'petite', 'plus', 'tall', 'big_and_tall', 'maternity']
+                  }, 'null'
+                ], default: 'regular'
+              }
+            ]
+          }.to_json
+        )
+      end
+
+      let(:reader) do
+        StringIO.new(
+          csv_string = CSV.generate do |csv|
+            csv << %w[id size_type]
+            csv << %w[1 regular]
+            csv << %W[2 big\sand\stall]
+            csv << %w[3 ]
+          end
+        )
+      end
+
+      let(:schema) { CSV2Avro::Schema.new(schema_io) }
+
+      let(:writer) { CSV2Avro::AvroWriter.new(StringIO.new, schema) }
+
+      let(:bad_rows_writer) { StringIO.new }
+
+      let(:error_writer) { StringIO.new }
+
+      before do
+        CSV2Avro::Converter.new(reader, writer, bad_rows_writer, error_writer, { write_defaults: true }, schema: schema).convert
+      end
+
+      it 'should not have any bad rows' do
+        expect(bad_rows_writer.read).to eq("")
+      end
+
+      it 'should not have any errors' do
+        expect(error_writer.read).to eq("")
+      end
+
+      it 'should store the data with the given schema' do
+        expect(AvroReader.new(writer).read).to eq(
+          [
+            { 'id'=>1, 'size_type'=>'regular' },
+            { 'id'=>2, 'size_type'=>'big_and_tall' },
+            { 'id'=>3, 'size_type'=>'regular' }
+          ]
+        )
+      end
+    end
+
+    context 'data with bad rows' do
+      let(:schema_io) do
+        StringIO.new(
+          {
+            name: 'categories',
+            type: 'record',
+            fields: [
+              { name: 'id', type: 'int' },
+              { name: 'name', type: 'string', aliases: ['title'] },
+              { name: 'description', type: ['string', 'null'] }
+            ]
+          }.to_json
+        )
+      end
+
+      let(:reader) do
+        StringIO.new(
+          csv_string = CSV.generate({col_sep: "\t"}) do |csv|
+            csv << %w[id title description]
+            csv << ['1', nil, 'dresses']
+            csv << %w[2 female-tops]
+            csv << %w[3 female-bottoms]
+            csv << ['4', nil, 'female-shoes']
+          end
+          )
+      end
+
+      let(:schema) { CSV2Avro::Schema.new(schema_io) }
+
+      let(:writer) { CSV2Avro::AvroWriter.new(StringIO.new, schema) }
+
+      let(:bad_rows_writer) { StringIO.new }
+
+      let(:error_writer) { StringIO.new }
+
+      before do
+        CSV2Avro::Converter.new(reader, writer, bad_rows_writer, error_writer, { delimiter: "\t" }, schema: schema).convert
+      end
+
+      it 'should have the bad data in the original form' do
+        expect(bad_rows_writer.string).to eq(
+          "id\ttitle\tdescription\n1\t\tdresses\n4\t\tfemale-shoes\n"
+        )
+      end
+
+      it 'should have an error' do
+        expect(error_writer.string).to eq(
+          "line 2: Missing value at name\nline 5: Missing value at name\n"
+        )
+      end
+
+      it 'should store the data with the given schema' do
+        expect(AvroReader.new(writer).read).to eq(
+          [
+            { 'id'=>2, 'name'=>'female-tops', 'description'=>nil },
+            { 'id'=>3, 'name'=>'female-bottoms', 'description'=>nil }
+          ]
+        )
+      end
+    end
+  end
+end

data/spec/csv2avro/schema_spec.rb

@@ -0,0 +1,85 @@
+require 'spec_helper'
+
+RSpec.describe CSV2Avro::Schema do
+  describe '#defaults' do
+    context 'shema with default values' do
+      let(:schema_io) do
+        StringIO.new(
+          {
+            name: 'product',
+            type: 'record',
+            fields: [
+              { name: 'id', type: 'int' },
+              { name: 'category', type: 'string', default: 'unknown' },
+              { name: 'enabled', type: ['boolean', 'null'], default: false }
+            ]
+          }.to_json
+        )
+      end
+
+      subject(:schema) do
+        CSV2Avro::Schema.new(schema_io)
+      end
+
+      it 'should return a hash with the field - default value pairs' do
+        expect(schema.defaults).to eq({ 'category'=>'unknown', 'enabled'=>false })
+      end
+    end
+  end
+
+  describe '#types' do
+    context 'shema with different types' do
+      let(:schema_io) do
+        StringIO.new(
+          {
+            name: 'product',
+            type: 'record',
+            fields: [
+              { name: 'id', type: 'int' },
+              { name: 'category', type: 'string' },
+              { name: 'reviews', type: { type: 'array', items: 'string' }},
+              { name: 'enabled', type: ['boolean', 'null'] },
+              { name: 'availability', type: {
+                  type:'enum', name:'availability_values', symbols:['in_stock', 'out_of_stock', 'preorder']
+                }, default: 'in_stock'
+              }
+            ]
+          }.to_json
+        )
+      end
+
+      subject(:schema) do
+        CSV2Avro::Schema.new(schema_io)
+      end
+
+      it 'should return a hash with the field - default value pairs' do
+        expect(schema.types).to eq({ 'id'=>:int, 'category'=>:string, 'reviews'=>:array, 'enabled'=>:boolean, 'availability'=>:enum })
+      end
+    end
+  end
+
+  describe '#aliases' do
+    context 'shema with aliases' do
+      let(:schema_io) do
+        StringIO.new(
+          {
+            name: 'product',
+            type: 'record',
+            fields: [
+              { name: 'id', type: 'int' },
+              { name: 'look_id', type: 'string', aliases: ['color_id', 'photo_group_id'] }
+            ]
+          }.to_json
+        )
+      end
+
+      subject(:schema) do
+        CSV2Avro::Schema.new(schema_io)
+      end
+
+      it 'should return a hash with the alias - name mapping' do
+        expect(schema.aliases).to eq({ 'color_id'=>'look_id', 'photo_group_id'=>'look_id' })
+      end
+    end
+  end
+end

data/spec/csv2avro_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+RSpec.describe CSV2Avro do
+  describe '#convert' do
+    let(:options) do
+      {
+        schema: './spec/support/schema.avsc'
+      }
+    end
+
+    subject(:converter) do
+      ARGV.replace ['./spec/support/data.csv']
+
+      CSV2Avro.new(options)
+    end
+
+    it 'should write the problems to STDERR' do
+      expect { converter.convert }.to output("line 4: Missing value at name\n").to_stderr
+    end
+
+    it 'should have a bad row' do
+      File.open('./spec/support/data.bad.csv', 'r') do |file|
+        expect(file.read).to eq("id,name,description\n3,,Bras\n")
+      end
+    end
+
+    it 'should contain the avro data' do
+      File.open('./spec/support/data.avro', 'r') do |file|
+        expect(AvroReader.new(file).read).to eq(
+          [
+            { 'id'=>1, 'name'=>'dresses',     'description'=>'Dresses' },
+            { 'id'=>2, 'name'=>'female-tops', 'description'=>nil }
+          ]
+        )
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+require 'csv2avro'
+require 'csv2avro/converter'
+require 'csv2avro/avro_writer'
+
+require 'json'
+
+Dir[File.dirname(__FILE__) + '/support/**/*.rb'].each {|f| require f }
+
+RSpec.configure do |config|
+  config.after(:all) do
+    Dir["./spec/support/*.avro", "./spec/support/*.bad*"].each do |file|
+      File.delete(file)
+    end
+  end
+end

data/spec/support/avro_reader.rb

@@ -0,0 +1,22 @@
+class AvroReader
+  attr_reader :io
+
+  def initialize(io, schema=nil)
+    @io = io
+    @reader = if schema
+      schema_file = Avro::Schema.parse(schema)
+      Avro::IO::DatumReader.new(nil, schema_file)
+    else
+      Avro::IO::DatumReader.new
+    end
+  end
+
+  def read
+    dr = Avro::DataFile::Reader.new(io, @reader)
+
+    rows = []
+    dr.each { |record| rows << record }
+
+    rows
+  end
+end

data/spec/support/data.csv

@@ -0,0 +1,4 @@
+id,name,description
+1,dresses,Dresses
+2,female-tops,
+3,,Bras

data/spec/support/schema.avsc

@@ -0,0 +1,17 @@
+{
+  "type": "record",
+  "name": "categories",
+  "fields": [{
+      "name": "id",
+      "type": "int"
+    },
+    {
+      "name": "name",
+      "type": "string"
+    },
+    {
+      "name": "description",
+      "type": ["string", "null"]
+    }
+  ]
+}