csv2avro 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a5d99afbda08e7b21d3045731ad738ae3d5c129
-  data.tar.gz: 9acd1c36181e07032710143a1cf6e0d43d956153
+  metadata.gz: 9ca0f6dd6e1c7b69c3318b727e1b61b98adf9d6b
+  data.tar.gz: 50dc0ace77d0c56ac1cc048b8f4e778ecd76fd43
 SHA512:
-  metadata.gz: f2bc2091b05c25a1fc4272d5b90386e24ca0e4dd51d0a0c0304a15c0c2db1cc799fcfd30b8100985023fb9d93e61ca1f28fef7911bbac0fc7c98cb57d0ea92f4
-  data.tar.gz: e53e649a5fba8a62e29d60bf84701b0084a360395af377985e5c18dad584d4610101eded0bb90cde16ac7ff6eeadf87fb932c14e939f40f2e98318d93555eee7
+  metadata.gz: ad0c57ddf4f596c74e2f545aac86f7e663a11d6d5ac9ac85dc5246cb45465237c8d33068ec175e9f7dec3809c78bb52fc76b6a26935c722585f7023aed1a47a6
+  data.tar.gz: a9e8c252d834cfd0d48f4739f21243eba47823f870fefddb3e5092bc88df113ae4dc79d88865410c1df93fbd8f5d9d292f6d7f35417e83f16f5dd283b6340aff

data/CHANGELOG.md

@@ -3,6 +3,12 @@
 All notable changes to this project are documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## 1.3.0 (2016-02-16) [compare](https://github.com/sspinc/csv2avro/compare/1.2.0...1.3.0))
+Support for custom line endings
+
+### Added
+* New line ending command line parameter
+
 ## 1.2.0 (2015-11-18) [compare](https://github.com/sspinc/csv2avro/compare/1.1.0...1.2.0))
 Structured logging and metrics
 

data/README.md

@@ -2,6 +2,8 @@
 
 Convert CSV files to Avro like a boss.
 
+![CircleCI](https://circleci.com/gh/sspinc/csv2avro.svg?style=shield&circle-token=:circle-token)
+
 ## Installation
 
     $ gem install csv2avro
@@ -47,11 +49,12 @@ This will uncompress the file and converts it to avro, leaving the original file
 For a full list of available options, run `csv2avro --help`
 ```
 $ csv2avro --help
-Version 1.0.1 of CSV2Avro
+Version 1.3.0 of CSV2Avro
 Usage: csv2avro [options] [file]
     -s, --schema SCHEMA              A file containing the Avro schema. This value is required.
     -b, --bad-rows [BAD_ROWS]        The output location of the bad rows report file.
     -d, --delimiter [DELIMITER]      Field delimiter. If none specified, then comma is used as the delimiter.
+    -l, --line-ending [LINE_ENDING]  Line ending character used as row separator in CSV parsing
     -a [ARRAY_DELIMITER],            Array field delimiter. If none specified, then comma is used as the delimiter.
         --array-delimiter
     -D, --write-defaults             Write default values.

data/bin/csv2avro

@@ -22,6 +22,10 @@ option_parser = OptionParser.new do |opts|
     options[:delimiter] = char.gsub("\\t", "\t")
   end
 
+  opts.on('-l', '--line-ending [LINE_ENDING]', 'Line ending character used as row separator in CSV parsing') do |char|
+    options[:line_ending] = char.gsub("\\r", "\r").gsub("\\n", "\n")
+  end
+
   opts.on('-a', '--array-delimiter [ARRAY_DELIMITER]', 'Array field delimiter. If none specified, then comma is used as the delimiter.') do |char|
     options[:array_delimiter] = char
   end

data/circle.yml

@@ -0,0 +1,7 @@
+machine:
+  ruby:
+    version: 2.2.2
+
+dependencies:
+  pre:
+    - gem install bundler --version 1.10

data/csv2avro.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.2"
   spec.add_development_dependency "pry", "~> 0.10"

data/lib/csv2avro/converter.rb

@@ -67,9 +67,14 @@ class CSV2Avro
       @options[:delimiter] || ','
     end
 
+    def row_sep
+      @options[:line_ending] || :auto
+    end
+
     def csv_options
       {
         col_sep: col_sep,
+        row_sep: row_sep,
         headers: @header,
         header_converters: :aliases,
         skip_blanks: true,
@@ -82,7 +87,6 @@ class CSV2Avro
       CSV::HeaderConverters[:aliases] = lambda do |header|
         @schema.aliases[header] || header
       end
-
       @csv ||= CSV.new(@reader, csv_options)
     end
 

data/lib/csv2avro/version.rb

@@ -1,3 +1,3 @@
 class CSV2Avro
-  VERSION = "1.2.0"
+  VERSION = "1.3.0"
 end

data/spec/csv2avro_spec.rb

@@ -59,5 +59,36 @@ RSpec.describe CSV2Avro do
         end
       end
     end
+
+    context "CRLF line endings" do
+      let(:options) { { schema: './spec/support/schema.avsc',
+                        line_ending: "\r\n" } }
+      subject(:converter) do
+        CSV2Avro.new(options)
+      end
+
+      before do
+        ARGV.replace ['./spec/support/data_crlf.csv']
+        converter.convert
+      end
+
+      it 'should have a bad row' do
+        File.open('./spec/support/data_crlf.bad', 'r') do |file|
+          expect(file.read).to eq("L4: Missing value at name\n")
+        end
+      end
+
+      it 'should contain the avro data' do
+        File.open('./spec/support/data_crlf.avro', 'r') do |file|
+          expect(AvroReader.new(file).read).to eq(
+            [
+              { 'id'=>1, 'name'=>'dresses',     'description'=>'Dresses' },
+              { 'id'=>2, 'name'=>'female-tops', 'description'=>nil },
+            ]
+          )
+        end
+      end
+    end
+
   end
 end

data/spec/support/data_crlf.csv

@@ -0,0 +1,4 @@
+id,name,description
+1,dresses,Dresses
+"2","female-tops",
+"3",,"Bras"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: csv2avro
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Peter Ableda
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-19 00:00:00.000000000 Z
+date: 2016-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.10'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -126,6 +126,7 @@ files:
 - README.md
 - Rakefile
 - bin/csv2avro
+- circle.yml
 - csv2avro.gemspec
 - lib/csv2avro.rb
 - lib/csv2avro/avro_writer.rb
@@ -140,6 +141,7 @@ files:
 - spec/spec_helper.rb
 - spec/support/avro_reader.rb
 - spec/support/data.csv
+- spec/support/data_crlf.csv
 - spec/support/data_quoted.csv
 - spec/support/schema.avsc
 homepage: ''
@@ -173,6 +175,7 @@ test_files:
 - spec/spec_helper.rb
 - spec/support/avro_reader.rb
 - spec/support/data.csv
+- spec/support/data_crlf.csv
 - spec/support/data_quoted.csv
 - spec/support/schema.avsc
 has_rdoc: