csv2avro 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6e7e9a8d86d5cd8e85b957ffecb5e60ccfe9c8b5
-  data.tar.gz: 2b12a6828c601dfe19e6d93bc39b481ec1e15118
+  metadata.gz: 8a5d99afbda08e7b21d3045731ad738ae3d5c129
+  data.tar.gz: 9acd1c36181e07032710143a1cf6e0d43d956153
 SHA512:
-  metadata.gz: 1caef810f21aa9f9b8dd1562c253a967f5b1c94382296f677dd6703624ac51d3d13774457c76820bf52c4a88795829d28a1b46017384ab306abf9f62bb50a078
-  data.tar.gz: cf9f67c9316d2840f883a36082a30187ae8aebbe24db71a4100496ea833b20568b9ec16f4e92258b38705b7334f07b78b52766db78245cc81c2a32e3c6244d95
+  metadata.gz: f2bc2091b05c25a1fc4272d5b90386e24ca0e4dd51d0a0c0304a15c0c2db1cc799fcfd30b8100985023fb9d93e61ca1f28fef7911bbac0fc7c98cb57d0ea92f4
+  data.tar.gz: e53e649a5fba8a62e29d60bf84701b0084a360395af377985e5c18dad584d4610101eded0bb90cde16ac7ff6eeadf87fb932c14e939f40f2e98318d93555eee7

data/CHANGELOG.md

@@ -3,6 +3,16 @@
 All notable changes to this project are documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## 1.2.0 (2015-11-18) [compare](https://github.com/sspinc/csv2avro/compare/1.1.0...1.2.0))
+Structured logging and metrics
+
+### Changed
+* Log in JSON format using Logr (https://github.com/sspinc/logr)
+
+### Added
+* New started_converting and finished_converting events
+* New lines_processed metric
+
 ## 1.1.0 (2015-09-16) [compare](https://github.com/sspinc/csv2avro/compare/1.0.2...1.1.0))
 
 ### Changed

data/bin/csv2avro

@@ -48,10 +48,12 @@ begin
 
   CSV2Avro.new(options).convert
 rescue OptionParser::MissingArgument => ex
+  CSV2Avro.logger.fatal(ex.message)
   $stderr.puts ex.message
   $stderr.puts option_parser
   exit 2
 rescue Exception => e
+  CSV2Avro.logger.fatal("processing failed: #{e.message}")
   $stderr.puts 'Uh oh, something went wrong!'
   $stderr.puts e.message
   $stderr.puts e.backtrace.join("\n")

data/csv2avro.gemspec

@@ -25,4 +25,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bump", "~> 0.5"
 
   spec.add_runtime_dependency "avro", "~> 1.7"
+  spec.add_runtime_dependency "logr", "~> 0.1"
 end

data/lib/csv2avro/converter.rb

@@ -1,14 +1,24 @@
 require 'csv2avro/schema'
 require 'csv2avro/avro_writer'
 require 'csv'
+require 'logr'
 
 class CSV2Avro
   class Converter
-    def initialize(reader, writer, bad_rows_writer, error_writer, options, schema: schema)
+
+    def self.logger
+      @logger ||= Logr::Logger.new('csv2avro.converter')
+    end
+
+    def logger
+      self.class.logger
+    end
+
+    def initialize(reader, writer, bad_rows_writer, filename, options, schema: schema)
       @reader = reader
       @writer = writer
       @bad_rows_writer = bad_rows_writer
-      @error_writer = error_writer
+      @filename = filename
       @options = options
       @schema = schema
 
@@ -22,7 +32,8 @@ class CSV2Avro
           row = csv.shift
         rescue CSV::MalformedCSVError
           error_msg = "L#{row_number}: Unable to parse"
-          @error_writer.puts(error_msg)
+          logger.event('parse_error', filename: @filename, line: row_number)
+                .error(error_msg)
           @bad_rows_writer.puts(error_msg)
           next
         end
@@ -35,11 +46,15 @@ class CSV2Avro
           @writer.write(hash)
         rescue CSV2Avro::SchemaValidationError => e
           error_msg = "L#{row_number}: #{e.errors.join(', ')}"
-          @error_writer.puts(error_msg)
+          e.errors.each do |error|
+            logger.event('schema_violation', filename: @filename, line: row_number, cause: error)
+                  .error(error_msg)
+          end
           @bad_rows_writer.puts(error_msg)
         end
       end
       @writer.flush
+      row_number
     end
 
     private

data/lib/csv2avro/version.rb

@@ -1,3 +1,3 @@
 class CSV2Avro
-  VERSION = "1.1.0"
+  VERSION = "1.2.0"
 end

data/lib/csv2avro.rb

@@ -1,9 +1,19 @@
 require 'csv2avro/converter'
 require 'csv2avro/version'
 
+require 'logr'
+
 class CSV2Avro
   attr_reader :input_path, :schema_path, :bad_rows_path, :stdout_option, :options
 
+  def self.logger
+    @logger ||= Logr::Logger.new('csv2avro')
+  end
+
+  def logger
+    self.class.logger
+  end
+
   def initialize(options)
     @input_path = ARGV.first
     @schema_path = options.delete(:schema)
@@ -14,7 +24,16 @@ class CSV2Avro
   end
 
   def convert
-    Converter.new(reader, writer, bad_rows_writer, error_writer, options, schema: schema).convert
+    logger.event('started_converting', filename: input_filename)
+          .monitored("Started converting #{input_filename}", "Started converting #{input_filename}")
+          .info("Started converting #{input_filename}")
+
+    lines = Converter.new(reader, writer, bad_rows_writer, input_filename, options, schema: schema).convert
+
+    logger.event('finished_converting', filename: input_filename)
+          .metric('lines_processed', lines)
+          .monitored("Finished converting #{input_filename}", "Finished converting #{input_filename}, processed #{lines} lines in total.")
+          .info("Finished converting #{input_filename}")
   ensure
     writer.close if writer
     bad_rows_writer.close
@@ -45,6 +64,10 @@ class CSV2Avro
     end
   end
 
+  def input_filename
+    File.basename(input_path)
+  end
+
   def avro_uri
     dir = File.dirname(input_path)
     ext = File.extname(input_path)
@@ -53,10 +76,6 @@ class CSV2Avro
     "#{dir}/#{name}.avro"
   end
 
-  def error_writer
-    $stderr
-  end
-
   def bad_rows_writer
     @__bad_rows_writer ||= File.open(bad_rows_uri, 'w')
   end

data/spec/csv2avro/converter_spec.rb

@@ -6,7 +6,6 @@ RSpec.describe CSV2Avro::Converter do
     let(:writer) { StringIO.new }
     let(:avro_writer) { CSV2Avro::AvroWriter.new(writer, schema) }
     let(:bad_rows_writer) { StringIO.new }
-    let(:error_writer) { StringIO.new }
 
     context 'schema with string and integer columns' do
       let(:schema_reader) do
@@ -35,17 +34,13 @@ RSpec.describe CSV2Avro::Converter do
         end
 
         before do
-          CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, error_writer, {}, schema: schema).convert
+          CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, 'data.csv', {}, schema: schema).convert
         end
 
         it 'should not have any bad rows' do
           expect(bad_rows_writer.read).to be_empty
         end
 
-        it 'should not have any errors' do
-          expect(error_writer.read).to be_empty
-        end
-
         it 'should store the data with the given schema' do
           expect(AvroReader.new(avro_writer).read).to eq(
             [
@@ -68,17 +63,13 @@ RSpec.describe CSV2Avro::Converter do
         end
 
         before do
-          CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, error_writer, { delimiter: "\t" }, schema: schema).convert
+          CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, 'data.csv', { delimiter: "\t" }, schema: schema).convert
         end
 
         it 'should not have any bad rows' do
           expect(bad_rows_writer.read).to be_empty
         end
 
-        it 'should not have any errors' do
-          expect(error_writer.read).to be_empty
-        end
-
         it 'should store the data with the given schema' do
           expect(AvroReader.new(writer).read).to eq(
             [
@@ -117,17 +108,13 @@ RSpec.describe CSV2Avro::Converter do
         end
 
         before do
-          CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, error_writer, {}, schema: schema).convert
+          CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, 'data.csv', {}, schema: schema).convert
         end
 
         it 'should not have any bad rows' do
           expect(bad_rows_writer.read).to be_empty
         end
 
-        it 'should not have any errors' do
-          expect(error_writer.read).to be_empty
-        end
-
         it 'should store the data with the given schema' do
           expect(AvroReader.new(writer).read).to eq(
             [
@@ -150,17 +137,13 @@ RSpec.describe CSV2Avro::Converter do
         end
 
         before do
-          CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, error_writer, { delimiter: "\t", array_delimiter: ';' }, schema: schema).convert
+          CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, 'data.csv', { delimiter: "\t", array_delimiter: ';' }, schema: schema).convert
         end
 
         it 'should not have any bad rows' do
           expect(bad_rows_writer.read).to be_empty
         end
 
-        it 'should not have any errors' do
-          expect(error_writer.read).to be_empty
-        end
-
         it 'should store the data with the given schema' do
           expect(AvroReader.new(writer).read).to eq(
             [
@@ -199,17 +182,13 @@ RSpec.describe CSV2Avro::Converter do
       end
 
       before do
-        CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, error_writer, { write_defaults: true }, schema: schema).convert
+        CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, 'data.csv', { write_defaults: true }, schema: schema).convert
       end
 
       it 'should not have any bad rows' do
         expect(bad_rows_writer.read).to be_empty
       end
 
-      it 'should not have any errors' do
-        expect(error_writer.read).to be_empty
-      end
-
       it 'should store the defaults data' do
         expect(AvroReader.new(writer).read).to eq(
           [
@@ -245,17 +224,13 @@ RSpec.describe CSV2Avro::Converter do
       end
 
       before do
-        CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, error_writer, {}, schema: schema).convert
+        CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, 'data.csv', {}, schema: schema).convert
       end
 
       it 'should not have any bad rows' do
         expect(bad_rows_writer.read).to be_empty
       end
 
-      it 'should not have any errors' do
-        expect(error_writer.read).to be_empty
-      end
-
       it 'should store the data with the given schema' do
         expect(AvroReader.new(writer).read).to eq(
           [
@@ -298,17 +273,13 @@ RSpec.describe CSV2Avro::Converter do
       end
 
       before do
-        CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, error_writer, { write_defaults: true }, schema: schema).convert
+        CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, 'data.csv', { write_defaults: true }, schema: schema).convert
       end
 
       it 'should not have any bad rows' do
         expect(bad_rows_writer.read).to be_empty
       end
 
-      it 'should not have any errors' do
-        expect(error_writer.read).to be_empty
-      end
-
       it 'should store the data with the given schema' do
         expect(AvroReader.new(writer).read).to eq(
           [
@@ -348,7 +319,7 @@ RSpec.describe CSV2Avro::Converter do
       end
 
       before do
-        CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, error_writer, { delimiter: "\t" }, schema: schema).convert
+        CSV2Avro::Converter.new(reader, avro_writer, bad_rows_writer, 'data.csv', { delimiter: "\t" }, schema: schema).convert
       end
 
       it 'should report the bad rows correctly' do
@@ -357,12 +328,6 @@ RSpec.describe CSV2Avro::Converter do
         )
       end
 
-      it 'should have an error' do
-        expect(error_writer.string).to eq(
-          "L2: Missing value at name\nL5: Missing value at name\n"
-        )
-      end
-
       it 'should store the data with the given schema' do
         expect(AvroReader.new(writer).read).to eq(
           [

data/spec/csv2avro_spec.rb

@@ -10,13 +10,10 @@ RSpec.describe CSV2Avro do
     context "Unquoted header" do
       before do
         ARGV.replace ['./spec/support/data.csv']
+        converter.convert
       end
 
       bad_rows_output = "L4: Missing value at name\nL7: Unable to parse\nL9: Missing value at id, Missing value at name\nL10: 'male-shoes' at id doesn't match the type '\"int\"', Missing value at name\n"
-      it 'should write errors to STDERR' do
-        expect { converter.convert }.to output(bad_rows_output).to_stderr
-      end
-
       it 'should have bad rows' do
         File.open('./spec/support/data.bad', 'r') do |file|
           expect(file.read).to eq(bad_rows_output)
@@ -40,10 +37,7 @@ RSpec.describe CSV2Avro do
     context "Quoted header" do
       before do
         ARGV.replace ['./spec/support/data_quoted.csv']
-      end
-
-      it 'should write errors to STDERR' do
-        expect { converter.convert }.to output("L4: Missing value at name\nL7: Unable to parse\n").to_stderr
+        converter.convert
       end
 
       it 'should have a bad row' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: csv2avro
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Peter Ableda
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-16 00:00:00.000000000 Z
+date: 2015-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -95,6 +95,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: logr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 description: Convert CSV files to Avro like a boss.
 email:
 - scotty@secretsaucepartners.com