streamingly 0.2.0 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3868d732dd4b4cae20b4e659401dfb3d30f2d1f8
-  data.tar.gz: 4123fd67e1617b8d8f08e414989508760b7b1014
+  metadata.gz: dee9c9df4012d58362750ed8c7c9d1562f4f927f
+  data.tar.gz: 3572f514fc6650de3a820e2a5728a5fcac9bc584
 SHA512:
-  metadata.gz: c56860e644409b403cee9670c6706360ee04e617c8875d7eebb6df7bbc87c397a6361cd401b5407caf057a7377b17f6d8ef296b92a90622f4da286ee188d834b
-  data.tar.gz: b99332851d72cb7ec2363db6524985ac6dd7a12e51c8ac7d42955fb46c26fec771bc257ed866376e58e8da127f3c0c1a550b29f1b4d3b68ade217fc4b4da6083
+  metadata.gz: ccc461ab15a8bbdea0db4faf2bf668b574e9c926c8a65bbb5672682ac4e4f80fcb1d92c75c0d9b993c8c20a985ea6c5683fc7f541bd13562a159093f6d4ec974
+  data.tar.gz: 01b0db0a463137f8f24e4320448ae75770879eb9ef180c1ef4213297120c14cc4c79ae11b40ff19d7d38aba40e36938f0c20d6a0d08b9902e70a6eccbc3275d9

data/lib/streamingly/reducer.rb

@@ -23,7 +23,7 @@ module Streamingly
   private
 
     def flush
-      @accumulator ? @accumulator.flush : []
+      @accumulator ? (@accumulator.flush || []).compact : []
     rescue StandardError => error
       on_error(error, {})
       []
@@ -33,8 +33,10 @@ module Streamingly
       # Streaming Hadoop only treats the first tab as the delimiter between
       # the key and value.  Additional tabs are grouped into the value:
       # http://hadoop.apache.org/docs/r0.18.3/streaming.html#How+Does+Streaming+Work
-      key, value = line.split("\t", 2)
+      key, value = (line || '').split("\t", 2)
+      return [] unless key && value
 
+      results = nil
       if @prev_key != key
         results = flush
 
@@ -42,16 +44,16 @@ module Streamingly
         @accumulator = new_accumulator(key)
       end
 
-      @accumulator.apply_value(value)
+      @accumulator.apply_value(value) if @accumulator
 
       results || []
     rescue StandardError => error
-      on_error(error, { :line => line })
+      on_error(error, line: line)
       []
     end
 
     def on_error(error, error_context)
-      raise error unless @error_callback_defined
+      raise error unless @error_callback_defined && !@accumulator.nil?
       @accumulator.on_error(error, error_context)
     end
 

data/lib/streamingly/serde.rb

@@ -34,6 +34,7 @@ module Streamingly
 
     def self.from_tabbed_csv(string)
       k, v = string.split("\t", 2)
+      return if k.nil? || v.nil?
       key = from_string_or_csv(k)
       value = if v.include? "\t"
                 from_tabbed_csv(v)
@@ -44,11 +45,13 @@ module Streamingly
     end
 
     def self.from_string_or_csv(string)
-      if string.include? ','
-        from_csv(string)
+      if string.include? ','  # Likely a CSV
+        from_csv(string)  # Attempt to parse
       else
         string
       end
+    rescue CSV::MalformedCSVError  # Not actually CSV, fallback to string
+      string
     end
 
     def self.resolve_class(class_name)

data/lib/streamingly/version.rb

@@ -1,3 +1,3 @@
 module Streamingly
-  VERSION = "0.2.0"
+  VERSION = '0.2.2'
 end

data/spec/streamingly/serde_spec.rb

@@ -28,4 +28,43 @@ describe Streamingly::SerDe do
     end
   end
 
-end
+  describe '.to_csv' do
+    it 'is identity function for a string' do
+      record = 'test_string'
+      expect(described_class.to_csv(record)).to eq record
+    end
+
+    it 'is equal to string version of Streamingly kv' do
+      record = Streamingly::KV.new('key', 'value')
+      expect(described_class.to_csv(record)).to eq record.to_s
+    end
+
+    it 'serializes struct to CSV, interpreting decimal fields as floats' do
+      Record = Struct.new(:number, :string)
+      record = Record.new(1, 'string_value')
+      expect(described_class.to_csv(record)).to eq 'Record,1,string_value'
+    end
+  end
+
+  describe '.from_string_or_csv' do
+    it 'returns CSV serialization for CSV string' do
+      expect(described_class.from_string_or_csv('1,2')).to eq ['1', '2']
+    end
+
+    it 'returns string if not containing a comma' do
+      expect(described_class.from_string_or_csv('foo')).to eq 'foo'
+    end
+
+    it 'returns string if containing a comma but not valid CSV' do
+      expect(described_class.from_string_or_csv('"foo,bar')).to eq '"foo,bar'
+    end
+  end
+
+  describe '.from_tabbed_csv' do
+    it 'returns nested KV pair structure with the first tab as the split' do
+      expect(described_class.from_tabbed_csv("a\tb\tc")).to eq(
+        Streamingly::KV.new('a', Streamingly::KV.new('b', 'c'))
+      )
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: streamingly
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.2
 platform: ruby
 authors:
 - Matt Gillooly
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-31 00:00:00.000000000 Z
+date: 2015-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler