streamingly 0.0.8 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f507114d8202daadc401f6de1d18054035a7c557
-  data.tar.gz: 4cab496f8fb9940b12aca0eab8735ef00344e124
+  metadata.gz: 58821c716d9d1b756cfca76758fe86c538017a4d
+  data.tar.gz: b85d8a2398ab33ba1a08fa2770af84d0b2b0ebee
 SHA512:
-  metadata.gz: 7a33cace50cfb7ebf35723738474b0026f415228366df9385b63abc645f0c046017c174aa633fd2bf1953f358ec4c31be6d3577741c7bb83c4286abeb0002d91
-  data.tar.gz: a3f0251f7359e1ada162f7cf5378ecdd458fae05eb3bff8dde27e1d9f26e0cc68210de99e01ce25313555cbec1c58050fcb4476bcb76a73a8013a7d50a31546a
+  metadata.gz: 75cc3e16e31928a46816eb3dce6248e129a21ba5124716bcb258de9c01e75b9a7650befc0602d78e469110f5f00800fd22a7a7afba4ba0a9e528cd964a4bbb07
+  data.tar.gz: d65f1bfd7b08897bb985674276a1a098185583123115941bc2d2f845225ae7e8f22a892ff0e12878ed9e4382aeb5e3369bd76bef0f2ab71d582e0198d771949a

data/lib/streamingly/reducer.rb

@@ -1,9 +1,10 @@
 module Streamingly
   class Reducer
 
-    def initialize(accumulator_class, accumulator_options=nil)
+    def initialize(accumulator_class, accumulator_options = nil)
       @accumulator_class = accumulator_class
       @accumulator_options = accumulator_options
+      @error_callback_defined = @accumulator_class.method_defined?(:on_error)
     end
 
     def reduce_over(enumerator)
@@ -16,12 +17,16 @@ module Streamingly
       flush.each do |out|
         yield out
       end
+
     end
 
   private
 
     def flush
       @accumulator ? @accumulator.flush : []
+    rescue StandardError => error
+      on_error(error, {})
+      []
     end
 
     def reduce(line)
@@ -40,6 +45,14 @@ module Streamingly
       @accumulator.apply_value(value)
 
       results || []
+    rescue StandardError => error
+      on_error(error, { :line => line })
+      []
+    end
+
+    def on_error(error, error_context)
+      raise error unless @error_callback_defined
+      @accumulator.on_error(error, error_context)
     end
 
     def new_accumulator(key)

data/lib/streamingly/serde.rb

@@ -8,6 +8,8 @@ module Streamingly
       case record
       when String
         record
+      when Streamingly::KV
+        record.to_s
       when Struct
         tokens = *record.map { |token|
           case token

data/lib/streamingly/version.rb

@@ -1,3 +1,3 @@
 module Streamingly
-  VERSION = "0.0.8"
+  VERSION = "0.1.0"
 end

data/spec/streamingly/reducer_spec.rb

@@ -1,8 +1,44 @@
 require 'spec_helper'
 
+class TestAccumulator
+  attr_reader :applied_values, :raised_errors, :error_contexts
+
+  def initialize
+    @applied_values = []
+    @raised_errors = []
+    @error_contexts = []
+  end
+
+  def apply_value(value)
+    @applied_values << value
+  end
+
+  def flush
+    []
+  end
+
+  def on_error(error, error_context)
+    @raised_errors << error
+    @error_contexts << error_context
+  end
+end
+
+class RaisesOnSingleCharAccumulator < TestAccumulator
+  def apply_value(value)
+    super(value)
+    raise ArgumentError, "Must not be a single character" if value.size == 1
+  end
+end
+
+class RaisesOnFlushAccumulator < TestAccumulator
+  def flush
+    raise RuntimeError, "Cannot flush when ya only have two pairs"
+  end
+end
+
 describe Streamingly::Reducer do
 
-  let(:accumulator_class) { double }
+  let(:accumulator_class) { double(:accumulator_class, :method_defined? => true) }
   subject { described_class.new(accumulator_class) }
 
   describe "#reduce_over" do
@@ -85,6 +121,125 @@ describe Streamingly::Reducer do
       end
     end
 
+    context "given a record which will cause an exception" do
+      let(:key) { 'key1' }
+      let(:value1) { "a" }
+      let(:value2) { "abc" }
+      let(:records) {
+        [
+          [key, value1].join("\t"),
+          [key, value2].join("\t")
+        ]
+      }
+
+      let(:accumulator) { RaisesOnSingleCharAccumulator.new }
+
+      context "with error callback specified" do
+        before do
+          accumulator_class.stub(:new).with(key) { accumulator }
+        end
+
+        it "keeps processing after error applying value" do
+          subject.reduce_over(records)
+
+          expect(accumulator.applied_values).to eq([value1, value2])
+        end
+
+        it "calls supplied error callback with correct context" do
+          subject.reduce_over(records)
+
+          expect(accumulator.raised_errors.size).to eq(1)
+          raised_error = accumulator.raised_errors[0]
+          expect(raised_error.class).to eq(ArgumentError)
+
+          expect(accumulator.error_contexts.size).to eq(1)
+          error_context = accumulator.error_contexts[0]
+          expect(error_context[:line]).to eq([key, value1].join("\t"))
+        end
+      end
+
+      context "without error callback specified" do
+        let(:accumulator_class) { double(:accumulator_class, :method_defined? => false) }
+        subject { described_class.new(accumulator_class) }
+
+        before do
+            accumulator_class.stub(:new).with(key) { accumulator }
+        end
+
+        it "stops processing after error applying value" do
+          expect{ subject.reduce_over(records) }.to raise_error(ArgumentError)
+
+          expect(accumulator.applied_values).to eq([value1])
+          expect(accumulator.raised_errors.size).to eq(0)
+        end
+      end
+    end
+
+    context "given accumulator which fails on flush" do
+      let(:key1) { 'key1' }
+      let(:key2) { 'key2' }
+      let(:value1) { 'asdf' }
+      let(:value2) { 'qwerty' }
+
+      let(:records) {
+        [
+          [key1, value1].join("\t"),
+          [key2, value2].join("\t")
+        ]
+      }
+
+      let(:accumulator1) { RaisesOnFlushAccumulator.new }
+      let(:accumulator2) { TestAccumulator.new }
+
+      context "with error callback specified" do
+        before do
+          accumulator_class.stub(:new).with(key1) { accumulator1 }
+          accumulator_class.stub(:new).with(key2) { accumulator2 }
+        end
+
+        it "keeps processing after error applying value" do
+          subject.reduce_over(records)
+
+          expect(accumulator1.applied_values).to eq([value1])
+
+          expect(accumulator2.applied_values).to eq([value2])
+        end
+
+        it "calls supplied error callback" do
+          subject.reduce_over(records)
+
+          expect(accumulator1.raised_errors.size).to eq(1)
+          raised_error = accumulator1.raised_errors[0]
+          expect(raised_error.class).to eq(RuntimeError)
+
+          expect(accumulator1.error_contexts.size).to eq(1)
+          expect(accumulator1.error_contexts[0]).to be_empty
+
+          expect(accumulator2.raised_errors.size).to eq(0)
+        end
+      end
+
+      context "without error callback specified" do
+        let(:accumulator_class) { double(:accumulator_class, :method_defined? => false) }
+        subject { described_class.new(accumulator_class) }
+
+        before do
+          accumulator_class.stub(:new).with(key1) { accumulator1 }
+          accumulator_class.stub(:new).with(key2) { accumulator2 }
+        end
+
+        it "stops processing after error applying value" do
+          expect{ subject.reduce_over(records) }.to raise_error(RuntimeError)
+
+          expect(accumulator1.applied_values).to eq([value1])
+          expect(accumulator1.raised_errors.size).to eq(0)
+
+          expect(accumulator2.applied_values).to eq([])
+          expect(accumulator2.raised_errors.size).to eq(0)
+        end
+      end
+    end
+
     context "when supplied with accumulator options" do
       let(:accumulator_options) { { foo: 'bar' } }
       subject { described_class.new(accumulator_class, accumulator_options) }

data/streamingly.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "pry"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec", "~> 2.11"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: streamingly
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.1.0
 platform: ruby
 authors:
 - Matt Gillooly
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-08 00:00:00.000000000 Z
+date: 2014-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement