stockboy 0.8.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 90e9bfd03d069c571ffd1e6e03dd018cf477289c
-  data.tar.gz: f0f88cdbbf0e65a8eadddf0e390d91f3c1a5f4c4
+  metadata.gz: e9478014be3b55b4e3c8c106a418e4f8fb9940bd
+  data.tar.gz: 46ca2d7e3bd6d567eca0f3ae26842a522c231d3b
 SHA512:
-  metadata.gz: d0686581b72dd43ef079e93f8b3336669f50919e5f8adaa5019b198d4ce39b61771584ffcab122640cadbe1bdb372d4745bb4f444ee3d3d799833f2f73215b2f
-  data.tar.gz: 1b11eae674ea31430894a4e4f050e91f2011841a94f72c6b580269f446eab4a5e4baacdda616c70e3b0a889b13fdd8fa51633fec9e936401bd01399f45857ab1
+  metadata.gz: 74ae4dc70a0676ef4677e7fc62f700e291a6681996b27e9dae7112a136ba5ccb835ac53c5cced0cfedf8d3b9bc21fa83052e70b1fd5dc0288bc57ae129648451
+  data.tar.gz: 4ae8f8e91b4368e3aa5cc1a486ffa4cff87e9eb9aa68f0c198d840d80fa0346c2ba2a012ae60f8d25ac94d080d50917b732725af67f8e22325a7c7dbea8ce0ae

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.8.1 / 2014-04-24
+
+* [BUGFIX]      Support `data_size` for non-string providers
+* [BUGFIX]      Checks for nil data and nil configuration options
+
 ## 0.8.0 / 2014-04-02
 
 * [ENHANCEMENT] Expose `data_time` and `data_size` in repeater

data/README.md

@@ -5,10 +5,10 @@
 [![Build Status](https://travis-ci.org/avit/stockboy.png)][travis]
 [![Code Climate](https://codeclimate.com/github/avit/stockboy.png)][climate]
 
-Stockboy helps you receive and unpack data onto your shelves. You might
-consider using it to synchronize data exported from external sources, or
-migrating your own data from legacy systems. (TL;DR, Stockboy is a Ruby
-[DSL][dsl] for doing [ETL][etl].)
+Stockboy helps you receive and unpack data onto your shelves. It provides a DSL
+for configuring data processing pipelines. You might consider using it to
+synchronize data exported from external sources, as part of an [ETL][etl]
+workflow, or migrating your own data from legacy systems. 
 
 Full documentation available at [rdoc.info/gems/stockboy][rdoc]
 

data/lib/stockboy/job.rb

@@ -116,6 +116,14 @@ module Stockboy
       provider.errors.empty?
     end
 
+    def data(&block)
+      provider.data(&block)
+    end
+
+    def data?(reduction=:all?)
+      provider.data?(reduction)
+    end
+
     # Count of all processed records
     #
     # @!attribute [r] total_records
@@ -209,6 +217,7 @@ module Stockboy
     def load_records
       reset
       load_all_records
+      return unless provider.data?
       partition_all_records
       @processed = true
     end
@@ -234,14 +243,20 @@ module Stockboy
       end
     end
 
+    # Allows for a data method that either yields or returns giving preference
+    # to the yield. It will ignore the data return value if it has yielded.
+    #
     def with_provider_data
       return to_enum(__method__) unless block_given?
       yielded = nil
       provider.data do |data|
-        yielded = true
-        yield data
+        if data
+          yielded = true
+          yield data
+        end
       end
-      yield provider.data unless yielded
+      return if yielded
+      yield(provider.data) if provider.data
     end
 
     def record_partition(record)

data/lib/stockboy/provider.rb

@@ -81,8 +81,11 @@ module Stockboy
       @data
     end
 
-    def data?
-      @data_size && @data_size > 0
+    # Determine if there is returned data
+    #
+    def data?(_=nil)
+      return nil unless @data
+      @data && !@data.empty?
     end
 
     # Reset received data

data/lib/stockboy/provider_repeater.rb

@@ -5,22 +5,57 @@ module Stockboy
 
     YIELD_ONCE = proc { |output, provider| output << provider }
 
+    ProviderStats = Struct.new(:data_time, :data_size, :data_) do
+      def self.from(provider)
+        new(provider.data_time, provider.data_size, provider.data?)
+      end
+
+      alias :data? :data_ # 1.9 compatibility
+    end
+
     attr_reader :base_provider
-    attr_reader :data_size
-    attr_reader :data_time
 
     def initialize(provider, &yielder)
       @orig_provider = provider
       @base_provider = provider.dup
+      @iterations = []
       @yielder = yielder || YIELD_ONCE
     end
 
-    def data?
-      @data_size && @data_size > 0
+    # Determine if there was any returned data after processing iterations
+    # @param [:all?,:any?,:one?] reduction
+    #   Specify if all iterations must return data to be valid, or just any
+    #
+    def data?(reduction = :all?)
+      return nil if data_iterations == 0
+      @iterations.send(reduction, &:data?)
+    end
+
+    # Get the total data size returned after processing iterations
+    #
+    def data_size
+      @iterations.reduce(nil) { |sum, source|
+        source.data_size ? source.data_size + sum.to_i : sum
+      }
+    end
+
+    # Get the last data time returned after processing iterations
+    #
+    def data_time
+      @iterations.reduce(nil) { |max, source|
+        source.data_time && (max.nil? || source.data_time > max) ? source.data_time : max
+      }
+    end
+
+    def data_iterations
+      @iterations.size
     end
 
     def data
-      raise ArgumentError, "expects a block for yielding each data set" unless block_given?
+      unless block_given?
+        raise ArgumentError, "expect a block for yielding data iterations"
+      end
+
       each do |nth_provider|
         yield fetch_iteration_data(nth_provider)
       end
@@ -28,7 +63,7 @@ module Stockboy
 
     def clear
       @base_provider = @orig_provider.dup
-      @data_time, @data_size = nil, nil
+      @iterations.clear
       super
     end
 
@@ -67,8 +102,7 @@ module Stockboy
 
     def fetch_iteration_data(provider)
       if provider.data
-        @data_size = [@data_size, provider.data_size].compact.reduce(&:+)
-        @data_time = [@data_time, provider.data_time].compact.max
+        @iterations << ProviderStats.from(provider)
       end
       provider.data
     end

data/lib/stockboy/providers/ftp.rb

@@ -116,6 +116,7 @@ module Stockboy::Providers
     rescue Net::FTPError => e
       errors << e.message
       logger.warn e.message
+      nil
     end
 
     def matching_file

data/lib/stockboy/providers/http.rb

@@ -61,7 +61,7 @@ module Stockboy::Providers
     dsl_attr :password
 
     def uri
-      return nil if @uri.nil? || @uri.empty?
+      return nil if @uri.nil? || @uri.to_s.empty?
       URI(@uri).tap { |u| u.query = URI.encode_www_form(@query) if @query }
     end
 
@@ -126,18 +126,18 @@ module Stockboy::Providers
 
     def client
       orig_logger, HTTPI.logger = HTTPI.logger, logger
-      req = HTTPI::Request.new(uri)
+      req = HTTPI::Request.new.tap { |c| c.url = uri }
       req.auth.basic(username, password) if username && password
-      yield req if block_given?
+      block_given? ? yield(req) : req
+    ensure
       HTTPI.logger = orig_logger
-      req
     end
 
     private
 
     def validate
-      errors << "uri must be specified" if uri.blank?
-      errors << "method (:get, :post) must be specified" if method.blank?
+      errors << "uri must be specified" unless uri
+      errors << "method (:get, :post) must be specified" unless method
       errors.empty?
     end
 

data/lib/stockboy/version.rb

@@ -1,3 +1,3 @@
 module Stockboy
-  VERSION = "0.8.0"
+  VERSION = "0.8.1"
 end

data/spec/stockboy/job_spec.rb

@@ -12,6 +12,7 @@ class TestProvider
   end
   def data_size; @data && @data.size end
   def data_time; @data && Time.now end
+  def data?; true end
 end
 
 class TestReader

data/spec/stockboy/provider_spec.rb

@@ -79,7 +79,7 @@ module Stockboy
       its(:data_size) { should be_nil }
 
       context "after fetching" do
-        before { provider.data }
+        before { provider.fetch_data }
         its(:data_size) { should be > 0 }
       end
     end
@@ -88,8 +88,8 @@ module Stockboy
       subject(:provider) { ProviderSubclass.new }
       its(:data_time) { should be_nil }
 
-      context "after iterating" do
-        before { provider.data }
+      context "after fetching" do
+        before { provider.fetch_data }
         its(:data_time) { should be_a Time }
       end
     end
@@ -98,8 +98,8 @@ module Stockboy
       subject(:provider) { ProviderSubclass.new }
       its(:data?) { should be_nil }
 
-      context "after iterating" do
-        before { provider.data }
+      context "after fetching" do
+        before { provider.fetch_data }
         its(:data?) { should be true }
       end
     end

data/spec/stockboy/providers/http_spec.rb

@@ -72,6 +72,20 @@ module Stockboy
       end
     end
 
+    describe "#client" do
+      subject(:client) { provider.client }
+
+      before { provider.uri = "http://example.com/" }
+
+      it "should configure the base url" do
+        client.url.host.should == "example.com"
+      end
+
+      it "returns the value of the passed block" do
+        provider.client { |http| "DATA" }.should == "DATA"
+      end
+    end
+
     describe "#data" do
       let(:response)   { HTTPI::Response.new(200, {}, '{"success":true}') }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stockboy
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Andrew Vit
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-03 00:00:00.000000000 Z
+date: 2014-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake