arc-furnace 0.1.0 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 38e438f57e6350ddce2937cf2ef03d6d93ea20a8
-  data.tar.gz: 17ee74aba7edef53647c2a2e4efe6b4d3bda477f
+  metadata.gz: 2ab49afa557ed851dd866c6dd1486bde91062890
+  data.tar.gz: f66fa651a0e3ac6df2c3824d5daecfe1244859c6
 SHA512:
-  metadata.gz: ee8d88b55edd486dbd496d2831096ed53eaad124e6043629747b9503bd6473267c1f18086edae0425152b08638fc7ed4961cfecc79b72a92d709768c49316b25
-  data.tar.gz: 39114078eab439a660c83479b9251b0e9f1690aa8e0110b51a4dc061aee5f7d64421189c69f0705349d95339b14e7e5b118124b1822a7c3e747446313c585ec7
+  metadata.gz: 236b5f55f9914dc37a89579cc543304880af745d49096186e94397e6eca5ea41024437bb16461e5c457786e47fbc3db67fed268a424f72b7c7c057be026d796e
+  data.tar.gz: 6833e0e9e9e84442ce686cdd68e55efdd3c8535ec57b7fe225c10f2811e390e5af02525086ab90cac8ee158533fe619f7984f4f0cb834fda25f2dd57670ae694

data/README.md

@@ -1,6 +1,8 @@
 # ArcFurnace
+[![Gem Version](https://badge.fury.io/rb/arc-furnace.png)][gem]
 [![Build Status](https://travis-ci.org/salsify/arc-furnace.svg?branch=master)][travis]
 
+[gem]: https://rubygems.org/gems/arc-furnace
 [travis]: http://travis-ci.org/salsify/arc-furnace
 
 ArcFurnace melts, melds, and transforms your scrap data into perfectly crafted data for ingest into applications,
@@ -71,7 +73,7 @@ require a stream of data (`Hash`, `Transform`, `Join`, `Sink`) will have one.
 
 #### Hashes
 
-A `Hash` provides indexed access to a `Source` but pre-computing the index based on a key. The processing happens during the 
+A `Hash` provides indexed access to a `Source` but pre-computing the index based on a key. The processing happens during the
 prepare stage of pipeline processing. Hashes have a simple interface, `#get(primary_key)`, to requesting data. Hashes
 are almost exclusively used as inputs to one side of joins.
 
@@ -82,6 +84,12 @@ key is the key that the hash was rolled-up on, however, the `key_column` option
 may override this. Note the default join is an inner join, which will drop source rows if the hash does not contain
 a matching row.
 
+#### Filters
+
+A `Filter` acts as a source, however, takes a source as an input and determines whether to pass each row to
+the next downstream node by calling the `#filter` method on itself. There is an associated `BlockFilter` and
+sugar on `Pipeline` to make this easy.
+
 #### Transforms
 
 A `Transform` acts as a source, however, takes a source as an input and transforms each input. The `BlockTransform` and
@@ -100,7 +108,7 @@ subscribe to the `#row(hash)` interace--each output row is passed to this method
 
 ### General pipeline development process
 
-1. Define a source. Choose an existing `Source` implementation in this library (`CSVSource` or `ExcelSource`), 
+1. Define a source. Choose an existing `Source` implementation in this library (`CSVSource` or `ExcelSource`),
    extend the `EnumeratorSource`, or implement the `row()` method for a new source.
 2. Define any transformations, or joins. This may cause you to revisit #1.
 3. Define the sink. This is generally custom, or, may be one of the provided `CSVSink` types.
@@ -114,9 +122,8 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## TODOs
 
-1. Add a `filter` node and implementation to `Pipeline`
-2. Add examples for `ErrorHandler` interface.
-3. Add sugar to define a `BlockTransform` on a `Source` definition in a `Pipeline`.
+1. Add examples for `ErrorHandler` interface.
+2. Add sugar to define a `BlockTransform` on a `Source` definition in a `Pipeline`.
 
 ## Contributing
 

data/lib/arc-furnace/block_filter.rb

@@ -0,0 +1,18 @@
+require 'arc-furnace/filter'
+
+module ArcFurnace
+  class BlockFilter < Filter
+    private_attr_reader :block
+
+    def initialize(source:, block:)
+      raise 'Must specify a block' if block.nil?
+      @block = block
+      super(source: source)
+    end
+
+    def filter(row)
+      block.call(row)
+    end
+
+  end
+end

data/lib/arc-furnace/enumerator_source.rb

@@ -30,7 +30,7 @@ module ArcFurnace
 
     # Return the enumerator
     def build_enumerator
-      raise "Unimplemented!"
+      raise "Unimplemented"
     end
   end
 end

data/lib/arc-furnace/filter.rb

@@ -0,0 +1,37 @@
+require 'arc-furnace/source'
+
+# Filters limit rows to downstream nodes. They act just like Enumerable#filter:
+# when the #filter method returns true, the row is passed downstream. when
+# it returns false, the row is skipped.
+module ArcFurnace
+  class Filter < Source
+
+    private_attr_reader :source
+    attr_reader :value
+
+    def initialize(source:)
+      @source = source
+      advance
+    end
+
+    # Given a row from the source, tell if it should be passed down to the next
+    # node downstream from this node.
+    #
+    # This method must return a boolean
+    def filter(row)
+      raise "Unimplemented"
+    end
+
+    def empty?
+      value.nil? && source.empty?
+    end
+
+    def advance
+      loop do
+        @value = source.row
+        break if value.nil? || filter(value)
+      end
+    end
+
+  end
+end

data/lib/arc-furnace/nodes.rb

@@ -1,10 +1,12 @@
 require 'arc-furnace/all_fields_csv_sink'
 require 'arc-furnace/binary_key_merging_hash'
+require 'arc-furnace/block_filter'
 require 'arc-furnace/block_transform'
 require 'arc-furnace/block_unfold'
 require 'arc-furnace/csv_sink'
 require 'arc-furnace/csv_source'
 require 'arc-furnace/enumerator_source'
+require 'arc-furnace/filter'
 require 'arc-furnace/fixed_column_csv_sink'
 require 'arc-furnace/hash'
 require 'arc-furnace/inner_join'

data/lib/arc-furnace/pipeline.rb

@@ -23,58 +23,70 @@ module ArcFurnace
       end
 
       @sink_node = -> do
-        type.new(resolve_parameters(params))
+        type.new(resolve_parameters(:sink, params))
       end
       @sink_source = source
     end
 
     # Define a hash node, processing all rows from it's source and caching them
     # in-memory.
-    def self.hash_node(name, type: ArcFurnace::Hash, params:)
-      define_intermediate(name, type: type, params: params)
+    def self.hash_node(node_id, type: ArcFurnace::Hash, params:)
+      define_intermediate(node_id, type: type, params: params)
     end
 
     # A source that has row semantics, delivering a hash per row (or per entity)
     # for the source.
-    def self.source(name, type:, params:)
+    def self.source(node_id, type:, params:)
       raise "Source #{type} is not a Source!" unless type <= Source
-      define_intermediate(name, type: type, params: params)
+      define_intermediate(node_id, type: type, params: params)
     end
 
     # Define an inner join node where rows from the source are dropped
     # if an associated entity is not found in the hash for the join key
-    def self.inner_join(name, type: ArcFurnace::InnerJoin, params:)
-      define_intermediate(name, type: type, params: params)
+    def self.inner_join(node_id, type: ArcFurnace::InnerJoin, params:)
+      define_intermediate(node_id, type: type, params: params)
     end
 
     # Define an outer join nod  e where rows from the source are kept
     # even if an associated entity is not found in the hash for the join key
-    def self.outer_join(name, type: ArcFurnace::OuterJoin, params:)
-      define_intermediate(name, type: type, params: params)
+    def self.outer_join(node_id, type: ArcFurnace::OuterJoin, params:)
+      define_intermediate(node_id, type: type, params: params)
     end
 
     # Define a node that transforms rows. By default you get a BlockTransform
     # (and when this metaprogramming method is passed a block) that will be passed
     # a hash for each row. The result of the block becomes the row for the next
     # downstream node.
-    def self.transform(name, type: BlockTransform, params: {}, &block)
-      if block
+    def self.transform(node_id, type: BlockTransform, params: {}, &block)
+      if block_given? && type <= BlockTransform
         params[:block] = block
       end
       raise "Transform #{type} is not a Transform!" unless type <= Transform
-      define_intermediate(name, type: type, params: params)
+      define_intermediate(node_id, type: type, params: params)
     end
 
-    # Define a node that unfolds rows. By default you get a BlocUnfold
+    # Define a node that unfolds rows. By default you get a BlockUnfold
     # (and when this metaprogramming method is passed a block) that will be passed
     # a hash for each row. The result of the block becomes the set of rows for the next
     # downstream node.
-    def self.unfold(name, type: BlockUnfold, params: {}, &block)
-      if block
+    def self.unfold(node_id, type: BlockUnfold, params: {}, &block)
+      if block_given? && type <= BlockUnfold
         params[:block] = block
       end
       raise "Unfold #{type} is not an Unfold!" unless type <= Unfold
-      define_intermediate(name, type: type, params: params)
+      define_intermediate(node_id, type: type, params: params)
+    end
+
+    # Define a node that filters rows. By default you get a BlockFilter
+    # (and when this metaprogramming method is passed a block) that will be passed
+    # a hash for each row. The result of the block determines if a given row
+    # flows to a downstream node
+    def self.filter(node_id, type: BlockFilter, params: {}, &block)
+      if block_given? && type <= BlockFilter
+        params[:block] = block
+      end
+      raise "Filter #{type} is not a Filter!" unless type <= Filter
+      define_intermediate(node_id, type: type, params: params)
     end
 
     # Create an instance to run a transformation, passing the parameters to
@@ -82,18 +94,18 @@ module ArcFurnace
     # will have a single public method--#execute, which will perform the
     # transformation.
     def self.instance(params = {})
-      DSLInstance.new(self, params)
+      PipelineInstance.new(self, params)
     end
 
     private
 
-    def self.define_intermediate(name, type:, params:)
-      intermediates_map[name] = -> do
-        type.new(resolve_parameters(params))
+    def self.define_intermediate(node_id, type:, params:)
+      intermediates_map[node_id] = -> do
+        type.new(resolve_parameters(node_id, params))
       end
     end
 
-    class DSLInstance
+    class PipelineInstance
       attr_reader :sink_node, :sink_source, :intermediates_map, :params, :dsl_class, :error_handler
 
       def initialize(dsl_class, error_handler: ErrorHandler.new, **params)
@@ -135,22 +147,22 @@ module ArcFurnace
         @sink_source = intermediates_map[dsl_class.sink_source]
       end
 
-      def resolve_parameters(params_to_resolve)
+      def resolve_parameters(node_id, params_to_resolve)
         params_to_resolve.each_with_object({}) do |(key, value), result|
           result[key] =
             if value.is_a?(Symbol)
               # Allow resolution of intermediates
-              resolve_parameter(value)
+              resolve_parameter(node_id, value)
             elsif value.nil?
-              resolve_parameter(key)
+              resolve_parameter(node_id, key)
             else
               value
             end
         end
       end
 
-      def resolve_parameter(key)
-        self.params[key] || self.intermediates_map[key] || (raise "Unknown key #{key}!")
+      def resolve_parameter(node_id, key)
+        self.params[key] || self.intermediates_map[key] || (raise "When processing node #{node_id}: Unknown key #{key}!")
       end
 
     end

data/lib/arc-furnace/source.rb

@@ -4,6 +4,8 @@ module ArcFurnace
   class Source < Node
     extend Forwardable
 
+    # Called to prepare anything this source needs to do before providing rows.
+    # For instance, opening a source file or database connection.
     def prepare
 
     end
@@ -17,23 +19,23 @@ module ArcFurnace
 
     # Is this source empty?
     def empty?
-
+      raise 'Unimplemented'
     end
 
     # The current value this source points at
     # This is generally the only method required to implement a source.
     def value
-
+      raise 'Unimplemented'
     end
 
-    # Close the source
+    # Close the source. Called by the framework at the end of processing.
     def close
 
     end
 
     # Advance this source by one. #advance specifies no return value contract
     def advance
-
+      raise 'Unimplemented'
     end
 
   end

data/lib/arc-furnace/version.rb

@@ -1,3 +1,3 @@
 module ArcFurnace
-  VERSION = "0.1.0"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: arc-furnace
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.3
 platform: ruby
 authors:
 - Daniel Spangenberger
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-10-04 00:00:00.000000000 Z
+date: 2015-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -129,6 +129,7 @@ files:
 - lib/arc-furnace/abstract_join.rb
 - lib/arc-furnace/all_fields_csv_sink.rb
 - lib/arc-furnace/binary_key_merging_hash.rb
+- lib/arc-furnace/block_filter.rb
 - lib/arc-furnace/block_transform.rb
 - lib/arc-furnace/block_unfold.rb
 - lib/arc-furnace/csv_sink.rb
@@ -138,6 +139,7 @@ files:
 - lib/arc-furnace/enumerator_source.rb
 - lib/arc-furnace/error_handler.rb
 - lib/arc-furnace/excel_source.rb
+- lib/arc-furnace/filter.rb
 - lib/arc-furnace/fixed_column_csv_sink.rb
 - lib/arc-furnace/hash.rb
 - lib/arc-furnace/inner_join.rb