arc-furnace 0.1.0

data/lib/arc-furnace/private_attr.rb

@@ -0,0 +1,28 @@
+class Module
+
+  private
+
+  # Meta-programming to easily create private attribute reader methods.
+  def private_attr_reader(*attrs)
+    attr_reader(*attrs)
+    private(*attrs)
+  end
+
+  # Meta-programming to easily create private attribute writer methods.
+  def private_attr_writer(*attrs)
+    attr_writer(*attrs)
+    private(*attrs.map { |attr| "#{attr}=".to_sym })
+  end
+
+  # Meta-programming to easily create private attribute accessor methods.
+  def private_attr_accessor(*attrs)
+    private_attr_reader(*attrs)
+    private_attr_writer(*attrs)
+  end
+
+  def private_alias_method(new_name, old_name)
+    alias_method(new_name, old_name)
+    private(new_name)
+  end
+
+end

data/lib/arc-furnace/sink.rb

@@ -0,0 +1,21 @@
+module ArcFurnace
+  class Sink
+
+    # The only required method to implement. #row is called for each output row and
+    # a sink must handle each.
+    def row(row)
+      raise "Unimplemented!"
+    end
+
+    # Handle any pre-processing here.
+    def prepare
+
+    end
+
+    # If the sink needs to perform any clean-up (closing file handles, etc),
+    # do it here.
+    def finalize
+
+    end
+  end
+end

data/lib/arc-furnace/source.rb

@@ -0,0 +1,40 @@
+require 'arc-furnace/node'
+
+module ArcFurnace
+  class Source < Node
+    extend Forwardable
+
+    def prepare
+
+    end
+
+    # Advance this source by one, returning the row as a hash
+    def row
+      result = value
+      advance
+      result
+    end
+
+    # Is this source empty?
+    def empty?
+
+    end
+
+    # The current value this source points at
+    # This is generally the only method required to implement a source.
+    def value
+
+    end
+
+    # Close the source
+    def close
+
+    end
+
+    # Advance this source by one. #advance specifies no return value contract
+    def advance
+
+    end
+
+  end
+end

data/lib/arc-furnace/suffixed_fixed_column_csv_sink.rb

@@ -0,0 +1,18 @@
+require 'arc-furnace/fixed_column_csv_sink'
+
+module ArcFurnace
+  class SuffixedFixedColumnCSVSink < FixedColumnCSVSink
+    private_attr_reader :fields
+
+    def write_header
+      csv << fields.each_with_object([]) do |(key, count), result|
+        if count > 1
+          count.times { |index| result << "#{key} #{index + 1}" }
+        else
+          result << key
+        end
+      end
+    end
+
+  end
+end

data/lib/arc-furnace/transform.rb

@@ -0,0 +1,28 @@
+require 'arc-furnace/source'
+
+module ArcFurnace
+  class Transform < Source
+
+    private_attr_reader :source
+
+    def initialize(source:)
+      @source = source
+    end
+
+    def prepare
+      source.prepare
+    end
+
+    def value
+      value = source.value.deep_dup
+      transform(value) if value
+    end
+
+    def transform(row)
+      row
+    end
+
+    delegate [:empty?, :advance] => :source
+
+  end
+end

data/lib/arc-furnace/unfold.rb

@@ -0,0 +1,45 @@
+require 'arc-furnace/source'
+
+# An unfold is the reverse of a fold--it takes a single value an blows it out
+# into an enumeration of values. Useful for splitting up rows into multiple output
+# rows and whatnot. Only #unfold is required for implementation.
+module ArcFurnace
+  class Unfold < Source
+
+    private_attr_reader :source, :unfolded
+    attr_reader :value
+
+    def initialize(source:)
+      @source = source
+      advance
+    end
+
+    def prepare
+      source.prepare
+    end
+
+    # Given a row from the source, produce the unfolded rows as a result. This method must return
+    # an array.
+    def unfold(row)
+      raise "Unimplemented!"
+    end
+
+    def empty?
+      value.nil? && source.empty?
+    end
+
+    def advance
+      while (unfolded.nil? || unfolded.empty?) && !source.empty?
+        # Use reverse since we want to process in-order, but, #pop is much faster than #unshift
+        @unfolded = unfold(source.row.deep_dup)
+        unfolded.reverse!
+      end
+      if unfolded && !unfolded.empty?
+        @value = unfolded.pop
+      else
+        @value = nil
+      end
+    end
+
+  end
+end

data/lib/arc-furnace/version.rb

@@ -0,0 +1,3 @@
+module ArcFurnace
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,182 @@
+--- !ruby/object:Gem::Specification
+name: arc-furnace
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Daniel Spangenberger
+- Brian Tenggren
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-10-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: eigenclass
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: roo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: ice_nine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.11'
+description: 'An ETL library for Ruby that performs the basic actions of ETL: extract,
+  transform, and load. Easily extensible.'
+email:
+- dan@salsify.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- arc-furnace.gemspec
+- lib/arc-furnace.rb
+- lib/arc-furnace/abstract_join.rb
+- lib/arc-furnace/all_fields_csv_sink.rb
+- lib/arc-furnace/binary_key_merging_hash.rb
+- lib/arc-furnace/block_transform.rb
+- lib/arc-furnace/block_unfold.rb
+- lib/arc-furnace/csv_sink.rb
+- lib/arc-furnace/csv_source.rb
+- lib/arc-furnace/csv_to_hash_with_duplicate_headers.rb
+- lib/arc-furnace/dsl.rb
+- lib/arc-furnace/enumerator_source.rb
+- lib/arc-furnace/error_handler.rb
+- lib/arc-furnace/excel_source.rb
+- lib/arc-furnace/fixed_column_csv_sink.rb
+- lib/arc-furnace/hash.rb
+- lib/arc-furnace/inner_join.rb
+- lib/arc-furnace/logging_error_handler.rb
+- lib/arc-furnace/merging_hash.rb
+- lib/arc-furnace/multi_csv_source.rb
+- lib/arc-furnace/node.rb
+- lib/arc-furnace/nodes.rb
+- lib/arc-furnace/outer_join.rb
+- lib/arc-furnace/pipeline.rb
+- lib/arc-furnace/private_attr.rb
+- lib/arc-furnace/sink.rb
+- lib/arc-furnace/source.rb
+- lib/arc-furnace/suffixed_fixed_column_csv_sink.rb
+- lib/arc-furnace/transform.rb
+- lib/arc-furnace/unfold.rb
+- lib/arc-furnace/version.rb
+homepage: http://github.com/salsify/arc-furnace
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.7
+signing_key: 
+specification_version: 4
+summary: Melds and transforms data from multiple sources into a single stream
+test_files: []