arc-furnace 0.1.0

data/lib/arc-furnace/csv_sink.rb

@@ -0,0 +1,21 @@
+require 'arc-furnace/sink'
+
+module ArcFurnace
+  class CSVSink < Sink
+    private_attr_reader :csv, :fields
+
+    def initialize(filename: , fields: , encoding: 'UTF-8', force_quotes: false)
+      @csv = CSV.open(filename, 'wb', encoding: encoding, headers: true, force_quotes: force_quotes)
+      @fields = fields
+      csv << fields
+    end
+
+    def finalize
+      csv.close
+    end
+
+    def row(hash)
+      csv << fields.map { |field_id| hash[field_id] }
+    end
+  end
+end

data/lib/arc-furnace/csv_source.rb

@@ -0,0 +1,33 @@
+require 'arc-furnace/source'
+require 'arc-furnace/csv_to_hash_with_duplicate_headers'
+require 'csv'
+
+module ArcFurnace
+  class CSVSource < Source
+    include CSVToHashWithDuplicateHeaders
+    private_attr_reader :csv, :file
+    attr_reader :value
+
+    def initialize(filename: , encoding: 'UTF-8')
+      @file = File.open(filename, encoding: encoding)
+      @csv = CSV.new(file, encoding: encoding, headers: true).each
+      advance
+    end
+
+    # Is this source empty?
+    def empty?
+      !value
+    end
+
+    def advance
+      @value =
+        begin
+          csv_to_hash_with_duplicates(csv.next) if csv
+        rescue StopIteration
+          file.close
+          @csv = nil
+          nil
+        end
+    end
+  end
+end

data/lib/arc-furnace/csv_to_hash_with_duplicate_headers.rb

@@ -0,0 +1,19 @@
+module ArcFurnace
+  module CSVToHashWithDuplicateHeaders
+    def csv_to_hash_with_duplicates(row)
+      result = {}
+      row.each do |column, value|
+        unless value.nil?
+          existing_value = result[column]
+          result[column] =
+            if existing_value
+              Array.wrap(existing_value) + [ value ]
+            else
+              value
+            end
+        end
+      end
+      result
+    end
+  end
+end

data/lib/arc-furnace/dsl.rb

@@ -0,0 +1,6 @@
+require 'arc-furnace/pipeline'
+
+# Backwards-compatibility
+module ArcFurnace
+  DSL = Pipeline
+end

data/lib/arc-furnace/enumerator_source.rb

@@ -0,0 +1,36 @@
+require 'arc-furnace/source'
+
+module ArcFurnace
+  class EnumeratorSource < Source
+
+    private_attr_reader :enumerator
+    attr_reader :value
+
+    def initialize
+      @enumerator = build_enumerator
+      advance
+    end
+
+    # Is this source empty?
+    def empty?
+      !value
+    end
+
+    def advance
+      @value =
+        begin
+          enumerator.next if enumerator
+        rescue StopIteration
+          @enumerator = nil
+          nil
+        end
+    end
+
+    protected
+
+    # Return the enumerator
+    def build_enumerator
+      raise "Unimplemented!"
+    end
+  end
+end

data/lib/arc-furnace/error_handler.rb

@@ -0,0 +1,27 @@
+require 'arc-furnace/source'
+
+module ArcFurnace
+  class ErrorHandler
+
+    # Called during a join operation when a source row is missing a value for the join key.
+    def missing_join_key(source_row:, node_id:)
+      # nothing
+    end
+
+    # Called during a join operation when the hash is missing a value for the join key.
+    def missing_hash_key(key:, source_row:, node_id:)
+      # nothing
+    end
+
+    # Called when a hash node is missing a primary key during the build process.
+    def missing_primary_key(source_row:, node_id:)
+      # nothing
+    end
+
+    # Called when a hash node has duplicate source rows
+    def duplicate_primary_key(duplicate_row:, key:, node_id:)
+      # nothing
+    end
+
+  end
+end

data/lib/arc-furnace/excel_source.rb

@@ -0,0 +1,38 @@
+require 'arc-furnace/enumerator_source'
+require 'roo'
+
+module ArcFurnace
+  class ExcelSource < EnumeratorSource
+
+    private_attr_reader :excel, :enumerator
+    attr_reader :value
+
+    def initialize(filename: )
+      @excel = Roo::Excelx.new(filename)
+      super()
+    end
+
+    def close
+      @excel.close if @excel
+    end
+
+    def build_enumerator
+      header_row = excel.row(1)
+
+      last_row_index = excel.last_row
+      current_row_index = 2
+
+      Enumerator.new do |yielder|
+        until current_row_index > last_row_index
+          row = header_row.each_with_object(::Hash.new).each_with_index do |(header, result), index|
+            value = excel.cell(current_row_index, index + 1)
+            coerced_value = (value.is_a?(String) ? value : excel.excelx_value(current_row_index, index + 1)).try(:to_s).try(:strip)
+            result[header] = coerced_value unless coerced_value.blank?
+          end
+          current_row_index += 1
+          yielder << row
+        end
+      end
+    end
+  end
+end

data/lib/arc-furnace/fixed_column_csv_sink.rb

@@ -0,0 +1,37 @@
+require 'arc-furnace/sink'
+
+module ArcFurnace
+  class FixedColumnCSVSink < Sink
+    private_attr_reader :fields, :csv
+
+    # Expects filename to a filename to open the csv
+    # Expects fields to a hash of Column name => column count
+    def initialize(filename: , fields: , encoding: 'UTF-8', force_quotes: false)
+      @csv = CSV.open(filename, 'wb', encoding: encoding, headers: true, force_quotes: force_quotes)
+      @fields = fields
+      write_header
+    end
+
+    def write_header
+      csv << fields.each_with_object([]) do |(key, count), result|
+        count.times { result << key }
+      end
+    end
+
+    def finalize
+      csv.close
+    end
+
+    def row(hash)
+      row = []
+      fields.each do |column_name, count|
+        values = Array.wrap(hash[column_name])
+        (values.slice(0, count) || []).each do |value|
+          row << value
+        end
+        (count - values.length).times { row << nil }
+      end
+      csv << row
+    end
+  end
+end

data/lib/arc-furnace/hash.rb

@@ -0,0 +1,41 @@
+require 'arc-furnace/node'
+
+module ArcFurnace
+  class Hash < Node
+    attr_reader :key_column
+    private_attr_reader :source, :hash
+
+    def initialize(source: , key_column:)
+      @source = source
+      @key_column = key_column
+      @hash = {}
+    end
+
+    # Pass a block that accepts two argument, the join key
+    # for each value and the value
+    def each(&block)
+      hash.each(&block)
+    end
+
+    def prepare
+      loop do
+        break if source.empty?
+        row = source.row
+        key = row[key_column]
+        if key
+          if hash.include?(key)
+            error_handler.duplicate_primary_key(duplicate_row: row, key: key, node_id: node_id)
+          end
+          hash[key] = row
+        else
+          error_handler.missing_primary_key(source_row: row, node_id: node_id)
+        end
+      end
+    end
+
+    def get(primary_key)
+      hash[primary_key]
+    end
+
+  end
+end

data/lib/arc-furnace/inner_join.rb

@@ -0,0 +1,27 @@
+require 'arc-furnace/abstract_join'
+
+module ArcFurnace
+  # Perform a join between a hash and a source, only producing rows
+  # from the source that match a row from the hash. The resulting row
+  # will merge the source "into" the hash, that is, values from the
+  # source that share the same keys will overwrite values in the hash
+  # value for the corresponding source row.
+  #
+  # Example:
+  # Source row { id: "foo", key1: "boo", key2: "bar" }
+  # Matching hash row { id: "foo", key1: "bar", key3: "baz" }
+  # Result row: { id: "foo", key1: "boo", key2: "bar", key3: "baz" }
+  class InnerJoin < AbstractJoin
+
+    def advance
+      loop do
+        @value = source.row
+        break if value.nil?
+        if merge_source_row(value)
+          break
+        end
+      end
+    end
+
+  end
+end

data/lib/arc-furnace/merging_hash.rb

@@ -0,0 +1,41 @@
+module ArcFurnace
+  # This allows one to merge multiple rows into one such as:
+  #   key, value
+  #     1, foo
+  #     1, bar
+  #     2, baz
+  # Results in:
+  #   1 => { key => 1, value: [foo, bar] }
+  #   2 => { key => 2, value: baz }
+  class MergingHash < ::ArcFurnace::Hash
+    private_attr_reader :source, :hash
+
+    def prepare
+      loop do
+        break if source.empty?
+        row = source.row
+        row_key = row[key_column]
+        if row_key
+          row_entry = hash[row_key] ||= {}
+          row.each do |column, values|
+            existing_column_values = row_entry[column]
+            if existing_column_values && column != key_column
+              if existing_column_values.is_a?(Array)
+                existing_column_values.concat(Array.wrap(values))
+              else
+                new_row_entry = Array.wrap(existing_column_values)
+                new_row_entry.concat(Array.wrap(values))
+                row_entry[column] = new_row_entry
+              end
+            else
+              unless values.nil?
+                row_entry[column] = values.dup
+              end
+            end
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/arc-furnace/multi_csv_source.rb

@@ -0,0 +1,50 @@
+require 'arc-furnace/source'
+require 'arc-furnace/csv_to_hash_with_duplicate_headers'
+require 'csv'
+
+module ArcFurnace
+  class MultiCSVSource < Source
+    include CSVToHashWithDuplicateHeaders
+    private_attr_reader :csv, :file, :filenames, :encoding
+    attr_reader :value
+
+    def initialize(filenames: , encoding: 'UTF-8')
+      @encoding = encoding
+      @filenames = filenames.reverse
+      open_next_file
+    end
+
+    # Is this source empty?
+    def empty?
+      !value
+    end
+
+    def advance
+      advance_in_current_file || open_next_file
+    end
+
+    private
+
+    def advance_in_current_file
+      @value =
+          begin
+            csv_to_hash_with_duplicates(csv.next)
+          rescue StopIteration
+            nil
+          end
+      value
+    end
+
+    def open_next_file
+      file.close if file
+      @file = nil
+      if filenames.empty?
+        nil
+      else
+        @file = File.open(filenames.pop, encoding: encoding)
+        @csv = CSV.new(file, encoding: encoding, headers: true).each
+        advance_in_current_file || open_next_file
+      end
+    end
+  end
+end

data/lib/arc-furnace/node.rb

@@ -0,0 +1,5 @@
+module ArcFurnace
+  class Node
+    attr_accessor :error_handler, :node_id
+  end
+end

data/lib/arc-furnace/nodes.rb

@@ -0,0 +1,19 @@
+require 'arc-furnace/all_fields_csv_sink'
+require 'arc-furnace/binary_key_merging_hash'
+require 'arc-furnace/block_transform'
+require 'arc-furnace/block_unfold'
+require 'arc-furnace/csv_sink'
+require 'arc-furnace/csv_source'
+require 'arc-furnace/enumerator_source'
+require 'arc-furnace/fixed_column_csv_sink'
+require 'arc-furnace/hash'
+require 'arc-furnace/inner_join'
+require 'arc-furnace/merging_hash'
+require 'arc-furnace/multi_csv_source'
+require 'arc-furnace/node'
+require 'arc-furnace/outer_join'
+require 'arc-furnace/sink'
+require 'arc-furnace/source'
+require 'arc-furnace/suffixed_fixed_column_csv_sink'
+require 'arc-furnace/transform'
+require 'arc-furnace/unfold'

data/lib/arc-furnace/outer_join.rb

@@ -0,0 +1,14 @@
+require 'arc-furnace/abstract_join'
+
+module ArcFurnace
+  class OuterJoin < AbstractJoin
+
+    def advance
+      @value = source.row
+      unless value.nil?
+        merge_source_row(value)
+      end
+    end
+
+  end
+end

data/lib/arc-furnace/pipeline.rb

@@ -0,0 +1,158 @@
+require 'eigenclass'
+require 'arc-furnace/nodes'
+require 'arc-furnace/error_handler'
+
+module ArcFurnace
+  class Pipeline
+
+    eattr_accessor :sink_node, :sink_source, :intermediates_map
+    @intermediates_map = {}
+
+    # Ensure that subclasses don't overwrite the parent's transform
+    # node definitions
+    def self.inherited(subclass)
+      subclass.intermediates_map = intermediates_map.dup
+    end
+
+    # Define the sink for this transformation. Only a single sink may be
+    # specified per transformation. The sink is delivered a hash per row or
+    # entity, and feeds them from the graph of nodes above it.
+    def self.sink(type: , source:, params:)
+      if sink_node
+        raise 'Sink already defined!'
+      end
+
+      @sink_node = -> do
+        type.new(resolve_parameters(params))
+      end
+      @sink_source = source
+    end
+
+    # Define a hash node, processing all rows from it's source and caching them
+    # in-memory.
+    def self.hash_node(name, type: ArcFurnace::Hash, params:)
+      define_intermediate(name, type: type, params: params)
+    end
+
+    # A source that has row semantics, delivering a hash per row (or per entity)
+    # for the source.
+    def self.source(name, type:, params:)
+      raise "Source #{type} is not a Source!" unless type <= Source
+      define_intermediate(name, type: type, params: params)
+    end
+
+    # Define an inner join node where rows from the source are dropped
+    # if an associated entity is not found in the hash for the join key
+    def self.inner_join(name, type: ArcFurnace::InnerJoin, params:)
+      define_intermediate(name, type: type, params: params)
+    end
+
+    # Define an outer join nod  e where rows from the source are kept
+    # even if an associated entity is not found in the hash for the join key
+    def self.outer_join(name, type: ArcFurnace::OuterJoin, params:)
+      define_intermediate(name, type: type, params: params)
+    end
+
+    # Define a node that transforms rows. By default you get a BlockTransform
+    # (and when this metaprogramming method is passed a block) that will be passed
+    # a hash for each row. The result of the block becomes the row for the next
+    # downstream node.
+    def self.transform(name, type: BlockTransform, params: {}, &block)
+      if block
+        params[:block] = block
+      end
+      raise "Transform #{type} is not a Transform!" unless type <= Transform
+      define_intermediate(name, type: type, params: params)
+    end
+
+    # Define a node that unfolds rows. By default you get a BlocUnfold
+    # (and when this metaprogramming method is passed a block) that will be passed
+    # a hash for each row. The result of the block becomes the set of rows for the next
+    # downstream node.
+    def self.unfold(name, type: BlockUnfold, params: {}, &block)
+      if block
+        params[:block] = block
+      end
+      raise "Unfold #{type} is not an Unfold!" unless type <= Unfold
+      define_intermediate(name, type: type, params: params)
+    end
+
+    # Create an instance to run a transformation, passing the parameters to
+    # instantiate the transform instance with. The resulting class instance
+    # will have a single public method--#execute, which will perform the
+    # transformation.
+    def self.instance(params = {})
+      DSLInstance.new(self, params)
+    end
+
+    private
+
+    def self.define_intermediate(name, type:, params:)
+      intermediates_map[name] = -> do
+        type.new(resolve_parameters(params))
+      end
+    end
+
+    class DSLInstance
+      attr_reader :sink_node, :sink_source, :intermediates_map, :params, :dsl_class, :error_handler
+
+      def initialize(dsl_class, error_handler: ErrorHandler.new, **params)
+        @dsl_class = dsl_class
+        @params = params
+        @intermediates_map = {}
+        @error_handler = error_handler
+      end
+
+      def execute
+        build
+        prepare
+        run
+      end
+
+      private
+
+      def run
+        while (row = sink_source.row)
+          sink_node.row(row)
+        end
+        sink_node.finalize
+      end
+
+      def prepare
+        intermediates_map.each do |node_id, instance|
+          instance.error_handler = error_handler
+          instance.node_id = node_id
+          instance.prepare
+        end
+        sink_node.prepare
+      end
+
+      def build
+        dsl_class.intermediates_map.each do |key, instance|
+          intermediates_map[key] = instance_exec(&instance) if instance
+        end
+        @sink_node = instance_exec(&dsl_class.sink_node)
+        @sink_source = intermediates_map[dsl_class.sink_source]
+      end
+
+      def resolve_parameters(params_to_resolve)
+        params_to_resolve.each_with_object({}) do |(key, value), result|
+          result[key] =
+            if value.is_a?(Symbol)
+              # Allow resolution of intermediates
+              resolve_parameter(value)
+            elsif value.nil?
+              resolve_parameter(key)
+            else
+              value
+            end
+        end
+      end
+
+      def resolve_parameter(key)
+        self.params[key] || self.intermediates_map[key] || (raise "Unknown key #{key}!")
+      end
+
+    end
+  end
+end