itiel 0.1.0

data/lib/itiel/lookup/chained_step.rb

@@ -0,0 +1,35 @@
+module Itiel
+  module Lookup
+    #
+    # This module defines the input and output behavior of Lookup Steps.
+    #
+    # Whenever the instance receives input, it calls lookup! and set its return value
+    # as the input of its next_step if defined.
+    #
+    # All the classes that include this moudle must implement lookup!
+    #
+    module ChainedStep
+      module InstanceMethods
+        attr_accessor :next_step, :output
+        alias :>> :next_step=
+
+        def input=(input_stream)
+          Itiel::Logger.log_received(self, input_stream.size)
+          self.output = lookup!(input_stream)
+          #puts output if table_name == "profiles"
+          #puts input_stream if table_name == "profiles"
+          self.next_step.input = output if next_step
+          Itiel::Logger.log_processed(self, input_stream.size)
+        end
+
+        def lookup!(input_stream)
+          raise "lookup is not implemented"
+        end
+      end
+
+      def self.included(receiver)
+        receiver.send :include, InstanceMethods
+      end
+    end
+  end
+end

data/lib/itiel/lookup/csv_file.rb

@@ -0,0 +1,16 @@
+module Itiel
+  module Lookup
+    class CSVFile
+      include ChainedStep
+      include HashLookup
+
+      def initialize(file_name)
+        @file_name = file_name
+      end
+
+      def lookup_source
+        CSV.table @file_name
+      end
+    end
+  end
+end

data/lib/itiel/lookup/database_table.rb

@@ -0,0 +1,36 @@
+module Itiel
+  module Lookup
+    #
+    # Joins the main data stream with another lookup stream and appends the specified
+    # columns from the lookup stream to the input stream.
+    #
+    # Example:
+    #
+    # Set up a lookup to the *authors* table:
+    #
+    # @database_lookup = Itiel::Lookup::DatabaseTable.new
+    # @database_lookup.connection = :test
+    # @database_lookup.table_name = "authors"
+    #
+    # Join the "author_name" column on the input stream with the "name" column in the lookup stream.
+    #
+    # @database_lookup.lookup_columns = { "author_name" => "name" }
+    #
+    # Join the "id" column in the lookup stream as "author_id"
+    #
+    # @database_lookup.joined_columns = { "id" => "author_id" }
+    #
+    class DatabaseTable
+      include ChainedStep
+      include HashLookup
+      include Itiel::DB::SQLConnectable
+
+      attr_accessor :table_name
+
+      def lookup_source
+        db = self.class.sequel_connection(connection)
+        db[table_name.to_sym.to_sym].all
+      end
+    end
+  end
+end

data/lib/itiel/lookup/hash_lookup.rb

@@ -0,0 +1,35 @@
+module Itiel
+  module Lookup
+    module HashLookup
+      attr_accessor :lookup_columns, :joined_columns
+
+      def lookup!(input_stream)
+        input_stream.collect do |row|
+          origin_column = lookup_columns.first[0].to_sym
+          merge_data    = lookup_stream[row[origin_column]] || empty_joined_columns #{ origin_column => nil }
+          row.merge!(merge_data)
+        end
+      end
+
+      def lookup_stream
+        @lookup ||= lookup_source.inject({}) do |memory, row|
+          clean_row = {}
+          joined_columns.each do |original, target|
+            clean_row[target.to_sym] = row[original.to_sym]
+          end
+
+          memory[row[lookup_columns.first[1].to_sym]] = clean_row
+          memory
+        end
+      end
+
+      def empty_joined_columns
+        joined_columns.inject({}) {|m,h| m[h[1].to_sym] = nil; m }
+      end
+
+      def lookup_source
+        raise 'lookup_source is not implemented'
+      end
+    end
+  end
+end

data/lib/itiel/nameable.rb

@@ -0,0 +1,6 @@
+module Itiel
+  module Nameable
+    attr_accessor :step_name, :debug
+
+  end
+end

data/lib/itiel/script/chained_step.rb

@@ -0,0 +1,18 @@
+module Itiel
+  module Script
+    module ChainedStep
+      attr_accessor :next_step
+
+      alias :>> :next_step=
+
+      def input=(input_stream)
+        self.execute(input_stream)
+        next_step.input = input_stream if next_step
+      end
+
+      def execute(*)
+        raise "execute is not implemented"
+      end
+    end
+  end
+end

data/lib/itiel/script/ruby_script.rb

@@ -0,0 +1,31 @@
+module Itiel
+  module Script
+    #
+    # Process the stream with ruby
+    #
+    # Initialize it with a block that will yield each row of the stream.
+    #
+    # Usage:
+    #
+    #   Itiel::Script::RubyScript.new do |row|
+    #     row["slug"] = row["title"]
+    #   end
+    #
+    class RubyScript
+      include ChainedStep
+
+      attr_accessor :block
+
+      def initialize(&block)
+        self.block = block
+      end
+
+      def execute(input_stream)
+        input_stream.each do |row|
+          block.call(row)
+        end
+      end
+    end
+  end
+end
+

data/lib/itiel/script/sql_script.rb

@@ -0,0 +1,29 @@
+module Itiel
+  module Script
+    #
+    # Executes a SQL script or command on the specified
+    # connection
+    #
+    class SQLScript
+      include ChainedStep
+      include Itiel::DB::SQLConnectable
+
+      attr_accessor :connection
+      attr_accessor :sql
+
+      def initialize(*args)
+        self.sql = args[0]
+      end
+
+      def execute(*)
+        db = self.class.sequel_connection(connection)
+        db << sql
+      end
+
+      def sanity_check
+        raise Itiel::MissingConnection unless self.connection
+        raise Itiel::SQLSentenceNotProvided.new unless self.sql
+      end
+    end
+  end
+end

data/lib/itiel/transform/calculated_columns.rb

@@ -0,0 +1,47 @@
+module Itiel
+  module Transform
+    #
+    # This class allow us to create multiple columns or replace the current value of a column
+    # by the result of the execution of a block, you must return a hash with the columns you
+    # wish to add as the hash keys.
+    #
+    # Example:
+    #
+    # calculated = CalculatedColumns.new do |row|
+    #   total = row['price'] * row['quantity']
+    #   tax   = total * 0.2
+    #   { :total => total, :tax => tax }
+    # end
+    #
+    # calculated.input = [{:price => 12.50, :quantity => 3},{:price => 4.95, :quantity => 5}]
+    # calculated.output
+    # => {:price => 12.50, :quantity => 3, :total => 37.5, :tax => 7.5},{:price => 4.95, :quantity => 5, :total => 24.75, :tax => 4.95}
+    #
+    # It is important to note that you have to use casting on numeric fields
+    # just to make sure that it is from the right type. Stored column types
+    # may vary depending on the source.
+    #
+    class CalculatedColumns
+      include ChainedStep
+      include Itiel::Nameable
+
+      attr_accessor :arguments
+
+      def initialize(&block)
+        raise "Missing block" unless block_given?
+        @block = block
+      end
+
+      def transform!(input_stream)
+        sanity_check
+        input_stream.each do |object|
+          object.merge! @block.call(object)
+        end
+      end
+
+      def sanity_check
+        raise Itiel::UndefinedNextStepException.new "Undefined next_step" unless self.next_step
+      end
+    end
+  end
+end

data/lib/itiel/transform/chained_step.rb

@@ -0,0 +1,27 @@
+module Itiel
+  module Transform
+    #
+    # Defined the behavior of the Transformation
+    # and how they handle the data stream.
+    #
+    # All classes including this module should implement a
+    # transform! method that does the real transformation.
+    #
+    # Than method will be called as soon as the transformation
+    # receives input.
+    #
+    module ChainedStep
+      attr_accessor :next_step
+
+      alias :>> :next_step=
+
+      def input=(stream)
+        next_step.input = transform!(stream)
+      end
+
+      def transform!(stream)
+        raise Itiel::MethodNotImplementedException.new "You need to define the transform! method in the implementing class"
+      end
+    end
+  end
+end

data/lib/itiel/transform/constant_column.rb

@@ -0,0 +1,35 @@
+module Itiel
+  module Transform
+    #
+    # This transformation appends a new Column
+    # with a constant value to the data stream
+    #
+    #     Itiel::Transform::ConstantColumn.new( "column" => "Constant Value")
+    #
+    # Or, specify the column and the value later:
+    #
+    #     transformation = Itiel::Transform::ConstantColumn.new
+    #
+    #     transformation.append = { "column" => "Constant Value" }
+    #
+  	# The resulting stream will have a column named "column" with the value
+  	# "Constant Value" for all rows.
+  	#
+    class ConstantColumn
+      include ChainedStep
+      include Itiel::Nameable
+
+      attr_accessor :append
+
+      def initialize(*args)
+        self.append = args.first
+      end
+
+      def transform!(input_stream)
+        input_stream.collect do |row|
+          row.merge(self.append)
+        end
+      end
+    end
+  end
+end

data/lib/itiel/transform/input_output_behavior.rb

@@ -0,0 +1,44 @@
+module Itiel
+  module Transform
+    #
+    # This module defines the behavior for input and output
+    # on all our transformations.
+    #
+    # Whenever a transformation receives input, it stores the result of
+    # transform! on the output
+    #
+    # All the classes in tiel::Transformation should implement ransform!
+    #
+    module InputOutputBehavior
+      module InstanceMethods
+        def input=(input_stream)
+          Itiel::Logger.log_received(self, input_stream.size)
+          @input = input_stream
+        end
+
+        def input
+          @input
+        end
+
+        #
+        # Returns cached output by default, call it with true to run the
+        # transformation before returning the output
+        #
+        def output(retransform = false)
+          @output = (retransform ? transform!(self.input) : @output ||= transform!(self.input))
+          Itiel::Logger.log_processed(self, @output.size)
+          @output
+        end
+
+        #
+        # This method has to be implemented in the class
+        #
+        def transform!(input_stream) ; end
+      end
+
+      def self.included(receiver)
+        receiver.send :include, InstanceMethods
+      end
+    end
+  end
+end

data/lib/itiel/transform/map_values.rb

@@ -0,0 +1,43 @@
+module Itiel
+  module Transform
+    #
+    # Maps a field value to different values
+    #
+    # Usage:
+    #
+    #      @transformation = Itiel::Transform::MapValues.new(
+    #        {
+    #          "active" => { true => "yes", false => "no" }
+    #        }
+    #      )
+    #
+    # This would map all the values on the active column, true to yes and false to no
+    #
+    class MapValues
+      include ChainedStep
+      include Nameable
+
+      attr_accessor :mapping
+
+      def initialize(mapping)
+        self.mapping = mapping
+      end
+
+      def transform!(input_stream)
+        output = []
+        input_stream.each do |stream_row|
+          new_row = {}
+          stream_row.each do |column, value|
+            if self.mapping.keys.include?(column)
+              new_row[column] = self.mapping[column][value] || value
+            else
+              new_row[column] = value
+            end
+          end
+          output << new_row
+        end
+        output
+      end
+    end
+  end
+end

data/lib/itiel/transform/remove_column.rb

@@ -0,0 +1,33 @@
+module Itiel
+  module Transform
+    #
+    # This transformation only selects specific columns on the data stream
+    #
+    # Usage:
+    #
+    #     @transformer = Itiel::Transform::RemoveColumn.new("order_id")
+    #
+    # In the example, the output stream would not have the order_id column
+    #
+    class RemoveColumn
+      include ChainedStep
+      include Itiel::Nameable
+
+      attr_accessor :mappings
+
+      def initialize(*args)
+        self.mappings = args
+      end
+
+      def transform!(input_stream)
+        selected_output = []
+        input_stream.each do |object|
+          selected_output << object.select {|key, value| !self.mappings.include? key }
+        end
+
+        selected_output
+      end
+    end
+  end
+end
+