itiel 0.1.0

data/lib/itiel/db/truncator.rb

@@ -0,0 +1,30 @@
+module Itiel
+  module DB
+    #
+    # Truncates specified tables
+    #
+    # Usage:
+    #
+    #     @truncator = Itiel::DB::Truncator.new "tables", "to", "truncate"
+    #     @truncator.connection = :database
+    #     @truncator.truncate!
+    #
+    class Truncator
+      include Itiel::Nameable
+      include Itiel::DB::SQLConnectable
+
+      attr_accessor :tables
+
+      def initialize(*tables)
+        self.tables = tables
+      end
+
+      def truncate!
+        tables.each do |table|
+          db = self.class.sequel_connection(connection)
+          db[table.to_sym].truncate
+        end
+      end
+    end
+  end
+end

data/lib/itiel/extract/chained_step.rb

@@ -0,0 +1,22 @@
+module Itiel
+  module Extract
+    #
+    # Defines how the initial extractors behave
+    #
+    # All classes including this module must define the in_batches method
+    #
+    module ChainedStep
+      attr_accessor :next_step
+
+      alias :>> :next_step=
+
+      def start
+        self.next_step.input = extract
+      end
+
+      def extract
+        raise Itiel::MethodNotImplementedException.new "extract is not implemented"
+      end
+    end
+  end
+end

data/lib/itiel/extract/csv_file.rb

@@ -0,0 +1,31 @@
+require 'csv'
+
+module Itiel
+  module Extract
+    #
+    # Extracts all specified CSV file rows and sends it in batches to
+    # its next step
+    #
+    # Usage:
+    #
+    #    csv_file            = Itiel::Extract::CSVFile.new('FileName.csv')
+    #    csv_file.batch_size = 15
+    #    csv.file.start
+    #
+    class CSVFile
+      include ChainedStep
+      include Itiel::Nameable
+
+      attr_accessor :file_name
+
+      def initialize(file_name)
+        self.file_name = file_name
+      end
+
+      def extract
+        lines = CSV.read(self.file_name, :headers => true)
+        lines.collect(&:to_hash)
+      end
+    end
+  end
+end

data/lib/itiel/extract/custom_sql.rb

@@ -0,0 +1,38 @@
+require 'sequel'
+
+module Itiel
+  module Extract
+    #
+    # Creates a stream from the specified SQL query.
+    # Connection must be defined on a file that's on
+    # config/database.yml by default.
+    #
+    # Usage:
+    #
+    #     @custom_sql = Itiel::Extract::CustomSQL.new
+    #     @custom_sql.connection = :test
+    #     @custom_sql.script = 'SELECT * FROM some_table'
+    #
+    #
+    # You can set a different path for the config file at class level
+    #
+    #     Itiel::Extract::CustomSQL.connection_file_path = 'path_to_my_config/database.yml'
+    #
+    class CustomSQL
+      include ChainedStep
+      include Itiel::DB::SQLConnectable
+      include Itiel::Nameable
+
+      attr_accessor :script
+
+      def initialize(*args)
+        self.script = args[0]
+      end
+
+      def extract
+        db = self.class.sequel_connection(connection)
+        db[script].all
+      end
+    end
+  end
+end

data/lib/itiel/extract/database_table.rb

@@ -0,0 +1,23 @@
+module Itiel
+  module Extract
+    #
+    # Extracts all the contents from a Database table into the stream
+    # and passes it on to it's next_step
+    #
+    # Usage:
+    #
+    #     @extractor            = Itiel::Extract::DatabaseTable.new
+    #     @extractor.connection = :test
+    #     @extractor.table_name = 'test_table'
+    #
+    #
+    class DatabaseTable < CustomSQL
+      attr_accessor :table_name
+
+      def extract
+        db = self.class.sequel_connection(connection)
+        db[table_name.to_sym].all
+      end
+    end
+  end
+end

data/lib/itiel/job.rb

@@ -0,0 +1,116 @@
+module Itiel
+  class Job
+    attr_accessor :block
+    #
+    # Pass a block to this method to process the ETL steps in order
+    #
+    #
+    # Understands a single line per step:
+    #
+    #     Itiel::Job.run do |job|
+    #       job.step @source
+    #       job.step @destination
+    #
+    #       ...
+    #     end
+    #
+    #
+    # In this case the @source.output is sent to the @destination's input
+    # A third step on the list would send the @destination's output to its
+    # input
+    #
+    #
+    # Another way to do this is by passing hashes to the step method:
+    #
+    #     Itiel::Job.run do |job|
+    #       job.step @source => @destination
+    #     end
+    #
+    #
+    # You want to use this sintax when creating more complex flows. For example,
+    # you could send a step output to several inputs
+    #
+    #     Itiel::Job.run do |job|
+    #       job.step @source => [ @destination, @second_destination ]
+    #     end
+    #
+    def self.run(&block)
+      Itiel::Logger.log_start_job(self)
+      yield self.new
+      Itiel::Logger.log_end_job(self)
+    end
+
+    #
+    # Use it to define the job steps and then run the job at a later date.
+    #
+    # It returns an instance of a job, you can call run! on that instance later
+    # to actually run the steps defined on the block of the job
+    #
+    # Uses the same syntax as run:
+    #
+    #     job = Itiel::Job.define do |job|
+    #       job.step @source => @destination
+    #     end
+    #
+    #     job.run!
+    #
+    # Or
+    #
+    #     job = Itiel::Job.define do |job|
+    #       job.step @source
+    #       job.step @destination
+    #
+    #       ...
+    #     end
+    #
+    #     job.run!
+    #
+    def self.define(&block)
+      self.new(&block)
+    end
+
+    #
+    # Use it to run the steps on a previously defined Job
+    #
+    def run!
+      Itiel::Logger.log_start_job(self)
+      self.block.call(self)
+      Itiel::Logger.log_end_job(self)
+    end
+
+    #
+    # Call inside the run block to denote a data flow
+    #
+    def step(*args)
+      if args[0].is_a?(Hash)
+        hash_based_step(args[0])
+      else
+        single_line_step(args[0])
+      end
+    end
+
+    private
+    def initialize(&block)
+      self.block = block
+      super
+    end
+
+    def hash_based_step(*args)
+      source, destination = args[0].first.to_a
+      if destination.is_a?(Array)
+        destination.each { |object| object.input = source.output }
+      else
+        destination.input = source.output
+      end
+    end
+
+    def single_line_step(object)
+      unless @stream
+        @stream = object.output
+      else
+        object.input = @stream
+        @stream = object.output
+      end
+    end
+  end
+end

data/lib/itiel/load/chained_step.rb

@@ -0,0 +1,37 @@
+module Itiel
+  module Load
+    #
+    # This module defines the input and output behavior for Loader steps
+    #
+    # Whenever the instance receives input, it calls persist! and then
+    # wires the input and the output
+    #
+    # All the clasess in Itiel::Output should implement persist!
+    #
+    module ChainedStep
+      module InstanceMethods
+        attr_accessor :next_step
+
+        alias :>> :next_step=
+
+        def input=(input_stream)
+          Itiel::Logger.log_received(self, input_stream.size)
+          persist(input_stream)
+          self.next_step.input = input_stream if next_step
+          Itiel::Logger.log_processed(self, input_stream.size)
+        end
+
+        #
+        # This method must be implemented in the class
+        #
+        def persist(input_stream)
+          raise Itiel::MethodNotImplementedException.new "persist is not implemented"
+        end
+      end
+
+      def self.included(receiver)
+        receiver.send :include, InstanceMethods
+      end
+    end
+  end
+end

data/lib/itiel/load/csv_file.rb

@@ -0,0 +1,45 @@
+require 'csv'
+
+module Itiel
+  module Load
+    #
+    # Loads the data stream into a CSV file
+    #
+    # Usage:
+    #
+    #     @csv_file = Itiel::Load::CSVFile.new('filename.csv')
+    #     @csv_file.input = []
+    #
+    class CSVFile
+      include ChainedStep
+      include Itiel::Nameable
+
+      def initialize(file_name, append=true)
+        @append    = append
+        @file_name = file_name
+      end
+
+      def persist(input_stream)
+        headers = input_stream.collect(&:keys).flatten.uniq
+        mode    = @append ? "ab" : "w"
+        skip_headers = skip_headers?
+
+        CSV.open(@file_name, mode) do |csv|
+          csv << headers unless skip_headers
+          input_stream.each do |row|
+            csv_row = []
+            headers.each do |h|
+              csv_row << row[h]
+            end
+            csv << csv_row
+          end
+        end
+      end
+
+      private
+      def skip_headers?
+        File.exist?(@file_name) && @append
+      end
+    end
+  end
+end

data/lib/itiel/load/database_table.rb

@@ -0,0 +1,34 @@
+module Itiel
+  module Load
+    #
+    # Loads the stream into a database table.
+    #
+    # Usage:
+    #
+    #     @loader = Itiel::Load::DatabaseTable.new :connection, "table_name"
+    #
+    class DatabaseTable
+      include ChainedStep
+      include Itiel::Nameable
+      include Itiel::DB::SQLConnectable
+
+      attr_accessor :table_name
+
+      def initialize(connection, table_name)
+        self.connection = connection
+        self.table_name = table_name
+      end
+
+      def persist(input_stream)
+        input_stream.each do |element|
+          table.insert(element)
+        end
+      end
+
+      def table
+        @@db ||= self.class.sequel_connection(connection)
+        @@db[table_name.to_sym]
+      end
+    end
+  end
+end

data/lib/itiel/load/input_output_behavior.rb

@@ -0,0 +1,36 @@
+module Itiel
+  module Load
+    #
+    # This module defines the input and output behavior for Loader steps
+    #
+    # Whenever an Output receives input, it calls persist! and then
+    # wires the input and the output
+    #
+    # All the clasess in Itiel::Output should implement persist!
+    #
+    module ChainedStep
+      module InstanceMethods
+        def input=(input_stream)
+          Itiel::Logger.log_received(self, input_stream.size)
+          persist(input_stream)
+          self.next_step = input_stream if self.next_step
+          Itiel::Logger.log_processed(self, input_stream.size)
+        end
+
+        #
+        # This method has to be implemented in the class
+        #
+        def persist(input_stream)
+          raise "persist is not implemented"
+        end
+
+        private
+        attr_writer :output
+      end
+
+      def self.included(receiver)
+        receiver.send :include, InstanceMethods
+      end
+    end
+  end
+end

data/lib/itiel/logger.rb

@@ -0,0 +1,47 @@
+module Itiel
+  class Logger
+    class << self
+      attr_accessor :logger
+    end
+
+    def self.log_received(object, size)
+      msg = "#{object_name(object)} received #{size}.rows"
+      self.log_with_time(msg)
+    end
+
+    def self.log_processed(object, size)
+      msg = "#{object_name(object)} processed #{size}.rows"
+      self.log_with_time(msg)
+    end
+
+    def self.log_start_job(object)
+      msg = "#{object_name(object)} run at #{Time.now}"
+      self.enclosed_with_time(msg)
+    end
+
+    def self.log_end_job(object)
+      msg = "#{object_name(object)} finished at #{Time.now}"
+      self.enclosed_with_time(msg)
+    end
+
+    private
+    def self.object_name(object)
+      ( object.respond_to?(:name) ? object.name : object.class.name )
+    end
+
+    def self.info(message)
+      self.logger ||= ::Logger.new(STDOUT)
+      self.logger.info(message)
+    end
+
+    def self.log_with_time(msg)
+      self.info("#{Time.now} - #{msg}")
+    end
+
+    def self.enclosed_with_time(msg)
+      self.info("\n==================================================\n")
+      self.info(msg)
+      self.info("\n==================================================\n")
+    end
+  end
+end