itiel 0.1.0

data/lib/itiel/transform/rename_column.rb

@@ -0,0 +1,43 @@
+module Itiel
+  module Transform
+    #
+    # Renames a column in the data stream
+    #
+    # Usage:
+    #
+    #     @transformer = Itiel::Transform::RenameColumn.new("order_id" => "id")
+    #
+    # This would rename the order_id column in the input stream to id
+    #
+    class RenameColumn
+      include ChainedStep
+      include Itiel::Nameable
+
+      attr_accessor :mappings
+
+      def initialize(*args)
+        self.mappings = args
+      end
+
+      def transform!(input_stream)
+        old_keys = self.mappings.first.keys
+        all_keys = input_stream.first.keys
+
+        transformed_output = []
+        input_stream.each do |object|
+          element = {}
+          all_keys.each do |k|
+            if old_keys.include?(k)
+              element[self.mappings.first[k]] = object[k]
+            else
+              element[k] = object[k]
+            end
+          end
+          transformed_output << element
+        end
+
+        transformed_output
+      end
+    end
+  end
+end

data/lib/itiel/transform/select_column.rb

@@ -0,0 +1,37 @@
+module Itiel
+  module Transform
+    #
+    # This transformation only selects specific columns on the data stream
+    #
+    # Usage:
+    #
+    #     @transformer = Itiel::Transform::SelectColumn.new("order_id", "name")
+    #
+    # In the example, the output stream would only have the order_id and the name column
+    # All other columns will be ignored
+    #
+    class SelectColumn
+      include ChainedStep
+      include Itiel::Nameable
+
+      attr_accessor :mappings
+
+      def input=(stream)
+        next_step.input = transform!(stream)
+      end
+
+      def initialize(*args)
+        self.mappings = args
+      end
+
+      def transform!(input_stream)
+        selected_output = []
+        input_stream.each do |object|
+          selected_output << object.select {|key, value| self.mappings.include? key }
+        end
+
+        selected_output
+      end
+    end
+  end
+end

data/lib/itiel/version.rb

@@ -0,0 +1,3 @@
+module Itiel
+  VERSION = "0.1.0"
+end

data/spec/db/sql_connectable_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe Itiel::DB::SQLConnectable do
+  before :each do
+    class Step
+      include Itiel::DB::SQLConnectable
+    end
+  end
+
+  it "sets and gets connection_file_path" do
+    expect(Step).to respond_to :connection_file_path
+    expect(Step).to respond_to :connection_file_path=
+  end
+
+  it "returns a sequel_connection object based on the connection file" do
+    Step.connection_file_path = File.dirname(__FILE__) + '/../support/config/database.yml'
+    expect(Step.sequel_connection(:test)).to_not be_nil
+  end
+end
+

data/spec/extract/chained_step_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe Itiel::Extract::ChainedStep do
+  before :each do
+    klass = Class.new
+    klass.send(:include, Itiel::Extract::ChainedStep)
+    @step = klass.new
+
+    @stream = double
+  end
+
+  describe "#start" do
+    before :each do
+      @next_step = double
+      @step.next_step = @next_step
+    end
+
+    it "extracts and sends the stream to the next step" do
+      allow(@step).to receive(:extract).and_return(@stream)
+      allow(@next_step).to receive(:input=).and_return(@stream)
+
+      @step.start
+    end
+  end
+
+  describe "#extract" do
+    it "raises an exception" do
+      expect { @step.extract }.to raise_error Itiel::MethodNotImplementedException
+    end
+  end
+end

data/spec/extract/csv_file_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe Itiel::Extract::CSVFile do
+  before :each do
+    @step = Itiel::Extract::CSVFile.new 'FILENAME'
+  end
+
+  describe "#extract" do
+    it "reads a csv file and returns it as a hash to the stream" do
+      row = double
+      allow(row).to receive(:to_hash).and_return({data: true})
+
+      @stream = [ row ]
+      allow(CSV).to receive(:read).
+          with('FILENAME', headers: true).
+          and_return(@stream)
+
+      expect(@step.extract).to eq [{ data: true }]
+    end
+  end
+end
+

data/spec/extract/custom_sql_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe Itiel::Extract::CustomSQL do
+  describe "#extract" do
+    before :each do
+      @step = Itiel::Extract::CustomSQL.new 'SCRIPT'
+      @step.connection = :test
+    end
+
+    it "Runs a script on the database and returns its results to the stream" do
+      result = double
+      allow(result).to receive(:all).and_return double
+      db = { 'SCRIPT' => result }
+      allow(Itiel::Extract::CustomSQL).to receive(:sequel_connection).with(:test).and_return db
+
+      expect(@step.extract).to eq result.all
+    end
+  end
+end

data/spec/extract/database_table_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe Itiel::Extract::DatabaseTable do
+  before :each do
+    @step            = Itiel::Extract::DatabaseTable.new
+    @step.connection = :test
+    @step.table_name = 'table_name'
+  end
+
+  describe "#extract" do
+    it "returns all the rows in the specified database table" do
+      result = double
+      db = { table_name: result }
+      allow(result).to receive(:all).and_return double
+      expect(Itiel::Extract::DatabaseTable).
+          to receive(:sequel_connection).
+          with(:test).and_return db
+
+      expect(@step.extract).to eq result.all
+    end
+  end
+end

data/spec/job_spec.rb

@@ -0,0 +1,80 @@
+require 'spec_helper'
+
+describe Itiel::Job do
+  describe "#run" do
+    it "yields a new instance of Itiel::Job" do
+      Itiel::Job.run { |object| expect(object).to be_instance_of(Itiel::Job) }
+    end
+  end
+
+  describe "#define" do
+    it "returns an instance of Itiel::Job" do
+      expect(Itiel::Job.define).to be_instance_of(Itiel::Job)
+    end
+
+    it "yields a new instance of Itiel::Job" do
+      Itiel::Job.define {|object| assert_instance_of(Itiel::Job, object)}
+    end
+
+    it "sets given block as the block attribute of the instance it returns" do
+      block = Proc.new {}
+      job   = Itiel::Job.define(&block)
+      expect(block).to eq job.block
+    end
+  end
+
+  describe "#run!" do
+    it "calls the block on self.block" do
+      block = Proc.new {}
+      expect(block).to receive(:call)
+      job = Itiel::Job.define(&block)
+      job.run!
+    end
+
+    it "responds to step inside the block" do
+      @object = double
+      instanced_job = Itiel::Job.define do |job|
+        job.step @object
+      end
+
+      expect(instanced_job).to receive(:step).with(@object)
+      instanced_job.run!
+    end
+  end
+
+  describe "#step" do
+    before :each do
+      @job         = Itiel::Job.new
+      @source      = double
+      @destination = double
+      @stream      = double
+
+      allow(@source).to receive(:output).and_return @stream
+      expect(@destination).to receive(:input=).and_return @stream
+    end
+
+    describe "@source => @destination" do
+      it "hardwires the @destination's input with the @source output" do
+        @job.step @source => @destination
+      end
+    end
+
+    describe "step @source; step @destination" do
+      it "hardwires the @destination's input with the @source output" do
+        expect(@destination).to receive(:output)
+        @job.step @source
+        @job.step @destination
+      end
+    end
+
+    describe "step @source => [ @destination, @second_destination ]" do
+      it "hardwires both destination's inputs with the @source output" do
+        @second_destination = double
+        expect(@second_destination).to receive(:input=).and_return @stream
+
+        @job.step @source => [@destination, @second_destination]
+      end
+    end
+  end
+end
+

data/spec/loader/chained_step_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+describe Itiel::Load::ChainedStep do
+  before :each do
+    klass = Class.new
+    klass.send :include, Itiel::Load::ChainedStep
+
+    @step = klass.new
+  end
+
+  it "defines next_step" do
+    expect(@step).to respond_to(:next_step)
+    expect(@step).to respond_to(:next_step=)
+  end
+
+  describe :persist do
+    it "raises an error if undefined" do
+      expect { @step.persist([]) }.to raise_error Itiel::MethodNotImplementedException
+    end
+  end
+
+  describe :input= do
+    before :each do
+      @next_step      = double
+      @input          = [ double ]
+      @step.next_step = @next_step
+
+      expect(Itiel::Logger).to receive(:log_received).with(@step, @input.size)
+      expect(Itiel::Logger).to receive(:log_processed).with(@step, @input.size)
+    end
+
+    it "calls persist with the stream received through input= and sends it to next_step" do
+      expect(@step).to receive(:persist).with @input
+      expect(@next_step).to receive(:input=).with @input
+      @step.input = @input
+    end
+  end
+end
+

data/spec/loader/csv_file_spec.rb

@@ -0,0 +1,69 @@
+require 'spec_helper'
+
+describe Itiel::Load::CSVFile do
+  before :each do
+    @filename = File.expand_path("#{File.dirname(__FILE__)}/../../tmp/output.csv")
+    File.unlink(@filename) if File.exist?(@filename)
+
+    @input = [
+        {
+          "id"   => 1,
+          "name" => "Subject Name"
+        },
+        {
+          "id"   => 2,
+          "name" => "Subject Name"
+        }
+    ]
+
+    @expected_result = [
+        [ "id" , "name"         ] ,
+        [ "1"  , "Subject Name" ] ,
+        [ "2"  , "Subject Name" ]
+    ]
+
+    @csv_output = Itiel::Load::CSVFile.new(@filename)
+  end
+
+  describe "the file does not exist" do
+    before :each do
+      @csv_output.persist(@input)
+      @result = CSV.read(@filename, :headers => true)
+    end
+
+    it "generates a CSV file with the input data" do
+      expect(@expected_result).to eq @result.to_a
+    end
+  end
+
+  describe "the file already exists" do
+    before :each do
+      FileUtils.touch(@filename)
+      @csv_output.persist(@input)
+      @result = CSV.read(@filename)
+    end
+
+    it "appends to the existing data" do
+      expect(@expected_result - [[ "id", "name" ]]).to eq @result
+    end
+  end
+
+  describe "use append = false" do
+    before do
+      @csv_output = Itiel::Load::CSVFile.new(@filename, false)
+    end
+
+    describe "the file already exists" do
+      before :each do
+        FileUtils.touch(@filename)
+        @csv_output.persist(@input)
+        @result = CSV.read(@filename)
+      end
+
+      it "replaces the existing file with the input data" do
+        expect(@expected_result).to eq @result
+      end
+    end
+  end
+end
+

data/spec/loader/database_table_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+describe Itiel::Load::DatabaseTable do
+  before(:each) do
+    @output = Itiel::Load::DatabaseTable.new :test, "users"
+    @input = [
+        {
+          "id"   => "1",
+          "name" => "Some Name"
+        },
+        {
+          "id"   => "2",
+          "name" => "Some Other Name"
+        },
+    ]
+  end
+
+  it "inserts a record for each row" do
+    table = double
+    allow(@output).to receive(:table).and_return table
+
+    @input.each do |row|
+      expect(table).to receive(:insert).and_return row
+    end
+
+    @output.persist(@input)
+  end
+end
+

data/spec/lookup/hash_lookup_spec.rb

@@ -0,0 +1,108 @@
+require 'spec_helper'
+
+describe Itiel::Lookup::HashLookup do
+  before(:each) do
+    @input = [
+      {
+        :id     => 1,
+        :author => "Isaac"
+      },
+      {
+        :id     => 2,
+        :author => "Carver"
+      }
+    ]
+
+    @lookup_source = [
+      {
+        :id     => 1,
+        :name   => "Isaac",
+        :active => "t"
+      },
+      {
+        :id     => 2,
+        :name   => "Carver",
+        :active => "f"
+        }
+    ]
+
+    @lookup_stream = {
+      "Isaac" => {
+        :author_id => 1,
+        :active    => "t"
+      },
+
+      "Carver" => {
+        :author_id => 2,
+        :active    => "f"
+      }
+    }
+
+    @expected_output = [
+      {
+        :id        => 1,
+        :author    => "Isaac",
+        :author_id => 1,
+        :active    => "t"
+      },
+      {
+        :id        => 2,
+        :author    => "Carver",
+        :author_id => 2,
+        :active    => "f"
+      }
+    ]
+
+    class FooLookup
+      include Itiel::Lookup::HashLookup
+    end
+
+    @lookup = FooLookup.new
+    @lookup.lookup_columns = { "author" => "name" }
+    @lookup.joined_columns = { "id" => "author_id", "active" => "active" }
+  end
+
+  describe "#lookup!" do
+    it "joins the data using the specified column" do
+      allow(@lookup).to receive(:lookup_stream).and_return @lookup_stream
+      expect(@expected_output).to eq @lookup.lookup!(@input)
+    end
+  end
+
+  describe "#lookup_stream" do
+    it do
+      allow(@lookup).to receive(:lookup_source).and_return @lookup_source
+      expect(@lookup_stream).to eq @lookup.lookup_stream
+    end
+  end
+
+  describe 'with nil values in the lookup column' do
+    before do
+      @input.append({ id: 3, author: nil })
+      @expected_output.append({ id: 3, author: nil, author_id: nil, active: nil })
+    end
+
+    describe "#lookup!" do
+      it 'joins the data setting the value to nil' do
+        allow(@lookup).to receive(:lookup_stream).and_return @lookup_stream
+        expect(@expected_output).to eq @lookup.lookup!(@input)
+      end
+    end
+  end
+
+  describe "When the lookup column doesn't exist in the input_stream" do
+    before do
+      @input.append({ id: 3 })
+      @expected_output.append({ id: 3, author_id: nil, active: nil })
+    end
+
+    describe "#lookup!" do
+      it 'joins the data setting the value to nil' do
+        allow(@lookup).to receive(:lookup_stream).and_return @lookup_stream
+
+        expect(@expected_output).to eq @lookup.lookup!(@input)
+      end
+    end
+  end
+end
+