itiel 0.1.0

data/spec/nameable_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+describe Itiel::Nameable do
+  before :each do
+    step_class = Class.new do
+      include Itiel::Nameable
+    end
+
+    @step = step_class.new
+  end
+
+  it "responds to name= and name" do
+    expect(@step).to respond_to :step_name
+    expect(@step).to respond_to :step_name=
+  end
+end
+

data/spec/script/chained_step_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+describe Itiel::Script::ChainedStep do
+  before :each do
+    klass = Class.new
+    klass.send :include, Itiel::Script::ChainedStep
+
+    @step = klass.new
+  end
+
+  describe :input= do
+    before :each do
+      @input_stream = double
+      allow(@step).to receive(:next_step).and_return double
+    end
+
+    it "executes with input stream, sets output as next_step input" do
+      allow(@step).to receive(:execute).with(@input_stream)
+      allow(@step.next_step).to receive(:input=).with(@input_stream)
+      @step.input = @input_stream
+    end
+  end
+end
+

data/spec/script/ruby_script_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+describe Itiel::Script::RubyScript do
+  before :each do
+    @step = Itiel::Script::RubyScript.new do |row|
+      row["data"] = "bar"
+    end
+  end
+
+  describe "#execute" do
+    it "iterates through the input_stream and yields to given block" do
+      stream = [ { "data" => "foo" } ]
+      @step.execute(stream)
+      expect(stream).to eq [ { "data" => "bar" } ]
+    end
+  end
+end
+

data/spec/script/sql_script_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+
+describe Itiel::Script::SQLScript do
+  before :each do
+    @sql_script = Itiel::Script::SQLScript.new("DELETE FROM orders")
+    @data_stream = { "c" => "v" }
+  end
+
+  describe :sanity_check do
+    describe "no connection specified" do
+      it "raises Itiel::MissingConnection" do
+        expect { @sql_script.sanity_check }.to raise_error Itiel::MissingConnection
+      end
+    end
+
+    describe "No SQL sentence is given" do
+      before :each do
+        @sql_script.connection = Itiel::DB::Connection.new
+        @sql_script.sql = nil
+      end
+
+      it "raises Itiel::SQLSentenceNotProvided" do
+        expect { @sql_script.sanity_check }.to raise_error Itiel::SQLSentenceNotProvided
+      end
+    end
+  end
+
+  describe :execute do
+    before :each do
+      @connection = double
+      @sql_script.connection = :test
+      allow(Itiel::Script::SQLScript).to receive(:sequel_connection).with(:test).and_return @connection
+    end
+
+    it "Executes the specified SQL script with the given connection" do
+      expect(@connection).to receive(:<<).with(@sql_script.sql)
+      @sql_script.execute
+    end
+  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,24 @@
+require 'simplecov'
+require 'itiel'
+
+SimpleCov.start
+
+TMP_DIR = File.expand_path("#{File.dirname(__FILE__)}/../tmp")
+Dir.mkdir(TMP_DIR) unless File.exist?(TMP_DIR)
+
+log = Logger.new("#{File.dirname(__FILE__)}/../tmp/test.log")
+log.level = Logger::INFO
+Itiel::Logger.logger = log
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |expectations|
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+
+  config.order = :random
+  # config.warnings = true
+end

data/spec/support/config/database.yml

@@ -0,0 +1 @@
+test: 'sqlite://test.db'

data/spec/support/config/sources.yml

@@ -0,0 +1,9 @@
+legacy:
+  adapter: sqlite3
+  database: tmp/source.sqlite3
+  pool: 5
+
+destination:
+  adapter: sqlite3
+  database: tmp/destination.sqlite3
+  pool: 5

data/spec/transform/calculated_columns_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe Itiel::Transform::CalculatedColumns do
+  before(:each) do
+    @transformer = Itiel::Transform::CalculatedColumns.new do |row|
+      total = row['price'] * row['quantity']
+      tax   = total * 0.2
+      { 'total' => total, 'tax' => tax }
+    end
+
+    @transformer.next_step = double
+
+    @input = [
+        { "id" => 1 , "price" => 1.5 , "quantity" => 5 },
+        { "id" => 2 , "price" => 3   , "quantity" => 9 },
+        { "id" => 3 , "price" => 4.5 , "quantity" => 3 }
+    ]
+  end
+
+  it "raises an exception when it's not defined" do
+    @transformer.next_step = nil
+
+    expect { @transformer.transform!(@input) }.to raise_error Itiel::UndefinedNextStepException
+  end
+
+  it "creates new columns and calculate to fill it" do
+    expected_result = [
+        { "id" => 1 , "price" => 1.5 , "quantity" => 5 , "total" => 7.5,  "tax" => 1.5 },
+        { "id" => 2 , "price" => 3   , "quantity" => 9 , "total" => 27,   "tax" => 5.4 },
+        { "id" => 3 , "price" => 4.5 , "quantity" => 3 , "total" => 13.5, "tax" => 2.7 }
+    ]
+
+    expect(@transformer.transform!(@input)).to eq expected_result
+  end
+end
+

data/spec/transform/chained_step_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe Itiel::Transform::ChainedStep do
+  before :each do
+    class Step
+      include Itiel::Transform::ChainedStep
+    end
+
+    @step = Step.new
+  end
+
+  it "defines input=" do
+    expect(@step).to respond_to :input=
+  end
+
+  it "defines setters and getters for next_step" do
+    expect(@step).to respond_to :next_step
+    expect(@step).to respond_to :next_step=
+  end
+
+  it "sets next_step.input as the result of transform!" do
+    stream    = double
+    next_step = double
+
+    expect(@step).to receive(:transform!).with(stream).and_return stream
+    expect(next_step).to receive(:input=).with(stream)
+
+    @step.next_step = next_step
+    @step.input     = stream
+  end
+
+  it "raises an error if transform! is not defined" do
+    expect { @step.transform!(double) }.to raise_error Itiel::MethodNotImplementedException
+  end
+end
+

data/spec/transform/constant_column_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe Itiel::Transform::ConstantColumn do
+  before :each do
+    @input = [
+        { "id" => "1" },
+        { "id" => "2" }
+    ]
+
+    @expected_output = [
+        { "id" => "1", "append" => "value" },
+        { "id" => "2", "append" => "value" }
+    ]
+
+    @transformation = Itiel::Transform::ConstantColumn.new("append" => "value")
+  end
+
+  it "appends a field with a constant value to the stream" do
+    expect(@transformation.transform!(@input)).to eq @expected_output
+  end
+end
+

data/spec/transform/map_values_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe Itiel::Transform::MapValues do
+  before :each do
+    @input = [
+      { "id" => 1 , "active" => true  } ,
+      { "id" => 2 , "active" => false }
+    ]
+
+    @output = [
+      { "id" => 1 , "active" => "yes" } ,
+      { "id" => 2 , "active" => "no"  }
+    ]
+
+    @transformation = Itiel::Transform::MapValues.new(
+      {
+        "active" => { true => "yes", false => "no" }
+      }
+    )
+  end
+
+  it "maps a column depending on it's value" do
+    expect(@output).to eq @transformation.transform!(@input)
+  end
+end
+

data/spec/transform/rename_column_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe Itiel::Transform::RenameColumn do
+  before :each do
+    @transformer = Itiel::Transform::RenameColumn.new(
+        "order_id" => "id",
+        "name" => "client_name"
+    )
+
+    @input = [
+        { "order_id" => 1 , "name" => "bucket"        , "price" => 1.5 },
+        { "order_id" => 2 , "name" => "second bucket" , "price" => 3   }
+    ]
+  end
+
+  it "renames the elements in the input hash" do
+    expected_result = [
+        { "id" => 1 , "client_name" => "bucket"        , "price" => 1.5 },
+        { "id" => 2 , "client_name" => "second bucket" , "price" => 3   }
+    ]
+
+    expect(@transformer.transform!(@input)).to eq expected_result
+  end
+end
+

data/spec/transform/select_column_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe Itiel::Transform::SelectColumn do
+  before :each do
+    @transformer = Itiel::Transform::SelectColumn.new("order_id", "name")
+    @input = [
+         { "order_id" => 1 , "name" => "bucket"        , "price" => 1.5 },
+         { "order_id" => 2 , "name" => "second bucket" , "price" => 3   }
+    ]
+  end
+
+  it "returns only specified fields" do
+    expected_result = [
+        {"order_id" => 1, "name" => "bucket"        } ,
+        {"order_id" => 2, "name" => "second bucket" }
+    ]
+
+    expect(@transformer.transform!(@input)).to eq expected_result
+  end
+end
+

metadata

@@ -0,0 +1,344 @@
+--- !ruby/object:Gem::Specification
+name: itiel
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- David Padilla
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-04-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.5.0
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: turn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: DSL for ETL in Ruby
+email:
+- david@padilla.cc
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".autotest"
+- ".gitignore"
+- ".gitlab-ci.yml"
+- ".rspec"
+- ".ruby-version"
+- ".travis.yml"
+- Gemfile
+- Gemfile.rails.4.0
+- Gemfile.rails.4.1
+- Gemfile.rails.4.2
+- README.markdown
+- Rakefile
+- build.sh
+- features/extract/database_table.feature
+- features/extract/sql_script.feature
+- features/load/database_table_loader.feature
+- features/lookup/csv_file.feature
+- features/lookup/database_table.feature
+- features/script/ruby_script.feature
+- features/step_definitions/csv_steps.rb
+- features/step_definitions/extractor/csv_file_steps.rb
+- features/step_definitions/extractor/custom_sql_steps.rb
+- features/step_definitions/extractor/database_steps.rb
+- features/step_definitions/extractor/database_table_steps.rb
+- features/step_definitions/extractor/extraction_steps.rb
+- features/step_definitions/flow_steps.rb
+- features/step_definitions/loader/csv_file_steps.rb
+- features/step_definitions/loader/database_table_steps.rb
+- features/step_definitions/lookup/lookup_steps.rb
+- features/step_definitions/scripting/ruby_script_steps.rb
+- features/step_definitions/stream_steps.rb
+- features/step_definitions/transformation/calculated_column_steps.rb
+- features/step_definitions/transformation/calculated_columns_steps.rb
+- features/step_definitions/transformation/constant_column_steps.rb
+- features/step_definitions/transformation/map_values_step.rb
+- features/step_definitions/transformation/rename_column_steps.rb
+- features/step_definitions/transformation/select_column_steps.rb
+- features/step_definitions/transformation/single_column_sort_steps.rb
+- features/support/database.yml
+- features/support/env.rb
+- features/transform/transformations.feature
+- itiel.gemspec
+- lib/itiel.rb
+- lib/itiel/db/connection.rb
+- lib/itiel/db/sql_connectable.rb
+- lib/itiel/db/truncator.rb
+- lib/itiel/extract/chained_step.rb
+- lib/itiel/extract/csv_file.rb
+- lib/itiel/extract/custom_sql.rb
+- lib/itiel/extract/database_table.rb
+- lib/itiel/job.rb
+- lib/itiel/load/chained_step.rb
+- lib/itiel/load/csv_file.rb
+- lib/itiel/load/database_table.rb
+- lib/itiel/load/input_output_behavior.rb
+- lib/itiel/logger.rb
+- lib/itiel/lookup/chained_step.rb
+- lib/itiel/lookup/csv_file.rb
+- lib/itiel/lookup/database_table.rb
+- lib/itiel/lookup/hash_lookup.rb
+- lib/itiel/nameable.rb
+- lib/itiel/script/chained_step.rb
+- lib/itiel/script/ruby_script.rb
+- lib/itiel/script/sql_script.rb
+- lib/itiel/transform/calculated_columns.rb
+- lib/itiel/transform/chained_step.rb
+- lib/itiel/transform/constant_column.rb
+- lib/itiel/transform/input_output_behavior.rb
+- lib/itiel/transform/map_values.rb
+- lib/itiel/transform/remove_column.rb
+- lib/itiel/transform/rename_column.rb
+- lib/itiel/transform/select_column.rb
+- lib/itiel/version.rb
+- spec/db/sql_connectable_spec.rb
+- spec/extract/chained_step_spec.rb
+- spec/extract/csv_file_spec.rb
+- spec/extract/custom_sql_spec.rb
+- spec/extract/database_table_spec.rb
+- spec/job_spec.rb
+- spec/loader/chained_step_spec.rb
+- spec/loader/csv_file_spec.rb
+- spec/loader/database_table_spec.rb
+- spec/lookup/hash_lookup_spec.rb
+- spec/nameable_spec.rb
+- spec/script/chained_step_spec.rb
+- spec/script/ruby_script_spec.rb
+- spec/script/sql_script_spec.rb
+- spec/spec_helper.rb
+- spec/support/config/database.yml
+- spec/support/config/sources.yml
+- spec/transform/calculated_columns_spec.rb
+- spec/transform/chained_step_spec.rb
+- spec/transform/constant_column_spec.rb
+- spec/transform/map_values_spec.rb
+- spec/transform/rename_column_spec.rb
+- spec/transform/select_column_spec.rb
+homepage: https://gitlab.com/dabit/itiel
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: itiel
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: DSL for ETL in Ruby
+test_files:
+- features/extract/database_table.feature
+- features/extract/sql_script.feature
+- features/load/database_table_loader.feature
+- features/lookup/csv_file.feature
+- features/lookup/database_table.feature
+- features/script/ruby_script.feature
+- features/step_definitions/csv_steps.rb
+- features/step_definitions/extractor/csv_file_steps.rb
+- features/step_definitions/extractor/custom_sql_steps.rb
+- features/step_definitions/extractor/database_steps.rb
+- features/step_definitions/extractor/database_table_steps.rb
+- features/step_definitions/extractor/extraction_steps.rb
+- features/step_definitions/flow_steps.rb
+- features/step_definitions/loader/csv_file_steps.rb
+- features/step_definitions/loader/database_table_steps.rb
+- features/step_definitions/lookup/lookup_steps.rb
+- features/step_definitions/scripting/ruby_script_steps.rb
+- features/step_definitions/stream_steps.rb
+- features/step_definitions/transformation/calculated_column_steps.rb
+- features/step_definitions/transformation/calculated_columns_steps.rb
+- features/step_definitions/transformation/constant_column_steps.rb
+- features/step_definitions/transformation/map_values_step.rb
+- features/step_definitions/transformation/rename_column_steps.rb
+- features/step_definitions/transformation/select_column_steps.rb
+- features/step_definitions/transformation/single_column_sort_steps.rb
+- features/support/database.yml
+- features/support/env.rb
+- features/transform/transformations.feature
+- spec/db/sql_connectable_spec.rb
+- spec/extract/chained_step_spec.rb
+- spec/extract/csv_file_spec.rb
+- spec/extract/custom_sql_spec.rb
+- spec/extract/database_table_spec.rb
+- spec/job_spec.rb
+- spec/loader/chained_step_spec.rb
+- spec/loader/csv_file_spec.rb
+- spec/loader/database_table_spec.rb
+- spec/lookup/hash_lookup_spec.rb
+- spec/nameable_spec.rb
+- spec/script/chained_step_spec.rb
+- spec/script/ruby_script_spec.rb
+- spec/script/sql_script_spec.rb
+- spec/spec_helper.rb
+- spec/support/config/database.yml
+- spec/support/config/sources.yml
+- spec/transform/calculated_columns_spec.rb
+- spec/transform/chained_step_spec.rb
+- spec/transform/constant_column_spec.rb
+- spec/transform/map_values_spec.rb
+- spec/transform/rename_column_spec.rb
+- spec/transform/select_column_spec.rb