itiel 0.1.0

data/features/step_definitions/extractor/csv_file_steps.rb

@@ -0,0 +1,3 @@
+When /^I create a Extractor::CSVFile object with "([^"]*)"$/ do |filename|
+  @source = Itiel::Extract::CSVFile.new File.join("tmp", filename)
+end

data/features/step_definitions/extractor/custom_sql_steps.rb

@@ -0,0 +1,6 @@
+When /^I create a Itiel::Extract::CustomSQL object with the query:$/ do |string|
+  @custom_sql = Itiel::Extract::CustomSQL.new string
+  Itiel::Extract::CustomSQL.connection_file_path = @database_config_path
+  @custom_sql.connection = :test
+  @extraction = @custom_sql.extract
+end

data/features/step_definitions/extractor/database_steps.rb

@@ -0,0 +1,27 @@
+Given /^an? (.+) database table with rows:$/ do |table_name, table|
+  step "a #{table_name} database table"
+
+  rows = Itiel::Extract::CustomSQL.sequel_connection(:test)[table_name.to_sym]
+  table.hashes.each do |row|
+    rows.insert row
+  end
+end
+
+Given /^a blogposts database table$/ do
+  Itiel::Extract::CustomSQL.connection_file_path = @database_config_path
+
+  Itiel::Extract::CustomSQL.sequel_connection(:test).create_table :blogposts do
+    Integer :id
+    String :title
+    String :author
+  end
+end
+
+Given /^an? authors database table$/ do
+  Itiel::Extract::CustomSQL.connection_file_path = @database_config_path
+
+  Itiel::Extract::CustomSQL.sequel_connection(:test).create_table :authors do
+    Integer :id
+    String :name
+  end
+end

data/features/step_definitions/extractor/database_table_steps.rb

@@ -0,0 +1,8 @@
+When /^I create a Itiel::Extract::DatabaseTable object for the '(.*?)' table$/ do |table_name|
+  @database_table = Itiel::Extract::DatabaseTable.new
+  @database_table.connection = :test
+  @database_table.table_name = table_name
+  @source ||= @database_table
+
+  @extraction = @database_table.extract
+end

data/features/step_definitions/extractor/extraction_steps.rb

@@ -0,0 +1,3 @@
+Then /^the extraction for that object should have (\d+) rows?$/ do |count|
+  expect(@extraction.count).to eq count.to_i
+end

data/features/step_definitions/flow_steps.rb

@@ -0,0 +1,9 @@
+When /^the data flows in the following direction:$/ do |table|
+  last_step = nil
+
+  instance_eval table.raw.flatten.join(" >> ")
+end
+
+When /^I start the source$/ do
+  @source.start
+end

data/features/step_definitions/loader/csv_file_steps.rb

@@ -0,0 +1,4 @@
+When /^I create a Loader::CSVFile object with "([^"]*)"$/ do |filename|
+  @destination = Itiel::Load::CSVFile.new(File.join('tmp', filename), false)
+end
+

data/features/step_definitions/loader/database_table_steps.rb

@@ -0,0 +1,14 @@
+Given /^I create a Loader::DatabaseTable object for the "([^"]*)" table$/ do |arg1|
+  @destination  = Itiel::Load::DatabaseTable.new :test, "blogposts"
+end
+
+When /^I load the source into the destination$/ do
+  @source >> @destination
+  @source.start
+end
+
+Then /^the "([^"]*)" table should have (\d+) records$/ do |table_name, record_count|
+  Itiel::Extract::CustomSQL.connection_file_path = @database_config_path
+
+  expect(Itiel::Extract::CustomSQL.sequel_connection(:test)[table_name.to_sym].count).to eq record_count.to_i
+end

data/features/step_definitions/lookup/lookup_steps.rb

@@ -0,0 +1,35 @@
+Given(/^I create a Itiel::Lookup::DatabaseTable object for the "(.*?)"$/) do |table_name|
+  @lookup = Itiel::Lookup::DatabaseTable.new
+  @lookup.connection = :test
+  @lookup.table_name = table_name
+end
+
+Given(/^I create a Itiel::Lookup::CSVFile object with the "(.*?)" file$/) do |file_name|
+  @lookup = Itiel::Lookup::CSVFile.new(File.join('tmp', file_name))
+end
+
+Given(/^the lookup joins the "(.*?)" column in the source with the "(.*?)" column in the lookup stream$/) do |source_column, lookup_column|
+  @lookup.lookup_columns = { source_column => lookup_column }
+end
+
+Given(/^the lookup appends the "(.*?)" column on the lookup stream to the source as "(.*?)"$/) do |original_name, new_name|
+  @lookup.joined_columns = { original_name => new_name }
+end
+
+Then(/^the output for the lookup should be:$/) do |table|
+  result = @lookup.output
+  table.hashes.each_with_index do |row, index|
+    table.headers.each do |column|
+      expect(result[index][column.to_sym].to_s).to eq row[column]
+    end
+  end
+end
+
+Given /^an? orders database table$/ do
+  Itiel::Extract::CustomSQL.connection_file_path = @database_config_path
+
+  Itiel::Extract::CustomSQL.sequel_connection(:test).create_table :orders do
+    Integer :id
+    String :email
+  end
+end

data/features/step_definitions/scripting/ruby_script_steps.rb

@@ -0,0 +1,5 @@
+Given /^I create a Itiel::Script::RubyScript object with the following block of code:$/ do |script|
+  @ruby_script = Itiel::Script::RubyScript.new do |row|
+    instance_eval script
+  end
+end

data/features/step_definitions/stream_steps.rb

@@ -0,0 +1,8 @@
+Given /^the following data is in the stream:$/ do |table|
+  @stream = table.hashes
+end
+
+Then /^the resulting output stream should be:$/ do |table|
+  output = @ruby_script.execute(@stream)
+  expect(output).to eq table.hashes
+end

data/features/step_definitions/transformation/calculated_column_steps.rb

@@ -0,0 +1,5 @@
+Given /^I create a Transformation::CalculatedColumn object that multiplies "([^"]*)" times (\d+)$/ do |column_name, times|
+  @calculated_column = Itiel::Transform::CalculatedColumn.new 'new_id' do |columns|
+    columns['id'].to_i * 3
+  end
+end

data/features/step_definitions/transformation/calculated_columns_steps.rb

@@ -0,0 +1,7 @@
+Given /^I create a Transformation::CalculatedColumns object$/ do
+  @calculated_columns = Itiel::Transform::CalculatedColumns.new do |row|
+     name_state = "#{row['name']}-#{row['state']}"
+     id_name    = "#{row['id']}-#{row['name']}"
+     { 'name_state' => name_state, 'id_name' => id_name }
+  end
+end

data/features/step_definitions/transformation/constant_column_steps.rb

@@ -0,0 +1,3 @@
+When /^I create a Transformation::ConstantColumn object with "([^"]*)" => "([^"]*)"$/ do |column, value|
+  @constant_column = Itiel::Transform::ConstantColumn.new({ column => value })
+end

data/features/step_definitions/transformation/map_values_step.rb

@@ -0,0 +1,4 @@
+Given /^I create a Transformation::MapValues object for the state column to integer$/ do
+  @map_values = Itiel::Transform::MapValues.new(
+      { "state" => { "active" => 1, "inactive" => 0 }})
+end

data/features/step_definitions/transformation/rename_column_steps.rb

@@ -0,0 +1,3 @@
+Given /^I create a Transformation::RenameColumn object from "([^"]*)" to "([^"]*)"$/ do |original_name, new_name|
+  @rename_column = Itiel::Transform::RenameColumn.new({ original_name => new_name })
+end

data/features/step_definitions/transformation/select_column_steps.rb

@@ -0,0 +1,3 @@
+Given /^I create a Transformation::SelectColumn object with "([^"]*)" and "([^"]*)"$/ do |column1, column2|
+  @select_column = Itiel::Transform::SelectColumn.new column1, column2
+end

data/features/step_definitions/transformation/single_column_sort_steps.rb

@@ -0,0 +1,3 @@
+When /^I create a Transformation::SingleColumnSort object by "([^"]*)"$/ do |column_name|
+  @single_column_sort = Itiel::Transform::SingleColumnSort.new column_name
+end

data/features/support/database.yml

@@ -0,0 +1 @@
+test: 'sqlite://test.db'

data/features/support/env.rb

@@ -0,0 +1,13 @@
+require 'simplecov'
+
+SimpleCov.start
+
+require 'itiel'
+require 'csv'
+
+Itiel::Logger.logger = Logger.new File.join('tmp', 'itiel.log')
+
+Before '@database' do
+  File.unlink 'test.db' if File.exists?('test.db')
+  @database_config_path = File.join('features', 'support', 'database.yml')
+end

data/features/transform/transformations.feature

@@ -0,0 +1,123 @@
+Feature: Transformations
+
+  Background:
+    Given a "source.csv" file with the following rows:
+      | id | name  | state    |
+      | 1  | john  | active   |
+      | 2  | ruby  | active   |
+      | 3  | rails | inactive |
+      | 4  | pete  | active   |
+
+    # @source = Itiel::Extracts::CSVFile.new('source.csv')
+    And I create a Extractor::CSVFile object with "source.csv"
+    # @destination  = Itiel::Loads::CSVFile.new('destination.csv')
+    And I create a Loader::CSVFile object with "destination.csv"
+
+
+  Scenario: Append a constant value column to the original stream
+    # @constant_column = Itiel::Transform::ConstantColumn.new("constant" => "value")
+    Given I create a Transformation::ConstantColumn object with "constant" => "value"
+
+    # @source >> @constant_column >> @destination
+    And the data flows in the following direction:
+      | @source          |
+      | @constant_column |
+      | @destination     |
+
+    # @source.start
+    When I start the source
+
+    Then the "destination.csv" file should exist with the following content:
+      | id | name  | state    | constant |
+      | 1  | john  | active   | value    |
+      | 2  | ruby  | active   | value    |
+      | 3  | rails | inactive | value    |
+      | 4  | pete  | active   | value    |
+
+  Scenario: Map Values in a column
+    # @map_values = Itiel::Transform::MapValues.new("state" => { "active" => 1, "inactive" => 0 })
+    Given I create a Transformation::MapValues object for the state column to integer
+
+    # @source.next_step     = @map_values
+    # @map_values.next_step = @destination
+    And the data flows in the following direction:
+      | @source      |
+      | @map_values  |
+      | @destination |
+
+    # @source.start
+    When I start the source
+
+    Then the "destination.csv" file should exist with the following content:
+      | id | name  | state |
+      | 1  | john  | 1     |
+      | 2  | ruby  | 1     |
+      | 3  | rails | 0     |
+      | 4  | pete  | 1     |
+
+
+  Scenario: Rename an existing column
+    # @rename_column = Itiel::Transform::RenameColumn.new("state" => "status")
+    Given I create a Transformation::RenameColumn object from "state" to "status"
+
+    # @source.next_step     = @rename_column
+    # @rename_column.next_step = @destination
+    And the data flows in the following direction:
+      | @source        |
+      | @rename_column |
+      | @destination   |
+
+    # @source.start
+    When I start the source
+
+    Then the "destination.csv" file should exist with the following content:
+      | id | name  | status   |
+      | 1  | john  | active   |
+      | 2  | ruby  | active   |
+      | 3  | rails | inactive |
+      | 4  | pete  | active   |
+
+  Scenario: Select specific columns from the stream
+    # @select_column = Itiel::Transform::SelectColumn.new("id", "state")
+    Given I create a Transformation::SelectColumn object with "id" and "state"
+
+    # @source.next_step     = @select_column
+    # @select_column.next_step = @destination
+    And the data flows in the following direction:
+      | @source        |
+      | @select_column |
+      | @destination   |
+
+    # @source.start
+    When I start the source
+
+    Then the "destination.csv" file should exist with the following content:
+      | id | state    |
+      | 1  | active   |
+      | 2  | active   |
+      | 3  | inactive |
+      | 4  | active   |
+
+  Scenario: Create multiple columns with the result of a calculation with other columns
+
+    # @calculated_columns = Itiel::Transform::CalculatedColumns.new do |row|
+    #   name_state = "#{row['name']}-#{row['state']}"
+    #   id_name    = "#{row['id']}-#{row['name']}"
+    #   { 'name_state' => name_state, 'id_name' => id_name }
+    # end
+    Given I create a Transformation::CalculatedColumns object
+
+    And the data flows in the following direction:
+      | @source             |
+      | @calculated_columns |
+      | @destination        |
+
+    # @source.start
+    When I start the source
+
+    Then the "destination.csv" file should exist with the following content:
+      | id | name  | state    | name_state     | id_name |
+      | 1  | john  | active   | john-active    | 1-john  |
+      | 2  | ruby  | active   | ruby-active    | 2-ruby  |
+      | 3  | rails | inactive | rails-inactive | 3-rails |
+      | 4  | pete  | active   | pete-active    | 4-pete  |

data/itiel.gemspec

@@ -0,0 +1,34 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "itiel/version"
+
+Gem::Specification.new do |s|
+  s.name        = "itiel"
+  s.version     = Itiel::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["David Padilla"]
+  s.email       = ["david@padilla.cc"]
+  s.homepage    = "https://gitlab.com/dabit/itiel"
+  s.summary     = %q{DSL for ETL in Ruby}
+  s.description = %q{DSL for ETL in Ruby}
+
+  s.rubyforge_project = "itiel"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'activesupport', '~> 4.0'
+  s.add_dependency 'activerecord' , '~> 4.0'
+  s.add_dependency 'sequel'
+
+  s.add_development_dependency 'cucumber'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'rspec', '~> 3.5.0'
+  s.add_development_dependency 'rr'
+  s.add_development_dependency 'minitest'
+  s.add_development_dependency 'sqlite3'
+  s.add_development_dependency 'simplecov'
+  s.add_development_dependency 'turn'
+end

data/lib/itiel.rb

@@ -0,0 +1,45 @@
+require 'yaml'
+require 'active_support'
+require 'active_support/core_ext'
+
+module Itiel
+  class MethodNotImplementedException < Exception; end
+  class MissingConnection < Exception ; end
+  class UndefinedNextStepException < Exception ; end
+  class SQLSentenceNotProvided < Exception ; end
+end
+
+require 'itiel/job'
+require 'itiel/logger'
+require 'itiel/nameable'
+
+require 'itiel/db/connection'
+require 'itiel/db/sql_connectable.rb'
+require 'itiel/db/truncator.rb'
+
+require 'itiel/extract/chained_step'
+require 'itiel/extract/csv_file'
+require 'itiel/extract/custom_sql'
+require 'itiel/extract/database_table'
+
+require 'itiel/load/chained_step'
+require 'itiel/load/csv_file'
+require 'itiel/load/database_table'
+
+require 'itiel/lookup/chained_step'
+require 'itiel/lookup/hash_lookup'
+require 'itiel/lookup/database_table'
+require 'itiel/lookup/csv_file'
+
+require 'itiel/script/chained_step'
+require 'itiel/script/sql_script'
+require 'itiel/script/ruby_script'
+
+require 'itiel/transform/chained_step'
+require 'itiel/transform/calculated_columns'
+require 'itiel/transform/constant_column'
+require 'itiel/transform/rename_column'
+require 'itiel/transform/remove_column'
+require 'itiel/transform/select_column'
+require 'itiel/transform/input_output_behavior'
+require 'itiel/transform/map_values'

data/lib/itiel/db/connection.rb

@@ -0,0 +1,24 @@
+module Itiel
+  module DB
+    class Connection
+      include Itiel::Nameable
+
+      attr_accessor :sources_file
+      attr_accessor :connection_name
+
+      def initialize
+        self.sources_file = 'config/sources.yml'
+      end
+
+      def connection_string(refresh = false)
+        @connection = ( refresh ? load_from_file : @connection ||= load_from_file)
+      end
+
+      private
+      def load_from_file
+        yaml = YAML.load_file(self.sources_file)
+        yaml[self.connection_name]
+      end
+    end
+  end
+end

data/lib/itiel/db/sql_connectable.rb

@@ -0,0 +1,33 @@
+require 'sequel'
+
+module Itiel
+  module DB
+    module SQLConnectable
+      attr_accessor :connection, :connection_file_path
+
+      module ClassMethods
+        def connection_file_path
+          @@connection_file_path ||= 'config/database.yml'
+        end
+
+        def connection_file_path=(value)
+          @@connection_file_path = value
+        end
+
+        def sequel_connection(connection_name)
+          Sequel.connect read_connection_string(connection_name)
+        end
+
+        protected
+        def read_connection_string(connection_name)
+          @connections ||= YAML.load_file(connection_file_path)
+          @connections[connection_name.to_s]
+        end
+      end
+
+      def self.included(receiver)
+        receiver.extend ClassMethods
+      end
+    end
+  end
+end