itiel 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 319124d81efc2cd9b0963c9f1693862d7347170e
+  data.tar.gz: 9e89b3071f090742e273d80a0023f703064f7325
+SHA512:
+  metadata.gz: e4b71fb0ccdcec9e48cae895382e069b67680924b56901e034a10e6e452448081b2ea0fa5538e63e334ed2c15760dfa0143799795cbd753488fc72ba5b0d638f
+  data.tar.gz: cc404b8fa1e0404e93caa8ab29d031a1496f7d74fe4440d55a524ff623d6cc6afc4ca62e87c19806ae18955768e14d1facb30728f0a41a4c1453f75fc0419620

data/.autotest

@@ -0,0 +1,17 @@
+require 'autotest/fsevent'
+require 'autotest/growl'
+require 'autotest/fsevent'
+require 'autotest/timestamp'
+
+Autotest.add_hook :initialize do |at|
+  at.clear_mappings
+
+  %w{.git coverage/ README gemspec DS_Store features/ Gemfile.lock Rakefile Gemfile}.each do |exception|
+    at.add_exception exception
+  end
+
+  at.add_mapping %r%^lib/itiel/(.*)\.rb$% do |_, m|
+    ["test/unit/#{m[1]}_test.rb"]
+  end
+
+end

data/.gitignore

@@ -0,0 +1,13 @@
+*.gem
+.bundle
+Gemfile*.lock
+pkg/*
+**.swp
+.rvmrc
+**.sqlite3
+tmp
+doc
+coverage
+test.db
+tags
+.idea

data/.gitlab-ci.yml

@@ -0,0 +1,36 @@
+cache:
+  paths:
+    - vendor/ruby
+
+before_script:
+  - ruby -v                                   # Print out ruby version for debugging
+  - gem install bundler  --no-ri --no-rdoc    # Bundler is not installed with the image
+  - bundle install -j $(nproc) --path vendor  # Install dependencies into ./vendor/ruby
+
+tests:2.3.rails.4.0:
+  image: "ruby:2.3"
+  variables:
+    BUNDLE_GEMFILE: "Gemfile.rails.4.0"
+  script:
+    - bundle exec rake
+
+tests:2.3.rails.4.1:
+  image: "ruby:2.3"
+  variables:
+    BUNDLE_GEMFILE: "Gemfile.rails.4.1"
+  script:
+    - bundle exec rake
+
+tests:2.2.rails.4.0:
+  image: "ruby:2.2"
+  variables:
+    BUNDLE_GEMFILE: "Gemfile.rails.4.0"
+  script:
+    - bundle exec rake
+
+tests:2.2.rails.4.1:
+  image: "ruby:2.2"
+  variables:
+    BUNDLE_GEMFILE: "Gemfile.rails.4.1"
+  script:
+    - bundle exec rake

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.ruby-version

@@ -0,0 +1 @@
+2.3.2

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+
+rvm:
+  - 2.3.4
+  - 2.2.7
+
+gemfile:
+  - Gemfile.rails.4.0
+  - Gemfile.rails.4.1

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in itiel.gemspec
+gemspec

data/Gemfile.rails.4.0

@@ -0,0 +1,7 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in itiel.gemspec
+gemspec
+
+gem 'activerecord', '~> 4.0.0'
+gem 'activesupport', '~> 4.0.0'

data/Gemfile.rails.4.1

@@ -0,0 +1,7 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in itiel.gemspec
+gemspec
+
+gem 'activerecord', '~> 4.1.0'
+gem 'activesupport', '~> 4.1.0'

data/Gemfile.rails.4.2

@@ -0,0 +1,7 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in itiel.gemspec
+gemspec
+
+gem 'activerecord', '~> 4.2.0'
+gem 'activesupport', '~> 4.2.0'

data/README.markdown

@@ -0,0 +1,106 @@
+# Itiel
+
+[![build status](https://gitlab.com/dabit/itiel/badges/master/build.svg)](https://gitlab.com/dabit/itiel/commits/master)
+[![coverage report](https://gitlab.com/dabit/itiel/badges/master/coverage.svg)](https://gitlab.com/dabit/itiel/commits/master)
+
+Hopefully, it will be an awesome Framework to do ETL with Ruby. It
+should only work with *Ruby 1.9*.
+
+[A working example](https://github.com/railsmx/rails-mx-blog/blob/master/itiel/posts.rb)
+
+# This README is work in progress
+
+You should not trust it, until I clean it up.
+
+For better examples and documentation, refer to the *features* folder.
+
+
+# TODO: Everything I want it to be, below:
+
+### Define the Itiel project structure
+
+I have in mind something like this:
+
+    /itiel_example/
+    |~config/
+    | `-sources.yml
+    |~lib/
+    | `-transformation_class1.rb
+    |~jobs/
+    | `-main_job.rb
+    |~tasks/
+    | `-itiel.rake
+    |-Gemfile
+    |-Gemfile.lock
+
+### Controllers
+
+Take care of defining and running the jobs. It's where you put all the
+code like the example above.
+
+### A command line bin to generate the basic structure
+
+    itiel new project_name.
+
+Use thor?
+
+### Rake
+
+    rake itiel:run
+
+### Mongo
+
+Picture yourself migrating form SQL to Mongo
+
+### Joins, Scripts and lots more transformations
+
+Some on my mind:
+
+    Itiel::Joins::DatabaseTable
+    Itiel::Lookups::DatabaseTable
+    Itiel::Loads::FTP
+    Itiel::Loads::HTTPRequest
+    Itiel::Loads::MongoDBDocument
+    Itiel::Script::ExecuteInSystem
+    Itiel::Extracts::HTTP
+    Itiel::Extracts::MongoDBDocument
+    Itiel::Transforms::CustomSort
+    Itiel::Transforms::MapValues
+    Itiel::Transforms::IfNil
+
+Just ideas, I have nothing on how to implement most of them
+
+### Filters and conditional stream routing
+
+If the column X has a value of Y, send true output to this step, send
+false output to this other steps
+
+### Slowly Changing Dimentions
+
+Can't have ETL without these
+
+### Test helpers
+
+So you can TDD your ETL
+
+### Logging
+
+It would be extra nice if all the steps logged statistics
+
+* Processing X rows of Y
+* Time spent
+* Transformation Name
+* Configurable log level
+
+### Viz
+
+    rake itiel::graph
+
+Creates a graphical representation of your ETL flow
+
+### Ruby 1.8 and 1.9
+
+Not really, Ruby 2.0+ is enough. I don't have plans to make it
+work on 1.8 or 1.9
+
+

data/Rakefile

@@ -0,0 +1,13 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+require 'cucumber/rake/task'
+
+task :default => [:spec, :cucumber]
+
+RSpec::Core::RakeTask.new
+
+Cucumber::Rake::Task.new do |task|
+  task.cucumber_opts = %w{--format progress}
+end

data/build.sh

@@ -0,0 +1,10 @@
+#! /bin/bash -e
+
+source "$HOME/.rvm/scripts/rvm"
+rvm 1.9.2
+rvm gemset create itiel
+rvm use 1.9.2@itiel
+bundle -v || gem install bundler
+bundle install
+rake
+

data/features/extract/database_table.feature

@@ -0,0 +1,16 @@
+Feature: Extract data from a Database Table
+
+  @database
+  Scenario:
+    Given a blogposts database table with rows:
+       | id | title              | author          |
+       | 1  | The great tool     | Tim Taylor      |
+       | 2  | A footbal anecdote | John Miller     |
+       | 3  | Historical music   | John Mellencamp |
+
+    # @database_table = Itiel::Extract::DatabaseTable.new
+    # @database_table.connection = :test
+    # @database_table.table_name = 'blogposts'
+    When I create a Itiel::Extract::DatabaseTable object for the 'blogposts' table
+    Then the extraction for that object should have 3 rows
+

data/features/extract/sql_script.feature

@@ -0,0 +1,17 @@
+Feature: Extract rows from a database using a custom SQL script
+
+  @database
+  Scenario:
+    Given a blogposts database table with rows:
+       | id | title              | author          |
+       | 1  | The great tool     | Tim Taylor      |
+       | 2  | A footbal anecdote | John Miller     |
+       | 3  | Historical music   | John Mellencamp |
+
+     When I create a Itiel::Extract::CustomSQL object with the query:
+     """
+       SELECT * FROM blogposts WHERE id = 3;
+     """
+
+     Then the extraction for that object should have 1 row
+

data/features/load/database_table_loader.feature

@@ -0,0 +1,21 @@
+Feature: Load data into a database table
+
+  Background:
+    Given a blogposts database table
+    And a "source.csv" file with the following rows:
+      | id | title                 | author          |
+      | 1  | Pride and Prejudice   | Jane Austen     |
+      | 2  | Treasure Island       | R L Stevenson   |
+      | 3  | Steve Jobs            | Walter Isaacson |
+      | 4  | The thank you economy | Gary Vaynerchuk |
+
+    # @source = Itiel::Extracts::CSVFile.new('source.csv')
+    And I create a Extractor::CSVFile object with "source.csv"
+    # @destination  = Itiel::Load::DatabaseTable.new :test, "blogposts"
+    And I create a Loader::DatabaseTable object for the "blogposts" table
+
+  @database
+  Scenario:
+    When I load the source into the destination
+    Then the "blogposts" table should have 4 records
+

data/features/lookup/csv_file.feature

@@ -0,0 +1,41 @@
+Feature: Lookup data from a CSV file
+
+  @database
+  Scenario:
+    Given an orders database table with rows:
+      | id | email            |
+      | 1  | john@example.com |
+      | 2  | tim@test.com     |
+      | 3  | math@example.com |
+
+    And a "customers.csv" file with the following rows:
+      | id | email            |
+      | 1  | john@example.com |
+      | 2  | tim@test.com     |
+      | 3  | math@example.com |
+
+    # @database_table = Itiel::Extract::DatabaseTable.new
+    # @database_table.connection = :test
+    # @database_table.table_name = 'orders'
+    And I create a Itiel::Extract::DatabaseTable object for the 'orders' table
+
+    # @lookup = Itiel::Lookup::CSVFile.new('customers.csv')
+    And I create a Itiel::Lookup::CSVFile object with the "customers.csv" file
+
+    # @lookup.lookup_columns = { :email => :email }
+    And the lookup joins the "email" column in the source with the "email" column in the lookup stream
+
+    # @lookup.joined_columns = { :id => :customer_id }
+    And the lookup appends the "id" column on the lookup stream to the source as "customer_id"
+
+    And the data flows in the following direction:
+      | @database_table  |
+      | @lookup          |
+
+    When I start the source
+
+    Then the output for the lookup should be:
+      | id | email            | customer_id |
+      | 1  | john@example.com | 1           |
+      | 2  | tim@test.com     | 2           |
+      | 3  | math@example.com | 3           |

data/features/lookup/database_table.feature

@@ -0,0 +1,43 @@
+Feature: Lookup data from a Database Table
+
+  @database
+  Scenario:
+    Given a blogposts database table with rows:
+      | id | title               | author          |
+      | 1  | The great tool      | Tim Taylor      |
+      | 2  | A football anecdote | John Miller     |
+      | 3  | Historical music    | John Mellencamp |
+
+    And an authors database table with rows:
+      | id | name            |
+      | 1  | Tim Taylor      |
+      | 2  | John Miller     |
+      | 3  | John Mellencamp |
+
+    # @database_table = Itiel::Extract::DatabaseTable.new
+    # @database_table.connection = :test
+    # @database_table.table_name = 'blogposts'
+    And I create a Itiel::Extract::DatabaseTable object for the 'blogposts' table
+
+    # @lookup = Itiel::Lookup::DatabaseTable.new
+    # @lookup.connection    = :test
+    # @lookup.table_name    = 'author'
+    And I create a Itiel::Lookup::DatabaseTable object for the "authors"
+
+    # @lookup.lookup_columns = { :author => :name }
+    And the lookup joins the "author" column in the source with the "name" column in the lookup stream
+
+    # @lookup.joined_columns = { :id => :author_id }
+    And the lookup appends the "id" column on the lookup stream to the source as "author_id"
+
+    And the data flows in the following direction:
+      | @database_table  |
+      | @lookup |
+
+    When I start the source
+
+    Then the output for the lookup should be:
+      | id | title               | author          | author_id |
+      | 1  | The great tool      | Tim Taylor      | 1         |
+      | 2  | A football anecdote | John Miller     | 2         |
+      | 3  | Historical music    | John Mellencamp | 3         |

data/features/script/ruby_script.feature

@@ -0,0 +1,19 @@
+Feature: Ruby scripting
+
+  Scenario:
+    Given the following data is in the stream:
+       | id | title               | author          |
+       | 1  | The great tool      | Tim Taylor      |
+       | 2  | A football anecdote | John Miller     |
+       | 3  | Historical music    | John Mellencamp |
+
+    And I create a Itiel::Script::RubyScript object with the following block of code:
+      """
+        row["slug"] = row["title"].parameterize
+      """
+
+    Then the resulting output stream should be:
+       | id | title               | author          | slug                |
+       | 1  | The great tool      | Tim Taylor      | the-great-tool      |
+       | 2  | A football anecdote | John Miller     | a-football-anecdote |
+       | 3  | Historical music    | John Mellencamp | historical-music    |

data/features/step_definitions/csv_steps.rb

@@ -0,0 +1,15 @@
+Given /^a "([^"]*)" file with the following rows:$/ do |filename, table|
+  CSV.open File.join('tmp', filename), "wb" do |csv|
+    table.raw.each { |row| csv << row }
+  end
+end
+
+Then /^the "([^"]*)" file should exist with the following content:$/ do |filename, table|
+  expected_file = CSV.generate do |csv|
+    table.raw.each { |row| csv << row }
+  end
+
+  resulting_file = File.new(File.join('tmp', filename), "rb").read
+
+  expect(resulting_file).to eq expected_file
+end