datamancer 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5a09ebee71e5c2b54c0b790ae4dc2165e4e9e45a
+  data.tar.gz: eabae34cb0d1b34ce072796f3b0edab7117325e2
+SHA512:
+  metadata.gz: 84af02d3f38e7e861086bf268eb6670ef8528f6b70035d2bba6f3098b99dd897b29b727ae1037ba80abad412ff34ae543eec4e8c57ed71f4a74a939f35be4b15
+  data.tar.gz: 4c88628302bca9be9089ee2a6c1be2991c06201d66ce91bd88a62247b6ad45b64ff0cc90403dce9c1ebd0dcabb6fb807be58f52133394c0f57e5771473d558c2

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+.ruby-version
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in datamancer.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Matías Battocchia
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,33 @@
+# Datamancer
+
+**The Datamancer** is a form of sorcerer whose spells deal with data manipulation between databases.
+
+Data targets (sources and destinations) can be **databases** supported by ActiveRecord and **CSV files**. Multiple targets can be present in a single ETL process.
+
+Datamancer relies in bulk SQL reading and writing, and does not instantiate ActiveRecord objects, which is used for the sole purpose of connecting to databases.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'datamancer'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install datamancer
+
+## Usage
+
+*Please see the specs, for now.*
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/datamancer.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'datamancer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "datamancer"
+  spec.version       = Datamancer::VERSION
+  spec.authors       = ["Matías Battocchia"]
+  spec.email         = ["matias@riseup.net"]
+  spec.description   = %q{A magical extract, transform, load (ETL) library for data integration.}
+  spec.summary       = %q{}
+  spec.homepage      = "https://github.com/matiasbattocchia/datamancer"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "activerecord"
+  spec.add_development_dependency "sqlite3"
+  spec.add_development_dependency "activerecord-jdbcsqlite3-adapter"
+end

data/lib/datamancer/extract.rb

@@ -0,0 +1,183 @@
+module Datamancer
+
+  def extract args
+    raise ArgumentError,
+      'Extract requires a source, i.e. extract(from: source)' unless
+        args.is_a?(Hash) && args[:from]
+
+    headers = case args[:from]
+    when String
+
+      # TODO: The first row (headers row) is dropped; to drop more initial rows should be an option.
+     
+      # TODO: Test the separator option.
+
+      csv = CSV.read args[:from], col_sep: (args[:separator] || ',')
+      csv.shift
+    when Hash
+      ::ActiveRecord::Base.establish_connection args[:from]
+      db = ::ActiveRecord::Base.connection
+
+      # TODO: Test this.
+
+      table = args[:table] || args[:from][:table]
+      
+      raise ArgumentError,
+        'Extract requires a database table, i.e. extract(from: source, table: table_name)' unless table
+
+      db.columns(table).map(&:name)
+    end
+
+    @fields  = {}
+    @actions = {}
+    
+    headers.each do |header|
+      field   = header.to_sym
+      mapping = header
+      @fields[field] = mapping
+    end unless args[:exclude]
+
+    # The reason behind default_actions is the possibility to
+    # write reject_if: nil with the DSL.
+    default_actions = {reject_if: :ñil, reject_unless: :ñil}
+
+    define_singleton_method :field do |name, actions = {}, &block|
+      actions[:type] ||= actions[:type_default]
+      actions = default_actions.merge(actions)
+      mapping = actions[:map] || name.to_s
+
+      raise MissingField,
+        "Required field '#{mapping}' was not found in '#{args[:from]}'" unless headers.include? mapping
+      
+      field = name.to_sym
+      @fields.delete(actions[:map].to_sym) if actions[:map]
+      @fields[field]  = mapping
+      @actions[field] = actions
+      @actions[field][:block] = block
+    end
+
+    yield if block_given?
+    
+    output = case args[:from]
+    when String
+      @fields.each do |field, mapping|
+        index = headers.find_index(mapping)
+        @fields[field] = index
+      end
+
+      csv
+
+    when Hash
+      columns = @fields.map { |field, mapping| "#{mapping} AS #{field}" }.join(', ')
+
+      @fields.keys.each_with_index do |field, index|
+        @fields[field] = index
+      end
+
+      db.select_rows("SELECT #{columns} FROM #{table}")
+    end
+
+    output.map! do |array_row|
+      hash_row = {}
+
+      @fields.each do |field, index|
+        value = array_row[index]
+        hash_row[field] = field_actions field, value, @actions[field]
+      end
+
+      if hash_row.has_value?(:reject)
+        puts hash_row
+        nil
+      else
+        hash_row
+      end
+    end.compact!
+
+    output
+  end
+
+  def field_actions field, value, actions
+    return value unless actions
+
+    # TODO: Revisit the order of actions.
+
+    ## Block-passing ##
+    
+    # TODO: Test this.
+
+    if actions[:block]
+      value = actions[:block].call(value)
+    end
+
+    ## Stripping ##
+
+    # TODO: Test this.
+
+    if actions[:strip]
+      value.strip! if value.respond_to?(:strip!)
+    end
+
+    ## Casting ##
+
+    # Indexes and :type_default are not good friends.
+    # (Because of join while transforming.)
+
+    # TODO: Test this.
+
+    if value || actions[:type_default]
+
+      # TODO: Better data types support. From Mongoid:
+
+      # [ ] Array
+      # [ ] BigDecimal
+      # [ ] Boolean
+      # [x] Float
+      # [ ] Hash
+      # [x] Integer
+      # [ ] Range
+      # [ ] Regexp
+      # [x] String
+      # [x] Symbol
+      # [ ] Date
+      # [ ] DateTime
+      # [ ] Time
+      # [ ] TimeWithZone
+
+      case actions[:type].to_s
+      when 'Complex'
+        value = value.to_c
+      when 'Float'
+        value = value.to_f
+      when 'Integer'
+        value = value.to_i
+      when 'Rational'
+        value = value.to_r
+      when 'String'
+        value = value.to_s
+      when 'Symbol'
+        value = value.to_sym
+      end
+    end
+
+    ## Default value ##
+  
+    # TODO: Test this.
+
+    if value.nil? || (actions[:empty_default] && value.empty?)
+      value = actions[:default]
+    end
+
+    ## Validation ##
+
+    # TODO: Test this. Test to not reject nil by default.
+
+    if actions[:reject_if] == value ||
+      (actions[:reject_unless] != :ñil &&
+       actions[:reject_unless] != value)
+
+      value = :reject
+    end
+
+    value
+  end
+end

data/lib/datamancer/load.rb

@@ -0,0 +1,89 @@
+module Datamancer
+
+  def load input, args
+
+    raise ArgumentError,
+      'Load requires a destination, i.e. load(data, to: destination)' unless
+        args.is_a?(Hash) && args[:to]
+
+    ## COLUMNS ##
+
+    # define_singleton_method :field do |name, options = {}|
+    #   @columns << (options[:map] || name)
+    # end
+
+    # @columns = []
+
+    # yield if block_given?
+
+    ## INSERTS ##
+
+    define_singleton_method :field do |name, options = {}|
+      name = name.to_sym
+
+      raise MissingField,
+        "Required field '#{name}' was not found in '#{args[:to]}'" unless @input_row.include? name
+
+      @output_row[options[:map] || name] = @input_row[name]
+      @output_row.delete(name) if !args[:exclude] && options[:map]
+    end
+ 
+    inserts = []
+
+    input.each do |row|
+      @input_row = row
+      @output_row = args[:exclude] ? {} : row.dup
+
+      yield if block_given?
+
+      inserts << @output_row.values
+    end
+
+    columns = @output_row.keys
+
+    ## LOAD ##
+
+    # TODO: Set 'w' or 'w+' for CSV writing.
+    
+    if args[:to].is_a?(String)
+      mode = if args[:append] then 'a' else 'w' end
+      
+      CSV.open(args[:to], mode) do |csv|
+        csv << columns
+
+        inserts.each do |insert|
+          csv << insert
+        end
+      end
+
+    else
+      ::ActiveRecord::Base.establish_connection(args[:to])
+
+      # TODO: Test this.
+
+      table = args[:table] || args[:to][:table]
+      
+      raise ArgumentError,
+        'Load requires a database table, i.e. load(to: destination, table: table_name)' unless table
+
+      ::ActiveRecord::Base.connection.delete("DELETE FROM #{table}") unless args[:append]
+      batch_size = args[:batch] || 1000
+
+      pre_query = "INSERT INTO #{table} (#{columns.join(',')}) VALUES "
+
+      # String values must be enclosed by single quotes.
+      inserts.map! do |insert|
+        insert.map! do |field|
+          field.is_a?(String) ? "'#{field}'" : field
+        end
+
+        "(#{insert.join(',')})"
+      end
+
+      until inserts.empty?
+        query = pre_query + inserts.pop(batch_size).join(',')
+        ::ActiveRecord::Base.connection.execute query
+      end
+    end
+  end
+end

data/lib/datamancer/transform.rb

@@ -0,0 +1,121 @@
+module Datamancer
+
+  def join left, right, attribute
+
+    attribute = attribute.to_sym
+
+    left_groups  = Hash.new { |hash, key| hash[key] = [] }
+    right_groups = Hash.new { |hash, key| hash[key] = [] }
+
+    left.each do |tuple|
+      left_groups[tuple[attribute]] << tuple if tuple[attribute]
+    end
+    
+    right.each do |tuple|
+      right_groups[tuple[attribute]] << tuple if tuple[attribute]
+    end
+
+    output = Array.new
+ 
+    left_groups.each do |key, left_group|
+      
+      if right_group = right_groups[key]
+
+        left_group.each do |left_tuple|
+          right_group.each do |right_tuple|
+            output << left_tuple.merge(right_tuple)
+          end
+        end
+
+      end
+    end
+
+    output
+  end
+
+  def transform input, args = {}
+
+    if args[:join]
+      raise ArgumentError unless args[:on]
+
+      input = join input, args[:join], args[:on]
+    end
+
+    # TODO: Method-overriding safeguard.
+  
+    input.first.each_key do |key|
+      define_singleton_method key.downcase do
+      
+        # Some methods applied to fields might modify the original fields.
+        # Fields could be duplicated in case this be a common problem.
+
+        #@input_row[key].dup
+
+        @input_row[key]
+      end
+    end
+
+    define_singleton_method :field do |name, value = nil, *args|
+      raise MissingField,
+        "Required field '#{name}' was not found" unless respond_to?(name)
+        
+      @output_row[name.to_sym] = if value.is_a?(Symbol)
+                                   send(name).send *args.unshift(value)
+                                 else
+                                   value || send(name)
+                                 end
+    end
+
+    define_singleton_method :del_field do |name|
+      raise MissingField,
+        "Filtered field '#{name}' was not found" unless respond_to?(name)
+      
+      @output_row.delete(name.to_sym)
+    end
+
+    define_singleton_method :new_field do |name, value|
+      raise ExistingField,
+        "New field '#{name}' already exists" if respond_to?(name)
+
+      @output_row[name.to_sym] = value
+    end
+
+    input.map do |row|
+      @input_row = row
+      @output_row = args[:exclude] ? {} : @input_row.dup
+
+      yield if block_given?
+
+      @output_row
+    end
+  end
+
+  def aggregate input
+    
+    define_singleton_method :dim do |name|
+      name = name.to_sym
+      @dimensions[name] = @row[name]
+    end
+    
+    define_singleton_method :fact do |name|
+      name = name.to_sym
+      @facts[name] = @row[name]
+    end
+
+    aggregated_input = Hash.new { |hash, key| hash[key] = Hash.new }
+
+    input.each do |row|
+      @row = row
+      @dimensions = {}
+      @facts = {}
+
+      yield if block_given?
+
+      aggregated_input[@dimensions].merge!(@facts) { |_, fact, other_fact| fact + other_fact }
+    end
+
+    aggregated_input.map do |dimensions, facts|
+      dimensions.merge(facts)
+    end
+  end
+end

data/lib/datamancer/version.rb

@@ -0,0 +1,3 @@
+module Datamancer
+  VERSION = "0.0.2"
+end

data/lib/datamancer.rb

@@ -0,0 +1,56 @@
+require 'datamancer/version'
+require 'datamancer/extract'
+require 'datamancer/transform'
+require 'datamancer/load'
+
+module Datamancer
+  class MissingField < StandardError; end
+  class ExistingField < StandardError; end
+  
+  def keyvalue file
+    YAML::load_file file
+  end
+end
+
+# TODO: Decide on extract() default values issue; string?
+
+# TODO: Case insensitive and regex field-mapping.
+
+# TODO: Better errors and tests for them.
+
+# TODO: ETL by batch.
+
+# TODO: Methods alias.
+# field => f
+# unfield => u
+
+# TODO: Field inclusion policy.
+#
+# OPTIONS
+#
+#   :include_all (DEFAULT)
+#   :exclude_all
+# 
+# METHODS
+#
+#   unfield(name)
+#   field(name, value = nil)
+#
+# CASES
+#
+#   unfield(name)      => include all, exclude 'name'
+#   field(name)        => exclude all, include 'name'
+#   field(name, value) => include 'name'
+#
+#   unfield(name1) and field(name2)        => include all, exclude 'name1', include 'name2'
+#   unfield(name1) and field(name2, value) => include all, exclude 'name1', include 'name2'
+#   field(name1) and field(name2, value)   => exclude all, include 'name1', include 'name2'
+#
+#   unfield(name1) and field(name2) and field(name2, value) => include all, exclude 'name1', include 'name2', include 'name3'
+# 
+# USAGE
+#
+#   Basically :include_all is passed explicity to be used along with field(name) (which otherwise would cause a
+#   general exclusion of fields) for documentation purposes.
+#
+#   :exclude_all is useful for changing transform()'s default inclusive behaviour when using field(name, value) alone.

data/spec/data/destination.csv

@@ -0,0 +1,3 @@
+name,age
+Foo,27
+Bar,42

data/spec/data/left_source.csv

@@ -0,0 +1,5 @@
+name,some_id
+"Foo",1
+"Bar",2
+"Baz",2
+"Foobar",

data/spec/data/right_source.csv

@@ -0,0 +1,5 @@
+age,some_id
+0,
+27,1
+33,1
+42,2

data/spec/data/simple.yml

@@ -0,0 +1,2 @@
+Argentina: '+54'
+Brazil: '+55'

data/spec/data/source.csv

@@ -0,0 +1,3 @@
+name,age
+"Foo",27
+"Bar",42

data/spec/data/source2.csv

@@ -0,0 +1,6 @@
+store,coffee_cups,name,croissants
+"A",1,"Foo",0
+A,3,Bar,2
+A,5,Bar,4
+B,7,Foo,6
+B,9,Foo,8

data/spec/extract_spec.rb

@@ -0,0 +1,155 @@
+require 'spec_helper'
+
+describe Datamancer do
+  
+  context 'EXTRACT spell' do
+
+    sources = {'CSV file' => $dir + '/source.csv',
+               'database' => {adapter: 'sqlite3', database: $dir + '/source.sqlite3', table: 'users'}}
+    
+    sources.each do |source_type, source|
+
+      context "against a #{source_type.upcase}" do
+
+        before(:all) do
+          @source = source
+          @data = [{name: 'Foo', age: 27}, {name: 'Bar', age: 42}]
+        end
+
+        after(:all) do
+          ActiveRecord::Base.connection.close
+        end if source_type == 'database'
+
+
+        it 'reads data from source implicitly' do
+          expect(
+            extract(from: @source)
+          ).to eq(@data)
+
+          expect(
+            extract(from: @source) do
+              field :name
+            end
+          ).to eq(@data)
+
+          expect(
+            extract(from: @source) do
+              field :name
+              field :age
+            end
+          ).to eq(@data)
+        end
+
+        
+        it 'reads data from source explicitly' do
+          expect(
+
+            extract(from: @source, exclude: true) do
+              field :name
+            end
+
+          ).to eq([{name: 'Foo'}, {name: 'Bar'}])
+        end
+
+        
+        it 'raises an exception if data source is missing' do
+          expect {
+
+            extract(@source)
+
+          }.to raise_error(ArgumentError, 'Extract requires a source, i.e. extract(from: source)')
+        end
+
+        
+        it 'raises an exception if a required field is missing' do
+
+          # TODO: Better explanation for this error.
+
+          # source = case source_type
+          # when 'CSV file' then "#{@source} file"
+          # when 'database' then "#{@source[:database]}.#{@source[:table]} table"
+          # end
+
+          expect {
+
+            extract(from: @source) do
+              field :surname
+            end
+
+          }.to raise_error(MissingField,
+            "Required field 'surname' was not found in '#{@source}'")
+        end
+
+        
+        it 'lets fields to be strings' do
+          expect(
+
+            extract(from: @source) do
+              field 'name'
+              field 'age'
+            end
+
+          ).to eq(@data)
+        end
+
+        
+        it 'maps fields' do
+          expect(
+
+            extract(from: @source) do
+              field :years_old, map: 'age'
+            end
+
+          ).to eq([{name: 'Foo', years_old: 27}, {name: 'Bar', years_old: 42}])
+        end
+
+        
+        it 'casts types on fields' do
+          expect(
+
+            extract(from: @source) do
+              field :age, type: Integer
+            end
+
+          ).to eq([{name: 'Foo', age: 27}, {name: 'Bar', age: 42}])
+        end
+
+      end
+    end
+
+    # TODO: Validations
+
+    # exclusion
+    # inclusion
+    # format
+    # presence
+    # uniqueness (?)
+    # length (?)
+    # numericality (?)
+    # custom validations (?)
+
+    # When a field validation fails, the row gets dropped.
+    # This action is logged and optionally the process halted.
+    # Validations should accept options (:allow_nil, :if, etc).
+
+    # it 'validates fields' do
+      
+    #   pending 'Not now...'
+
+    #   expect(
+
+    #     extract(from: @csv_file) do
+    #       field :name
+    #       field :age, reject_if: '27'
+    #     end
+
+    #   ).to eq([{name: 'Bar', age: '42'}])
+    # end
+    
+    it 'imports YAML key-value tables' do
+      @yml_file = $dir + '/simple.yml'
+      expect(keyvalue(@yml_file)).to eq({'Argentina' => '+54', 'Brazil' => '+55'})
+    end
+
+  end
+end

data/spec/load_spec.rb

@@ -0,0 +1,114 @@
+require 'spec_helper'
+
+describe Datamancer do
+
+  context 'LOAD spell' do
+
+    destinations = {'CSV file' => $dir + '/destination.csv',
+                    'database' => {adapter: 'sqlite3', database: $dir + '/destination.sqlite3', table: 'users'}}
+    
+    destinations.each do |destination_type, destination|
+
+      context "against a #{destination_type.upcase}" do
+
+        before(:all) do
+          @destination = destination
+          @data = [{name: 'Foo', age: 27}, {name: 'Bar', age: 42}]
+          
+          if destination_type == 'database'
+            ActiveRecord::Base.establish_connection(destination)
+            ActiveRecord::Base.connection.delete('DELETE FROM users')
+          end
+        end
+
+        after(:all) do
+          ActiveRecord::Base.connection.close
+        end if destination_type == 'database'
+        
+        
+        it 'writes data to destination implicitly' do
+          load(@data, to: @destination, append: false)
+
+          expect(extract from: @destination).to eq(@data)
+
+          load(@data, to: @destination, append: false) do
+            field :name
+          end
+
+          expect(extract from: @destination).to eq(@data)
+
+          load(@data, to: @destination, append: false) do
+            field :name
+            field :age
+          end
+
+          expect(extract from: @destination).to eq(@data)
+        end
+
+        
+        it 'appends data to destination'
+
+        
+        it 'writes data to destination explicitly' do
+          load(@data, to: @destination, append: false, exclude: true) do
+            field :name
+          end
+
+          expect(extract from: @destination).to eq(
+            case destination_type
+            when 'CSV file'
+              [{name: 'Foo'}, {name: 'Bar'}]
+            when 'database'
+              
+              # TODO: Fix this empty DB columns thing.
+              
+              [{name: 'Foo', age: nil}, {name: 'Bar', age: nil}]
+            end)
+        end
+
+        
+        it 'raises an exception if data destination is missing' do
+          expect {
+
+            load(@data, @destination)
+
+          }.to raise_error(ArgumentError, 'Load requires a destination, i.e. load(data, to: destination)')
+        end
+        
+        
+        it 'raises an exception if a required field is missing' do
+          expect {
+
+            load(@data, to: @destination) do
+              field :surname
+            end
+
+          }.to raise_error(MissingField,
+            "Required field 'surname' was not found in '#{@destination}'")
+        end
+
+        
+        it 'lets fields to be strings' do
+          load(@data, to: @destination, append: false) do
+            field 'name'
+            field 'age'
+          end
+
+          expect(extract from: @destination).to eq(@data)
+        end
+
+        
+        it 'maps fields' do
+          data = [{name: 'Foo', years_old: '27'}, {name: 'Bar', years_old: '42'}]
+
+          load(data, to: @destination, append: false) do
+            field :years_old, map: 'age'
+          end
+
+          expect(extract from: @destination).to eq(@data)
+        end
+
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+require 'datamancer'
+require 'csv'
+require 'active_record'
+
+if defined? JRUBY_VERSION
+  require 'activerecord-jdbc-adapter'
+else
+  require 'sqlite3'
+end
+
+include Datamancer
+
+$dir = __dir__ + '/data'

data/spec/transform_spec.rb

@@ -0,0 +1,231 @@
+require 'spec_helper'
+
+describe Datamancer do
+
+  context 'TRANSFORMATION spell' do
+    
+    before(:all) do
+      csv_file = $dir + '/source.csv'
+      @data = extract from: csv_file
+    end
+ 
+    
+    it 'passes fields implicitly' do
+      expect(
+        transform(@data)
+      ).to eq(@data)
+
+      expect(
+        transform(@data) do
+          field :name
+        end
+      ).to eq(@data)
+    end
+
+    
+    it 'passes fields explicitly' do
+
+      # TODO: What happens when no field is passed?
+      
+      expect(
+
+        transform(@data, exclude: true) do
+          field :name
+        end
+
+      ).to eq([{name: 'Foo'}, {name: 'Bar'}])
+    end
+
+    
+    it 'filters fields explicitly' do
+
+      # TODO: What happens when all fields are deleted?
+      
+      expect(
+
+        transform(@data) do
+          del_field :age
+        end
+
+      ).to eq([{name: 'Foo'}, {name: 'Bar'}])
+    end
+ 
+    
+    it 'raises an exception if a field is missing' do
+      expect {
+
+        transform(@data) do
+          field :surname
+        end
+
+      }.to raise_error(MissingField,
+        "Required field 'surname' was not found")
+
+      expect {
+
+        transform(@data) do
+          del_field :surname
+        end
+
+      }.to raise_error(MissingField,
+        "Filtered field 'surname' was not found")
+    end
+
+    
+    it 'changes fields through *true* expressions' do
+      expect(
+
+        transform(@data) do
+          field :name, false
+          field :age, true
+        end
+
+      ).to eq([{name: 'Foo', age: true}, {name: 'Bar', age: true}])
+    end
+
+
+    it 'creates fields' do
+      expect(
+        transform(@data) do
+          new_field :new, true
+        end
+      ).to eq([{name: 'Foo', age: '27', new: true}, {name: 'Bar', age: '42', new: true}])
+
+      # TODO: A better explanation for this error.
+      
+      expect{
+        transform(@data) do
+          new_field :new
+        end
+      }.to raise_error(ArgumentError)
+      
+      expect {
+        transform(@data) do
+          new_field :name, true
+        end
+      }.to raise_error(ExistingField,
+          "New field 'name' already exists")
+    end
+
+    
+    it 'lets field names to be strings and allows field methods to co-occur' do
+      expect(
+
+        transform(@data) do
+          field 'name'
+          del_field 'age'
+          new_field 'new', true
+        end
+
+      ).to eq([{name: 'Foo', new: true}, {name: 'Bar', new: true}])
+    end
+
+    
+    it 'changes fields sending messages as symbols' do
+      expect(
+        transform(@data) do
+          field :name, :slice, 0
+          field :age, :to_i
+        end
+      ).to eq([{name: 'F', age: 27}, {name: 'B', age: 42}])
+      
+      expect(
+        transform(@data) do
+          field :name, 'slice', 0
+          field :age, 'to_i'
+        end
+      ).to eq([{name: 'slice', age: 'to_i'}, {name: 'slice', age: 'to_i'}])
+    end
+
+
+    it 'generates field getters per row' do
+      expect(
+
+        transform(@data) do
+          new_field :namage, name.downcase + age
+        end
+
+      ).to eq([{name: 'Foo', age: '27', namage: 'foo27'}, {name: 'Bar', age: '42', namage: 'bar42'}])
+    end
+
+
+    context 'combines records by' do
+
+      before(:all) do
+        @left_data = extract from: $dir + '/left_source.csv'
+        @right_data = extract from: $dir + '/right_source.csv'
+      end
+
+
+      it 'inner join' do
+
+        # TODO: A better explanation for this error.
+
+        expect {
+          transform(@left_data, join: @right_data)
+        }.to raise_error(ArgumentError)
+
+        expect(
+          transform(@left_data, join: @right_data, on: 'some_id') do
+            del_field :some_id
+            new_field :namage, name.downcase + age
+          end
+        ).to eq([{name: 'Foo', age: '27', namage: 'foo27'},
+                 {name: 'Foo', age: '33', namage: 'foo33'},
+                 {name: 'Bar', age: '42', namage: 'bar42'},
+                 {name: 'Baz', age: '42', namage: 'baz42'}])
+      end
+
+
+      it 'left outer join'
+      it 'right outer join'
+      it 'full outer join'
+
+    end
+  end
+
+
+  context 'aggregation' do
+
+    before(:all) do
+      csv_file = $dir + '/source2.csv'
+      @data = extract from: csv_file do
+        field :store
+        field :coffee_cups, type: Integer
+        field :name
+        field :croissants, type: Integer
+      end
+    end
+
+    it 'projects data to dimensions summing facts' do
+
+      # TODO: Facts must be additionable. Maybe raise an error?
+      # TODO: What happens with bad formatted data?
+
+      expect(
+
+        aggregate(@data) do
+          dim  :name
+          fact :coffee_cups
+          fact :croissants
+        end
+
+      ).to eq([{name: 'Foo', coffee_cups: 17, croissants: 14},
+               {name: 'Bar', coffee_cups:  8, croissants:  6}])
+    end
+    
+
+    it 'lets fields to be strings' do
+      expect(
+
+        aggregate(@data) do
+          dim  'name'
+          fact 'coffee_cups'
+        end
+
+      ).to eq([{name: 'Foo', coffee_cups: 17},
+               {name: 'Bar', coffee_cups:  8}])
+    end
+
+  end
+end

metadata

@@ -0,0 +1,163 @@
+--- !ruby/object:Gem::Specification
+name: datamancer
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Matías Battocchia
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord-jdbcsqlite3-adapter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A magical extract, transform, load (ETL) library for data integration.
+email:
+- matias@riseup.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- datamancer.gemspec
+- lib/datamancer.rb
+- lib/datamancer/extract.rb
+- lib/datamancer/load.rb
+- lib/datamancer/transform.rb
+- lib/datamancer/version.rb
+- spec/data/destination.csv
+- spec/data/destination.sqlite3
+- spec/data/left_source.csv
+- spec/data/right_source.csv
+- spec/data/simple.yml
+- spec/data/source.csv
+- spec/data/source.sqlite3
+- spec/data/source2.csv
+- spec/extract_spec.rb
+- spec/load_spec.rb
+- spec/spec_helper.rb
+- spec/transform_spec.rb
+homepage: https://github.com/matiasbattocchia/datamancer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.2
+signing_key: 
+specification_version: 4
+summary: ''
+test_files:
+- spec/data/destination.csv
+- spec/data/destination.sqlite3
+- spec/data/left_source.csv
+- spec/data/right_source.csv
+- spec/data/simple.yml
+- spec/data/source.csv
+- spec/data/source.sqlite3
+- spec/data/source2.csv
+- spec/extract_spec.rb
+- spec/load_spec.rb
+- spec/spec_helper.rb
+- spec/transform_spec.rb