datamancer 0.0.5 → 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 89a4e4a23249888271ce7e622cef69ac4872369a
+  data.tar.gz: e74f08a6e9b943a74e1527d39ded6ffdd96adcf3
+SHA512:
+  metadata.gz: 44b412595b669021ecc519d5d9e0eb70b9c8160a92480e5960c759d64afd55f7f0145393aa311d0831a668fb34cd0f1421245571adbd985b8681d90f114ee43b
+  data.tar.gz: 41c415bfd5434eac51d8d210fd29c5bb8ff92e959b8df9c93e208ca8d61898739e3cd801c7847cfbe645b4b50b81740f0785e34d65898e084423934307d6d7e9

data/lib/datamancer.rb

@@ -1,4 +1,5 @@
 require 'datamancer/version'
+require 'datamancer/datastream'
 require 'datamancer/extract'
 require 'datamancer/transform'
 require 'datamancer/load'

data/lib/datamancer/datastream.rb

@@ -0,0 +1,18 @@
+class Array
+  
+  def where criteria
+    self.select do |row|
+      result = true
+
+      criteria.each do |field, value|
+        unless row[field] == value
+          result = false
+          break
+        end
+      end
+
+      result
+    end
+  end
+
+end

data/lib/datamancer/extract.rb

@@ -69,20 +69,29 @@ module Datamancer
       csv
 
     when Hash
-      columns = @fields.map { |field, mapping| "#{mapping} AS #{field}" }.join(', ')
+
+      #TODO: Test for column names with spaces.
+      #TODO: Implement all the SQL escaping cases.
+
+      columns = @fields.map { |field, mapping| "[#{mapping}] AS [#{field}]" }.join(', ')
 
       @fields.keys.each_with_index do |field, index|
         @fields[field] = index
       end
 
-      # TODO: Test top.
+      # TODO: Test top, distinct, where.
       # TODO: Top for CSV.
       # TODO: Top for support several databases.
 
-      if args[:top]
-        db.select_rows("SELECT TOP #{args[:top]} #{columns} FROM #{table}" )
+      if args[:distinct]
+        db.select_rows(
+          "SELECT DISTINCT #{columns} FROM #{table}")
+      elsif args[:top]
+        db.select_rows(
+          "SELECT TOP #{args[:top]} #{columns} FROM #{table} #{'WHERE ' + args[:where] if args[:where]}")
       else
-        db.select_rows("SELECT #{columns} FROM #{table}")
+        db.select_rows(
+          "SELECT #{columns} FROM #{table} #{'WHERE ' + args[:where] if args[:where]}")
       end
     end
 

data/lib/datamancer/load.rb

@@ -1,5 +1,50 @@
 module Datamancer
 
+  def raw input, args
+
+    raise ArgumentError,
+      'Raw requires a destination, i.e. raw(data, to: destination)' unless
+        args.is_a?(Hash) && args[:to]
+
+    ::ActiveRecord::Base.establish_connection(args[:to])
+
+    # TODO: Test this.
+
+    # table = args[:table] || args[:to][:table]
+      
+    # raise ArgumentError,
+    #   'Raw requires a database table, i.e. raw(data, to: destination, table: table_name)' unless table
+
+    # TODO: Method-overriding safeguard.
+  
+    input.first.each_key do |key|
+      define_singleton_method key.downcase do
+      
+        # Some methods applied to fields might modify the original fields.
+        # Fields could be duplicated in case this be a common problem.
+
+        #@input_row[key].dup
+
+        @input_row[key]
+      end
+    end
+
+    define_singleton_method :db do
+      ::ActiveRecord::Base.connection
+    end
+    
+    define_singleton_method :query do |query|
+      ::ActiveRecord::Base.connection.execute query
+    end
+
+    input.each do |row|
+      @input_row = row
+      
+      yield if block_given?
+    end
+  end
+
+
   def load input, args
 
     raise ArgumentError,
@@ -64,7 +109,7 @@ module Datamancer
       table = args[:table] || args[:to][:table]
       
       raise ArgumentError,
-        'Load requires a database table, i.e. load(to: destination, table: table_name)' unless table
+        'Load requires a database table, i.e. load(data, to: destination, table: table_name)' unless table
 
       ::ActiveRecord::Base.connection.delete("DELETE FROM #{table}") unless args[:append]
       batch_size = args[:batch] || 1000

data/lib/datamancer/transform.rb

@@ -1,5 +1,35 @@
 module Datamancer
 
+  def add left, right
+    first_row = left.first.merge right.first
+
+    keys = first_row.keys
+
+    valores_por_defecto = {}
+
+    keys.each do |key|
+      valores_por_defecto[key] = case first_row[key]
+                                 when String then ''
+                                 when Numeric then 0
+                                 else nil end
+    end
+
+    output = []
+
+    (left + right).each do |input_row|
+
+      output_row = {}
+
+      keys.each do |key|
+        output_row[key] = input_row[key] || valores_por_defecto[key]
+      end
+
+      output << output_row
+    end
+
+    output
+  end
+
   def join left, right, attribute
 
     attribute = attribute.to_sym
@@ -59,6 +89,8 @@ module Datamancer
 
   def transform input, args = {}
 
+    # TODO: Mensajes que expliquen mejor los estos errores.
+
     if args[:join]
       raise ArgumentError unless args[:on]
       raise ArgumentError unless input.first.keys.include?(args[:on].to_sym)
@@ -67,6 +99,10 @@ module Datamancer
       input = join input, args[:join], args[:on]
     end
 
+    if args[:add]
+      input = add input, args[:add]
+    end
+
     if args[:unique]
       input = unique input, args[:unique]
     end
@@ -74,13 +110,13 @@ module Datamancer
     # TODO: Method-overriding safeguard.
   
     input.first.each_key do |key|
-      define_singleton_method key.downcase do
+      define_singleton_method key.to_s.gsub(' ', '_').downcase do
       
         # Some methods applied to fields might modify the original fields.
         # Fields could be duplicated in case this be a common problem.
 
         #@input_row[key].dup
-
+        
         @input_row[key]
       end
     end

data/lib/datamancer/version.rb

@@ -1,3 +1,3 @@
 module Datamancer
-  VERSION = '0.0.5'
+  VERSION = '0.1.0'
 end

data/spec/datastream_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe Datamancer do
+
+  before(:all) do
+    @datastream = [
+      {a: 1, b: 2, c: 3},
+      {a: 1, b: 1, c: 3},
+      {a: 1, b: 1, c: 1}
+    ]
+  end
+
+  it 'selects rows that match a criteria' do
+    expect(
+      @datastream.where(a: 1, b: 1)
+    ).to eq(
+      [{a: 1, b: 1, c: 3},
+       {a: 1, b: 1, c: 1}])
+  end
+end

metadata

@@ -1,112 +1,99 @@
 --- !ruby/object:Gem::Specification
 name: datamancer
 version: !ruby/object:Gem::Version
-  version: 0.0.5
-  prerelease:
+  version: 0.1.0
 platform: ruby
 authors:
 - Matías Battocchia
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-20 00:00:00.000000000 Z
+date: 2014-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.3'
-    none: false
-  requirement: !ruby/object:Gem::Requirement
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.3'
-    none: false
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
   name: rake
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rspec
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
+- !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: activerecord
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
+- !ruby/object:Gem::Dependency
+  name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: sqlite3
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
+- !ruby/object:Gem::Dependency
+  name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: activerecord-jdbcsqlite3-adapter
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
+- !ruby/object:Gem::Dependency
+  name: activerecord-jdbcsqlite3-adapter
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
-  prerelease: false
   type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A magical extract, transform, load (ETL) library for data integration.
 email:
 - matias@riseup.net
@@ -121,6 +108,7 @@ files:
 - Rakefile
 - datamancer.gemspec
 - lib/datamancer.rb
+- lib/datamancer/datastream.rb
 - lib/datamancer/extract.rb
 - lib/datamancer/load.rb
 - lib/datamancer/transform.rb
@@ -131,6 +119,7 @@ files:
 - spec/data/source.csv
 - spec/data/source.sqlite3
 - spec/data/source2.csv
+- spec/datastream_spec.rb
 - spec/extract_spec.rb
 - spec/load_spec.rb
 - spec/spec_helper.rb
@@ -138,7 +127,8 @@ files:
 homepage: https://github.com/matiasbattocchia/datamancer
 licenses:
 - MIT
-post_install_message:
+metadata: {}
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -147,18 +137,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-  none: false
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-  none: false
 requirements: []
-rubyforge_project:
-rubygems_version: 1.8.24
-signing_key:
-specification_version: 3
+rubyforge_project: 
+rubygems_version: 2.0.2
+signing_key: 
+specification_version: 4
 summary: ''
 test_files:
 - spec/data/destination.csv
@@ -167,6 +155,7 @@ test_files:
 - spec/data/source.csv
 - spec/data/source.sqlite3
 - spec/data/source2.csv
+- spec/datastream_spec.rb
 - spec/extract_spec.rb
 - spec/load_spec.rb
 - spec/spec_helper.rb