remi 0.0.1 → 0.2.2

data/lib/remi/source_to_target_map.rb

@@ -0,0 +1,93 @@
+module Remi
+  class SourceToTargetMap
+    def initialize(source_df, target_df=nil)
+      @source_df = source_df
+      @target_df = target_df || @source_df
+
+      reset_map
+    end
+
+    def self.apply(source_df, target_df=nil, &block)
+      target_df ||= source_df
+      Docile.dsl_eval(SourceToTargetMap.new(source_df, target_df), &block)
+    end
+
+    def source(*source_fields)
+      @source_fields = Array(source_fields)
+      self
+    end
+
+    def transform(*transforms)
+      @transforms += Array(transforms)
+      self
+    end
+
+    def target(*target_fields)
+      @target_fields = Array(target_fields)
+      self
+    end
+
+    def reset_map
+      @source_fields = []
+      @target_fields = []
+      @transforms = []
+    end
+
+    def map(*args)
+      case
+      when @source_fields.include?(nil)
+        do_map_generic
+      when @source_fields.size == 1 && @transforms.size == 0
+        do_map_direct_copy
+      when @source_fields.size == 1 && @target_fields.size == 1
+        do_map_single_source_and_target_field
+      else
+        do_map_generic
+      end
+
+      reset_map
+    end
+
+
+
+    private
+
+    def do_map_direct_copy
+      @target_fields.each do |target_field|
+        @target_df[target_field] = @source_df[@source_fields.first].dup
+      end
+    end
+
+    def do_map_single_source_and_target_field
+      @target_df[@target_fields.first] = @source_df[@source_fields.first].recode do |field_value|
+        @transforms.reduce(field_value) { |value, tform| tform.call(*(value || [nil])) }
+      end
+    end
+
+    def do_map_generic
+      work_vector = if @source_fields.size == 1 && @source_fields.first != nil
+        @source_df[@source_fields.first].dup
+      elsif @source_fields.size > 1
+        # It's faster to zip together several vectors and recode those than it is to
+        # recode a dataframe row by row!
+        Daru::Vector.new(@source_df[@source_fields.first].zip(*@source_fields[1..-1].map { |name| @source_df[name] }), index: @source_df.index)
+      else
+        Daru::Vector.new([], index: @source_df.index)
+      end
+
+      work_vector.recode! do |field_value|
+        @transforms.reduce(field_value) { |value, tform| tform.call(*(value || [nil])) }
+      end
+
+      @target_fields.each_with_index do |target_field, field_idx|
+        @target_df[target_field] = work_vector.recode do |field_value|
+          if field_value.is_a?(Array) then
+            field_value[field_idx]
+          else
+            field_value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/remi/transform.rb

@@ -0,0 +1,137 @@
+module Remi
+  module Transform
+    extend self
+
+    def [](meth)
+      method(meth)
+    end
+
+    # We need to memoize each lambda with its static arguments so it's not recreated each row.
+    # Inspired by parameter memoization in http://www.justinweiss.com/articles/4-simple-memoization-patterns-in-ruby-and-one-gem/
+    def memoize_as_lambda(func, *args, &block)
+      iv = instance_variable_get("@#{func}")
+      return iv[args] if iv
+
+      hash_memo = Hash.new do |h, margs|
+        h[margs] = lambda { |*largs| block.call(margs, *largs) }
+      end
+      instance_variable_set("@#{func}", hash_memo)[args]
+    end
+
+    def prefix(prefix)
+      memoize_as_lambda(__method__, prefix) do |(mprefix), larg|
+        "#{mprefix}#{larg}"
+      end
+    end
+
+    def postfix(postfix)
+      memoize_as_lambda(__method__, postfix) do |(mpostfix), larg|
+        "#{larg}#{mpostfix}"
+      end
+    end
+
+    def concatenate(delimiter="")
+      memoize_as_lambda(__method__, delimiter) do |(mdelimiter), *largs|
+        Array(largs).join(mdelimiter)
+      end
+    end
+
+    def lookup(h_lookup, missing: nil)
+      memoize_as_lambda(__method__, h_lookup, missing) do |(mh_lookup, mmissing), larg|
+        result = mh_lookup[larg]
+
+        if !result.nil?
+          result
+        elsif mmissing.class == Proc
+          mmissing.call(larg)
+        else
+          mmissing
+        end
+      end
+    end
+
+    def nvl
+      memoize_as_lambda(__method__) do |*largs|
+        Array(largs).find('') { |arg| !arg.blank? }
+      end
+    end
+
+    def ifblank(replace_with)
+      memoize_as_lambda(__method__, replace_with) do |(mreplace_with), larg|
+        larg.blank? ? mreplace_with : larg
+      end
+    end
+
+    def format_date(from_fmt: '%m/%d/%Y', to_fmt: '%Y-%m-%d')
+      memoize_as_lambda(__method__, from_fmt, to_fmt) do |(mfrom_fmt, mto_fmt), larg|
+        begin
+          if larg.blank? then
+            ''
+          elsif larg.respond_to? :strftime
+            larg.strftime(mto_fmt)
+          else
+            Date.strptime(larg, mfrom_fmt).strftime(mto_fmt)
+          end
+        rescue ArgumentError => err
+          puts "Error parsing date (#{larg.class}): '#{larg}'"
+          raise err
+        end
+      end
+    end
+
+
+    def parse_date(format: '%Y-%m-%d', if_blank: nil)
+      memoize_as_lambda(__method__, format, if_blank.try(:to_sym)) do |(mformat, mif_blank), larg|
+        begin
+          if larg.blank? then
+            if mif_blank == :low
+              Date.new(1900,01,01)
+            elsif mif_blank == :high
+              Date.new(2999,12,31)
+            else
+              mif_blank
+            end
+          else
+            Date.strptime(larg, mformat)
+          end
+        rescue ArgumentError => err
+          puts "Error parsing date (#{larg.class}): '#{larg}')"
+          raise err
+        end
+      end
+    end
+
+    def date_diff(measure = :days)
+      memoize_as_lambda(__method__, measure.to_sym) do |(mmeasure), *larg|
+        if mmeasure == :days
+          (larg.last - larg.first).to_i
+        elsif mmeasure == :months
+          (larg.last.year * 12 + larg.last.month) - (larg.first.year * 12 + larg.first.month)
+        elsif mmeasure == :years
+          larg.last.year - larg.first.year
+        else
+          raise "I don't know how to handle #{mmeasure} yet"
+        end
+      end
+    end
+
+    def constant(const)
+      memoize_as_lambda(__method__, const) do |(mconst), larg|
+        mconst
+      end
+    end
+
+    def replace(regex, replace_with)
+      memoize_as_lambda(__method__, regex, replace_with) do |(mregex, mreplace_with), larg|
+        larg.gsub(regex, replace_with)
+      end
+    end
+
+    def validate_email(substitute='')
+      memoize_as_lambda(__method__, substitute) do |(msubstitute), larg|
+        larg.match(/^.+@[a-z0-9\-\.]+$/i) ? larg : msubstitute
+      end
+    end
+
+  end
+end

data/lib/remi/version.rb

@@ -0,0 +1,3 @@
+module Remi
+  VERSION = '0.2.2'
+end

data/remi.gemspec

@@ -1,18 +1,36 @@
-# -*- encoding: utf-8 -*-
+# -*- encoding: utf-8 mode: ruby -*-
 $:.push File.expand_path("../lib", __FILE__)
 require "remi/version"
 
 Gem::Specification.new do |s|
   s.name        = "remi"
-  s.version     = "0.0.1"
+  s.version     = Remi::VERSION
   s.authors     = ["Sterling Paramore"]
-  s.email       = ["gnilrets@gmail.com"]
-  s.homepage    = "https://github.com/gnilrets"
-  s.license     = "Apache-2.0"
-  s.summary     = "Remi (Ruby Extract Modify and Integrate)"
-  s.description = "Data manipulation in Ruby"
+  s.email       = ["sterling.paramore@insidetrack.com"]
+  s.homepage    = "https://github.com/inside-track/remi"
+  s.license     = "MIT"
+  s.summary     = "Remi (Ruby Extract Map Integrate)"
+  s.description = "Data manipulation and ETL in Ruby"
 
   s.rubyforge_project = "Remi"
+  s.add_runtime_dependency "daru", ["~> 0.1"]
+
+  s.add_runtime_dependency 'bond', ['~> 0.5']
+  s.add_runtime_dependency 'docile', ['~> 1.1']
+  s.add_runtime_dependency 'net-sftp', ['~> 2.1']
+  s.add_runtime_dependency 'pg', ['~> 0.18']
+
+  s.add_runtime_dependency "cucumber", ["~> 2.1"]
+  s.add_runtime_dependency "rspec", ["~> 3.3"]
+  s.add_runtime_dependency "regexp-examples", ["~> 1.1"]
+
+  s.add_runtime_dependency "activesupport", ["~> 4.2"]
+
+  # Move these into separate package
+  s.add_runtime_dependency 'restforce', ['~> 2.1']
+  s.add_runtime_dependency 'salesforce_bulk_api', ['0.0.12']
+
+  s.add_development_dependency 'iruby', ['0.2.7']
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/workbooks/sample_workbook.ipynb

@@ -0,0 +1,56 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "require_relative \"#{Dir.pwd}/workbook_helper\"\n",
+    "\n",
+    "require 'benchmark'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "# Safest to delete and rebuild class when developing\n",
+    "Object.send(:remove_const, :SomeJob) if Object.constants.include?(:SomeJob)\n",
+    "\n",
+    "class SomeJob\n",
+    "  include Remi::Job\n",
+    "  define_target :my_lookup, Hash\n",
+    "  define_transform :populate_lookup, targets: :my_lookup do\n",
+    "    my_lookup[:happy] = \"bear\"\n",
+    "    my_lookup[:smacks] = \"honey\"\n",
+    "  end\n",
+    "end\n",
+    "\n",
+    "job = SomeJob.new.run\n",
+    "job.my_lookup"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Ruby 2.2.2",
+   "language": "ruby",
+   "name": "ruby"
+  },
+  "language_info": {
+   "file_extension": "rb",
+   "mimetype": "application/x-ruby",
+   "name": "ruby",
+   "version": "2.2.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}

data/workbooks/workbook_helper.rb

@@ -0,0 +1 @@
+require_relative '../lib/remi'

metadata

@@ -1,32 +1,250 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-01 00:00:00.000000000 Z
-dependencies: []
-description: Data manipulation in Ruby
+date: 2016-01-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: daru
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: bond
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: docile
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: net-sftp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: regexp-examples
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: restforce
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: salesforce_bulk_api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.12
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.12
+- !ruby/object:Gem::Dependency
+  name: iruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.7
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.7
+description: Data manipulation and ETL in Ruby
 email:
-- gnilrets@gmail.com
-executables: []
+- sterling.paramore@insidetrack.com
+executables:
+- remi
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".bundle/config"
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
 - README.md
-- lib/noodling.rb
+- bin/remi
+- doc/install-rbenv-os_x.md
 - lib/remi.rb
+- lib/remi/cli.rb
+- lib/remi/core/daru.rb
+- lib/remi/core/refinements.rb
+- lib/remi/core/string.rb
+- lib/remi/cucumber.rb
+- lib/remi/cucumber/business_rules.rb
+- lib/remi/cucumber/data_source.rb
+- lib/remi/data_source.rb
+- lib/remi/data_source/csv_file.rb
+- lib/remi/data_source/data_frame.rb
+- lib/remi/data_source/postgres.rb
+- lib/remi/data_source/salesforce.rb
+- lib/remi/data_subject.rb
+- lib/remi/data_target.rb
+- lib/remi/data_target/csv_file.rb
+- lib/remi/data_target/data_frame.rb
+- lib/remi/data_target/salesforce.rb
+- lib/remi/extractor/sftp_file.rb
+- lib/remi/field_symbolizers.rb
+- lib/remi/job.rb
+- lib/remi/lookup/regex_sieve.rb
+- lib/remi/project/features/examples.feature
+- lib/remi/project/features/formulas.feature
+- lib/remi/project/features/sample_job.feature
+- lib/remi/project/features/step_definitions/remi_step.rb
+- lib/remi/project/features/support/env.rb
+- lib/remi/project/features/support/env_app.rb
+- lib/remi/project/features/transforms/date_diff.feature
+- lib/remi/project/features/transforms/parse_date.feature
+- lib/remi/project/features/transforms/prefix.feature
+- lib/remi/project/jobs/all_jobs_shared.rb
+- lib/remi/project/jobs/copy_source_job.rb
+- lib/remi/project/jobs/sample_job.rb
+- lib/remi/project/jobs/transforms/date_diff_job.rb
+- lib/remi/project/jobs/transforms/parse_date_job.rb
+- lib/remi/project/jobs/transforms/prefix_job.rb
+- lib/remi/project/jobs/transforms/transform_jobs.rb
+- lib/remi/settings.rb
+- lib/remi/sf_bulk_helper.rb
+- lib/remi/source_to_target_map.rb
+- lib/remi/transform.rb
 - lib/remi/version.rb
 - remi.gemspec
-- test/test_NAME.rb
-homepage: https://github.com/gnilrets
+- workbooks/sample_workbook.ipynb
+- workbooks/workbook_helper.rb
+homepage: https://github.com/inside-track/remi
 licenses:
-- Apache-2.0
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -34,19 +252,18 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: Remi
-rubygems_version: 2.2.1
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
-summary: Remi (Ruby Extract Modify and Integrate)
-test_files:
-- test/test_NAME.rb
+summary: Remi (Ruby Extract Map Integrate)
+test_files: []