remi 0.0.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 46611d84305353596aa2b69083c10515284f38fc
-  data.tar.gz: 455d0e91952b3d0a4961733ce3ee47c9baa6b10f
+  metadata.gz: 0e96cbd80b9948b85124da0bb1a7b5618c0692ec
+  data.tar.gz: 8eee478cbfdb78fc3c670c5fb2978f076b9d4673
 SHA512:
-  metadata.gz: 1ae4ff9142e4e0e856887e6b14922b43f224a4cfc759d70389f6cc09e724afddd73c8fed24d30bac4ea635087a8bc52af9fea58643a32015f3081dbea34b987b
-  data.tar.gz: fa19b1adbde11aa26792ca1b772540595218e847b69db5cc7c86f70db71d478c82660935b9df223fab0feec560fdb64bb8e1f7013c6bff29343a651f1ba9c789
+  metadata.gz: c355c76c0abbbd1bb9f37ff7d4b384b8c54772bbdfad3d1eff0a4531c3da063e9bb777aca0e69936cd5047c7f059b1f393da889b9ac047cbe0d77411d9e9fc18
+  data.tar.gz: 87df4f46a2f4f1127ccd885baf89e2466829dd56db840ce2d227c2ae8a7a96ec145873aac2ab9220a15b78158b8ff0afde56f314a6ecfa3a8c5a5523df89a154

data/.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_DISABLE_SHARED_GEMS: '1'

data/.gitignore

@@ -1,3 +1,4 @@
 *~
-/.ruby-version
-/.ruby-gemset
+/.yardoc
+/vendor
+/noodling

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/.ruby-version

@@ -0,0 +1 @@
+2.2.2

data/Gemfile

@@ -0,0 +1,4 @@
+# -*- mode: ruby -*-
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,123 @@
+PATH
+  remote: .
+  specs:
+    remi (0.2.1)
+      activesupport (~> 4.2)
+      bond (~> 0.5)
+      cucumber (~> 2.1)
+      daru (~> 0.1)
+      docile (~> 1.1)
+      net-sftp (~> 2.1)
+      pg (~> 0.18)
+      regexp-examples (~> 1.1)
+      restforce (~> 2.1)
+      rspec (~> 3.3)
+      salesforce_bulk_api (= 0.0.12)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.5)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    bond (0.5.1)
+    builder (3.2.2)
+    clbustos-rtf (0.4.2)
+    cucumber (2.3.0)
+      builder (>= 2.1.2)
+      cucumber-core (~> 1.4.0)
+      cucumber-wire (~> 0.0.1)
+      diff-lcs (>= 1.1.3)
+      event-bus (~> 0.1.0)
+      gherkin (~> 3.2.0)
+      multi_json (>= 1.7.5, < 2.0)
+      multi_test (>= 0.1.2)
+    cucumber-core (1.4.0)
+      gherkin (~> 3.2.0)
+    cucumber-wire (0.0.1)
+    daru (0.1.1)
+      reportbuilder (~> 1.4)
+      spreadsheet (~> 1.0.3)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    event-bus (0.1.0)
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
+    faraday_middleware (0.10.0)
+      faraday (>= 0.7.4, < 0.10)
+    gherkin (3.2.0)
+    hashie (3.4.3)
+    i18n (0.7.0)
+    iruby (0.2.7)
+      bond (~> 0.5)
+      mimemagic (~> 0.3)
+      multi_json (~> 1.11)
+      rbczmq (~> 1.7)
+    json (1.8.3)
+    mimemagic (0.3.1)
+    minitest (5.8.3)
+    multi_json (1.11.2)
+    multi_test (0.1.2)
+    multipart-post (2.0.0)
+    net-sftp (2.1.2)
+      net-ssh (>= 2.6.5)
+    net-ssh (3.0.2)
+    pg (0.18.4)
+    prawn (0.8.4)
+      prawn-core (>= 0.8.4, < 0.9)
+      prawn-layout (>= 0.8.4, < 0.9)
+      prawn-security (>= 0.8.4, < 0.9)
+    prawn-core (0.8.4)
+    prawn-layout (0.8.4)
+    prawn-security (0.8.4)
+    prawn-svg (0.9.1.11)
+      prawn (>= 0.8.4)
+    rbczmq (1.7.9)
+    regexp-examples (1.1.4)
+    reportbuilder (1.4.2)
+      clbustos-rtf (~> 0.4.0)
+      prawn (~> 0.8.4)
+      prawn-svg (~> 0.9.1)
+      text-table (~> 1.2)
+    restforce (2.1.2)
+      faraday (~> 0.9.0)
+      faraday_middleware (>= 0.8.8)
+      hashie (>= 1.2.0, < 4.0)
+      json (>= 1.7.5, < 1.9.0)
+    rspec (3.4.0)
+      rspec-core (~> 3.4.0)
+      rspec-expectations (~> 3.4.0)
+      rspec-mocks (~> 3.4.0)
+    rspec-core (3.4.1)
+      rspec-support (~> 3.4.0)
+    rspec-expectations (3.4.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-mocks (3.4.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-support (3.4.1)
+    ruby-ole (1.2.12)
+    salesforce_bulk_api (0.0.12)
+      json
+      xml-simple
+    spreadsheet (1.0.9)
+      ruby-ole (>= 1.0)
+    text-table (1.2.4)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    xml-simple (1.1.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  iruby (= 0.2.7)
+  remi!
+
+BUNDLED WITH
+   1.10.6

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 InsideTrack, Inc. ("REMI" name created by Sterling Paramore, all rights reserved.)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -1,4 +1,95 @@
-# Remi
-## Ruby Extract Modify and Integrate
+# Remi - Ruby Extract Map Integrate
 
-Something like SAS, but better.
+**Purpose:** Remi is a Ruby-based ETL package that provides an
+expressive data transformation language and facilitates the
+implementation and validation of non-technical business logic.
+
+Borrowing from principles of test/behavior-driven development
+(TDD/BDD), Remi is a system that supports business-rule-driven
+development (BRDD).  BRDD captures the idea that the rules that
+describe data transformations should both (1) be accessible to
+non-technical business users and (2) be strictly enforced in the logic
+that executes those transformations.  Remi is a Ruby application
+that allows a developer to write data transformation logic and have
+that logic validated according to business rule documentation.
+
+
+Remi will follow [semantic versioning](http://semver.org/) principles.
+Of course, while we're still on major version zero, little effort will
+be made to maintain backward compatibility.
+
+## Getting Started
+
+Add the gem to your Gemfile, `bundle install`, and then initialize your repository as
+Remi project
+
+    remi --init
+
+This command will create two directories: `jobs` and `features`.  The
+`jobs` directory contains an example of a Remi job that can be tested
+using the BRDD spec defined in the `features` directory.  Test to make
+sure this works by running
+
+    cucumber
+
+All of the test should pass.
+
+## Transforming Data
+
+TODO:
+
+Describe Daru foundation
+
+Examples setting up a job class with
+* csv source
+* sf source
+* dataframe intermediate target
+* csv target
+* parameters
+* maps
+
+## Business Rules
+
+TODO: Description of writing Business Rules.
+
+### Conventions to follow when writing features
+
+* Sources, targets, examples, field names enclosed in single quotes - `'field name'`
+* Field values enclosed in double quotes - `"field value"`
+* Special functions enclosed in stars - `*function*`
+* Example values encolsed in angular brackets - `<example>`
+
+Write whatever in scenario and feature descriptions
+
+### Common step library
+
+
+`Given the job is 'My Cool Job'`
+`Given the job source 'Client File'`
+`Given the job source 'Salesforce Extract'
+`Given the job target 'Salesforce Contact'`
+
+    Given the following example record called 'my killer example record':
+      | Id   | Name                |
+      | 1234 | OneTwoThreeFour     |
+
+... etc ...
+
+## Business Rule Validation
+
+TODO: Description of how to write Business Rule validations.
+
+
+
+
+
+## Contributing
+
+The best way to contribute would be to try it out and provide as much
+feedback as possible.
+
+If you want to develop the Remi framework then just fork, code, pull
+request, repeat.  Try to follow the
+[Ruby style guide](https://github.com/styleguide/ruby) and suggest
+other best practices.  I'm very interested in getting other ETL
+developers contribute their own perspective to the project.

data/bin/remi

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+File.expand_path(File.join(File.dirname(__FILE__),'../lib')).tap {|pwd| $LOAD_PATH.unshift(pwd) unless $LOAD_PATH.include?(pwd)}
+
+require 'bundler/setup'
+require 'remi'
+require 'remi/cli'
+
+Remi::Cli.execute

data/doc/install-rbenv-os_x.md

@@ -0,0 +1,47 @@
+## Installation using rbenv on OS X
+
+Using instructions found http://robots.thoughtbot.com/using-rbenv-to-manage-rubies-and-gems
+and http://dan.carley.co/blog/2012/02/07/rbenv-and-bundler/.  Here's how
+to get started with rbenv on os x
+
+    brew update
+    brew install rbenv
+
+Add the following lines to `.bashrc`:
+
+    # Homebrew rbenv
+    #To use Homebrew's directories rather than ~/.rbenv add to your profile:
+    export RBENV_ROOT=/usr/local/var/rbenv
+    if which rbenv > /dev/null; then eval "$(rbenv init -)"; fi
+
+Restart the shell and install
+
+    brew install rbenv-gem-rehash
+    brew install ruby-build
+
+Restart the shell and then install the rubies
+
+    rbenv install 2.2.2
+    rbenv global 2.2.2
+
+Install rbenv-bundler
+
+    brew install rbenv-bundler
+    rbenv bundler on
+
+Install bundler
+
+    gem update --system
+    gem install bundler
+    rbenv rehash
+
+Configure bundler by adding the following to `~/.bundle/config`
+
+    ---
+    BUNDLE_PATH: vendor/bundle
+    BUNDLE_DISABLE_SHARED_GEMS: "1"
+
+Finally, move into the Remi directory and install gems
+
+    bundle install
+    rbenv rehash

data/lib/remi.rb

@@ -1,12 +1,59 @@
-#!/usr/bin/ruby
+File.expand_path(File.dirname(__FILE__)).tap {|pwd| $LOAD_PATH.unshift(pwd) unless $LOAD_PATH.include?(pwd)}
 
-class Hola
-  def self.hi
-    puts "Hello world!"
-  end
-  def self.bye
-    puts "Goodbye, cruel world!"
-  end
-end
+# Core libraries
+require 'yaml'
+require 'json'
+require 'tmpdir'
 
+# Gems
+require 'daru'
+require 'docile'
+require 'net/sftp'
+require 'pg'
 
+# ActiveSupport extensions
+require 'active_support'
+require 'active_support/core_ext/object/conversions'
+require 'active_support/core_ext/object/blank'
+require 'active_support/core_ext/object/try'
+require 'active_support/core_ext/object/inclusion'
+require 'active_support/core_ext/string/inflections'
+require 'active_support/core_ext/numeric/time'
+require 'active_support/core_ext/numeric/conversions'
+require 'active_support/core_ext/date/calculations'
+require 'active_support/core_ext/time/calculations'
+
+# - Should separate SF stuff into separate SF support package
+require 'restforce'
+require 'salesforce_bulk_api'
+
+
+# Remi
+require 'remi/version.rb'
+require 'remi/core/string.rb'
+
+require 'remi/settings'
+require 'remi/job'
+require 'remi/source_to_target_map'
+require 'remi/field_symbolizers'
+require 'remi/data_subject'
+require 'remi/sf_bulk_helper' # separate into SF support package
+
+require 'remi/core/daru'
+require 'remi/core/refinements'
+
+require 'remi/extractor/sftp_file'
+
+require 'remi/data_source.rb'
+require 'remi/data_source/data_frame'
+require 'remi/data_source/csv_file'
+require 'remi/data_source/salesforce'
+require 'remi/data_source/postgres'
+
+require 'remi/data_target.rb'
+require 'remi/data_target/data_frame'
+require 'remi/data_target/salesforce'
+require 'remi/data_target/csv_file'
+
+require 'remi/lookup/regex_sieve'
+require 'remi/transform'

data/lib/remi/cli.rb

@@ -0,0 +1,56 @@
+require 'optparse'
+require 'fileutils'
+
+
+module Remi
+  module Cli
+    extend self
+
+    def execute
+      parse
+      initialize_project if @options[:init] == true
+    end
+
+    def parse(args = ARGV)
+      options = {}
+
+      opt_parser = OptionParser.new do |opts|
+        opts.banner = <<-EOT.unindent
+          Usage: Command line helpers for Remi.
+        EOT
+
+        opts.on('-h', '--help', 'Show this message') do
+          puts opts
+          exit
+        end
+
+        options[:init] = false
+        opts.on('-i', '--init', 'Initialze a new Remi project') do
+          options[:init] = true
+        end
+      end
+      opt_parser.parse!(args)
+
+      @options = options
+    end
+
+    def initialize_project
+      template_dir = File.expand_path(File.join(File.dirname(__FILE__),'project'))
+
+      FileUtils.mkdir_p "features"
+      FileUtils.cp(File.join(template_dir, 'features/sample_job.feature'), 'features')
+
+      FileUtils.mkdir_p "features/support"
+      FileUtils.cp(File.join(template_dir, 'features/support/env.rb'), 'features/support')
+      FileUtils.cp(File.join(template_dir, 'features/support/env_app.rb'), 'features/support') unless File.exist?('features/support/env_app.rb')
+
+      FileUtils.mkdir_p "features/step_definitions"
+      FileUtils.cp(File.join(template_dir, 'features/step_definitions/remi_step.rb'), 'features/step_definitions')
+
+      FileUtils.mkdir_p "jobs"
+      FileUtils.cp(File.join(template_dir, 'jobs/all_jobs_shared.rb'), 'jobs') unless File.exist?('jobs/all_jobs_shared.rb')
+      FileUtils.cp(File.join(template_dir, 'jobs/sample_job.rb'), 'jobs')
+    end
+
+  end
+end