forklift_etl 1.0.8

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NWRiNDNmYTQwZTQ3ZjY4ZmY4Y2FhYzQxOGRjMGY3ZjAxNjYzNmJjMw==
+  data.tar.gz: !binary |-
+    NTdjMWM2OTljNTIxYTk1YzA3ZWUyZmM3MTdlNmIxZmQ1YmMxNWM5Mw==
+SHA512:
+  metadata.gz: !binary |-
+    YjJjZDc2NjQxODY0ZjU0MGExNWIxY2VlZTcwMWVkYzNkZjA1MmZiZmM0NWRl
+    ZjFmNmFhNWI3NGNmYjE5NmU0Y2I4ODUxNTQxYjgzMjA2OTljOTA3M2UwZmM2
+    YTMzZTQ0ZjM5ODBjMjA3MDFkNjVjMGUzNDg3NjAyMmJiODBhZjA=
+  data.tar.gz: !binary |-
+    MDkxYTM1YTUzODczYzg1ZDJmNTJkZjM4NGU2NTE3ZWRjZWVkMWRiYmQwMWI3
+    YzVjNTU5NWQzZDY0YzY5OTIwNGRiZmViMDQwMjJjYjFjNzIxOTI1MjExN2Zm
+    Mjg4YzNjY2E2MDYxMzQ1M2ZmM2ZjNDY4NmI2Yjc5Yzc2MWY5Nzc=

data/.gitignore

@@ -0,0 +1,10 @@
+.DS_Store
+*.gem
+*/.DS_Store
+pid
+log
+config/databases.yml
+config/email.yml
+config/dump.yml
+/deploy
+/config

data/.rbenv-version

@@ -0,0 +1 @@
+1.9.3-p194

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+  - rbx
+  # - jruby-19mode # TODO: We'll need a ODBC variant of the mysql2 driver
+services:
+  - elasticsearch
+  - mysql

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in gemspec
+gemspec
+
+gem 'rake'
+gem 'rspec'
+gem 'awesome_print'
+gem 'email_spec'
+# gem 'debugger'

data/Gemfile.lock

@@ -0,0 +1,74 @@
+PATH
+  remote: .
+  specs:
+    forklift_etl (1.0.7)
+      activesupport (~> 4.0, >= 4.0.0)
+      elasticsearch (~> 1.0, >= 1.0.0)
+      lumberjack (~> 1.0, >= 1.0.0)
+      mysql2 (~> 0.0, >= 0.0.1)
+      pony (~> 1.0, >= 1.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.0.4)
+      i18n (~> 0.6, >= 0.6.9)
+      minitest (~> 4.2)
+      multi_json (~> 1.3)
+      thread_safe (~> 0.1)
+      tzinfo (~> 0.3.37)
+    addressable (2.3.6)
+    awesome_print (1.2.0)
+    diff-lcs (1.2.5)
+    elasticsearch (1.0.1)
+      elasticsearch-api (= 1.0.1)
+      elasticsearch-transport (= 1.0.1)
+    elasticsearch-api (1.0.1)
+      multi_json
+    elasticsearch-transport (1.0.1)
+      faraday
+      multi_json
+    email_spec (1.5.0)
+      launchy (~> 2.1)
+      mail (~> 2.2)
+    faraday (0.9.0)
+      multipart-post (>= 1.2, < 3)
+    i18n (0.6.9)
+    launchy (2.4.2)
+      addressable (~> 2.3)
+    lumberjack (1.0.5)
+    mail (2.5.4)
+      mime-types (~> 1.16)
+      treetop (~> 1.4.8)
+    mime-types (1.25.1)
+    minitest (4.7.5)
+    multi_json (1.9.2)
+    multipart-post (2.0.0)
+    mysql2 (0.3.15)
+    polyglot (0.3.4)
+    pony (1.8)
+      mail (>= 2.0)
+    rake (10.2.2)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.8)
+    rspec-expectations (2.14.5)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.6)
+    thread_safe (0.3.3)
+    treetop (1.4.15)
+      polyglot
+      polyglot (>= 0.3.1)
+    tzinfo (0.3.39)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  awesome_print
+  email_spec
+  forklift_etl!
+  rake
+  rspec

data/Rakefile

@@ -0,0 +1,13 @@
+#!/usr/bin/env rake
+require 'rake'
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = [ 
+    'spec/*/*_spec.rb', 
+    'spec/*/*/*_spec.rb',
+    'spec/*/*/*/*_spec.rb',
+  ]
+end
+task :default  => :spec

data/bin/forklift

@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'fileutils'
+
+begin
+  require 'forklift/forklift'
+rescue LoadError
+  require "#{File.expand_path(File.dirname(__FILE__))}/../lib/forklift/forklift.rb"
+end
+
+def generate
+  p = Dir.pwd
+
+  Dir.mkdir "#{p}/config"
+  Dir.mkdir "#{p}/config/connections"
+  Dir.mkdir "#{p}/config/connections/mysql"
+  Dir.mkdir "#{p}/log"
+  Dir.mkdir "#{p}/pid"
+  Dir.mkdir "#{p}/template"
+  Dir.mkdir "#{p}/transformations"
+  Dir.mkdir "#{p}/transports"
+  Dir.mkdir "#{p}/patterns"
+
+  template('source.yml',      "#{p}/config/connections/mysql/source.yml")
+  template('destination.yml', "#{p}/config/connections/mysql/destination.yml")
+  template('email.yml',       "#{p}/config/email.yml")
+  template('email.erb',       "#{p}/template/email.erb")
+  template('plan.rb',         "#{p}/plan.rb")
+end
+
+def template(source, destination)
+  t = "#{File.expand_path(File.dirname(__FILE__))}/../template"
+  FileUtils.copy("#{t}/#{source}", destination)
+  puts "Example plan generated"
+end
+
+
+def run_plan
+  file = "#{Dir.pwd}/#{ARGV[0]}"
+  if ARGV[0].nil? 
+    puts "[error] Please provide a plan.rb as the first argument"
+    exit(1)
+  end
+  Dir.chdir File.expand_path(File.dirname(ARGV[0]))
+  begin
+    require 'bundler'
+    Bundler.require(:default)
+  rescue Exception => e
+    puts "cannot load bundler: #{e}"
+  end
+  require file
+end
+
+############
+
+if ['--generate', '-generate'].include?(ARGV[0])
+  generate
+else
+  run_plan
+end

data/doc/EmailSuffix.html

@@ -0,0 +1,228 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+<meta content="text/html; charset=UTF-8" http-equiv="Content-Type">
+
+<title>class EmailSuffix - RDoc Documentation</title>
+
+<link type="text/css" media="screen" href="./rdoc.css" rel="stylesheet">
+
+<script type="text/javascript">
+  var rdoc_rel_prefix = "./";
+</script>
+
+<script type="text/javascript" charset="utf-8" src="./js/jquery.js"></script>
+<script type="text/javascript" charset="utf-8" src="./js/navigation.js"></script>
+<script type="text/javascript" charset="utf-8" src="./js/search_index.js"></script>
+<script type="text/javascript" charset="utf-8" src="./js/search.js"></script>
+<script type="text/javascript" charset="utf-8" src="./js/searcher.js"></script>
+<script type="text/javascript" charset="utf-8" src="./js/darkfish.js"></script>
+
+
+<body id="top" class="class">
+<nav id="metadata">
+  <nav id="home-section" class="section">
+  <h3 class="section-header">
+    <a href="./index.html">Home</a>
+    <a href="./table_of_contents.html#classes">Classes</a>
+    <a href="./table_of_contents.html#methods">Methods</a>
+  </h3>
+</nav>
+
+
+  <nav id="search-section" class="section project-section" class="initially-hidden">
+  <form action="#" method="get" accept-charset="utf-8">
+    <h3 class="section-header">
+      <input type="text" name="search" placeholder="Search" id="search-field"
+             title="Type to search, Up and Down to navigate, Enter to load">
+    </h3>
+  </form>
+
+  <ul id="search-results" class="initially-hidden"></ul>
+</nav>
+
+
+  <div id="file-metadata">
+    <nav id="file-list-section" class="section">
+  <h3 class="section-header">Defined In</h3>
+  <ul>
+    <li>example/transformations/email_suffix.rb
+  </ul>
+</nav>
+
+    
+  </div>
+
+  <div id="class-metadata">
+    
+    <nav id="parent-class-section" class="section">
+  <h3 class="section-header">Parent</h3>
+  
+  <p class="link"><a href="Object.html">Object</a>
+  
+</nav>
+
+    
+    <!-- Method Quickref -->
+<nav id="method-list-section" class="section">
+  <h3 class="section-header">Methods</h3>
+
+  <ul class="link-list">
+    
+    <li><a href="#method-i-do-21">#do!</a>
+    
+  </ul>
+</nav>
+
+  </div>
+
+  <div id="project-metadata">
+    <nav id="fileindex-section" class="section project-section">
+  <h3 class="section-header">Pages</h3>
+
+  <ul>
+  
+    <li class="file"><a href="./Gemfile.html">Gemfile</a>
+  
+    <li class="file"><a href="./Rakefile.html">Rakefile</a>
+  
+    <li class="file"><a href="./example/Gemfile.html">Gemfile</a>
+  
+    <li class="file"><a href="./spec/pid/pidfile.html">pidfile</a>
+  
+  </ul>
+</nav>
+
+    <nav id="classindex-section" class="section project-section">
+  <h3 class="section-header">Class and Module Index</h3>
+
+  <ul class="link-list">
+  
+    <li><a href="./Forklift.html">Forklift</a>
+  
+    <li><a href="./Forklift/Base.html">Forklift::Base</a>
+  
+    <li><a href="./Forklift/Base/Connection.html">Forklift::Base::Connection</a>
+  
+    <li><a href="./Forklift/Base/Logger.html">Forklift::Base::Logger</a>
+  
+    <li><a href="./Forklift/Base/Mailer.html">Forklift::Base::Mailer</a>
+  
+    <li><a href="./Forklift/Base/Mailer/ERBBinding.html">Forklift::Base::Mailer::ERBBinding</a>
+  
+    <li><a href="./Forklift/Base/Pid.html">Forklift::Base::Pid</a>
+  
+    <li><a href="./Forklift/Base/Utils.html">Forklift::Base::Utils</a>
+  
+    <li><a href="./Forklift/Connection.html">Forklift::Connection</a>
+  
+    <li><a href="./Forklift/Connection/Elasticsearch.html">Forklift::Connection::Elasticsearch</a>
+  
+    <li><a href="./Forklift/Connection/Mysql.html">Forklift::Connection::Mysql</a>
+  
+    <li><a href="./Forklift/Patterns.html">Forklift::Patterns</a>
+  
+    <li><a href="./Forklift/Patterns/Elasticsearch.html">Forklift::Patterns::Elasticsearch</a>
+  
+    <li><a href="./Forklift/Patterns/Mysql.html">Forklift::Patterns::Mysql</a>
+  
+    <li><a href="./Forklift/Plan.html">Forklift::Plan</a>
+  
+    <li><a href="./EmailSuffix.html">EmailSuffix</a>
+  
+    <li><a href="./Object.html">Object</a>
+  
+    <li><a href="./SpecClient.html">SpecClient</a>
+  
+    <li><a href="./SpecPlan.html">SpecPlan</a>
+  
+    <li><a href="./SpecSeeds.html">SpecSeeds</a>
+  
+  </ul>
+</nav>
+
+  </div>
+</nav>
+
+<div id="documentation">
+  <h1 class="class">class EmailSuffix</h1>
+
+  <div id="description" class="description">
+    
+  </div><!-- description -->
+
+  
+  
+  
+  <section id="5Buntitled-5D" class="documentation-section">
+    
+
+    
+
+    
+
+    
+
+    <!-- Methods -->
+    
+     <section id="public-instance-5Buntitled-5D-method-details" class="method-section section">
+      <h3 class="section-header">Public Instance Methods</h3>
+
+    
+      <div id="method-i-do-21" class="method-detail ">
+        
+        <div class="method-heading">
+          <span class="method-name">do!</span><span
+            class="method-args">(connection, forklift)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          
+          
+
+          
+          <div class="method-source-code" id="do-21-source">
+            <pre><span class="ruby-comment"># File example/transformations/email_suffix.rb, line 3</span>
+<span class="ruby-keyword">def</span> <span class="ruby-identifier">do!</span>(<span class="ruby-identifier">connection</span>, <span class="ruby-identifier">forklift</span>)
+  <span class="ruby-identifier">forklift</span>.<span class="ruby-identifier">logger</span>.<span class="ruby-identifier">log</span> <span class="ruby-string">&quot;collecting email suffixes...&quot;</span>
+
+  <span class="ruby-identifier">suffixes</span> = {}
+  <span class="ruby-identifier">connection</span>.<span class="ruby-identifier">read</span>(<span class="ruby-string">&quot;select email from users&quot;</span>){<span class="ruby-operator">|</span><span class="ruby-identifier">data</span><span class="ruby-operator">|</span>
+    <span class="ruby-identifier">data</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">row</span><span class="ruby-operator">|</span>
+      <span class="ruby-identifier">part</span> = <span class="ruby-identifier">row</span>[<span class="ruby-value">:email</span>].<span class="ruby-identifier">split</span>(<span class="ruby-string">'@'</span>).<span class="ruby-identifier">last</span>
+      <span class="ruby-identifier">suffixes</span>[<span class="ruby-identifier">part</span>] = <span class="ruby-value">0</span> <span class="ruby-keyword">if</span> <span class="ruby-identifier">suffixes</span>[<span class="ruby-identifier">part</span>].<span class="ruby-identifier">nil?</span>
+      <span class="ruby-identifier">suffixes</span>[<span class="ruby-identifier">part</span>] = <span class="ruby-identifier">suffixes</span>[<span class="ruby-identifier">part</span>] <span class="ruby-operator">+</span> <span class="ruby-value">1</span>
+    <span class="ruby-keyword">end</span>
+  }
+
+  <span class="ruby-identifier">suffixes</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">suffix</span>, <span class="ruby-identifier">count</span><span class="ruby-operator">|</span>
+    <span class="ruby-identifier">forklift</span>.<span class="ruby-identifier">logger</span>.<span class="ruby-identifier">log</span> <span class="ruby-node">&quot; &gt; #{suffix}: #{count}&quot;</span> <span class="ruby-keyword">if</span> <span class="ruby-identifier">count</span> <span class="ruby-operator">&gt;</span> <span class="ruby-value">5</span>
+  <span class="ruby-keyword">end</span>
+<span class="ruby-keyword">end</span></pre>
+          </div><!-- do-21-source -->
+          
+        </div>
+
+        
+
+        
+      </div><!-- do-21-method -->
+
+    
+    </section><!-- public-instance-method-details -->
+  
+  </section><!-- 5Buntitled-5D -->
+
+</div><!-- documentation -->
+
+
+<footer id="validator-badges">
+  <p><a href="http://validator.w3.org/check/referer">[Validate]</a>
+  <p>Generated by <a href="https://github.com/rdoc/rdoc">RDoc</a> 3.12.2.
+  <p>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish Rdoc Generator</a> 3.
+</footer>
+

data/doc/Forklift.html

@@ -0,0 +1,187 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+<meta content="text/html; charset=UTF-8" http-equiv="Content-Type">
+
+<title>module Forklift - RDoc Documentation</title>
+
+<link type="text/css" media="screen" href="./rdoc.css" rel="stylesheet">
+
+<script type="text/javascript">
+  var rdoc_rel_prefix = "./";
+</script>
+
+<script type="text/javascript" charset="utf-8" src="./js/jquery.js"></script>
+<script type="text/javascript" charset="utf-8" src="./js/navigation.js"></script>
+<script type="text/javascript" charset="utf-8" src="./js/search_index.js"></script>
+<script type="text/javascript" charset="utf-8" src="./js/search.js"></script>
+<script type="text/javascript" charset="utf-8" src="./js/searcher.js"></script>
+<script type="text/javascript" charset="utf-8" src="./js/darkfish.js"></script>
+
+
+<body id="top" class="module">
+<nav id="metadata">
+  <nav id="home-section" class="section">
+  <h3 class="section-header">
+    <a href="./index.html">Home</a>
+    <a href="./table_of_contents.html#classes">Classes</a>
+    <a href="./table_of_contents.html#methods">Methods</a>
+  </h3>
+</nav>
+
+
+  <nav id="search-section" class="section project-section" class="initially-hidden">
+  <form action="#" method="get" accept-charset="utf-8">
+    <h3 class="section-header">
+      <input type="text" name="search" placeholder="Search" id="search-field"
+             title="Type to search, Up and Down to navigate, Enter to load">
+    </h3>
+  </form>
+
+  <ul id="search-results" class="initially-hidden"></ul>
+</nav>
+
+
+  <div id="file-metadata">
+    <nav id="file-list-section" class="section">
+  <h3 class="section-header">Defined In</h3>
+  <ul>
+    <li>lib/forklift/base/connection.rb
+    <li>lib/forklift/base/logger.rb
+    <li>lib/forklift/base/mailer.rb
+    <li>lib/forklift/base/pid.rb
+    <li>lib/forklift/base/utils.rb
+    <li>lib/forklift/forklift.rb
+    <li>lib/forklift/patterns/elasticsearch_patterns.rb
+    <li>lib/forklift/patterns/mysql_patterns.rb
+    <li>lib/forklift/plan.rb
+    <li>lib/forklift/transports/elasticsearch.rb
+    <li>lib/forklift/transports/mysql.rb
+    <li>lib/forklift/version.rb
+  </ul>
+</nav>
+
+    
+  </div>
+
+  <div id="class-metadata">
+    
+    
+    
+    
+  </div>
+
+  <div id="project-metadata">
+    <nav id="fileindex-section" class="section project-section">
+  <h3 class="section-header">Pages</h3>
+
+  <ul>
+  
+    <li class="file"><a href="./Gemfile.html">Gemfile</a>
+  
+    <li class="file"><a href="./Rakefile.html">Rakefile</a>
+  
+    <li class="file"><a href="./example/Gemfile.html">Gemfile</a>
+  
+    <li class="file"><a href="./spec/pid/pidfile.html">pidfile</a>
+  
+  </ul>
+</nav>
+
+    <nav id="classindex-section" class="section project-section">
+  <h3 class="section-header">Class and Module Index</h3>
+
+  <ul class="link-list">
+  
+    <li><a href="./Forklift.html">Forklift</a>
+  
+    <li><a href="./Forklift/Base.html">Forklift::Base</a>
+  
+    <li><a href="./Forklift/Base/Connection.html">Forklift::Base::Connection</a>
+  
+    <li><a href="./Forklift/Base/Logger.html">Forklift::Base::Logger</a>
+  
+    <li><a href="./Forklift/Base/Mailer.html">Forklift::Base::Mailer</a>
+  
+    <li><a href="./Forklift/Base/Mailer/ERBBinding.html">Forklift::Base::Mailer::ERBBinding</a>
+  
+    <li><a href="./Forklift/Base/Pid.html">Forklift::Base::Pid</a>
+  
+    <li><a href="./Forklift/Base/Utils.html">Forklift::Base::Utils</a>
+  
+    <li><a href="./Forklift/Connection.html">Forklift::Connection</a>
+  
+    <li><a href="./Forklift/Connection/Elasticsearch.html">Forklift::Connection::Elasticsearch</a>
+  
+    <li><a href="./Forklift/Connection/Mysql.html">Forklift::Connection::Mysql</a>
+  
+    <li><a href="./Forklift/Patterns.html">Forklift::Patterns</a>
+  
+    <li><a href="./Forklift/Patterns/Elasticsearch.html">Forklift::Patterns::Elasticsearch</a>
+  
+    <li><a href="./Forklift/Patterns/Mysql.html">Forklift::Patterns::Mysql</a>
+  
+    <li><a href="./Forklift/Plan.html">Forklift::Plan</a>
+  
+    <li><a href="./EmailSuffix.html">EmailSuffix</a>
+  
+    <li><a href="./Object.html">Object</a>
+  
+    <li><a href="./SpecClient.html">SpecClient</a>
+  
+    <li><a href="./SpecPlan.html">SpecPlan</a>
+  
+    <li><a href="./SpecSeeds.html">SpecSeeds</a>
+  
+  </ul>
+</nav>
+
+  </div>
+</nav>
+
+<div id="documentation">
+  <h1 class="module">module Forklift</h1>
+
+  <div id="description" class="description">
+    
+  </div><!-- description -->
+
+  
+  
+  
+  <section id="5Buntitled-5D" class="documentation-section">
+    
+
+    
+
+    
+    <!-- Constants -->
+    <section id="constants-list" class="section">
+      <h3 class="section-header">Constants</h3>
+      <dl>
+      
+        <dt id="VERSION">VERSION
+        
+        <dd class="description">
+        
+      
+      </dl>
+    </section>
+    
+
+    
+
+    <!-- Methods -->
+    
+  </section><!-- 5Buntitled-5D -->
+
+</div><!-- documentation -->
+
+
+<footer id="validator-badges">
+  <p><a href="http://validator.w3.org/check/referer">[Validate]</a>
+  <p>Generated by <a href="https://github.com/rdoc/rdoc">RDoc</a> 3.12.2.
+  <p>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish Rdoc Generator</a> 3.
+</footer>
+