sip 0.0.0

data/README.rdoc

@@ -0,0 +1,32 @@
+= SIP: SQL to Hive Importer
+SIP is a ETL[http://en.wikipedia.org/wiki/Extract,_transform,_load] tool for extracting SQL databases and importing them into Hive[http://hive.apache.org].  It was created because the ability to transform columns and partition data was an absolute requirement, and no other tool provided that functionality.
+
+Unique features include:
+* The ability to transform columns (using Ruby code)
+* A single, simple, human readable configuration file
+* The ability to parition tables in Hive based on the value of any (possibly transformed) columns
+
+Bug reports and pull requests welcome on github[https://github.com/livingsocial/sip].
+
+== Requirements
+* A working Hadoop installation
+* Hive on the namenode
+* Ruby >= 1.8.6 on all datanodes
+* SQL Ruby libs
+
+== Installation, Configuration, and Use
+ gem install sip
+
+note: if no primary key (default: id), must set incremental_index to blank
+
+
+== How it Works
+Per table to be imported, SIP determines the queries necessary to perform an export and the creates scripts (one per datanode) that are then run individually in parallel.  Each script:
+1. Copies a transformation / partition (transpart) script to the datanode
+1. Performs the SQL extraction, piping output through the transpart script
+1. Uploads all partitions to HDFS
+
+Then, all of the partitions are imported from HDFS into Hive.  Easy squeezy.
+
+
+

data/Rakefile

@@ -0,0 +1,21 @@
+require 'rubygems'
+require 'bundler'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+Bundler::GemHelper.install_tasks
+
+desc "Run all unit tests"
+Rake::TestTask.new("test") { |t|
+  t.libs << "lib"
+  t.test_files = FileList['test/*_test.rb']
+  t.verbose = true
+}
+
+desc "Create documentation"
+Rake::RDocTask.new("docs") { |rdoc|
+  rdoc.title = "Sip - SQL to Hive importer"
+  rdoc.rdoc_dir = 'docs'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+}

data/bin/sip

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'rubygems'
+require 'sip'
+
+starttime = Time.now
+
+options = { :db => nil, :table => nil, :config => 'sip.yml', :debug => false, :tfile => nil, :overwrite => false }
+parser = OptionParser.new { |opts|
+  opts.banner = "Usage: sip [options]"
+
+  opts.on('-h', '--help', 'displays usage information') { 
+    puts opts
+    exit
+  }
+
+  opts.on('-o', '--overwrite', 'force full overwrite') {
+    options[:overwrite] = true
+  }
+
+  opts.on('-d', '--debug', 'Print debugging information') { |d|
+    options[:debug] = true
+  }
+
+  opts.on('-D <database>', '--database <database>', 'only import from given database') { |db|
+    options[:db] = db
+  }
+
+  opts.on('-c <config file>', '--config <config file>', 'use specified config') { |c|
+    options[:config] = c
+  }
+
+  opts.on('-t <table>', '--table <table>', 'only import from given table (must specify -D as well)') { |t|
+    options[:table] = t
+  }
+
+  opts.on('-f <file>', '--transpart-file <file>', 'translation/partition file location (default: looks for file in path)') { |t|
+    options[:tfile] = t
+  }
+}.parse!
+
+
+if ENV['HADOOP_HOME'].nil? or ENV['HIVE_HOME'].nil?
+   puts "You must first set both the HADOOP_HOME and HIVE_HOME environment variables."
+   exit 1
+end
+
+debug = Proc.new { |t| puts "#{Sip::Utils::hostname} #{Time.now.strftime("%Y-%m-%d %H:%M:%S")}: #{t}" if options[:debug] }
+
+debug.call "Using config file #{options[:config]}"
+config = Sip::Config.load_file options[:config]
+config.set_temp options, [:debug, :overwrite]
+
+config['tfile'] = Sip::Utils::tfile_path(options[:tfile] || config['tfile'])
+config['hdfs_tmpdir'] ||= "/tmp/sip"
+
+if config['tfile'].nil?
+  puts "Could not find transforation / partition (transpart) script.  Please use -f option to specify a location."
+  exit 1
+end
+debug.call "Using transformation / partitioning script '#{config['tfile']}'"
+
+sipper = Sip::Sipper.new config
+config['databases'].each { |dbname, dbconfig|
+  next if not options[:db].nil? and options[:db] != dbname
+  dbconfig['tables'].each { |tablename, tableconfig|
+    next if not options[:db].nil? and not options[:table].nil? and options[:table] != tablename
+    debug.call "creating script for table #{tablename} in database #{dbname}"
+    script_count = sipper.create_scripts dbconfig, tableconfig
+    if script_count > 0
+      debug.call "Executing #{script_count} scripts..."
+      sipper.run_scripts    
+      debug.call "Running hive import..."
+      sipper.run_hive_import tableconfig
+    end
+  }
+}
+
+# Save new version of config
+debug.call "Writing new version of config file to #{options[:config]}"
+sipper.config.save_file options[:config]
+
+debug.call "Finished running import in #{Time.now - starttime} seconds"

data/bin/transpart

@@ -0,0 +1,114 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'socket'
+require 'fileutils'
+require 'tmpdir'
+require 'date'
+require 'socket'
+
+class Transformer
+  def self.add_one(value)
+    value.to_i + 1
+  end
+
+  def self.datetime_to_date(value)
+    value.split(' ').first
+  end
+
+  # this takes forever to run
+  def self.to_date(value)
+    Date.parse(value).to_s
+  end
+end
+
+
+############################################################################
+# -c one,two,three,created_at -p created_at -t one:transone,two:transformtwo -o /tmp/sip -s 1234 -d 
+options = { :cols => [], :partition_by => nil, :trans => {}, :debug => false, :outdir => nil, :onehundreth => nil, :hive_tablename => nil }
+parser = OptionParser.new { |opts|
+  opts.banner = "Usage: transpart [options]"
+
+  opts.on('-c <colum list>', '--columns <column list>', 'list of columns in order') { |c|
+    options[:cols] = c.split(",")
+  }
+
+  opts.on('-p <partition column>', '--partition-by <partition column>', 'partition by given column') { |c|
+    options[:partition_by] = c
+  }
+
+  opts.on('-t <transformations>', '--transformations <transformations>', 'comma separated col:transmethod list') { |t|
+    t.split(',').each { |trans|
+      key, value = trans.split(':')
+      options[:trans][key] = value
+    }
+  }
+
+  opts.on('-H <hive tablename>', '--hive-table <hive tablename>', 'name of the hive table to import into') { |h|
+    options[:hive_tablename] = h
+  }
+
+  opts.on('-o <output dir>', '--output-dir <output dir>', 'Directory to output partitions to') { |o|
+    options[:outdir] = o
+  }
+  
+  opts.on('-s <num rows>', '--size <num rows>', 'Number of rows - used to estimate % done') { |s|
+    options[:onehundreth] = s.to_i / 100
+  }
+
+  opts.on('-d', '--debug', 'print debugging info') {
+    options[:debug] = true
+  }
+}.parse!
+
+
+if options[:cols].length == 0 and (options[:trans].length > 0 or not options[:partition_by].nil?)
+  puts "You must specify column names with the -c option if you also give transformation or parition options"
+  exit 1
+elsif not options[:partition_by].nil? and not options[:cols].include? options[:partition_by]
+  puts "Can not partition by a non-existant column"
+  exit 1
+end
+
+partfiles = {}
+outputdir = options[:outdir] || Dir.mktmpdir('sip_')
+hostname = Socket.gethostname
+tmpfname = Proc.new { |partition| 
+  odir = partition.nil? ? outputdir : File.join(outputdir, partition)
+  FileUtils.mkdir_p odir
+  File.join(odir, "#{hostname}_#{Time.now.to_f}_#{ (1...10).map { ('a'...'z').to_a.sort_by{rand}.first }.join }.part") 
+}
+debug = Proc.new { |t| puts "#{hostname} #{Time.now.strftime("%Y-%m-%d %H:%M:%S")}: #{t}" if options[:debug] }
+
+
+lineno = 0
+pdone = 0
+$stdin.each_line do |line|
+  lineno += 1
+  vals = line.chomp("\n").split("\t")
+  partval = nil
+  transvalues = []
+  vals.each_with_index { |value, index|
+    if options[:trans].has_key? options[:cols][index]
+      value = Transformer.send(options[:trans][options[:cols][index]], value)
+    end
+    if options[:cols][index] == options[:partition_by]
+      partval = value 
+    else
+      transvalues << value
+    end
+  }
+  
+  if not partfiles.has_key? partval
+    partfiles[partval] = File.open(tmpfname.call(partval), 'w') 
+  end
+  partfiles[partval].write(transvalues.join("\001") + "\n")
+
+  if not options[:onehundreth].nil? and lineno == options[:onehundreth]
+    lineno = 0
+    pdone += 1
+    debug.call "#{pdone}% finished exporting table"
+  end
+end
+
+debug.call "finished exporting table"
+partfiles.values.each { |f| f.close }

data/docs/classes/Sip.html

@@ -0,0 +1,169 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html 
+     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <title>Module: Sip</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+  <meta http-equiv="Content-Script-Type" content="text/javascript" />
+  <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
+  <script type="text/javascript">
+  // <![CDATA[
+
+  function popupCode( url ) {
+    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
+  }
+
+  function toggleCode( id ) {
+    if ( document.getElementById )
+      elem = document.getElementById( id );
+    else if ( document.all )
+      elem = eval( "document.all." + id );
+    else
+      return false;
+
+    elemStyle = elem.style;
+    
+    if ( elemStyle.display != "block" ) {
+      elemStyle.display = "block"
+    } else {
+      elemStyle.display = "none"
+    }
+
+    return true;
+  }
+  
+  // Make codeblocks hidden by default
+  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
+  
+  // ]]>
+  </script>
+
+</head>
+<body>
+
+
+
+    <div id="classHeader">
+        <table class="header-table">
+        <tr class="top-aligned-row">
+          <td><strong>Module</strong></td>
+          <td class="class-name-in-header">Sip</td>
+        </tr>
+        <tr class="top-aligned-row">
+            <td><strong>In:</strong></td>
+            <td>
+                <a href="../files/lib/sip/cmdopts_rb.html">
+                lib/sip/cmdopts.rb
+                </a>
+        <br />
+                <a href="../files/lib/sip/config_rb.html">
+                lib/sip/config.rb
+                </a>
+        <br />
+                <a href="../files/lib/sip/databases/dbbase_rb.html">
+                lib/sip/databases/dbbase.rb
+                </a>
+        <br />
+                <a href="../files/lib/sip/databases/mysql_rb.html">
+                lib/sip/databases/mysql.rb
+                </a>
+        <br />
+                <a href="../files/lib/sip/exceptions_rb.html">
+                lib/sip/exceptions.rb
+                </a>
+        <br />
+                <a href="../files/lib/sip/hive_rb.html">
+                lib/sip/hive.rb
+                </a>
+        <br />
+                <a href="../files/lib/sip/sipper_rb.html">
+                lib/sip/sipper.rb
+                </a>
+        <br />
+                <a href="../files/lib/sip/utils_rb.html">
+                lib/sip/utils.rb
+                </a>
+        <br />
+                <a href="../files/lib/sip/version_rb.html">
+                lib/sip/version.rb
+                </a>
+        <br />
+            </td>
+        </tr>
+
+        </table>
+    </div>
+  <!-- banner header -->
+
+  <div id="bodyContent">
+
+
+
+  <div id="contextContent">
+
+
+
+   </div>
+
+
+  </div>
+
+
+    <!-- if includes -->
+
+    <div id="section">
+
+    <div id="class-list">
+      <h3 class="section-bar">Classes and Modules</h3>
+
+      Module <a href="Sip/Hive.html" class="link">Sip::Hive</a><br />
+Module <a href="Sip/Utils.html" class="link">Sip::Utils</a><br />
+Class <a href="Sip/CmdOpts.html" class="link">Sip::CmdOpts</a><br />
+Class <a href="Sip/Config.html" class="link">Sip::Config</a><br />
+Class <a href="Sip/DBBase.html" class="link">Sip::DBBase</a><br />
+Class <a href="Sip/HadoopException.html" class="link">Sip::HadoopException</a><br />
+Class <a href="Sip/HiveQueryException.html" class="link">Sip::HiveQueryException</a><br />
+Class <a href="Sip/ImportScriptExecutionError.html" class="link">Sip::ImportScriptExecutionError</a><br />
+Class <a href="Sip/MySQLSipper.html" class="link">Sip::MySQLSipper</a><br />
+Class <a href="Sip/NoSuchColumn.html" class="link">Sip::NoSuchColumn</a><br />
+Class <a href="Sip/NoSuchTable.html" class="link">Sip::NoSuchTable</a><br />
+Class <a href="Sip/PastFailureException.html" class="link">Sip::PastFailureException</a><br />
+Class <a href="Sip/Sipper.html" class="link">Sip::Sipper</a><br />
+Class <a href="Sip/UnsupportedDatabaseType.html" class="link">Sip::UnsupportedDatabaseType</a><br />
+
+    </div>
+
+    <div id="constants-list">
+      <h3 class="section-bar">Constants</h3>
+
+      <div class="name-list">
+        <table summary="Constants">
+        <tr class="top-aligned-row context-row">
+          <td class="context-item-name">VERSION</td>
+          <td>=</td>
+          <td class="context-item-value">&quot;0.0.1&quot;</td>
+        </tr>
+        </table>
+      </div>
+    </div>
+
+
+
+      
+
+
+    <!-- if method_list -->
+
+
+  </div>
+
+
+<div id="validator-badges">
+  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
+</div>
+
+</body>
+</html>

data/docs/classes/Sip/CmdOpts.html

@@ -0,0 +1,179 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html 
+     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <title>Class: Sip::CmdOpts</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+  <meta http-equiv="Content-Script-Type" content="text/javascript" />
+  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
+  <script type="text/javascript">
+  // <![CDATA[
+
+  function popupCode( url ) {
+    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
+  }
+
+  function toggleCode( id ) {
+    if ( document.getElementById )
+      elem = document.getElementById( id );
+    else if ( document.all )
+      elem = eval( "document.all." + id );
+    else
+      return false;
+
+    elemStyle = elem.style;
+    
+    if ( elemStyle.display != "block" ) {
+      elemStyle.display = "block"
+    } else {
+      elemStyle.display = "none"
+    }
+
+    return true;
+  }
+  
+  // Make codeblocks hidden by default
+  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
+  
+  // ]]>
+  </script>
+
+</head>
+<body>
+
+
+
+    <div id="classHeader">
+        <table class="header-table">
+        <tr class="top-aligned-row">
+          <td><strong>Class</strong></td>
+          <td class="class-name-in-header">Sip::CmdOpts</td>
+        </tr>
+        <tr class="top-aligned-row">
+            <td><strong>In:</strong></td>
+            <td>
+                <a href="../../files/lib/sip/cmdopts_rb.html">
+                lib/sip/cmdopts.rb
+                </a>
+        <br />
+            </td>
+        </tr>
+
+        <tr class="top-aligned-row">
+            <td><strong>Parent:</strong></td>
+            <td>
+                Hash
+            </td>
+        </tr>
+        </table>
+    </div>
+  <!-- banner header -->
+
+  <div id="bodyContent">
+
+
+
+  <div id="contextContent">
+
+
+
+   </div>
+
+    <div id="method-list">
+      <h3 class="section-bar">Methods</h3>
+
+      <div class="name-list">
+      <a href="#M000040">set</a>&nbsp;&nbsp;
+      <a href="#M000041">to_s</a>&nbsp;&nbsp;
+      </div>
+    </div>
+
+  </div>
+
+
+    <!-- if includes -->
+
+    <div id="section">
+
+
+
+
+
+      
+
+
+    <!-- if method_list -->
+    <div id="methods">
+      <h3 class="section-bar">Public Instance methods</h3>
+
+      <div id="method-M000040" class="method-detail">
+        <a name="M000040"></a>
+
+        <div class="method-heading">
+          <a href="#M000040" class="method-signature">
+          <span class="method-name">set</span><span class="method-args">(*k)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000040-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000040-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/sip/cmdopts.rb, line 3</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">set</span>(<span class="ruby-operator">*</span><span class="ruby-identifier">k</span>)
+      <span class="ruby-identifier">k</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">key</span><span class="ruby-operator">|</span>
+        <span class="ruby-keyword kw">self</span>[<span class="ruby-identifier">key</span>] = <span class="ruby-keyword kw">nil</span>
+      }
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000041" class="method-detail">
+        <a name="M000041"></a>
+
+        <div class="method-heading">
+          <a href="#M000041" class="method-signature">
+          <span class="method-name">to_s</span><span class="method-args">(prefix=nil, suffix=nil)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000041-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000041-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/sip/cmdopts.rb, line 9</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">to_s</span>(<span class="ruby-identifier">prefix</span>=<span class="ruby-keyword kw">nil</span>, <span class="ruby-identifier">suffix</span>=<span class="ruby-keyword kw">nil</span>)
+      <span class="ruby-identifier">opts</span> = <span class="ruby-identifier">map</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">k</span>,<span class="ruby-identifier">v</span><span class="ruby-operator">|</span> 
+        <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">v</span>.<span class="ruby-identifier">nil?</span>
+          (<span class="ruby-identifier">k</span>.<span class="ruby-identifier">length</span> <span class="ruby-operator">&gt;</span> <span class="ruby-value">1</span>) <span class="ruby-operator">?</span> <span class="ruby-node">&quot;--#{k}&quot;</span> <span class="ruby-operator">:</span> <span class="ruby-node">&quot;-#{k}&quot;</span>
+        <span class="ruby-keyword kw">else</span>
+          (<span class="ruby-identifier">k</span>.<span class="ruby-identifier">length</span> <span class="ruby-operator">&gt;</span> <span class="ruby-value">1</span>) <span class="ruby-operator">?</span> <span class="ruby-node">&quot;--#{k}=#{v}&quot;</span> <span class="ruby-operator">:</span> <span class="ruby-node">&quot;-#{k} #{v}&quot;</span> 
+        <span class="ruby-keyword kw">end</span>
+      }.<span class="ruby-identifier">join</span>(<span class="ruby-value str">&quot; &quot;</span>)
+      [(<span class="ruby-identifier">prefix</span>.<span class="ruby-identifier">nil?</span> <span class="ruby-value">? </span><span class="ruby-value str">&quot;&quot;</span> <span class="ruby-operator">:</span> <span class="ruby-identifier">prefix</span>), <span class="ruby-identifier">opts</span>, (<span class="ruby-identifier">suffix</span>.<span class="ruby-identifier">nil?</span> <span class="ruby-value">? </span><span class="ruby-value str">&quot;&quot;</span> <span class="ruby-operator">:</span> <span class="ruby-identifier">suffix</span>)].<span class="ruby-identifier">select</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">v</span><span class="ruby-operator">|</span> <span class="ruby-identifier">v</span> <span class="ruby-operator">!=</span> <span class="ruby-value str">&quot;&quot;</span> }.<span class="ruby-identifier">join</span>(<span class="ruby-value str">&quot; &quot;</span>)
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+
+    </div>
+
+
+  </div>
+
+
+<div id="validator-badges">
+  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
+</div>
+
+</body>
+</html>