sip 0.0.0

data/lib/sip/exceptions.rb

@@ -0,0 +1,9 @@
+module Sip
+  class NoSuchTable < StandardError; end
+  class NoSuchColumn < StandardError; end
+  class ImportScriptExecutionError < StandardError; end
+  class PastFailureException < StandardError; end
+  class HiveQueryException < StandardError; end
+  class HadoopException < StandardError; end
+  class UnsupportedDatabaseType < StandardError; end
+end

data/lib/sip/extensions.rb

@@ -0,0 +1,5 @@
+class Struct
+  def get_binding
+    binding
+  end
+end

data/lib/sip/hive.rb

@@ -0,0 +1,62 @@
+module Sip
+  module Hive
+    def self.run_hsql_create_table(sipper, db, tableconf, method)
+      if method == :overwrite
+        Hive.run sipper, "DROP TABLE #{tableconf['hive_table_name']}"
+      end
+
+      partition = ""
+      cols = db.hive_columns tableconf['tablename']
+      if tableconf.has_key? 'partition_by'
+        if not cols.map { |k,v| k }.include? tableconf['partition_by']
+          raise NoSuchColumn, "Column to parition by '#{tableconf['partition_by']}' not found in table '#{tableconf['tablename']}'" 
+        end
+        partition_type = cols.select { |k,v| k == tableconf['partition_by'] }.map { |k,v| v }.first
+        partition = "PARTITIONED BY (#{tableconf['partition_by']} #{partition_type})"
+      end
+      colstring = cols.select { |k,v| k != tableconf.fetch('partition_by', nil) }.map { |k,v| "#{k} #{v}" }.join(", ")
+      Hive.run sipper, "CREATE TABLE IF NOT EXISTS #{tableconf['hive_table_name']} (#{colstring}) #{partition}"
+    end
+
+    def self.run(sipper, query)
+      cmd = "#{ENV['HIVE_HOME']}/bin/hive -S -e \"#{query}\""
+      sipper.log "Running Hive command: #{cmd}"
+      raise HiveQueryException, "Could not execute hive command #{cmd}" if not system(cmd)
+    end
+
+    def self.run_file(sipper, path)
+      cmd = "#{ENV['HIVE_HOME']}/bin/hive -S -f #{path}"
+      sipper.log "Running Hive cmd: #{cmd}"
+      raise HiveQueryException, "Could not execute hive command #{cmd}" if not system(cmd)
+    end
+
+    def self.run_import(sipper, tableconf)
+      base_path = "#{sipper.config['hdfs_tmpdir']}/#{tableconf['hive_table_name']}"
+      stmts = Hive.load_data_statements tableconf, base_path
+      path = File.join(sipper.config['tmpdir'], 'scripts', 'hive_import.hql')
+      open(path, 'w') { |f|
+        f.write stmts.join("\n")
+      }
+      Hive.run_file sipper, path
+    end
+
+    def self.load_data_statements(tableconf, base_path)
+      files = Hive.hdfs_ls(base_path).select { |p| p.slice(-5, 5) == '.part' }
+      if tableconf.has_key? 'partition_by'
+        files.map { |path|
+          partval = path.split('/')[-2]
+          "LOAD DATA INPATH '#{path}' INTO TABLE #{tableconf['hive_table_name']} PARTITION (#{tableconf['partition_by']}='#{partval}');"
+        }
+      else
+        ["LOAD DATA INPATH '#{base_path}' INTO TABLE #{tableconf['hive_table_name']};"]
+      end
+    end
+
+    def Hive.hdfs_ls(location)
+      result = `#{ENV['HADOOP_HOME']}/bin/hadoop dfs -lsr hdfs://#{location}`
+      lines = result.split("\n")
+      lines.shift
+      lines.map { |l| l.split(" ").last }
+    end
+  end
+end

data/lib/sip/sipper.rb

@@ -0,0 +1,118 @@
+module Sip
+  class Sipper
+    attr_reader :config
+
+    def initialize(config)
+      @config = config
+      slavefile = File.join(ENV['HADOOP_HOME'], 'conf', 'slaves')
+      log "Reading slaves from file #{slavefile}..."
+      begin
+        open(slavefile, 'r') { |f| @slaves = f.read.split("\n") }
+      rescue
+        raise HadoopException, "Could not read \"#{slavefile}\".  Is your HADOOP_HOME environment variable correct?"
+      end
+      Utils::sanity_check(@config)
+    end
+
+    def log(msg)
+      puts "#{Utils::hostname} #{Time.now.strftime '%Y-%m-%d %H:%M:%S'}: #{msg}" if @config[:debug]
+    end
+
+    # return number of scripts created
+    def create_scripts(dbconf, tableconf)
+      @scripts = []
+      db = DBBase.make_interface dbconf['type'], dbconf, self
+
+      # set default columns if necessary
+      tableconf['columns'] = db.columns(tableconf['tablename']).map { |c| c.first } if tableconf['columns'].nil?
+
+      if tableconf['incremental_index'].nil?
+        create_script_without_index(dbconf, tableconf, db)
+      else
+        create_scripts_with_index(dbconf, tableconf, db)
+      end
+
+      @scripts.length
+    end
+
+    # this is the case where there is no primary key index, so the whole
+    # table will need to be imported
+    def create_script_without_index(dbconf, tableconf, db)
+      Hive::run_hsql_create_table self, db, tableconf, :overwrite
+
+      log "Importing all rows from table #{dbconf['dbname']}.#{tableconf['tablename']}"
+      select = db.generate_command tableconf
+      transpart_opts = generate_transpart_options(tableconf, db)
+      @scripts << Utils::write_script(self, @slaves.first, select, dbconf['dbname'], tableconf['tablename'], transpart_opts)
+    end
+
+    def create_scripts_with_index(dbconf, tableconf, db)
+      max = db.get_column_max tableconf['tablename'], tableconf['incremental_index']
+
+      method = (tableconf['method'] == "append" and not @config[:overwrite]) ? :append : :overwrite
+      if method == :append and max == tableconf['incremental_index_value']
+        log "Ignoring #{dbconf['dbname']}.#{tableconf['tablename']} - already up to date"
+      else
+        Hive::run_hsql_create_table self, db, tableconf, method
+        @slaves.each_with_index { |slavename, index|
+          @scripts << create_script(slavename, index, dbconf, tableconf, db, max, method)
+        }
+        db.close
+      end
+
+      # update incremental index value if method in conf is append, regardless of whether or 
+      # not this is a forced overwrite
+      tableconf['incremental_index_value'] = max if tableconf['method'] == "append"
+    end
+
+    def create_script(slavename, index, dbconf, tableconf, db, max, method)
+      if method == :append
+        first, last = get_even_split(tableconf['incremental_index_value']+1, max, index, @slaves.length)
+      else
+        first, last = 1, max
+      end
+
+      log "Importing #{first} <= #{tableconf['incremental_index']} <= #{last} from table #{dbconf['dbname']}.#{tableconf['tablename']}"
+      select = db.generate_command tableconf, first, last
+      transpart_opts = generate_transpart_options(tableconf, db)
+      Utils::write_script self, slavename, select, dbconf['dbname'], tableconf['tablename'], transpart_opts
+    end
+
+    def generate_transpart_options(tableconf, db)
+      opts = CmdOpts.new
+      opts['c'] = db.order_column_list(tableconf['tablename'], tableconf['columns']).join(',')
+      opts['p'] = tableconf['partition_by'] if tableconf.has_key? "partition_by"
+      if tableconf.has_key? 'transformations' and tableconf['transformations'].length > 0
+        opts['t'] = tableconf['transformations'].map { |k,v| "#{k}:#{v}" }.join(',')
+      end
+      opts['H'] = tableconf['hive_table_name']
+      opts['o'] = File.join(@config['tmpdir'], 'partitions')
+      opts.set('d') if @config[:debug]
+      opts
+    end
+
+    def get_even_split(min, max, index, count)
+      size = ((max - min + 1).to_f / count.to_f).ceil
+      first = (size * index) + min
+      last = (size * (index+1)) + min - 1
+      [first, [max, last].min]
+    end
+
+    def run_scripts
+      pids = {}
+      @scripts.each { |script|
+        log "Running #{script}..."
+        pid = fork { Kernel.exit system("sh #{script}") }
+        pids[pid] = script
+        sleep 1
+      }
+      Process.waitall.map { |r| r.last }.each { |status|
+        raise ImportScriptExecutionError, "Error runing script '#{pids[status.pid]}'" if status.exitstatus != 0
+      }
+    end
+
+    def run_hive_import(tableconf)
+      Hive.run_import self, tableconf
+    end
+  end
+end

data/lib/sip/templates/export.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+HOST=<%= host %>
+DEBUG=<%= debug %>
+TFILE=<%= tfile %>
+TOPTS="<%= topts %>"
+TOPTS_ODIR=<%= output_dir %>
+TMPDIR=<%= tmpdir %>
+QUERY="<%= query %>"
+HADOOP_HOME=<%= hadoop_home %>
+HIVE_HOME=<%= hive_home %>
+OVERWRITE=<%= overwrite %>
+HIVE_TABLE_NAME=<%= hive_table_name %>
+HDFS_TMPDIR=<%= hdfs_tmpdir %>
+
+HDFS_PATH=hdfs://$HDFS_TMPDIR/$HIVE_TABLE_NAME/$HOST
+
+function debug { 
+    if [ "$DEBUG" == '1' ]; then
+	echo $HOST $(date +'%Y-%m-%d %H:%M:%S'): $1
+    fi
+}
+
+function error {
+    echo $HOST $(date +'%Y-%m-%d %H:%M:%S'): ERROR - $1
+    exit 1
+}
+
+function check_result {
+    if [ $? != 0 ]; then
+	error "$1"
+    fi
+}
+
+function hdfs_run {
+    debug "$2"
+    $HADOOP_HOME/bin/hadoop dfs $1
+    check_result "$2"
+}
+
+function remote_exec {
+    debug "$2"
+    ssh $HOST "$1"
+    check_result "$2"
+}
+
+$HADOOP_HOME/bin/hadoop dfs -test -e $HDFS_PATH || hdfs_run "-mkdir $HDFS_PATH" "Creating $HDFS_PATH"
+
+if [ "$OVERWRITE" == '0' ]; then
+    ssh $HOST "test ! -x $TOPTS_ODIR"
+    check_result "The directory $HOST:$TOPTS_ODIR exists, indicating a past failure.  Try a full reimport with the -o option."
+    FILE_COUNT=$($HADOOP_HOME/bin/hadoop dfs -ls $HDFS_PATH | wc -l)
+    if [ "$FILE_COUNT" != "0" ]; then
+	error "$HDFS_PATH is not empty, indicating a past failure.  Try a full reimport with the -o option."
+    fi
+else
+    remote_exec "rm -rf $TOPTS_ODIR" "removing directory $TOPTS_ODIR"
+    hdfs_run "-rmr $HDFS_PATH" "Clearing directory $HDFS_PATH"
+fi
+
+remote_exec "mkdir -p $TMPDIR" "making directory $HOST:$TMPDIR if it does not exist"
+
+debug "copying $TFILE to $HOST:$TMPDIR/transpart"
+scp -q $TFILE $HOST:$TMPDIR/transpart
+check_result "could not copy $TFILE to $HOST:$TMPDIR/transpart"
+
+remote_exec "$QUERY | ruby $TMPDIR/transpart $TOPTS" "executing query: $QUERY"
+
+remote_exec "$HADOOP_HOME/bin/hadoop dfs -moveFromLocal $TOPTS_ODIR $HDFS_PATH" "uploading files in $TOPTS_ODIR to HDFS"
+
+remote_exec "rm -rf $TOPTS_ODIR" "removing directory $TOPTS_ODIR"
+
+debug "finished import"

data/lib/sip/utils.rb

@@ -0,0 +1,58 @@
+require 'erb'
+require 'fileutils'
+require 'socket'
+
+module Sip
+  module Utils
+    def self.load_template(name, config)
+      path = File.join(File.dirname(__FILE__), "templates", name)
+      temp = ERB.new File.open(path, 'r') { |f| f.read }
+      klass = Struct.new *config.keys.map { |k| k.intern }
+      temp.result klass.new(*config.values).get_binding
+    end
+
+    def self.hostname
+      Socket.gethostname
+    end
+
+    def self.write_script(sipper, host, select, dbname, tablename, transpart_opts)      
+      hive_table_name = sipper.config.tconf(dbname, tablename)['hive_table_name']
+      args = { 
+        'host' => host,
+        'debug' => (sipper.config[:debug] ? '1' : '0'),
+        'tfile' => sipper.config['tfile'],
+        'topts' => transpart_opts.to_s,
+        'tmpdir' => sipper.config['tmpdir'],
+        'hdfs_tmpdir' => sipper.config['hdfs_tmpdir'],
+        'output_dir' => transpart_opts['o'],
+        'query' => select,
+        'hive_table_name' => sipper.config['databases'][dbname]['tables'][tablename]['hive_table_name'],
+        'hadoop_home' => ENV['HADOOP_HOME'],
+        'hive_home' => ENV['HIVE_HOME'],
+        'overwrite' => (sipper.config[:overwrite] ? '1' : '0')
+      }
+      script = Utils::load_template('export.sh', args)
+
+      d = File.join(sipper.config['tmpdir'], 'scripts')
+      FileUtils.mkdir_p d
+      fname = File.join(d, "#{host}-#{tablename}.sh")
+      open(fname, 'w') { |f| f.write(script) }
+      sipper.log "just wrote import script to #{fname}"
+      fname
+    end
+
+    # look for tfile - return nil if not found
+    def self.tfile_path(path=nil)
+      if path.nil?
+        path = (`which transpart`).strip
+        path = "./bin/transpart" if path == ""
+      end
+      path = File.expand_path path
+      test('f', path) ? path : nil
+    end
+
+    def self.sanity_check(config)
+      # eventually, check for table structure differences, etc
+    end
+  end
+end

data/lib/sip/version.rb

@@ -0,0 +1,3 @@
+module Sip
+  VERSION = "0.0.0"
+end

data/test/database_interaction_test.rb

@@ -0,0 +1,7 @@
+require File.join File.dirname(__FILE__), 'helper'
+
+class DatabaseInteractionTest < Test::Unit::TestCase
+  def test_connection
+    assert_equal 1, 1
+  end
+end

data/test/hive_test.rb

@@ -0,0 +1,28 @@
+require File.join File.dirname(__FILE__), 'helper'
+require File.join File.dirname(__FILE__), 'base'
+
+class HiveTest < Test::Unit::TestCase
+  include Base
+
+  def test_create_table
+    Sip::Hive.run_hsql_create_table @sipper, @db, @tableconf, :overwrite
+    #`#{ENV['HIVE_HOME']}/bin/hive -S -e "select 
+
+    # Sip::Hive.run @sipper, "create table #{@tableconf['hive_table_name']} (id integer, name string)"
+    # Sip::Hive.run @sipper, "
+
+  end
+
+  def test_run
+    assert_raise Sip::HiveQueryException do
+      hh = ENV['HIVE_HOME']
+      ENV['HIVE_HOME'] = "/tmp/asdfasdfasdf"
+      Sip::Hive.run @sipper, "show tables"
+      ENV['HIVE_HOME'] = hh
+    end
+
+    assert_raises Sip::HiveQueryException do
+      Sip::Hive.run @sipper, "this is not a real query"
+    end
+  end
+end

data/test/sipper_test.rb

@@ -0,0 +1,25 @@
+require File.join File.dirname(__FILE__), 'helper'
+
+class SipperTest < Test::Unit::TestCase
+  def test_bad_hadoop_env
+    assert_raise Sip::HadoopException do
+      o = ENV['HADOOP_HOME']
+      ENV['HADOOP_HOME'] = "/tmp/blahblahfake"
+      s = Sip::Sipper.new CONFIG
+      ENV['HADOOP_HOME'] = o
+    end
+  end
+
+  def test_split
+
+  end
+
+  def test_run_scripts
+
+  end
+
+  def test_transpart_opt_generation
+
+  end
+
+end

metadata

@@ -0,0 +1,125 @@
+--- !ruby/object:Gem::Specification 
+name: sip
+version: !ruby/object:Gem::Version 
+  hash: 31
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 0
+  version: 0.0.0
+platform: ruby
+authors: 
+- Brian Muller
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-25 00:00:00 -04:00
+default_executable: 
+dependencies: []
+
+description: SQL database importer for Hadoop / Hive
+email: 
+- brian.muller@livingsocial.com
+executables: 
+- sip
+- transpart
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/sip/cmdopts.rb
+- lib/sip/config.rb
+- lib/sip/databases/dbbase.rb
+- lib/sip/databases/mysql.rb
+- lib/sip/exceptions.rb
+- lib/sip/extensions.rb
+- lib/sip/hive.rb
+- lib/sip/sipper.rb
+- lib/sip/templates/export.sh
+- lib/sip/utils.rb
+- lib/sip/version.rb
+- lib/sip.rb
+- Gemfile
+- LICENSE
+- Rakefile
+- README.rdoc
+- docs/classes/Sip/CmdOpts.html
+- docs/classes/Sip/Config.html
+- docs/classes/Sip/DBBase.html
+- docs/classes/Sip/HadoopException.html
+- docs/classes/Sip/Hive.html
+- docs/classes/Sip/HiveQueryException.html
+- docs/classes/Sip/ImportScriptExecutionError.html
+- docs/classes/Sip/MySQLSipper.html
+- docs/classes/Sip/NoSuchColumn.html
+- docs/classes/Sip/NoSuchTable.html
+- docs/classes/Sip/PastFailureException.html
+- docs/classes/Sip/Sipper.html
+- docs/classes/Sip/UnsupportedDatabaseType.html
+- docs/classes/Sip/Utils.html
+- docs/classes/Sip.html
+- docs/classes/Struct.html
+- docs/created.rid
+- docs/files/lib/sip/cmdopts_rb.html
+- docs/files/lib/sip/config_rb.html
+- docs/files/lib/sip/databases/dbbase_rb.html
+- docs/files/lib/sip/databases/mysql_rb.html
+- docs/files/lib/sip/exceptions_rb.html
+- docs/files/lib/sip/extensions_rb.html
+- docs/files/lib/sip/hive_rb.html
+- docs/files/lib/sip/sipper_rb.html
+- docs/files/lib/sip/utils_rb.html
+- docs/files/lib/sip/version_rb.html
+- docs/files/lib/sip_rb.html
+- docs/files/README_rdoc.html
+- docs/fr_class_index.html
+- docs/fr_file_index.html
+- docs/fr_method_index.html
+- docs/index.html
+- docs/rdoc-style.css
+- test/database_interaction_test.rb
+- test/hive_test.rb
+- test/sipper_test.rb
+- bin/sip
+- bin/transpart
+has_rdoc: true
+homepage: https://github.com/livingsocial/sip
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: sip
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: SQL database importer for Hadoop / Hive
+test_files: 
+- test/database_interaction_test.rb
+- test/hive_test.rb
+- test/sipper_test.rb