rbpig 0.1.0

data/bin/rbpig

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'optparse'
+require 'ostruct'
+require 'rbpig'
+
+options = OpenStruct.new
+
+OptionParser.new do |opts|
+  opts.banner = "USAGE: rbpig [options]"
+
+  opts.on("-c", "--conf HADOOP_CONF", "Use this hadoop cluster xml config file.") do |config|
+    options.hadoop_config = config
+  end
+
+  opts.on("-m", "--mata-conf HIVE_CONF", "Use this hive metastore xml config file.") do |config|
+    options.hive_config = config
+  end
+  
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+exec(RBPig.executable(:hadoop_config => options.hadoop_config, :hive_config => options.hive_config))

data/lib/rbpig/dataset.rb

@@ -0,0 +1,23 @@
+module RBPig
+  class Dataset
+    class << self
+      HIVE_CONFIG = {:database => "default", :database_root => "/user/hive/warehouse"}
+      
+      def hive(table_name, storage_type = :text_file, config = {})
+        raise "storage type other than :text_file is not supported." unless storage_type == :text_file
+        
+        config = HIVE_CONFIG.merge(:field_separator => "\\t").merge(config)
+        new("#{table_name} = LOAD '#{config[:database_root]}/#{table_name}' USING HiveTableLoader('#{config[:field_separator]}', '#{config[:database]}');")
+      end
+    end
+    
+    def to_s
+      @load_script
+    end
+    
+    private
+    def initialize(load_script)
+      @load_script = load_script
+    end
+  end
+end

data/lib/rbpig.rb

@@ -0,0 +1,106 @@
+require 'fileutils'
+require 'rexml/document'
+require File.join(File.dirname(__FILE__), "rbpig", "dataset")
+
+module RBPig
+  class << self
+    def executable(configs)
+      configs = pig_configs(configs)
+      
+      pig_options = ["-Dudf.import.list=forward.pig.storage"]
+      unless configs[:hadoop_config].nil?
+        hadoop_config = {}
+        REXML::Document.new(File.new(configs[:hadoop_config])).elements.each('configuration/property') do |property|
+          hadoop_config[property.elements[1].text] = property.elements[2].text
+        end
+        pig_options << "-Dfs.default.name=#{hadoop_config["fs.default.name"]}" if hadoop_config.has_key?("fs.default.name")
+        pig_options << "-Dmapred.job.tracker=#{hadoop_config["mapred.job.tracker"]}" if hadoop_config.has_key?("mapred.job.tracker")
+      end
+      ["PIG_CLASSPATH='#{pig_classpath(configs)}'", "PIG_OPTS='#{pig_options.join(" ")}'", "pig", "-l /tmp"].join(" ")
+    end
+    
+    def connect(configs)
+      yield Pig.new(pig_configs(configs))
+    end
+    
+    private
+    def pig_configs(configs)
+      {:hadoop_config => nil, :hive_config => nil}.merge(configs || {})
+    end
+    
+    def pig_classpath(configs)
+      classpath = [
+        "#{File.join(File.dirname(__FILE__), %w[.. java dist porkchop.jar])}",
+        "#{File.join(File.dirname(__FILE__), %w[.. java lib hive hive-exec-0.5.0+32.jar])}",
+        "#{File.join(File.dirname(__FILE__), %w[.. java lib hive hive-metastore-0.5.0+32.jar])}",
+        "#{File.join(File.dirname(__FILE__), %w[.. java lib hive libfb303.jar])}",
+        "#{File.join(File.dirname(__FILE__), %w[.. java lib hive jdo2-api-2.3-SNAPSHOT.jar])}",
+        "#{File.join(File.dirname(__FILE__), %w[.. java lib hive datanucleus-core-1.1.2-patched.jar])}",
+        "#{File.join(File.dirname(__FILE__), %w[.. java lib hive datanucleus-enhancer-1.1.2.jar])}",
+        "#{File.join(File.dirname(__FILE__), %w[.. java lib hive datanucleus-rdbms-1.1.2.jar])}",
+        "#{File.join(File.dirname(__FILE__), %w[.. java lib pig jsp-2.1-6.1.14.jar])}"
+      ]
+      unless configs[:hive_config].nil?
+        raise "Rename '#{configs[:hive_config]}' to hive-site.xml for hive metastore configuration." unless File.basename(configs[:hive_config]) == "hive-site.xml"
+        classpath << File.dirname(configs[:hive_config])
+      end
+      classpath.join(":").freeze
+    end
+  end
+  
+  class Pig
+    def initialize(configs)
+      @configs = configs
+      @oink_oink = []
+    end
+    
+    def datasets(*datasets)
+       datasets.each {|e| @oink_oink << e.to_s}
+    end
+    
+    def grunt(oink)
+      @oink_oink << oink
+    end
+    
+    def fetch(*aliases)
+      alias_dump_dir = "/tmp/pigdump/#{Process.pid}_#{Time.now.to_i}"
+      aliases = aliases.map {|alias_to_fetch| "#{alias_dump_dir}/#{alias_to_fetch}"}
+      
+      pig_script_path = "/tmp/pigscript/#{Process.pid}_#{Time.now.to_i}"
+      FileUtils.mkdir_p(File.dirname(pig_script_path))
+      File.open(pig_script_path, "w") do |file|
+        @oink_oink.each {|oink| file << "#{oink}\n"}
+        aliases.each do |dump_file_path|
+          file << "STORE #{File.basename(dump_file_path)} INTO '#{dump_file_path}' USING PigStorage ('\\t');\n"
+        end
+      end
+      
+      pig_execution = "#{RBPig.executable(@configs)} -f #{pig_script_path} 2>&1"
+      pig_out = []
+      IO.popen(pig_execution) do |stdout|
+        puts pig_execution
+        until stdout.eof? do
+          pig_out << stdout.gets
+          puts pig_out.last
+        end
+      end
+      
+      if $?.success?
+        return *fetch_files_in_hdfs(aliases).map {|lines| lines.map{|e| e.chomp("\n").split("\t", -1)}}        
+      else
+        raise "#{pig_out.join("\n")}Failed executing #{pig_execution}"
+      end
+    end
+    
+    private
+    def fetch_files_in_hdfs(file_paths)
+      mandy_config = @configs[:hadoop_config].nil? && "" || "-c #{@configs[:hadoop_config]}"
+      return file_paths.map do |file_path|
+        FileUtils.remove_file(file_path, true) if File.exists?(file_path)
+        `mandy-get #{mandy_config} #{file_path} #{file_path}`
+        `mandy-rm #{mandy_config} #{file_path}`
+        File.open(file_path) {|file| file.readlines}
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification 
+name: rbpig
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Jae Lee
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-12-22 00:00:00 +00:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: mandy
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 5
+        - 0
+        version: 0.5.0
+  type: :runtime
+  version_requirements: *id001
+description: Simple lib for executing Pig queries, supports textfile based hive table loader with automatic schema discovery
+email: jlee@yetitrails.com
+executables: 
+- rbpig
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- bin/rbpig
+- lib/rbpig.rb
+- lib/rbpig/dataset.rb
+- java/dist/porkchop.jar
+- java/lib/hive/hive-exec-0.5.0+32.jar
+- java/lib/hive/hive-metastore-0.5.0+32.jar
+- java/lib/hive/libfb303.jar
+- java/lib/hive/jdo2-api-2.3-SNAPSHOT.jar
+- java/lib/hive/datanucleus-core-1.1.2-patched.jar
+- java/lib/hive/datanucleus-enhancer-1.1.2.jar
+- java/lib/hive/datanucleus-rdbms-1.1.2.jar
+- java/lib/pig/jsp-2.1-6.1.14.jar
+has_rdoc: true
+homepage: https://github.com/forward/rbpig
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Pig queries execution ruby binding
+test_files: []
+