remoteHadoopGem 0.0.0

data/lib/remoteHadoopGem.rb

@@ -0,0 +1,108 @@
+require "uuidtools"
+
+class RemoteHadoopGem
+  
+  #
+  #		****** commandToHadoop ****** 
+  #
+  #	function to run a command in a remote hadoop machine
+  #	require: 
+  #		1) username (to access the machine via ssh)
+  #		2) host (IP of the hadoop machine) 
+  #		3) keyPathFile (path of the private key to access the machine)
+  #		4) command to execute
+  
+  def self.commandToHadoop(username, host, keyPathFile, command)
+    res=`ssh -i "#{keyPathFile}" -o StrictHostKeyChecking=no "#{username}"@"#{host}" "export PATH=/hadoop/bin/:$PATH; cd /hadoop/; #{command}"`
+    return "#{res}"
+  end
+
+  #
+  #		****** commandToHadoopJobID ****** 
+  #
+  #	function to run a command in a remote hadoop machine; returns the <job-id>
+  #	require: 
+  #		1) username (to access the machine via ssh)
+  #		2) host (IP of the hadoop machine) 
+  #		3) keyPathFile (path of the private key to access the machine)
+  #		4) command to execute
+  
+  def self.commandToHadoopJobID(username, host, keyPathFile, command)
+    # create a file with a random name to obtained the jobID
+    filename="tempJobID_"+UUIDTools::UUID.random_create.to_s
+    `ssh -n -f -i "#{keyPathFile}" "#{username}"@"#{host}" 'sh -c "export PATH=/hadoop/bin/:$PATH; cd /hadoop/; nohup #{command} >/dev/null 2>#{filename}.txt & " '`
+    res=`sleep 5; ssh -i "#{keyPathFile}" "#{username}"@"#{host}" "cat /hadoop/#{filename}.txt | grep 'Running job' " `
+    `ssh -i "#{keyPathFile}" "#{username}"@"#{host}" "rm /hadoop/#{filename}.txt"`
+    jobid=res.split[6]
+    
+    return "#{jobid}"
+  end
+
+  #
+  #		****** copyFileTo ****** 
+  #
+  #	function to copy a file in a remote hadoop machine
+  #	require: 
+  #		1) username (to access the machine via ssh)
+  #		2) host (IP of the hadoop machine) 
+  #		3) keyPathFile (path of the private key to access the machine)
+  #		4) filePath (path of the local file)
+  #		5) destFilePath (path on the remote hadoop node)
+  
+  def self.copyFileTo(username, host, keyPathFile, filePath, destFilePath)
+    res=`scp -r -i "#{keyPathFile}" -o StrictHostKeyChecking=no #{filePath}  "#{username}"@"#{host}":#{destFilePath}`
+  end
+  
+  #
+  #		****** readFileFrom ****** 
+  #
+  #	function to read a file from a remote hadoop machine
+  #	require: 
+  #		1) username (to access the machine via ssh)
+  #		2) host (IP of the hadoop machine) 
+  #		3) keyPathFile (path of the private key to access the machine)
+  #		4) destFilePath (path of the remote file to read)
+  #	
+
+  def self.readFileFrom(username, host, keyPathFile, destFilePath)
+    res=`ssh -i "#{keyPathFile}" -o StrictHostKeyChecking=no "#{username}"@"#{host}" "cat #{destFilePath}"`
+    return "#{res}"
+  end
+
+  #
+  #		****** jobsList ****** 
+  #
+  #	function to list the running jobs in a remote hadoop machine
+  #	require: 
+  #		1) username (to access the machine via ssh)
+  #		2) host (IP of the hadoop machine) 
+  #		3) keyPathFile (path of the private key to access the machine)
+  #	
+
+  def self.jobsList(username, host, keyPathFile)
+    res=`ssh -i "#{keyPathFile}" -o StrictHostKeyChecking=no "#{username}"@"#{host}" "export PATH=/hadoop/bin/:$PATH; cd /hadoop/; hadoop job -list "`
+    return "#{res}"
+  end
+
+  #
+  #		****** jobStatus ****** 
+  #
+  #	function to read the status of a running job in a remote hadoop machine
+  #	require: 
+  #		1) username (to access the machine via ssh)
+  #		2) host (IP of the hadoop machine) 
+  #		3) keyPathFile (path of the private key to access the machine)
+  #		4) job_id (id of the job)
+  #	
+
+  def self.jobStatus(username, host, keyPathFile, job_id)
+    res=`ssh -i "#{keyPathFile}" -o StrictHostKeyChecking=no "#{username}"@"#{host}" "export PATH=/hadoop/bin/:$PATH; cd /hadoop/; hadoop job -status #{job_id} "`
+    return "#{res}"
+  end
+  
+
+  def self.cleanTempFiles(username, host, keyPathFile)
+    `ssh -i "#{keyPathFile}" -o StrictHostKeyChecking=no "#{username}"@"#{host}" "export PATH=/hadoop/bin/:$PATH; cd /hadoop/; rm ./tempJobID_* "`
+  end
+
+end

metadata

@@ -0,0 +1,46 @@
+--- !ruby/object:Gem::Specification
+name: remoteHadoopGem
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Fraternale
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-06 00:00:00.000000000 Z
+dependencies: []
+description: A simple test gem
+email: f.seby87@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/remoteHadoopGem.rb
+homepage: ''
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Hola!
+test_files: []