RubyGems - houdah - Versions diffs - 0.0.1 - Mend

houdah 0.0.1

Files changed (140) hide show

data/README.rdoc ADDED

@@ -0,0 +1,47 @@
+= houdah -- Ruby lib for interacting with a Hadoop JobTracker / TaskTrackers
+Houdah is a Ruby thrift lib for interfacing with a Hadoop jobtracker.
+To install:
+  gem install ganapati
+== Starting thrift server
+Houdah expects that you have Cloudera's Hue[https://ccp.cloudera.com/display/SUPPORT/Downloads] installed and working.  As a part of the Hue installation, you will compile and install a thrift interface that runs when the jobtracker is running.  To be clear,
+you do not have to run Hue in order for Houdah to work - you just need to make sure that the org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin is working.
+== Basic Usage
+  require 'rubygems'
+  require 'houdah'
+  # args are host, optional port (9290 default), optional username (default "houdah"), and optional timeout
+  client = Houdah::Client.new "192.168.1.1"
+  # get list of running jobs
+  client.jobs
+  # get a list of failed jobs
+  client.jobs(:failed)
+  # get jobtracker name
+  client.name
+  # get percent mapped / reduced for all running jobs
+  client.jobs.each { |j| puts "#{j.percent_mapped} mapped #{j.percent_reduced} reduced" }
+  # get percent done for the most recent job
+  client.jobs.first.percent_done
+  # get the mapred.local.dir config variable for the most recent job
+  puts client.jobs.first.config['mapred.local.dir']
+  # get the hive query for the hive job running
+  puts client.jobs.first.config['hive.query.string']
+  # kill all jobs
+  client.jobs.each { |j| j.kill! }
+  # close the client connection
+  client.close
+  # Quick and dirty way to get the number of killed jobs.  The run class method takes care of closing the client.
+  puts Houdah::Client.run("192.168.1.1") { |h| h.jobs(:killed).length }

data/Rakefile ADDED

@@ -0,0 +1,27 @@
+require 'rubygems'
+require 'bundler'
+require 'rake/testtask'
+require 'rake/rdoctask'
+Bundler::GemHelper.install_tasks
+desc "Create documentation"
+Rake::RDocTask.new("doc") { |rdoc|
+  rdoc.title = "Houdah - Riding on the back of Hadoop"
+  rdoc.rdoc_dir = 'docs'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+}
+desc "Re-generate thrift files"
+task "regen_thrift" do
+  if ENV['HUE_HOME'].nil?
+    puts "You must set your HUE_HOME variable before calling this task."
+    return
+  end
+  system "thrift --gen rb -o /tmp #{ENV['HUE_HOME']}/desktop/libs/hadoop/java/if/jobtracker.thrift"
+  system "thrift --gen rb -o /tmp #{ENV['HUE_HOME']}/desktop/libs/hadoop/java/if/common.thrift"
+  system "mv /tmp/gen-rb/* lib/thrift"
+end
+task :default => [ :gem, :doc ]

data/docs/classes/Hadoop.html ADDED

@@ -0,0 +1,140 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html
+     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <title>Module: Hadoop</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+  <meta http-equiv="Content-Script-Type" content="text/javascript" />
+  <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
+  <script type="text/javascript">
+  // <![CDATA[
+  function popupCode( url ) {
+    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
+  }
+  function toggleCode( id ) {
+    if ( document.getElementById )
+      elem = document.getElementById( id );
+    else if ( document.all )
+      elem = eval( "document.all." + id );
+    else
+      return false;
+    elemStyle = elem.style;
+    if ( elemStyle.display != "block" ) {
+      elemStyle.display = "block"
+    } else {
+      elemStyle.display = "none"
+    }
+    return true;
+  }
+  // Make codeblocks hidden by default
+  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
+  // ]]>
+  </script>
+</head>
+<body>
+    <div id="classHeader">
+        <table class="header-table">
+        <tr class="top-aligned-row">
+          <td><strong>Module</strong></td>
+          <td class="class-name-in-header">Hadoop</td>
+        </tr>
+        <tr class="top-aligned-row">
+            <td><strong>In:</strong></td>
+            <td>
+                <a href="../files/lib/thrift/common_constants_rb.html">
+                lib/thrift/common_constants.rb
+                </a>
+        <br />
+                <a href="../files/lib/thrift/common_types_rb.html">
+                lib/thrift/common_types.rb
+                </a>
+        <br />
+                <a href="../files/lib/thrift/hadoop_service_base_rb.html">
+                lib/thrift/hadoop_service_base.rb
+                </a>
+        <br />
+                <a href="../files/lib/thrift/jobtracker_rb.html">
+                lib/thrift/jobtracker.rb
+                </a>
+        <br />
+                <a href="../files/lib/thrift/jobtracker_constants_rb.html">
+                lib/thrift/jobtracker_constants.rb
+                </a>
+        <br />
+                <a href="../files/lib/thrift/jobtracker_types_rb.html">
+                lib/thrift/jobtracker_types.rb
+                </a>
+        <br />
+            </td>
+        </tr>
+        </table>
+    </div>
+  <!-- banner header -->
+  <div id="bodyContent">
+  <div id="contextContent">
+    <div id="description">
+      <p>
+Autogenerated by Thrift
+</p>
+<p>
+DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+</p>
+    </div>
+   </div>
+  </div>
+    <!-- if includes -->
+    <div id="section">
+    <div id="class-list">
+      <h3 class="section-bar">Classes and Modules</h3>
+      Module <a href="Hadoop/API.html" class="link">Hadoop::API</a><br />
+    </div>
+    <!-- if method_list -->
+  </div>
+<div id="validator-badges">
+  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
+</div>
+</body>
+</html>

data/docs/classes/Hadoop/API.html ADDED

@@ -0,0 +1,141 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html
+     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <title>Module: Hadoop::API</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+  <meta http-equiv="Content-Script-Type" content="text/javascript" />
+  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
+  <script type="text/javascript">
+  // <![CDATA[
+  function popupCode( url ) {
+    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
+  }
+  function toggleCode( id ) {
+    if ( document.getElementById )
+      elem = document.getElementById( id );
+    else if ( document.all )
+      elem = eval( "document.all." + id );
+    else
+      return false;
+    elemStyle = elem.style;
+    if ( elemStyle.display != "block" ) {
+      elemStyle.display = "block"
+    } else {
+      elemStyle.display = "none"
+    }
+    return true;
+  }
+  // Make codeblocks hidden by default
+  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
+  // ]]>
+  </script>
+</head>
+<body>
+    <div id="classHeader">
+        <table class="header-table">
+        <tr class="top-aligned-row">
+          <td><strong>Module</strong></td>
+          <td class="class-name-in-header">Hadoop::API</td>
+        </tr>
+        <tr class="top-aligned-row">
+            <td><strong>In:</strong></td>
+            <td>
+                <a href="../../files/lib/thrift/common_constants_rb.html">
+                lib/thrift/common_constants.rb
+                </a>
+        <br />
+                <a href="../../files/lib/thrift/common_types_rb.html">
+                lib/thrift/common_types.rb
+                </a>
+        <br />
+                <a href="../../files/lib/thrift/hadoop_service_base_rb.html">
+                lib/thrift/hadoop_service_base.rb
+                </a>
+        <br />
+                <a href="../../files/lib/thrift/jobtracker_rb.html">
+                lib/thrift/jobtracker.rb
+                </a>
+        <br />
+                <a href="../../files/lib/thrift/jobtracker_constants_rb.html">
+                lib/thrift/jobtracker_constants.rb
+                </a>
+        <br />
+                <a href="../../files/lib/thrift/jobtracker_types_rb.html">
+                lib/thrift/jobtracker_types.rb
+                </a>
+        <br />
+            </td>
+        </tr>
+        </table>
+    </div>
+  <!-- banner header -->
+  <div id="bodyContent">
+  <div id="contextContent">
+   </div>
+  </div>
+    <!-- if includes -->
+    <div id="section">
+    <div id="class-list">
+      <h3 class="section-bar">Classes and Modules</h3>
+      Module <a href="API/HadoopServiceBase.html" class="link">Hadoop::API::HadoopServiceBase</a><br />
+Module <a href="API/Jobtracker.html" class="link">Hadoop::API::Jobtracker</a><br />
+Class <a href="API/IOException.html" class="link">Hadoop::API::IOException</a><br />
+Class <a href="API/MetricsContext.html" class="link">Hadoop::API::MetricsContext</a><br />
+Class <a href="API/MetricsRecord.html" class="link">Hadoop::API::MetricsRecord</a><br />
+Class <a href="API/RequestContext.html" class="link">Hadoop::API::RequestContext</a><br />
+Class <a href="API/RuntimeInfo.html" class="link">Hadoop::API::RuntimeInfo</a><br />
+Class <a href="API/StackTraceElement.html" class="link">Hadoop::API::StackTraceElement</a><br />
+Class <a href="API/ThreadStackTrace.html" class="link">Hadoop::API::ThreadStackTrace</a><br />
+Class <a href="API/ThriftDelegationToken.html" class="link">Hadoop::API::ThriftDelegationToken</a><br />
+Class <a href="API/VersionInfo.html" class="link">Hadoop::API::VersionInfo</a><br />
+    </div>
+    <!-- if method_list -->
+  </div>
+<div id="validator-badges">
+  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
+</div>
+</body>
+</html>

data/docs/classes/Hadoop/API/HadoopServiceBase.html ADDED

@@ -0,0 +1,122 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html
+     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <title>Module: Hadoop::API::HadoopServiceBase</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+  <meta http-equiv="Content-Script-Type" content="text/javascript" />
+  <link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
+  <script type="text/javascript">
+  // <![CDATA[
+  function popupCode( url ) {
+    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
+  }
+  function toggleCode( id ) {
+    if ( document.getElementById )
+      elem = document.getElementById( id );
+    else if ( document.all )
+      elem = eval( "document.all." + id );
+    else
+      return false;
+    elemStyle = elem.style;
+    if ( elemStyle.display != "block" ) {
+      elemStyle.display = "block"
+    } else {
+      elemStyle.display = "none"
+    }
+    return true;
+  }
+  // Make codeblocks hidden by default
+  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
+  // ]]>
+  </script>
+</head>
+<body>
+    <div id="classHeader">
+        <table class="header-table">
+        <tr class="top-aligned-row">
+          <td><strong>Module</strong></td>
+          <td class="class-name-in-header">Hadoop::API::HadoopServiceBase</td>
+        </tr>
+        <tr class="top-aligned-row">
+            <td><strong>In:</strong></td>
+            <td>
+                <a href="../../../files/lib/thrift/hadoop_service_base_rb.html">
+                lib/thrift/hadoop_service_base.rb
+                </a>
+        <br />
+            </td>
+        </tr>
+        </table>
+    </div>
+  <!-- banner header -->
+  <div id="bodyContent">
+  <div id="contextContent">
+   </div>
+  </div>
+    <!-- if includes -->
+    <div id="section">
+    <div id="class-list">
+      <h3 class="section-bar">Classes and Modules</h3>
+      Class <a href="HadoopServiceBase/Client.html" class="link">Hadoop::API::HadoopServiceBase::Client</a><br />
+Class <a href="HadoopServiceBase/GetAllMetrics_args.html" class="link">Hadoop::API::HadoopServiceBase::GetAllMetrics_args</a><br />
+Class <a href="HadoopServiceBase/GetAllMetrics_result.html" class="link">Hadoop::API::HadoopServiceBase::GetAllMetrics_result</a><br />
+Class <a href="HadoopServiceBase/GetMetricsContext_args.html" class="link">Hadoop::API::HadoopServiceBase::GetMetricsContext_args</a><br />
+Class <a href="HadoopServiceBase/GetMetricsContext_result.html" class="link">Hadoop::API::HadoopServiceBase::GetMetricsContext_result</a><br />
+Class <a href="HadoopServiceBase/GetRuntimeInfo_args.html" class="link">Hadoop::API::HadoopServiceBase::GetRuntimeInfo_args</a><br />
+Class <a href="HadoopServiceBase/GetRuntimeInfo_result.html" class="link">Hadoop::API::HadoopServiceBase::GetRuntimeInfo_result</a><br />
+Class <a href="HadoopServiceBase/GetThreadDump_args.html" class="link">Hadoop::API::HadoopServiceBase::GetThreadDump_args</a><br />
+Class <a href="HadoopServiceBase/GetThreadDump_result.html" class="link">Hadoop::API::HadoopServiceBase::GetThreadDump_result</a><br />
+Class <a href="HadoopServiceBase/GetVersionInfo_args.html" class="link">Hadoop::API::HadoopServiceBase::GetVersionInfo_args</a><br />
+Class <a href="HadoopServiceBase/GetVersionInfo_result.html" class="link">Hadoop::API::HadoopServiceBase::GetVersionInfo_result</a><br />
+Class <a href="HadoopServiceBase/Processor.html" class="link">Hadoop::API::HadoopServiceBase::Processor</a><br />
+    </div>
+    <!-- if method_list -->
+  </div>
+<div id="validator-badges">
+  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
+</div>
+</body>
+</html>