RubyGems - houdah - Versions diffs - 0.0.1 - Mend

houdah 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (140) hide show

data/README.rdoc ADDED

@@ -0,0 +1,47 @@
+= houdah -- Ruby lib for interacting with a Hadoop JobTracker / TaskTrackers
+Houdah is a Ruby thrift lib for interfacing with a Hadoop jobtracker.
+To install:
+  gem install ganapati
+== Starting thrift server
+Houdah expects that you have Cloudera's Hue[https://ccp.cloudera.com/display/SUPPORT/Downloads] installed and working.  As a part of the Hue installation, you will compile and install a thrift interface that runs when the jobtracker is running.  To be clear,
+you do not have to run Hue in order for Houdah to work - you just need to make sure that the org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin is working.
+== Basic Usage
+  require 'rubygems'
+  require 'houdah'
+  # args are host, optional port (9290 default), optional username (default "houdah"), and optional timeout
+  client = Houdah::Client.new "192.168.1.1"
+  # get list of running jobs
+  client.jobs
+  # get a list of failed jobs
+  client.jobs(:failed)
+  # get jobtracker name
+  client.name
+  # get percent mapped / reduced for all running jobs
+  client.jobs.each { |j| puts "#{j.percent_mapped} mapped #{j.percent_reduced} reduced" }
+  # get percent done for the most recent job
+  client.jobs.first.percent_done
+  # get the mapred.local.dir config variable for the most recent job
+  puts client.jobs.first.config['mapred.local.dir']
+  # get the hive query for the hive job running
+  puts client.jobs.first.config['hive.query.string']
+  # kill all jobs
+  client.jobs.each { |j| j.kill! }
+  # close the client connection
+  client.close
+  # Quick and dirty way to get the number of killed jobs.  The run class method takes care of closing the client.
+  puts Houdah::Client.run("192.168.1.1") { |h| h.jobs(:killed).length }

data/Rakefile ADDED

@@ -0,0 +1,27 @@
+require 'rubygems'
+require 'bundler'
+require 'rake/testtask'
+require 'rake/rdoctask'
+Bundler::GemHelper.install_tasks
+desc "Create documentation"
+Rake::RDocTask.new("doc") { |rdoc|
+  rdoc.title = "Houdah - Riding on the back of Hadoop"
+  rdoc.rdoc_dir = 'docs'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+}
+desc "Re-generate thrift files"
+task "regen_thrift" do
+  if ENV['HUE_HOME'].nil?
+    puts "You must set your HUE_HOME variable before calling this task."
+    return
+  end
+  system "thrift --gen rb -o /tmp #{ENV['HUE_HOME']}/desktop/libs/hadoop/java/if/jobtracker.thrift"
+  system "thrift --gen rb -o /tmp #{ENV['HUE_HOME']}/desktop/libs/hadoop/java/if/common.thrift"
+  system "mv /tmp/gen-rb/* lib/thrift"
+end
+task :default => [ :gem, :doc ]

data/docs/classes/Hadoop.html ADDED

@@ -0,0 +1,140 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html
+     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <title>Module: Hadoop</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+  <meta http-equiv="Content-Script-Type" content="text/javascript" />
+  <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
+  <script type="text/javascript">
+  // <![CDATA[
+  function popupCode( url ) {
+    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
+  }
+  function toggleCode( id ) {
+    if ( document.getElementById )
+      elem = document.getElementById( id );
+    else if ( document.all )
+      elem = eval( "document.all." + id );
+    else
+      return false;
+    elemStyle = elem.style;
+    if ( elemStyle.display != "block" ) {
+      elemStyle.display = "block"
+    } else {
+      elemStyle.display = "none"
+    }
+    return true;
+  }
+  // Make codeblocks hidden by default
+  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
+  // ]]>
+  </script>
+</head>
+<body>
+    <div id="classHeader">
+        <table class="header-table">
+        <tr class="top-aligned-row">
+          <td><strong>Module</strong></td>
+          <td class="class-name-in-header">Hadoop</td>
+        </tr>
+        <tr class="top-aligned-row">
+            <td><strong>In:</strong></td>
+            <td>
+                <a href="../files/lib/thrift/common_constants_rb.html">
+                lib/thrift/common_constants.rb
+                </a>
+        <br />
+                <a href="../files/lib/thrift/common_types_rb.html">
+                lib/thrift/common_types.rb
+                </a>
+        <br />
+                <a href="../files/lib/thrift/hadoop_service_base_rb.html">
+                lib/thrift/hadoop_service_base.rb
+                </a>
+        <br />
+                <a href="../files/lib/thrift/jobtracker_rb.html">
+                lib/thrift/jobtracker.rb
+                </a>
+        <br />
+                <a href="../files/lib/thrift/jobtracker_constants_rb.html">
+                lib/thrift/jobtracker_constants.rb
+                </a>
+        <br />
+                <a href="../files/lib/thrift/jobtracker_types_rb.html">
+                lib/thrift/jobtracker_types.rb
+                </a>
+        <br />
+            </td>
+        </tr>
+        </table>
+    </div>
+  <!-- banner header -->
+  <div id="bodyContent">
+  <div id="contextContent">
+    <div id="description">
+      <p>
+Autogenerated by Thrift
+</p>
+<p>
+DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+</p>
+    </div>
+   </div>
+  </div>
+    <!-- if includes -->
+    <div id="section">
+    <div id="class-list">
+      <h3 class="section-bar">Classes and Modules</h3>
+      Module <a href="Hadoop/API.html" class="link">Hadoop::API</a><br />
+    </div>
+    <!-- if method_list -->
+  </div>
+<div id="validator-badges">
+  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
+</div>
+</body>
+</html>

data/docs/classes/Hadoop/API.html ADDED

@@ -0,0 +1,141 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html
+     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <title>Module: Hadoop::API</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+  <meta http-equiv="Content-Script-Type" content="text/javascript" />
+  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
+  <script type="text/javascript">
+  // <![CDATA[
+  function popupCode( url ) {
+    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
+  }
+  function toggleCode( id ) {
+    if ( document.getElementById )
+      elem = document.getElementById( id );
+    else if ( document.all )
+      elem = eval( "document.all." + id );
+    else
+      return false;
+    elemStyle = elem.style;
+    if ( elemStyle.display != "block" ) {
+      elemStyle.display = "block"
+    } else {
+      elemStyle.display = "none"
+    }
+    return true;
+  }
+  // Make codeblocks hidden by default
+  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
+  // ]]>
+  </script>
+</head>
+<body>
+    <div id="classHeader">
+        <table class="header-table">
+        <tr class="top-aligned-row">
+          <td><strong>Module</strong></td>
+          <td class="class-name-in-header">Hadoop::API</td>
+        </tr>
+        <tr class="top-aligned-row">
+            <td><strong>In:</strong></td>
+            <td>
+                <a href="../../files/lib/thrift/common_constants_rb.html">
+                lib/thrift/common_constants.rb
+                </a>
+        <br />
+                <a href="../../files/lib/thrift/common_types_rb.html">
+                lib/thrift/common_types.rb
+                </a>
+        <br />
+                <a href="../../files/lib/thrift/hadoop_service_base_rb.html">
+                lib/thrift/hadoop_service_base.rb
+                </a>
+        <br />
+                <a href="../../files/lib/thrift/jobtracker_rb.html">
+                lib/thrift/jobtracker.rb
+                </a>
+        <br />
+                <a href="../../files/lib/thrift/jobtracker_constants_rb.html">
+                lib/thrift/jobtracker_constants.rb
+                </a>
+        <br />
+                <a href="../../files/lib/thrift/jobtracker_types_rb.html">
+                lib/thrift/jobtracker_types.rb
+                </a>
+        <br />
+            </td>
+        </tr>
+        </table>
+    </div>
+  <!-- banner header -->
+  <div id="bodyContent">
+  <div id="contextContent">
+   </div>
+  </div>
+    <!-- if includes -->
+    <div id="section">
+    <div id="class-list">
+      <h3 class="section-bar">Classes and Modules</h3>
+      Module <a href="API/HadoopServiceBase.html" class="link">Hadoop::API::HadoopServiceBase</a><br />
+Module <a href="API/Jobtracker.html" class="link">Hadoop::API::Jobtracker</a><br />
+Class <a href="API/IOException.html" class="link">Hadoop::API::IOException</a><br />
+Class <a href="API/MetricsContext.html" class="link">Hadoop::API::MetricsContext</a><br />
+Class <a href="API/MetricsRecord.html" class="link">Hadoop::API::MetricsRecord</a><br />
+Class <a href="API/RequestContext.html" class="link">Hadoop::API::RequestContext</a><br />
+Class <a href="API/RuntimeInfo.html" class="link">Hadoop::API::RuntimeInfo</a><br />
+Class <a href="API/StackTraceElement.html" class="link">Hadoop::API::StackTraceElement</a><br />
+Class <a href="API/ThreadStackTrace.html" class="link">Hadoop::API::ThreadStackTrace</a><br />
+Class <a href="API/ThriftDelegationToken.html" class="link">Hadoop::API::ThriftDelegationToken</a><br />
+Class <a href="API/VersionInfo.html" class="link">Hadoop::API::VersionInfo</a><br />
+    </div>
+    <!-- if method_list -->
+  </div>
+<div id="validator-badges">
+  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
+</div>
+</body>
+</html>

data/docs/classes/Hadoop/API/HadoopServiceBase.html ADDED

@@ -0,0 +1,122 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html
+     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <title>Module: Hadoop::API::HadoopServiceBase</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+  <meta http-equiv="Content-Script-Type" content="text/javascript" />
+  <link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
+  <script type="text/javascript">
+  // <![CDATA[
+  function popupCode( url ) {
+    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
+  }
+  function toggleCode( id ) {
+    if ( document.getElementById )
+      elem = document.getElementById( id );
+    else if ( document.all )
+      elem = eval( "document.all." + id );
+    else
+      return false;
+    elemStyle = elem.style;
+    if ( elemStyle.display != "block" ) {
+      elemStyle.display = "block"
+    } else {
+      elemStyle.display = "none"
+    }
+    return true;
+  }
+  // Make codeblocks hidden by default
+  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
+  // ]]>
+  </script>
+</head>
+<body>
+    <div id="classHeader">
+        <table class="header-table">
+        <tr class="top-aligned-row">
+          <td><strong>Module</strong></td>
+          <td class="class-name-in-header">Hadoop::API::HadoopServiceBase</td>
+        </tr>
+        <tr class="top-aligned-row">
+            <td><strong>In:</strong></td>
+            <td>
+                <a href="../../../files/lib/thrift/hadoop_service_base_rb.html">
+                lib/thrift/hadoop_service_base.rb
+                </a>
+        <br />
+            </td>
+        </tr>
+        </table>
+    </div>
+  <!-- banner header -->
+  <div id="bodyContent">
+  <div id="contextContent">
+   </div>
+  </div>
+    <!-- if includes -->
+    <div id="section">
+    <div id="class-list">
+      <h3 class="section-bar">Classes and Modules</h3>
+      Class <a href="HadoopServiceBase/Client.html" class="link">Hadoop::API::HadoopServiceBase::Client</a><br />
+Class <a href="HadoopServiceBase/GetAllMetrics_args.html" class="link">Hadoop::API::HadoopServiceBase::GetAllMetrics_args</a><br />
+Class <a href="HadoopServiceBase/GetAllMetrics_result.html" class="link">Hadoop::API::HadoopServiceBase::GetAllMetrics_result</a><br />
+Class <a href="HadoopServiceBase/GetMetricsContext_args.html" class="link">Hadoop::API::HadoopServiceBase::GetMetricsContext_args</a><br />
+Class <a href="HadoopServiceBase/GetMetricsContext_result.html" class="link">Hadoop::API::HadoopServiceBase::GetMetricsContext_result</a><br />
+Class <a href="HadoopServiceBase/GetRuntimeInfo_args.html" class="link">Hadoop::API::HadoopServiceBase::GetRuntimeInfo_args</a><br />
+Class <a href="HadoopServiceBase/GetRuntimeInfo_result.html" class="link">Hadoop::API::HadoopServiceBase::GetRuntimeInfo_result</a><br />
+Class <a href="HadoopServiceBase/GetThreadDump_args.html" class="link">Hadoop::API::HadoopServiceBase::GetThreadDump_args</a><br />
+Class <a href="HadoopServiceBase/GetThreadDump_result.html" class="link">Hadoop::API::HadoopServiceBase::GetThreadDump_result</a><br />
+Class <a href="HadoopServiceBase/GetVersionInfo_args.html" class="link">Hadoop::API::HadoopServiceBase::GetVersionInfo_args</a><br />
+Class <a href="HadoopServiceBase/GetVersionInfo_result.html" class="link">Hadoop::API::HadoopServiceBase::GetVersionInfo_result</a><br />
+Class <a href="HadoopServiceBase/Processor.html" class="link">Hadoop::API::HadoopServiceBase::Processor</a><br />
+    </div>
+    <!-- if method_list -->
+  </div>
+<div id="validator-badges">
+  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
+</div>
+</body>
+</html>