ganapati 0.0.4 → 0.0.5

data/README.rdoc

@@ -0,0 +1,86 @@
+= ganapati -- Hadoop HDFS Thrift interface for Ruby
+
+Ganapati is a Ruby thrift lib for interfacing with Hadoop's distributed file system, HDFS.  It also includes a few command line client utilities.
+
+To install:
+  gem install ganapati
+
+== Starting thrift server
+Documentation in Hadoop for the thrift interface to HDFS is crap.  It can be found here[http://wiki.apache.org/hadoop/HDFS-APIs].
+  
+As a much simpler and safer way of auto compiling and then starting the thrift interface, use the provided script:
+  bin/hdfs_thrift_server <port>
+
+This will start a thrift server on the given port (after compiling the server code provided in the Hadoop distribution).
+
+== Basic Usage
+  require 'rubygems'
+  require 'ganapati'
+
+  # args are host, port, and optional timeout
+  client = Ganapati::Client.new 'localhost', 1234
+
+  # copy a file to hdfs
+  client.put("/some/file", "/some/hadoop/path")
+  
+  # get a file from hadoop
+  client.get("/some/hadoop/path", "/local/path")
+
+  # Create a file
+  f = client.create("/home/someuser/afile.txt")
+  f.write("this is some text")
+  # Always, always close the file
+  f.close 
+
+  # Create a file with code block
+  client.create("/home/someuser/afile.txt") { |f|
+    f.write("this is some text")
+  }
+
+  # Open a file for reading and read it
+  client.open('/home/someuser/afile.txt') { |f| 
+    puts f.read 
+    # or read for specific length from start
+    puts f.read(0, 4)
+  }
+
+  # read a file line by line
+  client.readlines('/home/someuser/afile.txt') { |line|
+    puts line
+  }
+
+  # Open a file for appending and append to it
+  client.append('/home/someuser/afile.txt') { |f| 
+    f.write "new data" 
+  }	  
+
+  ## Common file methods are available (chown, chmod, mkdir, stat, etc).  Examples:
+  # move a file
+  client.mv "/home/someuser/afile.txt", "/home/someuser/test.txt"
+
+  # remove a file
+  client.rm "/home/someuser/test.txt"
+
+  # test for file existance
+  client.exists? "/home/someuser/test.txt"
+
+  # get a list of all files
+  client.ls "/home"
+
+  client.close
+
+  # Quick and dirty way to print remote file.  The run class method takes care of closing the client.
+  puts Ganapati::Client.run('localhost', 1234) { |c| c.open('/home/someuser/afile.txt') { |f| f.read } } 
+
+== Command Line Utilities
+There are a few utility programs included in the bin directory.  *hls* provides a way to see the contents of HDFS (recursively and verbosely with appropriate command line options):
+  ./bin/hls hdfs://host:port/tmp
+
+*hcp* provides a way to copy to/from/between HDFS servers:
+  ./bin/hcp hdfs://host:port/some/path/to/file ./file
+  ./bin/hcp ./file hdfs://host:port/some/path/to/file
+  ./bin/hcp hdfs://anotherhost:port/some/path/to/file hdfs://host:port/some/path/to/file
+
+
+
+

data/Rakefile

@@ -0,0 +1,32 @@
+require 'rubygems'
+require 'bundler'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+Bundler::GemHelper.install_tasks
+
+desc "Create documentation"
+Rake::RDocTask.new("doc") { |rdoc|
+  rdoc.title = "Ganapati - Hadoop HDFS thrift interface for Ruby"
+  rdoc.rdoc_dir = 'docs'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+}
+
+desc "Re-generate thrift files"
+task "regen_thrift" do 
+  if ENV['HADOOP_HOME'].nil?
+    puts "You must set your HADOOP_HOME variable before calling this task."
+    return
+  end
+  system "thrift --gen rb -o /tmp #{ENV['HADOOP_HOME']}src/contrib/thriftfs/if/hadoopfs.thrift"
+  system "mv /tmp/gen-rb/* lib/thrift"
+end
+
+task :default => [ :gem, :doc ]
+
+
+
+
+
+

data/bin/hcp

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'rubygems'
+require 'ganapati'
+
+options = { :recursive => false, :verbose => false }
+op = OptionParser.new { |opts|
+  opts.banner = "Usage: hcp [options] source dest"
+
+  opts.on('-h', '--help', 'displays usage information') { 
+    puts opts
+    exit
+  }
+
+  opts.on('-v', '--verbose', 'verbose') {
+    options[:verbose] = true
+  }
+
+  opts.on('-r', '--recursive', 'copy recursively') {
+    puts "Recursive option not supported yet."
+    exit 1
+    #options[:recursive] = true
+  }
+}
+op.parse!
+
+if ARGV.length != 2
+  puts op.to_s
+  exit 1
+end
+
+class LocalFile
+  def create(location)
+    f = open(location, 'w')
+    yield f
+    f.close
+  end
+
+  def readlines(location)
+    f = open(location, 'r')
+    while (line = f.gets)
+      yield line
+    end
+    f.close
+  end
+  
+  def close
+  end
+end
+
+spath = Ganapati::FileUrl.new ARGV[0], :file
+dpath = Ganapati::FileUrl.new ARGV[1], :file
+
+source = spath.hdfs? ? Ganapati::Client.new(spath.host, spath.port) : LocalFile.new
+dest = dpath.hdfs? ? Ganapati::Client.new(dpath.host, dpath.port) : LocalFile.new
+
+dest.create(dpath.path) { |f|
+  source.readlines(spath.path) { |line|
+    f.write(line)
+  }
+}
+
+source.close
+dest.close

data/bin/hls

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'rubygems'
+require 'ganapati'
+
+VERBOSE_PROPS = [:path, :length, :isdir, :modification_time, :permission, :owner, :group]
+
+options = { :verbose => false, :recursive => false }
+op = OptionParser.new { |opts|
+  opts.banner = "Usage: hls [options] host:port/location"
+
+  opts.on('-h', '--help', 'displays usage information') { 
+    puts opts
+    exit
+  }
+
+  opts.on('-r', '--recursive', 'copy recursively') {
+    options[:recursive] = true
+  }
+
+  opts.on('-v', '--verbose', 'verbose output') {
+    puts VERBOSE_PROPS.map { |a| a.to_s }.join("\t")
+    options[:verbose] = true
+  }
+}
+op.parse!
+
+if ARGV.length != 1
+  puts op.to_s
+  exit 1
+end
+
+url = Ganapati::FileUrl.new ARGV.first
+client = Ganapati::Client.new url.host, url.port
+client.ls(url.path, options[:verbose], options[:recursive]).each { |s|
+  if options[:verbose]
+    puts VERBOSE_PROPS.map { |p| s.send p }.join("\t")
+  else
+    puts s
+  end
+}
+client.close

data/docs/classes/Ganapati.html

@@ -0,0 +1,138 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html 
+     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <title>Module: Ganapati</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+  <meta http-equiv="Content-Script-Type" content="text/javascript" />
+  <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
+  <script type="text/javascript">
+  // <![CDATA[
+
+  function popupCode( url ) {
+    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
+  }
+
+  function toggleCode( id ) {
+    if ( document.getElementById )
+      elem = document.getElementById( id );
+    else if ( document.all )
+      elem = eval( "document.all." + id );
+    else
+      return false;
+
+    elemStyle = elem.style;
+    
+    if ( elemStyle.display != "block" ) {
+      elemStyle.display = "block"
+    } else {
+      elemStyle.display = "none"
+    }
+
+    return true;
+  }
+  
+  // Make codeblocks hidden by default
+  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
+  
+  // ]]>
+  </script>
+
+</head>
+<body>
+
+
+
+    <div id="classHeader">
+        <table class="header-table">
+        <tr class="top-aligned-row">
+          <td><strong>Module</strong></td>
+          <td class="class-name-in-header">Ganapati</td>
+        </tr>
+        <tr class="top-aligned-row">
+            <td><strong>In:</strong></td>
+            <td>
+                <a href="../files/lib/ganapati/client_rb.html">
+                lib/ganapati/client.rb
+                </a>
+        <br />
+                <a href="../files/lib/ganapati/hfile_rb.html">
+                lib/ganapati/hfile.rb
+                </a>
+        <br />
+                <a href="../files/lib/ganapati/utils_rb.html">
+                lib/ganapati/utils.rb
+                </a>
+        <br />
+                <a href="../files/lib/ganapati/version_rb.html">
+                lib/ganapati/version.rb
+                </a>
+        <br />
+            </td>
+        </tr>
+
+        </table>
+    </div>
+  <!-- banner header -->
+
+  <div id="bodyContent">
+
+
+
+  <div id="contextContent">
+
+
+
+   </div>
+
+
+  </div>
+
+
+    <!-- if includes -->
+
+    <div id="section">
+
+    <div id="class-list">
+      <h3 class="section-bar">Classes and Modules</h3>
+
+      Class <a href="Ganapati/Client.html" class="link">Ganapati::Client</a><br />
+Class <a href="Ganapati/FileUrl.html" class="link">Ganapati::FileUrl</a><br />
+Class <a href="Ganapati/HFile.html" class="link">Ganapati::HFile</a><br />
+
+    </div>
+
+    <div id="constants-list">
+      <h3 class="section-bar">Constants</h3>
+
+      <div class="name-list">
+        <table summary="Constants">
+        <tr class="top-aligned-row context-row">
+          <td class="context-item-name">VERSION</td>
+          <td>=</td>
+          <td class="context-item-value">&quot;0.0.5&quot;</td>
+        </tr>
+        </table>
+      </div>
+    </div>
+
+
+
+      
+
+
+    <!-- if method_list -->
+
+
+  </div>
+
+
+<div id="validator-badges">
+  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
+</div>
+
+</body>
+</html>

data/docs/classes/Ganapati/Client.html

@@ -0,0 +1,666 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html 
+     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <title>Class: Ganapati::Client</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+  <meta http-equiv="Content-Script-Type" content="text/javascript" />
+  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
+  <script type="text/javascript">
+  // <![CDATA[
+
+  function popupCode( url ) {
+    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
+  }
+
+  function toggleCode( id ) {
+    if ( document.getElementById )
+      elem = document.getElementById( id );
+    else if ( document.all )
+      elem = eval( "document.all." + id );
+    else
+      return false;
+
+    elemStyle = elem.style;
+    
+    if ( elemStyle.display != "block" ) {
+      elemStyle.display = "block"
+    } else {
+      elemStyle.display = "none"
+    }
+
+    return true;
+  }
+  
+  // Make codeblocks hidden by default
+  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
+  
+  // ]]>
+  </script>
+
+</head>
+<body>
+
+
+
+    <div id="classHeader">
+        <table class="header-table">
+        <tr class="top-aligned-row">
+          <td><strong>Class</strong></td>
+          <td class="class-name-in-header">Ganapati::Client</td>
+        </tr>
+        <tr class="top-aligned-row">
+            <td><strong>In:</strong></td>
+            <td>
+                <a href="../../files/lib/ganapati/client_rb.html">
+                lib/ganapati/client.rb
+                </a>
+        <br />
+            </td>
+        </tr>
+
+        <tr class="top-aligned-row">
+            <td><strong>Parent:</strong></td>
+            <td>
+                Object
+            </td>
+        </tr>
+        </table>
+    </div>
+  <!-- banner header -->
+
+  <div id="bodyContent">
+
+
+
+  <div id="contextContent">
+
+
+
+   </div>
+
+    <div id="method-list">
+      <h3 class="section-bar">Methods</h3>
+
+      <div class="name-list">
+      <a href="#M000180">append</a>&nbsp;&nbsp;
+      <a href="#M000187">chmod</a>&nbsp;&nbsp;
+      <a href="#M000188">chown</a>&nbsp;&nbsp;
+      <a href="#M000172">close</a>&nbsp;&nbsp;
+      <a href="#M000178">create</a>&nbsp;&nbsp;
+      <a href="#M000184">exists?</a>&nbsp;&nbsp;
+      <a href="#M000175">get</a>&nbsp;&nbsp;
+      <a href="#M000186">ls</a>&nbsp;&nbsp;
+      <a href="#M000183">mkdir</a>&nbsp;&nbsp;
+      <a href="#M000182">mv</a>&nbsp;&nbsp;
+      <a href="#M000171">new</a>&nbsp;&nbsp;
+      <a href="#M000179">open</a>&nbsp;&nbsp;
+      <a href="#M000174">put</a>&nbsp;&nbsp;
+      <a href="#M000176">readchunks</a>&nbsp;&nbsp;
+      <a href="#M000177">readlines</a>&nbsp;&nbsp;
+      <a href="#M000181">rm</a>&nbsp;&nbsp;
+      <a href="#M000190">run</a>&nbsp;&nbsp;
+      <a href="#M000189">set_replication</a>&nbsp;&nbsp;
+      <a href="#M000173">shutdown</a>&nbsp;&nbsp;
+      <a href="#M000185">stat</a>&nbsp;&nbsp;
+      </div>
+    </div>
+
+  </div>
+
+
+    <!-- if includes -->
+
+    <div id="section">
+
+
+
+
+
+      
+
+
+    <!-- if method_list -->
+    <div id="methods">
+      <h3 class="section-bar">Public Class methods</h3>
+
+      <div id="method-M000171" class="method-detail">
+        <a name="M000171"></a>
+
+        <div class="method-heading">
+          <a href="#M000171" class="method-signature">
+          <span class="method-name">new</span><span class="method-args">(server, port, timeout=60)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000171-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000171-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 4</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>(<span class="ruby-identifier">server</span>, <span class="ruby-identifier">port</span>, <span class="ruby-identifier">timeout</span>=<span class="ruby-value">60</span>)
+      <span class="ruby-identifier">socket</span> = <span class="ruby-constant">Thrift</span><span class="ruby-operator">::</span><span class="ruby-constant">Socket</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">server</span>, <span class="ruby-identifier">port</span>)
+      <span class="ruby-ivar">@transport</span> = <span class="ruby-constant">Thrift</span><span class="ruby-operator">::</span><span class="ruby-constant">BufferedTransport</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">socket</span>)
+      <span class="ruby-ivar">@transport</span>.<span class="ruby-identifier">open</span>
+      <span class="ruby-identifier">protocol</span> = <span class="ruby-constant">Thrift</span><span class="ruby-operator">::</span><span class="ruby-constant">BinaryProtocol</span>.<span class="ruby-identifier">new</span>(<span class="ruby-ivar">@transport</span>)
+      <span class="ruby-ivar">@client</span> = <span class="ruby-constant">ThriftHadoopFileSystem</span><span class="ruby-operator">::</span><span class="ruby-constant">Client</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">protocol</span>)
+      <span class="ruby-ivar">@client</span>.<span class="ruby-identifier">setInactivityTimeoutPeriod</span>(<span class="ruby-identifier">timeout</span>)
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000190" class="method-detail">
+        <a name="M000190"></a>
+
+        <div class="method-heading">
+          <a href="#M000190" class="method-signature">
+          <span class="method-name">run</span><span class="method-args">(server, port) {|c| ...}</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000190-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000190-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 125</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-keyword kw">self</span>.<span class="ruby-identifier">run</span>(<span class="ruby-identifier">server</span>, <span class="ruby-identifier">port</span>)
+      <span class="ruby-identifier">c</span> = <span class="ruby-constant">Client</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">server</span>, <span class="ruby-identifier">port</span>)
+      <span class="ruby-identifier">result</span> = <span class="ruby-keyword kw">yield</span> <span class="ruby-identifier">c</span>
+      <span class="ruby-identifier">c</span>.<span class="ruby-identifier">close</span>
+      <span class="ruby-identifier">result</span>
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <h3 class="section-bar">Public Instance methods</h3>
+
+      <div id="method-M000180" class="method-detail">
+        <a name="M000180"></a>
+
+        <div class="method-heading">
+          <a href="#M000180" class="method-signature">
+          <span class="method-name">append</span><span class="method-args">(path, &amp;block)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p>
+for appending
+</p>
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000180-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000180-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 82</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">append</span>(<span class="ruby-identifier">path</span>, <span class="ruby-operator">&amp;</span><span class="ruby-identifier">block</span>)
+      <span class="ruby-identifier">file_handle</span> <span class="ruby-identifier">:append</span>, <span class="ruby-identifier">path</span>, <span class="ruby-operator">&amp;</span><span class="ruby-identifier">block</span>
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000187" class="method-detail">
+        <a name="M000187"></a>
+
+        <div class="method-heading">
+          <a href="#M000187" class="method-signature">
+          <span class="method-name">chmod</span><span class="method-args">(path, mode)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000187-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000187-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 113</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">chmod</span>(<span class="ruby-identifier">path</span>, <span class="ruby-identifier">mode</span>)
+      <span class="ruby-ivar">@client</span>.<span class="ruby-identifier">chmod</span> <span class="ruby-identifier">pname</span>(<span class="ruby-identifier">path</span>), <span class="ruby-identifier">mode</span>
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000188" class="method-detail">
+        <a name="M000188"></a>
+
+        <div class="method-heading">
+          <a href="#M000188" class="method-signature">
+          <span class="method-name">chown</span><span class="method-args">(path, owner, group)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000188-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000188-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 117</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">chown</span>(<span class="ruby-identifier">path</span>, <span class="ruby-identifier">owner</span>, <span class="ruby-identifier">group</span>)
+      <span class="ruby-ivar">@client</span>.<span class="ruby-identifier">chown</span> <span class="ruby-identifier">pname</span>(<span class="ruby-identifier">path</span>), <span class="ruby-identifier">owner</span>, <span class="ruby-identifier">group</span>
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000172" class="method-detail">
+        <a name="M000172"></a>
+
+        <div class="method-heading">
+          <a href="#M000172" class="method-signature">
+          <span class="method-name">close</span><span class="method-args">()</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000172-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000172-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 13</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">close</span>
+      <span class="ruby-ivar">@transport</span>.<span class="ruby-identifier">close</span>
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000178" class="method-detail">
+        <a name="M000178"></a>
+
+        <div class="method-heading">
+          <a href="#M000178" class="method-signature">
+          <span class="method-name">create</span><span class="method-args">(path, &amp;block)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p>
+for writing to a <a href="Client.html#M000171">new</a> file
+</p>
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000178-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000178-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 72</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">create</span>(<span class="ruby-identifier">path</span>, <span class="ruby-operator">&amp;</span><span class="ruby-identifier">block</span>)
+      <span class="ruby-identifier">file_handle</span> <span class="ruby-identifier">:create</span>, <span class="ruby-identifier">path</span>, <span class="ruby-operator">&amp;</span><span class="ruby-identifier">block</span>
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000184" class="method-detail">
+        <a name="M000184"></a>
+
+        <div class="method-heading">
+          <a href="#M000184" class="method-signature">
+          <span class="method-name">exists?</span><span class="method-args">(path)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000184-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000184-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 98</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">exists?</span>(<span class="ruby-identifier">path</span>)
+      <span class="ruby-ivar">@client</span>.<span class="ruby-identifier">exists</span> <span class="ruby-identifier">pname</span>(<span class="ruby-identifier">path</span>)
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000175" class="method-detail">
+        <a name="M000175"></a>
+
+        <div class="method-heading">
+          <a href="#M000175" class="method-signature">
+          <span class="method-name">get</span><span class="method-args">(remotepath, destpath)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p>
+copy remote file to local
+</p>
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000175-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000175-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 35</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">get</span>(<span class="ruby-identifier">remotepath</span>, <span class="ruby-identifier">destpath</span>)
+      <span class="ruby-constant">Kernel</span>.<span class="ruby-identifier">open</span>(<span class="ruby-identifier">destpath</span>, <span class="ruby-value str">'w'</span>) { <span class="ruby-operator">|</span><span class="ruby-identifier">dest</span><span class="ruby-operator">|</span>
+        <span class="ruby-identifier">readchunks</span>(<span class="ruby-identifier">remotepath</span>) { <span class="ruby-operator">|</span><span class="ruby-identifier">chunk</span><span class="ruby-operator">|</span>
+          <span class="ruby-identifier">dest</span>.<span class="ruby-identifier">write</span> <span class="ruby-identifier">chunk</span>
+        }
+      }
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000186" class="method-detail">
+        <a name="M000186"></a>
+
+        <div class="method-heading">
+          <a href="#M000186" class="method-signature">
+          <span class="method-name">ls</span><span class="method-args">(path, details=false, recursive=false)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000186-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000186-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 106</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">ls</span>(<span class="ruby-identifier">path</span>, <span class="ruby-identifier">details</span>=<span class="ruby-keyword kw">false</span>, <span class="ruby-identifier">recursive</span>=<span class="ruby-keyword kw">false</span>)
+      <span class="ruby-identifier">statuses</span> = <span class="ruby-ivar">@client</span>.<span class="ruby-identifier">listStatus</span> <span class="ruby-identifier">pname</span>(<span class="ruby-identifier">path</span>)
+      <span class="ruby-identifier">paths</span> = (<span class="ruby-identifier">details</span>) <span class="ruby-operator">?</span> <span class="ruby-identifier">statuses</span> <span class="ruby-operator">:</span> <span class="ruby-identifier">statuses</span>.<span class="ruby-identifier">map</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">s</span><span class="ruby-operator">|</span> <span class="ruby-identifier">s</span>.<span class="ruby-identifier">path</span> }
+      <span class="ruby-keyword kw">return</span> <span class="ruby-identifier">paths</span> <span class="ruby-keyword kw">if</span> <span class="ruby-keyword kw">not</span> <span class="ruby-identifier">recursive</span>
+      <span class="ruby-identifier">paths</span> <span class="ruby-operator">+</span> <span class="ruby-identifier">statuses</span>.<span class="ruby-identifier">select</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">s</span><span class="ruby-operator">|</span> <span class="ruby-identifier">s</span>.<span class="ruby-identifier">isdir</span> }.<span class="ruby-identifier">map</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">s</span><span class="ruby-operator">|</span> <span class="ruby-identifier">ls</span>(<span class="ruby-identifier">s</span>.<span class="ruby-identifier">path</span>, <span class="ruby-identifier">details</span>, <span class="ruby-identifier">recursive</span>) }.<span class="ruby-identifier">flatten</span>
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000183" class="method-detail">
+        <a name="M000183"></a>
+
+        <div class="method-heading">
+          <a href="#M000183" class="method-signature">
+          <span class="method-name">mkdir</span><span class="method-args">(path)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000183-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000183-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 94</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">mkdir</span>(<span class="ruby-identifier">path</span>)
+      <span class="ruby-ivar">@client</span>.<span class="ruby-identifier">mkdirs</span> <span class="ruby-identifier">pname</span>(<span class="ruby-identifier">path</span>)
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000182" class="method-detail">
+        <a name="M000182"></a>
+
+        <div class="method-heading">
+          <a href="#M000182" class="method-signature">
+          <span class="method-name">mv</span><span class="method-args">(source, dest)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000182-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000182-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 90</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">mv</span>(<span class="ruby-identifier">source</span>, <span class="ruby-identifier">dest</span>)
+      <span class="ruby-ivar">@client</span>.<span class="ruby-identifier">rename</span> <span class="ruby-identifier">pname</span>(<span class="ruby-identifier">source</span>), <span class="ruby-identifier">pname</span>(<span class="ruby-identifier">dest</span>)
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000179" class="method-detail">
+        <a name="M000179"></a>
+
+        <div class="method-heading">
+          <a href="#M000179" class="method-signature">
+          <span class="method-name">open</span><span class="method-args">(path, &amp;block)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p>
+for reading
+</p>
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000179-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000179-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 77</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">open</span>(<span class="ruby-identifier">path</span>, <span class="ruby-operator">&amp;</span><span class="ruby-identifier">block</span>)
+      <span class="ruby-identifier">file_handle</span> <span class="ruby-identifier">:open</span>, <span class="ruby-identifier">path</span>, <span class="ruby-operator">&amp;</span><span class="ruby-identifier">block</span>
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000174" class="method-detail">
+        <a name="M000174"></a>
+
+        <div class="method-heading">
+          <a href="#M000174" class="method-signature">
+          <span class="method-name">put</span><span class="method-args">(localpath, destpath)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p>
+copy local file to remote
+</p>
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000174-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000174-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 23</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">put</span>(<span class="ruby-identifier">localpath</span>, <span class="ruby-identifier">destpath</span>)
+      <span class="ruby-identifier">create</span>(<span class="ruby-identifier">destpath</span>) { <span class="ruby-operator">|</span><span class="ruby-identifier">dest</span><span class="ruby-operator">|</span>
+        <span class="ruby-constant">Kernel</span>.<span class="ruby-identifier">open</span>(<span class="ruby-identifier">localpath</span>) { <span class="ruby-operator">|</span><span class="ruby-identifier">source</span><span class="ruby-operator">|</span>
+          <span class="ruby-comment cmt"># read 1 MB at a time</span>
+          <span class="ruby-keyword kw">while</span> <span class="ruby-identifier">record</span> = <span class="ruby-identifier">source</span>.<span class="ruby-identifier">read</span>(<span class="ruby-value">1048576</span>)
+            <span class="ruby-identifier">dest</span>.<span class="ruby-identifier">write</span>(<span class="ruby-identifier">record</span>)
+          <span class="ruby-keyword kw">end</span>
+        }
+      }
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000176" class="method-detail">
+        <a name="M000176"></a>
+
+        <div class="method-heading">
+          <a href="#M000176" class="method-signature">
+          <span class="method-name">readchunks</span><span class="method-args">(path, chunksize=1048576) {|source.read(index, chunksize)| ...}</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p>
+yeild chunksize of path one chunk at a time
+</p>
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000176-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000176-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 44</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">readchunks</span>(<span class="ruby-identifier">path</span>, <span class="ruby-identifier">chunksize</span>=<span class="ruby-value">1048576</span>)
+      <span class="ruby-identifier">open</span>(<span class="ruby-identifier">path</span>) { <span class="ruby-operator">|</span><span class="ruby-identifier">source</span><span class="ruby-operator">|</span>
+        <span class="ruby-identifier">size</span> = <span class="ruby-identifier">source</span>.<span class="ruby-identifier">length</span>
+        <span class="ruby-identifier">index</span> = <span class="ruby-value">0</span>
+        <span class="ruby-keyword kw">while</span> <span class="ruby-identifier">index</span> <span class="ruby-operator">&lt;</span> <span class="ruby-identifier">size</span>
+          <span class="ruby-keyword kw">yield</span> <span class="ruby-identifier">source</span>.<span class="ruby-identifier">read</span>(<span class="ruby-identifier">index</span>, <span class="ruby-identifier">chunksize</span>)
+          <span class="ruby-identifier">index</span> <span class="ruby-operator">+=</span> <span class="ruby-identifier">chunksize</span>
+        <span class="ruby-keyword kw">end</span>
+      }
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000177" class="method-detail">
+        <a name="M000177"></a>
+
+        <div class="method-heading">
+          <a href="#M000177" class="method-signature">
+          <span class="method-name">readlines</span><span class="method-args">(path, sep=&quot;\n&quot;) {|lastbit if lastbit != &quot;&quot;| ...}</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000177-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000177-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 55</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">readlines</span>(<span class="ruby-identifier">path</span>, <span class="ruby-identifier">sep</span>=<span class="ruby-value str">&quot;\n&quot;</span>)
+      <span class="ruby-identifier">lastbit</span> = <span class="ruby-value str">&quot;&quot;</span>
+      <span class="ruby-identifier">readchunks</span>(<span class="ruby-identifier">path</span>) { <span class="ruby-operator">|</span><span class="ruby-identifier">chunk</span><span class="ruby-operator">|</span>
+        <span class="ruby-identifier">parts</span> = <span class="ruby-identifier">chunk</span>.<span class="ruby-identifier">split</span>(<span class="ruby-identifier">sep</span>)
+        <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">parts</span>.<span class="ruby-identifier">length</span> <span class="ruby-operator">==</span> <span class="ruby-value">0</span>
+          <span class="ruby-keyword kw">yield</span> <span class="ruby-identifier">lastbit</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">lastbit</span> <span class="ruby-operator">!=</span> <span class="ruby-value str">&quot;&quot;</span>
+        <span class="ruby-keyword kw">elsif</span> <span class="ruby-identifier">parts</span>.<span class="ruby-identifier">length</span> <span class="ruby-operator">==</span> <span class="ruby-value">1</span>
+          <span class="ruby-keyword kw">yield</span> <span class="ruby-identifier">lastbit</span> <span class="ruby-operator">+</span> <span class="ruby-identifier">parts</span>.<span class="ruby-identifier">first</span>
+        <span class="ruby-keyword kw">else</span>
+          <span class="ruby-keyword kw">yield</span> <span class="ruby-identifier">lastbit</span> <span class="ruby-operator">+</span> <span class="ruby-identifier">parts</span>.<span class="ruby-identifier">first</span>
+          <span class="ruby-identifier">parts</span>.<span class="ruby-identifier">slice</span>(<span class="ruby-value">1</span>, <span class="ruby-identifier">parts</span>.<span class="ruby-identifier">length</span>).<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">p</span><span class="ruby-operator">|</span> <span class="ruby-keyword kw">yield</span> <span class="ruby-identifier">p</span> }          
+        <span class="ruby-keyword kw">end</span>
+        <span class="ruby-identifier">lastbit</span> = <span class="ruby-value str">&quot;&quot;</span>
+      }
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000181" class="method-detail">
+        <a name="M000181"></a>
+
+        <div class="method-heading">
+          <a href="#M000181" class="method-signature">
+          <span class="method-name">rm</span><span class="method-args">(path, recursive=false)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000181-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000181-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 86</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">rm</span>(<span class="ruby-identifier">path</span>, <span class="ruby-identifier">recursive</span>=<span class="ruby-keyword kw">false</span>)
+      <span class="ruby-ivar">@client</span>.<span class="ruby-identifier">rm</span> <span class="ruby-identifier">pname</span>(<span class="ruby-identifier">path</span>), <span class="ruby-identifier">recursive</span>
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000189" class="method-detail">
+        <a name="M000189"></a>
+
+        <div class="method-heading">
+          <a href="#M000189" class="method-signature">
+          <span class="method-name">set_replication</span><span class="method-args">(path, level)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000189-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000189-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 121</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">set_replication</span>(<span class="ruby-identifier">path</span>, <span class="ruby-identifier">level</span>)
+      <span class="ruby-ivar">@client</span>.<span class="ruby-identifier">setReplication</span> <span class="ruby-identifier">pname</span>(<span class="ruby-identifier">path</span>), <span class="ruby-identifier">level</span>
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000173" class="method-detail">
+        <a name="M000173"></a>
+
+        <div class="method-heading">
+          <a href="#M000173" class="method-signature">
+          <span class="method-name">shutdown</span><span class="method-args">(status=0)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p>
+<a href="Client.html#M000173">shutdown</a> the thrift server
+</p>
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000173-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000173-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 18</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">shutdown</span>(<span class="ruby-identifier">status</span>=<span class="ruby-value">0</span>)
+      <span class="ruby-ivar">@client</span>.<span class="ruby-identifier">shutdown</span> <span class="ruby-identifier">status</span>
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+      <div id="method-M000185" class="method-detail">
+        <a name="M000185"></a>
+
+        <div class="method-heading">
+          <a href="#M000185" class="method-signature">
+          <span class="method-name">stat</span><span class="method-args">(path)</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000185-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000185-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/ganapati/client.rb, line 102</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">stat</span>(<span class="ruby-identifier">path</span>)
+      <span class="ruby-ivar">@client</span>.<span class="ruby-identifier">stat</span> <span class="ruby-identifier">pname</span>(<span class="ruby-identifier">path</span>)
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
+        </div>
+      </div>
+
+
+    </div>
+
+
+  </div>
+
+
+<div id="validator-badges">
+  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
+</div>
+
+</body>
+</html>