mandy 0.5.17 → 0.5.19

data/VERSION

@@ -1 +1 @@
-0.5.17
+0.5.19

data/bin/mandy

@@ -15,15 +15,22 @@ puts "Available Mandy Commands"
 puts '------------------------'
 
 {
+  'mandy-cat' => 'Concatenate and print files from HDFS',
+  'mandy-cp' => 'Copy file from HDFS',
+  'mandy-exists' => 'Check to see if a file exists in HDFS',
+  'mandy-get' => 'Copy and merge files to the local file system. The allowed formats are zip and TextRecordInputStream',
+  'mandy-hadoop' => 'Run a Map/Reduce task on hadoop using the provided cluster config',
+  'mandy-hdfs' => 'Execute generic Hadoop FS command on HDFS',
   'mandy-install' => 'Installs the Mandy Rubygem on several hosts via ssh.',
+  'mandy-kill' => 'Kill a specific Hadoop job',
   'mandy-local' => 'Run a Map/Reduce task locally without requiring hadoop',
-  'mandy-hadoop' => 'Run a Map/Reduce task on hadoop using the provided cluster config',
-  'mandy-rm' => 'remove a file or directory from HDFS',
-  'mandy-put' => 'upload a file into HDFS',
+  'mandy-lsr' => 'Recursively list files or directorys on HDFS',
   'mandy-map' => 'Run a map task reading on STDIN and writing to STDOUT',
+  'mandy-mkdir' => 'Takes path uri\'s as argument and creates directories',
+  'mandy-mv' => 'Moves files from source to destination in HDFS',
+  'mandy-put' => 'Upload a file into HDFS',
   'mandy-reduce' => 'Run a reduce task reading on STDIN and writing to STDOUT',
-  'mandy-lsr' => 'Recursively list files or directorys on HDFS'
-  
+  'mandy-rm' => 'Remove a file or directory from HDFS'
 }.each do |command, description|
 
   puts "#{command.ljust(15)} #{description}"

data/bin/mandy-get

@@ -33,4 +33,24 @@ remote_file = ARGV[0]
 local_file = ARGV[1]
 config = absolute_path(options.config || 'cluster.xml')
 
-`$HADOOP_HOME/bin/hadoop fs -conf #{config} -getmerge #{remote_file} #{local_file}`
+def file?(permissions)
+  permissions.chars.first.downcase != 'd'
+end
+
+def is_a_valid(path)
+  path =~ /\/(\S+)/
+end
+
+File.open("#{local_file}", "a") do |target|
+  IO.popen("$HADOOP_HOME/bin/hadoop fs -conf #{config} -ls #{remote_file}").each_line do |line|
+    permissions,replication,user,group,size,mod_date,mod_time,path = *line.split(/\s+/)
+    next unless is_a_valid(path)
+    if file?(permissions)
+      IO.popen("$HADOOP_HOME/bin/hadoop fs -conf #{config} -text #{path}") do |b|
+        while data = b.read(1024*50)
+          target.puts(data)
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,12 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: mandy
 version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 0
-  - 5
-  - 17
-  version: 0.5.17
+  prerelease: 
+  version: 0.5.19
 platform: ruby
 authors: 
 - Andy Kent
@@ -15,8 +11,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-20 00:00:00 +01:00
-default_executable: 
+date: 2010-08-20 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: json
@@ -26,8 +21,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 0
         version: "0"
   type: :runtime
   version_requirements: *id001
@@ -107,7 +100,6 @@ files:
 - lib/mandy/ruby-hbase/xml_decoder.rb
 - lib/mandy/test_runner.rb
 - bin/mandy-hdfs
-has_rdoc: true
 homepage: http://github.com/trafficbroker/mandy
 licenses: []
 
@@ -121,21 +113,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      segments: 
-      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.8.3
 signing_key: 
 specification_version: 2
 summary: Map/Reduce Framework