mandy 0.5.21 → 0.5.22

data/Rakefile

@@ -1,14 +1,14 @@
 require 'rubygems'
 require "rake"
 require File.expand_path(File.join(File.dirname(__FILE__), 'lib', 'mandy'))
-require 'spec/rake/spectask'
+# require 'spec/rake/spectask'
+# 
+# task :default => :spec
 
-task :default => :spec
-
-Spec::Rake::SpecTask.new(:spec) do |t|
-  t.spec_files = FileList['spec/lib/**/*_spec.rb']
-  t.spec_opts = %w{-f s -c -L mtime}
-end
+# Spec::Rake::SpecTask.new(:spec) do |t|
+#   t.spec_files = FileList['spec/lib/**/*_spec.rb']
+#   t.spec_opts = %w{-f s -c -L mtime}
+# end
 
 namespace :gem do
   task :build do
@@ -17,7 +17,7 @@ namespace :gem do
   end
 
   task :install => :build do
-    `sudo gem install pkg/mandy-*.gem --no-rdoc --no-ri`
+    `gem install pkg/mandy-*.gem --no-rdoc --no-ri`
   end
   
   task :push => :build do

data/VERSION

@@ -1 +1 @@
-0.5.21
+0.5.22

data/bin/mandy

@@ -18,7 +18,7 @@ puts '------------------------'
   'mandy-cat' => 'Concatenate and print files from HDFS',
   'mandy-cp' => 'Copy file from HDFS',
   'mandy-exists' => 'Check to see if a file exists in HDFS',
-  'mandy-get' => 'Copy and merge files to the local file system. The allowed formats are zip and TextRecordInputStream',
+  'mandy-get' => 'Copy and merge files to the local file system. The allowed formats are zip and TextRecordInputStream, use --raw for others',
   'mandy-hadoop' => 'Run a Map/Reduce task on hadoop using the provided cluster config',
   'mandy-hdfs' => 'Execute generic Hadoop FS command on HDFS',
   'mandy-install' => 'Installs the Mandy Rubygem on several hosts via ssh.',

data/bin/mandy-cp

@@ -29,4 +29,5 @@ source = ARGV[0]
 destination = ARGV[1]
 config = options.config || 'cluster.xml'
 
-`$HADOOP_HOME/bin/hadoop fs -conf #{config} -cp #{source} #{destination}`
+`$HADOOP_HOME/bin/hadoop fs -conf #{config} -cp #{source} #{destination}`
+exit($?.exitstatus)

data/bin/mandy-get

@@ -10,7 +10,7 @@ HadoopConfiguration.check_home_and_version
 
 exec('mandy-get -h') unless ARGV.size >= 2
 
-options = OpenStruct.new
+options = OpenStruct.new(:raw => false, :merge => false)
 
 OptionParser.new do |opts|
   opts.banner = "USAGE: mandy-get hdfs_file_location local_file_destination [options]"
@@ -19,6 +19,14 @@ OptionParser.new do |opts|
     options.config = config
   end
   
+  opts.on('-r', "--raw", "Get the raw file without decompression.") do |config|
+    options.raw = true
+  end
+  
+  opts.on('-m', "--merge", "Get a whole folder and merge results.") do |config|
+    options.merge = true
+  end
+  
   opts.on_tail("-h", "--help", "Show this message") do
     puts opts
     exit
@@ -41,6 +49,12 @@ end
 def is_a_valid(path)
   path =~ /\/(\S+)/
 end
+
+if options.raw
+  puts "$HADOOP_HOME/bin/hadoop fs -conf #{config} -get#{options.merge ? 'merge' : ''} #{remote_file} #{local_file}"
+  exit($?.exitstatus)
+end
+
 FileUtils.mkdir_p(File.split(local_file)[0])
 File.open("#{local_file}", "a") do |target|
   IO.popen("$HADOOP_HOME/bin/hadoop fs -conf #{config} -ls #{remote_file}").each_line do |line|

data/bin/mandy-mkdir

@@ -28,4 +28,5 @@ end.parse!
 dir = ARGV[0]
 config = options.config || 'cluster.xml'
 
-`$HADOOP_HOME/bin/hadoop fs -conf #{config} -mkdir #{dir}`
+`$HADOOP_HOME/bin/hadoop fs -conf #{config} -mkdir #{dir}`
+exit($?.exitstatus)

data/bin/mandy-mv

@@ -29,4 +29,5 @@ source = ARGV[0]
 destination = ARGV[1]
 config = options.config || 'cluster.xml'
 
-`$HADOOP_HOME/bin/hadoop fs -conf #{config} -mv #{source} #{destination}`
+`$HADOOP_HOME/bin/hadoop fs -conf #{config} -mv #{source} #{destination}`
+exit($?.exitstatus)

data/bin/mandy-put

@@ -33,4 +33,5 @@ source = absolute_path(ARGV[0])
 dest = ARGV[1]
 config = options.config || 'cluster.xml'
 
-`$HADOOP_HOME/bin/hadoop fs -conf #{config} -copyFromLocal #{source} #{dest}`
+`$HADOOP_HOME/bin/hadoop fs -conf #{config} -copyFromLocal #{source} #{dest}`
+exit($?.exitstatus)

data/bin/mandy-rm

@@ -42,4 +42,5 @@ if !options.force and File.expand_path(file).split("/").size <= 2
   exit
 end
 
-`$HADOOP_HOME/bin/hadoop fs -conf #{config} -rmr #{file}`
+`$HADOOP_HOME/bin/hadoop fs -conf #{config} -rmr #{file}`
+exit($?.exitstatus)

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mandy
 version: !ruby/object:Gem::Version 
-  hash: 33
+  hash: 39
   prerelease: 
   segments: 
   - 0
   - 5
-  - 21
-  version: 0.5.21
+  - 22
+  version: 0.5.22
 platform: ruby
 authors: 
 - Andy Kent
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-25 00:00:00 +01:00
+date: 2011-06-02 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -110,7 +110,7 @@ files:
 - lib/mandy/test_runner.rb
 - bin/mandy-hdfs
 has_rdoc: true
-homepage: http://github.com/trafficbroker/mandy
+homepage: http://github.com/forward/mandy
 licenses: []
 
 post_install_message: 
@@ -139,7 +139,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.4.2
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 2
 summary: Map/Reduce Framework