mandy 0.5.17 → 0.5.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. data/VERSION +1 -1
  2. data/bin/mandy +12 -5
  3. data/bin/mandy-get +21 -1
  4. metadata +4 -16
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.5.17
1
+ 0.5.19
data/bin/mandy CHANGED
@@ -15,15 +15,22 @@ puts "Available Mandy Commands"
15
15
  puts '------------------------'
16
16
 
17
17
  {
18
+ 'mandy-cat' => 'Concatenate and print files from HDFS',
19
+ 'mandy-cp' => 'Copy file from HDFS',
20
+ 'mandy-exists' => 'Check to see if a file exists in HDFS',
21
+ 'mandy-get' => 'Copy and merge files to the local file system. The allowed formats are zip and TextRecordInputStream',
22
+ 'mandy-hadoop' => 'Run a Map/Reduce task on hadoop using the provided cluster config',
23
+ 'mandy-hdfs' => 'Execute generic Hadoop FS command on HDFS',
18
24
  'mandy-install' => 'Installs the Mandy Rubygem on several hosts via ssh.',
25
+ 'mandy-kill' => 'Kill a specific Hadoop job',
19
26
  'mandy-local' => 'Run a Map/Reduce task locally without requiring hadoop',
20
- 'mandy-hadoop' => 'Run a Map/Reduce task on hadoop using the provided cluster config',
21
- 'mandy-rm' => 'remove a file or directory from HDFS',
22
- 'mandy-put' => 'upload a file into HDFS',
27
+ 'mandy-lsr' => 'Recursively list files or directorys on HDFS',
23
28
  'mandy-map' => 'Run a map task reading on STDIN and writing to STDOUT',
29
+ 'mandy-mkdir' => 'Takes path uri\'s as argument and creates directories',
30
+ 'mandy-mv' => 'Moves files from source to destination in HDFS',
31
+ 'mandy-put' => 'Upload a file into HDFS',
24
32
  'mandy-reduce' => 'Run a reduce task reading on STDIN and writing to STDOUT',
25
- 'mandy-lsr' => 'Recursively list files or directorys on HDFS'
26
-
33
+ 'mandy-rm' => 'Remove a file or directory from HDFS'
27
34
  }.each do |command, description|
28
35
 
29
36
  puts "#{command.ljust(15)} #{description}"
@@ -33,4 +33,24 @@ remote_file = ARGV[0]
33
33
  local_file = ARGV[1]
34
34
  config = absolute_path(options.config || 'cluster.xml')
35
35
 
36
- `$HADOOP_HOME/bin/hadoop fs -conf #{config} -getmerge #{remote_file} #{local_file}`
36
+ def file?(permissions)
37
+ permissions.chars.first.downcase != 'd'
38
+ end
39
+
40
+ def is_a_valid(path)
41
+ path =~ /\/(\S+)/
42
+ end
43
+
44
+ File.open("#{local_file}", "a") do |target|
45
+ IO.popen("$HADOOP_HOME/bin/hadoop fs -conf #{config} -ls #{remote_file}").each_line do |line|
46
+ permissions,replication,user,group,size,mod_date,mod_time,path = *line.split(/\s+/)
47
+ next unless is_a_valid(path)
48
+ if file?(permissions)
49
+ IO.popen("$HADOOP_HOME/bin/hadoop fs -conf #{config} -text #{path}") do |b|
50
+ while data = b.read(1024*50)
51
+ target.puts(data)
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
metadata CHANGED
@@ -1,12 +1,8 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mandy
3
3
  version: !ruby/object:Gem::Version
4
- prerelease: false
5
- segments:
6
- - 0
7
- - 5
8
- - 17
9
- version: 0.5.17
4
+ prerelease:
5
+ version: 0.5.19
10
6
  platform: ruby
11
7
  authors:
12
8
  - Andy Kent
@@ -15,8 +11,7 @@ autorequire:
15
11
  bindir: bin
16
12
  cert_chain: []
17
13
 
18
- date: 2010-08-20 00:00:00 +01:00
19
- default_executable:
14
+ date: 2010-08-20 00:00:00 Z
20
15
  dependencies:
21
16
  - !ruby/object:Gem::Dependency
22
17
  name: json
@@ -26,8 +21,6 @@ dependencies:
26
21
  requirements:
27
22
  - - ">="
28
23
  - !ruby/object:Gem::Version
29
- segments:
30
- - 0
31
24
  version: "0"
32
25
  type: :runtime
33
26
  version_requirements: *id001
@@ -107,7 +100,6 @@ files:
107
100
  - lib/mandy/ruby-hbase/xml_decoder.rb
108
101
  - lib/mandy/test_runner.rb
109
102
  - bin/mandy-hdfs
110
- has_rdoc: true
111
103
  homepage: http://github.com/trafficbroker/mandy
112
104
  licenses: []
113
105
 
@@ -121,21 +113,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
121
113
  requirements:
122
114
  - - ">="
123
115
  - !ruby/object:Gem::Version
124
- segments:
125
- - 0
126
116
  version: "0"
127
117
  required_rubygems_version: !ruby/object:Gem::Requirement
128
118
  none: false
129
119
  requirements:
130
120
  - - ">="
131
121
  - !ruby/object:Gem::Version
132
- segments:
133
- - 0
134
122
  version: "0"
135
123
  requirements: []
136
124
 
137
125
  rubyforge_project:
138
- rubygems_version: 1.3.7
126
+ rubygems_version: 1.8.3
139
127
  signing_key:
140
128
  specification_version: 2
141
129
  summary: Map/Reduce Framework