mandy 0.5.17 → 0.5.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/bin/mandy +12 -5
- data/bin/mandy-get +21 -1
- metadata +4 -16
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.5.
|
1
|
+
0.5.19
|
data/bin/mandy
CHANGED
@@ -15,15 +15,22 @@ puts "Available Mandy Commands"
|
|
15
15
|
puts '------------------------'
|
16
16
|
|
17
17
|
{
|
18
|
+
'mandy-cat' => 'Concatenate and print files from HDFS',
|
19
|
+
'mandy-cp' => 'Copy file from HDFS',
|
20
|
+
'mandy-exists' => 'Check to see if a file exists in HDFS',
|
21
|
+
'mandy-get' => 'Copy and merge files to the local file system. The allowed formats are zip and TextRecordInputStream',
|
22
|
+
'mandy-hadoop' => 'Run a Map/Reduce task on hadoop using the provided cluster config',
|
23
|
+
'mandy-hdfs' => 'Execute generic Hadoop FS command on HDFS',
|
18
24
|
'mandy-install' => 'Installs the Mandy Rubygem on several hosts via ssh.',
|
25
|
+
'mandy-kill' => 'Kill a specific Hadoop job',
|
19
26
|
'mandy-local' => 'Run a Map/Reduce task locally without requiring hadoop',
|
20
|
-
'mandy-
|
21
|
-
'mandy-rm' => 'remove a file or directory from HDFS',
|
22
|
-
'mandy-put' => 'upload a file into HDFS',
|
27
|
+
'mandy-lsr' => 'Recursively list files or directorys on HDFS',
|
23
28
|
'mandy-map' => 'Run a map task reading on STDIN and writing to STDOUT',
|
29
|
+
'mandy-mkdir' => 'Takes path uri\'s as argument and creates directories',
|
30
|
+
'mandy-mv' => 'Moves files from source to destination in HDFS',
|
31
|
+
'mandy-put' => 'Upload a file into HDFS',
|
24
32
|
'mandy-reduce' => 'Run a reduce task reading on STDIN and writing to STDOUT',
|
25
|
-
'mandy-
|
26
|
-
|
33
|
+
'mandy-rm' => 'Remove a file or directory from HDFS'
|
27
34
|
}.each do |command, description|
|
28
35
|
|
29
36
|
puts "#{command.ljust(15)} #{description}"
|
data/bin/mandy-get
CHANGED
@@ -33,4 +33,24 @@ remote_file = ARGV[0]
|
|
33
33
|
local_file = ARGV[1]
|
34
34
|
config = absolute_path(options.config || 'cluster.xml')
|
35
35
|
|
36
|
-
|
36
|
+
def file?(permissions)
|
37
|
+
permissions.chars.first.downcase != 'd'
|
38
|
+
end
|
39
|
+
|
40
|
+
def is_a_valid(path)
|
41
|
+
path =~ /\/(\S+)/
|
42
|
+
end
|
43
|
+
|
44
|
+
File.open("#{local_file}", "a") do |target|
|
45
|
+
IO.popen("$HADOOP_HOME/bin/hadoop fs -conf #{config} -ls #{remote_file}").each_line do |line|
|
46
|
+
permissions,replication,user,group,size,mod_date,mod_time,path = *line.split(/\s+/)
|
47
|
+
next unless is_a_valid(path)
|
48
|
+
if file?(permissions)
|
49
|
+
IO.popen("$HADOOP_HOME/bin/hadoop fs -conf #{config} -text #{path}") do |b|
|
50
|
+
while data = b.read(1024*50)
|
51
|
+
target.puts(data)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
metadata
CHANGED
@@ -1,12 +1,8 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mandy
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
prerelease:
|
5
|
-
|
6
|
-
- 0
|
7
|
-
- 5
|
8
|
-
- 17
|
9
|
-
version: 0.5.17
|
4
|
+
prerelease:
|
5
|
+
version: 0.5.19
|
10
6
|
platform: ruby
|
11
7
|
authors:
|
12
8
|
- Andy Kent
|
@@ -15,8 +11,7 @@ autorequire:
|
|
15
11
|
bindir: bin
|
16
12
|
cert_chain: []
|
17
13
|
|
18
|
-
date: 2010-08-20 00:00:00
|
19
|
-
default_executable:
|
14
|
+
date: 2010-08-20 00:00:00 Z
|
20
15
|
dependencies:
|
21
16
|
- !ruby/object:Gem::Dependency
|
22
17
|
name: json
|
@@ -26,8 +21,6 @@ dependencies:
|
|
26
21
|
requirements:
|
27
22
|
- - ">="
|
28
23
|
- !ruby/object:Gem::Version
|
29
|
-
segments:
|
30
|
-
- 0
|
31
24
|
version: "0"
|
32
25
|
type: :runtime
|
33
26
|
version_requirements: *id001
|
@@ -107,7 +100,6 @@ files:
|
|
107
100
|
- lib/mandy/ruby-hbase/xml_decoder.rb
|
108
101
|
- lib/mandy/test_runner.rb
|
109
102
|
- bin/mandy-hdfs
|
110
|
-
has_rdoc: true
|
111
103
|
homepage: http://github.com/trafficbroker/mandy
|
112
104
|
licenses: []
|
113
105
|
|
@@ -121,21 +113,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
121
113
|
requirements:
|
122
114
|
- - ">="
|
123
115
|
- !ruby/object:Gem::Version
|
124
|
-
segments:
|
125
|
-
- 0
|
126
116
|
version: "0"
|
127
117
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
128
118
|
none: false
|
129
119
|
requirements:
|
130
120
|
- - ">="
|
131
121
|
- !ruby/object:Gem::Version
|
132
|
-
segments:
|
133
|
-
- 0
|
134
122
|
version: "0"
|
135
123
|
requirements: []
|
136
124
|
|
137
125
|
rubyforge_project:
|
138
|
-
rubygems_version: 1.3
|
126
|
+
rubygems_version: 1.8.3
|
139
127
|
signing_key:
|
140
128
|
specification_version: 2
|
141
129
|
summary: Map/Reduce Framework
|