pmux 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/pmux/application.rb +62 -3
- data/lib/pmux/cleaner.rb +5 -5
- data/lib/pmux/handler.rb +15 -0
- data/lib/pmux/job.rb +10 -3
- data/lib/pmux/mapper.rb +3 -2
- data/lib/pmux/multi_session.rb +21 -8
- data/lib/pmux/reducer.rb +2 -1
- data/lib/pmux/storage_adapter.rb +44 -0
- data/lib/pmux/task_dispatcher.rb +27 -19
- data/lib/pmux/version.rb +1 -1
- data/test/log/1234567890.yml +57 -0
- data/test/test_application.rb +15 -0
- data/test/test_i_mapreduce.rb +3 -2
- metadata +19 -7
data/lib/pmux/application.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'optparse'
|
2
|
+
require 'yaml'
|
2
3
|
|
3
4
|
module Pmux
|
4
5
|
class Application
|
@@ -7,6 +8,7 @@ module Pmux
|
|
7
8
|
def run options=OPTS
|
8
9
|
optparser = optparse options
|
9
10
|
optparser.parse!
|
11
|
+
load_config options
|
10
12
|
options[:program_name] = optparser.program_name
|
11
13
|
options[:user] ||=
|
12
14
|
(ENV['USER'] || ENV['LOGNAME'] || Etc.getlogin || Etc.getpwuid.name)
|
@@ -30,11 +32,25 @@ module Pmux
|
|
30
32
|
show_status addrs, options
|
31
33
|
when options[:lookup]
|
32
34
|
lookup addrs, options
|
35
|
+
when options[:show_joblog]
|
36
|
+
show_joblog options
|
33
37
|
else
|
34
38
|
run_mr addrs, options
|
35
39
|
end
|
36
40
|
end
|
37
41
|
|
42
|
+
def load_config options
|
43
|
+
path = File.expand_path(options[:config_file] || "~/.pmux/config.yml")
|
44
|
+
if File.file? path
|
45
|
+
conf = YAML.load_file path
|
46
|
+
if conf.kind_of? Hash
|
47
|
+
for k, v in conf
|
48
|
+
options[k.intern] ||= v
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
38
54
|
def getaddr host
|
39
55
|
sa = Socket.pack_sockaddr_in 0, host
|
40
56
|
port, addr = Socket.unpack_sockaddr_in sa
|
@@ -55,6 +71,45 @@ module Pmux
|
|
55
71
|
File.unlink options[:sock_path] rescue nil
|
56
72
|
end
|
57
73
|
|
74
|
+
alias :putline :puts
|
75
|
+
|
76
|
+
def show_joblog options
|
77
|
+
log_dir = options[:log_dir]
|
78
|
+
if (job_id = options[:show_joblog]) == true
|
79
|
+
joblogs = Dir.glob(log_dir + '/*.yml').map {|path|
|
80
|
+
obj = YAML.load_file path
|
81
|
+
}
|
82
|
+
for h in joblogs.sort_by {|obj| obj[:invoked_at].to_f}
|
83
|
+
t = (h[:invoked_at] || h[:start_time]).strftime("%m/%d %H:%M")
|
84
|
+
line = format '%-10s %s "%s"', h[:id], t, h[:mapper]
|
85
|
+
putline line
|
86
|
+
end
|
87
|
+
else
|
88
|
+
path = File.join log_dir, "#{job_id}.yml"
|
89
|
+
els = {}
|
90
|
+
open(path) {|io|
|
91
|
+
s = YAML.load_stream io
|
92
|
+
header, tasks, footer = s[0], s[1], s[2]
|
93
|
+
if tasks
|
94
|
+
for task_id, task in tasks.sort_by {|k, v| k}
|
95
|
+
line = format '%5d %s %g',
|
96
|
+
task_id, task['node_addr'], task['welapse']
|
97
|
+
putline line
|
98
|
+
node_addr = task['node_addr']
|
99
|
+
els[node_addr] ||= [0, 0]
|
100
|
+
els[node_addr][0] += 1
|
101
|
+
els[node_addr][1] += task['welapse']
|
102
|
+
end
|
103
|
+
end
|
104
|
+
}
|
105
|
+
putline
|
106
|
+
for node_addr, v in els
|
107
|
+
putline format '%s %g/%d = %g',
|
108
|
+
node_addr, v[1], v[0], v[1] / v[0]
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
58
113
|
def show_status addrs, options
|
59
114
|
addrs = ['localhost'] if addrs.empty?
|
60
115
|
adapter = StorageAdapter.create options[:storage_name], addrs
|
@@ -93,7 +148,7 @@ module Pmux
|
|
93
148
|
end
|
94
149
|
|
95
150
|
def run_mr addrs, options, argv=ARGV
|
96
|
-
|
151
|
+
invoked_at = Time.now
|
97
152
|
if options[:storage_name] == 'local' and addrs.empty?
|
98
153
|
addrs = ['localhost']
|
99
154
|
end
|
@@ -101,11 +156,10 @@ module Pmux
|
|
101
156
|
locator_host = options[:locator_host] || addrs.first || 'localhost'
|
102
157
|
locator_port = options[:locator_port]
|
103
158
|
|
104
|
-
puts "storage: #{
|
159
|
+
puts "storage: #{adapter.class}" if options[:verbose]
|
105
160
|
begin
|
106
161
|
adapter.connect_to_storage locator_host, locator_port
|
107
162
|
files = adapter.get_files argv, options[:expand_glob]
|
108
|
-
options[:__get_files_time] = Time.now
|
109
163
|
rescue
|
110
164
|
STDERR.puts "Storage Error: #{$!}"
|
111
165
|
return
|
@@ -123,8 +177,10 @@ module Pmux
|
|
123
177
|
end
|
124
178
|
dispatcher = TaskDispatcher.new options, adapter, msession
|
125
179
|
job = Job.new options, files
|
180
|
+
job[:invoked_at] = invoked_at
|
126
181
|
job.mk_reducer_addrs adapter.addrs
|
127
182
|
dispatcher.run job
|
183
|
+
abort if job.failed
|
128
184
|
end
|
129
185
|
|
130
186
|
def optparse opts
|
@@ -132,6 +188,8 @@ module Pmux
|
|
132
188
|
op.on('--debug') {$debug = true; STDOUT.sync = true}
|
133
189
|
op.on('--server') {opts[:server] = true}
|
134
190
|
op.on('--argv=FILES') {}
|
191
|
+
op.on('--brick=HOST:/DIR', '-b') {|arg| (opts[:bricks] ||= []).push arg}
|
192
|
+
op.on('--config-file=FILE', '-F') {|arg| opts[:config_file] = arg}
|
135
193
|
op.on('--disable-plugins') {opts[:disable_plugins] = true}
|
136
194
|
op.on('--expand-glob') {opts[:expand_glob] = true}
|
137
195
|
op.on('--ff=FF', Integer) {|arg| opts[:ff] = arg}
|
@@ -147,6 +205,7 @@ module Pmux
|
|
147
205
|
op.on('--root-dir=DIR') {|arg| opts[:root_dir] = arg}
|
148
206
|
op.on('--ship-file=FILE', '--file=FILE') {|arg|
|
149
207
|
(opts[:ship_files] ||= []).push arg}
|
208
|
+
op.on('--show-joblog [job_id]') {|arg| opts[:show_joblog] = arg || true}
|
150
209
|
op.on('--status') {opts[:status] = true}
|
151
210
|
op.on('--storage=STORAGE_NAME') {|arg|
|
152
211
|
opts[:storage_name] = arg}
|
data/lib/pmux/cleaner.rb
CHANGED
@@ -2,18 +2,18 @@ require 'fileutils'
|
|
2
2
|
|
3
3
|
module Pmux
|
4
4
|
class Cleaner
|
5
|
-
def initialize
|
6
|
-
@
|
5
|
+
def initialize *glob_pats
|
6
|
+
@glob_pats = glob_pats
|
7
7
|
end
|
8
8
|
|
9
9
|
def run lim_time=nil
|
10
|
-
fork {fork {clean
|
10
|
+
fork {fork {@glob_pats.each {|glob_pat| clean glob_pat, lim_time}}}
|
11
11
|
Process.wait
|
12
12
|
end
|
13
13
|
|
14
14
|
def clean glob_pat=nil, lim_time=nil
|
15
|
-
glob_pat ||=
|
16
|
-
lim_time ||= Time.now - 3600*24
|
15
|
+
glob_pat ||= ''
|
16
|
+
lim_time ||= Time.now - 3600*24*7
|
17
17
|
paths = Dir.glob glob_pat
|
18
18
|
for path in paths
|
19
19
|
if File.exist? path
|
data/lib/pmux/handler.rb
CHANGED
@@ -194,6 +194,21 @@ module Pmux
|
|
194
194
|
'hello'
|
195
195
|
end
|
196
196
|
|
197
|
+
def ls dirs, args
|
198
|
+
res = []
|
199
|
+
for dir in dirs
|
200
|
+
for arg in args
|
201
|
+
Dir.chdir(dir) {
|
202
|
+
res += Dir.glob(arg).select {|path|
|
203
|
+
File.readable? path}.map {|path|
|
204
|
+
[path, File.join(dir, path)]
|
205
|
+
}
|
206
|
+
}
|
207
|
+
end
|
208
|
+
end
|
209
|
+
res
|
210
|
+
end
|
211
|
+
|
197
212
|
def quit
|
198
213
|
@server.loop.stop
|
199
214
|
cleaner = Cleaner.new "#{options[:tmp_dir]}/[0-9]*"
|
data/lib/pmux/job.rb
CHANGED
@@ -7,6 +7,7 @@ module Pmux
|
|
7
7
|
attr_reader :tasks, :num_t, :num_r
|
8
8
|
attr_reader :taskhash
|
9
9
|
attr_reader :reducers
|
10
|
+
attr_reader :failed
|
10
11
|
|
11
12
|
def initialize params, files
|
12
13
|
@params = params
|
@@ -20,9 +21,10 @@ module Pmux
|
|
20
21
|
@done_taskhash = {}
|
21
22
|
@tasks = mk_tasks files
|
22
23
|
@num_t = @tasks.size + @num_r
|
23
|
-
@h = {:
|
24
|
+
@h = {:job_started_at=>Time.now,
|
24
25
|
:map_tasks=>@tasks.size, :reduce_tasks=>@num_r,
|
25
26
|
}
|
27
|
+
@failed = false
|
26
28
|
end
|
27
29
|
|
28
30
|
def mk_tasks files
|
@@ -83,10 +85,15 @@ module Pmux
|
|
83
85
|
@taskhash.empty?
|
84
86
|
end
|
85
87
|
|
88
|
+
def set_failed
|
89
|
+
@taskhash.clear
|
90
|
+
@failed = true
|
91
|
+
end
|
92
|
+
|
86
93
|
def to_jlheader
|
87
94
|
h = {:id=>id, :files_first=>@files.first, :tasksize=>@tasks.size,
|
88
95
|
:params=>@params,
|
89
|
-
:
|
96
|
+
:invoked_at=>@h[:invoked_at], :job_started_at=>@h[:job_started_at],
|
90
97
|
:map_tasks=>@h[:map_tasks], :reduce_tasks=>@h[:reduce_tasks],
|
91
98
|
:storage_name=>@params[:storage_name],
|
92
99
|
:mapper=>@params[:mapper], :reducer=>@params[:reducer],
|
@@ -95,7 +102,7 @@ module Pmux
|
|
95
102
|
end
|
96
103
|
|
97
104
|
def to_jlfooter
|
98
|
-
h = {:
|
105
|
+
h = {:job_finished_at=>@h[:job_finished_at]}
|
99
106
|
end
|
100
107
|
end
|
101
108
|
end
|
data/lib/pmux/mapper.rb
CHANGED
@@ -75,18 +75,19 @@ module Pmux
|
|
75
75
|
mapper_cmd = @task['mapper'] || 'cat'
|
76
76
|
err_path = "#{tmp_dir}/.err.#{object_id}"
|
77
77
|
err_msg = nil
|
78
|
+
pipeio = nil
|
78
79
|
if @num_r <= 1
|
79
80
|
cmd_line = fix_cmd_line mapper_cmd,
|
80
81
|
@path, nil, err_path, tmp_dir
|
81
82
|
Log.debug "pipe: #{cmd_line}"
|
82
|
-
pipeio = PipeIO.new cmd_line
|
83
|
+
Dir.chdir(tmp_dir) {pipeio = PipeIO.new cmd_line}
|
83
84
|
out = open("#{@ifbase}-0", 'a')
|
84
85
|
pipeio.on_receive {|data| out.write data}
|
85
86
|
else # @num_r >= 2
|
86
87
|
partitioner = TextPartitioner.new @ifbase, @num_r,
|
87
88
|
:separator=>@task['separator']
|
88
89
|
cmd_line = fix_cmd_line mapper_cmd, @path, nil, err_path, tmp_dir
|
89
|
-
pipeio = PipeIO.new cmd_line
|
90
|
+
Dir.chdir(tmp_dir) {pipeio = PipeIO.new cmd_line}
|
90
91
|
pipeio.on_receive {|data| partitioner.emit data}
|
91
92
|
end
|
92
93
|
on_success = @on_success
|
data/lib/pmux/multi_session.rb
CHANGED
@@ -6,7 +6,7 @@ end
|
|
6
6
|
|
7
7
|
module Pmux
|
8
8
|
class SessionWrapper
|
9
|
-
attr_reader :addr
|
9
|
+
attr_reader :addr, :scp_channels
|
10
10
|
attr_accessor :ssh, :scp_session_count
|
11
11
|
|
12
12
|
def initialize addr
|
@@ -14,6 +14,7 @@ module Pmux
|
|
14
14
|
@ssh = nil
|
15
15
|
@scp = nil
|
16
16
|
@scp_session_count = 0
|
17
|
+
@scp_channels = {}
|
17
18
|
end
|
18
19
|
|
19
20
|
def scp
|
@@ -43,6 +44,7 @@ module Pmux
|
|
43
44
|
@buffers = {}
|
44
45
|
|
45
46
|
@on_error = nil
|
47
|
+
@session_lim = ($test ? 99 : 2)
|
46
48
|
end
|
47
49
|
|
48
50
|
def connect_to_addr addr, cmd=nil
|
@@ -95,6 +97,10 @@ module Pmux
|
|
95
97
|
end
|
96
98
|
|
97
99
|
def error_on_addr addr, err=nil
|
100
|
+
session = @sessions[addr]
|
101
|
+
session.scp_channels.each_value {|ch|
|
102
|
+
ch.on_open_failed.call ch, 0xfe000001, err.to_s
|
103
|
+
}
|
98
104
|
@err_addrs.push addr
|
99
105
|
@addrs.delete addr
|
100
106
|
@on_error.call addr, err if @on_error
|
@@ -116,13 +122,12 @@ module Pmux
|
|
116
122
|
@on_error = block
|
117
123
|
end
|
118
124
|
|
119
|
-
def scp_upload_files addr, files, remote, options={}
|
125
|
+
def scp_upload_files addr, files, remote, options={}
|
120
126
|
mf = MR::MultiFuture.new
|
121
127
|
for file in files
|
122
128
|
future = scp_upload addr, file, remote, options
|
123
129
|
mf.add future
|
124
130
|
end
|
125
|
-
mf.on_all &block if block
|
126
131
|
mf
|
127
132
|
end
|
128
133
|
|
@@ -140,7 +145,7 @@ module Pmux
|
|
140
145
|
|
141
146
|
def scp_upload_sub scp, addr, future, local, remote, options
|
142
147
|
session = @sessions[addr]
|
143
|
-
if !session or session.scp_session_count >
|
148
|
+
if !session or session.scp_session_count > @session_lim
|
144
149
|
queue = (@scp_queue[addr] ||= [])
|
145
150
|
queue.push [:up, future, addr, remote, local, options]
|
146
151
|
return
|
@@ -152,8 +157,10 @@ module Pmux
|
|
152
157
|
@scptable[scpid] = future
|
153
158
|
|
154
159
|
channel = scp.upload(local, remote, options)
|
160
|
+
session.scp_channels[scpid] = channel
|
155
161
|
channel.on_eof {|ch|
|
156
162
|
session.scp_session_count -= 1
|
163
|
+
session.scp_channels.delete scpid
|
157
164
|
@loop.set_timer(0) {process_scp_queue_once addr}
|
158
165
|
|
159
166
|
future.set_result(nil, options[:set_result])
|
@@ -161,12 +168,18 @@ module Pmux
|
|
161
168
|
}
|
162
169
|
channel.on_open_failed {|ch, code, desc|
|
163
170
|
Log.error "#{addr}: scp error: #{desc}"
|
164
|
-
err = RuntimeError.new "scp error: #{desc}"
|
165
|
-
@on_error.call addr, err
|
166
171
|
session.scp_session_count -= 1
|
172
|
+
session.scp_channels.delete scpid
|
167
173
|
@loop.set_timer(0) {process_scp_queue_once addr}
|
168
174
|
|
169
|
-
|
175
|
+
if code == 0xfe000001
|
176
|
+
err = desc
|
177
|
+
else
|
178
|
+
err = RuntimeError.new "scp error: #{desc}"
|
179
|
+
@on_error.call addr, err
|
180
|
+
err = nil
|
181
|
+
end
|
182
|
+
future.set_result(err, options[:set_result])
|
170
183
|
@scptable.delete scpid
|
171
184
|
}
|
172
185
|
end
|
@@ -185,7 +198,7 @@ module Pmux
|
|
185
198
|
|
186
199
|
def scp_download_sub scp, addr, future, remote, local, options
|
187
200
|
session = @sessions[addr]
|
188
|
-
if !session or session.scp_session_count >
|
201
|
+
if !session or session.scp_session_count > @session_lim
|
189
202
|
queue = (@scp_queue[addr] ||= [])
|
190
203
|
queue.push [:down, future, addr, remote, local, options]
|
191
204
|
return
|
data/lib/pmux/reducer.rb
CHANGED
@@ -59,7 +59,8 @@ module Pmux
|
|
59
59
|
cmd_line = fix_cmd_line reducer_cmd,
|
60
60
|
@paths.join(' '), nil, err_path, tmp_dir
|
61
61
|
Log.debug "popen: #{cmd_line}"
|
62
|
-
pipeio =
|
62
|
+
pipeio = nil
|
63
|
+
Dir.chdir(@tmp_dir) {pipeio = PipeIO.new cmd_line}
|
63
64
|
if @on_receive
|
64
65
|
pipeio.on_receive &@on_receive
|
65
66
|
else
|
data/lib/pmux/storage_adapter.rb
CHANGED
@@ -102,4 +102,48 @@ module Pmux
|
|
102
102
|
end
|
103
103
|
end
|
104
104
|
end
|
105
|
+
|
106
|
+
class BBFSAdapter < GlusterFSAdapter
|
107
|
+
Adapters['bbfs'] = self
|
108
|
+
|
109
|
+
def connect_to_storage locator_host, locator_port
|
110
|
+
@addr2dirs = {}
|
111
|
+
for b in @options[:bricks]
|
112
|
+
if b =~ %r{(\S+):(.+)}
|
113
|
+
dir = $2
|
114
|
+
addr = getaddr $1
|
115
|
+
(@addr2dirs[addr] ||= []).push dir
|
116
|
+
end
|
117
|
+
end
|
118
|
+
@addrs = @addr2dirs.keys
|
119
|
+
@h = init_node_hash @addrs
|
120
|
+
end
|
121
|
+
|
122
|
+
def get_files args, glob_flag=false
|
123
|
+
msession = MRSession.new @addrs, @options
|
124
|
+
msession.on_error {|addr, err| $stderr.printf "%s: %s\n", addr, err.to_s}
|
125
|
+
msession.connect
|
126
|
+
@locations = {}
|
127
|
+
mf = MR::MultiFuture.new
|
128
|
+
mf.on_success {|f|
|
129
|
+
addr = f.addr
|
130
|
+
res = f.get
|
131
|
+
for rpath, apath in res
|
132
|
+
(@locations[rpath] ||= []).push [addr, apath]
|
133
|
+
end
|
134
|
+
}
|
135
|
+
for addr in @addrs
|
136
|
+
future = msession.call_async addr, 'ls', @addr2dirs[addr], args
|
137
|
+
mf.add future
|
138
|
+
end
|
139
|
+
mf.join_all
|
140
|
+
@locations.keys
|
141
|
+
end
|
142
|
+
|
143
|
+
def lookup_file file
|
144
|
+
@locations[file]
|
145
|
+
end
|
146
|
+
|
147
|
+
private
|
148
|
+
end
|
105
149
|
end
|
data/lib/pmux/task_dispatcher.rb
CHANGED
@@ -57,42 +57,46 @@ module Pmux
|
|
57
57
|
scheduler.shipped[addr] = true
|
58
58
|
scheduler.process_queue
|
59
59
|
}
|
60
|
+
mf_scp.on_error {job.set_failed}
|
60
61
|
else
|
61
62
|
puts 'start scheduler' if @verbose
|
62
63
|
scheduler.shipped[addr] = true
|
63
64
|
scheduler.process_queue
|
64
65
|
end
|
65
66
|
}
|
66
|
-
mf_init.on_error {job.
|
67
|
+
mf_init.on_error {job.set_failed}
|
67
68
|
|
68
69
|
# wait for all map tasks to finish
|
69
70
|
until job.completed?
|
70
71
|
msession.loop.run_once
|
71
72
|
end
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
73
|
+
unless job.failed
|
74
|
+
if job.num_r.zero?
|
75
|
+
gatherer.join_all
|
76
|
+
else
|
77
|
+
mf_shuffle.join_all
|
78
|
+
# reduce phase
|
79
|
+
job.mk_reduce_tasks
|
80
|
+
scheduler.inject_tasks job.tasks
|
81
|
+
scheduler.process_queue
|
82
|
+
# wait for all reduce tasks to finish
|
83
|
+
until job.completed?
|
84
|
+
msession.loop.run_once
|
85
|
+
end
|
86
|
+
gatherer.join_all
|
83
87
|
end
|
84
|
-
gatherer.join_all
|
85
88
|
end
|
86
89
|
|
87
90
|
Log.info "END"
|
88
|
-
job[:
|
91
|
+
job[:job_finished_at] = Time.now
|
89
92
|
jl.dump_footer
|
90
93
|
jl.close
|
91
94
|
|
92
95
|
mf_quit = msession.multicast_call_async :quit
|
93
96
|
mf_quit.join_all
|
94
97
|
|
95
|
-
cleaner = Cleaner.new "#{options[:tmp_dir]}/w*"
|
98
|
+
cleaner = Cleaner.new "#{options[:tmp_dir]}/w*",
|
99
|
+
"#{options[:log_dir]}/*.yml"
|
96
100
|
cleaner.run
|
97
101
|
end
|
98
102
|
|
@@ -104,12 +108,11 @@ module Pmux
|
|
104
108
|
task_id, node_addr, ifbase =
|
105
109
|
result.values_at 'task_id', 'node_addr', 'ifbase'
|
106
110
|
Log.info "receive result #{task_id} from #{node_addr}"
|
107
|
-
jl.add task_id, :node_addr=>node_addr, :ifbase=>ifbase,
|
108
|
-
:welapse=>result['welapse']
|
109
111
|
puts "receive result #{task_id} from #{node_addr}" if @verbose
|
110
112
|
if result['error']
|
111
113
|
if @on_error
|
112
114
|
@on_error.call result
|
115
|
+
job.set_failed
|
113
116
|
else
|
114
117
|
end
|
115
118
|
end
|
@@ -146,8 +149,13 @@ module Pmux
|
|
146
149
|
end
|
147
150
|
|
148
151
|
task = job.get_task_by_id task_id
|
149
|
-
|
150
|
-
|
152
|
+
alloc_time = task[:alloc_time]
|
153
|
+
allocated_at = alloc_time - job[:job_started_at]
|
154
|
+
elapse = Time.now - alloc_time if alloc_time
|
155
|
+
#task[:welapse] = result['welapse']
|
156
|
+
jl.add task_id, :node_addr=>node_addr, :ifbase=>ifbase,
|
157
|
+
:welapse=>result['welapse'], :elapse=>elapse,
|
158
|
+
:allocated_at=>allocated_at
|
151
159
|
|
152
160
|
# delete task
|
153
161
|
scheduler.delete_task_from_job job, task, node_addr
|
data/lib/pmux/version.rb
CHANGED
@@ -0,0 +1,57 @@
|
|
1
|
+
---
|
2
|
+
:id: '1234567890'
|
3
|
+
:files_first: /dist/apache/access.20001101_0.log.gz
|
4
|
+
:tasksize: 6
|
5
|
+
:params:
|
6
|
+
:disable_plugins: true
|
7
|
+
:ship_files:
|
8
|
+
- tmp/apache_sum.rb
|
9
|
+
:mapper: apache_sum.rb
|
10
|
+
:reducer: apache_sum.rb
|
11
|
+
:program_name: pmux
|
12
|
+
:user: foo
|
13
|
+
:root_dir: /home/foo/.pmux
|
14
|
+
:tmp_dir: /home/foo/.pmux/tmp
|
15
|
+
:log_dir: /home/foo/.pmux/log
|
16
|
+
:__start_time: 2000-11-30 09:42:41.111131000 +09:00
|
17
|
+
:__get_files_time: 2000-11-30 09:42:41.130985000 +09:00
|
18
|
+
:num_r: 1
|
19
|
+
:job_name: ! '"apache_sum.rb"'
|
20
|
+
:start_time: 2000-11-30 09:42:41.158194000 +09:00
|
21
|
+
:map_tasks: 6
|
22
|
+
:reduce_tasks: 1
|
23
|
+
:storage_name:
|
24
|
+
:mapper: apache_sum.rb
|
25
|
+
:reducer: apache_sum.rb
|
26
|
+
:num_r: 1
|
27
|
+
---
|
28
|
+
2:
|
29
|
+
node_addr: "192.168.196.208"
|
30
|
+
ifbase: "/home/foo/.pmux/tmp/105459480/m2"
|
31
|
+
welapse: 0.037278
|
32
|
+
4:
|
33
|
+
node_addr: "192.168.196.208"
|
34
|
+
ifbase: "/home/foo/.pmux/tmp/105459480/m4"
|
35
|
+
welapse: 0.036954
|
36
|
+
1:
|
37
|
+
node_addr: "192.168.196.208"
|
38
|
+
ifbase: "/home/foo/.pmux/tmp/105459480/m1"
|
39
|
+
welapse: 5.009
|
40
|
+
5:
|
41
|
+
node_addr: "192.168.196.224"
|
42
|
+
ifbase: "/home/foo/.pmux/tmp/105459480/m5"
|
43
|
+
welapse: 14.709426
|
44
|
+
6:
|
45
|
+
node_addr: "192.168.196.224"
|
46
|
+
ifbase: "/home/foo/.pmux/tmp/105459480/m6"
|
47
|
+
welapse: 0.03236
|
48
|
+
3:
|
49
|
+
node_addr: "192.168.196.224"
|
50
|
+
ifbase: "/home/foo/.pmux/tmp/105459480/m3"
|
51
|
+
welapse: 18.065011
|
52
|
+
7:
|
53
|
+
node_addr: "192.168.196.208"
|
54
|
+
ifbase: nil
|
55
|
+
welapse: 0.0344
|
56
|
+
---
|
57
|
+
:end_time: 2000-11-30 09:43:00.172319000 +09:00
|
data/test/test_application.rb
CHANGED
@@ -1,6 +1,21 @@
|
|
1
1
|
require File.expand_path(File.dirname(__FILE__) + '/unittest_helper')
|
2
2
|
|
3
3
|
class TestMain < Test::Unit::TestCase
|
4
|
+
def test_show_joblog
|
5
|
+
main = Pmux::Application.new
|
6
|
+
class <<main
|
7
|
+
attr_reader :lines
|
8
|
+
def putline line=''; (@lines||=[]).push line; end
|
9
|
+
end
|
10
|
+
log_dir = File.expand_path(File.dirname(__FILE__) + '/log')
|
11
|
+
|
12
|
+
main.show_joblog :log_dir=>log_dir, :show_joblog=>true
|
13
|
+
ae 1, main.lines.size
|
14
|
+
main.lines.clear
|
15
|
+
main.show_joblog :log_dir=>log_dir, :show_joblog=>'1234567890'
|
16
|
+
ae 10, main.lines.size
|
17
|
+
end
|
18
|
+
|
4
19
|
def test_optparse
|
5
20
|
main = Pmux::Application.new
|
6
21
|
options = {}
|
data/test/test_i_mapreduce.rb
CHANGED
@@ -53,7 +53,7 @@ end
|
|
53
53
|
class Pmux::Cleaner
|
54
54
|
def run lim_time=nil
|
55
55
|
lim_time = Time.now - 3600
|
56
|
-
clean
|
56
|
+
@glob_pats.each {|glob_pat| clean glob_pat, lim_time}
|
57
57
|
end
|
58
58
|
end
|
59
59
|
|
@@ -147,8 +147,9 @@ class TestIMapreduce < Test::Unit::TestCase
|
|
147
147
|
job = Pmux::Job.new options, files
|
148
148
|
job.mk_reducer_addrs addrs
|
149
149
|
dispatcher.run job
|
150
|
+
assert job.failed
|
150
151
|
assert_match %r{/notexist: No such file or directory}, er['error_message']
|
151
|
-
ae
|
152
|
+
ae 0, writer.lines.size
|
152
153
|
end
|
153
154
|
|
154
155
|
# mr test; num_r=0, 4 files, ff=2
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pmux
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2013-01-11 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: msgpack-rpc
|
16
|
-
requirement:
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,15 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements:
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
25
30
|
- !ruby/object:Gem::Dependency
|
26
31
|
name: net-scp
|
27
|
-
requirement:
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
28
33
|
none: false
|
29
34
|
requirements:
|
30
35
|
- - ! '>='
|
@@ -32,7 +37,12 @@ dependencies:
|
|
32
37
|
version: '0'
|
33
38
|
type: :runtime
|
34
39
|
prerelease: false
|
35
|
-
version_requirements:
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
36
46
|
description: lightweight file-based MapReduce system
|
37
47
|
email:
|
38
48
|
executables:
|
@@ -71,6 +81,7 @@ files:
|
|
71
81
|
- lib/pmux/worker.rb
|
72
82
|
- lib/pmux/writer.rb
|
73
83
|
- pmux.gemspec
|
84
|
+
- test/log/1234567890.yml
|
74
85
|
- test/mock_mros.rb
|
75
86
|
- test/mock_pipeio.rb
|
76
87
|
- test/mock_world.rb
|
@@ -119,11 +130,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
119
130
|
version: '0'
|
120
131
|
requirements: []
|
121
132
|
rubyforge_project: pmux
|
122
|
-
rubygems_version: 1.8.
|
133
|
+
rubygems_version: 1.8.24
|
123
134
|
signing_key:
|
124
135
|
specification_version: 3
|
125
136
|
summary: lightweight file-based MapReduce system
|
126
137
|
test_files:
|
138
|
+
- test/log/1234567890.yml
|
127
139
|
- test/mock_mros.rb
|
128
140
|
- test/mock_pipeio.rb
|
129
141
|
- test/mock_world.rb
|