pmux 0.1.1 → 0.1.2

data/lib/pmux/application.rb

@@ -1,4 +1,5 @@
 require 'optparse'
+require 'yaml'
 
 module Pmux
   class Application
@@ -7,6 +8,7 @@ module Pmux
     def run options=OPTS
       optparser = optparse options
       optparser.parse!
+      load_config options
       options[:program_name] = optparser.program_name
       options[:user] ||=
         (ENV['USER'] || ENV['LOGNAME'] || Etc.getlogin || Etc.getpwuid.name)
@@ -30,11 +32,25 @@ module Pmux
         show_status addrs, options
       when options[:lookup]
         lookup addrs, options
+      when options[:show_joblog]
+        show_joblog options
       else
         run_mr addrs, options
       end
     end
 
+    def load_config options
+      path = File.expand_path(options[:config_file] || "~/.pmux/config.yml")
+      if File.file? path
+        conf = YAML.load_file path
+        if conf.kind_of? Hash
+          for k, v in conf
+            options[k.intern] ||= v
+          end
+        end
+      end
+    end
+
     def getaddr host
       sa = Socket.pack_sockaddr_in 0, host
       port, addr = Socket.unpack_sockaddr_in sa
@@ -55,6 +71,45 @@ module Pmux
       File.unlink options[:sock_path] rescue nil
     end
 
+    alias :putline :puts
+
+    def show_joblog options
+      log_dir = options[:log_dir]
+      if (job_id = options[:show_joblog]) == true
+        joblogs = Dir.glob(log_dir + '/*.yml').map {|path|
+          obj = YAML.load_file path
+        }
+        for h in joblogs.sort_by {|obj| obj[:invoked_at].to_f}
+          t = (h[:invoked_at] || h[:start_time]).strftime("%m/%d %H:%M")
+          line = format '%-10s %s "%s"', h[:id], t, h[:mapper]
+          putline line
+        end
+      else
+        path = File.join log_dir, "#{job_id}.yml"
+        els = {}
+        open(path) {|io|
+          s = YAML.load_stream io
+          header, tasks, footer = s[0], s[1], s[2]
+          if tasks
+            for task_id, task in tasks.sort_by {|k, v| k}
+              line = format '%5d %s %g',
+                task_id, task['node_addr'], task['welapse']
+              putline line
+              node_addr = task['node_addr']
+              els[node_addr] ||= [0, 0]
+              els[node_addr][0] += 1
+              els[node_addr][1] += task['welapse']
+            end
+          end
+        }
+        putline
+        for node_addr, v in els
+          putline format '%s %g/%d = %g',
+            node_addr, v[1], v[0], v[1] / v[0]
+        end
+      end
+    end
+
     def show_status addrs, options
       addrs = ['localhost'] if addrs.empty?
       adapter = StorageAdapter.create options[:storage_name], addrs
@@ -93,7 +148,7 @@ module Pmux
     end
 
     def run_mr addrs, options, argv=ARGV
-      options[:__start_time] = Time.now
+      invoked_at = Time.now
       if options[:storage_name] == 'local' and addrs.empty?
         addrs = ['localhost']
       end
@@ -101,11 +156,10 @@ module Pmux
       locator_host = options[:locator_host] || addrs.first || 'localhost'
       locator_port = options[:locator_port]
 
-      puts "storage: #{options[:storage_name]}" if options[:verbose]
+      puts "storage: #{adapter.class}" if options[:verbose]
       begin
         adapter.connect_to_storage locator_host, locator_port
         files = adapter.get_files argv, options[:expand_glob]
-        options[:__get_files_time] = Time.now
       rescue
         STDERR.puts "Storage Error: #{$!}"
         return
@@ -123,8 +177,10 @@ module Pmux
       end
       dispatcher = TaskDispatcher.new options, adapter, msession
       job = Job.new options, files
+      job[:invoked_at] = invoked_at
       job.mk_reducer_addrs adapter.addrs
       dispatcher.run job
+      abort if job.failed
     end
 
     def optparse opts
@@ -132,6 +188,8 @@ module Pmux
       op.on('--debug') {$debug = true; STDOUT.sync = true}
       op.on('--server') {opts[:server] = true}
       op.on('--argv=FILES') {}
+      op.on('--brick=HOST:/DIR', '-b') {|arg| (opts[:bricks] ||= []).push arg}
+      op.on('--config-file=FILE', '-F') {|arg| opts[:config_file] = arg}
       op.on('--disable-plugins') {opts[:disable_plugins] = true}
       op.on('--expand-glob') {opts[:expand_glob] = true}
       op.on('--ff=FF', Integer) {|arg| opts[:ff] = arg}
@@ -147,6 +205,7 @@ module Pmux
       op.on('--root-dir=DIR') {|arg| opts[:root_dir] = arg}
       op.on('--ship-file=FILE', '--file=FILE') {|arg|
         (opts[:ship_files] ||= []).push arg}
+      op.on('--show-joblog [job_id]') {|arg| opts[:show_joblog] = arg || true}
       op.on('--status') {opts[:status] = true}
       op.on('--storage=STORAGE_NAME') {|arg|
         opts[:storage_name] = arg}

data/lib/pmux/cleaner.rb

@@ -2,18 +2,18 @@ require 'fileutils'
 
 module Pmux
   class Cleaner
-    def initialize glob_pat
-      @glob_pat = glob_pat
+    def initialize *glob_pats
+      @glob_pats = glob_pats
     end
 
     def run lim_time=nil
-      fork {fork {clean @glob_pat, lim_time}}
+      fork {fork {@glob_pats.each {|glob_pat| clean glob_pat, lim_time}}}
       Process.wait
     end
 
     def clean glob_pat=nil, lim_time=nil
-      glob_pat ||= @glob_pat
-      lim_time ||= Time.now - 3600*24
+      glob_pat ||= ''
+      lim_time ||= Time.now - 3600*24*7
       paths = Dir.glob glob_pat
       for path in paths
         if File.exist? path

data/lib/pmux/handler.rb

@@ -194,6 +194,21 @@ module Pmux
       'hello'
     end
 
+    def ls dirs, args
+      res = []
+      for dir in dirs
+        for arg in args
+          Dir.chdir(dir) {
+            res += Dir.glob(arg).select {|path|
+              File.readable? path}.map {|path|
+              [path, File.join(dir, path)]
+            }
+          }
+        end
+      end
+      res
+    end
+
     def quit
       @server.loop.stop
       cleaner = Cleaner.new "#{options[:tmp_dir]}/[0-9]*"

data/lib/pmux/job.rb

@@ -7,6 +7,7 @@ module Pmux
     attr_reader :tasks, :num_t, :num_r
     attr_reader :taskhash
     attr_reader :reducers
+    attr_reader :failed
 
     def initialize params, files
       @params = params
@@ -20,9 +21,10 @@ module Pmux
       @done_taskhash = {}
       @tasks = mk_tasks files
       @num_t = @tasks.size + @num_r
-      @h = {:start_time=>Time.now,
+      @h = {:job_started_at=>Time.now,
         :map_tasks=>@tasks.size, :reduce_tasks=>@num_r,
       }
+      @failed = false
     end
 
     def mk_tasks files
@@ -83,10 +85,15 @@ module Pmux
       @taskhash.empty?
     end
 
+    def set_failed
+      @taskhash.clear
+      @failed = true
+    end
+
     def to_jlheader
       h = {:id=>id, :files_first=>@files.first, :tasksize=>@tasks.size,
         :params=>@params,
-        :start_time=>@h[:start_time],
+        :invoked_at=>@h[:invoked_at], :job_started_at=>@h[:job_started_at],
         :map_tasks=>@h[:map_tasks], :reduce_tasks=>@h[:reduce_tasks],
         :storage_name=>@params[:storage_name],
         :mapper=>@params[:mapper], :reducer=>@params[:reducer],
@@ -95,7 +102,7 @@ module Pmux
     end
 
     def to_jlfooter
-      h = {:end_time=>@h[:end_time]}
+      h = {:job_finished_at=>@h[:job_finished_at]}
     end
   end
 end

data/lib/pmux/mapper.rb

@@ -75,18 +75,19 @@ module Pmux
       mapper_cmd = @task['mapper'] || 'cat'
       err_path = "#{tmp_dir}/.err.#{object_id}"
       err_msg = nil
+      pipeio = nil
       if @num_r <= 1
         cmd_line = fix_cmd_line mapper_cmd,
           @path, nil, err_path, tmp_dir
         Log.debug "pipe: #{cmd_line}"
-        pipeio = PipeIO.new cmd_line
+        Dir.chdir(tmp_dir) {pipeio = PipeIO.new cmd_line}
         out = open("#{@ifbase}-0", 'a')
         pipeio.on_receive {|data| out.write data}
       else # @num_r >= 2
         partitioner = TextPartitioner.new @ifbase, @num_r,
           :separator=>@task['separator']
         cmd_line = fix_cmd_line mapper_cmd, @path, nil, err_path, tmp_dir
-        pipeio = PipeIO.new cmd_line
+        Dir.chdir(tmp_dir) {pipeio = PipeIO.new cmd_line}
         pipeio.on_receive {|data| partitioner.emit data}
       end
       on_success = @on_success

data/lib/pmux/multi_session.rb

@@ -6,7 +6,7 @@ end
 
 module Pmux
   class SessionWrapper
-    attr_reader :addr
+    attr_reader :addr, :scp_channels
     attr_accessor :ssh, :scp_session_count
 
     def initialize addr
@@ -14,6 +14,7 @@ module Pmux
       @ssh = nil
       @scp = nil
       @scp_session_count = 0
+      @scp_channels = {}
     end
 
     def scp
@@ -43,6 +44,7 @@ module Pmux
       @buffers = {}
 
       @on_error = nil
+      @session_lim = ($test ? 99 : 2)
     end
 
     def connect_to_addr addr, cmd=nil
@@ -95,6 +97,10 @@ module Pmux
     end
 
     def error_on_addr addr, err=nil
+      session = @sessions[addr]
+      session.scp_channels.each_value {|ch|
+        ch.on_open_failed.call ch, 0xfe000001, err.to_s
+      }
       @err_addrs.push addr
       @addrs.delete addr
       @on_error.call addr, err if @on_error
@@ -116,13 +122,12 @@ module Pmux
       @on_error = block
     end
 
-    def scp_upload_files addr, files, remote, options={}, &block
+    def scp_upload_files addr, files, remote, options={}
       mf = MR::MultiFuture.new
       for file in files
         future = scp_upload addr, file, remote, options
         mf.add future
       end
-      mf.on_all &block if block
       mf
     end
 
@@ -140,7 +145,7 @@ module Pmux
 
     def scp_upload_sub scp, addr, future, local, remote, options
       session = @sessions[addr]
-      if !session or session.scp_session_count > 5
+      if !session or session.scp_session_count > @session_lim
         queue = (@scp_queue[addr] ||= [])
         queue.push [:up, future, addr, remote, local, options]
         return
@@ -152,8 +157,10 @@ module Pmux
       @scptable[scpid] = future
 
       channel = scp.upload(local, remote, options)
+      session.scp_channels[scpid] = channel
       channel.on_eof {|ch|
         session.scp_session_count -= 1
+        session.scp_channels.delete scpid
         @loop.set_timer(0) {process_scp_queue_once addr}
 
         future.set_result(nil, options[:set_result])
@@ -161,12 +168,18 @@ module Pmux
       }
       channel.on_open_failed {|ch, code, desc|
         Log.error "#{addr}: scp error: #{desc}"
-        err = RuntimeError.new "scp error: #{desc}"
-        @on_error.call addr, err
         session.scp_session_count -= 1
+        session.scp_channels.delete scpid
         @loop.set_timer(0) {process_scp_queue_once addr}
 
-        future.set_result(nil, options[:set_result])
+        if code == 0xfe000001
+          err = desc
+        else
+          err = RuntimeError.new "scp error: #{desc}"
+          @on_error.call addr, err
+          err = nil
+        end
+        future.set_result(err, options[:set_result])
         @scptable.delete scpid
       }
     end
@@ -185,7 +198,7 @@ module Pmux
 
     def scp_download_sub scp, addr, future, remote, local, options
       session = @sessions[addr]
-      if !session or session.scp_session_count > 5
+      if !session or session.scp_session_count > @session_lim
         queue = (@scp_queue[addr] ||= [])
         queue.push [:down, future, addr, remote, local, options]
         return

data/lib/pmux/reducer.rb

@@ -59,7 +59,8 @@ module Pmux
       cmd_line = fix_cmd_line reducer_cmd,
         @paths.join(' '), nil, err_path, tmp_dir
       Log.debug "popen: #{cmd_line}"
-      pipeio = PipeIO.new cmd_line
+      pipeio = nil
+      Dir.chdir(@tmp_dir) {pipeio = PipeIO.new cmd_line}
       if @on_receive
         pipeio.on_receive &@on_receive
       else

data/lib/pmux/storage_adapter.rb

@@ -102,4 +102,48 @@ module Pmux
       end
     end
   end
+
+  class BBFSAdapter < GlusterFSAdapter
+    Adapters['bbfs'] = self
+
+    def connect_to_storage locator_host, locator_port
+      @addr2dirs = {}
+      for b in @options[:bricks]
+        if b =~ %r{(\S+):(.+)}
+          dir = $2
+          addr = getaddr $1
+          (@addr2dirs[addr] ||= []).push dir
+        end
+      end
+      @addrs = @addr2dirs.keys
+      @h = init_node_hash @addrs
+    end
+
+    def get_files args, glob_flag=false
+      msession = MRSession.new @addrs, @options
+      msession.on_error {|addr, err| $stderr.printf "%s: %s\n", addr, err.to_s}
+      msession.connect
+      @locations = {}
+      mf = MR::MultiFuture.new
+      mf.on_success {|f|
+        addr = f.addr
+        res = f.get
+        for rpath, apath in res
+          (@locations[rpath] ||= []).push [addr, apath]
+        end
+      }
+      for addr in @addrs
+        future = msession.call_async addr, 'ls', @addr2dirs[addr], args
+        mf.add future
+      end
+      mf.join_all
+      @locations.keys
+    end
+
+    def lookup_file file
+      @locations[file]
+    end
+
+    private
+  end
 end

data/lib/pmux/task_dispatcher.rb

@@ -57,42 +57,46 @@ module Pmux
             scheduler.shipped[addr] = true
             scheduler.process_queue
           }
+          mf_scp.on_error {job.set_failed}
         else
           puts 'start scheduler' if @verbose
           scheduler.shipped[addr] = true
           scheduler.process_queue
         end
       }
-      mf_init.on_error {job.taskhash.clear}
+      mf_init.on_error {job.set_failed}
 
       # wait for all map tasks to finish
       until job.completed?
         msession.loop.run_once
       end
-      if job.num_r.zero?
-        gatherer.join_all
-      else
-        mf_shuffle.join_all
-        # reduce phase
-        job.mk_reduce_tasks
-        scheduler.inject_tasks job.tasks
-        scheduler.process_queue
-        # wait for all reduce tasks to finish
-        until job.completed?
-          msession.loop.run_once
+      unless job.failed
+        if job.num_r.zero?
+          gatherer.join_all
+        else
+          mf_shuffle.join_all
+          # reduce phase
+          job.mk_reduce_tasks
+          scheduler.inject_tasks job.tasks
+          scheduler.process_queue
+          # wait for all reduce tasks to finish
+          until job.completed?
+            msession.loop.run_once
+          end
+          gatherer.join_all
         end
-        gatherer.join_all
       end
 
       Log.info "END"
-      job[:end_time] = Time.now
+      job[:job_finished_at] = Time.now
       jl.dump_footer
       jl.close
 
       mf_quit = msession.multicast_call_async :quit
       mf_quit.join_all
 
-      cleaner = Cleaner.new "#{options[:tmp_dir]}/w*"
+      cleaner = Cleaner.new "#{options[:tmp_dir]}/w*",
+        "#{options[:log_dir]}/*.yml"
       cleaner.run
     end
 
@@ -104,12 +108,11 @@ module Pmux
       task_id, node_addr, ifbase =
         result.values_at 'task_id', 'node_addr', 'ifbase'
       Log.info "receive result #{task_id} from #{node_addr}"
-      jl.add task_id, :node_addr=>node_addr, :ifbase=>ifbase,
-        :welapse=>result['welapse']
       puts "receive result #{task_id} from #{node_addr}" if @verbose
       if result['error']
         if @on_error
           @on_error.call result
+          job.set_failed
         else
         end
       end
@@ -146,8 +149,13 @@ module Pmux
       end
 
       task = job.get_task_by_id task_id
-      task[:elapse] = Time.now - task[:alloc_time] if task[:alloc_time]
-      task[:welapse] = result['welapse']
+      alloc_time = task[:alloc_time]
+      allocated_at = alloc_time - job[:job_started_at]
+      elapse = Time.now - alloc_time if alloc_time
+      #task[:welapse] = result['welapse']
+      jl.add task_id, :node_addr=>node_addr, :ifbase=>ifbase,
+        :welapse=>result['welapse'], :elapse=>elapse,
+        :allocated_at=>allocated_at
 
       # delete task
       scheduler.delete_task_from_job job, task, node_addr

data/lib/pmux/version.rb

@@ -1,3 +1,3 @@
 module Pmux
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/test/log/1234567890.yml

@@ -0,0 +1,57 @@
+---
+:id: '1234567890'
+:files_first: /dist/apache/access.20001101_0.log.gz
+:tasksize: 6
+:params:
+  :disable_plugins: true
+  :ship_files:
+  - tmp/apache_sum.rb
+  :mapper: apache_sum.rb
+  :reducer: apache_sum.rb
+  :program_name: pmux
+  :user: foo
+  :root_dir: /home/foo/.pmux
+  :tmp_dir: /home/foo/.pmux/tmp
+  :log_dir: /home/foo/.pmux/log
+  :__start_time: 2000-11-30 09:42:41.111131000 +09:00
+  :__get_files_time: 2000-11-30 09:42:41.130985000 +09:00
+  :num_r: 1
+  :job_name: ! '"apache_sum.rb"'
+:start_time: 2000-11-30 09:42:41.158194000 +09:00
+:map_tasks: 6
+:reduce_tasks: 1
+:storage_name: 
+:mapper: apache_sum.rb
+:reducer: apache_sum.rb
+:num_r: 1
+---
+2:
+  node_addr: "192.168.196.208"
+  ifbase: "/home/foo/.pmux/tmp/105459480/m2"
+  welapse: 0.037278
+4:
+  node_addr: "192.168.196.208"
+  ifbase: "/home/foo/.pmux/tmp/105459480/m4"
+  welapse: 0.036954
+1:
+  node_addr: "192.168.196.208"
+  ifbase: "/home/foo/.pmux/tmp/105459480/m1"
+  welapse: 5.009
+5:
+  node_addr: "192.168.196.224"
+  ifbase: "/home/foo/.pmux/tmp/105459480/m5"
+  welapse: 14.709426
+6:
+  node_addr: "192.168.196.224"
+  ifbase: "/home/foo/.pmux/tmp/105459480/m6"
+  welapse: 0.03236
+3:
+  node_addr: "192.168.196.224"
+  ifbase: "/home/foo/.pmux/tmp/105459480/m3"
+  welapse: 18.065011
+7:
+  node_addr: "192.168.196.208"
+  ifbase: nil
+  welapse: 0.0344
+---
+:end_time: 2000-11-30 09:43:00.172319000 +09:00

data/test/test_application.rb

@@ -1,6 +1,21 @@
 require File.expand_path(File.dirname(__FILE__) + '/unittest_helper')
 
 class TestMain < Test::Unit::TestCase
+  def test_show_joblog
+    main = Pmux::Application.new
+    class <<main
+      attr_reader :lines
+      def putline line=''; (@lines||=[]).push line; end
+    end
+    log_dir = File.expand_path(File.dirname(__FILE__) + '/log')
+
+    main.show_joblog :log_dir=>log_dir, :show_joblog=>true
+    ae 1, main.lines.size
+    main.lines.clear
+    main.show_joblog :log_dir=>log_dir, :show_joblog=>'1234567890'
+    ae 10, main.lines.size
+  end
+
   def test_optparse
     main = Pmux::Application.new
     options = {}

data/test/test_i_mapreduce.rb

@@ -53,7 +53,7 @@ end
 class Pmux::Cleaner
   def run lim_time=nil
     lim_time = Time.now - 3600
-    clean @glob_pat, lim_time
+    @glob_pats.each {|glob_pat| clean glob_pat, lim_time}
   end
 end
 
@@ -147,8 +147,9 @@ class TestIMapreduce < Test::Unit::TestCase
     job = Pmux::Job.new options, files
     job.mk_reducer_addrs addrs
     dispatcher.run job
+    assert job.failed
     assert_match %r{/notexist: No such file or directory}, er['error_message']
-    ae 420, writer.lines.size
+    ae 0, writer.lines.size
   end
 
   # mr test; num_r=0, 4 files, ff=2

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pmux
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-07 00:00:00.000000000 Z
+date: 2013-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack-rpc
-  requirement: &252597760 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *252597760
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: net-scp
-  requirement: &252597260 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +37,12 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *252597260
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: lightweight file-based MapReduce system
 email: 
 executables:
@@ -71,6 +81,7 @@ files:
 - lib/pmux/worker.rb
 - lib/pmux/writer.rb
 - pmux.gemspec
+- test/log/1234567890.yml
 - test/mock_mros.rb
 - test/mock_pipeio.rb
 - test/mock_world.rb
@@ -119,11 +130,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: pmux
-rubygems_version: 1.8.11
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: lightweight file-based MapReduce system
 test_files:
+- test/log/1234567890.yml
 - test/mock_mros.rb
 - test/mock_pipeio.rb
 - test/mock_world.rb