pmux 0.1.0

data/.gitignore

@@ -0,0 +1,8 @@
+*~
+.rbenv-version
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+test/.t
+tmp/*

data/README.md

@@ -0,0 +1,36 @@
+# Pmux: pipeline multiplexer
+
+Pmux is a lightweight file-based MapReduce system, written in Ruby.
+Applying the philosophy of Unix pipeline processing to distributed
+computing on a GlusterFS cluster, pmux provides a tool capable of
+handling large amounts of data stored in files.
+
+## Requirements
+ * ruby 1.9.2 or higher
+ * msgpack-rpc
+ * net-ssh, net-scp
+ * gflocator
+ * GlusterFS 3.3.0 or higher, native client (FUSE)
+
+## Install
+
+on all GlusterFS server nodes:
+
+    gem install pmux
+
+on the GlusterFS client node
+
+    gem install pmux
+    gem install gflocator
+    sudo gflocator
+
+## Usage
+
+show status
+
+    $ pmux --status
+    host0.example.com: pmux 0.1.0, num_cpu=8, ruby 1.9.3
+
+distributed grep
+
+    $ pmux --mapper="grep PATTERN" /glusterfs/xxx/*.log

data/Rakefile

@@ -0,0 +1,4 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new

data/bin/pmux

@@ -0,0 +1,5 @@
+#! /usr/bin/env ruby
+
+require 'pmux'
+
+Pmux::Application.new.run

data/lib/pmux/application.rb

@@ -0,0 +1,166 @@
+require 'optparse'
+
+module Pmux
+  class Application
+    OPTS = {}
+
+    def run options=OPTS
+      optparser = optparse options
+      optparser.parse!
+      options[:program_name] = optparser.program_name
+      options[:user] ||=
+        (ENV['USER'] || ENV['LOGNAME'] || Etc.getlogin || Etc.getpwuid.name)
+
+      root_dir = (options[:root_dir] ||=
+                  File.expand_path "~/.#{options[:program_name]}")
+      tmp_dir = (options[:tmp_dir] ||= root_dir + '/tmp')
+      log_dir = (options[:log_dir] ||= root_dir + '/log')
+      [root_dir, tmp_dir, log_dir].each {|dir|
+        Dir.mkdir dir unless File.exist? dir
+      }
+
+      Plugin.load_plugins unless options[:disable_plugins]
+      addrs = (options[:hosts] || '').split(',').map {|host| getaddr host}
+
+      case
+      when options[:server]
+        (options[:sock_path] ||= '/tmp/.pmuxsock') << ".#{$$}"
+        run_server options
+      when options[:status]
+        show_status addrs, options
+      when options[:lookup]
+        lookup addrs, options
+      else
+        run_mr addrs, options
+      end
+    end
+
+    def getaddr host
+      sa = Socket.pack_sockaddr_in 0, host
+      port, addr = Socket.unpack_sockaddr_in sa
+      addr
+    end
+
+    def run_server options
+      STDOUT.sync = true
+      server = MR::Server.new
+      handler = Pmux::Handler.new server, options
+      pipe_transport = MR::PipeTransport.new STDIN, STDOUT, STDERR
+      server.listen pipe_transport, handler
+      unix_transport = MR::UNIXServerTransport.new options[:sock_path]
+      server.listen unix_transport, handler
+      server.run
+    rescue SystemCallError
+    ensure
+      File.unlink options[:sock_path] rescue nil
+    end
+
+    def show_status addrs, options
+      addrs = ['localhost'] if addrs.empty?
+      adapter = StorageAdapter.create options[:storage_name], addrs
+      msession = MRSession.new addrs, options
+      msession.on_error {|addr, err| $stderr.printf "%s: %s\n", addr, err.to_s}
+      msession.connect
+
+      mf = msession.multicast_call_async :get_properties
+      templ = "%s: %s %s, num_cpu=%s, ruby %s\n"
+      mf.on_success {|f|
+        props = f.get
+        print templ % [props['hostname'],
+          props['program_name'], props['VERSION'],
+          props['num_cpu'], props['RUBY_VERSION']]
+      }
+      mf.on_error {|f| printf "%s: ERROR: %s\n", f.addr, f.error}
+      mf.join_all
+
+      msession.on_error {}
+      mf = msession.multicast_call_async :quit
+      mf.join_all
+    end
+
+    def lookup addrs, options
+      adapter = StorageAdapter.create options[:storage_name], addrs
+      name = options[:lookup]
+      locator_host = options[:locator_host] || addrs.first || 'localhost'
+      locator_port = options[:locator_port]
+      adapter.connect_to_storage locator_host, locator_port
+      adapter.get_files [name]
+      locations = adapter.lookup_file name
+      $stderr.puts "name: #{name}"
+      for host, path in locations
+        $stderr.puts "location: #{host}:#{path}"
+      end
+    end
+
+    def run_mr addrs, options, argv=ARGV
+      options[:__start_time] = Time.now
+      if options[:storage_name] == 'local' and addrs.empty?
+        addrs = ['localhost']
+      end
+      adapter = StorageAdapter.create options[:storage_name], addrs, options
+      locator_host = options[:locator_host] || addrs.first || 'localhost'
+      locator_port = options[:locator_port]
+
+      puts "storage: #{options[:storage_name]}" if options[:verbose]
+      begin
+        adapter.connect_to_storage locator_host, locator_port
+        files = adapter.get_files argv, options[:expand_glob]
+        options[:__get_files_time] = Time.now
+      rescue
+        STDERR.puts "Storage Error: #{$!}"
+        return
+      end
+      raise RuntimeError, "no hostname specified" if adapter.addrs.empty?
+
+      msession = MRSession.new adapter.addrs, options
+      msession.on_error {|addr, err|
+        $stderr.printf "%s: %s\n", addr, err.inspect if err
+      }
+      msession.connect
+
+      if options[:reducer]
+        options[:num_r] ||= 1
+      end
+      dispatcher = TaskDispatcher.new options, adapter, msession
+      job = Job.new options, files
+      job.mk_reducer_addrs adapter.addrs
+      dispatcher.run job
+    end
+
+    def optparse opts
+      op = OptionParser.new
+      op.on('--debug') {$debug = true; STDOUT.sync = true}
+      op.on('--server') {opts[:server] = true}
+      op.on('--argv=FILES') {}
+      op.on('--disable-plugins') {opts[:disable_plugins] = true}
+      op.on('--expand-glob') {opts[:expand_glob] = true}
+      op.on('--ff=FF', Integer) {|arg| opts[:ff] = arg}
+      op.on('--hosts=HOST,HOST,...') {|arg| opts[:hosts] = arg}
+      op.on('--ipaddr=ADDR') {|arg| opts[:ipaddr] = arg}
+      op.on('--locator-host=HOST') {|arg| opts[:locator_host] = arg}
+      op.on('--locator-port=PORT', Integer) {|arg|
+        opts[:locator_port] = arg}
+      op.on('--lookup=FILE') {|arg| opts[:lookup] = arg}
+      op.on('--mapper=CMD') {|arg| opts[:mapper] = arg}
+      op.on('--reducer=CMD') {|arg| opts[:reducer] = arg}
+      op.on('--num-r=NUM', Integer) {|arg| opts[:num_r] = arg}
+      op.on('--root-dir=DIR') {|arg| opts[:root_dir] = arg}
+      op.on('--sec', '--connect-secondary') {opts[:connect_secondary] = true}
+      op.on('--ship-file=FILE', '--file=FILE') {|arg|
+        (opts[:ship_files] ||= []).push arg}
+      op.on('--status') {opts[:status] = true}
+      op.on('--storage=STORAGE_NAME') {|arg|
+        opts[:storage_name] = arg}
+      op.on('--verbose') {opts[:verbose] = true}
+      op.on('--version') {
+        puts "#{op.program_name} #{VERSION}"
+        exit
+      }
+      class <<op
+        attr_accessor :options
+      end
+      op.options = opts
+      op
+    end
+  end
+end

data/lib/pmux/cleaner.rb

@@ -0,0 +1,28 @@
+require 'fileutils'
+
+module Pmux
+  class Cleaner
+    def initialize glob_pat
+      @glob_pat = glob_pat
+    end
+
+    def run lim_time=nil
+      fork {fork {clean @glob_pat, lim_time}}
+      Process.wait
+    end
+
+    def clean glob_pat=nil, lim_time=nil
+      glob_pat ||= @glob_pat
+      lim_time ||= Time.now - 3600*24
+      paths = Dir.glob glob_pat
+      for path in paths
+        if File.exist? path
+          mtime = File.mtime path
+          if mtime < lim_time
+            FileUtils.rm_rf path
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pmux/fiber18.rb

@@ -0,0 +1,64 @@
+# Thread based Fiber implementation for Ruby 1.8 by Aman Gupta
+unless defined? Fiber
+  require 'thread'
+  require 'singleton'
+  class FiberError < StandardError; end
+  class Fiber
+    def initialize
+      raise ArgumentError, 'new Fiber requires a block' unless block_given?
+ 
+      @yield = Queue.new
+      @resume = Queue.new
+
+      @thread = Thread.new{ @yield.push [ *yield(*@resume.pop) ] }
+      @thread.abort_on_exception = true
+      @thread[:fiber] = self
+    end
+    attr_reader :thread
+ 
+    def resume *args
+      raise FiberError, 'dead fiber called' unless @thread.alive?
+      @resume.push(args)
+      result = @yield.pop
+      result.size > 1 ? result : result.first
+    end
+    
+    def yield *args
+      @yield.push(args)
+      result = @resume.pop
+      result.size > 1 ? result : result.first
+    end
+    
+    def self.yield *args
+      raise FiberError, "can't yield from root fiber" unless fiber = Thread.current[:fiber]
+      fiber.yield(*args)
+    end
+ 
+    def self.current
+      Thread.current[:fiber] or raise FiberError, 'not inside a fiber'
+    end
+ 
+    def inspect
+      "#<#{self.class}:0x#{self.object_id.to_s(16)}>"
+    end
+  end
+
+  class RootFiber < Fiber
+    include Singleton
+    def initialize
+    end
+
+    def resume *args
+      raise FiberError, "can't resume root fiber"
+    end
+
+    def yield *args
+      raise FiberError, "can't yield from root fiber"
+    end
+  end
+
+  #attach the root fiber to the main thread
+  Thread.main[:fiber] = RootFiber.instance
+else
+  require 'fiber'
+end

data/lib/pmux/fixcmd.rb

@@ -0,0 +1,25 @@
+module Pmux
+  module FixCmdLine
+    def fix_cmd_line cmd_line, in_path=nil, out_path=nil, err_path=nil, tmp_dir=nil
+      res = []
+      cmds = cmd_line.split /\s*\|\s*/
+      n = 0
+      for cmd in cmds
+        c, = cmd.split
+        if tmp_dir and File.executable?(cc = "#{tmp_dir}/#{c}")
+          #cmd = cc
+          cmd = "#{tmp_dir}/#{cmd}"
+        end
+        if n == 0
+          cmd << " #{in_path}" if in_path
+          err_path ||= '/dev/null'
+          cmd << " 2>#{err_path}"
+        end
+        res.push cmd
+        n += 1
+      end
+      cmd << " >>#{out_path}" if out_path
+      res.join '|'
+    end
+  end
+end

data/lib/pmux/gatherer.rb

@@ -0,0 +1,23 @@
+module Pmux
+  class Gatherer
+    attr_accessor :writer
+    attr_reader :mf
+
+    def initialize writer=nil
+      @writer = writer
+      @mf = MR::MultiFuture.new
+      @mf.on_success {|f| writer.write f.get}
+    end
+
+    def gather msession, node_addr, remote, local, options={}
+      future = msession.scp_download node_addr, remote, local,
+        :set_result=>local
+      @mf.add future
+    end
+
+    def join_all
+      @mf.join_all
+      @writer.finish
+    end
+  end
+end

data/lib/pmux/handler.rb

@@ -0,0 +1,262 @@
+require 'fileutils'
+
+module Pmux
+  class Handler
+    attr_reader :options
+    attr_accessor :server
+
+    def initialize server=nil, options={}
+      @server = server
+      @options = options
+      @ipaddr = options[:ipaddr]
+
+      @wtq = []
+      @wq = []
+      @ases = {}
+      @msession = nil
+      @seqid = 0
+    end
+
+    def init_job job_id
+      path = "#{options[:tmp_dir]}/#{job_id}"
+      Dir.mkdir path
+      options[:job_dir] = path
+      Log.init "#{path}/worker.log", :log_level=>'debug'
+
+      num_cpu = get_num_cpu
+      #fork_worker num_cpu, options
+
+      {
+        'job_dir' => path,
+        'num_cpu' => num_cpu,
+      }
+    end
+
+    def get_num_cpu
+      cpuinfo_path = '/proc/cpuinfo'
+      if File.exist? cpuinfo_path
+        lines = File.readlines(cpuinfo_path).grep(/^processor/)
+        lines.size
+      else
+        2
+      end
+    end
+
+    # execute a task and return the result
+    def exec_streaming_task task
+      start_time = Time.now
+      as = MR::AsyncResult.new
+      if (task_keys = task['task_keys'])
+        error_ids = []
+        fusion_id = task['task_id']
+        fiber = Fiber.new {
+          for task_id, file in task_keys
+            ntask = task.merge 'fusion_id'=>fusion_id,
+              'task_id'=>task_id, 'path'=>file
+            result = do_one_task ntask, fiber
+          end
+          result.update :task_id=>fusion_id, :task_keys=>task_keys,
+            :welapse=>(Time.now - start_time)
+          as.result result
+        }
+      else
+        fiber = Fiber.new {
+          result = do_one_task(task, fiber)
+          result[:welapse] = Time.now - start_time
+          as.result result
+        }
+      end
+      fiber.resume
+      as
+    end
+
+    def do_one_task task, fiber=nil
+      job_id, task_id, node_addr =
+        task.values_at 'job_id', 'task_id', 'node_addr'
+      num_r = task['num_r'].to_i
+      result = {:job_id=>job_id, :task_id=>task_id, :node_addr=>node_addr}
+      exception = nil
+      error_level = nil
+      if @options and (otmp_dir = @options[:tmp_dir])
+        tmp_dir = "#{otmp_dir}/#{job_id}"
+      else
+        tmp_dir = "/var/tmp/#{job_id}"
+      end
+      if task['pindex']
+        # reduce task
+        result[:reduce] = true
+        reducer = StreamingReducer.new task, tmp_dir, @server.loop
+        reducer.on_success {
+          result[:output_path] = reducer.output_path
+          fiber.resume result if fiber
+        }
+        reducer.on_error {|e|
+          result.update :error=>e.class.to_s,
+            :error_message=>e.message, :backtrace=>e.backtrace[0]
+          fiber.resume result if fiber
+        }
+        reducer.do_streaming_reduce_task
+      else
+        # map task
+        result[:map] = true
+        mapper = StreamingMapper.new task, tmp_dir, @server.loop
+        #result[:ifbase] = mapper.do_map_task
+        mapper.on_success {
+          result[:ifbase] = mapper.ifbase
+          if num_r.zero?
+            if (rsize = mapper.result_size)
+              result[:result_body] = mapper.result_body if rsize < 1024
+            else
+              result[:result_body] = ''
+            end
+          end
+          fiber.resume result if fiber
+        }
+        mapper.on_error {|e|
+          result.update :error=>e.class.to_s,
+            :error_message=>e.message, :backtrace=>e.backtrace[0]
+          fiber.resume result if fiber
+        }
+        mapper.do_streaming_map_task
+      end
+      Fiber.yield
+    end
+    private :do_one_task
+
+    def notify_reduce params
+      job_id, task_id, pindex, node_addr, ifbase =
+        params.values_at 'job_id', 'task_id', 'pindex', 'node_addr', 'ifbase'
+      ifpath = "#{ifbase}-#{pindex}"
+      Log.debug "H: notify_reduce #{job_id}-#{task_id} #{ifbase}"
+
+      if @ipaddr == node_addr
+        # local
+        local = "#{options[:job_dir]}/#{File.basename(ifpath).sub(/^m/, 't')}"
+        File.rename ifpath, local
+        {:job_id=>job_id, :task_id=>task_id, :ifbase=>ifbase}
+      else
+        # remote
+        @msession ||=
+          MultiSession.new([], {:user=>@options[:user]}, @server.loop)
+        @msession.connect_to_addr node_addr
+        local = "#{options[:job_dir]}/#{File.basename(ifpath).sub(/^m/, 't')}"
+        future = @msession.scp_download node_addr, ifpath, local
+        future.attach_callback {|f|
+          if (as = @ases.delete "r#{job_id}-#{task_id}")
+            as.result :job_id=>job_id, :task_id=>task_id, :ifbase=>ifbase
+          end
+        }
+        @ases["r#{job_id}-#{task_id}"] = MR::AsyncResult.new
+      end
+    #rescue Exception
+    end
+
+
+    def init_scan addrs
+      log_path = "#{options[:log_dir]}/diffuser.log"
+      Log.init log_path, :log_level=>'debug'
+      @adapter = StorageAdapter.create 'pmuxfs', addrs
+      @fs_dir = options[:fs_dir]
+      @adapter.set_fs_dir @fs_dir
+      @fs_dir
+    end
+
+    def scan_once
+      files = @adapter.find
+    end
+
+    def close_download_channel node_addr
+      @msession.close_channel node_addr if @msession
+      @msession.class.to_s
+    end
+
+
+    def get_status
+      [
+        ['ruby_version', RUBY_VERSION, :string],
+        ['hoge', 1, :gauge],
+      ]
+    end
+
+    def get_properties
+      {
+        'hostname' => Socket.gethostname,
+        'program_name' => options[:program_name],
+        'root_dir' => options[:root_dir],
+        'tmp_dir' => options[:tmp_dir],
+        'VERSION' => VERSION,
+        'RUBY_VERSION' => RUBY_VERSION,
+        'num_cpu' => get_num_cpu,
+      }
+    end
+
+    def hello
+      'hello'
+    end
+
+    def quit
+      @server.loop.stop
+      cleaner = Cleaner.new "#{options[:tmp_dir]}/[0-9]*"
+      cleaner.run
+      nil
+    end
+
+=begin
+    private
+    def fork_worker num_worker, options
+      for n in 1..num_worker
+        pid = fork {
+          @server.loop.stop
+          begin
+            worker = Worker.new options
+          rescue Timeout::Error
+            Log.info "worker #{$$}: initialization timeout"
+            worker = nil
+          end
+          if worker
+            worker.run
+            Log.puts 'worker exit'
+          end
+          exit
+        }
+      end
+    end
+
+    def enq_task task
+      Log.debug "H: enq_task #{task['job_id']}-#{task['task_id']}"
+      @wtq.push task
+      process_task_queue
+      if $test and @fiber
+        @fiber.resume
+      end
+      @ases["#{task['job_id']}-#{task['task_id']}"] = MR::AsyncResult.new
+    end
+
+    def get_task
+      Log.debug "H: get_task"
+      if @wtq.empty?
+        as = MR::AsyncResult.new
+        @wq.push as
+        as
+      else
+        @wtq.shift
+      end
+    end
+
+    def put_result result
+      Log.debug "H: put_result #{result['task_id']}"
+      if (as = @ases.delete "#{result['job_id']}-#{result['task_id']}")
+        Log.debug "H: return as.result"
+        as.result result
+      end
+      nil
+    end
+
+    def process_task_queue
+      while !@wtq.empty? and (as = @wq.shift)
+        as.result @wtq.shift
+      end
+    end
+=end
+  end
+end