pmux 0.1.0 → 0.1.1

data/LICENSE

@@ -0,0 +1,26 @@
+Copyright (c) 2012, Internet Initiative Japan Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -6,7 +6,7 @@ computing on a GlusterFS cluster, pmux provides a tool capable of
 handling large amounts of data stored in files.
 
 ## Requirements
- * ruby 1.9.2 or higher
+ * ruby 1.8.7, 1.9.2 or higher
  * msgpack-rpc
  * net-ssh, net-scp
  * gflocator
@@ -24,6 +24,8 @@ on the GlusterFS client node
     gem install gflocator
     sudo gflocator
 
+Each of GlusterFS server nodes, SSH with authentication key is required.
+
 ## Usage
 
 show status
@@ -31,6 +33,11 @@ show status
     $ pmux --status
     host0.example.com: pmux 0.1.0, num_cpu=8, ruby 1.9.3
 
+show status of remote machine
+
+    $ pmux --status -h host1.example.com
+    host1.example.com: pmux 0.1.0, num_cpu=2, ruby 1.9.3
+
 distributed grep
 
     $ pmux --mapper="grep PATTERN" /glusterfs/xxx/*.log

data/lib/pmux/application.rb

@@ -145,7 +145,6 @@ module Pmux
       op.on('--reducer=CMD') {|arg| opts[:reducer] = arg}
       op.on('--num-r=NUM', Integer) {|arg| opts[:num_r] = arg}
       op.on('--root-dir=DIR') {|arg| opts[:root_dir] = arg}
-      op.on('--sec', '--connect-secondary') {opts[:connect_secondary] = true}
       op.on('--ship-file=FILE', '--file=FILE') {|arg|
         (opts[:ship_files] ||= []).push arg}
       op.on('--status') {opts[:status] = true}

data/lib/pmux/multi_session.rb

@@ -39,7 +39,6 @@ module Pmux
 
       @scptable = {}
       @scpid = 0
-      @scps = {}
       @scp_queue = {}
       @buffers = {}
 
@@ -132,9 +131,6 @@ module Pmux
       queue = (@scp_queue[addr] ||= [])
 
       if (scp = @sessions[addr].scp)
-      #if (scp = @scps[addr]) or
-      #     (@sessions[addr] and @sessions[addr].ssh and
-      #      scp = (@scps[addr] = @sessions[addr].scp))
         scp_upload_sub scp, addr, future, local, remote, options
       else
         queue.push [:up, future, addr, remote, local, options]
@@ -143,14 +139,35 @@ module Pmux
     end
 
     def scp_upload_sub scp, addr, future, local, remote, options
+      session = @sessions[addr]
+      if !session or session.scp_session_count > 5
+        queue = (@scp_queue[addr] ||= [])
+        queue.push [:up, future, addr, remote, local, options]
+        return
+      end
+      session.scp_session_count += 1
+
       scpid = @scpid
       @scpid += 1
       @scptable[scpid] = future
-      scp.upload(local, remote, options) {|ch, name, sent, total|
-        if sent >= total
-          future.set_result nil, sent
-          @scptable.delete scpid
-        end
+
+      channel = scp.upload(local, remote, options)
+      channel.on_eof {|ch|
+        session.scp_session_count -= 1
+        @loop.set_timer(0) {process_scp_queue_once addr}
+
+        future.set_result(nil, options[:set_result])
+        @scptable.delete scpid
+      }
+      channel.on_open_failed {|ch, code, desc|
+        Log.error "#{addr}: scp error: #{desc}"
+        err = RuntimeError.new "scp error: #{desc}"
+        @on_error.call addr, err
+        session.scp_session_count -= 1
+        @loop.set_timer(0) {process_scp_queue_once addr}
+
+        future.set_result(nil, options[:set_result])
+        @scptable.delete scpid
       }
     end
 
@@ -159,9 +176,6 @@ module Pmux
       queue = (@scp_queue[addr] ||= [])
 
       if (scp = @sessions[addr].scp)
-      #if (scp = @scps[addr]) or
-      #     (@sessions[addr] and @sessions[addr].ssh and
-      #      scp = (@scps[addr] = @sessions[addr].scp))
         scp_download_sub scp, addr, future, remote, local, options
       else
         queue.push [:down, future, addr, remote, local, options]
@@ -202,7 +216,6 @@ module Pmux
     end
 
     def process_scp_queue_once addr
-      #scp = @scps[addr]
       scp = @sessions[addr].scp
       queue = (@scp_queue[addr] ||= [])
       if scp and !queue.empty?

data/lib/pmux/version.rb

@@ -1,3 +1,3 @@
 module Pmux
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pmux
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-26 00:00:00.000000000 Z
+date: 2012-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack-rpc
-  requirement: &130782920 !ruby/object:Gem::Requirement
+  requirement: &252597760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *130782920
+  version_requirements: *252597760
 - !ruby/object:Gem::Dependency
   name: net-scp
-  requirement: &130782420 !ruby/object:Gem::Requirement
+  requirement: &252597260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *130782420
+  version_requirements: *252597260
 description: lightweight file-based MapReduce system
 email: 
 executables:
@@ -41,6 +41,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- LICENSE
 - README.md
 - Rakefile
 - bin/pmux