pmux 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/LICENSE ADDED
@@ -0,0 +1,26 @@
1
+ Copyright (c) 2012, Internet Initiative Japan Inc.
2
+ All rights reserved.
3
+
4
+ Redistribution and use in source and binary forms, with or without
5
+ modification, are permitted provided that the following conditions are
6
+ met:
7
+
8
+ * Redistributions of source code must retain the above copyright
9
+ notice, this list of conditions and the following disclaimer.
10
+
11
+ * Redistributions in binary form must reproduce the above
12
+ copyright notice, this list of conditions and the following
13
+ disclaimer in the documentation and/or other materials provided
14
+ with the distribution.
15
+
16
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
data/README.md CHANGED
@@ -6,7 +6,7 @@ computing on a GlusterFS cluster, pmux provides a tool capable of
6
6
  handling large amounts of data stored in files.
7
7
 
8
8
  ## Requirements
9
- * ruby 1.9.2 or higher
9
+ * ruby 1.8.7, 1.9.2 or higher
10
10
  * msgpack-rpc
11
11
  * net-ssh, net-scp
12
12
  * gflocator
@@ -24,6 +24,8 @@ on the GlusterFS client node
24
24
  gem install gflocator
25
25
  sudo gflocator
26
26
 
27
+ Each of GlusterFS server nodes, SSH with authentication key is required.
28
+
27
29
  ## Usage
28
30
 
29
31
  show status
@@ -31,6 +33,11 @@ show status
31
33
  $ pmux --status
32
34
  host0.example.com: pmux 0.1.0, num_cpu=8, ruby 1.9.3
33
35
 
36
+ show status of remote machine
37
+
38
+ $ pmux --status -h host1.example.com
39
+ host1.example.com: pmux 0.1.0, num_cpu=2, ruby 1.9.3
40
+
34
41
  distributed grep
35
42
 
36
43
  $ pmux --mapper="grep PATTERN" /glusterfs/xxx/*.log
@@ -145,7 +145,6 @@ module Pmux
145
145
  op.on('--reducer=CMD') {|arg| opts[:reducer] = arg}
146
146
  op.on('--num-r=NUM', Integer) {|arg| opts[:num_r] = arg}
147
147
  op.on('--root-dir=DIR') {|arg| opts[:root_dir] = arg}
148
- op.on('--sec', '--connect-secondary') {opts[:connect_secondary] = true}
149
148
  op.on('--ship-file=FILE', '--file=FILE') {|arg|
150
149
  (opts[:ship_files] ||= []).push arg}
151
150
  op.on('--status') {opts[:status] = true}
@@ -39,7 +39,6 @@ module Pmux
39
39
 
40
40
  @scptable = {}
41
41
  @scpid = 0
42
- @scps = {}
43
42
  @scp_queue = {}
44
43
  @buffers = {}
45
44
 
@@ -132,9 +131,6 @@ module Pmux
132
131
  queue = (@scp_queue[addr] ||= [])
133
132
 
134
133
  if (scp = @sessions[addr].scp)
135
- #if (scp = @scps[addr]) or
136
- # (@sessions[addr] and @sessions[addr].ssh and
137
- # scp = (@scps[addr] = @sessions[addr].scp))
138
134
  scp_upload_sub scp, addr, future, local, remote, options
139
135
  else
140
136
  queue.push [:up, future, addr, remote, local, options]
@@ -143,14 +139,35 @@ module Pmux
143
139
  end
144
140
 
145
141
  def scp_upload_sub scp, addr, future, local, remote, options
142
+ session = @sessions[addr]
143
+ if !session or session.scp_session_count > 5
144
+ queue = (@scp_queue[addr] ||= [])
145
+ queue.push [:up, future, addr, remote, local, options]
146
+ return
147
+ end
148
+ session.scp_session_count += 1
149
+
146
150
  scpid = @scpid
147
151
  @scpid += 1
148
152
  @scptable[scpid] = future
149
- scp.upload(local, remote, options) {|ch, name, sent, total|
150
- if sent >= total
151
- future.set_result nil, sent
152
- @scptable.delete scpid
153
- end
153
+
154
+ channel = scp.upload(local, remote, options)
155
+ channel.on_eof {|ch|
156
+ session.scp_session_count -= 1
157
+ @loop.set_timer(0) {process_scp_queue_once addr}
158
+
159
+ future.set_result(nil, options[:set_result])
160
+ @scptable.delete scpid
161
+ }
162
+ channel.on_open_failed {|ch, code, desc|
163
+ Log.error "#{addr}: scp error: #{desc}"
164
+ err = RuntimeError.new "scp error: #{desc}"
165
+ @on_error.call addr, err
166
+ session.scp_session_count -= 1
167
+ @loop.set_timer(0) {process_scp_queue_once addr}
168
+
169
+ future.set_result(nil, options[:set_result])
170
+ @scptable.delete scpid
154
171
  }
155
172
  end
156
173
 
@@ -159,9 +176,6 @@ module Pmux
159
176
  queue = (@scp_queue[addr] ||= [])
160
177
 
161
178
  if (scp = @sessions[addr].scp)
162
- #if (scp = @scps[addr]) or
163
- # (@sessions[addr] and @sessions[addr].ssh and
164
- # scp = (@scps[addr] = @sessions[addr].scp))
165
179
  scp_download_sub scp, addr, future, remote, local, options
166
180
  else
167
181
  queue.push [:down, future, addr, remote, local, options]
@@ -202,7 +216,6 @@ module Pmux
202
216
  end
203
217
 
204
218
  def process_scp_queue_once addr
205
- #scp = @scps[addr]
206
219
  scp = @sessions[addr].scp
207
220
  queue = (@scp_queue[addr] ||= [])
208
221
  if scp and !queue.empty?
@@ -1,3 +1,3 @@
1
1
  module Pmux
2
- VERSION = "0.1.0"
2
+ VERSION = "0.1.1"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pmux
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,11 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-10-26 00:00:00.000000000 Z
12
+ date: 2012-11-07 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: msgpack-rpc
16
- requirement: &130782920 !ruby/object:Gem::Requirement
16
+ requirement: &252597760 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
21
21
  version: '0'
22
22
  type: :runtime
23
23
  prerelease: false
24
- version_requirements: *130782920
24
+ version_requirements: *252597760
25
25
  - !ruby/object:Gem::Dependency
26
26
  name: net-scp
27
- requirement: &130782420 !ruby/object:Gem::Requirement
27
+ requirement: &252597260 !ruby/object:Gem::Requirement
28
28
  none: false
29
29
  requirements:
30
30
  - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
32
32
  version: '0'
33
33
  type: :runtime
34
34
  prerelease: false
35
- version_requirements: *130782420
35
+ version_requirements: *252597260
36
36
  description: lightweight file-based MapReduce system
37
37
  email:
38
38
  executables:
@@ -41,6 +41,7 @@ extensions: []
41
41
  extra_rdoc_files: []
42
42
  files:
43
43
  - .gitignore
44
+ - LICENSE
44
45
  - README.md
45
46
  - Rakefile
46
47
  - bin/pmux