cget 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/cget +56 -0
- data/bin/cget-remote +44 -0
- data/cget.gemspec +18 -0
- metadata +49 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 834d154a9fb4101e7af2d0323f38ec02bfbcc0e5
|
4
|
+
data.tar.gz: 3e3025dab55a3e03f1332b505bbee9d42e006afb
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: b43b9103ff1ee4e10dfe5515f39ea9eb02671957c84e7774eecb124aaf3eee29313efddb1175d91c46954119e46f8607c3f3be396b5e82cb0e0933cf251b4cc8
|
7
|
+
data.tar.gz: d75ee99136220d021c8a36fdb3ee23ccd64a11bfd498d65f41b6e21a1a939a35d585274ef48e3f493f19d60fd9c72749c9c8975b5e18e36cecf94ea49649cbba
|
data/bin/cget
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require "socket"
|
3
|
+
require "thread"
|
4
|
+
|
5
|
+
Thread.abort_on_exception = true
|
6
|
+
|
7
|
+
def usage!
|
8
|
+
$stderr.puts "Usage: cget [user@]host:remote_path local_path"
|
9
|
+
exit false
|
10
|
+
end
|
11
|
+
|
12
|
+
usage! if ARGV.size < 2
|
13
|
+
host, remote_path = ARGV[0].split(":", 2)
|
14
|
+
local_path = ARGV[1]
|
15
|
+
usage! unless host && remote_path
|
16
|
+
|
17
|
+
ssh = IO.popen(["ssh", host, "--", "cget-remote"], "r+b")
|
18
|
+
|
19
|
+
ssh.write("#{remote_path}\0")
|
20
|
+
connections, size, port, auth_token = ssh.gets.split
|
21
|
+
|
22
|
+
file = File.open(local_path, "wb")
|
23
|
+
file.truncate(size.to_i)
|
24
|
+
|
25
|
+
queue = Queue.new
|
26
|
+
|
27
|
+
threads = connections.to_i.times.map { |i|
|
28
|
+
Thread.new do
|
29
|
+
sock = TCPSocket.new(host, port.to_i)
|
30
|
+
sock.puts(auth_token)
|
31
|
+
|
32
|
+
while buffer = sock.readpartial(4096 + 8)
|
33
|
+
offset, = buffer[0, 8].unpack("Q>")
|
34
|
+
queue << { action: :write, offset: offset, data: buffer[8, 4096] }
|
35
|
+
end
|
36
|
+
|
37
|
+
queue << { action: :join, thread: Thread.current, number: i }
|
38
|
+
end
|
39
|
+
}
|
40
|
+
|
41
|
+
read = 0
|
42
|
+
|
43
|
+
while threads.any?
|
44
|
+
message = queue.pop
|
45
|
+
case message[:action]
|
46
|
+
when :write
|
47
|
+
file.seek(message[:offset])
|
48
|
+
read += file.write(message[:data])
|
49
|
+
$stderr.print "\r #{read / 1024 / 1024} / #{file.size / 1024 / 1024} MiB"
|
50
|
+
when :join
|
51
|
+
message[:thread].join
|
52
|
+
threads.delete(message[:thread])
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
$stderr.puts
|
data/bin/cget-remote
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require "socket"
|
3
|
+
require "securerandom"
|
4
|
+
|
5
|
+
$stdout.sync = true
|
6
|
+
|
7
|
+
CONNECTIONS = 20
|
8
|
+
|
9
|
+
path = $stdin.gets("\0").chop
|
10
|
+
file = File.open(path, "r", encoding: "BINARY")
|
11
|
+
|
12
|
+
server = TCPServer.new(0)
|
13
|
+
server_port = server.addr[1]
|
14
|
+
|
15
|
+
auth_token = SecureRandom.hex(32)
|
16
|
+
|
17
|
+
puts "#{CONNECTIONS} #{file.size} #{server_port} #{auth_token}"
|
18
|
+
|
19
|
+
sockets = CONNECTIONS.times.map { |i|
|
20
|
+
while sock = server.accept
|
21
|
+
break sock if sock.gets.chomp == auth_token
|
22
|
+
end
|
23
|
+
}
|
24
|
+
|
25
|
+
while sockets.any?
|
26
|
+
read, write, err = IO.select([], sockets, sockets)
|
27
|
+
|
28
|
+
err.each do |sock|
|
29
|
+
sock.close
|
30
|
+
sockets.delete(sock)
|
31
|
+
end
|
32
|
+
|
33
|
+
write.each do |sock|
|
34
|
+
offset = file.pos
|
35
|
+
if buffer = file.read(4096)
|
36
|
+
sock.write([offset].pack("Q>") + buffer)
|
37
|
+
else
|
38
|
+
sock.close
|
39
|
+
sockets.delete(sock)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
sockets.each(&:close)
|
data/cget.gemspec
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = "cget"
|
3
|
+
s.version = "0.0.1"
|
4
|
+
s.summary = "parallel downloader over ssh"
|
5
|
+
s.description = "a parallel downloader using ssh as a control connection"
|
6
|
+
|
7
|
+
s.author = "Charlie Somerville"
|
8
|
+
s.email = "charlie@charliesomerville.com"
|
9
|
+
s.homepage = "https://github.com/charliesome/cget"
|
10
|
+
|
11
|
+
s.license = "MIT"
|
12
|
+
s.files = `git ls-files`.lines.map(&:chomp)
|
13
|
+
|
14
|
+
s.executables = %w[
|
15
|
+
cget
|
16
|
+
cget-remote
|
17
|
+
]
|
18
|
+
end
|
metadata
ADDED
@@ -0,0 +1,49 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: cget
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Charlie Somerville
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-01-11 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description: a parallel downloader using ssh as a control connection
|
14
|
+
email: charlie@charliesomerville.com
|
15
|
+
executables:
|
16
|
+
- cget
|
17
|
+
- cget-remote
|
18
|
+
extensions: []
|
19
|
+
extra_rdoc_files: []
|
20
|
+
files:
|
21
|
+
- bin/cget
|
22
|
+
- bin/cget-remote
|
23
|
+
- cget.gemspec
|
24
|
+
homepage: https://github.com/charliesome/cget
|
25
|
+
licenses:
|
26
|
+
- MIT
|
27
|
+
metadata: {}
|
28
|
+
post_install_message:
|
29
|
+
rdoc_options: []
|
30
|
+
require_paths:
|
31
|
+
- lib
|
32
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
33
|
+
requirements:
|
34
|
+
- - ">="
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: '0'
|
37
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0'
|
42
|
+
requirements: []
|
43
|
+
rubyforge_project:
|
44
|
+
rubygems_version: 2.2.0
|
45
|
+
signing_key:
|
46
|
+
specification_version: 4
|
47
|
+
summary: parallel downloader over ssh
|
48
|
+
test_files: []
|
49
|
+
has_rdoc:
|