cget 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (5) hide show
  1. checksums.yaml +7 -0
  2. data/bin/cget +56 -0
  3. data/bin/cget-remote +44 -0
  4. data/cget.gemspec +18 -0
  5. metadata +49 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 834d154a9fb4101e7af2d0323f38ec02bfbcc0e5
4
+ data.tar.gz: 3e3025dab55a3e03f1332b505bbee9d42e006afb
5
+ SHA512:
6
+ metadata.gz: b43b9103ff1ee4e10dfe5515f39ea9eb02671957c84e7774eecb124aaf3eee29313efddb1175d91c46954119e46f8607c3f3be396b5e82cb0e0933cf251b4cc8
7
+ data.tar.gz: d75ee99136220d021c8a36fdb3ee23ccd64a11bfd498d65f41b6e21a1a939a35d585274ef48e3f493f19d60fd9c72749c9c8975b5e18e36cecf94ea49649cbba
@@ -0,0 +1,56 @@
1
+ #!/usr/bin/env ruby
2
+ require "socket"
3
+ require "thread"
4
+
5
+ Thread.abort_on_exception = true
6
+
7
+ def usage!
8
+ $stderr.puts "Usage: cget [user@]host:remote_path local_path"
9
+ exit false
10
+ end
11
+
12
+ usage! if ARGV.size < 2
13
+ host, remote_path = ARGV[0].split(":", 2)
14
+ local_path = ARGV[1]
15
+ usage! unless host && remote_path
16
+
17
+ ssh = IO.popen(["ssh", host, "--", "cget-remote"], "r+b")
18
+
19
+ ssh.write("#{remote_path}\0")
20
+ connections, size, port, auth_token = ssh.gets.split
21
+
22
+ file = File.open(local_path, "wb")
23
+ file.truncate(size.to_i)
24
+
25
+ queue = Queue.new
26
+
27
+ threads = connections.to_i.times.map { |i|
28
+ Thread.new do
29
+ sock = TCPSocket.new(host, port.to_i)
30
+ sock.puts(auth_token)
31
+
32
+ while buffer = sock.readpartial(4096 + 8)
33
+ offset, = buffer[0, 8].unpack("Q>")
34
+ queue << { action: :write, offset: offset, data: buffer[8, 4096] }
35
+ end
36
+
37
+ queue << { action: :join, thread: Thread.current, number: i }
38
+ end
39
+ }
40
+
41
+ read = 0
42
+
43
+ while threads.any?
44
+ message = queue.pop
45
+ case message[:action]
46
+ when :write
47
+ file.seek(message[:offset])
48
+ read += file.write(message[:data])
49
+ $stderr.print "\r #{read / 1024 / 1024} / #{file.size / 1024 / 1024} MiB"
50
+ when :join
51
+ message[:thread].join
52
+ threads.delete(message[:thread])
53
+ end
54
+ end
55
+
56
+ $stderr.puts
@@ -0,0 +1,44 @@
1
+ #!/usr/bin/env ruby
2
+ require "socket"
3
+ require "securerandom"
4
+
5
+ $stdout.sync = true
6
+
7
+ CONNECTIONS = 20
8
+
9
+ path = $stdin.gets("\0").chop
10
+ file = File.open(path, "r", encoding: "BINARY")
11
+
12
+ server = TCPServer.new(0)
13
+ server_port = server.addr[1]
14
+
15
+ auth_token = SecureRandom.hex(32)
16
+
17
+ puts "#{CONNECTIONS} #{file.size} #{server_port} #{auth_token}"
18
+
19
+ sockets = CONNECTIONS.times.map { |i|
20
+ while sock = server.accept
21
+ break sock if sock.gets.chomp == auth_token
22
+ end
23
+ }
24
+
25
+ while sockets.any?
26
+ read, write, err = IO.select([], sockets, sockets)
27
+
28
+ err.each do |sock|
29
+ sock.close
30
+ sockets.delete(sock)
31
+ end
32
+
33
+ write.each do |sock|
34
+ offset = file.pos
35
+ if buffer = file.read(4096)
36
+ sock.write([offset].pack("Q>") + buffer)
37
+ else
38
+ sock.close
39
+ sockets.delete(sock)
40
+ end
41
+ end
42
+ end
43
+
44
+ sockets.each(&:close)
@@ -0,0 +1,18 @@
1
+ Gem::Specification.new do |s|
2
+ s.name = "cget"
3
+ s.version = "0.0.1"
4
+ s.summary = "parallel downloader over ssh"
5
+ s.description = "a parallel downloader using ssh as a control connection"
6
+
7
+ s.author = "Charlie Somerville"
8
+ s.email = "charlie@charliesomerville.com"
9
+ s.homepage = "https://github.com/charliesome/cget"
10
+
11
+ s.license = "MIT"
12
+ s.files = `git ls-files`.lines.map(&:chomp)
13
+
14
+ s.executables = %w[
15
+ cget
16
+ cget-remote
17
+ ]
18
+ end
metadata ADDED
@@ -0,0 +1,49 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: cget
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Charlie Somerville
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-01-11 00:00:00.000000000 Z
12
+ dependencies: []
13
+ description: a parallel downloader using ssh as a control connection
14
+ email: charlie@charliesomerville.com
15
+ executables:
16
+ - cget
17
+ - cget-remote
18
+ extensions: []
19
+ extra_rdoc_files: []
20
+ files:
21
+ - bin/cget
22
+ - bin/cget-remote
23
+ - cget.gemspec
24
+ homepage: https://github.com/charliesome/cget
25
+ licenses:
26
+ - MIT
27
+ metadata: {}
28
+ post_install_message:
29
+ rdoc_options: []
30
+ require_paths:
31
+ - lib
32
+ required_ruby_version: !ruby/object:Gem::Requirement
33
+ requirements:
34
+ - - ">="
35
+ - !ruby/object:Gem::Version
36
+ version: '0'
37
+ required_rubygems_version: !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - ">="
40
+ - !ruby/object:Gem::Version
41
+ version: '0'
42
+ requirements: []
43
+ rubyforge_project:
44
+ rubygems_version: 2.2.0
45
+ signing_key:
46
+ specification_version: 4
47
+ summary: parallel downloader over ssh
48
+ test_files: []
49
+ has_rdoc: