cget 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 834d154a9fb4101e7af2d0323f38ec02bfbcc0e5
+  data.tar.gz: 3e3025dab55a3e03f1332b505bbee9d42e006afb
+SHA512:
+  metadata.gz: b43b9103ff1ee4e10dfe5515f39ea9eb02671957c84e7774eecb124aaf3eee29313efddb1175d91c46954119e46f8607c3f3be396b5e82cb0e0933cf251b4cc8
+  data.tar.gz: d75ee99136220d021c8a36fdb3ee23ccd64a11bfd498d65f41b6e21a1a939a35d585274ef48e3f493f19d60fd9c72749c9c8975b5e18e36cecf94ea49649cbba

data/bin/cget

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+require "socket"
+require "thread"
+
+Thread.abort_on_exception = true
+
+def usage!
+  $stderr.puts "Usage: cget [user@]host:remote_path local_path"
+  exit false
+end
+
+usage! if ARGV.size < 2
+host, remote_path = ARGV[0].split(":", 2)
+local_path = ARGV[1]
+usage! unless host && remote_path
+
+ssh = IO.popen(["ssh", host, "--", "cget-remote"], "r+b")
+
+ssh.write("#{remote_path}\0")
+connections, size, port, auth_token = ssh.gets.split
+
+file = File.open(local_path, "wb")
+file.truncate(size.to_i)
+
+queue = Queue.new
+
+threads = connections.to_i.times.map { |i|
+  Thread.new do
+    sock = TCPSocket.new(host, port.to_i)
+    sock.puts(auth_token)
+
+    while buffer = sock.readpartial(4096 + 8)
+      offset, = buffer[0, 8].unpack("Q>")
+      queue << { action: :write, offset: offset, data: buffer[8, 4096] }
+    end
+
+    queue << { action: :join, thread: Thread.current, number: i }
+  end
+}
+
+read = 0
+
+while threads.any?
+  message = queue.pop
+  case message[:action]
+  when :write
+    file.seek(message[:offset])
+    read += file.write(message[:data])
+    $stderr.print "\r #{read / 1024 / 1024} / #{file.size / 1024 / 1024} MiB"
+  when :join
+    message[:thread].join
+    threads.delete(message[:thread])
+  end
+end
+
+$stderr.puts

data/bin/cget-remote

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+require "socket"
+require "securerandom"
+
+$stdout.sync = true
+
+CONNECTIONS = 20
+
+path = $stdin.gets("\0").chop
+file = File.open(path, "r", encoding: "BINARY")
+
+server = TCPServer.new(0)
+server_port = server.addr[1]
+
+auth_token = SecureRandom.hex(32)
+
+puts "#{CONNECTIONS} #{file.size} #{server_port} #{auth_token}"
+
+sockets = CONNECTIONS.times.map { |i|
+  while sock = server.accept
+    break sock if sock.gets.chomp == auth_token
+  end
+}
+
+while sockets.any?
+  read, write, err = IO.select([], sockets, sockets)
+
+  err.each do |sock|
+    sock.close
+    sockets.delete(sock)
+  end
+
+  write.each do |sock|
+    offset = file.pos
+    if buffer = file.read(4096)
+      sock.write([offset].pack("Q>") + buffer)
+    else
+      sock.close
+      sockets.delete(sock)
+    end
+  end
+end
+
+sockets.each(&:close)

data/cget.gemspec

@@ -0,0 +1,18 @@
+Gem::Specification.new do |s|
+  s.name = "cget"
+  s.version = "0.0.1"
+  s.summary = "parallel downloader over ssh"
+  s.description = "a parallel downloader using ssh as a control connection"
+
+  s.author = "Charlie Somerville"
+  s.email = "charlie@charliesomerville.com"
+  s.homepage = "https://github.com/charliesome/cget"
+
+  s.license = "MIT"
+  s.files = `git ls-files`.lines.map(&:chomp)
+
+  s.executables = %w[
+    cget
+    cget-remote
+  ]
+end

metadata

@@ -0,0 +1,49 @@
+--- !ruby/object:Gem::Specification
+name: cget
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Charlie Somerville
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-11 00:00:00.000000000 Z
+dependencies: []
+description: a parallel downloader using ssh as a control connection
+email: charlie@charliesomerville.com
+executables:
+- cget
+- cget-remote
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/cget
+- bin/cget-remote
+- cget.gemspec
+homepage: https://github.com/charliesome/cget
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
+specification_version: 4
+summary: parallel downloader over ssh
+test_files: []
+has_rdoc: