nabit 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +53 -0
- data/README.mkd +60 -0
- data/Rakefile +5 -0
- data/bin/nabit +40 -0
- data/lib/nabit.rb +2 -0
- data/lib/nabit/downloader.rb +176 -0
- data/lib/nabit/version.rb +3 -0
- data/nabit.gemspec +24 -0
- metadata +58 -0
data/LICENSE
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
Copyright (c) 2012, Jon Maken
|
2
|
+
|
3
|
+
All rights reserved.
|
4
|
+
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
7
|
+
|
8
|
+
* Redistributions of source code must retain the above copyright notice,
|
9
|
+
this list of conditions and the following disclaimer.
|
10
|
+
|
11
|
+
* Redistributions in binary form must reproduce the above copyright notice,
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
13
|
+
and/or other materials provided with the distribution.
|
14
|
+
|
15
|
+
* Neither the name of Jon Maken nor the names of any contributors may be
|
16
|
+
used to endorse or promote products derived from this software without
|
17
|
+
specific prior written permission.
|
18
|
+
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
|
20
|
+
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
21
|
+
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
|
22
|
+
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
|
23
|
+
OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
24
|
+
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
25
|
+
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
26
|
+
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
27
|
+
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
28
|
+
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
29
|
+
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
30
|
+
|
31
|
+
|
32
|
+
=== MiniPortile License ===
|
33
|
+
|
34
|
+
Copyright (c) 2011 Luis Lavena.
|
35
|
+
|
36
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
37
|
+
a copy of this software and associated documentation files (the
|
38
|
+
"Software"), to deal in the Software without restriction, including
|
39
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
40
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
41
|
+
permit persons to whom the Software is furnished to do so, subject to
|
42
|
+
the following conditions:
|
43
|
+
|
44
|
+
The above copyright notice and this permission notice shall be
|
45
|
+
included in all copies or substantial portions of the Software.
|
46
|
+
|
47
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
48
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
49
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
50
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
51
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
52
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
53
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.mkd
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
## What?
|
2
|
+
|
3
|
+
A minimalistic, pure-Ruby HTTP/HTTPS/FTP file downloader library with a command
|
4
|
+
line interface.
|
5
|
+
|
6
|
+
## Usage
|
7
|
+
|
8
|
+
HTTP/HTTPS/FTP File Downloader, v0.2.0
|
9
|
+
Usage: nabit URL [FILE]
|
10
|
+
|
11
|
+
URL http/https/ftp location of the file to download
|
12
|
+
FILE local path at which to save downloaded file
|
13
|
+
|
14
|
+
|
15
|
+
influential environment variables:
|
16
|
+
|
17
|
+
HTTP_PROXY url to http proxy
|
18
|
+
CA_CERT_FILE full path to CA certificate file
|
19
|
+
|
20
|
+
### CLI Usage Examples
|
21
|
+
|
22
|
+
cd %USERPROFILE%\Downloads
|
23
|
+
nabit http://pyyaml.org/download/libyaml/yaml-0.1.4.tar.gz
|
24
|
+
|
25
|
+
nabit http://sqlite.org/sqlite-autoconf-3071100.tar.gz c:\temp\sqlite-latest.tar.gz
|
26
|
+
|
27
|
+
nabit ftp://sourceware.org/pub/libffi/libffi-3.0.10.tar.gz "$HOME/downloads/libffi-3.0.10.tar.gz"
|
28
|
+
|
29
|
+
set CA_CERT_FILE=c:\tools\cacert.pem
|
30
|
+
nabit https://github.com/downloads/thecodeshop/ruby/w32time.7z c:/temp/w32time.7z
|
31
|
+
|
32
|
+
### Library Usage Example
|
33
|
+
|
34
|
+
require 'rubygems' # only for Ruby < 1.9
|
35
|
+
require 'nabit'
|
36
|
+
|
37
|
+
ENV['CA_CERT_FILE'] ||= '/etc/ssl/certs/ca-certificates.crt
|
38
|
+
target = File.expand_path('~/downloads/secret-codes-20120327.tar.xz')
|
39
|
+
source = 'https://unbreakable.area51.io/secret-codes.tar.xz'
|
40
|
+
|
41
|
+
nab = Nabit::Downloader.new
|
42
|
+
nab.download_file(source, target)
|
43
|
+
|
44
|
+
### Local Build and Install
|
45
|
+
|
46
|
+
git clone https://github.com/jonforums/nabit.git
|
47
|
+
cd nabit
|
48
|
+
rake package
|
49
|
+
gem install pkg/nabit-X.Y.Z.gem [--user-install]
|
50
|
+
|
51
|
+
## License
|
52
|
+
|
53
|
+
3-clause BSD. Portions from MiniPortile's HTTP implementation. See project LICENSE file.
|
54
|
+
|
55
|
+
## TODO
|
56
|
+
|
57
|
+
* investigate low-level, cross-platform socket tweaks
|
58
|
+
* add user/pw FTP support
|
59
|
+
* test proxy impl
|
60
|
+
* test with JRuby on Windows and Arch
|
data/Rakefile
ADDED
data/bin/nabit
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require 'nabit/version'
|
3
|
+
|
4
|
+
module Nabit
|
5
|
+
|
6
|
+
def self.usage
|
7
|
+
<<-EOU
|
8
|
+
|
9
|
+
HTTP/HTTPS/FTP File Downloader, v#{Nabit::VERSION}
|
10
|
+
Usage: nabit URL [FILE]
|
11
|
+
|
12
|
+
URL http/https/ftp location of the file to download
|
13
|
+
FILE local path at which to save downloaded file
|
14
|
+
|
15
|
+
|
16
|
+
influential environment variables:
|
17
|
+
|
18
|
+
HTTP_PROXY url to http proxy
|
19
|
+
CA_CERT_FILE full path to CA certificate file
|
20
|
+
|
21
|
+
EOU
|
22
|
+
end
|
23
|
+
|
24
|
+
class Runner
|
25
|
+
def self.run(args, options = {})
|
26
|
+
# XXX dodgy `File.basename` on URI for default filename, do post `URI.parse`?
|
27
|
+
file_name = args[1] ? File.expand_path(args[1]) :
|
28
|
+
File.expand_path(File.basename(args[0]))
|
29
|
+
|
30
|
+
Nabit::Downloader.new.download_file(args[0], file_name)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
|
36
|
+
_args = ARGV.dup
|
37
|
+
abort Nabit.usage unless (1..2).include?(_args.length)
|
38
|
+
|
39
|
+
require 'nabit'
|
40
|
+
Nabit::Runner.run(_args)
|
data/lib/nabit.rb
ADDED
@@ -0,0 +1,176 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'net/https' if RUBY_VERSION < '1.9'
|
3
|
+
require 'net/ftp'
|
4
|
+
require 'fileutils'
|
5
|
+
require 'tempfile'
|
6
|
+
|
7
|
+
module Nabit
|
8
|
+
|
9
|
+
class Downloader
|
10
|
+
|
11
|
+
attr_accessor :logger, :max_ca_verify_depth, :ftp_data_chunk_size
|
12
|
+
|
13
|
+
def initialize
|
14
|
+
@logger = STDOUT.binmode
|
15
|
+
@max_ca_verify_depth = 5
|
16
|
+
@ftp_data_chunk_size = 32768
|
17
|
+
end
|
18
|
+
|
19
|
+
def download_file(url, full_path, count = 3)
|
20
|
+
return if File.exists?(full_path)
|
21
|
+
|
22
|
+
uri = URI.parse(url)
|
23
|
+
case uri.scheme.downcase
|
24
|
+
when /ftp/
|
25
|
+
ftp_download(uri, full_path)
|
26
|
+
when /http|https/
|
27
|
+
http_download(url, full_path, count)
|
28
|
+
else
|
29
|
+
raise "\nERROR: Unknown URI scheme '#{uri.scheme}'"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
private
|
35
|
+
def message(text)
|
36
|
+
@logger.print text
|
37
|
+
@logger.flush
|
38
|
+
end
|
39
|
+
|
40
|
+
def output(text = '')
|
41
|
+
@logger.puts text
|
42
|
+
@logger.flush
|
43
|
+
end
|
44
|
+
|
45
|
+
def http_download(url, full_path, count)
|
46
|
+
|
47
|
+
begin
|
48
|
+
uri = URI.parse(url)
|
49
|
+
filename = File.basename(uri.path)
|
50
|
+
|
51
|
+
if ENV['HTTP_PROXY']
|
52
|
+
protocol, userinfo, proxy_host, proxy_port = URI::split(ENV['HTTP_PROXY'])
|
53
|
+
proxy_user, proxy_pass = userinfo.split(/:/) if userinfo
|
54
|
+
http = Net::HTTP.new(uri.host, uri.port, proxy_host, proxy_port, proxy_user, proxy_pass)
|
55
|
+
else
|
56
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
57
|
+
end
|
58
|
+
|
59
|
+
if uri.scheme.downcase == 'https'
|
60
|
+
http.use_ssl = true
|
61
|
+
if ENV['CA_CERT_FILE']
|
62
|
+
cert_file = ENV['CA_CERT_FILE'].dup
|
63
|
+
cert_file.gsub!(File::ALT_SEPARATOR, File::SEPARATOR) if File::ALT_SEPARATOR
|
64
|
+
end
|
65
|
+
if cert_file && File.exists?(cert_file)
|
66
|
+
http.ca_file = cert_file
|
67
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
68
|
+
http.verify_depth = @max_ca_verify_depth
|
69
|
+
else
|
70
|
+
raise <<-EOT
|
71
|
+
To download using HTTPS you must first set the CA_CERT_FILE
|
72
|
+
environment variable to the path of a valid CA certificate file.
|
73
|
+
A file of bundled public CA certs may be downloaded from:
|
74
|
+
|
75
|
+
http://curl.haxx.se/ca/cacert.pem
|
76
|
+
|
77
|
+
EOT
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
http.request_get(uri.path) do |response|
|
82
|
+
case response
|
83
|
+
when Net::HTTPNotFound
|
84
|
+
output "404 - Not Found"
|
85
|
+
return false
|
86
|
+
|
87
|
+
when Net::HTTPClientError
|
88
|
+
output "Error: Client Error: #{response.inspect}"
|
89
|
+
return false
|
90
|
+
|
91
|
+
when Net::HTTPRedirection
|
92
|
+
raise "Too many redirections for the original URL, halting." if count <= 0
|
93
|
+
url = response["location"]
|
94
|
+
return http_download(url, full_path, count - 1)
|
95
|
+
|
96
|
+
when Net::HTTPOK
|
97
|
+
temp_file = Tempfile.new("download-#{filename}")
|
98
|
+
temp_file.binmode
|
99
|
+
|
100
|
+
size = 0
|
101
|
+
progress = 0
|
102
|
+
total = response.header["Content-Length"].to_i
|
103
|
+
|
104
|
+
response.read_body do |chunk|
|
105
|
+
temp_file << chunk
|
106
|
+
size += chunk.size
|
107
|
+
new_progress = (size * 100) / total
|
108
|
+
unless new_progress == progress
|
109
|
+
message "\rDownloading %s (%3d%%) " % [filename, new_progress]
|
110
|
+
end
|
111
|
+
progress = new_progress
|
112
|
+
end
|
113
|
+
|
114
|
+
output
|
115
|
+
|
116
|
+
temp_file.close
|
117
|
+
File.unlink full_path if File.exists?(full_path)
|
118
|
+
FileUtils.mkdir_p File.dirname(full_path)
|
119
|
+
FileUtils.mv temp_file.path, full_path, :force => true
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
rescue Exception => e
|
124
|
+
File.unlink full_path if File.exists?(full_path)
|
125
|
+
output "ERROR: #{e.message}"
|
126
|
+
raise "Failed to download file"
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def ftp_download(parsed_uri, full_path)
|
131
|
+
filename = File.basename(parsed_uri.path)
|
132
|
+
|
133
|
+
begin
|
134
|
+
temp_file = Tempfile.new("download-#{filename}")
|
135
|
+
temp_file.binmode
|
136
|
+
|
137
|
+
size = 0
|
138
|
+
progress = 0
|
139
|
+
|
140
|
+
# TODO add user/pw support
|
141
|
+
Net::FTP.open(parsed_uri.host) do |ftp|
|
142
|
+
ftp.passive = true
|
143
|
+
ftp.login
|
144
|
+
remote_dir = File.dirname(parsed_uri.path)
|
145
|
+
ftp.chdir(remote_dir) unless remote_dir == '.'
|
146
|
+
|
147
|
+
total = ftp.size(filename)
|
148
|
+
|
149
|
+
ftp.getbinaryfile(filename, nil, @ftp_data_chunk_size) do |chunk|
|
150
|
+
temp_file << chunk
|
151
|
+
size += chunk.size
|
152
|
+
new_progress = (size * 100) / total
|
153
|
+
unless new_progress == progress
|
154
|
+
message "\rDownloading %s (%3d%%) " % [filename, new_progress]
|
155
|
+
end
|
156
|
+
progress = new_progress
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
output
|
161
|
+
|
162
|
+
temp_file.close
|
163
|
+
File.unlink full_path if File.exists?(full_path)
|
164
|
+
FileUtils.mkdir_p File.dirname(full_path)
|
165
|
+
FileUtils.mv temp_file.path, full_path, :force => true
|
166
|
+
|
167
|
+
rescue Exception => e
|
168
|
+
File.unlink full_path if File.exists?(full_path)
|
169
|
+
output "ERROR: #{e.message}"
|
170
|
+
raise "Failed to download file"
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
end
|
175
|
+
|
176
|
+
end
|
data/nabit.gemspec
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
require File.expand_path('../lib/nabit/version', __FILE__)
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = 'nabit'
|
5
|
+
s.version = Nabit::VERSION
|
6
|
+
s.platform = Gem::Platform::RUBY
|
7
|
+
|
8
|
+
s.summary = 'A pure-Ruby HTTP/HTTPS/FTP file downloader'
|
9
|
+
s.description = <<-EOT
|
10
|
+
A pure-Ruby, minimal dependency HTTP/HTTPS/FTP file downloading library
|
11
|
+
with a sparse CLI.
|
12
|
+
EOT
|
13
|
+
s.license = '3-clause BSD'
|
14
|
+
s.homepage = 'http://github.com/jonforums/nabit.git'
|
15
|
+
|
16
|
+
s.author = 'Jon Maken'
|
17
|
+
s.email = 'jon.forums@gmail.com'
|
18
|
+
|
19
|
+
s.files = FileList["lib/**/*.rb", "Rakefile", "nabit.gemspec", "LICENSE", "README.mkd"]
|
20
|
+
s.executable = 'nabit'
|
21
|
+
|
22
|
+
s.required_ruby_version = ">= 1.8.7"
|
23
|
+
s.required_rubygems_version = ">= 1.7.2"
|
24
|
+
end
|
metadata
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: nabit
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.2.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Jon Maken
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-04-19 00:00:00.000000000 Z
|
13
|
+
dependencies: []
|
14
|
+
description: ! 'A pure-Ruby, minimal dependency HTTP/HTTPS/FTP file downloading library
|
15
|
+
|
16
|
+
with a sparse CLI.
|
17
|
+
|
18
|
+
'
|
19
|
+
email: jon.forums@gmail.com
|
20
|
+
executables:
|
21
|
+
- nabit
|
22
|
+
extensions: []
|
23
|
+
extra_rdoc_files: []
|
24
|
+
files:
|
25
|
+
- lib/nabit/downloader.rb
|
26
|
+
- lib/nabit/version.rb
|
27
|
+
- lib/nabit.rb
|
28
|
+
- Rakefile
|
29
|
+
- nabit.gemspec
|
30
|
+
- LICENSE
|
31
|
+
- README.mkd
|
32
|
+
- bin/nabit
|
33
|
+
homepage: http://github.com/jonforums/nabit.git
|
34
|
+
licenses:
|
35
|
+
- 3-clause BSD
|
36
|
+
post_install_message:
|
37
|
+
rdoc_options: []
|
38
|
+
require_paths:
|
39
|
+
- lib
|
40
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 1.8.7
|
46
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
47
|
+
none: false
|
48
|
+
requirements:
|
49
|
+
- - ! '>='
|
50
|
+
- !ruby/object:Gem::Version
|
51
|
+
version: 1.7.2
|
52
|
+
requirements: []
|
53
|
+
rubyforge_project:
|
54
|
+
rubygems_version: 1.8.22
|
55
|
+
signing_key:
|
56
|
+
specification_version: 3
|
57
|
+
summary: A pure-Ruby HTTP/HTTPS/FTP file downloader
|
58
|
+
test_files: []
|