puushload 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/puushload +17 -0
- data/lib/puushload/puush_downloader.rb +40 -0
- data/lib/puushload/puush_helper.rb +9 -0
- data/lib/puushload/puush_id_generator.rb +30 -0
- data/lib/puushload.rb +77 -0
- metadata +50 -0
data/bin/puushload
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'puushload'
|
4
|
+
gem 'trollop'
|
5
|
+
|
6
|
+
require 'trollop'
|
7
|
+
|
8
|
+
opts = Trollop::options do
|
9
|
+
opt :output_dir, "Output-Directory", :short => 'o', :default => "output", :type => String
|
10
|
+
opt :log_file, "Logile-Path", :short => 'l', :default => "STDOUT", :type => String
|
11
|
+
end
|
12
|
+
|
13
|
+
logger = Logger.new(opts[:log_file], 10, 1024000) if opts[:log_file] != "STDOUT"
|
14
|
+
|
15
|
+
#PuushDownloader.download_file('12345', File.absolute_path("test"))
|
16
|
+
puush_load = PuushLoad.new(opts[:output_dir], logger)
|
17
|
+
puush_load.download_files
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
class PuushDownloader
|
5
|
+
MAX_RETRIES = 5
|
6
|
+
|
7
|
+
def self.download_file(id, subfolder, logger, retries=0)
|
8
|
+
logger.error "Too many retries, this file doesn't seem to exist." and return false if retries > MAX_RETRIES
|
9
|
+
begin
|
10
|
+
Net::HTTP.start("puu.sh") do |http|
|
11
|
+
http.request_get('/' + URI.encode(id)) do |response|
|
12
|
+
filename = PuushHelper.get_file_name_from_response(response)
|
13
|
+
return false if !filename
|
14
|
+
|
15
|
+
filename = id + " - " + filename
|
16
|
+
path = subfolder + "/" + filename
|
17
|
+
|
18
|
+
return filename if File.exists?(path)
|
19
|
+
|
20
|
+
FileUtils.mkdir_p(subfolder)
|
21
|
+
FileUtils.touch(path)
|
22
|
+
|
23
|
+
file = open path, 'wb'
|
24
|
+
|
25
|
+
response.read_body do |segment|
|
26
|
+
file.write(segment)
|
27
|
+
end
|
28
|
+
|
29
|
+
return filename
|
30
|
+
end
|
31
|
+
end
|
32
|
+
rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError, Errno::EHOSTUNREACH, SocketError,
|
33
|
+
Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError => e
|
34
|
+
|
35
|
+
logger.warn "Connection error, retrying (#{retries})...: " + e.to_s
|
36
|
+
self.download_file(id, subfolder, logger, retries+1)
|
37
|
+
#return false
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
class PuushIDGenerator
|
2
|
+
CHARS = ("0".."9").to_a + ("a".."z").to_a + ("A".."Z").to_a
|
3
|
+
|
4
|
+
def self.generate(&block)
|
5
|
+
|
6
|
+
# this is really fuckin dirty atm, but you may optimize this if you want to
|
7
|
+
|
8
|
+
# Example-ID: 12345
|
9
|
+
|
10
|
+
# first char (1) Only Numbers at the moment. Should take a bit until chars are being used here
|
11
|
+
("1".."9").each do |char1|
|
12
|
+
# second char (2)
|
13
|
+
CHARS.each do |char2|
|
14
|
+
# third char (3)
|
15
|
+
CHARS.each do |char3|
|
16
|
+
# fourth char (4)
|
17
|
+
CHARS.each do |char4|
|
18
|
+
# fifth char (5)
|
19
|
+
CHARS.each do |char5|
|
20
|
+
# combine se IDs
|
21
|
+
id = char1 + char2 + char3 + char4 + char5
|
22
|
+
# call se block
|
23
|
+
block.call(id)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/lib/puushload.rb
ADDED
@@ -0,0 +1,77 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
|
3
|
+
gem "archive-tar-minitar", "~> 0.5.2"
|
4
|
+
|
5
|
+
require 'puushload/puush_downloader'
|
6
|
+
require 'puushload/puush_helper'
|
7
|
+
require 'puushload/puush_id_generator'
|
8
|
+
|
9
|
+
require 'zip/zip'
|
10
|
+
|
11
|
+
require 'logger'
|
12
|
+
require 'tmpdir'
|
13
|
+
|
14
|
+
class PuushLoad
|
15
|
+
MAX_THREADS = 50
|
16
|
+
ARCHIVE_SIZE = 1000
|
17
|
+
|
18
|
+
def initialize(outputFolder, logger)
|
19
|
+
@logger = logger
|
20
|
+
@logger = Logger.new(STDOUT) if @logger.nil?
|
21
|
+
@logger.level = Logger::INFO
|
22
|
+
@output_folder = File.absolute_path(outputFolder)
|
23
|
+
end
|
24
|
+
|
25
|
+
def download_files
|
26
|
+
@logger.info "Starting download!"
|
27
|
+
|
28
|
+
spawned_threads = []
|
29
|
+
downloaded_files = 0
|
30
|
+
archives = 0
|
31
|
+
current_tmp_dir = Dir.mktmpdir
|
32
|
+
|
33
|
+
FileUtils.mkdir_p(@output_folder)
|
34
|
+
|
35
|
+
PuushIDGenerator.generate do |id|
|
36
|
+
|
37
|
+
spawned_threads << Thread.new {
|
38
|
+
PuushDownloader.download_file(id, current_tmp_dir, @logger)
|
39
|
+
downloaded_files += 1
|
40
|
+
}
|
41
|
+
|
42
|
+
if spawned_threads.length >= MAX_THREADS
|
43
|
+
spawned_threads.each { |spawnedThread| spawnedThread.join }
|
44
|
+
spawned_threads.clear
|
45
|
+
@logger.info "Killed some threads. #{downloaded_files} Files so far!"
|
46
|
+
|
47
|
+
if downloaded_files % ARCHIVE_SIZE == 0 && downloaded_files > 0 then
|
48
|
+
make_archive((@output_folder + "/" + archive_name(archives.to_s)), (current_tmp_dir))
|
49
|
+
|
50
|
+
@logger.info "Created archive with id: #{archives} !"
|
51
|
+
|
52
|
+
current_tmp_dir = Dir.mktmpdir
|
53
|
+
archives += 1
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
@logger.info "Actually, this is impossible, but you finished it!"
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
def archive_name(archives)
|
63
|
+
"puush" + archives + ".zip"
|
64
|
+
end
|
65
|
+
|
66
|
+
def make_archive(out_file, in_folder)
|
67
|
+
files = Dir.entries(in_folder)
|
68
|
+
|
69
|
+
Zip::ZipFile.open(out_file, Zip::ZipFile::CREATE) do |zipfile|
|
70
|
+
files.each do |filename|
|
71
|
+
zipfile.add(filename, in_folder + '/' + filename)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
FileUtils.remove_entry_secure in_folder
|
76
|
+
end
|
77
|
+
end
|
metadata
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: puushload
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Jan-Henrik Bruhn
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-02-11 00:00:00.000000000 Z
|
13
|
+
dependencies: []
|
14
|
+
description:
|
15
|
+
email: jh.bruhn@me.com
|
16
|
+
executables:
|
17
|
+
- puushload
|
18
|
+
extensions: []
|
19
|
+
extra_rdoc_files: []
|
20
|
+
files:
|
21
|
+
- lib/puushload.rb
|
22
|
+
- lib/puushload/puush_downloader.rb
|
23
|
+
- lib/puushload/puush_helper.rb
|
24
|
+
- lib/puushload/puush_id_generator.rb
|
25
|
+
- bin/puushload
|
26
|
+
homepage:
|
27
|
+
licenses: []
|
28
|
+
post_install_message:
|
29
|
+
rdoc_options: []
|
30
|
+
require_paths:
|
31
|
+
- lib
|
32
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ! '>='
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
44
|
+
requirements: []
|
45
|
+
rubyforge_project:
|
46
|
+
rubygems_version: 1.8.25
|
47
|
+
signing_key:
|
48
|
+
specification_version: 3
|
49
|
+
summary: Download all files from Puush.me!
|
50
|
+
test_files: []
|