puushload 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/puushload.rb +22 -12
- metadata +1 -1
data/lib/puushload.rb
CHANGED
|
@@ -9,6 +9,9 @@ require 'zip/zip'
|
|
|
9
9
|
require 'logger'
|
|
10
10
|
require 'tmpdir'
|
|
11
11
|
|
|
12
|
+
Zip.options[:continue_on_exists_proc] = true
|
|
13
|
+
Zip.options[:on_exists_proc] = true
|
|
14
|
+
|
|
12
15
|
class PuushLoad
|
|
13
16
|
MAX_THREADS = 20
|
|
14
17
|
ARCHIVE_SIZE = 1000
|
|
@@ -27,29 +30,35 @@ class PuushLoad
|
|
|
27
30
|
downloaded_files = 0
|
|
28
31
|
archives = 0
|
|
29
32
|
current_tmp_dir = Dir.mktmpdir
|
|
30
|
-
|
|
33
|
+
archive_file_counter = 0
|
|
31
34
|
FileUtils.mkdir_p(@output_folder)
|
|
32
35
|
|
|
33
36
|
PuushIDGenerator.generate do |id|
|
|
34
37
|
spawned_threads << Thread.new {
|
|
35
|
-
|
|
38
|
+
if PuushDownloader.download_file(id, current_tmp_dir, @logger) then
|
|
39
|
+
downloaded_files += 1
|
|
40
|
+
archive_file_counter += 1
|
|
41
|
+
end
|
|
42
|
+
|
|
36
43
|
}
|
|
37
|
-
|
|
38
|
-
if spawned_threads.length >= MAX_THREADS
|
|
44
|
+
|
|
45
|
+
if spawned_threads.length >= MAX_THREADS then
|
|
39
46
|
spawned_threads.each { |spawnedThread| spawnedThread.join }
|
|
40
47
|
spawned_threads.clear
|
|
41
48
|
@logger.info "Killed some threads. #{downloaded_files} Files so far!"
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
if archive_file_counter >= ARCHIVE_SIZE then
|
|
52
|
+
archive_file_counter = 0
|
|
53
|
+
make_archive((@output_folder + "/" + archive_name(archives.to_s)), (current_tmp_dir))
|
|
47
54
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
55
|
+
@logger.info "Created archive with id: #{archives} !"
|
|
56
|
+
current_tmp_dir = Dir.mktmpdir
|
|
57
|
+
archives += 1
|
|
51
58
|
end
|
|
52
59
|
|
|
60
|
+
sleep 0.01
|
|
61
|
+
|
|
53
62
|
end
|
|
54
63
|
@logger.info "Actually, this is impossible, but you finished it!"
|
|
55
64
|
end
|
|
@@ -59,6 +68,7 @@ class PuushLoad
|
|
|
59
68
|
"puush" + archives + ".zip"
|
|
60
69
|
end
|
|
61
70
|
|
|
71
|
+
|
|
62
72
|
def make_archive(out_file, in_folder)
|
|
63
73
|
files = Dir.entries(in_folder)
|
|
64
74
|
|