image-dumper 0.7.9 → 0.8

Sign up to get free protection for your applications and to get access to all the features.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/bin/dumper +27 -20
  3. data/lib/dumper/profiles/fakku.rb +5 -2
  4. data/lib/dumper/utils.rb +4 -4
  5. data/lib/dumper/version.rb +1 -1
  6. data/lib/dumper.rb +0 -1
  7. metadata +52 -38
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 540402d705c54203cf18d899ee74a5f9c4873920
4
- data.tar.gz: 102783104ecaa46a5623fbf4d4af9e0a09ecd625
3
+ metadata.gz: a271949bdf94df1ddf5301b00800dd2e98d6df07
4
+ data.tar.gz: b70bb0f5fa24a749f3d6a47c0177265fe10b9f94
5
5
  SHA512:
6
- metadata.gz: e600b4a39246d6471e7a58b9082cdadc58122e92746b889565f6d409812d175afb7bb9dd73509dcc808d0dd136c26827efa860fc6e161855b04a079d6c2e6af1
7
- data.tar.gz: 88679f502090a13679ee8884a3ef363a2b06fe8bac7eb2847ebb6d01e57459c8df6e59b7a01e73feef824f11835a24e7111ace88865418f62dcbd1ca8ce3b8d1
6
+ metadata.gz: fbc65f3877081c9d09ac8378b92f2776551818a8cd2bfac69007bb445b916b034005443fd95dc5415a66a18df3b10f55a242e98550628d5f32151d0cc7aa5147
7
+ data.tar.gz: 6b1b41c11742cdfa79d2c6f2cdfb9b33328e938fb98991b6243768521e49ff426361492205ea7c206415f73250a3e1ffffb1d72ab531954304301406a440a312
data/bin/dumper CHANGED
@@ -18,13 +18,14 @@
18
18
  # along with Dumper. If not, see <http://www.gnu.org/licenses/>.
19
19
  #++
20
20
  require 'dumper'
21
+ require 'optparse'
21
22
 
22
23
  options = {}
23
24
 
24
25
  OptionParser.new do |o|
25
26
  options[:url ] = []
26
27
  options[:path] = []
27
-
28
+
28
29
  o.on '-l', '--list', 'Show available profiles' do
29
30
  abort 'Profiles available:'.tap { |s|
30
31
  Dumper.list.sort { |a, b| a <=> b }.each { |p| s << "\n" + (' ' * 3) + p }
@@ -33,7 +34,7 @@ OptionParser.new do |o|
33
34
 
34
35
  o.on '-i', '--info PROFILE', 'Get info about profiles' do |profile|
35
36
  if Dumper.list.include? profile
36
- method = ("info_#{profile}").to_sym
37
+ method = ("info_#{profile}").to_sym
37
38
  Dumper::Profiles.send(method).tap { |i|
38
39
  puts "Option 'from' is #{i[:from] ? 'available' : 'not available'}."
39
40
  puts "Option 'to' is #{i[:to] ? 'available' : 'not available'}."
@@ -43,44 +44,44 @@ OptionParser.new do |o|
43
44
  abort 'Profile not found.'
44
45
  end
45
46
  end
46
-
47
+
47
48
  o.on '-u', '--url URL', 'Target URL' do |url|
48
49
  options[:url] << url
49
50
  end
50
-
51
+
51
52
  o.on '-f', '--from PAGE', 'Start to save from... (if allowed)' do |pages|
52
53
  options[:from] = pages.to_i
53
54
  end
54
-
55
+
55
56
  o.on '-t', '--to PAGE', 'Finish to save at... (if allowed)' do |pages|
56
57
  options[:to] = pages.to_i
57
58
  end
58
-
59
+
59
60
  o.on '-p', '--path PATH', 'Target folder' do |path|
60
61
  options[:path] << File.expand_path(path)
61
62
  end
62
-
63
+
63
64
  o.on '-x', '--xpath XPATH', 'Use a custom xpath' do |xpath|
64
65
  options[:xpath] = xpath
65
66
  end
66
-
67
+
67
68
  o.on '-r', '--profile PROFILE', 'Force the use of the given profile' do |profile|
68
69
  options[:profile] = profile
69
70
  end
70
-
71
+
71
72
  o.on '-t', '--threads MIN:MAX', 'Set the number of threads to use' do |threads|
72
73
  threads = threads.split ?:
73
74
  Dumper.pool_size = threads[0].to_i, threads[1].to_i
74
75
  end
75
-
76
+
76
77
  o.on '-m', '--silence', 'Print only important messages' do
77
78
  Dumper.shut_up!
78
79
  end
79
-
80
+
80
81
  o.on '-o', '--output ', 'Print the logs on the given file' do |file|
81
82
  Dumper::Logger.redirect_on :file, file
82
83
  end
83
-
84
+
84
85
  o.on '-s', '--file FILE', 'File containing in each line "URL||Folder"' do |file|
85
86
  file = File.open(file).read.gsub(/\r\n?/, "\n")
86
87
  file.each_line { |line|
@@ -103,21 +104,27 @@ options[:url].each_with_index { |url, i|
103
104
 
104
105
  host = options[:profile] || URI.parse(url).host.split(?.)[-2]
105
106
 
106
- if options[:xpath].nil? && Dumper.list.include?(host.gsub(?-, ?_))
107
- method = "get_#{host.gsub ?-, ?_}".to_sym
108
- case
109
- when options[:from] && options[:to] then Dumper::Profiles.send method, url, options[:path][i], options[:from], options[:to]
110
- when options[:from] then Dumper::Profiles.send method, url, options[:path][i], options[:from]
111
- else Dumper::Profiles.send method, url, options[:path][i]
107
+ if Dumper.list.include?(host.gsub(?-, ?_))
108
+ if options[:xpath].nil?
109
+ method = "get_#{host.gsub ?-, ?_}".to_sym
110
+
111
+ case
112
+ when options[:from] && options[:to] then Dumper::Profiles.send method, url, options[:path][i], options[:from], options[:to]
113
+ when options[:from] then Dumper::Profiles.send method, url, options[:path][i], options[:from]
114
+ else Dumper::Profiles.send method, url, options[:path][i]
115
+ end
116
+ else
117
+ Dumper.get_generic url, options[:path][i], options[:xpath]
112
118
  end
113
119
  else
114
- Dumper.get_generic url, options[:path][i], options[:xpath]
120
+ puts 'Cannot find any profile for your source.'
121
+ puts 'Consider using `--profile` or `--xpath`.'
115
122
  end
116
123
  rescue Nokogiri::XML::XPath::SyntaxError => e
117
124
  puts e.to_s.gsub /expression/, 'xpath'
118
125
  puts 'Cannot dump.'
119
126
  rescue OpenURI::HTTPError => e
120
- puts "Error opening #{url}: #{e}"
127
+ puts "Error opening #{url}: #{e}"
121
128
  rescue URI::InvalidURIError => e
122
129
  puts "URL #{url} is not valid: #{e}"
123
130
  end
data/lib/dumper/profiles/fakku.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  #--
2
- # Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
2
+ # Copyright(C) 2015 Giovanni Capuano <webmaster@giovannicapuano.net>
3
3
  #
4
4
  # This file is part of Dumper.
5
5
  #
@@ -25,8 +25,11 @@ module Dumper
25
25
  url += '/read' unless url.end_with? '/read'
26
26
  errors = 0
27
27
 
28
- cdn = open(url).read.split('window.params.thumbs')[1].split('\/thumbs\/')[0].gsub(/\\\//m, ?/)[5..-1] + '/images/'
28
+ js_thumbs_slice = open(url).read.split('window.params.thumbs')[1].split('\/thumbs\/')
29
+ thumbs_count = js_thumbs_slice[1..-1].count
30
+ cdn = js_thumbs_slice[0].gsub(/\\\//m, ?/)[5..-1] + '/images/'
29
31
 
32
+ to = thumbs_count if to > thumbs_count
30
33
  from.upto(to) { |i|
31
34
  return if errors == 3
32
35
 
data/lib/dumper/utils.rb CHANGED
@@ -21,7 +21,7 @@ class String
21
21
  def numeric?
22
22
  self.to_i.to_s == self || self.to_f.to_s == self
23
23
  end
24
-
24
+
25
25
  def sanitize_filename
26
26
  self.split(/(?<=.)\.(?=[^.])(?!.*\.[^.])/m).map { |s| s.gsub /[^a-z0-9\-]+/i, ?_ }.join(?.)
27
27
  end
@@ -30,12 +30,12 @@ end
30
30
  class NilClass
31
31
  def numeric?
32
32
  false
33
- end
33
+ end
34
34
  end
35
35
 
36
36
  class URI::Parser
37
- def split url
37
+ def split(url)
38
38
  a = Addressable::URI::parse url
39
39
  [a.scheme, a.userinfo, a.host, a.port, nil, a.path, nil, a.query, a.fragment]
40
40
  end
41
- end
41
+ end
data/lib/dumper/version.rb CHANGED
@@ -19,6 +19,6 @@
19
19
 
20
20
  module Dumper
21
21
  def self.version
22
- '0.7.9'
22
+ '0.8'
23
23
  end
24
24
  end
data/lib/dumper.rb CHANGED
@@ -20,7 +20,6 @@
20
20
  require 'open-uri'
21
21
  require 'net/http'
22
22
  require 'uri'
23
- require 'optparse'
24
23
  require 'base64'
25
24
  require 'nokogiri'
26
25
  require 'openssl'
metadata CHANGED
@@ -1,169 +1,183 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: image-dumper
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.9
4
+ version: '0.8'
5
5
  platform: ruby
6
6
  authors:
7
7
  - Giovanni Capuano
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-04-16 00:00:00.000000000 Z
11
+ date: 2015-06-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
+ version_requirements: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - "~>"
17
+ - !ruby/object:Gem::Version
18
+ version: '1.6'
14
19
  name: nokogiri
20
+ prerelease: false
15
21
  requirement: !ruby/object:Gem::Requirement
16
22
  requirements:
17
23
  - - "~>"
18
24
  - !ruby/object:Gem::Version
19
25
  version: '1.6'
20
26
  type: :runtime
21
- prerelease: false
27
+ - !ruby/object:Gem::Dependency
22
28
  version_requirements: !ruby/object:Gem::Requirement
23
29
  requirements:
24
30
  - - "~>"
25
31
  - !ruby/object:Gem::Version
26
- version: '1.6'
27
- - !ruby/object:Gem::Dependency
32
+ version: '1.0'
28
33
  name: certified
34
+ prerelease: false
29
35
  requirement: !ruby/object:Gem::Requirement
30
36
  requirements:
31
37
  - - "~>"
32
38
  - !ruby/object:Gem::Version
33
39
  version: '1.0'
34
40
  type: :runtime
35
- prerelease: false
41
+ - !ruby/object:Gem::Dependency
36
42
  version_requirements: !ruby/object:Gem::Requirement
37
43
  requirements:
38
44
  - - "~>"
39
45
  - !ruby/object:Gem::Version
40
- version: '1.0'
41
- - !ruby/object:Gem::Dependency
46
+ version: '2.3'
42
47
  name: addressable
48
+ prerelease: false
43
49
  requirement: !ruby/object:Gem::Requirement
44
50
  requirements:
45
51
  - - "~>"
46
52
  - !ruby/object:Gem::Version
47
53
  version: '2.3'
48
54
  type: :runtime
49
- prerelease: false
55
+ - !ruby/object:Gem::Dependency
50
56
  version_requirements: !ruby/object:Gem::Requirement
51
57
  requirements:
52
58
  - - "~>"
53
59
  - !ruby/object:Gem::Version
54
- version: '2.3'
55
- - !ruby/object:Gem::Dependency
60
+ version: '1.0'
56
61
  name: iconv
62
+ prerelease: false
57
63
  requirement: !ruby/object:Gem::Requirement
58
64
  requirements:
59
65
  - - "~>"
60
66
  - !ruby/object:Gem::Version
61
67
  version: '1.0'
62
68
  type: :runtime
63
- prerelease: false
69
+ - !ruby/object:Gem::Dependency
64
70
  version_requirements: !ruby/object:Gem::Requirement
65
71
  requirements:
66
72
  - - "~>"
67
73
  - !ruby/object:Gem::Version
68
- version: '1.0'
69
- - !ruby/object:Gem::Dependency
74
+ version: '1.8'
70
75
  name: json
76
+ prerelease: false
71
77
  requirement: !ruby/object:Gem::Requirement
72
78
  requirements:
73
79
  - - "~>"
74
80
  - !ruby/object:Gem::Version
75
81
  version: '1.8'
76
82
  type: :runtime
77
- prerelease: false
83
+ - !ruby/object:Gem::Dependency
78
84
  version_requirements: !ruby/object:Gem::Requirement
79
85
  requirements:
80
86
  - - "~>"
81
87
  - !ruby/object:Gem::Version
82
- version: '1.8'
83
- - !ruby/object:Gem::Dependency
88
+ version: '0.1'
84
89
  name: thread
90
+ prerelease: false
85
91
  requirement: !ruby/object:Gem::Requirement
86
92
  requirements:
87
93
  - - "~>"
88
94
  - !ruby/object:Gem::Version
89
95
  version: '0.1'
90
96
  type: :runtime
91
- prerelease: false
97
+ - !ruby/object:Gem::Dependency
92
98
  version_requirements: !ruby/object:Gem::Requirement
93
99
  requirements:
94
100
  - - "~>"
95
101
  - !ruby/object:Gem::Version
96
- version: '0.1'
102
+ version: '1.1'
103
+ name: rainbow
104
+ prerelease: false
105
+ requirement: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - "~>"
108
+ - !ruby/object:Gem::Version
109
+ version: '1.1'
110
+ type: :runtime
97
111
  - !ruby/object:Gem::Dependency
112
+ version_requirements: !ruby/object:Gem::Requirement
113
+ requirements:
114
+ - - "~>"
115
+ - !ruby/object:Gem::Version
116
+ version: '0.4'
98
117
  name: alakazam
118
+ prerelease: false
99
119
  requirement: !ruby/object:Gem::Requirement
100
120
  requirements:
101
121
  - - "~>"
102
122
  - !ruby/object:Gem::Version
103
123
  version: '0.4'
104
124
  type: :runtime
105
- prerelease: false
125
+ - !ruby/object:Gem::Dependency
106
126
  version_requirements: !ruby/object:Gem::Requirement
107
127
  requirements:
108
128
  - - "~>"
109
129
  - !ruby/object:Gem::Version
110
- version: '0.4'
111
- - !ruby/object:Gem::Dependency
130
+ version: '10.3'
112
131
  name: rake
132
+ prerelease: false
113
133
  requirement: !ruby/object:Gem::Requirement
114
134
  requirements:
115
135
  - - "~>"
116
136
  - !ruby/object:Gem::Version
117
137
  version: '10.3'
118
138
  type: :development
119
- prerelease: false
139
+ - !ruby/object:Gem::Dependency
120
140
  version_requirements: !ruby/object:Gem::Requirement
121
141
  requirements:
122
142
  - - "~>"
123
143
  - !ruby/object:Gem::Version
124
- version: '10.3'
125
- - !ruby/object:Gem::Dependency
144
+ version: '3.1'
126
145
  name: rspec
146
+ prerelease: false
127
147
  requirement: !ruby/object:Gem::Requirement
128
148
  requirements:
129
149
  - - "~>"
130
150
  - !ruby/object:Gem::Version
131
151
  version: '3.1'
132
152
  type: :development
133
- prerelease: false
153
+ - !ruby/object:Gem::Dependency
134
154
  version_requirements: !ruby/object:Gem::Requirement
135
155
  requirements:
136
156
  - - "~>"
137
157
  - !ruby/object:Gem::Version
138
- version: '3.1'
139
- - !ruby/object:Gem::Dependency
158
+ version: '1.6'
140
159
  name: fastimage
160
+ prerelease: false
141
161
  requirement: !ruby/object:Gem::Requirement
142
162
  requirements:
143
163
  - - "~>"
144
164
  - !ruby/object:Gem::Version
145
165
  version: '1.6'
146
166
  type: :development
147
- prerelease: false
167
+ - !ruby/object:Gem::Dependency
148
168
  version_requirements: !ruby/object:Gem::Requirement
149
169
  requirements:
150
170
  - - "~>"
151
171
  - !ruby/object:Gem::Version
152
- version: '1.6'
153
- - !ruby/object:Gem::Dependency
172
+ version: '1.0'
154
173
  name: parallel_tests
174
+ prerelease: false
155
175
  requirement: !ruby/object:Gem::Requirement
156
176
  requirements:
157
177
  - - "~>"
158
178
  - !ruby/object:Gem::Version
159
179
  version: '1.0'
160
180
  type: :development
161
- prerelease: false
162
- version_requirements: !ruby/object:Gem::Requirement
163
- requirements:
164
- - - "~>"
165
- - !ruby/object:Gem::Version
166
- version: '1.0'
167
181
  description: A dumper to download whole galleries from board like 4chan, imagebam,
168
182
  mangaeden, deviantart, etc.
169
183
  email: webmaster@giovannicapuano.net
@@ -232,7 +246,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
232
246
  version: '0'
233
247
  requirements: []
234
248
  rubyforge_project:
235
- rubygems_version: 2.2.2
249
+ rubygems_version: 2.4.8
236
250
  signing_key:
237
251
  specification_version: 4
238
252
  summary: Fetch and download image gallery.