image-dumper 0.7.9 → 0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/bin/dumper +27 -20
  3. data/lib/dumper/profiles/fakku.rb +5 -2
  4. data/lib/dumper/utils.rb +4 -4
  5. data/lib/dumper/version.rb +1 -1
  6. data/lib/dumper.rb +0 -1
  7. metadata +52 -38
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 540402d705c54203cf18d899ee74a5f9c4873920
4
- data.tar.gz: 102783104ecaa46a5623fbf4d4af9e0a09ecd625
3
+ metadata.gz: a271949bdf94df1ddf5301b00800dd2e98d6df07
4
+ data.tar.gz: b70bb0f5fa24a749f3d6a47c0177265fe10b9f94
5
5
  SHA512:
6
- metadata.gz: e600b4a39246d6471e7a58b9082cdadc58122e92746b889565f6d409812d175afb7bb9dd73509dcc808d0dd136c26827efa860fc6e161855b04a079d6c2e6af1
7
- data.tar.gz: 88679f502090a13679ee8884a3ef363a2b06fe8bac7eb2847ebb6d01e57459c8df6e59b7a01e73feef824f11835a24e7111ace88865418f62dcbd1ca8ce3b8d1
6
+ metadata.gz: fbc65f3877081c9d09ac8378b92f2776551818a8cd2bfac69007bb445b916b034005443fd95dc5415a66a18df3b10f55a242e98550628d5f32151d0cc7aa5147
7
+ data.tar.gz: 6b1b41c11742cdfa79d2c6f2cdfb9b33328e938fb98991b6243768521e49ff426361492205ea7c206415f73250a3e1ffffb1d72ab531954304301406a440a312
data/bin/dumper CHANGED
@@ -18,13 +18,14 @@
18
18
  # along with Dumper. If not, see <http://www.gnu.org/licenses/>.
19
19
  #++
20
20
  require 'dumper'
21
+ require 'optparse'
21
22
 
22
23
  options = {}
23
24
 
24
25
  OptionParser.new do |o|
25
26
  options[:url ] = []
26
27
  options[:path] = []
27
-
28
+
28
29
  o.on '-l', '--list', 'Show available profiles' do
29
30
  abort 'Profiles available:'.tap { |s|
30
31
  Dumper.list.sort { |a, b| a <=> b }.each { |p| s << "\n" + (' ' * 3) + p }
@@ -33,7 +34,7 @@ OptionParser.new do |o|
33
34
 
34
35
  o.on '-i', '--info PROFILE', 'Get info about profiles' do |profile|
35
36
  if Dumper.list.include? profile
36
- method = ("info_#{profile}").to_sym
37
+ method = ("info_#{profile}").to_sym
37
38
  Dumper::Profiles.send(method).tap { |i|
38
39
  puts "Option 'from' is #{i[:from] ? 'available' : 'not available'}."
39
40
  puts "Option 'to' is #{i[:to] ? 'available' : 'not available'}."
@@ -43,44 +44,44 @@ OptionParser.new do |o|
43
44
  abort 'Profile not found.'
44
45
  end
45
46
  end
46
-
47
+
47
48
  o.on '-u', '--url URL', 'Target URL' do |url|
48
49
  options[:url] << url
49
50
  end
50
-
51
+
51
52
  o.on '-f', '--from PAGE', 'Start to save from... (if allowed)' do |pages|
52
53
  options[:from] = pages.to_i
53
54
  end
54
-
55
+
55
56
  o.on '-t', '--to PAGE', 'Finish to save at... (if allowed)' do |pages|
56
57
  options[:to] = pages.to_i
57
58
  end
58
-
59
+
59
60
  o.on '-p', '--path PATH', 'Target folder' do |path|
60
61
  options[:path] << File.expand_path(path)
61
62
  end
62
-
63
+
63
64
  o.on '-x', '--xpath XPATH', 'Use a custom xpath' do |xpath|
64
65
  options[:xpath] = xpath
65
66
  end
66
-
67
+
67
68
  o.on '-r', '--profile PROFILE', 'Force the use of the given profile' do |profile|
68
69
  options[:profile] = profile
69
70
  end
70
-
71
+
71
72
  o.on '-t', '--threads MIN:MAX', 'Set the number of threads to use' do |threads|
72
73
  threads = threads.split ?:
73
74
  Dumper.pool_size = threads[0].to_i, threads[1].to_i
74
75
  end
75
-
76
+
76
77
  o.on '-m', '--silence', 'Print only important messages' do
77
78
  Dumper.shut_up!
78
79
  end
79
-
80
+
80
81
  o.on '-o', '--output ', 'Print the logs on the given file' do |file|
81
82
  Dumper::Logger.redirect_on :file, file
82
83
  end
83
-
84
+
84
85
  o.on '-s', '--file FILE', 'File containing in each line "URL||Folder"' do |file|
85
86
  file = File.open(file).read.gsub(/\r\n?/, "\n")
86
87
  file.each_line { |line|
@@ -103,21 +104,27 @@ options[:url].each_with_index { |url, i|
103
104
 
104
105
  host = options[:profile] || URI.parse(url).host.split(?.)[-2]
105
106
 
106
- if options[:xpath].nil? && Dumper.list.include?(host.gsub(?-, ?_))
107
- method = "get_#{host.gsub ?-, ?_}".to_sym
108
- case
109
- when options[:from] && options[:to] then Dumper::Profiles.send method, url, options[:path][i], options[:from], options[:to]
110
- when options[:from] then Dumper::Profiles.send method, url, options[:path][i], options[:from]
111
- else Dumper::Profiles.send method, url, options[:path][i]
107
+ if Dumper.list.include?(host.gsub(?-, ?_))
108
+ if options[:xpath].nil?
109
+ method = "get_#{host.gsub ?-, ?_}".to_sym
110
+
111
+ case
112
+ when options[:from] && options[:to] then Dumper::Profiles.send method, url, options[:path][i], options[:from], options[:to]
113
+ when options[:from] then Dumper::Profiles.send method, url, options[:path][i], options[:from]
114
+ else Dumper::Profiles.send method, url, options[:path][i]
115
+ end
116
+ else
117
+ Dumper.get_generic url, options[:path][i], options[:xpath]
112
118
  end
113
119
  else
114
- Dumper.get_generic url, options[:path][i], options[:xpath]
120
+ puts 'Cannot find any profile for your source.'
121
+ puts 'Consider using `--profile` or `--xpath`.'
115
122
  end
116
123
  rescue Nokogiri::XML::XPath::SyntaxError => e
117
124
  puts e.to_s.gsub /expression/, 'xpath'
118
125
  puts 'Cannot dump.'
119
126
  rescue OpenURI::HTTPError => e
120
- puts "Error opening #{url}: #{e}"
127
+ puts "Error opening #{url}: #{e}"
121
128
  rescue URI::InvalidURIError => e
122
129
  puts "URL #{url} is not valid: #{e}"
123
130
  end
data/lib/dumper/profiles/fakku.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  #--
2
- # Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
2
+ # Copyright(C) 2015 Giovanni Capuano <webmaster@giovannicapuano.net>
3
3
  #
4
4
  # This file is part of Dumper.
5
5
  #
@@ -25,8 +25,11 @@ module Dumper
25
25
  url += '/read' unless url.end_with? '/read'
26
26
  errors = 0
27
27
 
28
- cdn = open(url).read.split('window.params.thumbs')[1].split('\/thumbs\/')[0].gsub(/\\\//m, ?/)[5..-1] + '/images/'
28
+ js_thumbs_slice = open(url).read.split('window.params.thumbs')[1].split('\/thumbs\/')
29
+ thumbs_count = js_thumbs_slice[1..-1].count
30
+ cdn = js_thumbs_slice[0].gsub(/\\\//m, ?/)[5..-1] + '/images/'
29
31
 
32
+ to = thumbs_count if to > thumbs_count
30
33
  from.upto(to) { |i|
31
34
  return if errors == 3
32
35
 
data/lib/dumper/utils.rb CHANGED
@@ -21,7 +21,7 @@ class String
21
21
  def numeric?
22
22
  self.to_i.to_s == self || self.to_f.to_s == self
23
23
  end
24
-
24
+
25
25
  def sanitize_filename
26
26
  self.split(/(?<=.)\.(?=[^.])(?!.*\.[^.])/m).map { |s| s.gsub /[^a-z0-9\-]+/i, ?_ }.join(?.)
27
27
  end
@@ -30,12 +30,12 @@ end
30
30
  class NilClass
31
31
  def numeric?
32
32
  false
33
- end
33
+ end
34
34
  end
35
35
 
36
36
  class URI::Parser
37
- def split url
37
+ def split(url)
38
38
  a = Addressable::URI::parse url
39
39
  [a.scheme, a.userinfo, a.host, a.port, nil, a.path, nil, a.query, a.fragment]
40
40
  end
41
- end
41
+ end
data/lib/dumper/version.rb CHANGED
@@ -19,6 +19,6 @@
19
19
 
20
20
  module Dumper
21
21
  def self.version
22
- '0.7.9'
22
+ '0.8'
23
23
  end
24
24
  end
data/lib/dumper.rb CHANGED
@@ -20,7 +20,6 @@
20
20
  require 'open-uri'
21
21
  require 'net/http'
22
22
  require 'uri'
23
- require 'optparse'
24
23
  require 'base64'
25
24
  require 'nokogiri'
26
25
  require 'openssl'
metadata CHANGED
@@ -1,169 +1,183 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: image-dumper
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.9
4
+ version: '0.8'
5
5
  platform: ruby
6
6
  authors:
7
7
  - Giovanni Capuano
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-04-16 00:00:00.000000000 Z
11
+ date: 2015-06-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
+ version_requirements: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - "~>"
17
+ - !ruby/object:Gem::Version
18
+ version: '1.6'
14
19
  name: nokogiri
20
+ prerelease: false
15
21
  requirement: !ruby/object:Gem::Requirement
16
22
  requirements:
17
23
  - - "~>"
18
24
  - !ruby/object:Gem::Version
19
25
  version: '1.6'
20
26
  type: :runtime
21
- prerelease: false
27
+ - !ruby/object:Gem::Dependency
22
28
  version_requirements: !ruby/object:Gem::Requirement
23
29
  requirements:
24
30
  - - "~>"
25
31
  - !ruby/object:Gem::Version
26
- version: '1.6'
27
- - !ruby/object:Gem::Dependency
32
+ version: '1.0'
28
33
  name: certified
34
+ prerelease: false
29
35
  requirement: !ruby/object:Gem::Requirement
30
36
  requirements:
31
37
  - - "~>"
32
38
  - !ruby/object:Gem::Version
33
39
  version: '1.0'
34
40
  type: :runtime
35
- prerelease: false
41
+ - !ruby/object:Gem::Dependency
36
42
  version_requirements: !ruby/object:Gem::Requirement
37
43
  requirements:
38
44
  - - "~>"
39
45
  - !ruby/object:Gem::Version
40
- version: '1.0'
41
- - !ruby/object:Gem::Dependency
46
+ version: '2.3'
42
47
  name: addressable
48
+ prerelease: false
43
49
  requirement: !ruby/object:Gem::Requirement
44
50
  requirements:
45
51
  - - "~>"
46
52
  - !ruby/object:Gem::Version
47
53
  version: '2.3'
48
54
  type: :runtime
49
- prerelease: false
55
+ - !ruby/object:Gem::Dependency
50
56
  version_requirements: !ruby/object:Gem::Requirement
51
57
  requirements:
52
58
  - - "~>"
53
59
  - !ruby/object:Gem::Version
54
- version: '2.3'
55
- - !ruby/object:Gem::Dependency
60
+ version: '1.0'
56
61
  name: iconv
62
+ prerelease: false
57
63
  requirement: !ruby/object:Gem::Requirement
58
64
  requirements:
59
65
  - - "~>"
60
66
  - !ruby/object:Gem::Version
61
67
  version: '1.0'
62
68
  type: :runtime
63
- prerelease: false
69
+ - !ruby/object:Gem::Dependency
64
70
  version_requirements: !ruby/object:Gem::Requirement
65
71
  requirements:
66
72
  - - "~>"
67
73
  - !ruby/object:Gem::Version
68
- version: '1.0'
69
- - !ruby/object:Gem::Dependency
74
+ version: '1.8'
70
75
  name: json
76
+ prerelease: false
71
77
  requirement: !ruby/object:Gem::Requirement
72
78
  requirements:
73
79
  - - "~>"
74
80
  - !ruby/object:Gem::Version
75
81
  version: '1.8'
76
82
  type: :runtime
77
- prerelease: false
83
+ - !ruby/object:Gem::Dependency
78
84
  version_requirements: !ruby/object:Gem::Requirement
79
85
  requirements:
80
86
  - - "~>"
81
87
  - !ruby/object:Gem::Version
82
- version: '1.8'
83
- - !ruby/object:Gem::Dependency
88
+ version: '0.1'
84
89
  name: thread
90
+ prerelease: false
85
91
  requirement: !ruby/object:Gem::Requirement
86
92
  requirements:
87
93
  - - "~>"
88
94
  - !ruby/object:Gem::Version
89
95
  version: '0.1'
90
96
  type: :runtime
91
- prerelease: false
97
+ - !ruby/object:Gem::Dependency
92
98
  version_requirements: !ruby/object:Gem::Requirement
93
99
  requirements:
94
100
  - - "~>"
95
101
  - !ruby/object:Gem::Version
96
- version: '0.1'
102
+ version: '1.1'
103
+ name: rainbow
104
+ prerelease: false
105
+ requirement: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - "~>"
108
+ - !ruby/object:Gem::Version
109
+ version: '1.1'
110
+ type: :runtime
97
111
  - !ruby/object:Gem::Dependency
112
+ version_requirements: !ruby/object:Gem::Requirement
113
+ requirements:
114
+ - - "~>"
115
+ - !ruby/object:Gem::Version
116
+ version: '0.4'
98
117
  name: alakazam
118
+ prerelease: false
99
119
  requirement: !ruby/object:Gem::Requirement
100
120
  requirements:
101
121
  - - "~>"
102
122
  - !ruby/object:Gem::Version
103
123
  version: '0.4'
104
124
  type: :runtime
105
- prerelease: false
125
+ - !ruby/object:Gem::Dependency
106
126
  version_requirements: !ruby/object:Gem::Requirement
107
127
  requirements:
108
128
  - - "~>"
109
129
  - !ruby/object:Gem::Version
110
- version: '0.4'
111
- - !ruby/object:Gem::Dependency
130
+ version: '10.3'
112
131
  name: rake
132
+ prerelease: false
113
133
  requirement: !ruby/object:Gem::Requirement
114
134
  requirements:
115
135
  - - "~>"
116
136
  - !ruby/object:Gem::Version
117
137
  version: '10.3'
118
138
  type: :development
119
- prerelease: false
139
+ - !ruby/object:Gem::Dependency
120
140
  version_requirements: !ruby/object:Gem::Requirement
121
141
  requirements:
122
142
  - - "~>"
123
143
  - !ruby/object:Gem::Version
124
- version: '10.3'
125
- - !ruby/object:Gem::Dependency
144
+ version: '3.1'
126
145
  name: rspec
146
+ prerelease: false
127
147
  requirement: !ruby/object:Gem::Requirement
128
148
  requirements:
129
149
  - - "~>"
130
150
  - !ruby/object:Gem::Version
131
151
  version: '3.1'
132
152
  type: :development
133
- prerelease: false
153
+ - !ruby/object:Gem::Dependency
134
154
  version_requirements: !ruby/object:Gem::Requirement
135
155
  requirements:
136
156
  - - "~>"
137
157
  - !ruby/object:Gem::Version
138
- version: '3.1'
139
- - !ruby/object:Gem::Dependency
158
+ version: '1.6'
140
159
  name: fastimage
160
+ prerelease: false
141
161
  requirement: !ruby/object:Gem::Requirement
142
162
  requirements:
143
163
  - - "~>"
144
164
  - !ruby/object:Gem::Version
145
165
  version: '1.6'
146
166
  type: :development
147
- prerelease: false
167
+ - !ruby/object:Gem::Dependency
148
168
  version_requirements: !ruby/object:Gem::Requirement
149
169
  requirements:
150
170
  - - "~>"
151
171
  - !ruby/object:Gem::Version
152
- version: '1.6'
153
- - !ruby/object:Gem::Dependency
172
+ version: '1.0'
154
173
  name: parallel_tests
174
+ prerelease: false
155
175
  requirement: !ruby/object:Gem::Requirement
156
176
  requirements:
157
177
  - - "~>"
158
178
  - !ruby/object:Gem::Version
159
179
  version: '1.0'
160
180
  type: :development
161
- prerelease: false
162
- version_requirements: !ruby/object:Gem::Requirement
163
- requirements:
164
- - - "~>"
165
- - !ruby/object:Gem::Version
166
- version: '1.0'
167
181
  description: A dumper to download whole galleries from board like 4chan, imagebam,
168
182
  mangaeden, deviantart, etc.
169
183
  email: webmaster@giovannicapuano.net
@@ -232,7 +246,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
232
246
  version: '0'
233
247
  requirements: []
234
248
  rubyforge_project:
235
- rubygems_version: 2.2.2
249
+ rubygems_version: 2.4.8
236
250
  signing_key:
237
251
  specification_version: 4
238
252
  summary: Fetch and download image gallery.