hotpixiv 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/ChangeLog +20 -0
- data/README.rdoc +64 -0
- data/Rakefile +144 -0
- data/bin/hotpixiv +21 -0
- data/lib/hotpixiv.rb +59 -0
- data/lib/hotpixiv/download.rb +131 -0
- data/lib/hotpixiv/utils.rb +98 -0
- data/spec/hotpixiv_helper.rb +79 -0
- data/spec/hotpixiv_spec.rb +68 -0
- metadata +116 -0
data/ChangeLog
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
== 0.0.1 / 2010-07-10
|
2
|
+
* initial release
|
3
|
+
|
4
|
+
== 0.0.2 / 2010-07-17
|
5
|
+
* bug fixed. fail download when URL with the page or image server name are wrong.
|
6
|
+
|
7
|
+
== 0.0.3 / 2010-07-22
|
8
|
+
* deal with pixiv renewal.
|
9
|
+
|
10
|
+
== 0.2.0 / 2010-11-30
|
11
|
+
* support Ruby1.9.2
|
12
|
+
|
13
|
+
== 0.2.1 / 2010-12-11
|
14
|
+
* append test case.
|
15
|
+
|
16
|
+
== 0.2.2 / 2010-12-30
|
17
|
+
* bug fixed.
|
18
|
+
|
19
|
+
== 0.2.3 / 2011-01-02
|
20
|
+
* bug fixed.
|
data/README.rdoc
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
= hotpixiv
|
2
|
+
|
3
|
+
hotpixiv is auto image collection tool in pixiv(http://www.pixiv.net).
|
4
|
+
|
5
|
+
== Installation
|
6
|
+
|
7
|
+
Generate and install the gem, execute a command.
|
8
|
+
|
9
|
+
rake gemspec
|
10
|
+
gem build hotpixiv.gemspec
|
11
|
+
|
12
|
+
=== Gem Installation
|
13
|
+
|
14
|
+
gem install hotpixiv
|
15
|
+
|
16
|
+
== How to use
|
17
|
+
|
18
|
+
The available option are:
|
19
|
+
|
20
|
+
-h # help
|
21
|
+
-l # output log file path.
|
22
|
+
-k # set the search keyword (only one).
|
23
|
+
-f # set the search keywords from textfile (more than one).
|
24
|
+
(choose either of -k or -f)
|
25
|
+
-p # acquire the image which is bigger than pixiv image's total point.
|
26
|
+
-s # set the directory path. downloaded image stores there.
|
27
|
+
-b # threshold of pixiv point.
|
28
|
+
-d # show debug message.
|
29
|
+
-C # config file path. (Please refer to example/config.yml)
|
30
|
+
|
31
|
+
== Synopsis
|
32
|
+
|
33
|
+
execute command example:
|
34
|
+
|
35
|
+
hotpixiv -p 50 -f c:/keywords.txt -s c:/pixiv/img -l e:/pixiv/log
|
36
|
+
|
37
|
+
== Copyright
|
38
|
+
|
39
|
+
* Author:: mapserver2007(Ryuichi TANAKA) <mapserver2007@gmail.com>
|
40
|
+
* Copyright:: Copyright (c) 2010 mapserver2007
|
41
|
+
|
42
|
+
== License
|
43
|
+
|
44
|
+
(The MIT License)
|
45
|
+
|
46
|
+
Copyright (c) 2010 Ryuichi Tanaka
|
47
|
+
|
48
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
49
|
+
of this software and associated documentation files (the "Software"), to deal
|
50
|
+
in the Software without restriction, including without limitation the rights
|
51
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
52
|
+
copies of the Software, and to permit persons to whom the Software is
|
53
|
+
furnished to do so, subject to the following conditions:
|
54
|
+
|
55
|
+
The above copyright notice and this permission notice shall be included in
|
56
|
+
all copies or substantial portions of the Software.
|
57
|
+
|
58
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
59
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
60
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
61
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
62
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
63
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
64
|
+
THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,144 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
require 'rake/clean'
|
4
|
+
require 'rake/testtask'
|
5
|
+
require 'rake/packagetask'
|
6
|
+
require 'rake/gempackagetask'
|
7
|
+
require 'rake/rdoctask'
|
8
|
+
require 'rake/contrib/rubyforgepublisher'
|
9
|
+
require 'rake/contrib/sshpublisher'
|
10
|
+
require 'fileutils'
|
11
|
+
include FileUtils
|
12
|
+
|
13
|
+
NAME = "hotpixiv"
|
14
|
+
PROJECT_NAME = "hotpixiv2"
|
15
|
+
AUTHOR = "mapserver2007"
|
16
|
+
EMAIL = "mapserver2007@gmail.com"
|
17
|
+
DESCRIPTION = "hotpixiv. Auto collection tool in pixiv."
|
18
|
+
HOMEPATH = "http://github.com/#{AUTHOR}/#{PROJECT_NAME}"
|
19
|
+
RUBYFORGE_PROJECT = "hotpixiv2"
|
20
|
+
BIN_FILES = %w( hotpixiv )
|
21
|
+
VERS = '0.2.3'
|
22
|
+
REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil
|
23
|
+
CLEAN.include ['**/.*.sw?', '*.gem', '.config']
|
24
|
+
RDOC_OPTS = [
|
25
|
+
'--title', "#{NAME} documentation",
|
26
|
+
"--charset", "utf-8",
|
27
|
+
"--opname", "index.html",
|
28
|
+
"--line-numbers",
|
29
|
+
"--main", "README.rdoc",
|
30
|
+
"--inline-source",
|
31
|
+
]
|
32
|
+
|
33
|
+
task :default => [:test]
|
34
|
+
task :package => [:clean]
|
35
|
+
|
36
|
+
Rake::TestTask.new("test") do |t|
|
37
|
+
t.libs << "test"
|
38
|
+
t.pattern = "test/**/*_test.rb"
|
39
|
+
t.verbose = true
|
40
|
+
end
|
41
|
+
|
42
|
+
spec = Gem::Specification.new do |s|
|
43
|
+
s.name = NAME
|
44
|
+
s.version = VERS
|
45
|
+
s.platform = Gem::Platform::RUBY
|
46
|
+
s.has_rdoc = true
|
47
|
+
s.extra_rdoc_files = ["README.rdoc", "ChangeLog"]
|
48
|
+
s.rdoc_options += RDOC_OPTS + ['--exclude', '^(examples|extras)/']
|
49
|
+
s.summary = DESCRIPTION
|
50
|
+
s.description = DESCRIPTION
|
51
|
+
s.author = AUTHOR
|
52
|
+
s.email = EMAIL
|
53
|
+
s.homepage = HOMEPATH
|
54
|
+
s.executables = BIN_FILES
|
55
|
+
s.rubyforge_project = RUBYFORGE_PROJECT
|
56
|
+
s.bindir = "bin"
|
57
|
+
s.require_path = "lib"
|
58
|
+
#s.autorequire = ""
|
59
|
+
s.test_files = Dir["spec/*.rb"]
|
60
|
+
|
61
|
+
s.add_dependency('parallel_runner', '>=0.0.1')
|
62
|
+
s.add_dependency('mechanize', '>=1.0.0')
|
63
|
+
s.required_ruby_version = '>= 1.9.2'
|
64
|
+
|
65
|
+
s.files = %w(README.rdoc ChangeLog Rakefile) +
|
66
|
+
Dir.glob("{bin,doc,test,lib,templates,generator,extras,website,script}/**/*") +
|
67
|
+
Dir.glob("ext/**/*.{h,c,rb}") +
|
68
|
+
Dir.glob("examples/**/*.rb") +
|
69
|
+
Dir.glob("tools/*.rb") +
|
70
|
+
Dir.glob("rails/*.rb")
|
71
|
+
|
72
|
+
s.extensions = FileList["ext/**/extconf.rb"].to_a
|
73
|
+
end
|
74
|
+
|
75
|
+
Rake::GemPackageTask.new(spec) do |p|
|
76
|
+
p.need_tar = true
|
77
|
+
p.gem_spec = spec
|
78
|
+
end
|
79
|
+
|
80
|
+
task :install do
|
81
|
+
name = "#{NAME}-#{VERS}.gem"
|
82
|
+
sh %{rake package}
|
83
|
+
sh %{sudo gem install pkg/#{name}}
|
84
|
+
end
|
85
|
+
|
86
|
+
task :uninstall => [:clean] do
|
87
|
+
sh %{sudo gem uninstall #{NAME}}
|
88
|
+
end
|
89
|
+
|
90
|
+
|
91
|
+
Rake::RDocTask.new do |rdoc|
|
92
|
+
rdoc.rdoc_dir = 'html'
|
93
|
+
rdoc.options += RDOC_OPTS
|
94
|
+
rdoc.template = "resh"
|
95
|
+
#rdoc.template = "#{ENV['template']}.rb" if ENV['template']
|
96
|
+
if ENV['DOC_FILES']
|
97
|
+
rdoc.rdoc_files.include(ENV['DOC_FILES'].split(/,\s*/))
|
98
|
+
else
|
99
|
+
rdoc.rdoc_files.include('README.rdoc', 'ChangeLog')
|
100
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
101
|
+
rdoc.rdoc_files.include('ext/**/*.c')
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
desc "Publish to RubyForge"
|
106
|
+
task :rubyforge => [:rdoc, :package] do
|
107
|
+
require 'rubyforge'
|
108
|
+
Rake::RubyForgePublisher.new(RUBYFORGE_PROJECT, 'stay').upload
|
109
|
+
end
|
110
|
+
|
111
|
+
desc 'Package and upload the release to rubyforge.'
|
112
|
+
task :release => [:clean, :package] do |t|
|
113
|
+
v = ENV["VERSION"] or abort "Must supply VERSION=x.y.z"
|
114
|
+
abort "Versions don't match #{v} vs #{VERS}" unless v == VERS
|
115
|
+
pkg = "pkg/#{NAME}-#{VERS}"
|
116
|
+
|
117
|
+
require 'rubyforge'
|
118
|
+
rf = RubyForge.new.configure
|
119
|
+
puts "Logging in"
|
120
|
+
rf.login
|
121
|
+
|
122
|
+
c = rf.userconfig
|
123
|
+
# c["release_notes"] = description if description
|
124
|
+
# c["release_changes"] = changes if changes
|
125
|
+
c["preformatted"] = true
|
126
|
+
|
127
|
+
files = [
|
128
|
+
"#{pkg}.tgz",
|
129
|
+
"#{pkg}.gem"
|
130
|
+
].compact
|
131
|
+
|
132
|
+
puts "Releasing #{NAME} v. #{VERS}"
|
133
|
+
rf.add_release RUBYFORGE_PROJECT, NAME, VERS, *files
|
134
|
+
end
|
135
|
+
|
136
|
+
desc 'Show information about the gem.'
|
137
|
+
task :debug_gem do
|
138
|
+
puts spec.to_ruby
|
139
|
+
end
|
140
|
+
|
141
|
+
desc 'Update gem spec'
|
142
|
+
task :gemspec do
|
143
|
+
open("#{NAME}.gemspec", 'w').write spec.to_ruby
|
144
|
+
end
|
data/bin/hotpixiv
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
$: << File.dirname(File.expand_path($PROGRAM_NAME)) + "/../lib"
|
4
|
+
require 'hotpixiv'
|
5
|
+
require 'optparse'
|
6
|
+
|
7
|
+
config = {}
|
8
|
+
OptionParser.new do |opt|
|
9
|
+
opt.on('-l', '--log LOG') {|v| config[:log] = v}
|
10
|
+
opt.on('-k', '--keyword KEYWORD') {|v| config[:keyword] = v}
|
11
|
+
opt.on('-f', '--file_keyword FILE_KEYWORD') {|v| config[:file_keyword] = v}
|
12
|
+
opt.on('-s', '--save_dir SAVE_DIR') {|v| config[:save_dir] = v}
|
13
|
+
opt.on('-p', '--page PAGE') {|v| config[:page] = v}
|
14
|
+
opt.on('-b', '--point_border POINT_BORDER') {|v| config[:point_border] = v}
|
15
|
+
opt.on('-d', '--debug') {|boolean| config[:debug] = boolean}
|
16
|
+
opt.on('-C', '--config CONFIG') {|v| config[:config] = v}
|
17
|
+
opt.on('-v', '--version') {puts HotPixiv::VERSION; exit}
|
18
|
+
opt.parse!
|
19
|
+
end
|
20
|
+
|
21
|
+
HotPixiv.get(config)
|
data/lib/hotpixiv.rb
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
require 'hotpixiv/download'
|
3
|
+
require 'hotpixiv/utils'
|
4
|
+
|
5
|
+
module HotPixiv
|
6
|
+
VERSION = '0.2.3'
|
7
|
+
PIXIV_API_URL = 'http://iphone.pxv.jp/iphone/'
|
8
|
+
REFERER = 'http://www.pixiv.net/'
|
9
|
+
USER_AGENT = 'Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us)
|
10
|
+
AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16'
|
11
|
+
DOWNLOAD_TIMEOUT = 10
|
12
|
+
PAGE = 10
|
13
|
+
POINT_BORDER = 0
|
14
|
+
LOG_LEVEL = 1
|
15
|
+
URL_REGEXP = /s?http?:\/\/img\d{2}.pixiv.net\/img\/.*?\/\d{8}\.(jpg|gif|png)/
|
16
|
+
|
17
|
+
class << self
|
18
|
+
# 共通設定
|
19
|
+
def init(params)
|
20
|
+
begin
|
21
|
+
params.merge!(Utils.load(params[:config]))
|
22
|
+
rescue
|
23
|
+
unless params[:config].nil? || Utils.file?(params[:config])
|
24
|
+
raise "Config file not found."
|
25
|
+
end
|
26
|
+
ensure
|
27
|
+
@log = Log.new(params[:log], params[:debug])
|
28
|
+
end
|
29
|
+
params[:point_border] ||= POINT_BORDER
|
30
|
+
keywords = Utils.read_text(params[:file_keyword])
|
31
|
+
keywords << params[:keyword] if keywords.length == 0
|
32
|
+
params[:keywords] = keywords
|
33
|
+
end
|
34
|
+
|
35
|
+
# 画像を取得する
|
36
|
+
def get(params)
|
37
|
+
begin
|
38
|
+
# 基本設定
|
39
|
+
init(params)
|
40
|
+
# 画像のダウンロード
|
41
|
+
Download.new.run(params) do |level, msg|
|
42
|
+
# 処理が成功した場合
|
43
|
+
if level == :info
|
44
|
+
@log.info = msg
|
45
|
+
puts "[OK]\t#{msg}\n"
|
46
|
+
# 処理が失敗した場合
|
47
|
+
elsif level == :error
|
48
|
+
@log.error = msg
|
49
|
+
puts "[NG]\t#{msg}\n"
|
50
|
+
end
|
51
|
+
end
|
52
|
+
rescue => e
|
53
|
+
@log.error = e.message
|
54
|
+
end
|
55
|
+
# エラーがあった場合は出力
|
56
|
+
@log.error.print
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,131 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
require 'mechanize'
|
3
|
+
require 'open-uri'
|
4
|
+
require 'parallel_runner'
|
5
|
+
|
6
|
+
module HotPixiv
|
7
|
+
class Download
|
8
|
+
# リクエストパラメータを作成
|
9
|
+
def query(keyword, page)
|
10
|
+
{:s_mode => "s_tag",
|
11
|
+
:word => keyword,
|
12
|
+
:p => page}.map {|k, v| "#{k.to_s}=#{CGI.escape(v.to_s.encode('UTF-8'))}"}.join("&")
|
13
|
+
end
|
14
|
+
|
15
|
+
# パラメータを整形
|
16
|
+
def trim(data, i)
|
17
|
+
return data.split(/\//)[4] if i == 6
|
18
|
+
return "%02d" % data if i == 4 && i.to_s.length == 1
|
19
|
+
data
|
20
|
+
end
|
21
|
+
|
22
|
+
# 文字列にダブルクオートが含まれている場合は置換する
|
23
|
+
def esc(data)
|
24
|
+
data.gsub(/\"\"\"/, "\"'").gsub(/\"\"/, "'")
|
25
|
+
end
|
26
|
+
|
27
|
+
def valiable_set(params)
|
28
|
+
params.each do |k, v|
|
29
|
+
self.instance_variable_set("@#{k.to_s}", v)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
# 画像URLを取得
|
34
|
+
def get_url(data)
|
35
|
+
point = POINT_BORDER
|
36
|
+
e = []
|
37
|
+
i = 0
|
38
|
+
# 文字列にダブルクオートが含まれている場合は置換する
|
39
|
+
esc(data).scan(/\"(.*?)\"/) do |d|
|
40
|
+
e << trim(d[0], i)
|
41
|
+
i += 1
|
42
|
+
end
|
43
|
+
# 総合点:e[16], 評価回数:e[15], 閲覧回数:e[17]
|
44
|
+
# サーバID:e[4], ユーザID:e[6], 画像ID:e[0], 拡張子:e[2]
|
45
|
+
url = "http://img#{e[4]}.pixiv.net/img/#{e[6]}/#{e[0]}.#{e[2]}"
|
46
|
+
e[16].to_i >= point && !!(URL_REGEXP =~ url) ? url : nil
|
47
|
+
end
|
48
|
+
|
49
|
+
# 画像ダウンロード
|
50
|
+
def download(list, keyword, &block)
|
51
|
+
Runner.parallel(list) do |url|
|
52
|
+
(download_from url, keyword).each do |res|
|
53
|
+
block.call(res.keys.pop, res.values.pop)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
# 画像をダウンロード
|
59
|
+
def download_from(url, keyword)
|
60
|
+
message_list = []
|
61
|
+
begin
|
62
|
+
save_image(url, keyword)
|
63
|
+
message_list << {:info => url}
|
64
|
+
# マンガの場合
|
65
|
+
rescue OpenURI::HTTPError
|
66
|
+
begin
|
67
|
+
0.upto(Float::INFINITY) do |page|
|
68
|
+
url_with_page = url.gsub(/(\d*)\.[jpg|png|gif]{3}/) do |matched|
|
69
|
+
f = matched.split(/\./)
|
70
|
+
"#{f[0]}_p#{page}.#{f[1]}"
|
71
|
+
end
|
72
|
+
save_image(url_with_page, keyword)
|
73
|
+
message_list << {:info => url_with_page}
|
74
|
+
end
|
75
|
+
rescue SocketError => e
|
76
|
+
message_list << {:error => "#{url} - #{e.message}"}
|
77
|
+
rescue OpenURI::HTTPError; end
|
78
|
+
end
|
79
|
+
message_list
|
80
|
+
end
|
81
|
+
|
82
|
+
# 画像を保存する
|
83
|
+
def save_image(url, keyword)
|
84
|
+
dir = Utils.create_path(@save_dir, keyword)
|
85
|
+
open(url, "Referer" => REFERER) do |f|
|
86
|
+
open(dir + "/" + File.basename(url), 'wb') do |output|
|
87
|
+
output.write(f.read)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
# ダウンロードを開始する
|
93
|
+
def run(params, &block)
|
94
|
+
valiable_set(params)
|
95
|
+
agent = Mechanize.new
|
96
|
+
agent.read_timeout = DOWNLOAD_TIMEOUT
|
97
|
+
|
98
|
+
# ぺージ、キーワードのURLリストを作成
|
99
|
+
page_list = []
|
100
|
+
1.upto(@page.to_i) do |page_no|
|
101
|
+
@keywords.each do |keyword|
|
102
|
+
page_list << "#{PIXIV_API_URL}search.php?#{query(keyword, page_no)}"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
# 日付のディレクトリを作成
|
107
|
+
raise "Invalid save directory path. - #{@save_dir}" unless Utils.directory?(@save_dir)
|
108
|
+
@save_dir = Utils.create_path(@save_dir, DateTime.now.strftime("%Y%m%d"))
|
109
|
+
Utils.create_dir(@save_dir)
|
110
|
+
|
111
|
+
# 並列ダウンロード開始
|
112
|
+
puts "Downloading ..."
|
113
|
+
Runner.parallel(page_list) do |page|
|
114
|
+
# URLからキーワードを抽出
|
115
|
+
keyword = URI.decode($1).encode(Utils.os_encoding) if /word=(.*?)&/ =~ page
|
116
|
+
# キーワードのディレクトリを作成
|
117
|
+
Utils.create_dir(@save_dir, keyword)
|
118
|
+
url_list = []
|
119
|
+
site = agent.get(page)
|
120
|
+
lines = (site/ '//body/p').inner_html.split(/\n/)
|
121
|
+
lines.each do |line|
|
122
|
+
url_list << get_url(line)
|
123
|
+
end
|
124
|
+
# nilのデータを削除
|
125
|
+
url_list.compact!
|
126
|
+
# 画像をダウンロードする
|
127
|
+
download(url_list, keyword, &block) unless url_list.empty?
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'pathname'
|
3
|
+
|
4
|
+
module HotPixiv
|
5
|
+
module Utils
|
6
|
+
class << self
|
7
|
+
def read_text(path)
|
8
|
+
data = []
|
9
|
+
if file?(path)
|
10
|
+
open(path) do |f|
|
11
|
+
while line = f.gets do
|
12
|
+
data << line.chomp unless line.chomp.empty?
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
data
|
17
|
+
end
|
18
|
+
|
19
|
+
def file?(e)
|
20
|
+
!!(FileTest.exist?(e) rescue nil)
|
21
|
+
end
|
22
|
+
|
23
|
+
def directory?(e)
|
24
|
+
!!(FileTest::directory?(e) rescue nil)
|
25
|
+
end
|
26
|
+
|
27
|
+
def create_path(*args)
|
28
|
+
Pathname.new(args.join("/")).cleanpath.to_s.encode(Utils.os_encoding)
|
29
|
+
end
|
30
|
+
|
31
|
+
def create_dir(*args)
|
32
|
+
Dir::mkdir(create_path(args)) rescue nil
|
33
|
+
end
|
34
|
+
|
35
|
+
def load(path)
|
36
|
+
config = YAML.load_file(path)
|
37
|
+
config.inject({}){|r, entry| r[entry[0].to_sym] = entry[1]; r}
|
38
|
+
end
|
39
|
+
|
40
|
+
def os_encoding
|
41
|
+
RUBY_PLATFORM.downcase =~ /mswin(?!ce)|mingw|cygwin|bccwin/ ?
|
42
|
+
"Windows-31J" : "UTF-8"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
class Log
|
48
|
+
def initialize(path, debug_on)
|
49
|
+
@debug = {}
|
50
|
+
@debug_on = debug_on
|
51
|
+
if Utils.directory? path
|
52
|
+
level = [0, 1, 2, 3, 4].index(LOG_LEVEL) || Logger::DEBUG
|
53
|
+
@log = Logger.new(logfile(path))
|
54
|
+
@log.level = level
|
55
|
+
else
|
56
|
+
unless path.nil?
|
57
|
+
@debug["error"] = []
|
58
|
+
@debug["error"] << "Invalid log directory path - #{path}"
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def logfile(dir)
|
64
|
+
Pathname.new(dir + "/" + "#{Time.now.strftime("%Y%m%d")}.log").cleanpath
|
65
|
+
end
|
66
|
+
|
67
|
+
def write(name, *args)
|
68
|
+
# ログ出力処理、デバッグスタックに保存
|
69
|
+
if name =~ /=$/
|
70
|
+
name.chop!
|
71
|
+
@debug[name] = [] unless @debug[name].instance_of?(Array)
|
72
|
+
args[0].each do |msg|
|
73
|
+
@log.send(name, msg.encode(Utils.os_encoding)) unless @log.nil?
|
74
|
+
@debug[name] << msg
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def debug(name)
|
80
|
+
# デバッグメッセージを標準出力
|
81
|
+
if name == 'print'
|
82
|
+
@debug[@name].each do |msg|
|
83
|
+
puts msg
|
84
|
+
end if @debug[@name].instance_of?(Array) && @debug_on
|
85
|
+
else
|
86
|
+
# ログメソッドを保存
|
87
|
+
@name = name
|
88
|
+
self
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def method_missing(name, *args)
|
93
|
+
attr = name.to_s
|
94
|
+
write(attr, args)
|
95
|
+
debug(attr)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
require 'rspec'
|
2
|
+
require File.dirname(__FILE__) + "/../lib/hotpixiv"
|
3
|
+
require File.dirname(__FILE__) + "/../lib/hotpixiv/download"
|
4
|
+
require File.dirname(__FILE__) + "/../lib/hotpixiv/utils"
|
5
|
+
|
6
|
+
module HotPixiv
|
7
|
+
class Rspec
|
8
|
+
def method_missing(name, *args)
|
9
|
+
attr = name.to_s
|
10
|
+
if @attr == "params"
|
11
|
+
config = Utils.load(File.dirname(__FILE__) + "/spec_config.yml")
|
12
|
+
@attr = nil
|
13
|
+
if attr == "common" || attr == "other"
|
14
|
+
config[attr.to_sym].inject({}){|r, entry| r[entry[0].to_sym] = entry[1]; r}
|
15
|
+
else
|
16
|
+
config[:common][attr] || config[:other][attr]
|
17
|
+
end
|
18
|
+
elsif attr == "params"
|
19
|
+
@attr = attr
|
20
|
+
self
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# テスト用に画像を1件のみ取得するようにメソッドを書き換える
|
25
|
+
def rewrite_download
|
26
|
+
HotPixiv::Download.class_eval do
|
27
|
+
def download(list, keyword, &block)
|
28
|
+
test_list = [list[0]]
|
29
|
+
Runner.parallel(test_list) do |url|
|
30
|
+
(download_from url, keyword).each do |res|
|
31
|
+
block.call(res.keys.pop, res.values.pop)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
# ダウンロード処理の実行結果を監視するメソッドを動的に定義
|
39
|
+
def add_download_inspector(keyword)
|
40
|
+
dir = Utils.create_path(self.params.save_dir,
|
41
|
+
DateTime.now.strftime("%Y%m%d"), keyword)
|
42
|
+
dir = Dir::entries(dir).reject! {|e| e == "." || e == ".."}
|
43
|
+
dir.should be_empty
|
44
|
+
end
|
45
|
+
|
46
|
+
# ログ処理の実行結果を監視するメソッドを動的に定義
|
47
|
+
def add_log_inspector
|
48
|
+
Log.class_eval do
|
49
|
+
def ignore_error(list)
|
50
|
+
list.reject! do |e|
|
51
|
+
!!(e.index("getaddrinfo: no address associated with hostname."))
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def log_inspector(level, error_ignore)
|
56
|
+
# 特定のエラーを無視する
|
57
|
+
@debug[level] ||= []
|
58
|
+
if error_ignore
|
59
|
+
ignore_error @debug[level]
|
60
|
+
end
|
61
|
+
@debug[level]
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
class << HotPixiv
|
66
|
+
def log_inspector(level, error_ignore = false)
|
67
|
+
@log.log_inspector(level.to_s, error_ignore)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
# 実行結果エラーログを確認する
|
73
|
+
def error_inspector(msg)
|
74
|
+
HotPixiv.log_inspector(:error, true).each do |inspect_msg|
|
75
|
+
inspect_msg.should == msg
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
require File.dirname(__FILE__) + "/hotpixiv_helper.rb"
|
3
|
+
|
4
|
+
describe HotPixiv, 'が実行する処理' do
|
5
|
+
before do
|
6
|
+
@rspec = HotPixiv::Rspec.new
|
7
|
+
@rspec.rewrite_download
|
8
|
+
@rspec.add_log_inspector
|
9
|
+
@params = @rspec.params.common
|
10
|
+
end
|
11
|
+
|
12
|
+
describe '正常系' do
|
13
|
+
after do
|
14
|
+
HotPixiv.log_inspector(:error, true).should be_empty
|
15
|
+
end
|
16
|
+
|
17
|
+
it "コマンドライン引数による実行が正常にできること" do
|
18
|
+
HotPixiv.get(@params)
|
19
|
+
end
|
20
|
+
|
21
|
+
it "設定ファイルによる実行が正常にできること" do
|
22
|
+
@params[:config] = @rspec.params.config
|
23
|
+
HotPixiv.get(@params)
|
24
|
+
end
|
25
|
+
|
26
|
+
it "キーワードをコマンドライン引数から指定して正常に実行出来ること" do
|
27
|
+
@params[:keyword] = "イカ娘"
|
28
|
+
HotPixiv.get(@params)
|
29
|
+
end
|
30
|
+
|
31
|
+
it "キーワードをファイルから指定して正常に実行出来ること" do
|
32
|
+
@params[:file_keyword] = @rspec.params.file_keyword
|
33
|
+
p @params[:file_keyword]
|
34
|
+
HotPixiv.get(@params)
|
35
|
+
end
|
36
|
+
|
37
|
+
it "ファイルパス設定に日本語を含む場合、実行が正常にできること" do
|
38
|
+
@params[:config] = @rspec.params.config_ja
|
39
|
+
HotPixiv.get(@params)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
describe '異常系' do
|
44
|
+
it "存在しないキーワードで検索した場合、画像が取得できないこと" do
|
45
|
+
@params[:keyword] = @rspec.params.dummy_value
|
46
|
+
HotPixiv.get(@params)
|
47
|
+
@rspec.add_download_inspector(@params[:keyword])
|
48
|
+
end
|
49
|
+
|
50
|
+
it "ログ出力ディレクトリパスが間違っている場合「Invalid log directory path.」と表示されること" do
|
51
|
+
@params[:log] = @rspec.params.dummy_value
|
52
|
+
HotPixiv.get(@params)
|
53
|
+
@rspec.error_inspector("Invalid log directory path - #{@params[:log]}")
|
54
|
+
end
|
55
|
+
|
56
|
+
it "画像保存ディレクトリパスが間違っている場合「No such file or direcotry - ディレクトリ名」と表示されること" do
|
57
|
+
@params[:save_dir] = @rspec.params.dummy_value
|
58
|
+
HotPixiv.get(@params)
|
59
|
+
@rspec.error_inspector("Invalid save directory path. - #{@params[:save_dir] }")
|
60
|
+
end
|
61
|
+
|
62
|
+
it "設定ファイルのパスが間違っている場合「Config file not found.」と表示されること" do
|
63
|
+
@params[:config] = @rspec.params.dummy_value
|
64
|
+
HotPixiv.get(@params)
|
65
|
+
@rspec.error_inspector("Config file not found.")
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
metadata
ADDED
@@ -0,0 +1,116 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: hotpixiv
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 2
|
8
|
+
- 3
|
9
|
+
version: 0.2.3
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- mapserver2007
|
13
|
+
autorequire:
|
14
|
+
bindir: bin
|
15
|
+
cert_chain: []
|
16
|
+
|
17
|
+
date: 2011-01-02 00:00:00 +09:00
|
18
|
+
default_executable: hotpixiv
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: parallel_runner
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
none: false
|
25
|
+
requirements:
|
26
|
+
- - ">="
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
segments:
|
29
|
+
- 0
|
30
|
+
- 0
|
31
|
+
- 1
|
32
|
+
version: 0.0.1
|
33
|
+
type: :runtime
|
34
|
+
version_requirements: *id001
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: mechanize
|
37
|
+
prerelease: false
|
38
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
segments:
|
44
|
+
- 1
|
45
|
+
- 0
|
46
|
+
- 0
|
47
|
+
version: 1.0.0
|
48
|
+
type: :runtime
|
49
|
+
version_requirements: *id002
|
50
|
+
description: hotpixiv. Auto collection tool in pixiv.
|
51
|
+
email: mapserver2007@gmail.com
|
52
|
+
executables:
|
53
|
+
- hotpixiv
|
54
|
+
extensions: []
|
55
|
+
|
56
|
+
extra_rdoc_files:
|
57
|
+
- README.rdoc
|
58
|
+
- ChangeLog
|
59
|
+
files:
|
60
|
+
- README.rdoc
|
61
|
+
- ChangeLog
|
62
|
+
- Rakefile
|
63
|
+
- bin/hotpixiv
|
64
|
+
- lib/hotpixiv/download.rb
|
65
|
+
- lib/hotpixiv/utils.rb
|
66
|
+
- lib/hotpixiv.rb
|
67
|
+
- spec/hotpixiv_helper.rb
|
68
|
+
- spec/hotpixiv_spec.rb
|
69
|
+
has_rdoc: true
|
70
|
+
homepage: http://github.com/mapserver2007/hotpixiv2
|
71
|
+
licenses: []
|
72
|
+
|
73
|
+
post_install_message:
|
74
|
+
rdoc_options:
|
75
|
+
- --title
|
76
|
+
- hotpixiv documentation
|
77
|
+
- --charset
|
78
|
+
- utf-8
|
79
|
+
- --opname
|
80
|
+
- index.html
|
81
|
+
- --line-numbers
|
82
|
+
- --main
|
83
|
+
- README.rdoc
|
84
|
+
- --inline-source
|
85
|
+
- --exclude
|
86
|
+
- ^(examples|extras)/
|
87
|
+
require_paths:
|
88
|
+
- lib
|
89
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
90
|
+
none: false
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
segments:
|
95
|
+
- 1
|
96
|
+
- 9
|
97
|
+
- 2
|
98
|
+
version: 1.9.2
|
99
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
100
|
+
none: false
|
101
|
+
requirements:
|
102
|
+
- - ">="
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
segments:
|
105
|
+
- 0
|
106
|
+
version: "0"
|
107
|
+
requirements: []
|
108
|
+
|
109
|
+
rubyforge_project: hotpixiv2
|
110
|
+
rubygems_version: 1.3.7
|
111
|
+
signing_key:
|
112
|
+
specification_version: 3
|
113
|
+
summary: hotpixiv. Auto collection tool in pixiv.
|
114
|
+
test_files:
|
115
|
+
- spec/hotpixiv_helper.rb
|
116
|
+
- spec/hotpixiv_spec.rb
|